Mistral¶

And Llama¶

Demonstrates how to use the ChatInterface widget to create a chatbot using Llama2 and Mistral.

"""
Demonstrates how to use the ChatInterface widget to create a chatbot using
Llama2 and Mistral.
"""

import panel as pn
from ctransformers import AutoModelForCausalLM

pn.extension()

MODEL_ARGUMENTS = {
    "llama": {
        "args": ["TheBloke/Llama-2-7b-Chat-GGUF"],
        "kwargs": {"model_file": "llama-2-7b-chat.Q5_K_M.gguf"},
    },
    "mistral": {
        "args": ["TheBloke/Mistral-7B-Instruct-v0.1-GGUF"],
        "kwargs": {"model_file": "mistral-7b-instruct-v0.1.Q4_K_M.gguf"},
    },
}


def callback(contents: str, user: str, instance: pn.chat.ChatInterface):
    for model in MODEL_ARGUMENTS:
        if model not in pn.state.cache:
            pn.state.cache[model] = AutoModelForCausalLM.from_pretrained(
                *MODEL_ARGUMENTS[model]["args"],
                **MODEL_ARGUMENTS[model]["kwargs"],
                gpu_layers=1,
            )

        llm = pn.state.cache[model]
        response = llm(contents, max_new_tokens=512, stream=True)

        message = None
        for chunk in response:
            message = instance.stream(chunk, user=model.title(), message=message)


chat_interface = pn.chat.ChatInterface(callback=callback)
chat_interface.send(
    "Send a message to get a reply from both Llama 2 and Mistral (7B)!",
    user="System",
    respond=False,
)
chat_interface.servable()

Api Chat¶

Source code for mistral_api_chat.py

import panel as pn
from mistralai.client import MistralClient
from mistralai.models.chat_completion import ChatMessage

pn.extension()


async def callback(contents: str, user: str, instance: pn.chat.ChatInterface):
    messages.append(ChatMessage(role="user", content=contents))

    mistral_response = ""
    for chunk in client.chat_stream(model="mistral-tiny", messages=messages):
        response = chunk.choices[0].delta.content
        if response is not None:
            mistral_response += response
            yield mistral_response

    if mistral_response:
        messages.append(ChatMessage(role="assistant", content=mistral_response))


messages = []
client = MistralClient()  # api_key=os.environ.get("MISTRAL_API_KEY", None)
chat_interface = pn.chat.ChatInterface(callback=callback, callback_user="Mistral AI")
chat_interface.send(
    "Send a message to get a reply from Mistral AI!", user="System", respond=False
)
chat_interface.servable()

Chat¶

Demonstrates how to use the ChatInterface to create a chatbot using Mistral through CTransformers.

Source code for mistral_chat.py

"""
Demonstrates how to use the `ChatInterface` to create a chatbot using
[Mistral](https://docs.mistral.ai) through
[CTransformers](https://github.com/marella/ctransformers).
"""

import panel as pn
from ctransformers import AutoConfig, AutoModelForCausalLM, Config

pn.extension()

llms = pn.state.cache["llms"] = pn.state.cache.get("llms", {})

INSTRUCTIONS = "You are a friendly chat bot willing to help out the user."


def apply_template(instructions, contents):
    text_row = f"""<s>[INST]{instructions} {contents}[/INST]"""
    return text_row


async def callback(contents: str, user: str, instance: pn.chat.ChatInterface):
    if "mistral" not in llms:
        instance.placeholder_text = "Downloading model; please wait..."
        config = AutoConfig(
            config=Config(
                temperature=0.5, max_new_tokens=2048, context_length=2048, gpu_layers=1
            ),
        )
        llms["mistral"] = AutoModelForCausalLM.from_pretrained(
            "TheBloke/Mistral-7B-Instruct-v0.1-GGUF",
            model_file="mistral-7b-instruct-v0.1.Q4_K_M.gguf",
            config=config,
        )

    llm = llms["mistral"]
    response = llm(apply_template(INSTRUCTIONS, contents), stream=True)
    message = ""
    for token in response:
        message += token
        yield message


chat_interface = pn.chat.ChatInterface(
    callback=callback,
    callback_user="Mistral",
    reset_on_send=True,
)
chat_interface.send(
    "Send a message to get a reply from Mistral!", user="System", respond=False
)
chat_interface.servable()

With Memory¶

Demonstrates how to use the ChatInterface to create a chatbot using Mistral through CTransformers. The chatbot includes a memory of the conversation history.

Source code for mistral_with_memory.py

"""
Demonstrates how to use the `ChatInterface` to create a chatbot using
[Mistral](https://docs.mistral.ai) through
[CTransformers](https://github.com/marella/ctransformers). The chatbot includes a
memory of the conversation history.
"""

import panel as pn
from ctransformers import AutoConfig, AutoModelForCausalLM, Config

pn.extension()

SYSTEM_INSTRUCTIONS = "Do what the user requests."


def apply_template(history):
    history = [message for message in history if message.user != "System"]
    prompt = ""
    for i, message in enumerate(history):
        if i == 0:
            prompt += f"<s>[INST]{SYSTEM_INSTRUCTIONS} {message.object}[/INST]"
        else:
            if message.user == "Mistral":
                prompt += f"{message.object}</s>"
            else:
                prompt += f"""[INST]{message.object}[/INST]"""
    return prompt


async def callback(contents: str, user: str, instance: pn.chat.ChatInterface):
    if "mistral" not in llms:
        instance.placeholder_text = "Downloading model; please wait..."
        config = AutoConfig(
            config=Config(
                temperature=0.5, max_new_tokens=2048, context_length=2048, gpu_layers=1
            ),
        )
        llms["mistral"] = AutoModelForCausalLM.from_pretrained(
            "TheBloke/Mistral-7B-Instruct-v0.1-GGUF",
            model_file="mistral-7b-instruct-v0.1.Q4_K_M.gguf",
            config=config,
        )

    llm = llms["mistral"]
    history = [message for message in instance.objects]
    prompt = apply_template(history)
    response = llm(prompt, stream=True)
    message = ""
    for token in response:
        message += token
        yield message


llms = {}
chat_interface = pn.chat.ChatInterface(
    callback=callback,
    callback_user="Mistral",
)
chat_interface.send(
    "Send a message to get a reply from Mistral!", user="System", respond=False
)
chat_interface.servable()