IMP new mistral

app.py

@@ -9,32 +9,32 @@ load_dotenv()
 
 async def handle(system_input: str, user_input: str):
     print(system_input, user_input)
-    buffers = ["", "", "", ""]
+    buffers = ["", "", "", "", ""]
     async for outputs in async_zip_stream(
             generate_gpt2(system_input, user_input),
             generate_mistral_7bvo1(system_input, user_input),
             generate_llama2(system_input, user_input),
             generate_llama3(system_input, user_input),
+            generate_mistral_7bvo3(system_input, user_input),
     ):
         # gpt_output, mistral_output, llama_output, llama2_output, llama3_output, llama4_output = outputs
         for i, b in enumerate(buffers):
             buffers[i] += str(outputs[i])
 
         yield list(buffers) + ["", ""]
-    yield list(buffers) + [(openllama_generation := generate_openllama(system_input, user_input)), '']
-    yield list(buffers) + [openllama_generation, generate_bloom(system_input, user_input)]
+    yield list(buffers) + [generate_bloom(system_input, user_input)]
 
 
 with gr.Blocks() as demo:
     system_input = gr.Textbox(label='System Input', value='You are AI assistant', lines=2)
     with gr.Row():
         gpt = gr.Textbox(label='gpt-2', lines=4, interactive=False)
-        mistral = gr.Textbox(label='mistral', lines=4, interactive=False)
-        llama = gr.Textbox(label='openllama', lines=4, interactive=False)
+        mistral = gr.Textbox(label='mistral-v01', lines=4, interactive=False)
+        mistral_new = gr.Textbox(label='mistral-v03', lines=4, interactive=False)
     with gr.Row():
         llama2 = gr.Textbox(label='llama-2', lines=4, interactive=False)
         llama3 = gr.Textbox(label='llama-3', lines=4, interactive=False)
-        bloom = gr.Textbox(label='bloom', lines=4, interactive=False)
+        bloom = gr.Textbox(label='bloom [GPU]', lines=4, interactive=False)
 
     user_input = gr.Textbox(label='User Input', lines=2)
     gen_button = gr.Button('Generate')
@@ -42,7 +42,7 @@ with gr.Blocks() as demo:
     gen_button.click(
         fn=handle,
         inputs=[system_input, user_input],
-        outputs=[gpt, mistral, llama2, llama3, llama, bloom],
+        outputs=[gpt, mistral, llama2, llama3, mistral_new, bloom],
     )
 
 demo.launch()

generators.py

@@ -38,6 +38,22 @@ async def generate_mistral_7bvo1(system_input, user_input):
         yield message.choices[0].delta.content
 
 
+async def generate_mistral_7bvo3(system_input, user_input):
+    client = AsyncInferenceClient(
+        "mistralai/Mistral-7B-Instruct-v0.3",
+        token=os.getenv('HF_TOKEN'),
+    )
+
+    async for message in await client.chat_completion(
+            messages=[
+                {"role": "system", "content": system_input},
+                {"role": "user", "content": user_input}, ],
+            max_tokens=256,
+            stream=True,
+    ):
+        yield message.choices[0].delta.content
+
+
 async def generate_gpt2(system_input, user_input):
     output = await query_llm({
         "inputs": (inputs:=f"{system_input}\n{user_input}"),