IMP t5

app.py

@@ -15,7 +15,7 @@ async def handle(system_input: str, user_input: str):
             generate_mistral_7bvo1(system_input, user_input),
             generate_llama2(system_input, user_input),
             generate_llama3(system_input, user_input),
-            generate_mistral_7bvo3(system_input, user_input),
+            generate_t5(system_input, user_input),
     ):
         # gpt_output, mistral_output, llama_output, llama2_output, llama3_output, llama4_output = outputs
         for i, b in enumerate(buffers):
@@ -30,7 +30,7 @@ with gr.Blocks() as demo:
     with gr.Row():
         gpt = gr.Textbox(label='gpt-2', lines=4, interactive=False)
         mistral = gr.Textbox(label='mistral-v01', lines=4, interactive=False)
-        mistral_new = gr.Textbox(label='mistral-v03', lines=4, interactive=False)
+        t5 = gr.Textbox(label='t5', lines=4, interactive=False)
     with gr.Row():
         llama2 = gr.Textbox(label='llama-2', lines=4, interactive=False)
         llama3 = gr.Textbox(label='llama-3', lines=4, interactive=False)
@@ -42,7 +42,7 @@ with gr.Blocks() as demo:
     gen_button.click(
         fn=handle,
         inputs=[system_input, user_input],
-        outputs=[gpt, mistral, llama2, llama3, mistral_new, bloom],
+        outputs=[gpt, mistral, llama2, llama3, t5, bloom],
     )
 
 demo.launch()

generators.py

@@ -38,25 +38,16 @@ async def generate_mistral_7bvo1(system_input, user_input):
         yield message.choices[0].delta.content
 
 
-async def generate_mistral_7bvo3(system_input, user_input):
-    client = AsyncInferenceClient(
-        "mistralai/Mistral-7B-Instruct-v0.3",
-        token=os.getenv('HF_TOKEN'),
-    )
-
-    async for message in await client.chat_completion(
-            messages=[
-                {"role": "system", "content": system_input},
-                {"role": "user", "content": user_input}, ],
-            max_tokens=256,
-            stream=True,
-    ):
-        yield message.choices[0].delta.content
+async def generate_t5(system_input, user_input):
+    output = await query_llm({
+        "inputs": (inputs := f"{system_input}\n{user_input}"),
+    }, "google/flan-t5-large")
+    yield output[0]["generated_text"]
 
 
 async def generate_gpt2(system_input, user_input):
     output = await query_llm({
-        "inputs": (inputs:=f"{system_input}\n{user_input}"),
+        "inputs": (inputs := f"{system_input}\n{user_input}"),
     }, "openai-community/gpt2")
     yield output[0]["generated_text"]
 
@@ -103,7 +94,6 @@ def generate_bloom(system_input, user_input):
     return tokenizer.decode(output[0], skip_special_tokens=True)
 
 
-
 async def generate_llama3(system_input, user_input):
     client = AsyncInferenceClient(
         "meta-llama/Meta-Llama-3.1-8B-Instruct",