Update app.py

app.py

@@ -1,51 +1,33 @@
-import gradio as gr
-from transformers import AutoTokenizer, AutoModelForCausalLM
-import torch
-import tempfile
 import os
+import gradio as gr
+from huggingface_hub import InferenceClient
 
-# Create the offload folder
-offload_dir = './offload'
-os.makedirs(offload_dir, exist_ok=True)
+# Retrieve the token from environment variable
+token = os.getenv("HF_TOKEN")
 
-#"tiiuae/falcon-7b-instruct",
-model = AutoModelForCausalLM.from_pretrained(
-    "meta-llama/Meta-Llama-3-8B",    
-    torch_dtype=torch.bfloat16,
-    device_map="auto",
-    low_cpu_mem_usage=True,
-    offload_folder=offload_dir
+client = InferenceClient(
+    "meta-llama/Llama-3.2-3B-Instruct",
+    token=token,
 )
-tokenizer = AutoTokenizer.from_pretrained("tiiuae/meta-llama/Meta-Llama-3-8B")
-
-
-def generate_text(input_text):
-    input_ids = tokenizer.encode(input_text, return_tensors="pt")
-    attention_mask = torch.ones(input_ids.shape)
-
-    output = model.generate(
-        input_ids,
-        attention_mask=attention_mask,
-        max_length=200,
-        do_sample=True,
-        top_k=10,
-        num_return_sequences=1,
-        eos_token_id=tokenizer.eos_token_id,
-    )
-
-    output_text = tokenizer.decode(output[0], skip_special_tokens=True)
-    print(output_text)
-
-    # Remove Prompt Echo from Generated Text
-    cleaned_output_text = output_text.replace(input_text, "")
-    return cleaned_output_text
 
+def chat_with_llama(user_input):
+    response = ""
+    for message in client.chat_completion(
+        messages=[{"role": "user", "content": user_input}],
+        max_tokens=500,
+        stream=True,
+    ):
+        response += message.choices[0].delta.content
+    return response
+
+# Create a Gradio interface
+interface = gr.Interface(
+    fn=chat_with_llama,
+    inputs=gr.Textbox(label="Input Text", placeholder="Ask something..."),
+    outputs="text",
+    title="Chat with Llama 3",
+    description="Enter your message to chat with Llama 3. Type your question or prompt below.",
+)
 
-text_generation_interface = gr.Interface(
-    fn=generate_text,
-    inputs=[
-        gr.inputs.Textbox(label="Input Text"),
-    ],
-    outputs=gr.inputs.Textbox(label="Generated Text"),
-    title="---LLM---",
-).launch()
+if __name__ == "__main__":
+    interface.launch()