Update app.py

app.py

@@ -24,12 +24,15 @@ LICENSE = """
 if not torch.cuda.is_available():
     DESCRIPTION += "\n<p>Running on CPU 🥶 This demo does not work on CPU.</p>"
 
-# Define the available models
-MODELS = [
-    {"name": "Nekochu/Luminia-13B-v3", "id": "Nekochu/Luminia-13B-v3"},
-    {"name": "Nekochu/Llama-2-13B-German-ORPO", "id": "Nekochu/Llama-2-13B-German-ORPO"},
-    # Add more models here in the future
-]
+
+if torch.cuda.is_available():
+    model_id = "Nekochu/Luminia-13B-v3"
+    model = AutoModelForCausalLM.from_pretrained(model_id, device_map="auto", load_in_4bit=True)
+    tokenizer = AutoTokenizer.from_pretrained(model_id)
+    tokenizer.use_default_system_prompt = False
+
+
+models_cache = {}
 
 @spaces.GPU(duration=120)
 def generate(
@@ -43,12 +46,13 @@ def generate(
     top_k: int = 50,
     repetition_penalty: float = 1.2,
 ) -> Iterator[str]:
-    # Load the model and tokenizer based on the selected model ID
-    model = AutoModelForCausalLM.from_pretrained(model_id, device_map="auto", load_in_4bit=True)
-    tokenizer = AutoTokenizer.from_pretrained(model_id)
-    tokenizer.use_default_system_prompt = False
-
-    conversation = []
+    if model_id not in models_cache:
+        model = AutoModelForCausalLM.from_pretrained(model_id, device_map="auto", load_in_4bit=True)
+        tokenizer = AutoTokenizer.from_pretrained(model_id)
+        tokenizer.use_default_system_prompt = False
+        models_cache[model_id] = (model, tokenizer)
+    else:
+        model, tokenizer = models_cache[model_id]
     if system_prompt:
         conversation.append({"role": "system", "content": system_prompt})
     for user, assistant in chat_history:
@@ -81,17 +85,11 @@ def generate(
         outputs.append(text)
         yield "".join(outputs)
 
-# Add a dropdown for model selection
-model_dropdown = gr.Dropdown(
-    label="Select Model",
-    choices=[model["name"] for model in MODELS],
-    value=MODELS[0]["name"], # Default to the first model
-)
 
 chat_interface = gr.ChatInterface(
     fn=generate,
     additional_inputs=[
-        model_dropdown,
+        gr.Textbox(label="Model ID", default="Nekochu/Luminia-13B-v3"),
         gr.Textbox(label="System prompt", lines=6),
         gr.Slider(
             label="Max new tokens",