Update app.py

app.py

@@ -7,6 +7,10 @@ import aiohttp
 import json
 import torch
 import re
+import nest_asyncio
+from hashlib import md5
+
+nest_asyncio.apply()
 
 repo_name = "BeardedMonster/SabiYarn-125M"
 device = "cuda" if torch.cuda.is_available() else "cpu"
@@ -147,6 +151,14 @@ async def generate_from_api(user_input, generation_config):
 
     return "FAILED"
 
+def generate_cache_key(user_input, generation_config):
+    key_data = f"{user_input}_{json.dumps(generation_config, sort_keys=True)}"
+    return md5(key_data.encode()).hexdigest()
+
+@st.cache_data(show_spinner=False)
+def get_cached_response(user_input, generation_config):
+    return asyncio.run(generate_from_api(user_input, generation_config))
+
 
 # Sample texts
 sample_texts = {
@@ -245,10 +257,12 @@ if st.button("Generate"):
             # Attempt the asynchronous API call
         generation_config["max_new_tokens"] = min(max_new_tokens, 1024 - len(tokenizer.tokenize(wrapped_input)))
         # generated_text = asyncio.run(generate_from_api(wrapped_input, generation_config))
+        cache_key = generate_cache_key(user_input, generation_config)
+        generated_text = get_cached_response(user_input, generation_config)
         
-        loop = asyncio.new_event_loop()
-        asyncio.set_event_loop(loop)
-        generated_text = loop.run_until_complete(generate_from_api(wrapped_input, generation_config))
+        # loop = asyncio.new_event_loop()
+        # asyncio.set_event_loop(loop)
+        # generated_text = loop.run_until_complete(generate_from_api(wrapped_input, generation_config))
         # except Exception as e:
             # print(f"API call failed: {e}. Using local model for text generation.")                
             # Use the locally loaded model for text generation