implemented background service for async launching of registered (in app) library functions

.gitignore

@@ -0,0 +1,2 @@
+test.ipynb
+allkeys.py

app.py

@@ -5,6 +5,9 @@ import anvil.server
 import pathlib
 import textwrap
 import google.generativeai as genai
+import import_ipynb
+from library import call_gpt, call_gemini
+from background_service import BackgroundTaskService
 
 anvil.server.connect('PLMOIU5VCGGUOJH2XORIBWV3-ZXZVFLWX7QFIIAF4')
 
@@ -15,14 +18,41 @@ MESSAGED={'title':'API Server',
 tokenizer = AutoTokenizer.from_pretrained('allenai/specter')
 encoder = AutoModel.from_pretrained('allenai/specter')
 
-GOOGLE_API_KEY=os.getenv('GOOGLE_API_KEY')
-genai.configure(api_key=GOOGLE_API_KEY)
+# GOOGLE_API_KEY=os.getenv('GOOGLE_API_KEY')
+# genai.configure(api_key=GOOGLE_API_KEY)
+
+service=BackgroundTaskService(max_tasks=10)
+service.register(call_gpt)
+service.register(call_gemini)
+
+@anvil.server.callable
+def launch(func_name,*args):
+    global service
+    # Launch task
+    task_id = service.launch_task(func_name, *args)
+    print(f"Task launched with ID: {task_id}")
+    return task_id
 
 @anvil.server.callable
-def call_gemini(text):
-    model = genai.GenerativeModel('gemini-pro')
-    response = model.generate_content(text)
-    return response.text
+def poll(task_id):
+    global service
+    # Poll for completion; if not complete return "In Progress" else return result
+    result = service.get_result(task_id)
+    if result=='No such task': return str(result)
+    elif result!='In Progress': 
+        del service.results[task_id]
+        if isinstance(result, (int, float, str, list, dict, tuple)): 
+            return result
+        else: 
+            print(str(result))
+            return str(result)
+    else: return str(result)
+
+# @anvil.server.callable
+# def call_gemini(text):
+#     model = genai.GenerativeModel('gemini-pro')
+#     response = model.generate_content(text)
+#     return response.text
 
 @anvil.server.callable
 def encode_anvil(text):

background_service.ipynb

@@ -0,0 +1,84 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import threading\n",
+    "import queue\n",
+    "import secrets\n",
+    "import concurrent.futures\n",
+    "from typing import Callable, Any, Dict"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class BackgroundTaskService:\n",
+    "    def __init__(self, max_tasks: int):\n",
+    "        self.max_tasks = max_tasks\n",
+    "        self.task_queue = queue.Queue()\n",
+    "        self.results = {}\n",
+    "        self.executor = concurrent.futures.ThreadPoolExecutor(max_workers=max_tasks)\n",
+    "        self.lock = threading.Lock()  # To handle concurrent access to results dictionary\n",
+    "        threading.Thread(target=self._worker, daemon=True).start()\n",
+    "        self.registry={}\n",
+    "    def register(self,func):\n",
+    "        self.registry[func.__name__]=func\n",
+    "    def _worker(self):\n",
+    "        while True:\n",
+    "            task_id, func, args = self.task_queue.get()\n",
+    "            result = self.executor.submit(func, *args).result()\n",
+    "            with self.lock:\n",
+    "                self.results[task_id] = result\n",
+    "\n",
+    "    def launch_task(self, func_name, *args) -> Any:\n",
+    "        func=self.registry[func_name]\n",
+    "        if self.task_queue.qsize() >= self.max_tasks:\n",
+    "            return \"Queue Full\"\n",
+    "        task_id = secrets.token_hex(16)\n",
+    "        self.task_queue.put((task_id, func, args))\n",
+    "        with self.lock:\n",
+    "            self.results[task_id] = \"In Progress\"\n",
+    "        return task_id\n",
+    "\n",
+    "    def get_result(self, task_id) -> Any:\n",
+    "        with self.lock:\n",
+    "            return self.results.get(task_id, \"No such task\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "py310all",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.13"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

library.ipynb

@@ -0,0 +1,75 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import anvil.server\n",
+    "import openai\n",
+    "import pathlib\n",
+    "import textwrap\n",
+    "import google.generativeai as genai"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def call_gemini(text,key):\n",
+    "    # response=f'calling gemini with key {key} and text {text}'\n",
+    "    # return response\n",
+    "    genai.configure(api_key=key)\n",
+    "    model = genai.GenerativeModel('gemini-pro')\n",
+    "    response = model.generate_content(text)\n",
+    "    return response.text"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def call_gpt(prompt,key,model):\n",
+    "    openai.api_key=key\n",
+    "    try:\n",
+    "        messages=[{\"role\": \"system\", \"content\": \"You are a helpful assistant.\"}]\n",
+    "        messages+=[{\"role\": \"user\", \"content\": prompt}]\n",
+    "        completions=openai.chat.completions.create(  #for new version >.28 ) \n",
+    "        # completions=openai.ChatCompletion.create(\n",
+    "        model=model,   \n",
+    "        messages=messages)\n",
+    "        # prediction=completions['choices'][0]['message']['content']\n",
+    "        prediction=completions.choices[0].message.content.strip() # for new version >.28\n",
+    "    except Exception as e:\n",
+    "        return -1,str(e)\n",
+    "    return 0,prediction"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "py310all",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.13"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

test.ipynb

@@ -11,7 +11,9 @@
     "import requests\n",
     "import json\n",
     "from urllib.request import urlretrieve\n",
-    "import pandas as pd"
+    "import pandas as pd\n",
+    "import time\n",
+    "from allkeys import OPENAIKEY, GEMENIKEY"
    ]
   },
   {
@@ -24,6 +26,130 @@
     "anvil.server.connect('PLMOIU5VCGGUOJH2XORIBWV3-ZXZVFLWX7QFIIAF4')"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def fetch_result(task_id):\n",
+    "    while True:\n",
+    "        result=anvil.server.call('poll',task_id)\n",
+    "        if result!='In Progress' or result=='No such task': break\n",
+    "        else: \n",
+    "            time.sleep(1)\n",
+    "            print(result)\n",
+    "    print(result)\n",
+    "    return result"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "text='write a python function to compute the nth digit of pi'\n",
+    "model='gpt-3.5-turbo'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "task_id=anvil.server.call('launch','call_gemini',text,GEMENIKEY)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "task_id=anvil.server.call('launch','call_gpt',text,OPENAIKEY,model)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "fetch_result(task_id)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "print(result)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "print(result[1],end='\\n')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pathlib\n",
+    "import textwrap\n",
+    "from IPython.display import display\n",
+    "from IPython.display import Markdown\n",
+    "\n",
+    "def to_markdown(text):\n",
+    "  text = text.replace('•', '  *')\n",
+    "  return Markdown(textwrap.indent(text, '> ', predicate=lambda _: True))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "prompt='write code that defines a transformer network from scratch in pytorch'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "response=anvil.server.call('call_gemini',prompt)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "anvil.server.call('encode_anvil',prompt)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "to_markdown(response)"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -173,6 +299,48 @@
    "source": [
     "df=pd.read_parquet('/tmp/validation_subset_int8.parquet')"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import torch\n",
+    "import torch.nn as nn\n",
+    "import torch.nn.functional as F\n",
+    "\n",
+    "class Transformer(nn.Module):\n",
+    "    def __init__(self, d_model, nhead, num_encoder_layers, num_decoder_layers, dim_feedforward, dropout=0.1):\n",
+    "        super(Transformer, self).__init__()\n",
+    "        self.transformer = nn.Transformer(d_model, nhead, num_encoder_layers, num_decoder_layers, dim_feedforward, dropout)\n",
+    "\n",
+    "    def forward(self, src, tgt):\n",
+    "        output = self.transformer(src, tgt)\n",
+    "        return output\n",
+    "\n",
+    "# Example usage:\n",
+    "# Define the model parameters\n",
+    "d_model = 512\n",
+    "nhead = 8\n",
+    "num_encoder_layers = 6\n",
+    "num_decoder_layers = 6\n",
+    "dim_feedforward = 2048\n",
+    "dropout = 0.1\n",
+    "\n",
+    "# Initialize the model\n",
+    "model = Transformer(d_model, nhead, num_encoder_layers, num_decoder_layers, dim_feedforward, dropout)\n",
+    "\n",
+    "# Generate some sample data\n",
+    "src = torch.rand(10, 32, 512)\n",
+    "tgt = torch.rand(20, 32, 512)\n",
+    "\n",
+    "# Pass the data through the model\n",
+    "output = model(src, tgt)\n",
+    "\n",
+    "# Print the output shape\n",
+    "print(output.shape)"
+   ]
   }
  ],
  "metadata": {