added sentence-transformers encode service

app.py

@@ -11,9 +11,19 @@ from background_service import BackgroundTaskService
 # anvil.server.connect('PLMOIU5VCGGUOJH2XORIBWV3-ZXZVFLWX7QFIIAF4')
 anvil.server.connect("S3SLHUQ2BB33NVTP7FWRAOHS-NDSRD7CDALRPSPLL")
 
+from sentence_transformers import SentenceTransformer
+from sentence_transformers.util import cos_sim
+# model = SentenceTransformer('thenlper/gte-large')
+model = SentenceTransformer('BAAI/bge-large-en')
+
+@anvil.server.callable
+def encode(sentence = None):
+    vec = model.encode(sentence)
+    return [float(val) if isinstance(val, (int, float, np.float32)) else 0.0 for val in vec]
+
 app=Flask(__name__)
-MESSAGED={'title':'API Server',
-          'messageL':['published server functions:','encode(text)', 
+MESSAGED={'title':'API Server for ICAPP',
+          'messageL':['published server functions:','encode_anvil(text)', 'encode(sentence)',
                       'call_gemini(text,key)','call_gpt(text,key,model)',
                       'task_id<=launch(func_name,*args)','poll(task_id)']}
 
@@ -89,27 +99,6 @@ def encode():
     embedding=emb_array.tolist()
     return jsonify({'embedding': embedding})
 
-@app.route("/file/<string:filename>")
-def return_file(filename):
-    return send_file('./data/'+filename)
-
-@app.route('/run',methods=['GET','POST'])
-def run_script():
-    script=''
-    # print(request.method)
-    print(request)
-    if request.method=='GET': 
-        script=request.args.get('script')
-        print('I am in get')
-    elif request.method=='POST':
-        print('I am in post')
-        data=request.get_json()
-        if 'script' in data: script=data['script']
-    if script=='' or script is None: return 'INVALID'
-    os.system(script+' > ./out.txt')
-    with open('./out.txt','r') as f: output=f.read()
-    return output
-
 @app.route('/',methods=['GET', 'POST'])
 def home():
     return render_template('home.html',messageD=MESSAGED)

app_gradio.py

@@ -1,49 +0,0 @@
-import gradio as gr
-from threading import Thread
-import time
-import anvil.server
-import os
-anvil.server.connect('55MH4EBKM22EP4E6D5T6CVSL-VGO5X4SM6JEXGJVT')
-import json
-import ast
-
-def run_script(scriptname):
-    # return scriptname
-    os.system(scriptname+' > ./out.txt')
-    with open('./out.txt','r') as f: output=f.read()
-    return output
-
-@anvil.server.callable
-def run_command(scriptname):
-    os.system(scriptname+' > ./out.txt')
-    with open('./out.txt','r') as f: output=f.read()
-    return output
-
-@anvil.server.callable
-def get_file(filename):
-    m = BlobMedia('text/plain', 'Hello, world!', name='hello.txt')
-    return m
-
-gradio_interface = gr.Interface(
-  fn=run_script,
-  inputs="text",
-  outputs="text",
-  title="REST API with Gradio and Huggingface Spaces",
-  description='''Inputs should be json of test item e.g., as a dictionary; 
-  output right now is just returning the input; later label will be returned.
-
-  This is how to call the API from Python: 
-
-  import requests
-
-  response = requests.post("https://gmshroff-gmserver.hf.space/run/predict", json={
-  "data": [
-    "\<put some json string here\>",
-  ]}).json()
-
-  data = response["data"])
-
-  ''')
-
-gradio_interface.launch()
-

requirements.txt

@@ -7,3 +7,4 @@ numpy
 transformers
 google-generativeai
 openai
+sentence-transformers

scripts/update_valdata.py

@@ -1,35 +0,0 @@
-#!/usr/bin/env python
-# coding: utf-8
-
-# In[ ]:
-
-
-from numerapi import NumerAPI
-import os
-import pandas as pd
-import numpy as np
-
-
-# In[ ]:
-
-
-napi = NumerAPI()
-data_path='./data/'
-
-
-# In[ ]:
-
-
-napi.download_dataset("v4.2/validation_int8.parquet", data_path+"validation_int8.parquet")
-validation_data=pd.read_parquet(data_path+"validation_int8.parquet")
-recent_eras=list(validation_data.loc[validation_data['data_type']=='validation']['era'].unique()[-2:])
-validation_subset=validation_data[validation_data['era'].isin(recent_eras)]
-validation_subset.to_parquet(data_path+"validation_subset_int8.parquet",index=False)
-# napi.download_dataset('v4.2/live_int8.parquet',data_path+'live_int8.parquet')
-
-
-# In[ ]:
-
-
-# print("Now please copy the file to server via: scp ../../data/validation_subset_int8.parquet gms@gms1:/home/gms/numerai/data/.")
-