add handler and requirements

handler.py

@@ -0,0 +1,49 @@
+from typing import Any, Dict
+
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer, AutoConfig
+
+# from peft import PeftConfig, PeftModel
+
+
+class EndpointHandler:
+    def __init__(self, path=""):
+        # load model and processor from path
+        self.tokenizer = AutoTokenizer.from_pretrained(path)
+        # try:
+        # config = AutoConfig.from_pretrained(path)
+        model = AutoModelForCausalLM.from_pretrained(
+            path,
+            # return_dict=True,
+            # load_in_8bit=True,
+            device_map="auto",
+            torch_dtype=torch.float16,
+            # trust_remote_code=True,
+        )
+        # model.resize_token_embeddings(len(self.tokenizer))
+        # model = PeftModel.from_pretrained(model, path)
+        # except Exception:
+        #     model = AutoModelForCausalLM.from_pretrained(
+        #         path, device_map="auto", load_in_8bit=True, torch_dtype=torch.float16, trust_remote_code=True
+        #     )
+        self.model = model
+        self.device = "cuda" if torch.cuda.is_available() else "cpu"
+
+    def __call__(self, data: Dict[str, Any]) -> Dict[str, str]:
+        # process input
+        inputs = data.pop("inputs", data)
+        parameters = data.pop("parameters", None)
+
+        # preprocess
+        inputs = self.tokenizer(f"User: {inputs}\n\n", return_tensors="pt")
+
+        # pass inputs with all kwargs in data
+        if parameters is not None:
+            outputs = self.model.generate(**inputs.to(self.device), max_new_tokens=880, **parameters)
+        else:
+            outputs = self.model.generate(**inputs.to(self.device), max_new_tokens=880)
+
+        # postprocess the prediction
+        prediction = self.tokenizer.decode(outputs[0], skip_special_tokens=True)
+
+        return [{"generated_text": prediction}]

requirements.txt

@@ -0,0 +1,156 @@
+accelerate==0.26.1
+aiohttp==3.9.3
+aiosignal==1.3.1
+annotated-types==0.6.0
+anyio==4.2.0
+appdirs==1.4.4
+argon2-cffi==23.1.0
+argon2-cffi-bindings==21.2.0
+arrow==1.3.0
+asttokens==2.4.1
+async-lru==2.0.4
+attrs==23.2.0
+Babel==2.14.0
+beautifulsoup4==4.12.3
+bitsandbytes==0.42.0
+bleach==6.1.0
+certifi==2024.2.2
+cffi==1.16.0
+charset-normalizer==3.3.2
+click==8.1.7
+comm==0.2.1
+datasets==2.16.1
+debugpy==1.8.0
+decorator==5.1.1
+deepspeed==0.13.1
+defusedxml==0.7.1
+dill==0.3.7
+docker-pycreds==0.4.0
+executing==2.0.1
+fastjsonschema==2.19.1
+filelock==3.13.1
+fqdn==1.5.1
+frozenlist==1.4.1
+fsspec==2023.10.0
+gitdb==4.0.11
+GitPython==3.1.41
+h11==0.14.0
+hf_transfer==0.1.5
+hjson==3.1.0
+httpcore==1.0.2
+httpx==0.26.0
+huggingface-hub==0.20.3
+idna==3.6
+ipykernel==6.29.1
+ipython==8.21.0
+ipywidgets==8.1.1
+isoduration==20.11.0
+jedi==0.19.1
+Jinja2==3.1.3
+json5==0.9.14
+jsonpointer==2.4
+jsonschema==4.21.1
+jsonschema-specifications==2023.12.1
+jupyter-events==0.9.0
+jupyter-lsp==2.2.2
+jupyter_client==8.6.0
+jupyter_core==5.7.1
+jupyter_server==2.12.5
+jupyter_server_terminals==0.5.2
+jupyterlab==4.1.0
+jupyterlab-widgets==3.0.9
+jupyterlab_pygments==0.3.0
+jupyterlab_server==2.25.2
+MarkupSafe==2.1.5
+matplotlib-inline==0.1.6
+mistune==3.0.2
+mpmath==1.3.0
+multidict==6.0.5
+multiprocess==0.70.15
+nbclient==0.9.0
+nbconvert==7.15.0
+nbformat==5.9.2
+nest-asyncio==1.6.0
+networkx==3.2.1
+ninja==1.11.1.1
+notebook==7.0.7
+notebook_shim==0.2.3
+numpy==1.26.4
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==8.9.2.26
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-nccl-cu12==2.19.3
+nvidia-nvjitlink-cu12==12.3.101
+nvidia-nvtx-cu12==12.1.105
+overrides==7.7.0
+packaging==23.2
+pandas==2.2.0
+pandocfilters==1.5.1
+parso==0.8.3
+peft==0.8.2
+pexpect==4.9.0
+platformdirs==4.2.0
+prometheus-client==0.19.0
+prompt-toolkit==3.0.43
+protobuf==4.25.2
+psutil==5.9.8
+ptyprocess==0.7.0
+pure-eval==0.2.2
+py-cpuinfo==9.0.0
+pyarrow==15.0.0
+pyarrow-hotfix==0.6
+pycparser==2.21
+pydantic==2.6.1
+pydantic_core==2.16.2
+Pygments==2.17.2
+pynvml==11.5.0
+python-dateutil==2.8.2
+python-json-logger==2.0.7
+pytz==2024.1
+PyYAML==6.0.1
+pyzmq==25.1.2
+referencing==0.33.0
+regex==2023.12.25
+requests==2.31.0
+rfc3339-validator==0.1.4
+rfc3986-validator==0.1.1
+rpds-py==0.17.1
+safetensors==0.4.2
+scipy==1.12.0
+Send2Trash==1.8.2
+sentry-sdk==1.40.2
+setproctitle==1.3.3
+six==1.16.0
+smmap==5.0.1
+sniffio==1.3.0
+soupsieve==2.5
+stack-data==0.6.3
+sympy==1.12
+terminado==0.18.0
+tinycss2==1.2.1
+tokenizers==0.15.1
+torch==2.2.0
+tornado==6.4
+tqdm==4.66.1
+traitlets==5.14.1
+transformers==4.37.2
+triton==2.2.0
+types-python-dateutil==2.8.19.20240106
+typing_extensions==4.9.0
+tzdata==2023.4
+uri-template==1.3.0
+urllib3==2.2.0
+wandb==0.16.3
+wcwidth==0.2.13
+webcolors==1.13
+webencodings==0.5.1
+websocket-client==1.7.0
+widgetsnbextension==4.0.9
+xxhash==3.4.1
+yarl==1.9.4