"""Run codes""" # import gradio # gradio.load("models/WizardLM/WizardCoder-15B-V1.0").launch() import os import time from types import SimpleNamespace import gradio as gr from about_time import about_time from ctransformers import AutoConfig, AutoModelForCausalLM from huggingface_hub import hf_hub_download from loguru import logger os.environ["TZ"] = "Asia/Shanghai" try: time.tzset() # type: ignore # pylint: disable=no-member except Exception: # Windows logger.warning("Windows, cant run time.tzset()") ns = SimpleNamespace( response="", generator=[], ) def predict(prompt, bot): # logger.debug(f"{prompt=}, {bot=}, {timeout=}") logger.debug(f"{prompt=}, {bot=}") ns.response = "" with about_time() as atime: # type: ignore try: # user_prompt = prompt generator = generate(llm, generation_config, system_prompt, prompt.strip()) print(assistant_prefix, end=" ", flush=True) response = "" buff.update(value="diggin...") for word in generator: # print(word, end="", flush=True) print(word, flush=True) # vertical stream response += word ns.response = response buff.update(value=response) print("") logger.debug(f"{response=}") except Exception as exc: logger.error(exc) response = f"{exc=}" # bot = {"inputs": [response]} _ = ( f"(time elapsed: {atime.duration_human}, " # type: ignore f"{atime.duration/(len(prompt) + len(response)):.1f}s/char)" # type: ignore ) bot.append([prompt, f"{response} {_}"]) return prompt, bot def predict_api(prompt): logger.debug(f"{prompt=}") ns.response = "" try: # user_prompt = prompt generator = generate(llm, generation_config, system_prompt, prompt.strip()) print(assistant_prefix, end=" ", flush=True) response = "" buff.update(value="diggin...") for word in generator: print(word, end="", flush=True) response += word ns.response = response buff.update(value=response) print("") logger.debug(f"{response=}") except Exception as exc: logger.error(exc) response = f"{exc=}" # bot = {"inputs": [response]} # bot = [(prompt, response)] return response def download_quant(destination_folder: str, repo_id: str, model_filename: str): local_path = os.path.abspath(destination_folder) return hf_hub_download( repo_id=repo_id, filename=model_filename, local_dir=local_path, local_dir_use_symlinks=True, ) logger.info("start dl") _ = """full url: https://huggingface.co/TheBloke/mpt-30B-chat-GGML/blob/main/mpt-30b-chat.ggmlv0.q4_1.bin""" # https://huggingface.co/TheBloke/mpt-30B-chat-GGML _ = """ mpt-30b-chat.ggmlv0.q4_0.bin q4_0 4 16.85 GB 19.35 GB 4-bit. mpt-30b-chat.ggmlv0.q4_1.bin q4_1 4 18.73 GB 21.23 GB 4-bit. Higher accuracy than q4_0 but not as high as q5_0. However has quicker inference than q5 models. mpt-30b-chat.ggmlv0.q5_0.bin q5_0 5 20.60 GB 23.10 GB mpt-30b-chat.ggmlv0.q5_1.bin q5_1 5 22.47 GB 24.97 GB mpt-30b-chat.ggmlv0.q8_0.bin q8_0 8 31.83 GB 34.33 GB """ model_filename = "mpt-30b-chat.ggmlv0.q4_1.bin" model_filename = "WizardCoder-15B-1.0.ggmlv3.q4_0.bin" # 10.7G model_filename = "WizardCoder-15B-1.0.ggmlv3.q4_1.bin" # 11.9G destination_folder = "models" repo_id = "TheBloke/mpt-30B-chat-GGML" if "WizardCoder" in model_filename: repo_id = "TheBloke/WizardCoder-15B-1.0-GGML" download_quant(destination_folder, repo_id, model_filename) logger.info("done dl") # if "mpt" in model_filename: # config = AutoConfig.from_pretrained("mosaicml/mpt-30b-cha t", context_length=8192) # llm = AutoModelForCausalLM.from_pretrained( # os.path.abspath(f"models/{model_filename}"), # model_type="mpt", # config=config, # ) # https://huggingface.co/spaces/matthoffner/wizardcoder-ggml/blob/main/main.py if "WizardCoder" in model_filename: llm = AutoModelForCausalLM.from_pretrained("TheBloke/WizardCoder-15B-1.0-GGML", model_file="", model_type="starcoder", threads=8) default_system_prompt = "A conversation between a user and an LLM-based AI assistant named Local Assistant. Local Assistant gives helpful and honest answers." user_prefix = "[user]: " assistant_prefix = "[assistant]: " css = """ .importantButton { background: linear-gradient(45deg, #7e0570,#5d1c99, #6e00ff) !important; border: none !important; } .importantButton:hover { background: linear-gradient(45deg, #ff00e0,#8500ff, #6e00ff) !important; border: none !important; } .disclaimer {font-variant-caps: all-small-caps; font-size: xx-small;} .xsmall {font-size: x-small;} """ with gr.Blocks( # title="mpt-30b-chat-ggml", title=f"{model_filename}", theme=gr.themes.Soft(text_size="sm", spacing_size="sm"), css=css, ) as block: with gr.Accordion("🎈 Info", open=False): # gr.HTML( # """