Spaces:
Runtime error
Runtime error
Update app.py
Browse files
app.py
CHANGED
@@ -1,3 +1,295 @@
|
|
1 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2 |
|
3 |
-
gradio.load("models/WizardLM/WizardCoder-15B-V1.0").launch()
|
|
|
1 |
+
"""Run codes"""
|
2 |
+
# import gradio
|
3 |
+
|
4 |
+
# gradio.load("models/WizardLM/WizardCoder-15B-V1.0").launch()
|
5 |
+
import os
|
6 |
+
import time
|
7 |
+
from types import SimpleNamespace
|
8 |
+
|
9 |
+
import gradio as gr
|
10 |
+
from about_time import about_time
|
11 |
+
from ctransformers import AutoConfig, AutoModelForCausalLM
|
12 |
+
from huggingface_hub import hf_hub_download
|
13 |
+
from loguru import logger
|
14 |
+
|
15 |
+
os.environ["TZ"] = "Asia/Shanghai"
|
16 |
+
try:
|
17 |
+
time.tzset() # type: ignore # pylint: disable=no-member
|
18 |
+
except Exception:
|
19 |
+
# Windows
|
20 |
+
logger.warning("Windows, cant run time.tzset()")
|
21 |
+
|
22 |
+
ns = SimpleNamespace(
|
23 |
+
response="",
|
24 |
+
generator=[],
|
25 |
+
)
|
26 |
+
|
27 |
+
|
28 |
+
def predict(prompt, bot):
|
29 |
+
# logger.debug(f"{prompt=}, {bot=}, {timeout=}")
|
30 |
+
logger.debug(f"{prompt=}, {bot=}")
|
31 |
+
|
32 |
+
ns.response = ""
|
33 |
+
with about_time() as atime: # type: ignore
|
34 |
+
try:
|
35 |
+
# user_prompt = prompt
|
36 |
+
generator = generate(llm, generation_config, system_prompt, prompt.strip())
|
37 |
+
print(assistant_prefix, end=" ", flush=True)
|
38 |
+
|
39 |
+
response = ""
|
40 |
+
buff.update(value="diggin...")
|
41 |
+
|
42 |
+
for word in generator:
|
43 |
+
# print(word, end="", flush=True)
|
44 |
+
print(word, flush=True) # vertical stream
|
45 |
+
response += word
|
46 |
+
ns.response = response
|
47 |
+
buff.update(value=response)
|
48 |
+
print("")
|
49 |
+
logger.debug(f"{response=}")
|
50 |
+
except Exception as exc:
|
51 |
+
logger.error(exc)
|
52 |
+
response = f"{exc=}"
|
53 |
+
|
54 |
+
# bot = {"inputs": [response]}
|
55 |
+
_ = (
|
56 |
+
f"(time elapsed: {atime.duration_human}, " # type: ignore
|
57 |
+
f"{atime.duration/(len(prompt) + len(response)):.1f}s/char)" # type: ignore
|
58 |
+
)
|
59 |
+
|
60 |
+
bot.append([prompt, f"{response} {_}"])
|
61 |
+
|
62 |
+
return prompt, bot
|
63 |
+
|
64 |
+
|
65 |
+
def predict_api(prompt):
|
66 |
+
logger.debug(f"{prompt=}")
|
67 |
+
ns.response = ""
|
68 |
+
try:
|
69 |
+
# user_prompt = prompt
|
70 |
+
generator = generate(llm, generation_config, system_prompt, prompt.strip())
|
71 |
+
print(assistant_prefix, end=" ", flush=True)
|
72 |
+
|
73 |
+
response = ""
|
74 |
+
buff.update(value="diggin...")
|
75 |
+
for word in generator:
|
76 |
+
print(word, end="", flush=True)
|
77 |
+
response += word
|
78 |
+
ns.response = response
|
79 |
+
buff.update(value=response)
|
80 |
+
print("")
|
81 |
+
logger.debug(f"{response=}")
|
82 |
+
except Exception as exc:
|
83 |
+
logger.error(exc)
|
84 |
+
response = f"{exc=}"
|
85 |
+
# bot = {"inputs": [response]}
|
86 |
+
# bot = [(prompt, response)]
|
87 |
+
|
88 |
+
return response
|
89 |
+
|
90 |
+
|
91 |
+
def download_quant(destination_folder: str, repo_id: str, model_filename: str):
|
92 |
+
local_path = os.path.abspath(destination_folder)
|
93 |
+
return hf_hub_download(
|
94 |
+
repo_id=repo_id,
|
95 |
+
filename=model_filename,
|
96 |
+
local_dir=local_path,
|
97 |
+
local_dir_use_symlinks=True,
|
98 |
+
)
|
99 |
+
|
100 |
+
|
101 |
+
logger.info("start dl")
|
102 |
+
_ = """full url: https://huggingface.co/TheBloke/mpt-30B-chat-GGML/blob/main/mpt-30b-chat.ggmlv0.q4_1.bin"""
|
103 |
+
|
104 |
+
# https://huggingface.co/TheBloke/mpt-30B-chat-GGML
|
105 |
+
_ = """
|
106 |
+
mpt-30b-chat.ggmlv0.q4_0.bin q4_0 4 16.85 GB 19.35 GB 4-bit.
|
107 |
+
mpt-30b-chat.ggmlv0.q4_1.bin q4_1 4 18.73 GB 21.23 GB 4-bit. Higher accuracy than q4_0 but not as high as q5_0. However has quicker inference than q5 models.
|
108 |
+
mpt-30b-chat.ggmlv0.q5_0.bin q5_0 5 20.60 GB 23.10 GB
|
109 |
+
mpt-30b-chat.ggmlv0.q5_1.bin q5_1 5 22.47 GB 24.97 GB
|
110 |
+
mpt-30b-chat.ggmlv0.q8_0.bin q8_0 8 31.83 GB 34.33 GB
|
111 |
+
"""
|
112 |
+
model_filename = "mpt-30b-chat.ggmlv0.q4_1.bin"
|
113 |
+
model_filename = "WizardCoder-15B-1.0.ggmlv3.q4_0.bin" # 10.7G
|
114 |
+
model_filename = "WizardCoder-15B-1.0.ggmlv3.q4_1.bin" # 11.9G
|
115 |
+
destination_folder = "models"
|
116 |
+
|
117 |
+
repo_id = "TheBloke/mpt-30B-chat-GGML"
|
118 |
+
if "WizardCoder" in model_filename:
|
119 |
+
repo_id = "TheBloke/WizardCoder-15B-1.0-GGML"
|
120 |
+
|
121 |
+
download_quant(destination_folder, repo_id, model_filename)
|
122 |
+
|
123 |
+
logger.info("done dl")
|
124 |
+
|
125 |
+
if "mpt" in model_filename:
|
126 |
+
config = AutoConfig.from_pretrained("mosaicml/mpt-30b-chat", context_length=8192)
|
127 |
+
llm = AutoModelForCausalLM.from_pretrained(
|
128 |
+
os.path.abspath(f"models/{model_filename}"),
|
129 |
+
model_type="mpt",
|
130 |
+
config=config,
|
131 |
+
)
|
132 |
+
|
133 |
+
# https://huggingface.co/spaces/matthoffner/wizardcoder-ggml/blob/main/main.py
|
134 |
+
if "WizardCoder" in model_filename:
|
135 |
+
llm = AutoModelForCausalLM.from_pretrained("TheBloke/WizardCoder-15B-1.0-GGML",
|
136 |
+
model_file="",
|
137 |
+
model_type="starcoder",
|
138 |
+
threads=8)
|
139 |
+
|
140 |
+
system_prompt = "A conversation between a user and an LLM-based AI assistant named Local Assistant. Local Assistant gives helpful and honest answers."
|
141 |
+
|
142 |
+
user_prefix = "[user]: "
|
143 |
+
assistant_prefix = "[assistant]: "
|
144 |
+
|
145 |
+
css = """
|
146 |
+
.importantButton {
|
147 |
+
background: linear-gradient(45deg, #7e0570,#5d1c99, #6e00ff) !important;
|
148 |
+
border: none !important;
|
149 |
+
}
|
150 |
+
.importantButton:hover {
|
151 |
+
background: linear-gradient(45deg, #ff00e0,#8500ff, #6e00ff) !important;
|
152 |
+
border: none !important;
|
153 |
+
}
|
154 |
+
.disclaimer {font-variant-caps: all-small-caps; font-size: xx-small;}
|
155 |
+
.xsmall {font-size: x-small;}
|
156 |
+
"""
|
157 |
+
|
158 |
+
with gr.Blocks(
|
159 |
+
# title="mpt-30b-chat-ggml",
|
160 |
+
title=f"{model_filename}",
|
161 |
+
theme=gr.themes.Soft(text_size="sm", spacing_size="sm"),
|
162 |
+
css=css,
|
163 |
+
) as block:
|
164 |
+
with gr.Accordion("🎈 Info", open=False):
|
165 |
+
# gr.HTML(
|
166 |
+
# """<center><a href="https://huggingface.co/spaces/mikeee/mpt-30b-chat?duplicate=true"><img src="https://bit.ly/3gLdBN6" alt="Duplicate"></a> and spin a CPU UPGRADE to avoid the queue</center>"""
|
167 |
+
# )
|
168 |
+
gr.Markdown(
|
169 |
+
f"""<h4><center>{model_filename}</center></h4>
|
170 |
+
|
171 |
+
The examples are meant for another model. You should try with
|
172 |
+
some coder-related prompts.
|
173 |
+
|
174 |
+
Try to refresh the browser and try again when occasionally errors occur.
|
175 |
+
|
176 |
+
It takes about >100 seconds to get a response. Restarting the space takes about 5 minutes if the space is asleep due to inactivity. If the space crashes for some reason, it will also take about 5 minutes to restart. You need to refresh the browser to reload the new space.
|
177 |
+
""",
|
178 |
+
elem_classes="xsmall",
|
179 |
+
)
|
180 |
+
conversation = Chat()
|
181 |
+
chatbot = gr.Chatbot(scroll_to_output=True).style(height=700) # 500
|
182 |
+
buff = gr.Textbox(show_label=False)
|
183 |
+
with gr.Row():
|
184 |
+
with gr.Column(scale=1):
|
185 |
+
msg = gr.Textbox(
|
186 |
+
label="Chat Message Box",
|
187 |
+
placeholder="Ask me anything (press Enter or click Submit to send)",
|
188 |
+
show_label=False,
|
189 |
+
).style(container=False)
|
190 |
+
with gr.Column(scale=0.1):
|
191 |
+
with gr.Row():
|
192 |
+
submit = gr.Button("Submit", elem_classes="xsmall")
|
193 |
+
stop = gr.Button("Stop", visible=False)
|
194 |
+
clear = gr.Button("Clear History", visible=True)
|
195 |
+
with gr.Row(visible=False):
|
196 |
+
with gr.Accordion("Advanced Options:", open=False):
|
197 |
+
with gr.Row():
|
198 |
+
with gr.Column(scale=2):
|
199 |
+
system = gr.Textbox(
|
200 |
+
label="System Prompt",
|
201 |
+
value=Chat.default_system_prompt,
|
202 |
+
show_label=False,
|
203 |
+
).style(container=False)
|
204 |
+
with gr.Column():
|
205 |
+
with gr.Row():
|
206 |
+
change = gr.Button("Change System Prompt")
|
207 |
+
reset = gr.Button("Reset System Prompt")
|
208 |
+
|
209 |
+
with gr.Accordion("Example inputs", open=True):
|
210 |
+
etext = """In America, where cars are an important part of the national psyche, a decade ago people had suddenly started to drive less, which had not happened since the oil shocks of the 1970s. """
|
211 |
+
examples = gr.Examples(
|
212 |
+
examples=[
|
213 |
+
["Explain the plot of Cinderella in a sentence."],
|
214 |
+
[
|
215 |
+
"How long does it take to become proficient in French, and what are the best methods for retaining information?"
|
216 |
+
],
|
217 |
+
["What are some common mistakes to avoid when writing code?"],
|
218 |
+
["Build a prompt to generate a beautiful portrait of a horse"],
|
219 |
+
["Suggest four metaphors to describe the benefits of AI"],
|
220 |
+
["Write a pop song about leaving home for the sandy beaches."],
|
221 |
+
["Write a summary demonstrating my ability to tame lions"],
|
222 |
+
["鲁迅和周树人什么关系 说中文"],
|
223 |
+
["鲁迅和周树人什么关系"],
|
224 |
+
["鲁迅和周树人什么关系 用英文回答"],
|
225 |
+
["从前有一头牛,这头牛后面有什么?"],
|
226 |
+
["正无穷大加一大于正无穷大吗?"],
|
227 |
+
["正无穷大加正无穷大大于正无穷大吗?"],
|
228 |
+
["-2的平方根等于什么"],
|
229 |
+
["树上有5只鸟,猎人开枪打死了一只。树上还有几只鸟?"],
|
230 |
+
["树上有11只鸟,猎人开枪打死了一只。树上还有几只鸟?提示:需考虑鸟可能受惊吓飞走。"],
|
231 |
+
["以红楼梦的行文风格写一张委婉的请假条。不少于320字。"],
|
232 |
+
[f"{etext} 翻成中文,列出3个版本"],
|
233 |
+
[f"{etext} \n 翻成中文,保留原意,但使用文学性的语言。不要写解释。列出3个版本"],
|
234 |
+
["js 判断一个数是不是质数"],
|
235 |
+
["js 实现python 的 range(10)"],
|
236 |
+
["js 实现python 的 [*(range(10)]"],
|
237 |
+
["假定 1 + 2 = 4, 试求 7 + 8"],
|
238 |
+
["Erkläre die Handlung von Cinderella in einem Satz."],
|
239 |
+
["Erkläre die Handlung von Cinderella in einem Satz. Auf Deutsch"],
|
240 |
+
],
|
241 |
+
inputs=[msg],
|
242 |
+
examples_per_page=40,
|
243 |
+
)
|
244 |
+
|
245 |
+
# with gr.Row():
|
246 |
+
with gr.Accordion("Disclaimer", open=False):
|
247 |
+
gr.Markdown(
|
248 |
+
f"Disclaimer: {"-".join(model_filename.split("-")[:2])} can produce factually incorrect output, and should not be relied on to produce "
|
249 |
+
"factually accurate information. MPT-30B was trained on various public datasets; while great efforts "
|
250 |
+
"have been taken to clean the pretraining data, it is possible that this model could generate lewd, "
|
251 |
+
"biased, or otherwise offensive outputs.",
|
252 |
+
elem_classes=["disclaimer"],
|
253 |
+
)
|
254 |
+
|
255 |
+
msg.submit(
|
256 |
+
# fn=conversation.user_turn,
|
257 |
+
fn=predict0,
|
258 |
+
inputs=[msg, chatbot],
|
259 |
+
outputs=[msg, chatbot],
|
260 |
+
queue=True,
|
261 |
+
show_progress="full",
|
262 |
+
api_name="predict",
|
263 |
+
)
|
264 |
+
submit.click(
|
265 |
+
# fn=predict0,
|
266 |
+
fn=lambda x, y: ("",) + predict0(x, y)[1:], # clear msg
|
267 |
+
inputs=[msg, chatbot],
|
268 |
+
outputs=[msg, chatbot],
|
269 |
+
queue=True,
|
270 |
+
show_progress="full",
|
271 |
+
)
|
272 |
+
clear.click(lambda: None, None, chatbot, queue=False)
|
273 |
+
|
274 |
+
# update buff Textbox, every: units in seconds)
|
275 |
+
# https://huggingface.co/spaces/julien-c/nvidia-smi/discussions
|
276 |
+
# does not work
|
277 |
+
# AttributeError: 'Blocks' object has no attribute 'run_forever'
|
278 |
+
# block.run_forever(lambda: ns.response, None, [buff], every=1)
|
279 |
+
|
280 |
+
with gr.Accordion("For Chat/Translation API", open=False, visible=False):
|
281 |
+
input_text = gr.Text()
|
282 |
+
api_btn = gr.Button("Go", variant="primary")
|
283 |
+
out_text = gr.Text()
|
284 |
+
api_btn.click(
|
285 |
+
predict_api,
|
286 |
+
input_text,
|
287 |
+
out_text,
|
288 |
+
# show_progress="full",
|
289 |
+
api_name="api",
|
290 |
+
)
|
291 |
+
|
292 |
+
# concurrency_count=5, max_size=20
|
293 |
+
# max_size=36, concurrency_count=14
|
294 |
+
block.queue(concurrency_count=5, max_size=20).launch(debug=True)
|
295 |
|
|