Spaces:
Running
Running
Commit
•
a09c96f
1
Parent(s):
0ea985e
Update app.py
Browse filesAdded the support for mistral codestral throught simple, inference client.
(Note : Still in development)
app.py
CHANGED
@@ -4,7 +4,7 @@ import os
|
|
4 |
|
5 |
API_URL = {
|
6 |
"Mistral" : "https://api-inference.huggingface.co/models/mistralai/Mixtral-8x7B-Instruct-v0.1",
|
7 |
-
"Codestral" : ""
|
8 |
}
|
9 |
|
10 |
HF_TOKEN = os.environ['HF_TOKEN']
|
@@ -16,7 +16,7 @@ mistralClient = InferenceClient(
|
|
16 |
)
|
17 |
|
18 |
codestralClient = InferenceClient(
|
19 |
-
API_URL["Codestral"],
|
20 |
headers = {"Authorization" : f"Bearer {HF_TOKEN}"},
|
21 |
)
|
22 |
|
@@ -32,11 +32,13 @@ def format_prompt(message, history, enable_hinglish=False):
|
|
32 |
prompt += f"[INST] {message} [/INST]"
|
33 |
return prompt
|
34 |
|
35 |
-
def generate(prompt, history, temperature=0.9, max_new_tokens=256, top_p=0.95, repetition_penalty=1.0
|
36 |
-
temperature = float(temperature)
|
37 |
if temperature < 1e-2:
|
38 |
temperature = 1e-2
|
|
|
39 |
top_p = float(top_p)
|
|
|
40 |
generate_kwargs = dict(
|
41 |
temperature=temperature,
|
42 |
max_new_tokens=max_new_tokens,
|
@@ -45,6 +47,9 @@ def generate(prompt, history, temperature=0.9, max_new_tokens=256, top_p=0.95, r
|
|
45 |
do_sample=True,
|
46 |
seed=42,
|
47 |
)
|
|
|
|
|
|
|
48 |
|
49 |
formatted_prompt = format_prompt(prompt, history, enable_hinglish)
|
50 |
stream = client.text_generation(formatted_prompt, **generate_kwargs, stream=True, details=True, return_full_text=False)
|
@@ -55,6 +60,19 @@ def generate(prompt, history, temperature=0.9, max_new_tokens=256, top_p=0.95, r
|
|
55 |
return output
|
56 |
|
57 |
additional_inputs=[
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
58 |
gr.Slider(
|
59 |
label="Temperature",
|
60 |
value=0.9,
|
@@ -91,12 +109,7 @@ additional_inputs=[
|
|
91 |
interactive=True,
|
92 |
info="Penalize repeated tokens",
|
93 |
),
|
94 |
-
|
95 |
-
label="Hinglish",
|
96 |
-
value=False,
|
97 |
-
interactive=True,
|
98 |
-
info="Enables the MistralTalk to talk in Hinglish (Combination of Hindi and English)",
|
99 |
-
)
|
100 |
]
|
101 |
|
102 |
css = """
|
@@ -115,7 +128,7 @@ with gr.Blocks(css=css) as demo:
|
|
115 |
generate,
|
116 |
additional_inputs=additional_inputs,
|
117 |
theme = gr.themes.Soft(),
|
118 |
-
examples=[["What is the secret to life?"], ["How the universe works?"],["What can you do?"],["What is quantum mechanics?"],["Do you belive in after life?"]]
|
119 |
)
|
120 |
|
121 |
demo.queue(max_size=100).launch(debug=True)
|
|
|
4 |
|
5 |
API_URL = {
|
6 |
"Mistral" : "https://api-inference.huggingface.co/models/mistralai/Mixtral-8x7B-Instruct-v0.1",
|
7 |
+
"Codestral" : "mistralai/Codestral-22B-v0.1"
|
8 |
}
|
9 |
|
10 |
HF_TOKEN = os.environ['HF_TOKEN']
|
|
|
16 |
)
|
17 |
|
18 |
codestralClient = InferenceClient(
|
19 |
+
model = API_URL["Codestral"],
|
20 |
headers = {"Authorization" : f"Bearer {HF_TOKEN}"},
|
21 |
)
|
22 |
|
|
|
32 |
prompt += f"[INST] {message} [/INST]"
|
33 |
return prompt
|
34 |
|
35 |
+
def generate(prompt, history, model = "Mistral", enable_hinglish=False, temperature=0.9, max_new_tokens=256, top_p=0.95, repetition_penalty=1.0):
|
36 |
+
temperature = float(temperature) # Generation arguments
|
37 |
if temperature < 1e-2:
|
38 |
temperature = 1e-2
|
39 |
+
|
40 |
top_p = float(top_p)
|
41 |
+
|
42 |
generate_kwargs = dict(
|
43 |
temperature=temperature,
|
44 |
max_new_tokens=max_new_tokens,
|
|
|
47 |
do_sample=True,
|
48 |
seed=42,
|
49 |
)
|
50 |
+
|
51 |
+
# Selecting model to be used
|
52 |
+
client = mistralClient if(model == "Mistral") else codestralClient
|
53 |
|
54 |
formatted_prompt = format_prompt(prompt, history, enable_hinglish)
|
55 |
stream = client.text_generation(formatted_prompt, **generate_kwargs, stream=True, details=True, return_full_text=False)
|
|
|
60 |
return output
|
61 |
|
62 |
additional_inputs=[
|
63 |
+
gr.Dropdown(
|
64 |
+
choices = ["Mistral","Codestral"],
|
65 |
+
value = "Mistral",
|
66 |
+
label = "Model to be used",
|
67 |
+
interactive=True,
|
68 |
+
info = "Mistral for general-purpose chatting and codestral for code related task (Supports 80+ languages)"
|
69 |
+
),
|
70 |
+
gr.Checkbox(
|
71 |
+
label="Hinglish",
|
72 |
+
value=False,
|
73 |
+
interactive=True,
|
74 |
+
info="Enables the MistralTalk to talk in Hinglish (Combination of Hindi and English)",
|
75 |
+
),
|
76 |
gr.Slider(
|
77 |
label="Temperature",
|
78 |
value=0.9,
|
|
|
109 |
interactive=True,
|
110 |
info="Penalize repeated tokens",
|
111 |
),
|
112 |
+
|
|
|
|
|
|
|
|
|
|
|
113 |
]
|
114 |
|
115 |
css = """
|
|
|
128 |
generate,
|
129 |
additional_inputs=additional_inputs,
|
130 |
theme = gr.themes.Soft(),
|
131 |
+
examples=[["What is the secret to life?"], ["How the universe works?"],["What can you do?"],["What is quantum mechanics?"],["Do you belive in after life?"], ["Java function to check if URL is valid or not."]]
|
132 |
)
|
133 |
|
134 |
demo.queue(max_size=100).launch(debug=True)
|