v1

.gitignore

@@ -0,0 +1,3 @@
+.vscode
+__pycache__
+*/__pycache__

app.py

@@ -38,8 +38,6 @@ model_3_8, tokenizer_3_8 = load_trol(link='TroL-3.8B')
 # loading model
 model_7, tokenizer_7 = load_trol(link='TroL-7B')
 
-print()
-
 def threading_function(inputs, image_token_number, streamer, device, model, tokenizer, temperature, new_max_token, top_p):
 
     # propagation
@@ -85,7 +83,7 @@ def bot_streaming(message, history, link, temperature, new_max_token, top_p):
             if "3.8B" not in link:
                 image_token_number = 1225
                 image = F.interpolate(image.unsqueeze(0), size=(490, 490), mode='bicubic').squeeze(0)
-            inputs = [{'image': image, 'question': message['text']}]
+            inputs = [{'image': image.to(accel.device), 'question': message['text']}]
         elif len(message['files']) > 1:
             raise Exception("No way!")
         else:
@@ -116,7 +114,7 @@ def bot_streaming(message, history, link, temperature, new_max_token, top_p):
 
         # Text decoding
         response = output_filtering(generated_text, model)
-    
+
     except:
         response = "There may be unsupported format: ex) pdf, video, sound. Only supported is a single image in this version."
 
@@ -138,7 +136,7 @@ def bot_streaming(message, history, link, temperature, new_max_token, top_p):
         yield buffer
 
 demo = gr.ChatInterface(fn=bot_streaming,
-                        additional_inputs = [gr.Radio(["3.8B"], label="Size", info="Select one model size", value="3.8B"), gr.Slider(0, 1, 0.9, label="temperature"), gr.Slider(1, 1024, 128, label="new_max_token"), gr.Slider(0, 1, 0.95, label="top_p")],
+                        additional_inputs = [gr.Radio(["1.8B", "3.8B"], label="Size", info="Select one model size", value="3.8B"), gr.Slider(0, 1, 0.9, label="temperature"), gr.Slider(1, 1024, 128, label="new_max_token"), gr.Slider(0, 1, 0.95, label="top_p")],
                         additional_inputs_accordion="Generation Hyperparameters",
                         theme=gr.themes.Soft(),
                         title="TroL",

trol/load_trol.py

@@ -23,7 +23,7 @@ def load_trol(link):
         from transformers import LlamaTokenizerFast as TroLTokenizer
         bits = 8
         path = TROL_3_8B
-        bit_quant_skip = ["vision_model", "mlp1", "lm_head"]
+        bit_quant_skip = ["vision_model", "vision_proj", "lm_head"]
 
     elif link == 'TroL-7B':
         from .arch_internlm2.modeling_trol import TroLForCausalLM