Spaces:
Running
Running
Commit
•
54801aa
1
Parent(s):
8f2b56f
Upload 2 files
Browse files- app.py +50 -0
- requirements.txt +5 -0
app.py
ADDED
@@ -0,0 +1,50 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import gradio as gr
|
2 |
+
import os
|
3 |
+
import torch
|
4 |
+
from PIL import Image
|
5 |
+
from peft import PeftConfig, PeftModel
|
6 |
+
from transformers import AutoProcessor, Blip2ForConditionalGeneration
|
7 |
+
from transformers import pipeline
|
8 |
+
|
9 |
+
device = "cuda" if torch.cuda.is_available() else "cpu"
|
10 |
+
|
11 |
+
processor = AutoProcessor.from_pretrained("Salesforce/blip2-opt-2.7b")
|
12 |
+
|
13 |
+
peft_model_id = "leoreigoto/Data2_V2_BLIP2_Finetune_Caption_First_Epoch"
|
14 |
+
config = PeftConfig.from_pretrained(peft_model_id)
|
15 |
+
blip_finetune = Blip2ForConditionalGeneration.from_pretrained(config.base_model_name_or_path)#, load_in_8bit=True, device_map="auto")
|
16 |
+
blip_finetune = PeftModel.from_pretrained(blip_finetune, peft_model_id)
|
17 |
+
|
18 |
+
qa_model = pipeline("question-answering", "timpal0l/mdeberta-v3-base-squad2")
|
19 |
+
|
20 |
+
|
21 |
+
def generate_caption(pred_image):
|
22 |
+
|
23 |
+
inputs = processor(pred_image, return_tensors="pt").to(device, torch.float16)
|
24 |
+
generated_ids = blip_finetune.generate(**inputs, max_new_tokens=50)
|
25 |
+
generated_text = processor.batch_decode(generated_ids, skip_special_tokens=True)[0].strip()
|
26 |
+
return generated_text
|
27 |
+
|
28 |
+
|
29 |
+
def prompt_run(pred_image,prompt_box):
|
30 |
+
generated_text = generate_caption(pred_image)
|
31 |
+
generated_text = qa_model(question = prompt_box, context = generated_text)
|
32 |
+
return generated_text['answer']
|
33 |
+
|
34 |
+
|
35 |
+
|
36 |
+
with gr.Blocks() as gradio_app:
|
37 |
+
with gr.Row():
|
38 |
+
pred_image = gr.Image(height=480,width= 480,image_mode='RGB',type="pil")
|
39 |
+
with gr.Column():
|
40 |
+
button_caption = gr.Button(value='Get image caption (description)',size='sm')
|
41 |
+
prompt_box = gr.Textbox(label="Prompt",placeholder='enter prompt here')
|
42 |
+
button_prompt = gr.Button(value='Run prompt',size='sm')
|
43 |
+
with gr.Column():
|
44 |
+
output_box = gr.Textbox(label="Model output", placeholder='model output')
|
45 |
+
|
46 |
+
button_prompt.click(prompt_run,inputs=[pred_image,prompt_box], outputs=[output_box])
|
47 |
+
button_caption.click(generate_caption,inputs=[pred_image], outputs=[output_box])
|
48 |
+
|
49 |
+
gradio_app.launch()
|
50 |
+
|
requirements.txt
ADDED
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
transformers
|
2 |
+
torch
|
3 |
+
peft
|
4 |
+
bitsandbytes
|
5 |
+
accelerate
|