# import os # os.system("pip uninstall -y gradio") # os.system("pip install gradio==3.43.2") import argparse import sys import os # import cv2 import glob import gradio as gr import numpy as np import json from PIL import Image from tqdm import tqdm from pathlib import Path import uvicorn from fastapi.staticfiles import StaticFiles import random import time import requests from fastapi import FastAPI from conversation import SeparatorStyle, conv_templates, default_conversation from utils import ( build_logger, moderation_msg, server_error_msg, ) from config import cur_conv logger = build_logger("gradio_web_server", "gradio_web_server.log") headers = {"Content-Type": "application/json"} # create a FastAPI app app = FastAPI() # # create a static directory to store the static files # static_dir = Path('/data/Multimodal-RAG/GenerativeAIExamples/ChatQnA/langchain/redis/chips-making-deals/') static_dir = Path('/') # mount FastAPI StaticFiles server app.mount("/static", StaticFiles(directory=static_dir), name="static") theme = gr.themes.Base( primary_hue=gr.themes.Color( c100="#dbeafe", c200="#bfdbfe", c300="#93c5fd", c400="#60a5fa", c50="#eff6ff", c500="#0054ae", c600="#00377c", c700="#00377c", c800="#1e40af", c900="#1e3a8a", c950="#0a0c2b"), secondary_hue=gr.themes.Color( c100="#dbeafe", c200="#bfdbfe", c300="#93c5fd", c400="#60a5fa", c50="#eff6ff", c500="#0054ae", c600="#0054ae", c700="#0054ae", c800="#1e40af", c900="#1e3a8a", c950="#1d3660"), ).set( body_background_fill_dark='*primary_950', body_text_color_dark='*neutral_300', border_color_accent='*primary_700', border_color_accent_dark='*neutral_800', block_background_fill_dark='*primary_950', block_border_width='2px', block_border_width_dark='2px', button_primary_background_fill_dark='*primary_500', button_primary_border_color_dark='*primary_500', ) css=''' @font-face { font-family: IntelOne; src: url("file/assets/intelone-bodytext-font-family-regular.ttf"); } ''' ## html_title = '''

Cognitive AI:
Multimodal RAG on Videos

''' # force dark theme js_func = """ function refresh() { const url = new URL(window.location); if (url.searchParams.get('__theme') !== 'dark') { url.searchParams.set('__theme', 'dark'); window.location.href = url.href; } } """ debug = False def print_debug(t): if debug: print(t) # https://stackoverflow.com/a/57781047 # Resizes a image and maintains aspect ratio # def maintain_aspect_ratio_resize(image, width=None, height=None, inter=cv2.INTER_AREA): # # Grab the image size and initialize dimensions # dim = None # (h, w) = image.shape[:2] # # Return original image if no need to resize # if width is None and height is None: # return image # # We are resizing height if width is none # if width is None: # # Calculate the ratio of the height and construct the dimensions # r = height / float(h) # dim = (int(w * r), height) # # We are resizing width if height is none # else: # # Calculate the ratio of the width and construct the dimensions # r = width / float(w) # dim = (width, int(h * r)) # # Return the resized image # return cv2.resize(image, dim, interpolation=inter) def time_to_frame(time, fps): ''' convert time in seconds into frame number ''' return int(time * fps - 1) def str2time(strtime): strtime = strtime.strip('"') hrs, mins, seconds = [float(c) for c in strtime.split(':')] total_seconds = hrs * 60**2 + mins * 60 + seconds return total_seconds def get_iframe(video_path: str, start: int = -1, end: int = -1): return f"""""" #TODO # def place(galleries, evt: gr.SelectData): # print(evt.value) # start_time = evt.value.split('||')[0].strip() # print(start_time) # # sub_video_id = evt.value.split('|')[-1] # if start_time in start_time_index_map.keys(): # sub_video_id = start_time_index_map[start_time] # else: # sub_video_id = 0 # path_to_sub_video = f"/static/video_embeddings/mp4.keynotes23/sub-videos/keynotes23_split{sub_video_id}.mp4" # # return evt.value # return get_iframe(path_to_sub_video) # def process(text_query): # tmp_dir = os.environ.get('VID_CACHE_DIR', os.environ.get('TMPDIR', './video_embeddings')) # frames, transcripts = run_query(text_query, path=tmp_dir) # # return video_file_path, [(image, caption) for image, caption in zip(frame_paths, transcripts)] # return [(frame, caption) for frame, caption in zip(frames, transcripts)], "" description = "This Space lets you engage with multimodal RAG on a video through a chat box." no_change_btn = gr.Button.update() enable_btn = gr.Button.update(interactive=True) disable_btn = gr.Button.update(interactive=False) # textbox = gr.Textbox( # show_label=False, placeholder="Enter text and press ENTER", container=False # ) def clear_history(request: gr.Request): logger.info(f"clear_history. ip: {request.client.host}") state = cur_conv.copy() return (state, state.to_gradio_chatbot(), "", None) + (disable_btn,) * 1 def add_text(state, text, request: gr.Request): logger.info(f"add_text. ip: {request.client.host}. len: {len(text)}") if len(text) <= 0 : state.skip_next = True return (state, state.to_gradio_chatbot(), "", None) + (no_change_btn,) * 1 text = text[:1536] # Hard cut-off state.append_message(state.roles[0], text) state.append_message(state.roles[1], None) state.skip_next = False return (state, state.to_gradio_chatbot(), "") + (disable_btn,) * 1 def http_bot( state, request: gr.Request ): logger.info(f"http_bot. ip: {request.client.host}") start_tstamp = time.time() if state.skip_next: # This generate call is skipped due to invalid inputs path_to_sub_videos = state.get_path_to_subvideos() yield (state, state.to_gradio_chatbot(), path_to_sub_videos) + (no_change_btn,) * 1 return if len(state.messages) == state.offset + 2: # First round of conversation new_state = cur_conv.copy() new_state.append_message(new_state.roles[0], state.messages[-2][1]) new_state.append_message(new_state.roles[1], None) state = new_state # Construct prompt prompt = state.get_prompt() all_images = state.get_images(return_pil=False) # Make requests is_very_first_query = True if len(all_images) == 0: # first query need to do RAG pload = { "query": prompt, } else: # subsequence queries, no need to do Retrieval is_very_first_query = False pload = { "prompt": prompt, "path-to-image": all_images[0], } if is_very_first_query: url = worker_addr + "/v1/rag/chat" else: url = worker_addr + "/v1/rag/multi_turn_chat" logger.info(f"==== request ====\n{pload}") logger.info(f"==== url request ====\n{url}") #uncomment this for testing UI only # state.messages[-1][-1] = f"response {len(state.messages)}" # yield (state, state.to_gradio_chatbot()) + (enable_btn,) * 1 # return state.messages[-1][-1] = "▌" path_to_sub_videos = state.get_path_to_subvideos() yield (state, state.to_gradio_chatbot(), path_to_sub_videos) + (disable_btn,) * 1 try: # Stream output response = requests.post(url, headers=headers, json=pload, timeout=100, stream=True) # print("response") # logger.info(response) for chunk in response.iter_lines(decode_unicode=False, delimiter=b"\0"): if chunk: res = json.loads(chunk.decode()) ## old_method # if response.status_code == 200: # cur_json = "" # for chunk in response: # # print('chunk is ---> ', chunk.decode('utf-8')) # cur_json += chunk.decode('utf-8') # try: # res = json.loads(cur_json) # except: # # a whole json does not include in this chunk, need to concatenate with next chunk # continue # # successfully load json into res # cur_json = "" if state.path_to_img is None and 'path-to-image' in res: state.path_to_img = res['path-to-image'] if state.video_title is None and 'title' in res: state.video_title = res['title'] if 'answer' in res: # print(f"answer is {res['answer']}") output = res["answer"] # print(f"state.messages is {state.messages[-1][-1]}") state.messages[-1][-1] = state.messages[-1][-1][:-1] + output + "▌" path_to_sub_videos = state.get_path_to_subvideos() yield (state, state.to_gradio_chatbot(), None) + (disable_btn,) * 1 time.sleep(0.03) # else: # raise requests.exceptions.RequestException() except requests.exceptions.RequestException as e: # print(e) state.messages[-1][-1] = server_error_msg yield (state, state.to_gradio_chatbot(), None) + ( enable_btn, ) return state.messages[-1][-1] = state.messages[-1][-1][:-1] path_to_sub_videos = state.get_path_to_subvideos() logger.info(path_to_sub_videos) yield (state, state.to_gradio_chatbot(), path_to_sub_videos) + (enable_btn,) * 1 finish_tstamp = time.time() logger.info(f"{state.messages[-1][-1]}") # with open(get_conv_log_filename(), "a") as fout: # data = { # "tstamp": round(finish_tstamp, 4), # "url": url, # "start": round(start_tstamp, 4), # "finish": round(start_tstamp, 4), # "state": state.dict(), # } # fout.write(json.dumps(data) + "\n") return dropdown_list = [ "What did Intel present at Nasdaq?", "From Chips Act Funding Announcement, by which year is Intel committed to Net Zero gas emissions?", "What percentage of renewable energy is Intel planning to use?", "a band playing music", "Which US state is Silicon Desert referred to?", "and which US state is Silicon Forest referred to?", "How do trigate fins work?", "What is the advantage of trigate over planar transistors?", "What are key objectives of transistor design?", "How fast can transistors switch?", ] with gr.Blocks(theme=theme, css=css) as demo: # gr.Markdown(description) state = gr.State(default_conversation.copy()) gr.HTML(value=html_title) demo.load(_js=js_func) with gr.Row(): with gr.Column(scale=4): video = gr.Video(height=512, width=512, elem_id="video" ) with gr.Column(scale=7): chatbot = gr.Chatbot( elem_id="chatbot", label="Multimodal RAG Chatbot", height=450 ) with gr.Row(): with gr.Column(scale=8): # textbox.render() textbox = gr.Dropdown( dropdown_list, allow_custom_value=True, # show_label=False, # container=False, label="Query", info="Enter your query here or choose a sample from the dropdown list!" ) with gr.Column(scale=1, min_width=50): submit_btn = gr.Button( value="Send", variant="primary", interactive=True ) with gr.Row(elem_id="buttons") as button_row: clear_btn = gr.Button(value="🗑️ Clear history", interactive=False) # Register listeners btn_list = [clear_btn] clear_btn.click( clear_history, None, [state, chatbot, textbox, video] + btn_list ) # textbox.submit( # add_text, # [state, textbox], # [state, chatbot, textbox,] + btn_list, # ).then( # http_bot, # [state, ], # [state, chatbot, video] + btn_list, # ) submit_btn.click( add_text, [state, textbox], [state, chatbot, textbox,] + btn_list, ).then( http_bot, [state, ], [state, chatbot, video] + btn_list, ) print_debug('Beginning') # btn.click(fn=process, # inputs=[text_query], # # outputs=[video_player, gallery], # outputs=[gallery, html], # ) # gallery.select(place, [gallery], [html]) demo.queue() app = gr.mount_gradio_app(app, demo, path='/') share = False enable_queue = True # try: # demo.queue(concurrency_count=3)#, enable_queue=False) # demo.launch(enable_queue=enable_queue, share=share, server_port=17808, server_name='0.0.0.0') # #BATCH -w isl-gpu48 # except: # demo.launch(enable_queue=False, share=share, server_port=17808, server_name='0.0.0.0') # serve the app if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("--host", type=str, default="0.0.0.0") parser.add_argument("--port", type=int, default=7860) parser.add_argument("--concurrency-count", type=int, default=20) parser.add_argument("--share", action="store_true") parser.add_argument("--worker-address", type=str, default="198.175.88.247") parser.add_argument("--worker-port", type=int, default=8080) args = parser.parse_args() logger.info(f"args: {args}") global worker_addr worker_addr = f"http://{args.worker_address}:{args.worker_port}" uvicorn.run(app, host=args.host, port=args.port) # for i in examples: # print(f'Processing {i[0]}') # results = process(*i) # print(f'{len(results[0])} results returned')