update app.py

app.py

@@ -130,7 +130,7 @@ class FoleyController:
         prompt_textbox,
         negative_prompt_textbox, 
         ip_adapter_scale,
-        # temporal_scale,
+        temporal_scale,
         sampler_dropdown,
         sample_step_slider,
         cfg_scale_slider,
@@ -154,7 +154,7 @@ class FoleyController:
         if seed_textbox != "":
             torch.manual_seed(int(seed_textbox))
             generator.manual_seed(int(seed_textbox))
-        max_frame_nums = 15
+        max_frame_nums = 150
         frames, duration  = read_frames_with_moviepy(input_video, max_frame_nums=max_frame_nums)
         if duration >= 10:
             duration = 10
@@ -169,7 +169,9 @@ class FoleyController:
         time_condition = time_condition + [-1] * (1024 - len(time_condition))
         # w -> b c h w
         time_condition = torch.FloatTensor(time_condition).unsqueeze(0).unsqueeze(0).unsqueeze(0).repeat(1, 1, 256, 1)
-        
+
+        # Note that clip need fewer frames
+        frames = frames[::10]
         images = self.image_processor(images=frames, return_tensors="pt").to(device)
         image_embeddings = self.image_encoder(**images).image_embeds
         image_embeddings = torch.mean(image_embeddings, dim=0, keepdim=True).unsqueeze(0).unsqueeze(0)
@@ -253,18 +255,20 @@ with gr.Blocks(css=css) as demo:
                     negative_prompt_textbox = gr.Textbox(value=N_PROMPT, label="Negative prompt", lines=1)
 
                 with gr.Row():
-                    sampler_dropdown = gr.Dropdown(
-                        label="Sampling method",
-                        choices=list(scheduler_dict.keys()),
-                        value=list(scheduler_dict.keys())[0],
-                    )
-                    sample_step_slider = gr.Slider(
-                        label="Sampling steps", value=25, minimum=10, maximum=100, step=1
-                    )
-
-                cfg_scale_slider = gr.Slider(label="CFG Scale", value=7.5, minimum=0, maximum=20)
-                ip_adapter_scale = gr.Slider(label="Visual Content Scale", value=1.0, minimum=0, maximum=1)
-                # temporal_scale = gr.Slider(label="Temporal Align Scale", value=0., minimum=0., maximum=1.0)
+                    ip_adapter_scale = gr.Slider(label="Visual Content Scale", value=1.0, minimum=0, maximum=1)
+                    temporal_scale = gr.Slider(label="Temporal Align Scale", value=0.2, minimum=0., maximum=1.0)
+
+                with gr.Accordion("Sampling Settings", open=False):
+                    with gr.Row():
+                        sampler_dropdown = gr.Dropdown(
+                            label="Sampling method",
+                            choices=list(scheduler_dict.keys()),
+                            value=list(scheduler_dict.keys())[0],
+                        )
+                        sample_step_slider = gr.Slider(
+                            label="Sampling steps", value=25, minimum=10, maximum=100, step=1
+                        )
+                    cfg_scale_slider = gr.Slider(label="CFG Scale", value=7.5, minimum=0, maximum=20)
 
                 with gr.Row():
                     seed_textbox = gr.Textbox(label="Seed", value=42)
@@ -273,7 +277,12 @@ with gr.Blocks(css=css) as demo:
 
                 generate_button = gr.Button(value="Generate", variant="primary")
 
-            result_video = gr.Video(label="Generated Audio", interactive=False)
+            with gr.Column():
+                result_video = gr.Video(label="Generated Audio", interactive=False)
+                gr.Markdown('**Tips**: <br> \
+                            1. With strong temporal visual cues in input video, you can scale up the **Temporal Align Scale**. <br>\
+                            2. **Visual content scale** is the level of semantic alignment with visual content. \
+                ')
 
         generate_button.click(
             fn=controller.foley,
@@ -282,7 +291,7 @@ with gr.Blocks(css=css) as demo:
                 prompt_textbox,
                 negative_prompt_textbox,
                 ip_adapter_scale,
-                # temporal_scale,
+                temporal_scale,
                 sampler_dropdown,
                 sample_step_slider,
                 cfg_scale_slider,
@@ -292,13 +301,22 @@ with gr.Blocks(css=css) as demo:
         )
 
         gr.Examples(
-            examples= [
-                ['examples/videos/51701454.mp4', 'seagulls', '', 1.0, 'DDIM', 25, 7.5, 10014024412012338098],
-                ['examples/videos/42.mp4', '', '', 1.0, 'DDIM', 25, 7.5, 42],
-                ['examples/videos/1.mp4', '', '', 1.0, 'DDIM', 25, 7.5, 93493458],
-                ['examples/videos/2.mp4', '', '', 1.0, 'DDIM', 25, 7.5, 16520432],
+            # examples= [
+            #     ['examples/videos/51701454.mp4', 'seagulls', '', 1.0, 'DDIM', 25, 7.5, 10014024412012338098],
+            #     ['examples/videos/42.mp4', '', '', 1.0, 'DDIM', 25, 7.5, 42],
+            #     ['examples/videos/1.mp4', '', '', 1.0, 'DDIM', 25, 7.5, 93493458],
+            #     ['examples/videos/2.mp4', '', '', 1.0, 'DDIM', 25, 7.5, 16520432],
+            # ],
+            examples=[
+                ['examples/input/case1.mp4', '', '', 1.0, 0.2, 'DDIM', 25, 7.5, 33817921],
+                ['examples/input/case3.mp4', '', '', 1.0, 0.2,'DDIM', 25, 7.5, 94667578],
+                ['examples/input/case5.mp4', '', '', 0.75, 0.2,'DDIM', 25, 7.5, 92890876],
+                ['examples/input/case6.mp4', '', '', 1.0, 0.2, 'DDIM', 25, 7.5, 77015909],
             ],
-            inputs=[init_img,prompt_textbox,negative_prompt_textbox,ip_adapter_scale,sampler_dropdown,sample_step_slider,cfg_scale_slider,seed_textbox],
+            inputs=[init_img,prompt_textbox,negative_prompt_textbox,ip_adapter_scale,temporal_scale,sampler_dropdown,sample_step_slider,cfg_scale_slider,seed_textbox],
+            cache_examples=True,
+            outputs=[result_video], 
+            fn=controller.foley,
         )
             
     demo.queue(10)