demo version2

spatial condition added

app.py

@@ -17,6 +17,7 @@ MAX_SEED = 100_000
 def generate(region1_concept,
                 region2_concept,
                 prompt,
+                pose_image_id,
                 region1_prompt,
                 region2_prompt,
                 negative_prompt,
@@ -33,13 +34,19 @@ def generate(region1_concept,
     region1_concept, region2_concept = region1_concept.lower(), region2_concept.lower()
     pretrained_model = merge(region1_concept, region2_concept)
 
-    keypose_condition = 'multi-concept/pose_data/two_apart.png'
-    region1 = '[0, 0, 512, 290]'
-    region2 = '[0, 650, 512, 910]'
+    with open('multi-concept/pose_data/pose.json') as f:
+        d = json.load(f)
+
+    pose_image = {obj.pop('pose_id'):obj for obj in d}[int(pose_image_id)]
+    print(pose_image)
+    keypose_condition = pose_image['keypose_condition']
+    region1 = pose_image['region1']
+    region2 = pose_image['region2']
 
     region1_prompt = f'[<{region1_concept}1> <{region1_concept}2>, {region1_prompt}]'
     region2_prompt = f'[<{region2_concept}1> <{region2_concept}2>, {region2_prompt}]'
     prompt_rewrite=f"{region1_prompt}-*-{region_neg_prompt}-*-{region1}|{region2_prompt}-*-{region_neg_prompt}-*-{region2}"
+    print(prompt_rewrite)
 
     result = infer(pretrained_model,
                   prompt,
@@ -164,6 +171,10 @@ def infer(pretrained_model,
     
     return image[0]
 
+
+def on_select(evt: gr.SelectData):  # SelectData is a subclass of EventData
+    return ''.join(c for c in evt.value['image']['orig_name'] if c.isdigit())
+
 examples_context = [
     'walking at Stanford university campus',
     'in a castle',
@@ -174,6 +185,10 @@ examples_context = [
 examples_region1 = ['wearing red hat, high resolution, best quality','bright smile, wearing pants, best quality']
 examples_region2 = ['smilling, wearing blue shirt, high resolution, best quality']
 
+with open('multi-concept/pose_data/pose.json') as f:
+    d = json.load(f)
+pose_image_list = [(obj['pose_id'],obj['keypose_condition']) for obj in d]
+
 css="""
 #col-container {
     margin: 0 auto;
@@ -182,124 +197,140 @@ css="""
 """
 
 with gr.Blocks(css=css) as demo:
+    gr.Markdown(f"""
+    # Orthogonal Adaptation
+    Currently running on : {power_device}
+    """)
     
-    with gr.Column(elem_id="col-container"):
-        gr.Markdown(f"""
-        # Orthogonal Adaptation
-        Currently running on {power_device}.
-        """)
-        prompt = gr.Text(
-                label="ContextPrompt",
-                show_label=False,
-                max_lines=1,
-                placeholder="Enter your context prompt for overall image",
-                container=False,
-            )
-        with gr.Row():
-            
-            region1_concept = gr.Dropdown(
-                ["Elsa", "Moana"],
-                label="Character 1",
-                info="Will add more characters later!"
-            )
-            region2_concept = gr.Dropdown(
-                ["Elsa", "Moana"], 
-                label="Character 2", 
-                info="Will add more characters later!"
-            )
-
-        with gr.Row():
-
-            region1_prompt = gr.Textbox(
-                label="Region1 Prompt",
-                show_label=False,
-                max_lines=2,
-                placeholder="Enter your prompt for character 1",
-                container=False,
-            )
-
-            region2_prompt = gr.Textbox(
-                label="Region2 Prompt",
-                show_label=False,
-                max_lines=2,
-                placeholder="Enter your prompt for character 2",
-                container=False,
-            )
-
-        run_button = gr.Button("Run", scale=1)
-        
-        result = gr.Image(label="Result", show_label=False)
-
-        with gr.Accordion("Advanced Settings", open=False):
-
-            negative_prompt = gr.Text(
-                label="Context Negative prompt",
-                max_lines=1,
-                value = 'saturated, cropped, worst quality, low quality',
-                visible=False,
-            )
-
-            region_neg_prompt =  gr.Text(
-                label="Regional Negative prompt",
-                max_lines=1,
-                value = 'shirtless, nudity, saturated, cropped, worst quality, low quality',
-                visible=False,
-            )
-            
-            seed = gr.Slider(
-                label="Seed",
-                minimum=0,
-                maximum=MAX_SEED,
-                step=1,
-                value=0,
-            )
-            
-            randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
+    with gr.Row():
+        with gr.Column(elem_id="col-container", scale=2):
+            gr.Markdown(f"""
+            ### 🕹️ Global and Region prompts:
+            """)
             
+            prompt = gr.Text(
+                    label="ContextPrompt",
+                    show_label=False,
+                    max_lines=1,
+                    placeholder="Enter your context(global) prompt",
+                    container=False,
+                )
             with gr.Row():
                 
-                sketch_adaptor_weight = gr.Slider(
-                        label="Sketch Adapter Weight",
-                        minimum = 0,
-                        maximum = 1,
-                        step=0.01,
-                        value=0,
+                region1_concept = gr.Dropdown(
+                    ["Elsa", "Moana"],
+                    label="Character 1",
+                    info="Will add more characters later!"
                 )
-                
-                keypose_adaptor_weight = gr.Slider(
-                        label="Keypose Adapter Weight",
-                        minimum = 0,
-                        maximum = 1,
-                        step= 0.01,
-                        value=1.0,
+                region2_concept = gr.Dropdown(
+                    ["Elsa", "Moana"], 
+                    label="Character 2", 
+                    info="Will add more characters later!"
+                )
+
+            with gr.Row():
+
+                region1_prompt = gr.Textbox(
+                    label="Region1 Prompt",
+                    show_label=False,
+                    max_lines=2,
+                    placeholder="Enter your regional prompt for character 1",
+                    container=False,
+                )
+
+                region2_prompt = gr.Textbox(
+                    label="Region2 Prompt",
+                    show_label=False,
+                    max_lines=2,
+                    placeholder="Enter your regional prompt for character 2",
+                    container=False,
                 )
+
+            gr.Markdown(f"### 🧭 Spatial Condition for regionally controllable sampling: ")
+            gallery = gr.Gallery(label = "Select pose for characters",
+                                value = [obj[1]for obj in pose_image_list], 
+                                elem_id = [obj[0]for obj in pose_image_list],
+                                interactive=False, show_download_button=False,
+                                preview=True, height = 200, object_fit="scale-down")
             
+            pose_image_id = gr.Textbox(visible=False)
+            gallery.select(on_select, None, pose_image_id)
 
-        gr.Examples(
-            label = 'Context Prompt example',
-            examples = examples_context,
-            inputs = [prompt]
-        )  
+            run_button = gr.Button("Run", scale=1)
 
-        with gr.Row():
-            gr.Examples(
-                label = 'Region1 Prompt example',
-                examples = examples_region1,
-                inputs = [region1_prompt]
-            )
+            with gr.Accordion("Advanced Settings", open=False):
+
+                negative_prompt = gr.Text(
+                    label="Context Negative prompt",
+                    max_lines=1,
+                    value = 'saturated, cropped, worst quality, low quality',
+                    visible=False,
+                )
+
+                region_neg_prompt =  gr.Text(
+                    label="Regional Negative prompt",
+                    max_lines=1,
+                    value = 'shirtless, nudity, saturated, cropped, worst quality, low quality',
+                    visible=False,
+                )
+                
+                seed = gr.Slider(
+                    label="Seed",
+                    minimum=0,
+                    maximum=MAX_SEED,
+                    step=1,
+                    value=0,
+                )
+                
+                randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
+                
+                with gr.Row():
+                    
+                    sketch_adaptor_weight = gr.Slider(
+                            label="Sketch Adapter Weight",
+                            minimum = 0,
+                            maximum = 1,
+                            step=0.01,
+                            value=0,
+                    )
+                    
+                    keypose_adaptor_weight = gr.Slider(
+                            label="Keypose Adapter Weight",
+                            minimum = 0,
+                            maximum = 1,
+                            step= 0.01,
+                            value=1.0,
+                    )
+                
+        with gr.Column(scale=1):
 
             gr.Examples(
-                label = 'Region2 Prompt example',
-                examples = [examples_region2],
-                inputs = [region2_prompt]
-            )
+                label = 'Global Prompt example',
+                examples = examples_context,
+                inputs = [prompt]
+            )  
+
+            with gr.Row():
+                gr.Examples(
+                    label = 'Region1 Prompt example',
+                    examples = examples_region1,
+                    inputs = [region1_prompt]
+                )
+
+                gr.Examples(
+                    label = 'Region2 Prompt example',
+                    examples = [examples_region2],
+                    inputs = [region2_prompt]
+                )
 
+            result = gr.Image(label="Result", show_label=False)
 
     run_button.click(
         fn = generate,
         inputs = [region1_concept,
                   region2_concept,
                   prompt,
+                  pose_image_id,
                   region1_prompt,
                   region2_prompt,
                   negative_prompt,