Commit
β’
4984c7e
1
Parent(s):
4783dcc
high quality (#17)
Browse files- High quality (0f59bfafcd98b4b4a93790a81be7de43d068135f)
- Update app.py (09d2a3966a11c5d694bde9833a8a514a0e291a14)
Co-authored-by: ApolinΓ‘rio from multimodal AI art <multimodalart@users.noreply.huggingface.co>
- app.py +75 -23
- checkers_mid.jpg +0 -0
- funky.jpeg +0 -0
- ultra_checkers.png +0 -0
app.py
CHANGED
@@ -8,6 +8,8 @@ from diffusers import (
|
|
8 |
StableDiffusionControlNetPipeline,
|
9 |
ControlNetModel,
|
10 |
StableDiffusionLatentUpscalePipeline,
|
|
|
|
|
11 |
DPMSolverMultistepScheduler, # <-- Added import
|
12 |
EulerDiscreteScheduler # <-- Added import
|
13 |
)
|
@@ -18,17 +20,22 @@ from illusion_style import css
|
|
18 |
BASE_MODEL = "SG161222/Realistic_Vision_V5.1_noVAE"
|
19 |
|
20 |
# Initialize both pipelines
|
21 |
-
vae = AutoencoderKL.from_pretrained("stabilityai/sd-vae-ft-mse")
|
22 |
#init_pipe = DiffusionPipeline.from_pretrained("SG161222/Realistic_Vision_V5.1_noVAE", torch_dtype=torch.float16)
|
23 |
-
controlnet = ControlNetModel.from_pretrained("monster-labs/control_v1p_sd15_qrcode_monster")#, torch_dtype=torch.float16)
|
24 |
main_pipe = StableDiffusionControlNetPipeline.from_pretrained(
|
25 |
BASE_MODEL,
|
26 |
controlnet=controlnet,
|
27 |
vae=vae,
|
28 |
safety_checker=None,
|
29 |
-
|
30 |
).to("cuda")
|
|
|
|
|
|
|
31 |
#model_id = "stabilityai/sd-x2-latent-upscaler"
|
|
|
|
|
32 |
#upscaler = StableDiffusionLatentUpscalePipeline.from_pretrained(model_id, torch_dtype=torch.float16)
|
33 |
#upscaler.to("cuda")
|
34 |
|
@@ -55,6 +62,31 @@ def center_crop_resize(img, output_size=(512, 512)):
|
|
55 |
|
56 |
return img
|
57 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
58 |
# Inference function
|
59 |
def inference(
|
60 |
control_image: Image.Image,
|
@@ -62,6 +94,9 @@ def inference(
|
|
62 |
negative_prompt: str,
|
63 |
guidance_scale: float = 8.0,
|
64 |
controlnet_conditioning_scale: float = 1,
|
|
|
|
|
|
|
65 |
seed: int = -1,
|
66 |
sampler = "DPM++ Karras SDE",
|
67 |
progress = gr.Progress(track_tqdm=True)
|
@@ -73,65 +108,82 @@ def inference(
|
|
73 |
#init_image = init_pipe(prompt).images[0]
|
74 |
|
75 |
# Rest of your existing code
|
76 |
-
|
77 |
main_pipe.scheduler = SAMPLER_MAP[sampler](main_pipe.scheduler.config)
|
78 |
generator = torch.manual_seed(seed) if seed != -1 else torch.Generator()
|
79 |
|
80 |
out = main_pipe(
|
81 |
prompt=prompt,
|
82 |
negative_prompt=negative_prompt,
|
83 |
-
image=
|
84 |
-
#control_image=control_image,
|
85 |
guidance_scale=float(guidance_scale),
|
86 |
controlnet_conditioning_scale=float(controlnet_conditioning_scale),
|
87 |
generator=generator,
|
88 |
-
|
89 |
-
|
90 |
-
|
91 |
-
|
92 |
-
|
93 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
94 |
|
95 |
with gr.Blocks(css=css) as app:
|
96 |
gr.Markdown(
|
97 |
'''
|
98 |
-
<center><h1>Illusion Diffusion π</h1></span>
|
99 |
-
<span font-size:16px;">Generate stunning illusion artwork with Stable Diffusion</span>
|
100 |
</center>
|
101 |
|
102 |
A space by AP [Follow me on Twitter](https://twitter.com/angrypenguinPNG)
|
103 |
|
104 |
This project works by using [Monster Labs QR Control Net](https://huggingface.co/monster-labs/control_v1p_sd15_qrcode_monster).
|
105 |
-
Given a prompt and your pattern, we use a QR code conditioned controlnet to create a stunning illusion! Credit to: MrUgleh
|
106 |
-
|
107 |
'''
|
108 |
)
|
109 |
|
110 |
with gr.Row():
|
111 |
with gr.Column():
|
112 |
control_image = gr.Image(label="Input Illusion", type="pil", elem_id="control_image")
|
113 |
-
controlnet_conditioning_scale = gr.Slider(minimum=0.0, maximum=5.0, step=0.01, value=0.8, label="Illusion strength", info="ControlNet conditioning scale"
|
114 |
-
gr.Examples(examples=["checkers.png", "pattern.png", "spiral.jpeg"], inputs=control_image)
|
115 |
prompt = gr.Textbox(label="Prompt", elem_id="prompt")
|
116 |
negative_prompt = gr.Textbox(label="Negative Prompt", value="low quality", elem_id="negative_prompt")
|
117 |
with gr.Accordion(label="Advanced Options", open=False):
|
118 |
-
#strength = gr.Slider(minimum=0.0, maximum=1.0, step=0.01, value=0.9, label="Strength")
|
119 |
guidance_scale = gr.Slider(minimum=0.0, maximum=50.0, step=0.25, value=7.5, label="Guidance Scale")
|
120 |
sampler = gr.Dropdown(choices=list(SAMPLER_MAP.keys()), value="Euler")
|
121 |
-
|
|
|
|
|
|
|
122 |
run_btn = gr.Button("Run")
|
123 |
with gr.Column():
|
124 |
-
result_image = gr.Image(label="Illusion Diffusion Output", elem_id="output")
|
125 |
with gr.Group(elem_id="share-btn-container", visible=False) as share_group:
|
126 |
community_icon = gr.HTML(community_icon_html)
|
127 |
loading_icon = gr.HTML(loading_icon_html)
|
128 |
share_button = gr.Button("Share to community", elem_id="share-btn")
|
129 |
|
130 |
history = show_gallery_history()
|
131 |
-
|
132 |
run_btn.click(
|
133 |
inference,
|
134 |
-
inputs=[control_image, prompt, negative_prompt, guidance_scale, controlnet_conditioning_scale, seed, sampler],
|
135 |
outputs=[result_image, share_group]
|
136 |
).then(
|
137 |
fn=fetch_gallery_history, inputs=[prompt, result_image], outputs=history, queue=False
|
|
|
8 |
StableDiffusionControlNetPipeline,
|
9 |
ControlNetModel,
|
10 |
StableDiffusionLatentUpscalePipeline,
|
11 |
+
StableDiffusionImg2ImgPipeline,
|
12 |
+
StableDiffusionControlNetImg2ImgPipeline,
|
13 |
DPMSolverMultistepScheduler, # <-- Added import
|
14 |
EulerDiscreteScheduler # <-- Added import
|
15 |
)
|
|
|
20 |
BASE_MODEL = "SG161222/Realistic_Vision_V5.1_noVAE"
|
21 |
|
22 |
# Initialize both pipelines
|
23 |
+
vae = AutoencoderKL.from_pretrained("stabilityai/sd-vae-ft-mse", torch_dtype=torch.float16)
|
24 |
#init_pipe = DiffusionPipeline.from_pretrained("SG161222/Realistic_Vision_V5.1_noVAE", torch_dtype=torch.float16)
|
25 |
+
controlnet = ControlNetModel.from_pretrained("monster-labs/control_v1p_sd15_qrcode_monster", torch_dtype=torch.float16)#, torch_dtype=torch.float16)
|
26 |
main_pipe = StableDiffusionControlNetPipeline.from_pretrained(
|
27 |
BASE_MODEL,
|
28 |
controlnet=controlnet,
|
29 |
vae=vae,
|
30 |
safety_checker=None,
|
31 |
+
torch_dtype=torch.float16,
|
32 |
).to("cuda")
|
33 |
+
#main_pipe.unet = torch.compile(main_pipe.unet, mode="reduce-overhead", fullgraph=True)
|
34 |
+
#main_pipe.unet.to(memory_format=torch.channels_last)
|
35 |
+
#main_pipe.unet = torch.compile(main_pipe.unet, mode="reduce-overhead", fullgraph=True)
|
36 |
#model_id = "stabilityai/sd-x2-latent-upscaler"
|
37 |
+
image_pipe = StableDiffusionControlNetImg2ImgPipeline.from_pretrained(BASE_MODEL, unet=main_pipe.unet, vae=vae, controlnet=controlnet, safety_checker=None, torch_dtype=torch.float16).to("cuda")
|
38 |
+
#image_pipe.unet = torch.compile(image_pipe.unet, mode="reduce-overhead", fullgraph=True)
|
39 |
#upscaler = StableDiffusionLatentUpscalePipeline.from_pretrained(model_id, torch_dtype=torch.float16)
|
40 |
#upscaler.to("cuda")
|
41 |
|
|
|
62 |
|
63 |
return img
|
64 |
|
65 |
+
def common_upscale(samples, width, height, upscale_method, crop=False):
|
66 |
+
if crop == "center":
|
67 |
+
old_width = samples.shape[3]
|
68 |
+
old_height = samples.shape[2]
|
69 |
+
old_aspect = old_width / old_height
|
70 |
+
new_aspect = width / height
|
71 |
+
x = 0
|
72 |
+
y = 0
|
73 |
+
if old_aspect > new_aspect:
|
74 |
+
x = round((old_width - old_width * (new_aspect / old_aspect)) / 2)
|
75 |
+
elif old_aspect < new_aspect:
|
76 |
+
y = round((old_height - old_height * (old_aspect / new_aspect)) / 2)
|
77 |
+
s = samples[:,:,y:old_height-y,x:old_width-x]
|
78 |
+
else:
|
79 |
+
s = samples
|
80 |
+
|
81 |
+
return torch.nn.functional.interpolate(s, size=(height, width), mode=upscale_method)
|
82 |
+
|
83 |
+
def upscale(samples, upscale_method, scale_by):
|
84 |
+
#s = samples.copy()
|
85 |
+
width = round(samples["images"].shape[3] * scale_by)
|
86 |
+
height = round(samples["images"].shape[2] * scale_by)
|
87 |
+
s = common_upscale(samples["images"], width, height, upscale_method, "disabled")
|
88 |
+
return (s)
|
89 |
+
|
90 |
# Inference function
|
91 |
def inference(
|
92 |
control_image: Image.Image,
|
|
|
94 |
negative_prompt: str,
|
95 |
guidance_scale: float = 8.0,
|
96 |
controlnet_conditioning_scale: float = 1,
|
97 |
+
control_guidance_start: float = 1,
|
98 |
+
control_guidance_end: float = 1,
|
99 |
+
upscaler_strength: float = 0.5,
|
100 |
seed: int = -1,
|
101 |
sampler = "DPM++ Karras SDE",
|
102 |
progress = gr.Progress(track_tqdm=True)
|
|
|
108 |
#init_image = init_pipe(prompt).images[0]
|
109 |
|
110 |
# Rest of your existing code
|
111 |
+
control_image_small = center_crop_resize(control_image)
|
112 |
main_pipe.scheduler = SAMPLER_MAP[sampler](main_pipe.scheduler.config)
|
113 |
generator = torch.manual_seed(seed) if seed != -1 else torch.Generator()
|
114 |
|
115 |
out = main_pipe(
|
116 |
prompt=prompt,
|
117 |
negative_prompt=negative_prompt,
|
118 |
+
image=control_image_small,
|
|
|
119 |
guidance_scale=float(guidance_scale),
|
120 |
controlnet_conditioning_scale=float(controlnet_conditioning_scale),
|
121 |
generator=generator,
|
122 |
+
control_guidance_start=float(control_guidance_start),
|
123 |
+
control_guidance_end=float(control_guidance_end),
|
124 |
+
num_inference_steps=15,
|
125 |
+
output_type="latent"
|
126 |
+
)
|
127 |
+
control_image_large = center_crop_resize(control_image, (1024, 1024))
|
128 |
+
upscaled_latents = upscale(out, "nearest-exact", 2)
|
129 |
+
out_image = image_pipe(
|
130 |
+
prompt=prompt,
|
131 |
+
negative_prompt=negative_prompt,
|
132 |
+
control_image=control_image_large,
|
133 |
+
image=upscaled_latents,
|
134 |
+
guidance_scale=float(guidance_scale),
|
135 |
+
generator=generator,
|
136 |
+
num_inference_steps=20,
|
137 |
+
strength=upscaler_strength,
|
138 |
+
control_guidance_start=float(control_guidance_start),
|
139 |
+
control_guidance_end=float(control_guidance_end),
|
140 |
+
controlnet_conditioning_scale=float(controlnet_conditioning_scale)
|
141 |
+
)
|
142 |
+
return out_image["images"][0], gr.update(visible=True)
|
143 |
+
|
144 |
+
#return out
|
145 |
|
146 |
with gr.Blocks(css=css) as app:
|
147 |
gr.Markdown(
|
148 |
'''
|
149 |
+
<center><h1>Illusion Diffusion HQ π</h1></span>
|
150 |
+
<span font-size:16px;">Generate stunning high quality illusion artwork with Stable Diffusion</span>
|
151 |
</center>
|
152 |
|
153 |
A space by AP [Follow me on Twitter](https://twitter.com/angrypenguinPNG)
|
154 |
|
155 |
This project works by using [Monster Labs QR Control Net](https://huggingface.co/monster-labs/control_v1p_sd15_qrcode_monster).
|
156 |
+
Given a prompt and your pattern, we use a QR code conditioned controlnet to create a stunning illusion! Credit to: [MrUgleh](https://twitter.com/MrUgleh) for discovering the workflow :)
|
|
|
157 |
'''
|
158 |
)
|
159 |
|
160 |
with gr.Row():
|
161 |
with gr.Column():
|
162 |
control_image = gr.Image(label="Input Illusion", type="pil", elem_id="control_image")
|
163 |
+
controlnet_conditioning_scale = gr.Slider(minimum=0.0, maximum=5.0, step=0.01, value=0.8, label="Illusion strength", elem_id="illusion_strength", info="ControlNet conditioning scale")
|
164 |
+
gr.Examples(examples=["checkers.png", "checkers_mid.jpg", "pattern.png", "ultra_checkers.png", "spiral.jpeg", "funky.jpeg" ], inputs=control_image)
|
165 |
prompt = gr.Textbox(label="Prompt", elem_id="prompt")
|
166 |
negative_prompt = gr.Textbox(label="Negative Prompt", value="low quality", elem_id="negative_prompt")
|
167 |
with gr.Accordion(label="Advanced Options", open=False):
|
|
|
168 |
guidance_scale = gr.Slider(minimum=0.0, maximum=50.0, step=0.25, value=7.5, label="Guidance Scale")
|
169 |
sampler = gr.Dropdown(choices=list(SAMPLER_MAP.keys()), value="Euler")
|
170 |
+
control_start = gr.Slider(minimum=0.0, maximum=1.0, step=0.1, value=0, label="Start of ControlNet")
|
171 |
+
control_end = gr.Slider(minimum=0.0, maximum=1.0, step=0.1, value=1, label="End of ControlNet")
|
172 |
+
strength = gr.Slider(minimum=0.0, maximum=1.0, step=0.1, value=1, label="Strength of the upscaler")
|
173 |
+
seed = gr.Slider(minimum=-1, maximum=9999999999, step=1, value=-1, label="Seed", info="-1 means random seed", randomize=True)
|
174 |
run_btn = gr.Button("Run")
|
175 |
with gr.Column():
|
176 |
+
result_image = gr.Image(label="Illusion Diffusion Output", interactive=False, elem_id="output")
|
177 |
with gr.Group(elem_id="share-btn-container", visible=False) as share_group:
|
178 |
community_icon = gr.HTML(community_icon_html)
|
179 |
loading_icon = gr.HTML(loading_icon_html)
|
180 |
share_button = gr.Button("Share to community", elem_id="share-btn")
|
181 |
|
182 |
history = show_gallery_history()
|
183 |
+
|
184 |
run_btn.click(
|
185 |
inference,
|
186 |
+
inputs=[control_image, prompt, negative_prompt, guidance_scale, controlnet_conditioning_scale, control_start, control_end, strength, seed, sampler],
|
187 |
outputs=[result_image, share_group]
|
188 |
).then(
|
189 |
fn=fetch_gallery_history, inputs=[prompt, result_image], outputs=history, queue=False
|
checkers_mid.jpg
ADDED
![]() |
funky.jpeg
ADDED
![]() |
ultra_checkers.png
ADDED
![]() |