Upload folder using huggingface_hub

app.py

@@ -0,0 +1,161 @@
+from app_settings import AppSettings
+from utils import show_system_info
+import constants
+from argparse import ArgumentParser
+from context import Context
+from constants import APP_VERSION, LCM_DEFAULT_MODEL_OPENVINO
+from models.interface_types import InterfaceType
+from constants import DEVICE
+
+parser = ArgumentParser(description=f"FAST SD CPU {constants.APP_VERSION}")
+parser.add_argument(
+    "-s",
+    "--share",
+    action="store_true",
+    help="Create sharable link(Web UI)",
+    required=False,
+)
+group = parser.add_mutually_exclusive_group(required=False)
+group.add_argument(
+    "-g",
+    "--gui",
+    action="store_true",
+    help="Start desktop GUI",
+)
+group.add_argument(
+    "-w",
+    "--webui",
+    action="store_true",
+    help="Start Web UI",
+)
+group.add_argument(
+    "-r",
+    "--realtime",
+    action="store_true",
+    help="Start realtime inference UI(experimental)",
+)
+group.add_argument(
+    "-v",
+    "--version",
+    action="store_true",
+    help="Version",
+)
+parser.add_argument(
+    "--lcm_model_id",
+    type=str,
+    help="Model ID or path,Default SimianLuo/LCM_Dreamshaper_v7",
+    default="SimianLuo/LCM_Dreamshaper_v7",
+)
+parser.add_argument(
+    "--prompt",
+    type=str,
+    help="Describe the image you want to generate",
+)
+parser.add_argument(
+    "--image_height",
+    type=int,
+    help="Height of the image",
+    default=512,
+)
+parser.add_argument(
+    "--image_width",
+    type=int,
+    help="Width of the image",
+    default=512,
+)
+parser.add_argument(
+    "--inference_steps",
+    type=int,
+    help="Number of steps,default : 4",
+    default=4,
+)
+parser.add_argument(
+    "--guidance_scale",
+    type=int,
+    help="Guidance scale,default : 1.0",
+    default=1.0,
+)
+
+parser.add_argument(
+    "--number_of_images",
+    type=int,
+    help="Number of images to generate ,default : 1",
+    default=1,
+)
+parser.add_argument(
+    "--seed",
+    type=int,
+    help="Seed,default : -1 (disabled) ",
+    default=-1,
+)
+parser.add_argument(
+    "--use_openvino",
+    action="store_true",
+    help="Use OpenVINO model",
+)
+
+parser.add_argument(
+    "--use_offline_model",
+    action="store_true",
+    help="Use offline model",
+)
+parser.add_argument(
+    "--use_safety_checker",
+    action="store_false",
+    help="Use safety checker",
+)
+parser.add_argument(
+    "--use_lcm_lora",
+    action="store_true",
+    help="Use LCM-LoRA",
+)
+parser.add_argument(
+    "--base_model_id",
+    type=str,
+    help="LCM LoRA base model ID,Default Lykon/dreamshaper-8",
+    default="Lykon/dreamshaper-8",
+)
+parser.add_argument(
+    "--lcm_lora_id",
+    type=str,
+    help="LCM LoRA model ID,Default latent-consistency/lcm-lora-sdv1-5",
+    default="latent-consistency/lcm-lora-sdv1-5",
+)
+parser.add_argument(
+    "-i",
+    "--interactive",
+    action="store_true",
+    help="Interactive CLI mode",
+)
+parser.add_argument(
+    "--use_tiny_auto_encoder",
+    action="store_true",
+    help="Use tiny auto encoder for SD (TAESD)",
+)
+args = parser.parse_args()
+
+if args.version:
+    print(APP_VERSION)
+    exit()
+
+# parser.print_help()
+show_system_info()
+print(f"Using device : {constants.DEVICE}")
+app_settings = AppSettings()
+app_settings.load()
+print(
+    f"Found {len(app_settings.stable_diffsuion_models)} stable diffusion models in config/stable-diffusion-models.txt"
+)
+print(
+    f"Found {len(app_settings.lcm_lora_models)} LCM-LoRA models in config/lcm-lora-models.txt"
+)
+print(
+    f"Found {len(app_settings.openvino_lcm_models)} OpenVINO LCM models in config/openvino-lcm-models.txt"
+)
+
+from frontend.webui.ui import start_webui
+
+print("Starting web UI mode")
+start_webui(
+    args.share,
+)

app_settings.py

@@ -0,0 +1,89 @@
+import yaml
+from os import path, makedirs
+from models.settings import Settings
+from paths import FastStableDiffusionPaths
+from utils import get_models_from_text_file
+from constants import (
+    OPENVINO_LCM_MODELS_FILE,
+    LCM_LORA_MODELS_FILE,
+    SD_MODELS_FILE,
+    LCM_MODELS_FILE,
+)
+from copy import deepcopy
+
+
+class AppSettings:
+    def __init__(self):
+        self.config_path = FastStableDiffusionPaths().get_app_settings_path()
+        self._stable_diffsuion_models = get_models_from_text_file(
+            FastStableDiffusionPaths().get_models_config_path(SD_MODELS_FILE)
+        )
+        self._lcm_lora_models = get_models_from_text_file(
+            FastStableDiffusionPaths().get_models_config_path(LCM_LORA_MODELS_FILE)
+        )
+        self._openvino_lcm_models = get_models_from_text_file(
+            FastStableDiffusionPaths().get_models_config_path(OPENVINO_LCM_MODELS_FILE)
+        )
+        self._lcm_models = get_models_from_text_file(
+            FastStableDiffusionPaths().get_models_config_path(LCM_MODELS_FILE)
+        )
+
+    @property
+    def settings(self):
+        return self._config
+
+    @property
+    def stable_diffsuion_models(self):
+        return self._stable_diffsuion_models
+
+    @property
+    def openvino_lcm_models(self):
+        return self._openvino_lcm_models
+
+    @property
+    def lcm_models(self):
+        return self._lcm_models
+
+    @property
+    def lcm_lora_models(self):
+        return self._lcm_lora_models
+
+    def load(self, skip_file=False):
+        if skip_file:
+            print("Skipping config file")
+            settings_dict = self._load_default()
+            self._config = Settings.parse_obj(settings_dict)
+        else:
+            if not path.exists(self.config_path):
+                base_dir = path.dirname(self.config_path)
+                if not path.exists(base_dir):
+                    makedirs(base_dir)
+                try:
+                    print("Settings not found creating default settings")
+                    with open(self.config_path, "w") as file:
+                        yaml.dump(
+                            self._load_default(),
+                            file,
+                        )
+                except Exception as ex:
+                    print(f"Error in creating settings : {ex}")
+                    exit()
+            try:
+                with open(self.config_path) as file:
+                    settings_dict = yaml.safe_load(file)
+                    self._config = Settings.parse_obj(settings_dict)
+            except Exception as ex:
+                print(f"Error in loading settings : {ex}")
+
+    def save(self):
+        try:
+            with open(self.config_path, "w") as file:
+                tmp_cfg = deepcopy(self._config)
+                tmp_cfg.lcm_diffusion_setting.init_image = None
+                yaml.dump(tmp_cfg.dict(), file)
+        except Exception as ex:
+            print(f"Error in saving settings : {ex}")
+
+    def _load_default(self) -> dict:
+        defult_config = Settings()
+        return defult_config.dict()

backend/device.py

@@ -0,0 +1,23 @@
+import platform
+from constants import DEVICE
+import torch
+import openvino as ov
+
+core = ov.Core()
+
+
+def is_openvino_device() -> bool:
+    if DEVICE.lower() == "cpu" or DEVICE.lower()[0] == "g":
+        return True
+    else:
+        return False
+
+
+def get_device_name() -> str:
+    if DEVICE == "cuda" or DEVICE == "mps":
+        default_gpu_index = torch.cuda.current_device()
+        return torch.cuda.get_device_name(default_gpu_index)
+    elif platform.system().lower() == "darwin":
+        return platform.processor()
+    elif is_openvino_device():
+        return core.get_property(DEVICE.upper(), "FULL_DEVICE_NAME")

backend/image_saver.py

@@ -0,0 +1,40 @@
+from os import path, mkdir
+from typing import Any
+from uuid import uuid4
+from backend.models.lcmdiffusion_setting import LCMDiffusionSetting
+import json
+
+
+class ImageSaver:
+    @staticmethod
+    def save_images(
+        output_path: str,
+        images: Any,
+        folder_name: str = "",
+        format: str = ".png",
+        lcm_diffusion_setting: LCMDiffusionSetting = None,
+    ) -> None:
+        gen_id = uuid4()
+
+        for index, image in enumerate(images):
+            if not path.exists(output_path):
+                mkdir(output_path)
+
+            if folder_name:
+                out_path = path.join(
+                    output_path,
+                    folder_name,
+                )
+            else:
+                out_path = output_path
+
+            if not path.exists(out_path):
+                mkdir(out_path)
+            image.save(path.join(out_path, f"{gen_id}-{index+1}{format}"))
+        if lcm_diffusion_setting:
+            with open(path.join(out_path, f"{gen_id}.json"), "w") as json_file:
+                json.dump(
+                    lcm_diffusion_setting.model_dump(exclude="init_image"),
+                    json_file,
+                    indent=4,
+                )

backend/lcm_text_to_image.py

@@ -0,0 +1,352 @@
+from typing import Any
+from diffusers import LCMScheduler
+import torch
+from backend.models.lcmdiffusion_setting import LCMDiffusionSetting
+import numpy as np
+from constants import DEVICE
+from backend.models.lcmdiffusion_setting import LCMLora
+from backend.device import is_openvino_device
+from backend.openvino.pipelines import (
+    get_ov_text_to_image_pipeline,
+    ov_load_taesd,
+    get_ov_image_to_image_pipeline,
+)
+from backend.pipelines.lcm import (
+    get_lcm_model_pipeline,
+    load_taesd,
+    get_image_to_image_pipeline,
+)
+from backend.pipelines.lcm_lora import get_lcm_lora_pipeline
+from backend.models.lcmdiffusion_setting import DiffusionTask
+from image_ops import resize_pil_image
+from math import ceil
+
+
+class LCMTextToImage:
+    def __init__(
+        self,
+        device: str = "cpu",
+    ) -> None:
+        self.pipeline = None
+        self.use_openvino = False
+        self.device = ""
+        self.previous_model_id = None
+        self.previous_use_tae_sd = False
+        self.previous_use_lcm_lora = False
+        self.previous_ov_model_id = ""
+        self.previous_safety_checker = False
+        self.previous_use_openvino = False
+        self.img_to_img_pipeline = None
+        self.is_openvino_init = False
+        self.torch_data_type = (
+            torch.float32 if is_openvino_device() or DEVICE == "mps" else torch.float16
+        )
+        print(f"Torch datatype : {self.torch_data_type}")
+
+    def _pipeline_to_device(self):
+        print(f"Pipeline device : {DEVICE}")
+        print(f"Pipeline dtype : {self.torch_data_type}")
+        self.pipeline.to(
+            torch_device=DEVICE,
+            torch_dtype=self.torch_data_type,
+        )
+
+    def _add_freeu(self):
+        pipeline_class = self.pipeline.__class__.__name__
+        if isinstance(self.pipeline.scheduler, LCMScheduler):
+            if pipeline_class == "StableDiffusionPipeline":
+                print("Add FreeU - SD")
+                self.pipeline.enable_freeu(
+                    s1=0.9,
+                    s2=0.2,
+                    b1=1.2,
+                    b2=1.4,
+                )
+            elif pipeline_class == "StableDiffusionXLPipeline":
+                print("Add FreeU - SDXL")
+                self.pipeline.enable_freeu(
+                    s1=0.6,
+                    s2=0.4,
+                    b1=1.1,
+                    b2=1.2,
+                )
+
+    def _update_lcm_scheduler_params(self):
+        if isinstance(self.pipeline.scheduler, LCMScheduler):
+            self.pipeline.scheduler = LCMScheduler.from_config(
+                self.pipeline.scheduler.config,
+                beta_start=0.001,
+                beta_end=0.01,
+            )
+
+    def init(
+        self,
+        device: str = "cpu",
+        lcm_diffusion_setting: LCMDiffusionSetting = LCMDiffusionSetting(),
+    ) -> None:
+        self.device = device
+        self.use_openvino = lcm_diffusion_setting.use_openvino
+        model_id = lcm_diffusion_setting.lcm_model_id
+        use_local_model = lcm_diffusion_setting.use_offline_model
+        use_tiny_auto_encoder = lcm_diffusion_setting.use_tiny_auto_encoder
+        use_lora = lcm_diffusion_setting.use_lcm_lora
+        lcm_lora: LCMLora = lcm_diffusion_setting.lcm_lora
+        ov_model_id = lcm_diffusion_setting.openvino_lcm_model_id
+
+        if lcm_diffusion_setting.diffusion_task == DiffusionTask.image_to_image.value:
+            lcm_diffusion_setting.init_image = resize_pil_image(
+                lcm_diffusion_setting.init_image,
+                lcm_diffusion_setting.image_width,
+                lcm_diffusion_setting.image_height,
+            )
+
+        if (
+            self.pipeline is None
+            or self.previous_model_id != model_id
+            or self.previous_use_tae_sd != use_tiny_auto_encoder
+            or self.previous_lcm_lora_base_id != lcm_lora.base_model_id
+            or self.previous_lcm_lora_id != lcm_lora.lcm_lora_id
+            or self.previous_use_lcm_lora != use_lora
+            or self.previous_ov_model_id != ov_model_id
+            or self.previous_safety_checker != lcm_diffusion_setting.use_safety_checker
+            or self.previous_use_openvino != lcm_diffusion_setting.use_openvino
+        ):
+            if self.use_openvino and is_openvino_device():
+                if self.pipeline:
+                    del self.pipeline
+                    self.pipeline = None
+                self.is_openvino_init = True
+                if (
+                    lcm_diffusion_setting.diffusion_task
+                    == DiffusionTask.text_to_image.value
+                ):
+                    print(f"***** Init Text to image (OpenVINO) - {ov_model_id} *****")
+                    self.pipeline = get_ov_text_to_image_pipeline(
+                        ov_model_id,
+                        use_local_model,
+                    )
+                elif (
+                    lcm_diffusion_setting.diffusion_task
+                    == DiffusionTask.image_to_image.value
+                ):
+                    print(f"***** Image to image (OpenVINO) - {ov_model_id} *****")
+                    self.pipeline = get_ov_image_to_image_pipeline(
+                        ov_model_id,
+                        use_local_model,
+                    )
+            else:
+                if self.pipeline:
+                    del self.pipeline
+                    self.pipeline = None
+                if self.img_to_img_pipeline:
+                    del self.img_to_img_pipeline
+                    self.img_to_img_pipeline = None
+
+                if use_lora:
+                    print(
+                        f"***** Init LCM-LoRA pipeline - {lcm_lora.base_model_id} *****"
+                    )
+                    self.pipeline = get_lcm_lora_pipeline(
+                        lcm_lora.base_model_id,
+                        lcm_lora.lcm_lora_id,
+                        use_local_model,
+                        torch_data_type=self.torch_data_type,
+                    )
+                else:
+                    print(f"***** Init LCM Model pipeline - {model_id} *****")
+                    self.pipeline = get_lcm_model_pipeline(
+                        model_id,
+                        use_local_model,
+                    )
+
+                if (
+                    lcm_diffusion_setting.diffusion_task
+                    == DiffusionTask.image_to_image.value
+                ):
+                    self.img_to_img_pipeline = get_image_to_image_pipeline(
+                        self.pipeline
+                    )
+                self._pipeline_to_device()
+
+            if use_tiny_auto_encoder:
+                if self.use_openvino and is_openvino_device():
+                    print("Using Tiny Auto Encoder (OpenVINO)")
+                    ov_load_taesd(
+                        self.pipeline,
+                        use_local_model,
+                    )
+                else:
+                    print("Using Tiny Auto Encoder")
+                    if (
+                        lcm_diffusion_setting.diffusion_task
+                        == DiffusionTask.text_to_image.value
+                    ):
+                        load_taesd(
+                            self.pipeline,
+                            use_local_model,
+                            self.torch_data_type,
+                        )
+                    elif (
+                        lcm_diffusion_setting.diffusion_task
+                        == DiffusionTask.image_to_image.value
+                    ):
+                        load_taesd(
+                            self.img_to_img_pipeline,
+                            use_local_model,
+                            self.torch_data_type,
+                        )
+
+            if (
+                lcm_diffusion_setting.diffusion_task
+                == DiffusionTask.image_to_image.value
+                and lcm_diffusion_setting.use_openvino
+            ):
+                self.pipeline.scheduler = LCMScheduler.from_config(
+                    self.pipeline.scheduler.config,
+                )
+            else:
+                self._update_lcm_scheduler_params()
+
+            if use_lora:
+                self._add_freeu()
+
+            self.previous_model_id = model_id
+            self.previous_ov_model_id = ov_model_id
+            self.previous_use_tae_sd = use_tiny_auto_encoder
+            self.previous_lcm_lora_base_id = lcm_lora.base_model_id
+            self.previous_lcm_lora_id = lcm_lora.lcm_lora_id
+            self.previous_use_lcm_lora = use_lora
+            self.previous_safety_checker = lcm_diffusion_setting.use_safety_checker
+            self.previous_use_openvino = lcm_diffusion_setting.use_openvino
+            if (
+                lcm_diffusion_setting.diffusion_task
+                == DiffusionTask.text_to_image.value
+            ):
+                print(f"Pipeline : {self.pipeline}")
+            elif (
+                lcm_diffusion_setting.diffusion_task
+                == DiffusionTask.image_to_image.value
+            ):
+                if self.use_openvino and is_openvino_device():
+                    print(f"Pipeline : {self.pipeline}")
+                else:
+                    print(f"Pipeline : {self.img_to_img_pipeline}")
+
+    def generate(
+        self,
+        lcm_diffusion_setting: LCMDiffusionSetting,
+        reshape: bool = False,
+    ) -> Any:
+        guidance_scale = lcm_diffusion_setting.guidance_scale
+        img_to_img_inference_steps = lcm_diffusion_setting.inference_steps
+        check_step_value = int(
+            lcm_diffusion_setting.inference_steps * lcm_diffusion_setting.strength
+        )
+        if (
+            lcm_diffusion_setting.diffusion_task == DiffusionTask.image_to_image.value
+            and check_step_value < 1
+        ):
+            img_to_img_inference_steps = ceil(1 / lcm_diffusion_setting.strength)
+            print(
+                f"Strength: {lcm_diffusion_setting.strength},{img_to_img_inference_steps}"
+            )
+
+        if lcm_diffusion_setting.use_seed:
+            cur_seed = lcm_diffusion_setting.seed
+            if self.use_openvino:
+                np.random.seed(cur_seed)
+            else:
+                torch.manual_seed(cur_seed)
+
+        is_openvino_pipe = lcm_diffusion_setting.use_openvino and is_openvino_device()
+        if is_openvino_pipe:
+            print("Using OpenVINO")
+            if reshape and not self.is_openvino_init:
+                print("Reshape and compile")
+                self.pipeline.reshape(
+                    batch_size=-1,
+                    height=lcm_diffusion_setting.image_height,
+                    width=lcm_diffusion_setting.image_width,
+                    num_images_per_prompt=lcm_diffusion_setting.number_of_images,
+                )
+                self.pipeline.compile()
+
+            if self.is_openvino_init:
+                self.is_openvino_init = False
+
+        if not lcm_diffusion_setting.use_safety_checker:
+            self.pipeline.safety_checker = None
+            if (
+                lcm_diffusion_setting.diffusion_task
+                == DiffusionTask.image_to_image.value
+                and not is_openvino_pipe
+            ):
+                self.img_to_img_pipeline.safety_checker = None
+
+        if (
+            not lcm_diffusion_setting.use_lcm_lora
+            and not lcm_diffusion_setting.use_openvino
+            and lcm_diffusion_setting.guidance_scale != 1.0
+        ):
+            print("Not using LCM-LoRA so setting guidance_scale 1.0")
+            guidance_scale = 1.0
+
+        if lcm_diffusion_setting.use_openvino:
+            if (
+                lcm_diffusion_setting.diffusion_task
+                == DiffusionTask.text_to_image.value
+            ):
+                result_images = self.pipeline(
+                    prompt=lcm_diffusion_setting.prompt,
+                    negative_prompt=lcm_diffusion_setting.negative_prompt,
+                    num_inference_steps=lcm_diffusion_setting.inference_steps,
+                    guidance_scale=guidance_scale,
+                    width=lcm_diffusion_setting.image_width,
+                    height=lcm_diffusion_setting.image_height,
+                    num_images_per_prompt=lcm_diffusion_setting.number_of_images,
+                ).images
+            elif (
+                lcm_diffusion_setting.diffusion_task
+                == DiffusionTask.image_to_image.value
+            ):
+                result_images = self.pipeline(
+                    image=lcm_diffusion_setting.init_image,
+                    strength=lcm_diffusion_setting.strength,
+                    prompt=lcm_diffusion_setting.prompt,
+                    negative_prompt=lcm_diffusion_setting.negative_prompt,
+                    num_inference_steps=img_to_img_inference_steps * 3,
+                    guidance_scale=guidance_scale,
+                    num_images_per_prompt=lcm_diffusion_setting.number_of_images,
+                ).images
+
+        else:
+            if (
+                lcm_diffusion_setting.diffusion_task
+                == DiffusionTask.text_to_image.value
+            ):
+                result_images = self.pipeline(
+                    prompt=lcm_diffusion_setting.prompt,
+                    negative_prompt=lcm_diffusion_setting.negative_prompt,
+                    num_inference_steps=lcm_diffusion_setting.inference_steps,
+                    guidance_scale=guidance_scale,
+                    width=lcm_diffusion_setting.image_width,
+                    height=lcm_diffusion_setting.image_height,
+                    num_images_per_prompt=lcm_diffusion_setting.number_of_images,
+                ).images
+            elif (
+                lcm_diffusion_setting.diffusion_task
+                == DiffusionTask.image_to_image.value
+            ):
+                result_images = self.img_to_img_pipeline(
+                    image=lcm_diffusion_setting.init_image,
+                    strength=lcm_diffusion_setting.strength,
+                    prompt=lcm_diffusion_setting.prompt,
+                    negative_prompt=lcm_diffusion_setting.negative_prompt,
+                    num_inference_steps=img_to_img_inference_steps,
+                    guidance_scale=guidance_scale,
+                    width=lcm_diffusion_setting.image_width,
+                    height=lcm_diffusion_setting.image_height,
+                    num_images_per_prompt=lcm_diffusion_setting.number_of_images,
+                ).images
+
+        return result_images

backend/models/lcmdiffusion_setting.py

@@ -0,0 +1,39 @@
+from typing import Optional, Any
+from enum import Enum
+from pydantic import BaseModel
+from constants import LCM_DEFAULT_MODEL, LCM_DEFAULT_MODEL_OPENVINO
+
+
+class LCMLora(BaseModel):
+    base_model_id: str = "Lykon/dreamshaper-8"
+    lcm_lora_id: str = "latent-consistency/lcm-lora-sdv1-5"
+
+
+class DiffusionTask(str, Enum):
+    """Diffusion task types"""
+
+    text_to_image = "text_to_image"
+    image_to_image = "image_to_image"
+
+
+class LCMDiffusionSetting(BaseModel):
+    lcm_model_id: str = LCM_DEFAULT_MODEL
+    openvino_lcm_model_id: str = LCM_DEFAULT_MODEL_OPENVINO
+    use_offline_model: bool = False
+    use_lcm_lora: bool = False
+    lcm_lora: Optional[LCMLora] = LCMLora()
+    use_tiny_auto_encoder: bool = False
+    use_openvino: bool = False
+    prompt: str = ""
+    negative_prompt: str = ""
+    init_image: Any = None
+    strength: Optional[float] = 0.6
+    image_height: Optional[int] = 512
+    image_width: Optional[int] = 512
+    inference_steps: Optional[int] = 1
+    guidance_scale: Optional[float] = 1
+    number_of_images: Optional[int] = 1
+    seed: Optional[int] = 123123
+    use_seed: bool = False
+    use_safety_checker: bool = False
+    diffusion_task: str = DiffusionTask.text_to_image.value

backend/openvino/custom_ov_model_vae_decoder.py

@@ -0,0 +1,21 @@
+from backend.device import is_openvino_device
+
+if is_openvino_device():
+    from optimum.intel.openvino.modeling_diffusion import OVModelVaeDecoder
+
+
+class CustomOVModelVaeDecoder(OVModelVaeDecoder):
+    def __init__(
+        self,
+        model,
+        parent_model,
+        ov_config=None,
+        model_dir=None,
+    ):
+        super(OVModelVaeDecoder, self).__init__(
+            model,
+            parent_model,
+            ov_config,
+            "vae_decoder",
+            model_dir,
+        )

backend/openvino/pipelines.py

@@ -0,0 +1,75 @@
+from constants import DEVICE, LCM_DEFAULT_MODEL_OPENVINO
+from backend.tiny_decoder import get_tiny_decoder_vae_model
+from typing import Any
+from backend.device import is_openvino_device
+from paths import get_base_folder_name
+
+if is_openvino_device():
+    from huggingface_hub import snapshot_download
+    from optimum.intel.openvino.modeling_diffusion import OVBaseModel
+
+    from optimum.intel.openvino.modeling_diffusion import (
+        OVStableDiffusionPipeline,
+        OVStableDiffusionImg2ImgPipeline,
+        OVStableDiffusionXLPipeline,
+        OVStableDiffusionXLImg2ImgPipeline,
+    )
+    from backend.openvino.custom_ov_model_vae_decoder import CustomOVModelVaeDecoder
+
+
+def ov_load_taesd(
+    pipeline: Any,
+    use_local_model: bool = False,
+):
+    taesd_dir = snapshot_download(
+        repo_id=get_tiny_decoder_vae_model(pipeline.__class__.__name__),
+        local_files_only=use_local_model,
+    )
+    pipeline.vae_decoder = CustomOVModelVaeDecoder(
+        model=OVBaseModel.load_model(f"{taesd_dir}/vae_decoder/openvino_model.xml"),
+        parent_model=pipeline,
+        model_dir=taesd_dir,
+    )
+
+
+def get_ov_text_to_image_pipeline(
+    model_id: str = LCM_DEFAULT_MODEL_OPENVINO,
+    use_local_model: bool = False,
+) -> Any:
+    if "xl" in get_base_folder_name(model_id).lower():
+        pipeline = OVStableDiffusionXLPipeline.from_pretrained(
+            model_id,
+            local_files_only=use_local_model,
+            ov_config={"CACHE_DIR": ""},
+            device=DEVICE.upper(),
+        )
+    else:
+        pipeline = OVStableDiffusionPipeline.from_pretrained(
+            model_id,
+            local_files_only=use_local_model,
+            ov_config={"CACHE_DIR": ""},
+            device=DEVICE.upper(),
+        )
+
+    return pipeline
+
+
+def get_ov_image_to_image_pipeline(
+    model_id: str = LCM_DEFAULT_MODEL_OPENVINO,
+    use_local_model: bool = False,
+) -> Any:
+    if "xl" in get_base_folder_name(model_id).lower():
+        pipeline = OVStableDiffusionXLImg2ImgPipeline.from_pretrained(
+            model_id,
+            local_files_only=use_local_model,
+            ov_config={"CACHE_DIR": ""},
+            device=DEVICE.upper(),
+        )
+    else:
+        pipeline = OVStableDiffusionImg2ImgPipeline.from_pretrained(
+            model_id,
+            local_files_only=use_local_model,
+            ov_config={"CACHE_DIR": ""},
+            device=DEVICE.upper(),
+        )
+    return pipeline

backend/pipelines/lcm.py

@@ -0,0 +1,90 @@
+from constants import LCM_DEFAULT_MODEL
+from diffusers import (
+    DiffusionPipeline,
+    AutoencoderTiny,
+    UNet2DConditionModel,
+    LCMScheduler,
+)
+import torch
+from backend.tiny_decoder import get_tiny_decoder_vae_model
+from typing import Any
+from diffusers import (
+    LCMScheduler,
+    StableDiffusionImg2ImgPipeline,
+    StableDiffusionXLImg2ImgPipeline,
+)
+
+
+def _get_lcm_pipeline_from_base_model(
+    lcm_model_id: str,
+    base_model_id: str,
+    use_local_model: bool,
+):
+    pipeline = None
+    unet = UNet2DConditionModel.from_pretrained(
+        lcm_model_id,
+        torch_dtype=torch.float32,
+        local_files_only=use_local_model,
+    )
+    pipeline = DiffusionPipeline.from_pretrained(
+        base_model_id,
+        unet=unet,
+        torch_dtype=torch.float32,
+        local_files_only=use_local_model,
+    )
+    pipeline.scheduler = LCMScheduler.from_config(pipeline.scheduler.config)
+    return pipeline
+
+
+def load_taesd(
+    pipeline: Any,
+    use_local_model: bool = False,
+    torch_data_type: torch.dtype = torch.float32,
+):
+    vae_model = get_tiny_decoder_vae_model(pipeline.__class__.__name__)
+    pipeline.vae = AutoencoderTiny.from_pretrained(
+        vae_model,
+        torch_dtype=torch_data_type,
+        local_files_only=use_local_model,
+    )
+
+
+def get_lcm_model_pipeline(
+    model_id: str = LCM_DEFAULT_MODEL,
+    use_local_model: bool = False,
+):
+    pipeline = None
+    if model_id == "latent-consistency/lcm-sdxl":
+        pipeline = _get_lcm_pipeline_from_base_model(
+            model_id,
+            "stabilityai/stable-diffusion-xl-base-1.0",
+            use_local_model,
+        )
+
+    elif model_id == "latent-consistency/lcm-ssd-1b":
+        pipeline = _get_lcm_pipeline_from_base_model(
+            model_id,
+            "segmind/SSD-1B",
+            use_local_model,
+        )
+    else:
+        pipeline = DiffusionPipeline.from_pretrained(
+            model_id,
+            local_files_only=use_local_model,
+        )
+
+    return pipeline
+
+
+def get_image_to_image_pipeline(pipeline: Any) -> Any:
+    components = pipeline.components
+    pipeline_class = pipeline.__class__.__name__
+    if (
+        pipeline_class == "LatentConsistencyModelPipeline"
+        or pipeline_class == "StableDiffusionPipeline"
+    ):
+        return StableDiffusionImg2ImgPipeline(**components)
+    elif pipeline_class == "StableDiffusionXLPipeline":
+        return StableDiffusionXLImg2ImgPipeline(**components)
+    else:
+        raise Exception(f"Unknown pipeline {pipeline_class}")

backend/pipelines/lcm_lora.py

@@ -0,0 +1,25 @@
+from diffusers import DiffusionPipeline, LCMScheduler
+import torch
+
+
+def get_lcm_lora_pipeline(
+    base_model_id: str,
+    lcm_lora_id: str,
+    use_local_model: bool,
+    torch_data_type: torch.dtype,
+):
+    pipeline = DiffusionPipeline.from_pretrained(
+        base_model_id,
+        torch_dtype=torch_data_type,
+        local_files_only=use_local_model,
+    )
+    pipeline.load_lora_weights(
+        lcm_lora_id,
+        local_files_only=use_local_model,
+    )
+    if "lcm" in lcm_lora_id.lower():
+        print("LCM LoRA model detected so using recommended LCMScheduler")
+        pipeline.scheduler = LCMScheduler.from_config(pipeline.scheduler.config)
+    pipeline.fuse_lora()
+    pipeline.unet.to(memory_format=torch.channels_last)
+    return pipeline

backend/tiny_decoder.py

@@ -0,0 +1,30 @@
+from constants import (
+    TAESD_MODEL,
+    TAESDXL_MODEL,
+    TAESD_MODEL_OPENVINO,
+    TAESDXL_MODEL_OPENVINO,
+)
+
+
+def get_tiny_decoder_vae_model(pipeline_class) -> str:
+    print(f"Pipeline class : {pipeline_class}")
+    if (
+        pipeline_class == "LatentConsistencyModelPipeline"
+        or pipeline_class == "StableDiffusionPipeline"
+        or pipeline_class == "StableDiffusionImg2ImgPipeline"
+    ):
+        return TAESD_MODEL
+    elif (
+        pipeline_class == "StableDiffusionXLPipeline"
+        or pipeline_class == "StableDiffusionXLImg2ImgPipeline"
+    ):
+        return TAESDXL_MODEL
+    elif (
+        pipeline_class == "OVStableDiffusionPipeline"
+        or pipeline_class == "OVStableDiffusionImg2ImgPipeline"
+    ):
+        return TAESD_MODEL_OPENVINO
+    elif pipeline_class == "OVStableDiffusionXLPipeline":
+        return TAESDXL_MODEL_OPENVINO
+    else:
+        raise Exception("No valid pipeline class found!")

constants.py

@@ -0,0 +1,18 @@
+from os import environ
+
+APP_VERSION = "v1.0.0 beta 22"
+LCM_DEFAULT_MODEL = "stabilityai/sd-turbo"
+LCM_DEFAULT_MODEL_OPENVINO = "rupeshs/sd-turbo-openvino"
+APP_NAME = "FastSD CPU"
+APP_SETTINGS_FILE = "settings.yaml"
+RESULTS_DIRECTORY = "results"
+CONFIG_DIRECTORY = "configs"
+DEVICE = environ.get("DEVICE", "cpu")
+SD_MODELS_FILE = "stable-diffusion-models.txt"
+LCM_LORA_MODELS_FILE = "lcm-lora-models.txt"
+OPENVINO_LCM_MODELS_FILE = "openvino-lcm-models.txt"
+TAESD_MODEL = "madebyollin/taesd"
+TAESDXL_MODEL = "madebyollin/taesdxl"
+TAESD_MODEL_OPENVINO = "deinferno/taesd-openvino"
+LCM_MODELS_FILE = "lcm-models.txt"
+TAESDXL_MODEL_OPENVINO = "rupeshs/taesdxl-openvino"

context.py

@@ -0,0 +1,46 @@
+from typing import Any
+from app_settings import Settings
+from models.interface_types import InterfaceType
+from backend.lcm_text_to_image import LCMTextToImage
+from time import perf_counter
+from backend.image_saver import ImageSaver
+from pprint import pprint
+from state import get_settings
+
+
+class Context:
+    def __init__(
+        self,
+        interface_type: InterfaceType,
+        device="cpu",
+    ):
+        self.interface_type = interface_type
+        self.lcm_text_to_image = LCMTextToImage(device)
+
+    def generate_text_to_image(
+        self,
+        settings: Settings,
+        reshape: bool = False,
+        device: str = "cpu",
+    ) -> Any:
+        get_settings().save()
+        tick = perf_counter()
+        pprint(settings.lcm_diffusion_setting.model_dump())
+        if not settings.lcm_diffusion_setting.lcm_lora:
+            return None
+        self.lcm_text_to_image.init(
+            device,
+            settings.lcm_diffusion_setting,
+        )
+        images = self.lcm_text_to_image.generate(
+            settings.lcm_diffusion_setting,
+            reshape,
+        )
+        elapsed = perf_counter() - tick
+        ImageSaver.save_images(
+            settings.results_path,
+            images=images,
+            lcm_diffusion_setting=settings.lcm_diffusion_setting,
+        )
+        print(f"Latency : {elapsed:.2f} seconds")
+        return images

frontend/gui/app_window.py

@@ -0,0 +1,604 @@
+from PyQt5.QtWidgets import (
+    QWidget,
+    QPushButton,
+    QHBoxLayout,
+    QVBoxLayout,
+    QLabel,
+    QLineEdit,
+    QMainWindow,
+    QSlider,
+    QTabWidget,
+    QSpacerItem,
+    QSizePolicy,
+    QComboBox,
+    QCheckBox,
+    QTextEdit,
+    QToolButton,
+    QFileDialog,
+)
+from PyQt5 import QtWidgets, QtCore
+from PyQt5.QtGui import QPixmap, QDesktopServices
+from PyQt5.QtCore import QSize, QThreadPool, Qt, QUrl
+
+from PIL.ImageQt import ImageQt
+from constants import (
+    LCM_DEFAULT_MODEL,
+    LCM_DEFAULT_MODEL_OPENVINO,
+    APP_NAME,
+    APP_VERSION,
+)
+from frontend.gui.image_generator_worker import ImageGeneratorWorker
+from app_settings import AppSettings
+from paths import FastStableDiffusionPaths
+from frontend.utils import is_reshape_required
+from context import Context
+from models.interface_types import InterfaceType
+from constants import DEVICE
+from frontend.utils import enable_openvino_controls, get_valid_model_id
+from backend.models.lcmdiffusion_setting import DiffusionTask
+
+# DPI scale fix
+QtWidgets.QApplication.setAttribute(QtCore.Qt.AA_EnableHighDpiScaling, True)
+QtWidgets.QApplication.setAttribute(QtCore.Qt.AA_UseHighDpiPixmaps, True)
+
+
+class MainWindow(QMainWindow):
+    def __init__(self, config: AppSettings):
+        super().__init__()
+        self.config = config
+        self.setWindowTitle(APP_NAME)
+        self.setFixedSize(QSize(600, 670))
+        self.init_ui()
+        self.pipeline = None
+        self.threadpool = QThreadPool()
+        self.device = "cpu"
+        self.previous_width = 0
+        self.previous_height = 0
+        self.previous_model = ""
+        self.previous_num_of_images = 0
+        self.context = Context(InterfaceType.GUI)
+        self.init_ui_values()
+        self.gen_images = []
+        self.image_index = 0
+        print(f"Output path : {  self.config.settings.results_path}")
+
+    def init_ui_values(self):
+        self.lcm_model.setEnabled(
+            not self.config.settings.lcm_diffusion_setting.use_openvino
+        )
+        self.guidance.setValue(
+            int(self.config.settings.lcm_diffusion_setting.guidance_scale * 10)
+        )
+        self.seed_value.setEnabled(self.config.settings.lcm_diffusion_setting.use_seed)
+        self.safety_checker.setChecked(
+            self.config.settings.lcm_diffusion_setting.use_safety_checker
+        )
+        self.use_openvino_check.setChecked(
+            self.config.settings.lcm_diffusion_setting.use_openvino
+        )
+        self.width.setCurrentText(
+            str(self.config.settings.lcm_diffusion_setting.image_width)
+        )
+        self.height.setCurrentText(
+            str(self.config.settings.lcm_diffusion_setting.image_height)
+        )
+        self.inference_steps.setValue(
+            int(self.config.settings.lcm_diffusion_setting.inference_steps)
+        )
+        self.seed_check.setChecked(self.config.settings.lcm_diffusion_setting.use_seed)
+        self.seed_value.setText(str(self.config.settings.lcm_diffusion_setting.seed))
+        self.use_local_model_folder.setChecked(
+            self.config.settings.lcm_diffusion_setting.use_offline_model
+        )
+        self.results_path.setText(self.config.settings.results_path)
+        self.num_images.setValue(
+            self.config.settings.lcm_diffusion_setting.number_of_images
+        )
+        self.use_tae_sd.setChecked(
+            self.config.settings.lcm_diffusion_setting.use_tiny_auto_encoder
+        )
+        self.use_lcm_lora.setChecked(
+            self.config.settings.lcm_diffusion_setting.use_lcm_lora
+        )
+        self.lcm_model.setCurrentText(
+            get_valid_model_id(
+                self.config.lcm_models,
+                self.config.settings.lcm_diffusion_setting.lcm_model_id,
+                LCM_DEFAULT_MODEL,
+            )
+        )
+        self.base_model_id.setCurrentText(
+            get_valid_model_id(
+                self.config.stable_diffsuion_models,
+                self.config.settings.lcm_diffusion_setting.lcm_lora.base_model_id,
+            )
+        )
+        self.lcm_lora_id.setCurrentText(
+            get_valid_model_id(
+                self.config.lcm_lora_models,
+                self.config.settings.lcm_diffusion_setting.lcm_lora.lcm_lora_id,
+            )
+        )
+        self.openvino_lcm_model_id.setCurrentText(
+            get_valid_model_id(
+                self.config.openvino_lcm_models,
+                self.config.settings.lcm_diffusion_setting.openvino_lcm_model_id,
+                LCM_DEFAULT_MODEL_OPENVINO,
+            )
+        )
+        self.neg_prompt.setEnabled(
+            self.config.settings.lcm_diffusion_setting.use_lcm_lora
+            or self.config.settings.lcm_diffusion_setting.use_openvino
+        )
+        self.openvino_lcm_model_id.setEnabled(
+            self.config.settings.lcm_diffusion_setting.use_openvino
+        )
+
+    def init_ui(self):
+        self.create_main_tab()
+        self.create_settings_tab()
+        self.create_about_tab()
+        self.show()
+
+    def create_main_tab(self):
+        self.img = QLabel("<<Image>>")
+        self.img.setAlignment(Qt.AlignCenter)
+        self.img.setFixedSize(QSize(512, 512))
+        self.vspacer = QSpacerItem(20, 40, QSizePolicy.Minimum, QSizePolicy.Expanding)
+
+        self.prompt = QTextEdit()
+        self.prompt.setPlaceholderText("A fantasy landscape")
+        self.prompt.setAcceptRichText(False)
+        self.neg_prompt = QTextEdit()
+        self.neg_prompt.setPlaceholderText("")
+        self.neg_prompt.setAcceptRichText(False)
+        self.neg_prompt_label = QLabel("Negative prompt (Set guidance scale > 1.0):")
+        self.generate = QPushButton("Generate")
+        self.generate.clicked.connect(self.text_to_image)
+        self.prompt.setFixedHeight(40)
+        self.neg_prompt.setFixedHeight(35)
+        self.browse_results = QPushButton("...")
+        self.browse_results.setFixedWidth(30)
+        self.browse_results.clicked.connect(self.on_open_results_folder)
+        self.browse_results.setToolTip("Open output folder")
+
+        hlayout = QHBoxLayout()
+        hlayout.addWidget(self.neg_prompt)
+        hlayout.addWidget(self.generate)
+        hlayout.addWidget(self.browse_results)
+
+        self.previous_img_btn = QToolButton()
+        self.previous_img_btn.setText("<")
+        self.previous_img_btn.clicked.connect(self.on_show_previous_image)
+        self.next_img_btn = QToolButton()
+        self.next_img_btn.setText(">")
+        self.next_img_btn.clicked.connect(self.on_show_next_image)
+        hlayout_nav = QHBoxLayout()
+        hlayout_nav.addWidget(self.previous_img_btn)
+        hlayout_nav.addWidget(self.img)
+        hlayout_nav.addWidget(self.next_img_btn)
+
+        vlayout = QVBoxLayout()
+        vlayout.addLayout(hlayout_nav)
+        vlayout.addItem(self.vspacer)
+        vlayout.addWidget(self.prompt)
+        vlayout.addWidget(self.neg_prompt_label)
+        vlayout.addLayout(hlayout)
+
+        self.tab_widget = QTabWidget(self)
+        self.tab_main = QWidget()
+        self.tab_settings = QWidget()
+        self.tab_about = QWidget()
+        self.tab_main.setLayout(vlayout)
+
+        self.tab_widget.addTab(self.tab_main, "Text to Image")
+        self.tab_widget.addTab(self.tab_settings, "Settings")
+        self.tab_widget.addTab(self.tab_about, "About")
+
+        self.setCentralWidget(self.tab_widget)
+        self.use_seed = False
+
+    def create_settings_tab(self):
+        self.lcm_model_label = QLabel("Latent Consistency Model:")
+        # self.lcm_model = QLineEdit(LCM_DEFAULT_MODEL)
+        self.lcm_model = QComboBox(self)
+        self.lcm_model.addItems(self.config.lcm_models)
+        self.lcm_model.currentIndexChanged.connect(self.on_lcm_model_changed)
+
+        self.use_lcm_lora = QCheckBox("Use LCM LoRA")
+        self.use_lcm_lora.setChecked(False)
+        self.use_lcm_lora.stateChanged.connect(self.use_lcm_lora_changed)
+
+        self.lora_base_model_id_label = QLabel("Lora base model ID :")
+        self.base_model_id = QComboBox(self)
+        self.base_model_id.addItems(self.config.stable_diffsuion_models)
+        self.base_model_id.currentIndexChanged.connect(self.on_base_model_id_changed)
+
+        self.lcm_lora_model_id_label = QLabel("LCM LoRA model ID :")
+        self.lcm_lora_id = QComboBox(self)
+        self.lcm_lora_id.addItems(self.config.lcm_lora_models)
+        self.lcm_lora_id.currentIndexChanged.connect(self.on_lcm_lora_id_changed)
+
+        self.inference_steps_value = QLabel("Number of inference steps: 4")
+        self.inference_steps = QSlider(orientation=Qt.Orientation.Horizontal)
+        self.inference_steps.setMaximum(25)
+        self.inference_steps.setMinimum(1)
+        self.inference_steps.setValue(4)
+        self.inference_steps.valueChanged.connect(self.update_steps_label)
+
+        self.num_images_value = QLabel("Number of images: 1")
+        self.num_images = QSlider(orientation=Qt.Orientation.Horizontal)
+        self.num_images.setMaximum(100)
+        self.num_images.setMinimum(1)
+        self.num_images.setValue(1)
+        self.num_images.valueChanged.connect(self.update_num_images_label)
+
+        self.guidance_value = QLabel("Guidance scale: 1")
+        self.guidance = QSlider(orientation=Qt.Orientation.Horizontal)
+        self.guidance.setMaximum(20)
+        self.guidance.setMinimum(10)
+        self.guidance.setValue(10)
+        self.guidance.valueChanged.connect(self.update_guidance_label)
+
+        self.width_value = QLabel("Width :")
+        self.width = QComboBox(self)
+        self.width.addItem("256")
+        self.width.addItem("512")
+        self.width.addItem("768")
+        self.width.addItem("1024")
+        self.width.setCurrentText("512")
+        self.width.currentIndexChanged.connect(self.on_width_changed)
+
+        self.height_value = QLabel("Height :")
+        self.height = QComboBox(self)
+        self.height.addItem("256")
+        self.height.addItem("512")
+        self.height.addItem("768")
+        self.height.addItem("1024")
+        self.height.setCurrentText("512")
+        self.height.currentIndexChanged.connect(self.on_height_changed)
+
+        self.seed_check = QCheckBox("Use seed")
+        self.seed_value = QLineEdit()
+        self.seed_value.setInputMask("9999999999")
+        self.seed_value.setText("123123")
+        self.seed_check.stateChanged.connect(self.seed_changed)
+
+        self.safety_checker = QCheckBox("Use safety checker")
+        self.safety_checker.setChecked(True)
+        self.safety_checker.stateChanged.connect(self.use_safety_checker_changed)
+
+        self.use_openvino_check = QCheckBox("Use OpenVINO")
+        self.use_openvino_check.setChecked(False)
+        self.openvino_model_label = QLabel("OpenVINO LCM model:")
+        self.use_local_model_folder = QCheckBox(
+            "Use locally cached model or downloaded model folder(offline)"
+        )
+        self.openvino_lcm_model_id = QComboBox(self)
+        self.openvino_lcm_model_id.addItems(self.config.openvino_lcm_models)
+        self.openvino_lcm_model_id.currentIndexChanged.connect(
+            self.on_openvino_lcm_model_id_changed
+        )
+
+        self.use_openvino_check.setEnabled(enable_openvino_controls())
+        self.use_local_model_folder.setChecked(False)
+        self.use_local_model_folder.stateChanged.connect(self.use_offline_model_changed)
+        self.use_openvino_check.stateChanged.connect(self.use_openvino_changed)
+
+        self.use_tae_sd = QCheckBox(
+            "Use Tiny Auto Encoder - TAESD (Fast, moderate quality)"
+        )
+        self.use_tae_sd.setChecked(False)
+        self.use_tae_sd.stateChanged.connect(self.use_tae_sd_changed)
+
+        hlayout = QHBoxLayout()
+        hlayout.addWidget(self.seed_check)
+        hlayout.addWidget(self.seed_value)
+        hspacer = QSpacerItem(20, 10, QSizePolicy.Expanding, QSizePolicy.Minimum)
+        slider_hspacer = QSpacerItem(20, 10, QSizePolicy.Expanding, QSizePolicy.Minimum)
+
+        self.results_path_label = QLabel("Output path:")
+        self.results_path = QLineEdit()
+        self.results_path.textChanged.connect(self.on_path_changed)
+        self.browse_folder_btn = QToolButton()
+        self.browse_folder_btn.setText("...")
+        self.browse_folder_btn.clicked.connect(self.on_browse_folder)
+
+        self.reset = QPushButton("Reset All")
+        self.reset.clicked.connect(self.reset_all_settings)
+
+        vlayout = QVBoxLayout()
+        vspacer = QSpacerItem(20, 20, QSizePolicy.Minimum, QSizePolicy.Expanding)
+        vlayout.addItem(hspacer)
+        vlayout.setSpacing(3)
+        vlayout.addWidget(self.lcm_model_label)
+        vlayout.addWidget(self.lcm_model)
+        vlayout.addWidget(self.use_local_model_folder)
+        vlayout.addWidget(self.use_lcm_lora)
+        vlayout.addWidget(self.lora_base_model_id_label)
+        vlayout.addWidget(self.base_model_id)
+        vlayout.addWidget(self.lcm_lora_model_id_label)
+        vlayout.addWidget(self.lcm_lora_id)
+        vlayout.addWidget(self.use_openvino_check)
+        vlayout.addWidget(self.openvino_model_label)
+        vlayout.addWidget(self.openvino_lcm_model_id)
+        vlayout.addWidget(self.use_tae_sd)
+        vlayout.addItem(slider_hspacer)
+        vlayout.addWidget(self.inference_steps_value)
+        vlayout.addWidget(self.inference_steps)
+        vlayout.addWidget(self.num_images_value)
+        vlayout.addWidget(self.num_images)
+        vlayout.addWidget(self.width_value)
+        vlayout.addWidget(self.width)
+        vlayout.addWidget(self.height_value)
+        vlayout.addWidget(self.height)
+        vlayout.addWidget(self.guidance_value)
+        vlayout.addWidget(self.guidance)
+        vlayout.addLayout(hlayout)
+        vlayout.addWidget(self.safety_checker)
+
+        vlayout.addWidget(self.results_path_label)
+        hlayout_path = QHBoxLayout()
+        hlayout_path.addWidget(self.results_path)
+        hlayout_path.addWidget(self.browse_folder_btn)
+        vlayout.addLayout(hlayout_path)
+        self.tab_settings.setLayout(vlayout)
+        hlayout_reset = QHBoxLayout()
+        hspacer = QSpacerItem(20, 20, QSizePolicy.Expanding, QSizePolicy.Minimum)
+        hlayout_reset.addItem(hspacer)
+        hlayout_reset.addWidget(self.reset)
+        vlayout.addLayout(hlayout_reset)
+        vlayout.addItem(vspacer)
+
+    def create_about_tab(self):
+        self.label = QLabel()
+        self.label.setAlignment(Qt.AlignCenter)
+        self.label.setText(
+            f"""<h1>FastSD CPU {APP_VERSION}</h1> 
+               <h3>(c)2023 - Rupesh Sreeraman</h3>
+                <h3>Faster stable diffusion on CPU</h3>
+                 <h3>Based on Latent Consistency Models</h3>
+                <h3>GitHub : https://github.com/rupeshs/fastsdcpu/</h3>"""
+        )
+
+        vlayout = QVBoxLayout()
+        vlayout.addWidget(self.label)
+        self.tab_about.setLayout(vlayout)
+
+    def show_image(self, pixmap):
+        image_width = self.config.settings.lcm_diffusion_setting.image_width
+        image_height = self.config.settings.lcm_diffusion_setting.image_height
+        if image_width > 512 or image_height > 512:
+            new_width = 512 if image_width > 512 else image_width
+            new_height = 512 if image_height > 512 else image_height
+            self.img.setPixmap(
+                pixmap.scaled(
+                    new_width,
+                    new_height,
+                    Qt.KeepAspectRatio,
+                )
+            )
+        else:
+            self.img.setPixmap(pixmap)
+
+    def on_show_next_image(self):
+        if self.image_index != len(self.gen_images) - 1 and len(self.gen_images) > 0:
+            self.previous_img_btn.setEnabled(True)
+            self.image_index += 1
+            self.show_image(self.gen_images[self.image_index])
+            if self.image_index == len(self.gen_images) - 1:
+                self.next_img_btn.setEnabled(False)
+
+    def on_open_results_folder(self):
+        QDesktopServices.openUrl(QUrl.fromLocalFile(self.config.settings.results_path))
+
+    def on_show_previous_image(self):
+        if self.image_index != 0:
+            self.next_img_btn.setEnabled(True)
+            self.image_index -= 1
+            self.show_image(self.gen_images[self.image_index])
+            if self.image_index == 0:
+                self.previous_img_btn.setEnabled(False)
+
+    def on_path_changed(self, text):
+        self.config.settings.results_path = text
+
+    def on_browse_folder(self):
+        options = QFileDialog.Options()
+        options |= QFileDialog.ShowDirsOnly
+
+        folder_path = QFileDialog.getExistingDirectory(
+            self, "Select a Folder", "", options=options
+        )
+
+        if folder_path:
+            self.config.settings.results_path = folder_path
+            self.results_path.setText(folder_path)
+
+    def on_width_changed(self, index):
+        width_txt = self.width.itemText(index)
+        self.config.settings.lcm_diffusion_setting.image_width = int(width_txt)
+
+    def on_height_changed(self, index):
+        height_txt = self.height.itemText(index)
+        self.config.settings.lcm_diffusion_setting.image_height = int(height_txt)
+
+    def on_lcm_model_changed(self, index):
+        model_id = self.lcm_model.itemText(index)
+        self.config.settings.lcm_diffusion_setting.lcm_model_id = model_id
+
+    def on_base_model_id_changed(self, index):
+        model_id = self.base_model_id.itemText(index)
+        self.config.settings.lcm_diffusion_setting.lcm_lora.base_model_id = model_id
+
+    def on_lcm_lora_id_changed(self, index):
+        model_id = self.lcm_lora_id.itemText(index)
+        self.config.settings.lcm_diffusion_setting.lcm_lora.lcm_lora_id = model_id
+
+    def on_openvino_lcm_model_id_changed(self, index):
+        model_id = self.openvino_lcm_model_id.itemText(index)
+        self.config.settings.lcm_diffusion_setting.openvino_lcm_model_id = model_id
+
+    def use_openvino_changed(self, state):
+        if state == 2:
+            self.lcm_model.setEnabled(False)
+            self.use_lcm_lora.setEnabled(False)
+            self.lcm_lora_id.setEnabled(False)
+            self.base_model_id.setEnabled(False)
+            self.neg_prompt.setEnabled(True)
+            self.openvino_lcm_model_id.setEnabled(True)
+            self.config.settings.lcm_diffusion_setting.use_openvino = True
+        else:
+            self.lcm_model.setEnabled(True)
+            self.use_lcm_lora.setEnabled(True)
+            self.lcm_lora_id.setEnabled(True)
+            self.base_model_id.setEnabled(True)
+            self.neg_prompt.setEnabled(False)
+            self.openvino_lcm_model_id.setEnabled(False)
+            self.config.settings.lcm_diffusion_setting.use_openvino = False
+
+    def use_tae_sd_changed(self, state):
+        if state == 2:
+            self.config.settings.lcm_diffusion_setting.use_tiny_auto_encoder = True
+        else:
+            self.config.settings.lcm_diffusion_setting.use_tiny_auto_encoder = False
+
+    def use_offline_model_changed(self, state):
+        if state == 2:
+            self.config.settings.lcm_diffusion_setting.use_offline_model = True
+        else:
+            self.config.settings.lcm_diffusion_setting.use_offline_model = False
+
+    def use_lcm_lora_changed(self, state):
+        if state == 2:
+            self.lcm_model.setEnabled(False)
+            self.lcm_lora_id.setEnabled(True)
+            self.base_model_id.setEnabled(True)
+            self.neg_prompt.setEnabled(True)
+            self.config.settings.lcm_diffusion_setting.use_lcm_lora = True
+        else:
+            self.lcm_model.setEnabled(True)
+            self.lcm_lora_id.setEnabled(False)
+            self.base_model_id.setEnabled(False)
+            self.neg_prompt.setEnabled(False)
+            self.config.settings.lcm_diffusion_setting.use_lcm_lora = False
+
+    def use_safety_checker_changed(self, state):
+        if state == 2:
+            self.config.settings.lcm_diffusion_setting.use_safety_checker = True
+        else:
+            self.config.settings.lcm_diffusion_setting.use_safety_checker = False
+
+    def update_steps_label(self, value):
+        self.inference_steps_value.setText(f"Number of inference steps: {value}")
+        self.config.settings.lcm_diffusion_setting.inference_steps = value
+
+    def update_num_images_label(self, value):
+        self.num_images_value.setText(f"Number of images: {value}")
+        self.config.settings.lcm_diffusion_setting.number_of_images = value
+
+    def update_guidance_label(self, value):
+        val = round(int(value) / 10, 1)
+        self.guidance_value.setText(f"Guidance scale: {val}")
+        self.config.settings.lcm_diffusion_setting.guidance_scale = val
+
+    def seed_changed(self, state):
+        if state == 2:
+            self.seed_value.setEnabled(True)
+            self.config.settings.lcm_diffusion_setting.use_seed = True
+        else:
+            self.seed_value.setEnabled(False)
+            self.config.settings.lcm_diffusion_setting.use_seed = False
+
+    def get_seed_value(self) -> int:
+        use_seed = self.config.settings.lcm_diffusion_setting.use_seed
+        seed_value = int(self.seed_value.text()) if use_seed else -1
+        return seed_value
+
+    def generate_image(self):
+        self.config.settings.lcm_diffusion_setting.seed = self.get_seed_value()
+        self.config.settings.lcm_diffusion_setting.prompt = self.prompt.toPlainText()
+        self.config.settings.lcm_diffusion_setting.negative_prompt = (
+            self.neg_prompt.toPlainText()
+        )
+        self.config.settings.lcm_diffusion_setting.lcm_lora.lcm_lora_id = (
+            self.lcm_lora_id.currentText()
+        )
+        self.config.settings.lcm_diffusion_setting.lcm_lora.base_model_id = (
+            self.base_model_id.currentText()
+        )
+
+        if self.config.settings.lcm_diffusion_setting.use_openvino:
+            model_id = self.openvino_lcm_model_id.currentText()
+        else:
+            model_id = self.lcm_model.currentText()
+
+        self.config.settings.lcm_diffusion_setting.lcm_model_id = model_id
+
+        reshape_required = False
+        if self.config.settings.lcm_diffusion_setting.use_openvino:
+            # Detect dimension change
+            reshape_required = is_reshape_required(
+                self.previous_width,
+                self.config.settings.lcm_diffusion_setting.image_width,
+                self.previous_height,
+                self.config.settings.lcm_diffusion_setting.image_height,
+                self.previous_model,
+                model_id,
+                self.previous_num_of_images,
+                self.config.settings.lcm_diffusion_setting.number_of_images,
+            )
+        self.config.settings.lcm_diffusion_setting.diffusion_task = (
+            DiffusionTask.text_to_image.value
+        )
+        images = self.context.generate_text_to_image(
+            self.config.settings,
+            reshape_required,
+            DEVICE,
+        )
+        self.image_index = 0
+        self.gen_images = []
+        for img in images:
+            im = ImageQt(img).copy()
+            pixmap = QPixmap.fromImage(im)
+            self.gen_images.append(pixmap)
+
+        if len(self.gen_images) > 1:
+            self.next_img_btn.setEnabled(True)
+            self.previous_img_btn.setEnabled(False)
+        else:
+            self.next_img_btn.setEnabled(False)
+            self.previous_img_btn.setEnabled(False)
+
+        self.show_image(self.gen_images[0])
+
+        self.previous_width = self.config.settings.lcm_diffusion_setting.image_width
+        self.previous_height = self.config.settings.lcm_diffusion_setting.image_height
+        self.previous_model = model_id
+        self.previous_num_of_images = (
+            self.config.settings.lcm_diffusion_setting.number_of_images
+        )
+
+    def text_to_image(self):
+        self.img.setText("Please wait...")
+        worker = ImageGeneratorWorker(self.generate_image)
+        self.threadpool.start(worker)
+
+    def closeEvent(self, event):
+        self.config.settings.lcm_diffusion_setting.seed = self.get_seed_value()
+        print(self.config.settings.lcm_diffusion_setting)
+        print("Saving settings")
+        self.config.save()
+
+    def reset_all_settings(self):
+        self.use_local_model_folder.setChecked(False)
+        self.width.setCurrentText("512")
+        self.height.setCurrentText("512")
+        self.inference_steps.setValue(4)
+        self.guidance.setValue(10)
+        self.use_openvino_check.setChecked(False)
+        self.seed_check.setChecked(False)
+        self.safety_checker.setChecked(False)
+        self.results_path.setText(FastStableDiffusionPaths().get_results_path())
+        self.use_tae_sd.setChecked(False)
+        self.use_lcm_lora.setChecked(False)

frontend/gui/image_generator_worker.py

@@ -0,0 +1,37 @@
+from PyQt5.QtCore import (
+    pyqtSlot,
+    QRunnable,
+    pyqtSignal,
+    pyqtSlot,
+)
+from PyQt5.QtCore import QObject
+import traceback
+import sys
+
+
+class WorkerSignals(QObject):
+    finished = pyqtSignal()
+    error = pyqtSignal(tuple)
+    result = pyqtSignal(object)
+
+
+class ImageGeneratorWorker(QRunnable):
+    def __init__(self, fn, *args, **kwargs):
+        super(ImageGeneratorWorker, self).__init__()
+        self.fn = fn
+        self.args = args
+        self.kwargs = kwargs
+        self.signals = WorkerSignals()
+
+    @pyqtSlot()
+    def run(self):
+        try:
+            result = self.fn(*self.args, **self.kwargs)
+        except:
+            traceback.print_exc()
+            exctype, value = sys.exc_info()[:2]
+            self.signals.error.emit((exctype, value, traceback.format_exc()))
+        else:
+            self.signals.result.emit(result)
+        finally:
+            self.signals.finished.emit()

frontend/gui/ui.py

@@ -0,0 +1,15 @@
+from typing import List
+from frontend.gui.app_window import MainWindow
+from PyQt5.QtWidgets import QApplication
+import sys
+from app_settings import AppSettings
+
+
+def start_gui(
+    argv: List[str],
+    app_settings: AppSettings,
+):
+    app = QApplication(sys.argv)
+    window = MainWindow(app_settings)
+    window.show()
+    app.exec()

frontend/utils.py

@@ -0,0 +1,54 @@
+from constants import DEVICE
+from typing import List
+import platform
+from backend.device import is_openvino_device
+
+
+def is_reshape_required(
+    prev_width: int,
+    cur_width: int,
+    prev_height: int,
+    cur_height: int,
+    prev_model: int,
+    cur_model: int,
+    prev_num_of_images: int,
+    cur_num_of_images: int,
+) -> bool:
+    reshape_required = False
+    if (
+        prev_width != cur_width
+        or prev_height != cur_height
+        or prev_model != cur_model
+        or prev_num_of_images != cur_num_of_images
+    ):
+        print("Reshape and compile")
+        reshape_required = True
+
+    return reshape_required
+
+
+def enable_openvino_controls() -> bool:
+    return is_openvino_device() and platform.system().lower() != "darwin"
+
+
+def get_valid_model_id(
+    models: List,
+    model_id: str,
+    default_model: str = "",
+) -> str:
+    if len(models) == 0:
+        print("Error: model configuration file is empty,please add some models.")
+        return ""
+    if model_id == "":
+        if default_model:
+            return default_model
+        else:
+            return models[0]
+
+    if model_id in models:
+        return model_id
+    else:
+        print(
+            f"Error:{model_id} Model not found in configuration file,so using first model : {models[0]}"
+        )
+        return models[0]

frontend/webui/css/style.css

@@ -0,0 +1,22 @@
+footer {
+    visibility: hidden
+}
+
+#generate_button {
+    color: white;
+    border-color: #007bff;
+    background: #2563eb;
+
+}
+
+#save_button {
+    color: white;
+    border-color: #028b40;
+    background: #01b97c;
+    width: 200px;
+}
+
+#settings_header {
+    background: rgb(245, 105, 105);
+
+}

frontend/webui/generation_settings_ui.py

@@ -0,0 +1,140 @@
+import gradio as gr
+from state import get_settings
+
+app_settings = get_settings()
+
+
+def on_change_inference_steps(steps):
+    app_settings.settings.lcm_diffusion_setting.inference_steps = steps
+
+
+def on_change_image_width(img_width):
+    app_settings.settings.lcm_diffusion_setting.image_width = img_width
+
+
+def on_change_image_height(img_height):
+    app_settings.settings.lcm_diffusion_setting.image_height = img_height
+
+
+def on_change_num_images(num_images):
+    app_settings.settings.lcm_diffusion_setting.number_of_images = num_images
+
+
+def on_change_guidance_scale(guidance_scale):
+    app_settings.settings.lcm_diffusion_setting.guidance_scale = guidance_scale
+
+
+def on_change_seed_value(seed):
+    app_settings.settings.lcm_diffusion_setting.seed = seed
+
+
+def on_change_seed_checkbox(seed_checkbox):
+    app_settings.settings.lcm_diffusion_setting.use_seed = seed_checkbox
+
+
+def on_change_safety_checker_checkbox(safety_checker_checkbox):
+    app_settings.settings.lcm_diffusion_setting.use_safety_checker = (
+        safety_checker_checkbox
+    )
+
+
+def on_change_tiny_auto_encoder_checkbox(tiny_auto_encoder_checkbox):
+    app_settings.settings.lcm_diffusion_setting.use_tiny_auto_encoder = (
+        tiny_auto_encoder_checkbox
+    )
+
+
+def on_offline_checkbox(offline_checkbox):
+    app_settings.settings.lcm_diffusion_setting.use_offline_model = offline_checkbox
+
+
+def get_generation_settings_ui() -> None:
+    with gr.Blocks():
+        with gr.Row():
+            with gr.Column():
+                num_inference_steps = gr.Slider(
+                    1,
+                    25,
+                    value=app_settings.settings.lcm_diffusion_setting.inference_steps,
+                    step=1,
+                    label="Inference Steps",
+                    interactive=True,
+                )
+
+                image_height = gr.Slider(
+                    256,
+                    1024,
+                    value=app_settings.settings.lcm_diffusion_setting.image_height,
+                    step=256,
+                    label="Image Height",
+                    interactive=True,
+                )
+                image_width = gr.Slider(
+                    256,
+                    1024,
+                    value=app_settings.settings.lcm_diffusion_setting.image_width,
+                    step=256,
+                    label="Image Width",
+                    interactive=True,
+                )
+                num_images = gr.Slider(
+                    1,
+                    50,
+                    value=app_settings.settings.lcm_diffusion_setting.number_of_images,
+                    step=1,
+                    label="Number of images to generate",
+                    interactive=True,
+                )
+                guidance_scale = gr.Slider(
+                    1.0,
+                    2.0,
+                    value=app_settings.settings.lcm_diffusion_setting.guidance_scale,
+                    step=0.1,
+                    label="Guidance Scale",
+                    interactive=True,
+                )
+
+                seed = gr.Slider(
+                    value=app_settings.settings.lcm_diffusion_setting.seed,
+                    minimum=0,
+                    maximum=999999999,
+                    label="Seed",
+                    step=1,
+                    interactive=True,
+                )
+                seed_checkbox = gr.Checkbox(
+                    label="Use seed",
+                    value=app_settings.settings.lcm_diffusion_setting.use_seed,
+                    interactive=True,
+                )
+
+                safety_checker_checkbox = gr.Checkbox(
+                    label="Use Safety Checker",
+                    value=app_settings.settings.lcm_diffusion_setting.use_safety_checker,
+                    interactive=True,
+                )
+                tiny_auto_encoder_checkbox = gr.Checkbox(
+                    label="Use tiny auto encoder for SD",
+                    value=app_settings.settings.lcm_diffusion_setting.use_tiny_auto_encoder,
+                    interactive=True,
+                )
+                offline_checkbox = gr.Checkbox(
+                    label="Use locally cached model or downloaded model folder(offline)",
+                    value=app_settings.settings.lcm_diffusion_setting.use_offline_model,
+                    interactive=True,
+                )
+
+        num_inference_steps.change(on_change_inference_steps, num_inference_steps)
+        image_height.change(on_change_image_height, image_height)
+        image_width.change(on_change_image_width, image_width)
+        num_images.change(on_change_num_images, num_images)
+        guidance_scale.change(on_change_guidance_scale, guidance_scale)
+        seed.change(on_change_seed_value, seed)
+        seed_checkbox.change(on_change_seed_checkbox, seed_checkbox)
+        safety_checker_checkbox.change(
+            on_change_safety_checker_checkbox, safety_checker_checkbox
+        )
+        tiny_auto_encoder_checkbox.change(
+            on_change_tiny_auto_encoder_checkbox, tiny_auto_encoder_checkbox
+        )
+        offline_checkbox.change(on_change_tiny_auto_encoder_checkbox, offline_checkbox)

frontend/webui/image_to_image_ui.py

@@ -0,0 +1,124 @@
+from typing import Any
+import gradio as gr
+from backend.models.lcmdiffusion_setting import DiffusionTask
+from context import Context
+from models.interface_types import InterfaceType
+from frontend.utils import is_reshape_required
+from constants import DEVICE
+from state import get_settings
+from concurrent.futures import ThreadPoolExecutor
+
+app_settings = get_settings()
+
+context = Context(InterfaceType.WEBUI)
+previous_width = 0
+previous_height = 0
+previous_model_id = ""
+previous_num_of_images = 0
+
+
+def generate_image_to_image(
+    prompt,
+    negative_prompt,
+    init_image,
+    strength,
+) -> Any:
+    global previous_height, previous_width, previous_model_id, previous_num_of_images, app_settings
+
+    app_settings.settings.lcm_diffusion_setting.prompt = prompt
+    app_settings.settings.lcm_diffusion_setting.negative_prompt = negative_prompt
+    app_settings.settings.lcm_diffusion_setting.init_image = init_image
+    app_settings.settings.lcm_diffusion_setting.strength = strength
+
+    app_settings.settings.lcm_diffusion_setting.diffusion_task = (
+        DiffusionTask.image_to_image.value
+    )
+    model_id = app_settings.settings.lcm_diffusion_setting.openvino_lcm_model_id
+    reshape = False
+    image_width = app_settings.settings.lcm_diffusion_setting.image_width
+    image_height = app_settings.settings.lcm_diffusion_setting.image_height
+    num_images = app_settings.settings.lcm_diffusion_setting.number_of_images
+    if app_settings.settings.lcm_diffusion_setting.use_openvino:
+        reshape = is_reshape_required(
+            previous_width,
+            image_width,
+            previous_height,
+            image_height,
+            previous_model_id,
+            model_id,
+            previous_num_of_images,
+            num_images,
+        )
+
+    with ThreadPoolExecutor(max_workers=1) as executor:
+        future = executor.submit(
+            context.generate_text_to_image,
+            app_settings.settings,
+            reshape,
+            DEVICE,
+        )
+        images = future.result()
+    # images = context.generate_text_to_image(
+    #     app_settings.settings,
+    #     reshape,
+    #     DEVICE,
+    # )
+    previous_width = image_width
+    previous_height = image_height
+    previous_model_id = model_id
+    previous_num_of_images = num_images
+    return images
+
+
+def get_image_to_image_ui() -> None:
+    with gr.Blocks():
+        with gr.Row():
+            with gr.Column():
+                input_image = gr.Image(label="Init image", type="pil")
+                with gr.Row():
+                    prompt = gr.Textbox(
+                        show_label=False,
+                        lines=3,
+                        placeholder="A fantasy landscape",
+                        container=False,
+                    )
+
+                    generate_btn = gr.Button(
+                        "Generate",
+                        elem_id="generate_button",
+                        scale=0,
+                    )
+                negative_prompt = gr.Textbox(
+                    label="Negative prompt (Works in LCM-LoRA mode, set guidance > 1.0):",
+                    lines=1,
+                    placeholder="",
+                )
+                strength = gr.Slider(
+                    0.1,
+                    1,
+                    value=app_settings.settings.lcm_diffusion_setting.strength,
+                    step=0.01,
+                    label="Strength",
+                )
+
+                input_params = [
+                    prompt,
+                    negative_prompt,
+                    input_image,
+                    strength,
+                ]
+
+            with gr.Column():
+                output = gr.Gallery(
+                    label="Generated images",
+                    show_label=True,
+                    elem_id="gallery",
+                    columns=2,
+                    height=512,
+                )
+
+    generate_btn.click(
+        fn=generate_image_to_image,
+        inputs=input_params,
+        outputs=output,
+    )

frontend/webui/models_ui.py

@@ -0,0 +1,85 @@
+from app_settings import AppSettings
+from typing import Any
+import gradio as gr
+from constants import LCM_DEFAULT_MODEL, LCM_DEFAULT_MODEL_OPENVINO
+from state import get_settings
+from frontend.utils import get_valid_model_id
+
+app_settings = get_settings()
+app_settings.settings.lcm_diffusion_setting.openvino_lcm_model_id = get_valid_model_id(
+    app_settings.openvino_lcm_models,
+    app_settings.settings.lcm_diffusion_setting.openvino_lcm_model_id,
+)
+
+
+def change_lcm_model_id(model_id):
+    app_settings.settings.lcm_diffusion_setting.lcm_model_id = model_id
+
+
+def change_lcm_lora_model_id(model_id):
+    app_settings.settings.lcm_diffusion_setting.lcm_lora.lcm_lora_id = model_id
+
+
+def change_lcm_lora_base_model_id(model_id):
+    app_settings.settings.lcm_diffusion_setting.lcm_lora.base_model_id = model_id
+
+
+def change_openvino_lcm_model_id(model_id):
+    app_settings.settings.lcm_diffusion_setting.openvino_lcm_model_id = model_id
+
+
+def get_models_ui() -> None:
+    with gr.Blocks():
+        with gr.Row():
+            lcm_model_id = gr.Dropdown(
+                app_settings.lcm_models,
+                label="LCM model",
+                info="Diffusers LCM model ID",
+                value=get_valid_model_id(
+                    app_settings.lcm_models,
+                    app_settings.settings.lcm_diffusion_setting.lcm_model_id,
+                    LCM_DEFAULT_MODEL,
+                ),
+                interactive=True,
+            )
+        with gr.Row():
+            lcm_lora_model_id = gr.Dropdown(
+                app_settings.lcm_lora_models,
+                label="LCM LoRA model",
+                info="Diffusers LCM LoRA model ID",
+                value=get_valid_model_id(
+                    app_settings.lcm_lora_models,
+                    app_settings.settings.lcm_diffusion_setting.lcm_lora.lcm_lora_id,
+                ),
+                interactive=True,
+            )
+            lcm_lora_base_model_id = gr.Dropdown(
+                app_settings.stable_diffsuion_models,
+                label="LCM LoRA base model",
+                info="Diffusers LCM LoRA base model ID",
+                value=get_valid_model_id(
+                    app_settings.stable_diffsuion_models,
+                    app_settings.settings.lcm_diffusion_setting.lcm_lora.base_model_id,
+                ),
+                interactive=True,
+            )
+        with gr.Row():
+            lcm_openvino_model_id = gr.Dropdown(
+                app_settings.openvino_lcm_models,
+                label="LCM OpenVINO model",
+                info="OpenVINO LCM-LoRA fused model ID",
+                value=get_valid_model_id(
+                    app_settings.openvino_lcm_models,
+                    app_settings.settings.lcm_diffusion_setting.openvino_lcm_model_id,
+                ),
+                interactive=True,
+            )
+
+        lcm_model_id.change(change_lcm_model_id, lcm_model_id)
+        lcm_lora_model_id.change(change_lcm_lora_model_id, lcm_lora_model_id)
+        lcm_lora_base_model_id.change(
+            change_lcm_lora_base_model_id, lcm_lora_base_model_id
+        )
+        lcm_openvino_model_id.change(
+            change_openvino_lcm_model_id, lcm_openvino_model_id
+        )

frontend/webui/realtime_ui.py

@@ -0,0 +1,144 @@
+import gradio as gr
+from backend.lcm_text_to_image import LCMTextToImage
+from backend.models.lcmdiffusion_setting import LCMLora, LCMDiffusionSetting
+from constants import DEVICE, LCM_DEFAULT_MODEL_OPENVINO
+from time import perf_counter
+import numpy as np
+from cv2 import imencode
+import base64
+from backend.device import get_device_name
+from constants import APP_VERSION
+from backend.device import is_openvino_device
+
+lcm_text_to_image = LCMTextToImage()
+lcm_lora = LCMLora(
+    base_model_id="Lykon/dreamshaper-8",
+    lcm_lora_id="latent-consistency/lcm-lora-sdv1-5",
+)
+
+
+# https://github.com/gradio-app/gradio/issues/2635#issuecomment-1423531319
+def encode_pil_to_base64_new(pil_image):
+    image_arr = np.asarray(pil_image)[:, :, ::-1]
+    _, byte_data = imencode(".png", image_arr)
+    base64_data = base64.b64encode(byte_data)
+    base64_string_opencv = base64_data.decode("utf-8")
+    return "data:image/png;base64," + base64_string_opencv
+
+
+# monkey patching encode pil
+gr.processing_utils.encode_pil_to_base64 = encode_pil_to_base64_new
+
+
+def predict(
+    prompt,
+    steps,
+    seed,
+):
+    lcm_diffusion_setting = LCMDiffusionSetting()
+    lcm_diffusion_setting.openvino_lcm_model_id = "rupeshs/LCM-dreamshaper-v7-openvino"
+    lcm_diffusion_setting.prompt = prompt
+    lcm_diffusion_setting.guidance_scale = 1.0
+    lcm_diffusion_setting.inference_steps = steps
+    lcm_diffusion_setting.seed = seed
+    lcm_diffusion_setting.use_seed = True
+    lcm_diffusion_setting.image_width = 256 if is_openvino_device() else 512
+    lcm_diffusion_setting.image_height = 256 if is_openvino_device() else 512
+    lcm_diffusion_setting.use_openvino = True if is_openvino_device() else False
+    lcm_text_to_image.init(
+        DEVICE,
+        lcm_diffusion_setting,
+    )
+    start = perf_counter()
+
+    images = lcm_text_to_image.generate(lcm_diffusion_setting)
+    latency = perf_counter() - start
+    print(f"Latency: {latency:.2f} seconds")
+    return images[0]
+
+
+css = """
+#container{
+    margin: 0 auto;
+    max-width: 40rem;
+}
+#intro{
+    max-width: 100%;
+    text-align: center;
+    margin: 0 auto;
+}
+#generate_button {
+    color: white;
+    border-color: #007bff;
+    background: #007bff;
+    width: 200px;
+    height: 50px;
+}
+footer {
+    visibility: hidden
+}
+"""
+
+
+def _get_footer_message() -> str:
+    version = f"<center><p> {APP_VERSION} "
+    footer_msg = version + (
+        '  © 2023 <a href="https://github.com/rupeshs">'
+        " Rupesh Sreeraman</a></p></center>"
+    )
+    return footer_msg
+
+
+with gr.Blocks(css=css) as demo:
+    with gr.Column(elem_id="container"):
+        use_openvino = "- OpenVINO" if is_openvino_device() else ""
+        gr.Markdown(
+            f"""# Realtime FastSD CPU {use_openvino}
+               **Device : {DEVICE} , {get_device_name()}**
+            """,
+            elem_id="intro",
+        )
+
+        with gr.Row():
+            with gr.Row():
+                prompt = gr.Textbox(
+                    placeholder="Describe the image you'd like to see",
+                    scale=5,
+                    container=False,
+                )
+                generate_btn = gr.Button(
+                    "Generate",
+                    scale=1,
+                    elem_id="generate_button",
+                )
+
+        image = gr.Image(type="filepath")
+        with gr.Accordion("Advanced options", open=False):
+            steps = gr.Slider(
+                label="Steps",
+                value=4 if is_openvino_device() else 3,
+                minimum=1,
+                maximum=6,
+                step=1,
+            )
+            seed = gr.Slider(
+                randomize=True,
+                minimum=0,
+                maximum=999999999,
+                label="Seed",
+                step=1,
+            )
+        gr.HTML(_get_footer_message())
+
+        inputs = [prompt, steps, seed]
+        prompt.input(fn=predict, inputs=inputs, outputs=image, show_progress=False)
+        generate_btn.click(
+            fn=predict, inputs=inputs, outputs=image, show_progress=False
+        )
+        steps.change(fn=predict, inputs=inputs, outputs=image, show_progress=False)
+        seed.change(fn=predict, inputs=inputs, outputs=image, show_progress=False)
+
+
+def start_realtime_text_to_image(share=False):
+    demo.queue()
+    demo.launch(share=share)

frontend/webui/text_to_image_ui.py

@@ -0,0 +1,104 @@
+import gradio as gr
+from typing import Any
+from backend.models.lcmdiffusion_setting import DiffusionTask
+from context import Context
+from models.interface_types import InterfaceType
+from constants import DEVICE
+from state import get_settings
+from frontend.utils import is_reshape_required
+from concurrent.futures import ThreadPoolExecutor
+from pprint import pprint
+
+app_settings = get_settings()
+context = Context(InterfaceType.WEBUI)
+previous_width = 0
+previous_height = 0
+previous_model_id = ""
+previous_num_of_images = 0
+
+
+def generate_text_to_image(
+    prompt,
+    neg_prompt,
+) -> Any:
+    global previous_height, previous_width, previous_model_id, previous_num_of_images, app_settings
+    app_settings.settings.lcm_diffusion_setting.prompt = prompt
+    app_settings.settings.lcm_diffusion_setting.negative_prompt = neg_prompt
+    app_settings.settings.lcm_diffusion_setting.diffusion_task = (
+        DiffusionTask.text_to_image.value
+    )
+    model_id = app_settings.settings.lcm_diffusion_setting.openvino_lcm_model_id
+    reshape = False
+    image_width = app_settings.settings.lcm_diffusion_setting.image_width
+    image_height = app_settings.settings.lcm_diffusion_setting.image_height
+    num_images = app_settings.settings.lcm_diffusion_setting.number_of_images
+    if app_settings.settings.lcm_diffusion_setting.use_openvino:
+        reshape = is_reshape_required(
+            previous_width,
+            image_width,
+            previous_height,
+            image_height,
+            previous_model_id,
+            model_id,
+            previous_num_of_images,
+            num_images,
+        )
+
+    with ThreadPoolExecutor(max_workers=1) as executor:
+        future = executor.submit(
+            context.generate_text_to_image,
+            app_settings.settings,
+            reshape,
+            DEVICE,
+        )
+        images = future.result()
+    # images = context.generate_text_to_image(
+    #     app_settings.settings,
+    #     reshape,
+    #     DEVICE,
+    # )
+    previous_width = image_width
+    previous_height = image_height
+    previous_model_id = model_id
+    previous_num_of_images = num_images
+    return images
+
+
+def get_text_to_image_ui() -> None:
+    with gr.Blocks():
+        with gr.Row():
+            with gr.Column():
+                with gr.Row():
+                    prompt = gr.Textbox(
+                        show_label=False,
+                        lines=3,
+                        placeholder="A fantasy landscape",
+                        container=False,
+                    )
+
+                    generate_btn = gr.Button(
+                        "Generate",
+                        elem_id="generate_button",
+                        scale=0,
+                    )
+                negative_prompt = gr.Textbox(
+                    label="Negative prompt (Works in LCM-LoRA mode, set guidance > 1.0) :",
+                    lines=1,
+                    placeholder="",
+                )
+
+                input_params = [prompt, negative_prompt]
+
+            with gr.Column():
+                output = gr.Gallery(
+                    label="Generated images",
+                    show_label=True,
+                    elem_id="gallery",
+                    columns=2,
+                    height=512,
+                )
+    generate_btn.click(
+        fn=generate_text_to_image,
+        inputs=input_params,
+        outputs=output,
+    )

frontend/webui/ui.py

@@ -0,0 +1,70 @@
+import gradio as gr
+from constants import APP_VERSION
+from frontend.webui.text_to_image_ui import get_text_to_image_ui
+from frontend.webui.image_to_image_ui import get_image_to_image_ui
+from frontend.webui.generation_settings_ui import get_generation_settings_ui
+from frontend.webui.models_ui import get_models_ui
+from paths import FastStableDiffusionPaths
+from state import get_settings
+
+app_settings = get_settings()
+
+
+def _get_footer_message() -> str:
+    version = f"<center><p> {APP_VERSION} "
+    footer_msg = version + (
+        '  © 2023 <a href="https://github.com/rupeshs">'
+        " Rupesh Sreeraman</a></p></center>"
+    )
+    return footer_msg
+
+
+def get_web_ui() -> gr.Blocks:
+    def change_mode(mode):
+        global app_settings
+        app_settings.settings.lcm_diffusion_setting.use_lcm_lora = False
+        app_settings.settings.lcm_diffusion_setting.use_openvino = False
+        if mode == "LCM-LoRA":
+            app_settings.settings.lcm_diffusion_setting.use_lcm_lora = True
+        elif mode == "LCM-OpenVINO":
+            app_settings.settings.lcm_diffusion_setting.use_openvino = True
+
+    with gr.Blocks(
+        css=FastStableDiffusionPaths.get_css_path(),
+        title="FastSD CPU",
+    ) as fastsd_web_ui:
+        gr.HTML("<center><H1>FastSD CPU</H1></center>")
+        current_mode = "LCM"
+        if app_settings.settings.lcm_diffusion_setting.use_openvino:
+            current_mode = "LCM-OpenVINO"
+        elif app_settings.settings.lcm_diffusion_setting.use_lcm_lora:
+            current_mode = "LCM-LoRA"
+
+        mode = gr.Radio(
+            ["LCM", "LCM-LoRA", "LCM-OpenVINO"],
+            label="Mode",
+            info="Current working mode",
+            value=current_mode,
+        )
+        mode.change(change_mode, inputs=mode)
+
+        with gr.Tabs():
+            with gr.TabItem("Text to Image"):
+                get_text_to_image_ui()
+            with gr.TabItem("Image to Image"):
+                get_image_to_image_ui()
+            with gr.TabItem("Generation Settings"):
+                get_generation_settings_ui()
+            with gr.TabItem("Models"):
+                get_models_ui()
+
+        gr.HTML(_get_footer_message())
+
+    return fastsd_web_ui
+
+
+def start_webui(
+    share: bool = False,
+):
+    webui = get_web_ui()
+    webui.launch(share=share)

image_ops.py

@@ -0,0 +1,15 @@
+from PIL import Image
+
+
+def resize_pil_image(
+    pil_image: Image,
+    image_width,
+    image_height,
+):
+    return pil_image.convert("RGB").resize(
+        (
+            image_width,
+            image_height,
+        ),
+        Image.Resampling.LANCZOS,
+    )