import os from typing import List, Optional import torch import yaml from toolkit.config_modules import GenerateImageConfig, ModelConfig from toolkit.models.base_model import BaseModel from toolkit.models.v2.text_encoders.mistral3 import Mistral3ModelEncoder from toolkit.models.v2.vae.autoencoder_kl_flux2 import Flux2KLVAE from toolkit.basic import flush from toolkit.advanced_prompt_embeds import AdvancedPromptEmbeds from toolkit.samplers.custom_flowmatch_sampler import ( CustomFlowMatchEulerDiscreteScheduler, ) from toolkit.accelerator import unwrap_model from transformers import AutoTokenizer, AutoModel try: from diffusers import ErnieImagePipeline, AutoencoderKLFlux2 from .transformer import ErnieImageTransformer2DModel except ImportError: raise ImportError( "Diffusers is out of date. Update diffusers to the latest version by doing pip uninstall diffusers and then pip install -r requirements.txt" ) scheduler_config = { "base_image_seq_len": 256, "base_shift": 0.5, "invert_sigmas": False, "max_image_seq_len": 4096, "max_shift": 1.15, "num_train_timesteps": 1000, "shift": 3.0, "shift_terminal": None, "stochastic_sampling": False, "time_shift_type": "exponential", "use_beta_sigmas": False, "use_dynamic_shifting": False, "use_exponential_sigmas": False, "use_karras_sigmas": False, } class ErnieImageModel(BaseModel): arch = "ernie_image" def __init__( self, device, model_config: ModelConfig, dtype="bf16", custom_pipeline=None, noise_scheduler=None, **kwargs, ): super().__init__( device, model_config, dtype, custom_pipeline, noise_scheduler, **kwargs ) self.is_flow_matching = True self.is_transformer = True self.target_lora_modules = ["ErnieImageTransformer2DModel"] # static method to get the noise scheduler @staticmethod def get_train_scheduler(): return CustomFlowMatchEulerDiscreteScheduler(**scheduler_config) def get_bucket_divisibility(self): return 16 * 2 # 16 for the VAE, 2 for patch size def load_model(self): dtype = self.torch_dtype self.print_and_status_update("Loading ErnieImage model") model_path = self.model_config.name_or_path base_model_path = self.model_config.extras_name_or_path self.print_and_status_update("Loading transformer") if os.path.exists(model_path): # check if the path is a full checkpoint. te_folder_path = os.path.join(model_path, "text_encoder") # if we have the te, this folder is a full checkpoint, use it as the base if os.path.exists(te_folder_path): base_model_path = model_path # load + quantize + offload + placement, all driven by model_config transformer = ErnieImageTransformer2DModel.load( model_path, **self.component_load_kwargs("transformer") ) flush() self.print_and_status_update("Text Encoder") tokenizer = AutoTokenizer.from_pretrained( base_model_path, subfolder="tokenizer", torch_dtype=dtype ) text_encoder = Mistral3ModelEncoder.load( base_model_path, subfolder="text_encoder", **self.component_load_kwargs("te") ) flush() self.print_and_status_update("Loading VAE") vae = Flux2KLVAE.load_model( base_model_path, dtype=dtype ).to(self.device_torch, dtype=dtype) self.noise_scheduler = ErnieImageModel.get_train_scheduler() self.print_and_status_update("Making pipe") kwargs = {} pipe: ErnieImagePipeline = ErnieImagePipeline( scheduler=self.noise_scheduler, text_encoder=None, tokenizer=tokenizer, vae=vae, transformer=None, **kwargs, ) # for quantization, it works best to do these after making the pipe pipe.text_encoder = text_encoder pipe.transformer = transformer self.print_and_status_update("Preparing Model") text_encoder = [pipe.text_encoder] tokenizer = [pipe.tokenizer] # leave it on cpu for now if not self.low_vram: pipe.transformer = pipe.transformer.to(self.device_torch) flush() # low_vram: the text encoder stays on cpu; get_prompt_embeds moves it # to the gpu on demand if not self.low_vram: text_encoder[0].to(self.device_torch) text_encoder[0].requires_grad_(False) text_encoder[0].eval() flush() # save it to the model class self.vae = vae self.text_encoder = text_encoder # list of text encoders self.tokenizer = tokenizer # list of tokenizers self.model = pipe.transformer self.pipeline = pipe self.print_and_status_update("Model Loaded") def get_generation_pipeline(self): scheduler = ErnieImageModel.get_train_scheduler() pipeline: ErnieImagePipeline = ErnieImagePipeline( scheduler=scheduler, text_encoder=unwrap_model(self.text_encoder[0]), tokenizer=self.tokenizer[0], vae=unwrap_model(self.vae), transformer=unwrap_model(self.transformer), ) pipeline = pipeline.to(self.device_torch) return pipeline def encode_images(self, image_list: List[torch.Tensor], device=None, dtype=None): if self.vae.device == torch.device("cpu"): self.vae.to(self.device_torch) if device is None: device = self.vae_device_torch if dtype is None: dtype = self.vae_torch_dtype self.vae.eval() self.vae.requires_grad_(False) image = image_list if isinstance(image, list): image = torch.stack(image, dim=0) image = image.to(device, dtype=dtype) latents = self.vae.encode(image).latent_dist.sample() latents = self.pipeline._patchify_latents(latents) bn_mean = self.vae.bn.running_mean.view(1, -1, 1, 1).to( device=latents.device, dtype=latents.dtype ) bn_std = torch.sqrt(self.vae.bn.running_var.view(1, -1, 1, 1) + 1e-5).to( device=latents.device, dtype=latents.dtype ) latents = (latents - bn_mean) / bn_std return latents def decode_latents(self, latents: torch.Tensor, device=None, dtype=None): if self.vae.device == torch.device("cpu"): self.vae.to(self.device_torch) if device is None: device = self.vae_device_torch if dtype is None: dtype = self.vae_torch_dtype latents = latents.to(device, dtype=dtype) bn_mean = self.vae.bn.running_mean.view(1, -1, 1, 1).to(device) bn_std = torch.sqrt(self.vae.bn.running_var.view(1, -1, 1, 1) + 1e-5).to(device) latents = latents * bn_std + bn_mean # Unpatchify latents = self.pipeline._unpatchify_latents(latents) # Decode images = self.vae.decode(latents, return_dict=False)[0] return images def generate_single_image( self, pipeline: ErnieImagePipeline, gen_config: GenerateImageConfig, conditional_embeds: AdvancedPromptEmbeds, unconditional_embeds: AdvancedPromptEmbeds, generator: torch.Generator, extra: dict, ): if self.model.device == torch.device("cpu"): self.model.to(self.device_torch) sc = self.get_bucket_divisibility() gen_config.width = int(gen_config.width // sc * sc) gen_config.height = int(gen_config.height // sc * sc) img = pipeline( prompt_embeds=conditional_embeds.text_embeds, negative_prompt_embeds=unconditional_embeds.text_embeds, height=gen_config.height, width=gen_config.width, num_inference_steps=gen_config.num_inference_steps, guidance_scale=gen_config.guidance_scale, latents=gen_config.latents, generator=generator, **extra, ).images[0] return img def get_noise_prediction( self, latent_model_input: torch.Tensor, timestep: torch.Tensor, # 0 to 1000 scale text_embeddings: AdvancedPromptEmbeds, **kwargs, ): if self.model.device == torch.device("cpu"): self.model.to(self.device_torch) text_bth, text_lens = self.pipeline._pad_text( text_hiddens=text_embeddings.text_embeds, device=self.device_torch, dtype=self.vae.dtype, text_in_dim=self.pipeline.transformer.config.text_in_dim, ) pred = self.transformer( hidden_states=latent_model_input, timestep=timestep, text_bth=text_bth, text_lens=text_lens, return_dict=False, )[0] return pred def get_prompt_embeds(self, prompt: str) -> AdvancedPromptEmbeds: if self.pipeline.text_encoder.device == torch.device("cpu"): self.pipeline.text_encoder.to(self.device_torch) if isinstance(prompt, str): prompt = [prompt] text_hiddens = [] for p in prompt: ids = self.pipeline.tokenizer( p, add_special_tokens=True, truncation=True, padding=False, )["input_ids"] if len(ids) == 0: if self.pipeline.tokenizer.bos_token_id is not None: ids = [self.pipeline.tokenizer.bos_token_id] else: ids = [0] input_ids = torch.tensor([ids], device=self.device_torch) outputs = self.pipeline.text_encoder( input_ids=input_ids, output_hidden_states=True, ) # Use second to last hidden state (matches training) hidden = outputs.hidden_states[-2][0] # [T, H] text_hiddens.append(hidden) pe = AdvancedPromptEmbeds(text_embeds=text_hiddens) return pe def get_model_has_grad(self): return False def get_te_has_grad(self): return False def save_model(self, output_path, meta, save_dtype): transformer: ErnieImageTransformer2DModel = unwrap_model(self.model) transformer.save_pretrained( save_directory=os.path.join(output_path, "transformer"), safe_serialization=True, ) meta_path = os.path.join(output_path, "aitk_meta.yaml") with open(meta_path, "w") as f: yaml.dump(meta, f) def get_loss_target(self, *args, **kwargs): noise = kwargs.get("noise") batch = kwargs.get("batch") return (noise - batch.latents).detach() def get_base_model_version(self): return self.arch def get_transformer_block_names(self) -> Optional[List[str]]: return ["layers"] lora_keys_use_comfy_prefix = True