Improved the prompt handeling of ideogram4 model. Now used advanced prompts class to store them smaller and allow longer prompts
This commit is contained in:
@@ -9,7 +9,7 @@ from toolkit.config_modules import GenerateImageConfig, ModelConfig
|
|||||||
from toolkit.models.base_model import BaseModel
|
from toolkit.models.base_model import BaseModel
|
||||||
from toolkit.basic import flush
|
from toolkit.basic import flush
|
||||||
from toolkit.print import print_acc
|
from toolkit.print import print_acc
|
||||||
from toolkit.prompt_utils import PromptEmbeds
|
from toolkit.advanced_prompt_embeds import AdvancedPromptEmbeds
|
||||||
from toolkit.samplers.custom_flowmatch_sampler import (
|
from toolkit.samplers.custom_flowmatch_sampler import (
|
||||||
CustomFlowMatchEulerDiscreteScheduler,
|
CustomFlowMatchEulerDiscreteScheduler,
|
||||||
)
|
)
|
||||||
@@ -29,6 +29,7 @@ from .src.latent_norm import get_latent_norm
|
|||||||
from .src.pipeline import (
|
from .src.pipeline import (
|
||||||
Ideogram4Pipeline,
|
Ideogram4Pipeline,
|
||||||
get_qwen3_vl_features,
|
get_qwen3_vl_features,
|
||||||
|
pad_text_features,
|
||||||
patchify_latents,
|
patchify_latents,
|
||||||
predict_velocity,
|
predict_velocity,
|
||||||
unpatchify_latents,
|
unpatchify_latents,
|
||||||
@@ -176,15 +177,21 @@ class Ideogram4Model(BaseModel):
|
|||||||
|
|
||||||
self.patch_size = 2
|
self.patch_size = 2
|
||||||
self.vae_scale_factor = 8
|
self.vae_scale_factor = 8
|
||||||
# Length the Qwen3-VL caption is padded/truncated to. Fixed so cached and
|
# Safety cap on caption token length (truncation only). Captions are stored
|
||||||
# batched embeddings always share a sequence length.
|
# per-sample at their natural length and padded to the batch max at the
|
||||||
|
# model call, so this is just an upper bound for very long JSON prompts.
|
||||||
self.max_text_length = int(
|
self.max_text_length = int(
|
||||||
self.model_config.model_kwargs.get("max_text_length", 512)
|
self.model_config.model_kwargs.get("max_text_length", 3072)
|
||||||
)
|
)
|
||||||
|
|
||||||
self._latent_shift = None
|
self._latent_shift = None
|
||||||
self._latent_scale = None
|
self._latent_scale = None
|
||||||
|
|
||||||
|
@property
|
||||||
|
def text_embedding_space_version(self):
|
||||||
|
# we changed the embeddings. invalidate cache.
|
||||||
|
return self.arch + "_te_v2"
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def get_train_scheduler():
|
def get_train_scheduler():
|
||||||
return CustomFlowMatchEulerDiscreteScheduler(**scheduler_config)
|
return CustomFlowMatchEulerDiscreteScheduler(**scheduler_config)
|
||||||
@@ -340,8 +347,8 @@ class Ideogram4Model(BaseModel):
|
|||||||
self,
|
self,
|
||||||
pipeline: Ideogram4Pipeline,
|
pipeline: Ideogram4Pipeline,
|
||||||
gen_config: GenerateImageConfig,
|
gen_config: GenerateImageConfig,
|
||||||
conditional_embeds: PromptEmbeds,
|
conditional_embeds: AdvancedPromptEmbeds,
|
||||||
unconditional_embeds: PromptEmbeds,
|
unconditional_embeds: AdvancedPromptEmbeds,
|
||||||
generator: torch.Generator,
|
generator: torch.Generator,
|
||||||
extra: dict,
|
extra: dict,
|
||||||
):
|
):
|
||||||
@@ -371,7 +378,7 @@ class Ideogram4Model(BaseModel):
|
|||||||
self,
|
self,
|
||||||
latent_model_input: torch.Tensor, # (B, 128, gh, gw)
|
latent_model_input: torch.Tensor, # (B, 128, gh, gw)
|
||||||
timestep: torch.Tensor, # 0 to 1000 scale
|
timestep: torch.Tensor, # 0 to 1000 scale
|
||||||
text_embeddings: PromptEmbeds,
|
text_embeddings: AdvancedPromptEmbeds,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
):
|
):
|
||||||
if self.model.device == torch.device("cpu"):
|
if self.model.device == torch.device("cpu"):
|
||||||
@@ -383,8 +390,10 @@ class Ideogram4Model(BaseModel):
|
|||||||
if t01.shape[0] != latent_model_input.shape[0]:
|
if t01.shape[0] != latent_model_input.shape[0]:
|
||||||
t01 = t01.expand(latent_model_input.shape[0])
|
t01 = t01.expand(latent_model_input.shape[0])
|
||||||
|
|
||||||
llm_features = text_embeddings.text_embeds.to(self.device_torch)
|
# Pad the per-sample caption features to the batch max here.
|
||||||
text_mask = text_embeddings.attention_mask.to(self.device_torch)
|
llm_features, text_mask = pad_text_features(
|
||||||
|
text_embeddings.text_embeds, self.device_torch, self.torch_dtype
|
||||||
|
)
|
||||||
|
|
||||||
pred = predict_velocity(
|
pred = predict_velocity(
|
||||||
self.transformer,
|
self.transformer,
|
||||||
@@ -395,7 +404,7 @@ class Ideogram4Model(BaseModel):
|
|||||||
)
|
)
|
||||||
return pred
|
return pred
|
||||||
|
|
||||||
def get_prompt_embeds(self, prompt) -> PromptEmbeds:
|
def get_prompt_embeds(self, prompt) -> AdvancedPromptEmbeds:
|
||||||
if isinstance(prompt, str):
|
if isinstance(prompt, str):
|
||||||
prompt = [prompt]
|
prompt = [prompt]
|
||||||
|
|
||||||
@@ -403,11 +412,11 @@ class Ideogram4Model(BaseModel):
|
|||||||
self.text_encoder.to(self.device_torch)
|
self.text_encoder.to(self.device_torch)
|
||||||
device = self.text_encoder.device
|
device = self.text_encoder.device
|
||||||
|
|
||||||
pad_id = self.tokenizer.pad_token_id
|
# Encode each caption at its natural length (no cross-sample padding) and
|
||||||
if pad_id is None:
|
# store one feature tensor per batch item. Padding to a common length is
|
||||||
pad_id = self.tokenizer.eos_token_id or 0
|
# deferred to the model call, so caching a prompt only stores its real
|
||||||
|
# length -- important for the long structured (JSON) captions.
|
||||||
token_id_list = []
|
features_list = []
|
||||||
for p in prompt:
|
for p in prompt:
|
||||||
messages = [{"role": "user", "content": [{"type": "text", "text": p}]}]
|
messages = [{"role": "user", "content": [{"type": "text", "text": p}]}]
|
||||||
text = self.tokenizer.apply_chat_template(
|
text = self.tokenizer.apply_chat_template(
|
||||||
@@ -419,29 +428,19 @@ class Ideogram4Model(BaseModel):
|
|||||||
truncation=True,
|
truncation=True,
|
||||||
max_length=self.max_text_length,
|
max_length=self.max_text_length,
|
||||||
)["input_ids"]
|
)["input_ids"]
|
||||||
token_id_list.append(ids)
|
if len(ids) == 0:
|
||||||
|
ids = [self.tokenizer.eos_token_id or 0]
|
||||||
|
|
||||||
seq_len = self.max_text_length
|
token_ids = torch.tensor([ids], dtype=torch.long, device=device)
|
||||||
batch_size = len(token_id_list)
|
attention_mask = torch.ones_like(token_ids)
|
||||||
token_ids = torch.full((batch_size, seq_len), pad_id, dtype=torch.long)
|
pos_2d = (attention_mask.cumsum(dim=-1) - 1).clamp(min=0).to(torch.long)
|
||||||
attention_mask = torch.zeros((batch_size, seq_len), dtype=torch.long)
|
|
||||||
for b, ids in enumerate(token_id_list):
|
|
||||||
n = min(len(ids), seq_len)
|
|
||||||
# left pad
|
|
||||||
token_ids[b, seq_len - n :] = torch.tensor(ids[:n], dtype=torch.long)
|
|
||||||
attention_mask[b, seq_len - n :] = 1
|
|
||||||
|
|
||||||
token_ids = token_ids.to(device)
|
features = get_qwen3_vl_features(
|
||||||
attention_mask = attention_mask.to(device)
|
self.text_encoder, token_ids, attention_mask, pos_2d
|
||||||
pos_2d = (attention_mask.cumsum(dim=-1) - 1).clamp(min=0).to(torch.long)
|
) # (1, Lt, D)
|
||||||
|
features_list.append(features[0].to(self.torch_dtype))
|
||||||
|
|
||||||
features = get_qwen3_vl_features(
|
return AdvancedPromptEmbeds(text_embeds=features_list)
|
||||||
self.text_encoder, token_ids, attention_mask, pos_2d
|
|
||||||
)
|
|
||||||
|
|
||||||
pe = PromptEmbeds(features.to(self.torch_dtype))
|
|
||||||
pe.attention_mask = attention_mask
|
|
||||||
return pe
|
|
||||||
|
|
||||||
def get_model_has_grad(self):
|
def get_model_has_grad(self):
|
||||||
return False
|
return False
|
||||||
|
|||||||
@@ -123,6 +123,32 @@ def get_qwen3_vl_features(
|
|||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def pad_text_features(
|
||||||
|
features_list: List[torch.Tensor],
|
||||||
|
device: torch.device,
|
||||||
|
dtype: torch.dtype,
|
||||||
|
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||||
|
"""Right-pad a list of per-sample (Lt_i, D) features into a batch.
|
||||||
|
|
||||||
|
Captions are stored at their natural length (one tensor per batch item) and
|
||||||
|
only padded to the batch max here, right before the model call. Returns
|
||||||
|
``(features (B, Lt, D), attention_mask (B, Lt))``; the mask is 1 for real
|
||||||
|
tokens and 0 for padding (which the transformer masks out anyway).
|
||||||
|
"""
|
||||||
|
lengths = [f.shape[0] for f in features_list]
|
||||||
|
max_len = max(lengths)
|
||||||
|
dim = features_list[0].shape[-1]
|
||||||
|
batch_size = len(features_list)
|
||||||
|
|
||||||
|
features = torch.zeros(batch_size, max_len, dim, device=device, dtype=dtype)
|
||||||
|
mask = torch.zeros(batch_size, max_len, dtype=torch.long, device=device)
|
||||||
|
for i, f in enumerate(features_list):
|
||||||
|
n = f.shape[0]
|
||||||
|
features[i, :n] = f.to(device, dtype)
|
||||||
|
mask[i, :n] = 1
|
||||||
|
return features, mask
|
||||||
|
|
||||||
|
|
||||||
def predict_velocity(
|
def predict_velocity(
|
||||||
transformer: Ideogram4Transformer2DModel,
|
transformer: Ideogram4Transformer2DModel,
|
||||||
latents: torch.Tensor, # (B, 128, gh, gw)
|
latents: torch.Tensor, # (B, 128, gh, gw)
|
||||||
@@ -281,11 +307,13 @@ class Ideogram4Pipeline:
|
|||||||
latents = latents.to(device, dtype=torch.float32)
|
latents = latents.to(device, dtype=torch.float32)
|
||||||
latents = latents * scheduler.init_noise_sigma
|
latents = latents * scheduler.init_noise_sigma
|
||||||
|
|
||||||
cond_feats = conditional_embeds.text_embeds.to(device, dtype=dtype)
|
cond_feats, cond_mask = pad_text_features(
|
||||||
cond_mask = conditional_embeds.attention_mask.to(device)
|
conditional_embeds.text_embeds, device, dtype
|
||||||
|
)
|
||||||
if do_cfg:
|
if do_cfg:
|
||||||
uncond_feats = unconditional_embeds.text_embeds.to(device, dtype=dtype)
|
uncond_feats, uncond_mask = pad_text_features(
|
||||||
uncond_mask = unconditional_embeds.attention_mask.to(device)
|
unconditional_embeds.text_embeds, device, dtype
|
||||||
|
)
|
||||||
|
|
||||||
for t in timesteps:
|
for t in timesteps:
|
||||||
t01 = (t / 1000.0).to(device).expand(latents.shape[0])
|
t01 = (t / 1000.0).to(device).expand(latents.shape[0])
|
||||||
|
|||||||
@@ -533,7 +533,7 @@ class AiToolkitDataset(LatentCachingMixin, ControlCachingMixin, CLIPCachingMixin
|
|||||||
size_database=self.size_database,
|
size_database=self.size_database,
|
||||||
dataset_root=dataset_folder,
|
dataset_root=dataset_folder,
|
||||||
encode_control_in_text_embeddings=self.sd.encode_control_in_text_embeddings if self.sd else False,
|
encode_control_in_text_embeddings=self.sd.encode_control_in_text_embeddings if self.sd else False,
|
||||||
text_embedding_space_version=self.sd.model_config.arch if self.sd else "sd1",
|
text_embedding_space_version=self.sd.text_embedding_space_version if self.sd else "sd1",
|
||||||
te_padding_side=self.sd.te_padding_side if self.sd else "right",
|
te_padding_side=self.sd.te_padding_side if self.sd else "right",
|
||||||
latent_space_version=latent_space_version,
|
latent_space_version=latent_space_version,
|
||||||
temporal_compression=temporal_compression,
|
temporal_compression=temporal_compression,
|
||||||
|
|||||||
@@ -255,6 +255,10 @@ class BaseModel:
|
|||||||
@property
|
@property
|
||||||
def is_lumina2(self):
|
def is_lumina2(self):
|
||||||
return self.arch == 'lumina2'
|
return self.arch == 'lumina2'
|
||||||
|
|
||||||
|
@property
|
||||||
|
def text_embedding_space_version(self):
|
||||||
|
return self.arch
|
||||||
|
|
||||||
def get_bucket_divisibility(self):
|
def get_bucket_divisibility(self):
|
||||||
if self.vae is None:
|
if self.vae is None:
|
||||||
|
|||||||
@@ -269,6 +269,10 @@ class StableDiffusion:
|
|||||||
def is_lumina2(self):
|
def is_lumina2(self):
|
||||||
return self.arch == 'lumina2'
|
return self.arch == 'lumina2'
|
||||||
|
|
||||||
|
@property
|
||||||
|
def text_embedding_space_version(self):
|
||||||
|
return self.arch
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def unet_unwrapped(self):
|
def unet_unwrapped(self):
|
||||||
return unwrap_model(self.unet)
|
return unwrap_model(self.unet)
|
||||||
|
|||||||
Reference in New Issue
Block a user