WIP more work on cogview4

This commit is contained in:
Jaret Burkett committed 2025-03-05 09:43:00 -07:00
1 parent 6f6fb90812
commit aa44828c0c
3 files changed
+19 -32

No files matched your search

+1 -1
View File
@@ -1,7 +1,7 @@
torch==2.5.1 torch==2.5.1
torchvision==0.20.1 torchvision==0.20.1
safetensors safetensors
git+https://github.com/huggingface/diffusers@97fda1b75c70705b245a462044fedb47abb17e56 git+https://github.com/huggingface/diffusers@24c062aaa19f5626d03d058daf8afffa2dfd49f7
transformers==4.49.0 transformers==4.49.0
lycoris-lora==1.8.3 lycoris-lora==1.8.3
flatten_json flatten_json
+17 -30
View File
@@ -20,7 +20,6 @@ from transformers import GlmModel, AutoTokenizer
from diffusers import FlowMatchEulerDiscreteScheduler from diffusers import FlowMatchEulerDiscreteScheduler
from typing import TYPE_CHECKING from typing import TYPE_CHECKING
from toolkit.accelerator import unwrap_model from toolkit.accelerator import unwrap_model
from toolkit.samplers.custom_flowmatch_sampler import CustomFlowMatchEulerDiscreteScheduler from toolkit.samplers.custom_flowmatch_sampler import CustomFlowMatchEulerDiscreteScheduler
if TYPE_CHECKING: if TYPE_CHECKING:
@@ -86,7 +85,6 @@ class CogView4(BaseModel):
base_model_path = "THUDM/CogView4-6B" base_model_path = "THUDM/CogView4-6B"
model_path = self.model_config.name_or_path model_path = self.model_config.name_or_path
# pipe = CogView4Pipeline.from_pretrained("THUDM/CogView4-6B", torch_dtype=torch.bfloat16)
self.print_and_status_update("Loading CogView4 model") self.print_and_status_update("Loading CogView4 model")
# base_model_path = "black-forest-labs/FLUX.1-schnell" # base_model_path = "black-forest-labs/FLUX.1-schnell"
base_model_path = self.model_config.name_or_path_original base_model_path = self.model_config.name_or_path_original
@@ -213,19 +211,6 @@ class CogView4(BaseModel):
generator: torch.Generator, generator: torch.Generator,
extra: dict, extra: dict,
): ):
# there is a bug in the check in diffusers code that requires the prompt embeds to be the same length for conditional and unconditional
# they are processed in 2 passes and the encoding code doesnt do this. So it shouldnt be needed. But, we will zero pad the shorter one. for now. Just inference here, so it should be fine.
if conditional_embeds.text_embeds.shape[1] < unconditional_embeds.text_embeds.shape[1]:
pad_len = unconditional_embeds.text_embeds.shape[1] - \
conditional_embeds.text_embeds.shape[1]
conditional_embeds.text_embeds = torch.cat([conditional_embeds.text_embeds, torch.zeros(conditional_embeds.text_embeds.shape[0], pad_len,
conditional_embeds.text_embeds.shape[2], device=conditional_embeds.text_embeds.device, dtype=conditional_embeds.text_embeds.dtype)], dim=1)
elif conditional_embeds.text_embeds.shape[1] > unconditional_embeds.text_embeds.shape[1]:
pad_len = conditional_embeds.text_embeds.shape[1] - \
unconditional_embeds.text_embeds.shape[1]
unconditional_embeds.text_embeds = torch.cat([unconditional_embeds.text_embeds, torch.zeros(unconditional_embeds.text_embeds.shape[0], pad_len,
unconditional_embeds.text_embeds.shape[2], device=unconditional_embeds.text_embeds.device, dtype=unconditional_embeds.text_embeds.dtype)], dim=1)
img = pipeline( img = pipeline(
prompt_embeds=conditional_embeds.text_embeds.to( prompt_embeds=conditional_embeds.text_embeds.to(
self.device_torch, dtype=self.torch_dtype), self.device_torch, dtype=self.torch_dtype),
@@ -259,12 +244,12 @@ class CogView4(BaseModel):
[target_size], dtype=self.torch_dtype, device=self.device_torch) [target_size], dtype=self.torch_dtype, device=self.device_torch)
target_size = original_size.clone() target_size = original_size.clone()
noise_pred_cond = self.model( noise_pred_cond = self.model(
hidden_states=latent_model_input, # torch.Size([1, 16, 128, 128]) hidden_states=latent_model_input,
encoder_hidden_states=text_embeddings.text_embeds, # torch.Size([1, 16, 4096]) encoder_hidden_states=text_embeddings.text_embeds,
timestep=timestep, timestep=timestep,
original_size=original_size, # [[1024., 1024.]] original_size=original_size,
target_size=target_size, # [[1024., 1024.]] target_size=target_size,
crop_coords=crops_coords_top_left, # [[0., 0.]] crop_coords=crops_coords_top_left,
return_dict=False, return_dict=False,
)[0] )[0]
return noise_pred_cond return noise_pred_cond
@@ -285,7 +270,7 @@ class CogView4(BaseModel):
return self.text_encoder.layers[0].mlp.down_proj.weight.requires_grad return self.text_encoder.layers[0].mlp.down_proj.weight.requires_grad
def save_model(self, output_path, meta, save_dtype): def save_model(self, output_path, meta, save_dtype):
# only save the unet # only save the unet
transformer: CogView4Transformer2DModel = unwrap_model(self.model) transformer: CogView4Transformer2DModel = unwrap_model(self.model)
transformer.save_pretrained( transformer.save_pretrained(
save_directory=os.path.join(output_path, 'transformer'), save_directory=os.path.join(output_path, 'transformer'),
@@ -305,21 +290,23 @@ class CogView4(BaseModel):
if noise is None: if noise is None:
raise ValueError("Noise is not provided") raise ValueError("Noise is not provided")
# return batch.latents # return batch.latents
# return (batch.latents - noise).detach()
return (noise - batch.latents).detach() return (noise - batch.latents).detach()
# return (batch.latents).detach()
# return (effective_noise - batch.latents).detach() # return (effective_noise - batch.latents).detach()
def _get_low_res_latents(self, latents): def _get_low_res_latents(self, latents):
# todo prevent needing to do this and grab the tensor another way. # todo prevent needing to do this and grab the tensor another way.
with torch.no_grad(): with torch.no_grad():
# Decode latents to image space # Decode latents to image space
images = self.decode_latents(latents, device=latents.device, dtype=latents.dtype) images = self.decode_latents(
latents, device=latents.device, dtype=latents.dtype)
# Downsample by a factor of 2 using bilinear interpolation # Downsample by a factor of 2 using bilinear interpolation
B, C, H, W = images.shape B, C, H, W = images.shape
low_res_images = torch.nn.functional.interpolate( low_res_images = torch.nn.functional.interpolate(
images, images,
size=(H // 4, W // 4), size=(H // 2, W // 2),
mode="bilinear", mode="bilinear",
align_corners=False align_corners=False
) )
@@ -333,10 +320,10 @@ class CogView4(BaseModel):
) )
# Encode the low-resolution images back to latent space # Encode the low-resolution images back to latent space
low_res_latents = self.encode_images(upsampled_low_res_images, device=latents.device, dtype=latents.dtype) low_res_latents = self.encode_images(
upsampled_low_res_images, device=latents.device, dtype=latents.dtype)
return low_res_latents return low_res_latents
# def add_noise( # def add_noise(
# self, # self,
# original_samples: torch.FloatTensor, # original_samples: torch.FloatTensor,
@@ -402,7 +389,6 @@ class CogView4(BaseModel):
# return noisy_latents # return noisy_latents
# def add_noise( # def add_noise(
# self, # self,
# original_samples: torch.FloatTensor, # original_samples: torch.FloatTensor,
@@ -437,12 +423,13 @@ class CogView4(BaseModel):
# t_01 = (t / 1000).to(original_samples_chunks[idx].device) # t_01 = (t / 1000).to(original_samples_chunks[idx].device)
# lrln = low_res_latents_chunks[idx] - original_samples_chunks[idx] # lrln = low_res_latents_chunks[idx] - original_samples_chunks[idx]
# lrln = lrln * (1 - t_01) # # lrln = lrln * (1 - t_01)
# # make the noise an interpolation between noise and low_res_latents with # # make the noise an interpolation between noise and low_res_latents with
# # being noise at t_01=1 and low_res_latents at t_01=0 # # being noise at t_01=1 and low_res_latents at t_01=0
# # new_noise = t_01 * noise_chunks[idx] + (1 - t_01) * lrln # new_noise = t_01 * noise_chunks[idx] + (1 - t_01) * lrln
# new_noise = noise_chunks[idx] + lrln # # new_noise = noise_chunks[idx] + lrln
# # new_noise = noise_chunks[idx] + lrln
# # Then apply flowmatching interpolation between this blended state and noise # # Then apply flowmatching interpolation between this blended state and noise
# noisy_latents = (1 - t_01) * original_samples + t_01 * new_noise # noisy_latents = (1 - t_01) * original_samples + t_01 * new_noise
+1 -1
View File
@@ -89,7 +89,7 @@ class CustomFlowMatchEulerDiscreteScheduler(FlowMatchEulerDiscreteScheduler):
) -> torch.Tensor: ) -> torch.Tensor:
t_01 = (timesteps / 1000).to(original_samples.device) t_01 = (timesteps / 1000).to(original_samples.device)
# forward ODE # forward ODE
noisy_model_input = (1 - t_01) * original_samples + t_01 * noise noisy_model_input = (1.0 - t_01) * original_samples + t_01 * noise
# reverse ODE # reverse ODE
# noisy_model_input = (1 - t_01) * noise + t_01 * original_samples # noisy_model_input = (1 - t_01) * noise + t_01 * original_samples
return noisy_model_input return noisy_model_input