WIP more work on cogview4
This commit is contained in:
1 parent
6f6fb90812
commit
aa44828c0c
3 files changed
+19
-32
No files matched your search
+1
-1
@@ -1,7 +1,7 @@
|
|||||||
torch==2.5.1
|
torch==2.5.1
|
||||||
torchvision==0.20.1
|
torchvision==0.20.1
|
||||||
safetensors
|
safetensors
|
||||||
git+https://github.com/huggingface/diffusers@97fda1b75c70705b245a462044fedb47abb17e56
|
git+https://github.com/huggingface/diffusers@24c062aaa19f5626d03d058daf8afffa2dfd49f7
|
||||||
transformers==4.49.0
|
transformers==4.49.0
|
||||||
lycoris-lora==1.8.3
|
lycoris-lora==1.8.3
|
||||||
flatten_json
|
flatten_json
|
||||||
|
|||||||
+17
-30
@@ -20,7 +20,6 @@ from transformers import GlmModel, AutoTokenizer
|
|||||||
from diffusers import FlowMatchEulerDiscreteScheduler
|
from diffusers import FlowMatchEulerDiscreteScheduler
|
||||||
from typing import TYPE_CHECKING
|
from typing import TYPE_CHECKING
|
||||||
from toolkit.accelerator import unwrap_model
|
from toolkit.accelerator import unwrap_model
|
||||||
|
|
||||||
from toolkit.samplers.custom_flowmatch_sampler import CustomFlowMatchEulerDiscreteScheduler
|
from toolkit.samplers.custom_flowmatch_sampler import CustomFlowMatchEulerDiscreteScheduler
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
@@ -86,7 +85,6 @@ class CogView4(BaseModel):
|
|||||||
base_model_path = "THUDM/CogView4-6B"
|
base_model_path = "THUDM/CogView4-6B"
|
||||||
model_path = self.model_config.name_or_path
|
model_path = self.model_config.name_or_path
|
||||||
|
|
||||||
# pipe = CogView4Pipeline.from_pretrained("THUDM/CogView4-6B", torch_dtype=torch.bfloat16)
|
|
||||||
self.print_and_status_update("Loading CogView4 model")
|
self.print_and_status_update("Loading CogView4 model")
|
||||||
# base_model_path = "black-forest-labs/FLUX.1-schnell"
|
# base_model_path = "black-forest-labs/FLUX.1-schnell"
|
||||||
base_model_path = self.model_config.name_or_path_original
|
base_model_path = self.model_config.name_or_path_original
|
||||||
@@ -213,19 +211,6 @@ class CogView4(BaseModel):
|
|||||||
generator: torch.Generator,
|
generator: torch.Generator,
|
||||||
extra: dict,
|
extra: dict,
|
||||||
):
|
):
|
||||||
# there is a bug in the check in diffusers code that requires the prompt embeds to be the same length for conditional and unconditional
|
|
||||||
# they are processed in 2 passes and the encoding code doesnt do this. So it shouldnt be needed. But, we will zero pad the shorter one. for now. Just inference here, so it should be fine.
|
|
||||||
if conditional_embeds.text_embeds.shape[1] < unconditional_embeds.text_embeds.shape[1]:
|
|
||||||
pad_len = unconditional_embeds.text_embeds.shape[1] - \
|
|
||||||
conditional_embeds.text_embeds.shape[1]
|
|
||||||
conditional_embeds.text_embeds = torch.cat([conditional_embeds.text_embeds, torch.zeros(conditional_embeds.text_embeds.shape[0], pad_len,
|
|
||||||
conditional_embeds.text_embeds.shape[2], device=conditional_embeds.text_embeds.device, dtype=conditional_embeds.text_embeds.dtype)], dim=1)
|
|
||||||
elif conditional_embeds.text_embeds.shape[1] > unconditional_embeds.text_embeds.shape[1]:
|
|
||||||
pad_len = conditional_embeds.text_embeds.shape[1] - \
|
|
||||||
unconditional_embeds.text_embeds.shape[1]
|
|
||||||
unconditional_embeds.text_embeds = torch.cat([unconditional_embeds.text_embeds, torch.zeros(unconditional_embeds.text_embeds.shape[0], pad_len,
|
|
||||||
unconditional_embeds.text_embeds.shape[2], device=unconditional_embeds.text_embeds.device, dtype=unconditional_embeds.text_embeds.dtype)], dim=1)
|
|
||||||
|
|
||||||
img = pipeline(
|
img = pipeline(
|
||||||
prompt_embeds=conditional_embeds.text_embeds.to(
|
prompt_embeds=conditional_embeds.text_embeds.to(
|
||||||
self.device_torch, dtype=self.torch_dtype),
|
self.device_torch, dtype=self.torch_dtype),
|
||||||
@@ -259,12 +244,12 @@ class CogView4(BaseModel):
|
|||||||
[target_size], dtype=self.torch_dtype, device=self.device_torch)
|
[target_size], dtype=self.torch_dtype, device=self.device_torch)
|
||||||
target_size = original_size.clone()
|
target_size = original_size.clone()
|
||||||
noise_pred_cond = self.model(
|
noise_pred_cond = self.model(
|
||||||
hidden_states=latent_model_input, # torch.Size([1, 16, 128, 128])
|
hidden_states=latent_model_input,
|
||||||
encoder_hidden_states=text_embeddings.text_embeds, # torch.Size([1, 16, 4096])
|
encoder_hidden_states=text_embeddings.text_embeds,
|
||||||
timestep=timestep,
|
timestep=timestep,
|
||||||
original_size=original_size, # [[1024., 1024.]]
|
original_size=original_size,
|
||||||
target_size=target_size, # [[1024., 1024.]]
|
target_size=target_size,
|
||||||
crop_coords=crops_coords_top_left, # [[0., 0.]]
|
crop_coords=crops_coords_top_left,
|
||||||
return_dict=False,
|
return_dict=False,
|
||||||
)[0]
|
)[0]
|
||||||
return noise_pred_cond
|
return noise_pred_cond
|
||||||
@@ -285,7 +270,7 @@ class CogView4(BaseModel):
|
|||||||
return self.text_encoder.layers[0].mlp.down_proj.weight.requires_grad
|
return self.text_encoder.layers[0].mlp.down_proj.weight.requires_grad
|
||||||
|
|
||||||
def save_model(self, output_path, meta, save_dtype):
|
def save_model(self, output_path, meta, save_dtype):
|
||||||
# only save the unet
|
# only save the unet
|
||||||
transformer: CogView4Transformer2DModel = unwrap_model(self.model)
|
transformer: CogView4Transformer2DModel = unwrap_model(self.model)
|
||||||
transformer.save_pretrained(
|
transformer.save_pretrained(
|
||||||
save_directory=os.path.join(output_path, 'transformer'),
|
save_directory=os.path.join(output_path, 'transformer'),
|
||||||
@@ -305,21 +290,23 @@ class CogView4(BaseModel):
|
|||||||
if noise is None:
|
if noise is None:
|
||||||
raise ValueError("Noise is not provided")
|
raise ValueError("Noise is not provided")
|
||||||
# return batch.latents
|
# return batch.latents
|
||||||
|
# return (batch.latents - noise).detach()
|
||||||
return (noise - batch.latents).detach()
|
return (noise - batch.latents).detach()
|
||||||
|
# return (batch.latents).detach()
|
||||||
# return (effective_noise - batch.latents).detach()
|
# return (effective_noise - batch.latents).detach()
|
||||||
|
|
||||||
|
|
||||||
def _get_low_res_latents(self, latents):
|
def _get_low_res_latents(self, latents):
|
||||||
# todo prevent needing to do this and grab the tensor another way.
|
# todo prevent needing to do this and grab the tensor another way.
|
||||||
with torch.no_grad():
|
with torch.no_grad():
|
||||||
# Decode latents to image space
|
# Decode latents to image space
|
||||||
images = self.decode_latents(latents, device=latents.device, dtype=latents.dtype)
|
images = self.decode_latents(
|
||||||
|
latents, device=latents.device, dtype=latents.dtype)
|
||||||
|
|
||||||
# Downsample by a factor of 2 using bilinear interpolation
|
# Downsample by a factor of 2 using bilinear interpolation
|
||||||
B, C, H, W = images.shape
|
B, C, H, W = images.shape
|
||||||
low_res_images = torch.nn.functional.interpolate(
|
low_res_images = torch.nn.functional.interpolate(
|
||||||
images,
|
images,
|
||||||
size=(H // 4, W // 4),
|
size=(H // 2, W // 2),
|
||||||
mode="bilinear",
|
mode="bilinear",
|
||||||
align_corners=False
|
align_corners=False
|
||||||
)
|
)
|
||||||
@@ -333,10 +320,10 @@ class CogView4(BaseModel):
|
|||||||
)
|
)
|
||||||
|
|
||||||
# Encode the low-resolution images back to latent space
|
# Encode the low-resolution images back to latent space
|
||||||
low_res_latents = self.encode_images(upsampled_low_res_images, device=latents.device, dtype=latents.dtype)
|
low_res_latents = self.encode_images(
|
||||||
|
upsampled_low_res_images, device=latents.device, dtype=latents.dtype)
|
||||||
return low_res_latents
|
return low_res_latents
|
||||||
|
|
||||||
|
|
||||||
# def add_noise(
|
# def add_noise(
|
||||||
# self,
|
# self,
|
||||||
# original_samples: torch.FloatTensor,
|
# original_samples: torch.FloatTensor,
|
||||||
@@ -402,7 +389,6 @@ class CogView4(BaseModel):
|
|||||||
|
|
||||||
# return noisy_latents
|
# return noisy_latents
|
||||||
|
|
||||||
|
|
||||||
# def add_noise(
|
# def add_noise(
|
||||||
# self,
|
# self,
|
||||||
# original_samples: torch.FloatTensor,
|
# original_samples: torch.FloatTensor,
|
||||||
@@ -437,12 +423,13 @@ class CogView4(BaseModel):
|
|||||||
# t_01 = (t / 1000).to(original_samples_chunks[idx].device)
|
# t_01 = (t / 1000).to(original_samples_chunks[idx].device)
|
||||||
|
|
||||||
# lrln = low_res_latents_chunks[idx] - original_samples_chunks[idx]
|
# lrln = low_res_latents_chunks[idx] - original_samples_chunks[idx]
|
||||||
# lrln = lrln * (1 - t_01)
|
# # lrln = lrln * (1 - t_01)
|
||||||
|
|
||||||
# # make the noise an interpolation between noise and low_res_latents with
|
# # make the noise an interpolation between noise and low_res_latents with
|
||||||
# # being noise at t_01=1 and low_res_latents at t_01=0
|
# # being noise at t_01=1 and low_res_latents at t_01=0
|
||||||
# # new_noise = t_01 * noise_chunks[idx] + (1 - t_01) * lrln
|
# new_noise = t_01 * noise_chunks[idx] + (1 - t_01) * lrln
|
||||||
# new_noise = noise_chunks[idx] + lrln
|
# # new_noise = noise_chunks[idx] + lrln
|
||||||
|
# # new_noise = noise_chunks[idx] + lrln
|
||||||
|
|
||||||
# # Then apply flowmatching interpolation between this blended state and noise
|
# # Then apply flowmatching interpolation between this blended state and noise
|
||||||
# noisy_latents = (1 - t_01) * original_samples + t_01 * new_noise
|
# noisy_latents = (1 - t_01) * original_samples + t_01 * new_noise
|
||||||
|
|||||||
@@ -89,7 +89,7 @@ class CustomFlowMatchEulerDiscreteScheduler(FlowMatchEulerDiscreteScheduler):
|
|||||||
) -> torch.Tensor:
|
) -> torch.Tensor:
|
||||||
t_01 = (timesteps / 1000).to(original_samples.device)
|
t_01 = (timesteps / 1000).to(original_samples.device)
|
||||||
# forward ODE
|
# forward ODE
|
||||||
noisy_model_input = (1 - t_01) * original_samples + t_01 * noise
|
noisy_model_input = (1.0 - t_01) * original_samples + t_01 * noise
|
||||||
# reverse ODE
|
# reverse ODE
|
||||||
# noisy_model_input = (1 - t_01) * noise + t_01 * original_samples
|
# noisy_model_input = (1 - t_01) * noise + t_01 * original_samples
|
||||||
return noisy_model_input
|
return noisy_model_input
|
||||||
|
|||||||
Reference in new issue
Block a user