More guidance work. Improved LoRA module resolver for unet. Added vega mappings and LoRA training for it. Various other bigfixes and changes

This commit is contained in:
Jaret Burkett committed 2023-12-15 06:02:10 -07:00
1 parent e5177833b2
commit 39870411d8
14 files changed
+3501 -106

No files matched your search

+282 -71
View File
@@ -7,7 +7,7 @@ from toolkit.prompt_utils import PromptEmbeds, concat_prompt_embeds
from toolkit.stable_diffusion_model import StableDiffusion
from toolkit.train_tools import get_torch_dtype
GuidanceType = Literal["targeted", "polarity", "targeted_polarity"]
GuidanceType = Literal["targeted", "polarity", "targeted_polarity", "direct"]
DIFFERENTIAL_SCALER = 0.2
@@ -118,8 +118,8 @@ def get_targeted_polarity_loss(
# )
# Disable the LoRA network so we can predict parent network knowledge without it
sd.network.is_active = False
sd.unet.eval()
# sd.network.is_active = False
# sd.unet.eval()
# Predict noise to get a baseline of what the parent network wants to do with the latents + noise.
# This acts as our control to preserve the unaltered parts of the image.
@@ -133,15 +133,15 @@ def get_targeted_polarity_loss(
# conditional_baseline_prediction, unconditional_baseline_prediction = torch.chunk(baseline_prediction, 2, dim=0)
negative_network_weights = [weight * -1.0 for weight in network_weight_list]
positive_network_weights = [weight * 1.0 for weight in network_weight_list]
cat_network_weight_list = positive_network_weights + negative_network_weights
# negative_network_weights = [weight * -1.0 for weight in network_weight_list]
# positive_network_weights = [weight * 1.0 for weight in network_weight_list]
# cat_network_weight_list = positive_network_weights + negative_network_weights
# turn the LoRA network back on.
sd.unet.train()
sd.network.is_active = True
# sd.network.is_active = True
sd.network.multiplier = cat_network_weight_list
# sd.network.multiplier = cat_network_weight_list
# do our prediction with LoRA active on the scaled guidance latents
prediction = sd.predict_noise(
@@ -183,9 +183,7 @@ def get_targeted_polarity_loss(
return loss
# targeted
def get_targeted_guidance_loss(
def get_direct_guidance_loss(
noisy_latents: torch.Tensor,
conditional_embeds: 'PromptEmbeds',
match_adapter_assist: bool,
@@ -206,81 +204,45 @@ def get_targeted_guidance_loss(
conditional_latents = batch.latents.to(device, dtype=dtype).detach()
unconditional_latents = batch.unconditional_latents.to(device, dtype=dtype).detach()
# # apply random offset to both latents
# offset = torch.randn((conditional_latents.shape[0], 1, 1, 1), device=device, dtype=dtype)
# offset = offset * 0.1
# conditional_latents = conditional_latents + offset
# unconditional_latents = unconditional_latents + offset
#
# # get random scale 0f 0.8 to 1.2
# scale = torch.rand((conditional_latents.shape[0], 1, 1, 1), device=device, dtype=dtype)
# scale = scale * 0.4
# scale = scale + 0.8
# conditional_latents = conditional_latents * scale
# unconditional_latents = unconditional_latents * scale
unconditional_diff = (unconditional_latents - conditional_latents)
# scale it to the timestep
unconditional_diff_noise = sd.add_noise(
torch.zeros_like(unconditional_latents),
unconditional_diff,
timesteps
)
unconditional_diff_noise = unconditional_diff_noise.detach().requires_grad_(False)
target_noise = noise + unconditional_diff_noise
noisy_latents = sd.add_noise(
conditional_noisy_latents = sd.add_noise(
conditional_latents,
target_noise,
# noise,
# target_noise,
noise,
timesteps
).detach()
# Disable the LoRA network so we can predict parent network knowledge without it
sd.network.is_active = False
sd.unet.eval()
# Predict noise to get a baseline of what the parent network wants to do with the latents + noise.
# This acts as our control to preserve the unaltered parts of the image.
baseline_prediction = sd.predict_noise(
latents=noisy_latents.to(device, dtype=dtype).detach(),
conditional_embeddings=conditional_embeds.to(device, dtype=dtype).detach(),
timestep=timesteps,
guidance_scale=1.0,
**pred_kwargs # adapter residuals in here
).detach().requires_grad_(False)
# determine the error for the baseline prediction
baseline_prediction_error = baseline_prediction - noise
prediction_target = baseline_prediction_error + unconditional_diff_noise
prediction_target = prediction_target.detach().requires_grad_(False)
unconditional_noisy_latents = sd.add_noise(
unconditional_latents,
noise,
timesteps
).detach()
# turn the LoRA network back on.
sd.unet.train()
sd.network.is_active = True
# sd.network.is_active = True
sd.network.multiplier = network_weight_list
# sd.network.multiplier = network_weight_list
# do our prediction with LoRA active on the scaled guidance latents
prediction = sd.predict_noise(
latents=noisy_latents.to(device, dtype=dtype).detach(),
conditional_embeddings=conditional_embeds.to(device, dtype=dtype).detach(),
timestep=timesteps,
latents=torch.cat([unconditional_noisy_latents, conditional_noisy_latents]).to(device, dtype=dtype).detach(),
conditional_embeddings=concat_prompt_embeds([conditional_embeds,conditional_embeds]).to(device, dtype=dtype).detach(),
timestep=torch.cat([timesteps, timesteps]),
guidance_scale=1.0,
**pred_kwargs # adapter residuals in here
)
prediction_error = prediction - noise
noise_pred_uncond, noise_pred_cond = torch.chunk(prediction, 2, dim=0)
guidance_scale = 1.0
guidance_pred = noise_pred_uncond + guidance_scale * (
noise_pred_cond - noise_pred_uncond
)
guidance_loss = torch.nn.functional.mse_loss(
prediction_error.float(),
# unconditional_diff_noise.float(),
prediction_target.float(),
guidance_pred.float(),
noise.detach().float(),
reduction="none"
)
guidance_loss = guidance_loss.mean([1, 2, 3])
guidance_loss = guidance_loss.mean()
@@ -297,6 +259,242 @@ def get_targeted_guidance_loss(
return loss
# targeted
def get_targeted_guidance_loss(
noisy_latents: torch.Tensor,
conditional_embeds: 'PromptEmbeds',
match_adapter_assist: bool,
network_weight_list: list,
timesteps: torch.Tensor,
pred_kwargs: dict,
batch: 'DataLoaderBatchDTO',
noise: torch.Tensor,
sd: 'StableDiffusion',
**kwargs
):
with torch.no_grad():
dtype = get_torch_dtype(sd.torch_dtype)
device = sd.device_torch
# create the differential mask from the actual tensors
conditional_imgs = batch.tensor.to(device, dtype=dtype).detach()
unconditional_imgs = batch.unconditional_tensor.to(device, dtype=dtype).detach()
differential_mask = torch.abs(conditional_imgs - unconditional_imgs)
differential_mask = differential_mask - differential_mask.min(dim=1, keepdim=True)[0].min(dim=2, keepdim=True)[0].min(dim=3, keepdim=True)[0]
differential_mask = differential_mask / differential_mask.max(dim=1, keepdim=True)[0].max(dim=2, keepdim=True)[0].max(dim=3, keepdim=True)[0]
# differential_mask is (bs, 3, width, height)
# latents are (bs, 4, width, height)
# reduce the mean on dim 1 to get a single channel mask and stack it to match latents
differential_mask = differential_mask.mean(dim=1, keepdim=True)
differential_mask = torch.cat([differential_mask] * 4, dim=1)
# scale the mask down to latent size
differential_mask = torch.nn.functional.interpolate(
differential_mask,
size=noisy_latents.shape[2:],
mode="nearest"
)
conditional_noisy_latents = noisy_latents
conditional_latents = batch.latents.to(device, dtype=dtype).detach()
unconditional_latents = batch.unconditional_latents.to(device, dtype=dtype).detach()
# unconditional_as_noise = unconditional_latents - conditional_latents
# conditional_as_noise = conditional_latents - unconditional_latents
# Encode the unconditional image into latents
unconditional_noisy_latents = sd.noise_scheduler.add_noise(
unconditional_latents,
noise,
timesteps
)
conditional_noisy_latents = sd.noise_scheduler.add_noise(
conditional_latents,
noise,
timesteps
)
# was_network_active = self.network.is_active
sd.network.is_active = False
sd.unet.eval()
# calculate the differential between our conditional (target image) and out unconditional ("bad" image)
# target_differential = unconditional_noisy_latents - conditional_noisy_latents
target_differential = unconditional_latents - conditional_latents
# target_differential = conditional_latents - unconditional_latents
# scale the target differential by the scheduler
# todo, scale it the right way
# target_differential = sd.noise_scheduler.add_noise(
# torch.zeros_like(target_differential),
# target_differential,
# timesteps
# )
# noise_abs_mean = torch.abs(noise + 1e-6).mean(dim=[1, 2, 3], keepdim=True)
# target_differential = target_differential.detach()
# target_differential_abs_mean = torch.abs(target_differential + 1e-6).mean(dim=[1, 2, 3], keepdim=True)
# # determins scaler to adjust to same abs mean as noise
# scaler = noise_abs_mean / target_differential_abs_mean
target_differential_knowledge = target_differential
target_differential_knowledge = target_differential_knowledge.detach()
# add the target differential to the target latents as if it were noise with the scheduler scaled to
# the current timestep. Scaling the noise here is IMPORTANT and will lead to a blurry targeted area if not done
# properly
# guidance_latents = sd.noise_scheduler.add_noise(
# conditional_noisy_latents,
# target_differential,
# timesteps
# )
# guidance_latents = conditional_noisy_latents + target_differential
# target_noise = conditional_noisy_latents + target_differential
# With LoRA network bypassed, predict noise to get a baseline of what the network
# wants to do with the latents + noise. Pass our target latents here for the input.
target_unconditional = sd.predict_noise(
latents=unconditional_noisy_latents.to(device, dtype=dtype).detach(),
conditional_embeddings=conditional_embeds.to(device, dtype=dtype).detach(),
timestep=timesteps,
guidance_scale=1.0,
**pred_kwargs # adapter residuals in here
).detach()
# target_conditional = sd.predict_noise(
# latents=conditional_noisy_latents.to(device, dtype=dtype).detach(),
# conditional_embeddings=conditional_embeds.to(device, dtype=dtype).detach(),
# timestep=timesteps,
# guidance_scale=1.0,
# **pred_kwargs # adapter residuals in here
# ).detach()
# we calculate the networks current knowledge so we do not overlearn what we know
# parent_knowledge = target_unconditional - target_conditional
# parent_knowledge = parent_knowledge.detach()
# del target_conditional
# del target_unconditional
# we now have the differential noise prediction needed to create our convergence target
# target_unknown_knowledge = target_differential + parent_knowledge
# del parent_knowledge
prior_prediction_loss = torch.nn.functional.mse_loss(
target_unconditional.float(),
noise.float(),
reduction="none"
).detach().clone()
# turn the LoRA network back on.
sd.unet.train()
sd.network.is_active = True
sd.network.multiplier = network_weight_list
# with LoRA active, predict the noise with the scaled differential latents added. This will allow us
# the opportunity to predict the differential + noise that was added to the latents.
prediction_conditional = sd.predict_noise(
latents=conditional_noisy_latents.to(device, dtype=dtype).detach(),
conditional_embeddings=conditional_embeds.to(device, dtype=dtype).detach(),
timestep=timesteps,
guidance_scale=1.0,
**pred_kwargs # adapter residuals in here
)
# remove the baseline conditional prediction. This will leave only the divergence from the baseline and
# the prediction of the added differential noise
# prediction_positive = prediction_unconditional - target_unconditional
# current_knowledge = target_unconditional - prediction_conditional
# current_differential_knowledge = prediction_conditional - target_unconditional
# current_unknown_knowledge = parent_knowledge - current_knowledge
#
# current_unknown_knowledge_abs_mean = torch.abs(current_unknown_knowledge + 1e-6).mean(dim=[1, 2, 3], keepdim=True)
# current_unknown_knowledge_std = current_unknown_knowledge / current_unknown_knowledge_abs_mean
# for loss, we target ONLY the unscaled differential between our conditional and unconditional latents
# this is the diffusion training process.
# This will guide the network to make identical predictions it previously did for everything EXCEPT our
# differential between the conditional and unconditional images
# positive_loss = torch.nn.functional.mse_loss(
# current_differential_knowledge.float(),
# target_differential_knowledge.float(),
# reduction="none"
# )
normal_loss = torch.nn.functional.mse_loss(
prediction_conditional.float(),
noise.float(),
reduction="none"
)
#
# # scale positive and neutral loss to the same scale
# positive_loss_abs_mean = torch.abs(positive_loss + 1e-6).mean(dim=[1, 2, 3], keepdim=True)
# normal_loss_abs_mean = torch.abs(normal_loss + 1e-6).mean(dim=[1, 2, 3], keepdim=True)
# scaler = normal_loss_abs_mean / positive_loss_abs_mean
# positive_loss = positive_loss * scaler
# positive_loss = positive_loss * differential_mask
# positive_loss = positive_loss
# masked_normal_loss = normal_loss * differential_mask
prior_loss = torch.abs(
normal_loss.float() - prior_prediction_loss.float(),
# ) * (1 - differential_mask)
)
decouple = True
# positive_loss_full = positive_loss
# prior_loss_full = prior_loss
#
# current_scaler = (prior_loss_full.max() / positive_loss_full.max())
# # positive_loss = positive_loss * current_scaler
# avg_scaler_arr.append(current_scaler.item())
# avg_scaler = sum(avg_scaler_arr) / len(avg_scaler_arr)
# print(f"avg scaler: {avg_scaler}, current scaler: {current_scaler.item()}")
# # remove extra scalers more than 100
# if len(avg_scaler_arr) > 100:
# avg_scaler_arr.pop(0)
#
# # positive_loss = positive_loss * avg_scaler
# positive_loss = positive_loss * avg_scaler * 0.1
if decouple:
# positive_loss = positive_loss.mean([1, 2, 3])
prior_loss = prior_loss.mean([1, 2, 3])
# masked_normal_loss = masked_normal_loss.mean([1, 2, 3])
positive_loss = prior_loss
# positive_loss = positive_loss + prior_loss
else:
# positive_loss = positive_loss + prior_loss
positive_loss = prior_loss
positive_loss = positive_loss.mean([1, 2, 3])
# positive_loss = positive_loss + adain_loss.mean([1, 2, 3])
# send it backwards BEFORE switching network polarity
# positive_loss = self.apply_snr(positive_loss, timesteps)
positive_loss = positive_loss.mean()
positive_loss.backward()
# loss = positive_loss.detach() + negative_loss.detach()
loss = positive_loss.detach()
# add a grad so other backward does not fail
loss.requires_grad_(True)
# restore network
sd.network.multiplier = network_weight_list
return loss
def get_guided_loss_polarity(
noisy_latents: torch.Tensor,
conditional_embeds: PromptEmbeds,
@@ -360,17 +558,17 @@ def get_guided_loss_polarity(
noise.float(),
reduction="none"
)
pred_loss = pred_loss.mean([1, 2, 3])
# pred_loss = pred_loss.mean([1, 2, 3])
pred_neg_loss = torch.nn.functional.mse_loss(
pred_neg.float(),
noise.float(),
reduction="none"
)
pred_neg_loss = pred_neg_loss.mean([1, 2, 3])
loss = pred_loss + pred_neg_loss
loss = loss.mean([1, 2, 3])
loss = loss.mean()
loss.backward()
@@ -437,5 +635,18 @@ def get_guidance_loss(
sd,
**kwargs
)
elif guidance_type == "direct":
return get_direct_guidance_loss(
noisy_latents,
conditional_embeds,
match_adapter_assist,
network_weight_list,
timesteps,
pred_kwargs,
batch,
noise,
sd,
**kwargs
)
else:
raise NotImplementedError(f"Guidance type {guidance_type} is not implemented")