WIP. just need to put it here
This commit is contained in:
1 parent
2305e55c82
commit
6ab8b8b0f1
4 files changed
+279
-63
No files matched your search
@@ -13,10 +13,12 @@ from toolkit.optimizer import get_optimizer
|
||||
from toolkit.paths import REPOS_ROOT
|
||||
import sys
|
||||
|
||||
from toolkit.pipelines import CustomStableDiffusionXLPipeline
|
||||
|
||||
sys.path.append(REPOS_ROOT)
|
||||
sys.path.append(os.path.join(REPOS_ROOT, 'leco'))
|
||||
|
||||
from diffusers import StableDiffusionPipeline, StableDiffusionXLPipeline, DDPMScheduler
|
||||
from diffusers import StableDiffusionPipeline, StableDiffusionXLPipeline
|
||||
|
||||
from jobs.process import BaseTrainProcess
|
||||
from toolkit.metadata import get_meta_for_safetensors, load_metadata_from_safetensors
|
||||
@@ -100,15 +102,7 @@ class BaseSDTrainProcess(BaseTrainProcess):
|
||||
# self.sd.tokenizer.to(self.device_torch)
|
||||
# TODO add clip skip
|
||||
if self.sd.is_xl:
|
||||
pipeline = StableDiffusionXLPipeline(
|
||||
vae=self.sd.vae,
|
||||
unet=self.sd.unet,
|
||||
text_encoder=self.sd.text_encoder[0],
|
||||
text_encoder_2=self.sd.text_encoder[1],
|
||||
tokenizer=self.sd.tokenizer[0],
|
||||
tokenizer_2=self.sd.tokenizer[1],
|
||||
scheduler=self.sd.noise_scheduler,
|
||||
)
|
||||
pipeline = self.sd.pipeline
|
||||
else:
|
||||
pipeline = StableDiffusionPipeline(
|
||||
vae=self.sd.vae,
|
||||
@@ -209,7 +203,8 @@ class BaseSDTrainProcess(BaseTrainProcess):
|
||||
img.save(output_path)
|
||||
|
||||
# clear pipeline and cache to reduce vram usage
|
||||
del pipeline
|
||||
if not self.sd.is_xl:
|
||||
del pipeline
|
||||
torch.cuda.empty_cache()
|
||||
|
||||
# restore training state
|
||||
@@ -363,6 +358,20 @@ class BaseSDTrainProcess(BaseTrainProcess):
|
||||
else:
|
||||
return None
|
||||
|
||||
def predict_noise_xl(
|
||||
self,
|
||||
latents: torch.FloatTensor,
|
||||
positive_prompt: str,
|
||||
negative_prompt: str,
|
||||
timestep: int,
|
||||
guidance_scale=7.5,
|
||||
guidance_rescale=0.7,
|
||||
add_time_ids=None,
|
||||
**kwargs,
|
||||
):
|
||||
pass
|
||||
|
||||
|
||||
def predict_noise(
|
||||
self,
|
||||
latents: torch.FloatTensor,
|
||||
@@ -492,12 +501,12 @@ class BaseSDTrainProcess(BaseTrainProcess):
|
||||
|
||||
if self.model_config.is_xl:
|
||||
|
||||
pipe = StableDiffusionXLPipeline.from_single_file(
|
||||
pipe = CustomStableDiffusionXLPipeline.from_single_file(
|
||||
self.model_config.name_or_path,
|
||||
dtype=dtype,
|
||||
scheduler_type='pndm',
|
||||
scheduler_type='dpm',
|
||||
device=self.device_torch
|
||||
)
|
||||
).to(self.device_torch)
|
||||
text_encoders = [pipe.text_encoder, pipe.text_encoder_2]
|
||||
tokenizer = [pipe.tokenizer, pipe.tokenizer_2]
|
||||
unet = pipe.unet
|
||||
@@ -513,7 +522,6 @@ class BaseSDTrainProcess(BaseTrainProcess):
|
||||
|
||||
text_encoder = text_encoders
|
||||
tokenizer = tokenizer
|
||||
del pipe
|
||||
flush()
|
||||
|
||||
|
||||
@@ -529,6 +537,7 @@ class BaseSDTrainProcess(BaseTrainProcess):
|
||||
text_encoder.eval()
|
||||
vae = load_vae(self.model_config.name_or_path, dtype=dtype).to('cpu', dtype=dtype)
|
||||
vae.eval()
|
||||
pipe = None
|
||||
flush()
|
||||
|
||||
|
||||
@@ -536,7 +545,7 @@ class BaseSDTrainProcess(BaseTrainProcess):
|
||||
# put on cpu for now, we only need it when sampling
|
||||
# vae = load_vae(self.model_config.name_or_path, dtype=dtype).to('cpu', dtype=dtype)
|
||||
# vae.eval()
|
||||
self.sd = StableDiffusion(vae, tokenizer, text_encoder, unet, noise_scheduler, is_xl=self.model_config.is_xl)
|
||||
self.sd = StableDiffusion(vae, tokenizer, text_encoder, unet, noise_scheduler, is_xl=self.model_config.is_xl, pipeline=pipe)
|
||||
|
||||
unet.to(self.device_torch, dtype=dtype)
|
||||
if self.train_config.xformers:
|
||||
|
||||
@@ -157,33 +157,19 @@ class TrainSDRescaleProcess(BaseSDTrainProcess):
|
||||
torch.randint(0, len(self.prompt_txt_list), (1,)).item()
|
||||
]
|
||||
prompt = self.prompt_cache[prompt_txt].to(device=self.device_torch, dtype=dtype)
|
||||
prompt.text_embeds.to(device=self.device_torch, dtype=dtype)
|
||||
prompt.pooled_embeds.to(device=self.device_torch, dtype=dtype)
|
||||
neutral = self.prompt_cache[""].to(device=self.device_torch, dtype=dtype)
|
||||
neutral.text_embeds.to(device=self.device_torch, dtype=dtype)
|
||||
neutral.pooled_embeds.to(device=self.device_torch, dtype=dtype)
|
||||
if prompt is None:
|
||||
raise ValueError(f"Prompt {prompt_txt} is not in cache")
|
||||
|
||||
prompt_batch = train_tools.concat_prompt_embeddings(
|
||||
prompt,
|
||||
neutral,
|
||||
self.train_config.batch_size,
|
||||
)
|
||||
|
||||
noise_scheduler = self.sd.noise_scheduler
|
||||
optimizer = self.optimizer
|
||||
lr_scheduler = self.lr_scheduler
|
||||
loss_function = torch.nn.MSELoss()
|
||||
|
||||
def get_noise_pred(p, n, gs, cts, dn):
|
||||
return self.predict_noise(
|
||||
latents=dn,
|
||||
text_embeddings=train_tools.concat_prompt_embeddings(
|
||||
p, # unconditional
|
||||
n, # positive
|
||||
self.train_config.batch_size,
|
||||
),
|
||||
timestep=cts,
|
||||
guidance_scale=gs,
|
||||
)
|
||||
|
||||
with torch.no_grad():
|
||||
self.sd.noise_scheduler.set_timesteps(
|
||||
self.train_config.max_denoising_steps, device=self.device_torch
|
||||
@@ -195,52 +181,60 @@ class TrainSDRescaleProcess(BaseSDTrainProcess):
|
||||
timesteps_to = torch.randint(
|
||||
1, self.train_config.max_denoising_steps, (1,)
|
||||
).item()
|
||||
absolute_total_timesteps = 1000
|
||||
|
||||
# get noise
|
||||
noise = self.get_latent_noise(
|
||||
latents = self.get_latent_noise(
|
||||
pixel_height=self.rescale_config.from_resolution,
|
||||
pixel_width=self.rescale_config.from_resolution,
|
||||
).to(self.device_torch, dtype=dtype)
|
||||
|
||||
# get latents
|
||||
latents = noise * self.sd.noise_scheduler.init_noise_sigma
|
||||
latents = latents.to(self.device_torch, dtype=dtype)
|
||||
#
|
||||
# # predict without network
|
||||
# assert self.network.is_active is False
|
||||
# denoised_latents = self.diffuse_some_steps(
|
||||
# latents, # pass simple noise latents
|
||||
# prompt_batch,
|
||||
# start_timesteps=0,
|
||||
# total_timesteps=timesteps_to,
|
||||
# guidance_scale=3,
|
||||
# )
|
||||
# noise_scheduler.set_timesteps(1000)
|
||||
#
|
||||
# current_timestep = noise_scheduler.timesteps[
|
||||
# int(timesteps_to * 1000 / self.train_config.max_denoising_steps)
|
||||
# ]
|
||||
denoised_fraction = timesteps_to / absolute_total_timesteps
|
||||
|
||||
current_timestep = 0
|
||||
denoised_latents = latents
|
||||
# get noise prediction at full scale
|
||||
from_prediction = get_noise_pred(
|
||||
prompt, neutral, 1, current_timestep, denoised_latents
|
||||
denoised_latents = self.sd.pipeline(
|
||||
num_inference_steps=1000,
|
||||
denoising_end=denoised_fraction,
|
||||
latents=latents,
|
||||
prompt_embeds=prompt.text_embeds,
|
||||
negative_prompt_embeds=neutral.text_embeds,
|
||||
pooled_prompt_embeds=prompt.pooled_embeds,
|
||||
negative_pooled_prompt_embeds=neutral.pooled_embeds,
|
||||
output_type="latent",
|
||||
num_images_per_prompt=self.train_config.batch_size,
|
||||
guidance_scale=3,
|
||||
).images.to(self.device_torch, dtype=dtype)
|
||||
|
||||
current_timestep = timesteps_to
|
||||
|
||||
from_prediction = self.sd.pipeline.predict_noise(
|
||||
latents=denoised_latents,
|
||||
prompt_embeds=prompt.text_embeds,
|
||||
negative_prompt_embeds=neutral.text_embeds,
|
||||
pooled_prompt_embeds=prompt.pooled_embeds,
|
||||
negative_pooled_prompt_embeds=neutral.pooled_embeds,
|
||||
timestep=current_timestep,
|
||||
guidance_scale=2
|
||||
)
|
||||
|
||||
reduced_from_prediction = self.reduce_size_fn(from_prediction).to("cpu", dtype=torch.float32)
|
||||
|
||||
# get noise prediction at reduced scale
|
||||
to_denoised_latents = self.reduce_size_fn(denoised_latents)
|
||||
to_denoised_latents = self.reduce_size_fn(denoised_latents).to(self.device_torch, dtype=dtype)
|
||||
|
||||
# start gradient
|
||||
optimizer.zero_grad()
|
||||
self.network.multiplier = 1.0
|
||||
with self.network:
|
||||
assert self.network.is_active is True
|
||||
to_prediction = get_noise_pred(
|
||||
prompt, neutral, 1, current_timestep, to_denoised_latents
|
||||
).to("cpu", dtype=torch.float32)
|
||||
to_prediction = self.sd.pipeline.predict_noise(
|
||||
latents=to_denoised_latents,
|
||||
prompt_embeds=prompt.text_embeds,
|
||||
negative_prompt_embeds=neutral.text_embeds,
|
||||
pooled_prompt_embeds=prompt.pooled_embeds,
|
||||
negative_pooled_prompt_embeds=neutral.pooled_embeds,
|
||||
timestep=current_timestep,
|
||||
guidance_scale=2
|
||||
)
|
||||
|
||||
reduced_from_prediction.requires_grad = False
|
||||
from_prediction.requires_grad = False
|
||||
|
||||
Reference in new issue
Block a user