Only add a blank control if the model has to have it, for unconditionals. Previously it always encoded a blank control on unconditional if the model could take one. Affects DOP and blank prompt preservations.
This commit is contained in:
@@ -120,6 +120,24 @@ class SDTrainer(BaseSDTrainProcess):
|
||||
|
||||
def before_model_load(self):
|
||||
pass
|
||||
|
||||
def get_blank_control_image(self):
|
||||
# noise instead of a black image so the fallback does not read as a
|
||||
# meaningful (solid black) reference
|
||||
control_image = torch.rand((1, 3, 224, 224), device=self.sd.device_torch, dtype=self.sd.torch_dtype)
|
||||
if self.sd.has_multiple_control_images:
|
||||
control_image = [control_image]
|
||||
return control_image
|
||||
|
||||
def encode_static_prompt(self, prompt, **kwargs):
|
||||
# static embeds (blank/trigger/uncond/DOP class) are always plain text.
|
||||
# Some models (edit models) cannot encode a prompt without control images
|
||||
# and raise, only then fall back to a blank control image. Real errors
|
||||
# surface on the fallback call.
|
||||
try:
|
||||
return self.sd.encode_prompt(prompt, **kwargs)
|
||||
except Exception:
|
||||
return self.sd.encode_prompt(prompt, control_images=self.get_blank_control_image(), **kwargs)
|
||||
|
||||
def cache_sample_prompts(self):
|
||||
if self.train_config.disable_sampling:
|
||||
@@ -259,18 +277,9 @@ class SDTrainer(BaseSDTrainProcess):
|
||||
|
||||
# cache unconditional embeds (blank prompt)
|
||||
with torch.no_grad():
|
||||
kwargs = {}
|
||||
if self.sd.encode_control_in_text_embeddings:
|
||||
# just do a blank image for unconditionals
|
||||
control_image = torch.zeros((1, 3, 224, 224), device=self.sd.device_torch, dtype=self.sd.torch_dtype)
|
||||
if self.sd.has_multiple_control_images:
|
||||
control_image = [control_image]
|
||||
|
||||
kwargs['control_images'] = control_image
|
||||
self.unconditional_embeds = self.sd.encode_prompt(
|
||||
self.unconditional_embeds = self.encode_static_prompt(
|
||||
[self.train_config.unconditional_prompt],
|
||||
long_prompts=self.do_long_prompts,
|
||||
**kwargs
|
||||
).to(
|
||||
self.device_torch,
|
||||
dtype=self.sd.torch_dtype
|
||||
@@ -324,18 +333,11 @@ class SDTrainer(BaseSDTrainProcess):
|
||||
raise ValueError("Cannot unload text encoder if training text encoder")
|
||||
# cache embeddings
|
||||
self.sd.text_encoder_to(self.device_torch)
|
||||
encode_kwargs = {}
|
||||
if self.sd.encode_control_in_text_embeddings:
|
||||
# just do a blank image for unconditionals
|
||||
control_image = torch.zeros((1, 3, 224, 224), device=self.sd.device_torch, dtype=self.sd.torch_dtype)
|
||||
if self.sd.has_multiple_control_images:
|
||||
control_image = [control_image]
|
||||
encode_kwargs['control_images'] = control_image
|
||||
self.cached_blank_embeds = self.sd.encode_prompt("", **encode_kwargs)
|
||||
self.cached_blank_embeds = self.encode_static_prompt("")
|
||||
if self.trigger_word is not None:
|
||||
self.cached_trigger_embeds = self.sd.encode_prompt(self.trigger_word, **encode_kwargs)
|
||||
self.cached_trigger_embeds = self.encode_static_prompt(self.trigger_word)
|
||||
if self.train_config.diff_output_preservation:
|
||||
self.cached_dop_class_embeds = self.sd.encode_prompt(self.train_config.diff_output_preservation_class)
|
||||
self.cached_dop_class_embeds = self.encode_static_prompt(self.train_config.diff_output_preservation_class)
|
||||
self.diff_output_preservation_embeds = self.cached_dop_class_embeds
|
||||
|
||||
self.cache_sample_prompts()
|
||||
|
||||
Reference in New Issue
Block a user