Fix audio loss when doing do_guidance_loss

This commit is contained in:
Jaret Burkett
2026-08-04 21:43:31 -06:00
parent d870e9b68a
commit 0f9094db95
5 changed files with 132 additions and 20 deletions

View File

@@ -736,6 +736,32 @@ class SDTrainer(BaseSDTrainProcess):
unconditional_target = unconditional_target * alpha
target = unconditional_target + guidance_scale * (target - unconditional_target)
# joint audio models (ltx2, minimax_h3, flux3) carry their audio
# target/pred on the batch. Extrapolate the audio target the
# same way so the audio stream trains contrastively as well.
audio_uncond = getattr(batch, 'audio_pred_uncond', None)
if batch.audio_target is not None and audio_uncond is not None:
audio_target = batch.audio_target.float()
audio_uncond = audio_uncond.float()
audio_dims = [1] * (audio_target.dim() - 1)
if self.train_config.do_guidance_loss_cfg_zero:
batch_size = audio_target.shape[0]
a_pos_flat = audio_target.view(batch_size, -1)
a_neg_flat = audio_uncond.view(batch_size, -1)
a_dot = torch.sum(a_pos_flat * a_neg_flat, dim=1, keepdim=True)
a_squared_norm = torch.sum(a_neg_flat ** 2, dim=1, keepdim=True) + 1e-8
audio_uncond = audio_uncond * (a_dot / a_squared_norm).view(-1, *audio_dims)
audio_guidance_scale = self._guidance_loss_target_batch
if isinstance(audio_guidance_scale, list):
audio_guidance_scale = torch.tensor(audio_guidance_scale).to(
audio_target.device, dtype=audio_target.dtype
).view(-1, *audio_dims)
batch.audio_target = (
audio_uncond + audio_guidance_scale * (audio_target - audio_uncond)
).to(batch.audio_target.dtype).detach()
if self.train_config.do_differential_guidance:
with torch.no_grad():
guidance_scale = self.train_config.differential_guidance_scale