github/ai-toolkit
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Fix audio issues with ltx2 models. Silent codec fails now raised. Auto convert surround sound audio to stereo. Invalidate old caches just to be safe so they recache now.

Browse Source
This commit is contained in:
Jaret Burkett
2026-03-23 20:08:33 +00:00
parent 561e6f201c
commit 5642b656b9
8 changed files with 36 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
toolkit/dataloader_mixins.py
View File

@@ -106,6 +106,22 @@ def clean_caption(caption):
# caption = ', '.join(caption_split)
return caption
def waveform_to_stereo(waveform):
c = waveform.shape[0]
if c == 2:
return waveform
if c == 1:
return waveform.expand(2, -1)
if c == 6: # 5.1: FL, FR, FC, LFE, BL, BR
fl, fr, fc, _, bl, br = waveform
k = 0.7071
return torch.stack([fl + k * fc + k * bl, fr + k * fc + k * br])
if c == 8: # 7.1: FL, FR, FC, LFE, BL, BR, SL, SR
fl, fr, fc, _, bl, br, sl, sr = waveform
k = 0.7071
return torch.stack([fl + k * fc + k * (bl + sl), fr + k * fc + k * (br + sr)])
return waveform.mean(0, keepdim=True).expand(2, -1)
class CaptionMixin:
def get_caption_item(self: 'AiToolkitDataset', index):
@@ -632,6 +648,8 @@ class ImageProcessingDTOMixin:
waveform, sample_rate = torchaudio.load(self.path) # [channels, samples]
waveform = waveform_to_stereo(waveform) # Convert to stereo if not already
if self.dataset_config.audio_normalize:
peak = waveform.abs().amax() # global peak across channels
eps = 1e-9
@@ -670,11 +688,8 @@ class ImageProcessingDTOMixin:
self.audio_data = {"waveform": waveform, "sample_rate": int(sample_rate)}
except Exception as e:
# Keep behavior identical for non-audio datasets; for audio datasets, just skip if missing/broken.
if hasattr(self.dataset_config, 'debug') and self.dataset_config.debug:
print_acc(f"Could not extract/stretch audio for {self.path}: {e}")
self.audio_data = None
self.audio_tensor = None
# if issue with libtorchcodec "Could not load libtorchcodec"
raise Exception(f"** WARNING ** - Error Processing audio for {self.path}. Error: {e}")
# Only log success in debug mode
if hasattr(self.dataset_config, 'debug') and self.dataset_config.debug: