Add support for MiniMax H3 T2V and I2V training
This commit is contained in:
1 parent
73cab2acf5
commit
8502a845b1
28 files changed
+4095
-41
No files matched your search
@@ -508,15 +508,19 @@ class ImageProcessingDTOMixin:
|
||||
if self.dataset_config.auto_frame_count:
|
||||
# allow for any length video here but make sure it is temporally compressable.
|
||||
vid_length_seconds = total_frames / video_fps
|
||||
|
||||
|
||||
desired_num_frames = int(vid_length_seconds * self.dataset_config.fps)
|
||||
|
||||
# make sure it is divisible by temporal_compression
|
||||
desired_num_frames = desired_num_frames // self.temporal_compression * self.temporal_compression
|
||||
|
||||
# TODO, all models currently add a key frame, but future models may not, update here if this changes.
|
||||
desired_num_frames += 1 # add one for the key frame that is always added
|
||||
|
||||
|
||||
if getattr(self, 'frame_count_snapper', None) is not None:
|
||||
# model-specific valid-frame-count grid (e.g. minimax_h3's 17n+5)
|
||||
desired_num_frames = self.frame_count_snapper(desired_num_frames)
|
||||
else:
|
||||
# make sure it is divisible by temporal_compression
|
||||
desired_num_frames = desired_num_frames // self.temporal_compression * self.temporal_compression
|
||||
|
||||
# TODO, all models currently add a key frame, but future models may not, update here if this changes.
|
||||
desired_num_frames += 1 # add one for the key frame that is always added
|
||||
|
||||
self.num_frames = desired_num_frames
|
||||
|
||||
|
||||
@@ -705,31 +709,39 @@ class ImageProcessingDTOMixin:
|
||||
else:
|
||||
target_duration = source_duration
|
||||
|
||||
waveform, sample_rate = torchaudio.load(self.path) # [channels, samples]
|
||||
|
||||
waveform = waveform_to_stereo(waveform) # Convert to stereo if not already
|
||||
|
||||
if self.dataset_config.audio_normalize:
|
||||
peak = waveform.abs().amax() # global peak across channels
|
||||
eps = 1e-9
|
||||
target_peak = 0.999 # ~ -0.01 dBFS
|
||||
gain = target_peak / (peak + eps)
|
||||
waveform = waveform * gain
|
||||
# torchcodec's AudioDecoder raises when a video has no audio
|
||||
# track, so probe for a stream before decoding.
|
||||
import av
|
||||
with av.open(self.path) as container:
|
||||
has_audio_stream = len(container.streams.audio) > 0
|
||||
|
||||
# Slice to the selected clip region (when we have a meaningful time range)
|
||||
if source_duration > 0.0:
|
||||
start_sample = int(round(clip_start_time * sample_rate))
|
||||
end_sample = int(round(clip_end_time * sample_rate))
|
||||
start_sample = max(0, min(start_sample, waveform.shape[-1]))
|
||||
end_sample = max(0, min(end_sample, waveform.shape[-1]))
|
||||
if end_sample > start_sample:
|
||||
waveform = waveform[..., start_sample:end_sample]
|
||||
waveform = None
|
||||
if has_audio_stream:
|
||||
waveform, sample_rate = torchaudio.load(self.path) # [channels, samples]
|
||||
|
||||
waveform = waveform_to_stereo(waveform) # Convert to stereo if not already
|
||||
|
||||
if self.dataset_config.audio_normalize:
|
||||
peak = waveform.abs().amax() # global peak across channels
|
||||
eps = 1e-9
|
||||
target_peak = 0.999 # ~ -0.01 dBFS
|
||||
gain = target_peak / (peak + eps)
|
||||
waveform = waveform * gain
|
||||
|
||||
# Slice to the selected clip region (when we have a meaningful time range)
|
||||
if source_duration > 0.0:
|
||||
start_sample = int(round(clip_start_time * sample_rate))
|
||||
end_sample = int(round(clip_end_time * sample_rate))
|
||||
start_sample = max(0, min(start_sample, waveform.shape[-1]))
|
||||
end_sample = max(0, min(end_sample, waveform.shape[-1]))
|
||||
if end_sample > start_sample:
|
||||
waveform = waveform[..., start_sample:end_sample]
|
||||
else:
|
||||
# No valid audio segment
|
||||
waveform = None
|
||||
else:
|
||||
# No valid audio segment
|
||||
# If we can't compute a meaningful time range, treat as no-audio
|
||||
waveform = None
|
||||
else:
|
||||
# If we can't compute a meaningful time range, treat as no-audio
|
||||
waveform = None
|
||||
|
||||
if waveform is not None and waveform.numel() > 0:
|
||||
target_samples = int(round(target_duration * sample_rate))
|
||||
|
||||
Reference in new issue
Block a user