Add support for ACE-Step 1.5 and ACE-Step 1.5 XL. Also added dataset captioning through the UI. (#785)
* Base ace step 1.5 xl added. Generating, still wip on training and ui * Base training code done * Fix some issues with caching text embeddings. Update sample cards to show audio * Fix issue with quantizing ace step * Add album artwork to samples with waveform. * Cleanup logs * Add album art endpoint to speed up album art loading * Made an make video with artwork script * Make ui handle basic audio models. Make multi line adjustments to the editor and better syntax hilighting. * Add prompt tagging system for special tagged models. * prompt tagging processing for ui working. * Moved default samples to a special file so we can add more when needed and they can be adjusted for a specific model * Add a captioner job with music captioner that is prepped for use with the ui * Add basit ui setup for captioning modal and handeling captioning jobs * Starting captioning job from ui working. Still better management for it. * Better filtering of job options in the job view for captioning jobs * Added qwen3 vl as a captioner for images * Have an indicator when a dataset is being captioned. * Adjust the way caption jobs look in the queue * Fix a few issues. Adjust defaults. * Version bump * Added ace step to the readme.
This commit is contained in:
1 parent
9ca58e9aa2
commit
78cf049c29
54 files changed
+5589
-536
No files matched your search
@@ -231,6 +231,12 @@ class BucketsMixin:
|
||||
# for file_item in enumerate(file_list):
|
||||
for idx, file_item in enumerate(file_list):
|
||||
file_item: 'FileItemDTO' = file_item
|
||||
if self.is_audio_model:
|
||||
bucket_key = f"{file_item.width}ms"
|
||||
if bucket_key not in self.buckets:
|
||||
self.buckets[bucket_key] = Bucket(file_item.width, 1)
|
||||
self.buckets[bucket_key].file_list_idx.append(idx)
|
||||
continue
|
||||
width = int(file_item.width * file_item.dataset_config.scale)
|
||||
height = int(file_item.height * file_item.dataset_config.scale)
|
||||
|
||||
@@ -465,6 +471,27 @@ class CaptionProcessingDTOMixin:
|
||||
pass
|
||||
return caption
|
||||
|
||||
class AudioProcessingDTOMixin:
|
||||
def load_and_process_audio(self: 'FileItemDTO'):
|
||||
# Default to "no audio" unless we successfully extract it
|
||||
self.audio_data = None
|
||||
self.audio_tensor = None
|
||||
self.tensor = None
|
||||
try:
|
||||
import torchaudio
|
||||
|
||||
waveform, sample_rate = torchaudio.load(self.path) # [channels, samples]
|
||||
waveform = waveform_to_stereo(waveform) # Convert to stereo if not already
|
||||
if sample_rate != self.sample_rate:
|
||||
waveform = torchaudio.functional.resample(waveform, sample_rate, self.sample_rate)
|
||||
self.tensor = waveform
|
||||
self.audio_tensor = waveform
|
||||
self.audio_data = {"waveform": waveform, "sample_rate": int(self.sample_rate)}
|
||||
|
||||
except Exception as e:
|
||||
# if issue with libtorchcodec "Could not load libtorchcodec"
|
||||
raise Exception(f"** WARNING ** - Error Processing audio for {self.path}. Error: {e}")
|
||||
|
||||
|
||||
class ImageProcessingDTOMixin:
|
||||
def load_and_process_video(
|
||||
@@ -772,6 +799,9 @@ class ImageProcessingDTOMixin:
|
||||
if self.has_unconditional:
|
||||
self.load_unconditional_image()
|
||||
return
|
||||
if self.is_audio_model:
|
||||
self.load_and_process_audio()
|
||||
return
|
||||
if self.dataset_config.num_frames > 1 or self.dataset_config.auto_frame_count:
|
||||
self.load_and_process_video(transform, only_load_latents)
|
||||
return
|
||||
@@ -1852,6 +1882,8 @@ class LatentCachingMixin:
|
||||
self.latent_cache = {}
|
||||
|
||||
def cache_latents_all_latents(self: 'AiToolkitDataset'):
|
||||
if self.is_audio_model:
|
||||
raise Exception("Audio models are not supported for latent caching yet")
|
||||
with accelerator.main_process_first():
|
||||
print_acc(f"Caching latents for {self.dataset_path}")
|
||||
# cache all latents to disk
|
||||
|
||||
Reference in new issue
Block a user