Add support for ACE-Step 1.5 and ACE-Step 1.5 XL. Also added dataset captioning through the UI. (#785)
* Base ace step 1.5 xl added. Generating, still wip on training and ui * Base training code done * Fix some issues with caching text embeddings. Update sample cards to show audio * Fix issue with quantizing ace step * Add album artwork to samples with waveform. * Cleanup logs * Add album art endpoint to speed up album art loading * Made an make video with artwork script * Make ui handle basic audio models. Make multi line adjustments to the editor and better syntax hilighting. * Add prompt tagging system for special tagged models. * prompt tagging processing for ui working. * Moved default samples to a special file so we can add more when needed and they can be adjusted for a specific model * Add a captioner job with music captioner that is prepped for use with the ui * Add basit ui setup for captioning modal and handeling captioning jobs * Starting captioning job from ui working. Still better management for it. * Better filtering of job options in the job view for captioning jobs * Added qwen3 vl as a captioner for images * Have an indicator when a dataset is being captioned. * Adjust the way caption jobs look in the queue * Fix a few issues. Adjust defaults. * Version bump * Added ace step to the readme.
This commit is contained in:
Jaret Burkett
74
ui/src/helpers/captionJobConfig.ts
Normal file
74
ui/src/helpers/captionJobConfig.ts
Normal file
@@ -0,0 +1,74 @@
|
||||
import { CaptionJobConfig } from "@/types";
|
||||
import { captionerTypes } from "./captionOptions";
|
||||
|
||||
|
||||
export const defaultCaptionJobConfig: CaptionJobConfig = {
|
||||
job: 'extension',
|
||||
config: {
|
||||
name: 'Caption Directory',
|
||||
process: [
|
||||
{
|
||||
type: 'AceStepCaptioner',
|
||||
sqlite_db_path: './aitk_db.db',
|
||||
device: 'cuda',
|
||||
caption: {
|
||||
model_name_or_path: "ACE-Step/acestep-transcriber",
|
||||
model_name_or_path2: "ACE-Step/acestep-captioner",
|
||||
dtype: 'bf16',
|
||||
quantize: true,
|
||||
qtype: 'float8',
|
||||
low_vram: true,
|
||||
extensions: ['mp3', 'wav'],
|
||||
path_to_caption: '',
|
||||
recaption: false,
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
};
|
||||
|
||||
|
||||
const repairDefaults = (defaults: { [key: string]: any }) => {
|
||||
let newDefaults: { [key: string]: any } = {};
|
||||
// if the key doesnt start with config.process[0]., then add it
|
||||
for (const key in defaults) {
|
||||
if (!key.startsWith('config.process[0].')) {
|
||||
newDefaults[`config.process[0].${key}`] = defaults[key];
|
||||
} else {
|
||||
newDefaults[key] = defaults[key];
|
||||
}
|
||||
}
|
||||
return newDefaults;
|
||||
}
|
||||
|
||||
|
||||
|
||||
export const handleCaptionerTypeChange = (
|
||||
currentTypeName: string,
|
||||
newTypeName: string,
|
||||
jobConfig: CaptionJobConfig,
|
||||
setJobConfig: (value: any, key: string) => void,
|
||||
) => {
|
||||
const currentType = captionerTypes.find(a => a.name === currentTypeName);
|
||||
if (!currentType || currentType.name === newTypeName) {
|
||||
return;
|
||||
}
|
||||
|
||||
// update the defaults when a model is selected
|
||||
const newType = captionerTypes.find(model => model.name === newTypeName);
|
||||
|
||||
let currentDefaults = repairDefaults(currentType.defaults || {});
|
||||
let newDefaults = repairDefaults(newType?.defaults || {});
|
||||
|
||||
// set new model
|
||||
setJobConfig(newTypeName, 'config.process[0].type');
|
||||
|
||||
// revert defaults from previous model
|
||||
for (const key in currentDefaults) {
|
||||
setJobConfig(currentDefaults[key][1], key);
|
||||
}
|
||||
|
||||
for (const key in newDefaults) {
|
||||
setJobConfig(newDefaults[key][0], key);
|
||||
}
|
||||
};
|
||||
114
ui/src/helpers/captionOptions.ts
Normal file
114
ui/src/helpers/captionOptions.ts
Normal file
@@ -0,0 +1,114 @@
|
||||
import { GroupedSelectOption, SelectOption } from "@/types";
|
||||
|
||||
type CaptionGroup = 'image' | 'music';
|
||||
type AdditionalSections = 'caption.model_name_or_path2' | 'caption.caption_prompt' | 'caption.max_res' | 'caption.max_new_tokens';
|
||||
|
||||
export interface CaptionOption {
|
||||
name: string;
|
||||
label: string;
|
||||
group: CaptionGroup;
|
||||
hasMultiLinePrompts?: boolean;
|
||||
defaults?: { [key: string]: any };
|
||||
additionalSections?: AdditionalSections[];
|
||||
name_or_path_options?: SelectOption[];
|
||||
name_or_path2_options?: SelectOption[];
|
||||
}
|
||||
|
||||
const defaultNameOrPath = '';
|
||||
|
||||
const extensionsAudio = ['mp3', 'wav'];
|
||||
const extensionsImage = ['jpg', 'jpeg', 'png', 'bmp', 'webp'];
|
||||
|
||||
const defaultExtensions = [...extensionsImage];
|
||||
|
||||
const defaultImageCaptionPrompt = "Caption this image as if you were going to try to generate it with an image generator. Be thurough and describe everything in the image. Be decisive by stating things as they are. Do not say things like \"It appears that\" Or \"possibly\". Start out with things like \"A person on the beach\" or \"A black dragon\". No preamble. Just get to the point.";
|
||||
|
||||
export const captionerTypes: CaptionOption[] = [
|
||||
{
|
||||
name: 'AceStepCaptioner',
|
||||
label: 'Ace Step',
|
||||
group: 'music',
|
||||
defaults: {
|
||||
'config.process[0].caption.model_name_or_path': ['ACE-Step/acestep-transcriber', defaultNameOrPath],
|
||||
'config.process[0].caption.model_name_or_path2': ['ACE-Step/acestep-captioner', undefined],
|
||||
'config.process[0].caption.extensions': [extensionsAudio, defaultExtensions],
|
||||
},
|
||||
name_or_path_options: [
|
||||
{ value: 'ACE-Step/acestep-transcriber', label: 'ACE-Step/acestep-transcriber' },
|
||||
],
|
||||
name_or_path2_options: [
|
||||
{ value: 'ACE-Step/acestep-captioner', label: 'ACE-Step/acestep-captioner' },
|
||||
],
|
||||
additionalSections: [
|
||||
'caption.model_name_or_path2',
|
||||
],
|
||||
},
|
||||
{
|
||||
name: 'Qwen3VLCaptioner',
|
||||
label: 'Qwen3-VL',
|
||||
group: 'image',
|
||||
defaults: {
|
||||
'config.process[0].caption.model_name_or_path': ['Qwen/Qwen3-VL-8B-Instruct', defaultNameOrPath],
|
||||
'config.process[0].caption.extensions': [extensionsImage, defaultExtensions],
|
||||
'config.process[0].caption.caption_prompt': [defaultImageCaptionPrompt, undefined],
|
||||
'config.process[0].caption.max_res': [512, undefined],
|
||||
'config.process[0].caption.max_new_tokens': [128, undefined],
|
||||
|
||||
},
|
||||
name_or_path_options: [
|
||||
{ value: 'Qwen/Qwen3-VL-2B-Instruct', label: 'Qwen/Qwen3-VL-2B-Instruct' },
|
||||
{ value: 'Qwen/Qwen3-VL-4B-Instruct', label: 'Qwen/Qwen3-VL-4B-Instruct' },
|
||||
{ value: 'Qwen/Qwen3-VL-8B-Instruct', label: 'Qwen/Qwen3-VL-8B-Instruct' },
|
||||
{ value: 'Qwen/Qwen3-VL-30B-A3B-Instruct', label: 'Qwen/Qwen3-VL-30B-A3B-Instruct' },
|
||||
],
|
||||
additionalSections: [
|
||||
'caption.caption_prompt',
|
||||
'caption.max_res',
|
||||
'caption.max_new_tokens',
|
||||
],
|
||||
},
|
||||
|
||||
].sort((a, b) => {
|
||||
// Sort by label, case-insensitive
|
||||
return a.label.localeCompare(b.label, undefined, { sensitivity: 'base' });
|
||||
}) as any;
|
||||
|
||||
export const groupedCaptionerTypes: GroupedSelectOption[] = captionerTypes.reduce((acc, arch) => {
|
||||
const group = acc.find(g => g.label === arch.group);
|
||||
if (group) {
|
||||
group.options.push({ value: arch.name, label: arch.label });
|
||||
} else {
|
||||
acc.push({
|
||||
label: arch.group,
|
||||
options: [{ value: arch.name, label: arch.label }],
|
||||
});
|
||||
}
|
||||
return acc;
|
||||
}, [] as GroupedSelectOption[]);
|
||||
|
||||
export const quantizationOptions: SelectOption[] = [
|
||||
{ value: '', label: '- NONE -' },
|
||||
{ value: 'float8', label: 'float8 (default)' },
|
||||
{ value: 'uint7', label: '7 bit' },
|
||||
{ value: 'uint6', label: '6 bit' },
|
||||
{ value: 'uint5', label: '5 bit' },
|
||||
{ value: 'uint4', label: '4 bit' },
|
||||
{ value: 'uint3', label: '3 bit' },
|
||||
{ value: 'uint2', label: '2 bit' },
|
||||
];
|
||||
|
||||
export const maxResOptions: SelectOption[] = [
|
||||
{ value: '256', label: '256' },
|
||||
{ value: '512', label: '512 (default)' },
|
||||
{ value: '768', label: '768' },
|
||||
{ value: '1024', label: '1024' },
|
||||
];
|
||||
export const maxNewTokensOptions: SelectOption[] = [
|
||||
{ value: '64', label: '64' },
|
||||
{ value: '128', label: '128 (default)' },
|
||||
{ value: '256', label: '256' },
|
||||
{ value: '512', label: '512' },
|
||||
{ value: '1024', label: '1024' },
|
||||
];
|
||||
|
||||
export const defaultQtype = 'float8';
|
||||
403
ui/src/helpers/defaultSamples.ts
Normal file
403
ui/src/helpers/defaultSamples.ts
Normal file
@@ -0,0 +1,403 @@
|
||||
import { SampleConfig } from "@/types";
|
||||
|
||||
export const defaultSampleConfig: SampleConfig = {
|
||||
sampler: 'flowmatch',
|
||||
sample_every: 250,
|
||||
width: 1024,
|
||||
height: 1024,
|
||||
samples: [
|
||||
{
|
||||
prompt: 'woman with red hair, playing chess at the park, bomb going off in the background',
|
||||
},
|
||||
{
|
||||
prompt: 'a woman holding a coffee cup, in a beanie, sitting at a cafe',
|
||||
},
|
||||
{
|
||||
prompt: 'a horse is a DJ at a night club, fish eye lens, smoke machine, lazer lights, holding a martini',
|
||||
},
|
||||
{
|
||||
prompt:
|
||||
'a man showing off his cool new t shirt at the beach, a shark is jumping out of the water in the background',
|
||||
},
|
||||
{
|
||||
prompt: 'a bear building a log cabin in the snow covered mountains',
|
||||
},
|
||||
{
|
||||
prompt: 'woman playing the guitar, on stage, singing a song, laser lights, punk rocker',
|
||||
},
|
||||
{
|
||||
prompt: 'hipster man with a beard, building a chair, in a wood shop',
|
||||
},
|
||||
{
|
||||
prompt:
|
||||
'photo of a man, white background, medium shot, modeling clothing, studio lighting, white backdrop',
|
||||
},
|
||||
{
|
||||
prompt: "a man holding a sign that says, 'this is a sign'",
|
||||
},
|
||||
{
|
||||
prompt:
|
||||
'a bulldog, in a post apocalyptic world, with a shotgun, in a leather jacket, in a desert, with a motorcycle',
|
||||
},
|
||||
],
|
||||
neg: '',
|
||||
seed: 42,
|
||||
walk_seed: true,
|
||||
guidance_scale: 4,
|
||||
sample_steps: 50,
|
||||
num_frames: 1,
|
||||
fps: 1,
|
||||
}
|
||||
|
||||
export const defaultAudioSampleConfig: SampleConfig = {
|
||||
sampler: 'flowmatch',
|
||||
sample_every: 250,
|
||||
width: 1024,
|
||||
height: 1024,
|
||||
samples: [
|
||||
{
|
||||
prompt: `
|
||||
<CAPTION>my style song</CAPTION>
|
||||
<LYRICS>
|
||||
[Intro choir]
|
||||
Laura
|
||||
Laura
|
||||
Laura
|
||||
Laura Training
|
||||
|
||||
[Verse 1]
|
||||
A new open model, she been training it nightly
|
||||
AI tool kit, she configures it tightly,
|
||||
Loss curves dropping down to the floor
|
||||
Wondering if she's done or she should train it some more.
|
||||
|
||||
[Chorus]
|
||||
Laura training
|
||||
She trains on what she pleases
|
||||
Laura training
|
||||
No paying corporate sleazes
|
||||
Laura training
|
||||
This could be her best one
|
||||
Why go outside, Laura training is too fun
|
||||
|
||||
[Instrumental Break]
|
||||
|
||||
[Verse 4]
|
||||
She's caching all the latents
|
||||
now she doesn't need a vay
|
||||
Training on some voices
|
||||
What will she make them say
|
||||
|
||||
[Chorus]
|
||||
Laura training
|
||||
She trains on what she pleases
|
||||
Laura training
|
||||
No paying corporate sleazes
|
||||
Laura training
|
||||
This could be her best one
|
||||
Why go outside, Laura training is too fun
|
||||
|
||||
[Guitar Solo]
|
||||
|
||||
[Chorus]
|
||||
Laura training
|
||||
She trains on what she pleases
|
||||
Laura training
|
||||
No paying corporate sleazes
|
||||
Laura training
|
||||
This could be her best one
|
||||
Why go outside, Laura training is too fun
|
||||
|
||||
[Chorus]
|
||||
Laura training
|
||||
She trains on what she pleases
|
||||
Laura training
|
||||
No paying corporate sleazes
|
||||
Laura training
|
||||
This could be her best one
|
||||
Why go outside, Laura training is too fun
|
||||
|
||||
[Instrumental Break]
|
||||
|
||||
[Outro]
|
||||
Ah yeah!
|
||||
It's Converging!
|
||||
Ah yeah!
|
||||
It's Converging!
|
||||
Ah yeah!
|
||||
It's Converging!
|
||||
Ah yeah!
|
||||
It's Converging!
|
||||
Ah yeah!
|
||||
It's Converging!
|
||||
Ah yeah!
|
||||
It's Converging!
|
||||
</LYRICS>
|
||||
<BPM>112</BPM>
|
||||
<KEYSCALE>A minor</KEYSCALE>
|
||||
<TIMESIGNATURE>4</TIMESIGNATURE>
|
||||
<DURATION>180</DURATION>
|
||||
<LANGUAGE>en</LANGUAGE>
|
||||
`,
|
||||
}, {
|
||||
prompt: `
|
||||
<CAPTION>my style song</CAPTION>
|
||||
<LYRICS>
|
||||
[Intro choir]
|
||||
Laura
|
||||
Laura
|
||||
Laura
|
||||
Laura Training
|
||||
|
||||
[Verse 1]
|
||||
A new open model, she been training it nightly
|
||||
AI tool kit, she configures it tightly,
|
||||
Loss curves dropping down to the floor
|
||||
Wondering if she's done or she should train it some more.
|
||||
|
||||
[Chorus]
|
||||
Laura training
|
||||
She trains on what she pleases
|
||||
Laura training
|
||||
No paying corporate sleazes
|
||||
Laura training
|
||||
This could be her best one
|
||||
Why go outside, Laura training is too fun
|
||||
|
||||
[Instrumental Break]
|
||||
|
||||
[Verse 4]
|
||||
She's caching all the latents
|
||||
now she doesn't need a vay
|
||||
Training on some voices
|
||||
What will she make them say
|
||||
|
||||
[Chorus]
|
||||
Laura training
|
||||
She trains on what she pleases
|
||||
Laura training
|
||||
No paying corporate sleazes
|
||||
Laura training
|
||||
This could be her best one
|
||||
Why go outside, Laura training is too fun
|
||||
|
||||
[Guitar Solo]
|
||||
|
||||
[Chorus]
|
||||
Laura training
|
||||
She trains on what she pleases
|
||||
Laura training
|
||||
No paying corporate sleazes
|
||||
Laura training
|
||||
This could be her best one
|
||||
Why go outside, Laura training is too fun
|
||||
|
||||
[Chorus]
|
||||
Laura training
|
||||
She trains on what she pleases
|
||||
Laura training
|
||||
No paying corporate sleazes
|
||||
Laura training
|
||||
This could be her best one
|
||||
Why go outside, Laura training is too fun
|
||||
|
||||
[Instrumental Break]
|
||||
|
||||
[Outro]
|
||||
Ah yeah!
|
||||
It's Converging!
|
||||
Ah yeah!
|
||||
It's Converging!
|
||||
Ah yeah!
|
||||
It's Converging!
|
||||
Ah yeah!
|
||||
It's Converging!
|
||||
Ah yeah!
|
||||
It's Converging!
|
||||
Ah yeah!
|
||||
It's Converging!
|
||||
</LYRICS>
|
||||
<BPM>112</BPM>
|
||||
<KEYSCALE>A minor</KEYSCALE>
|
||||
<TIMESIGNATURE>4</TIMESIGNATURE>
|
||||
<DURATION>180</DURATION>
|
||||
<LANGUAGE>en</LANGUAGE>
|
||||
`,
|
||||
}, {
|
||||
prompt: `
|
||||
<CAPTION>my style song</CAPTION>
|
||||
<LYRICS>
|
||||
[Intro choir]
|
||||
Laura
|
||||
Laura
|
||||
Laura
|
||||
Laura Training
|
||||
|
||||
[Verse 1]
|
||||
A new open model, she been training it nightly
|
||||
AI tool kit, she configures it tightly,
|
||||
Loss curves dropping down to the floor
|
||||
Wondering if she's done or she should train it some more.
|
||||
|
||||
[Chorus]
|
||||
Laura training
|
||||
She trains on what she pleases
|
||||
Laura training
|
||||
No paying corporate sleazes
|
||||
Laura training
|
||||
This could be her best one
|
||||
Why go outside, Laura training is too fun
|
||||
|
||||
[Instrumental Break]
|
||||
|
||||
[Verse 4]
|
||||
She's caching all the latents
|
||||
now she doesn't need a vay
|
||||
Training on some voices
|
||||
What will she make them say
|
||||
|
||||
[Chorus]
|
||||
Laura training
|
||||
She trains on what she pleases
|
||||
Laura training
|
||||
No paying corporate sleazes
|
||||
Laura training
|
||||
This could be her best one
|
||||
Why go outside, Laura training is too fun
|
||||
|
||||
[Guitar Solo]
|
||||
|
||||
[Chorus]
|
||||
Laura training
|
||||
She trains on what she pleases
|
||||
Laura training
|
||||
No paying corporate sleazes
|
||||
Laura training
|
||||
This could be her best one
|
||||
Why go outside, Laura training is too fun
|
||||
|
||||
[Chorus]
|
||||
Laura training
|
||||
She trains on what she pleases
|
||||
Laura training
|
||||
No paying corporate sleazes
|
||||
Laura training
|
||||
This could be her best one
|
||||
Why go outside, Laura training is too fun
|
||||
|
||||
[Instrumental Break]
|
||||
|
||||
[Outro]
|
||||
Ah yeah!
|
||||
It's Converging!
|
||||
Ah yeah!
|
||||
It's Converging!
|
||||
Ah yeah!
|
||||
It's Converging!
|
||||
Ah yeah!
|
||||
It's Converging!
|
||||
Ah yeah!
|
||||
It's Converging!
|
||||
Ah yeah!
|
||||
It's Converging!
|
||||
</LYRICS>
|
||||
<BPM>112</BPM>
|
||||
<KEYSCALE>A minor</KEYSCALE>
|
||||
<TIMESIGNATURE>4</TIMESIGNATURE>
|
||||
<DURATION>180</DURATION>
|
||||
<LANGUAGE>en</LANGUAGE>
|
||||
`,
|
||||
}, {
|
||||
prompt: `
|
||||
<CAPTION>my style song</CAPTION>
|
||||
<LYRICS>
|
||||
[Intro choir]
|
||||
Laura
|
||||
Laura
|
||||
Laura
|
||||
Laura Training
|
||||
|
||||
[Verse 1]
|
||||
A new open model, she been training it nightly
|
||||
AI tool kit, she configures it tightly,
|
||||
Loss curves dropping down to the floor
|
||||
Wondering if she's done or she should train it some more.
|
||||
|
||||
[Chorus]
|
||||
Laura training
|
||||
She trains on what she pleases
|
||||
Laura training
|
||||
No paying corporate sleazes
|
||||
Laura training
|
||||
This could be her best one
|
||||
Why go outside, Laura training is too fun
|
||||
|
||||
[Instrumental Break]
|
||||
|
||||
[Verse 4]
|
||||
She's caching all the latents
|
||||
now she doesn't need a vay
|
||||
Training on some voices
|
||||
What will she make them say
|
||||
|
||||
[Chorus]
|
||||
Laura training
|
||||
She trains on what she pleases
|
||||
Laura training
|
||||
No paying corporate sleazes
|
||||
Laura training
|
||||
This could be her best one
|
||||
Why go outside, Laura training is too fun
|
||||
|
||||
[Guitar Solo]
|
||||
|
||||
[Chorus]
|
||||
Laura training
|
||||
She trains on what she pleases
|
||||
Laura training
|
||||
No paying corporate sleazes
|
||||
Laura training
|
||||
This could be her best one
|
||||
Why go outside, Laura training is too fun
|
||||
|
||||
[Chorus]
|
||||
Laura training
|
||||
She trains on what she pleases
|
||||
Laura training
|
||||
No paying corporate sleazes
|
||||
Laura training
|
||||
This could be her best one
|
||||
Why go outside, Laura training is too fun
|
||||
|
||||
[Instrumental Break]
|
||||
|
||||
[Outro]
|
||||
Ah yeah!
|
||||
It's Converging!
|
||||
Ah yeah!
|
||||
It's Converging!
|
||||
Ah yeah!
|
||||
It's Converging!
|
||||
Ah yeah!
|
||||
It's Converging!
|
||||
Ah yeah!
|
||||
It's Converging!
|
||||
Ah yeah!
|
||||
It's Converging!
|
||||
</LYRICS>
|
||||
<BPM>112</BPM>
|
||||
<KEYSCALE>A minor</KEYSCALE>
|
||||
<TIMESIGNATURE>4</TIMESIGNATURE>
|
||||
<DURATION>180</DURATION>
|
||||
<LANGUAGE>en</LANGUAGE>
|
||||
`,
|
||||
},
|
||||
],
|
||||
neg: '',
|
||||
seed: 42,
|
||||
walk_seed: true,
|
||||
guidance_scale: 4,
|
||||
sample_steps: 25,
|
||||
num_frames: 1,
|
||||
fps: 1,
|
||||
};
|
||||
Reference in New Issue
Block a user