Let omni captioner handle just images as well. Set it as the new default captioner.

This commit is contained in:
Jaret Burkett
2026-08-13 10:30:17 -06:00
parent 6b7fb60a22
commit ab18528fdb
3 changed files with 97 additions and 118 deletions

View File

@@ -8,21 +8,25 @@ export const defaultCaptionJobConfig: CaptionJobConfig = {
name: 'Caption Directory',
process: [
{
type: 'AceStepCaptioner',
type: 'Qwen3OmniCaptioner',
sqlite_db_path: './aitk_db.db',
device: 'cuda',
caption: {
model_name_or_path: "ACE-Step/acestep-transcriber",
model_name_or_path2: "ACE-Step/acestep-captioner",
model_name_or_path: 'ai-toolkit/Qwen3-Omni-30B-A3B-Thinking',
dtype: 'bf16',
quantize: true,
qtype: 'convrot8',
low_vram: true,
extensions: ['mp3', 'wav', 'flac', 'ogg'],
low_vram: false,
extensions: ['mp4', 'mov', 'webm', 'mkv', 'avi', 'jpg', 'jpeg', 'png', 'bmp', 'webp'],
path_to_caption: '',
recaption: false,
compile: false,
compile: true,
caption_extension: 'txt',
caption_prompt:
"Caption this video as if you were going to try to generate it with a video generator. Describe the visual content, how it moves and changes over time, and the camera work. Also describe the audio, including any speech, music, or sound effects, and transcribe spoken dialogue verbatim in quotes. Be decisive by stating things as they are. Do not say things like \"It appears that\" Or \"possibly\". No preamble. Just get to the point.",
max_res: 512,
max_new_tokens: 512,
batch_size: 1,
},
},
],

View File

@@ -1,6 +1,6 @@
import { GroupedSelectOption, SelectOption } from "@/types";
type CaptionGroup = 'image' | 'music' | 'video';
type CaptionGroup = 'image' | 'music' | 'video' | 'image/video/sound';
type AdditionalSections = 'caption.model_name_or_path2' | 'caption.caption_prompt' | 'caption.max_res' | 'caption.max_new_tokens' | 'caption.fixed_caption' | 'caption.thinking' | 'caption.batch_size' | 'caption.layer_offloading';
export interface CaptionOption {
@@ -51,6 +51,22 @@ non_diegetic_music: 1-3 sentences describing background music the characters can
Describe only what is actually seen and heard. Be decisive. No preamble and no extra text - output only the three fields.`;
// The T2VA format applied to a still image: one static [Shot 1], no timeline,
// no audio fields beyond the required N/A placeholders.
const minimaxImageCaptionPrompt = `Caption this image as a MiniMax training prompt for a single still frame. Output exactly three fields in this order, each starting on its own line with these exact field names:
integrated_multimodal_description: [Shot 1] ...
overall_soundscape: N/A
non_diegetic_music: N/A
Rules for integrated_multimodal_description: Write a single [Shot 1] with no timestamp, no cuts, and no camera motion - the camera holds a static shot on a still frame. State the overall visual style first (Live-action, cinematic, 2D-animated, 3D CG, claymation, watercolor, or vintage film) and the framing (extreme wide shot, wide shot, medium shot, medium close-up, close-up, or extreme close-up, plus the viewpoint). Then describe everything visible decisively and specifically: subject appearance, pose, and position; clothing, colors, and materials; the scene, lighting, and key props; and the spatial relationships between them. There is no motion, no dialogue, and no sound - do not invent any. Put any legible text in the image (signs, labels, packaging) in double quotation marks verbatim without translating it.
overall_soundscape and non_diegetic_music are always exactly N/A for a still image.
Describe only what is actually visible. Be decisive. No preamble and no extra text - output only the three fields.`;
// Editable ADDITIONAL INSTRUCTIONS block injected into the Ideogram system prompt.
// Users can tweak this for dataset-specific guidance without altering the fixed
// output contract, element/background rules, or bbox format.
@@ -108,10 +124,10 @@ export const captionerTypes: CaptionOption[] = [
{
name: 'Qwen3OmniCaptioner',
label: 'Qwen3-Omni',
group: 'video',
group: 'image/video/sound',
defaults: {
'config.process[0].caption.model_name_or_path': ['ai-toolkit/Qwen3-Omni-30B-A3B-Instruct', defaultNameOrPath],
'config.process[0].caption.extensions': [extensionsVideo, defaultExtensions],
'config.process[0].caption.model_name_or_path': ['ai-toolkit/Qwen3-Omni-30B-A3B-Thinking', defaultNameOrPath],
'config.process[0].caption.extensions': [[...extensionsVideo, ...extensionsImage], defaultExtensions],
'config.process[0].caption.caption_prompt': [defaultVideoCaptionPrompt, undefined],
'config.process[0].caption.max_res': [512, undefined],
'config.process[0].caption.max_new_tokens': [512, undefined],
@@ -126,6 +142,7 @@ export const captionerTypes: CaptionOption[] = [
captionPrompts: {
'General': defaultVideoCaptionPrompt,
'MiniMax H4 T2V': minimaxT2VCaptionPrompt,
'MiniMax H4 Image': minimaxImageCaptionPrompt,
},
additionalSections: [
'caption.caption_prompt',