github/ai-toolkit
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Add caption prompt template picker to the ui. A few styling improvements thrown in for a good time.

Browse Source
This commit is contained in:
Jaret Burkett
2026-08-12 21:23:28 -06:00
parent ca42a72f4c
commit 81adcc2176
4 changed files with 84 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
ui/src/app/datasets/[datasetName]/page.tsx
View File

@@ -159,7 +159,12 @@ export default function DatasetPage({ params }: { params: { datasetName: string
</Button>
</div>
</TopBar>
<MainContent ref={scrollParentCallback}>
<MainContent
ref={scrollParentCallback}
belowTopBar
className="transition-[bottom] duration-300"
style={{ bottom: `${captionBarHeight}px` }}
>
{PageInfoContent}
{status === 'success' && imgList.length > 0 && scrollParent && (
<VirtuosoGrid
@@ -186,9 +191,9 @@ export default function DatasetPage({ params }: { params: { datasetName: string
computeItemKey={index => imgList[index]?.img_path ?? index}
/>
)}
{/* Spacer so the last cards stay accessible above the floating caption bar.
Always keeps a baseline gap, plus the bar height when it is showing. */}
<div style={{ height: `${captionBarHeight + 24}px` }} className="transition-[height] duration-300" />
{/* Baseline gap below the last row of cards. The caption bar itself is handled by
shrinking MainContent's bottom to the bar height, so no dynamic spacer is needed. */}
<div className="h-6" />
</MainContent>
<AddImagesModal />
{isSettingsLoaded && (

23
ui/src/components/CaptionSimpleJob.tsx
View File

@@ -31,6 +31,8 @@ type Props = {
const CaptionSimpleJob: React.FC<Props> = ({ jobConfig, setJobConfig, gpuIDs, setGpuIDs, gpuList, showGPUSelect }) => {
const selectedCaptionOption = captionerTypes.find(option => option.name === jobConfig.config.process[0].type);
const additionalSections = selectedCaptionOption?.additionalSections || [];
const captionPrompts = selectedCaptionOption?.captionPrompts || {};
const promptPresetNames = Object.keys(captionPrompts);
const minNewTokens = selectedCaptionOption?.minNewTokens ?? 0;
const newTokensOptions = maxNewTokensOptions.filter(option => parseInt(option.value) >= minNewTokens);
@@ -211,6 +213,27 @@ const CaptionSimpleJob: React.FC<Props> = ({ jobConfig, setJobConfig, gpuIDs, se
</div>
{additionalSections.includes('caption.caption_prompt') && (
<div className="mt-4">
{promptPresetNames.length > 1 && (
<div className="mb-4">
<SelectInput
label="Prompt Preset"
value={
promptPresetNames.find(
name => captionPrompts[name] === jobConfig.config.process[0].caption.caption_prompt,
) || ''
}
onChange={value => {
if (captionPrompts[value] !== undefined) {
setJobConfig(captionPrompts[value], 'config.process[0].caption.caption_prompt');
}
}}
options={[
{ value: '', label: '- Custom -' },
...promptPresetNames.map(name => ({ value: name, label: name })),
]}
/>
</div>
)}
<TextAreaInput
label="Caption Prompt"
value={jobConfig.config.process[0].caption.caption_prompt || ''}

33
ui/src/components/layout.tsx
View File

@@ -6,6 +6,7 @@ import { mobileSidebarState } from './Sidebar';
interface Props {
className?: string;
style?: React.CSSProperties;
children?: React.ReactNode;
}
@@ -36,14 +37,26 @@ export const TopBar: React.FC<Props> = ({ children, className }) => {
);
};
export const MainContent = React.forwardRef<HTMLDivElement, Props>(({ children, className }, ref) => {
return (
<div
ref={ref}
className={classNames('pt-14 px-2 sm:px-4 absolute top-0 left-0 w-full h-full overflow-auto', className)}
>
{children ? children : null}
</div>
);
});
interface MainContentProps extends Props {
/** Start the scroll container below the fixed TopBar so the scrollbar doesn't run behind it. */
belowTopBar?: boolean;
}
export const MainContent = React.forwardRef<HTMLDivElement, MainContentProps>(
({ children, className, style, belowTopBar }, ref) => {
return (
<div
ref={ref}
style={style}
className={classNames(
'px-2 sm:px-4 absolute left-0 w-full overflow-auto',
belowTopBar ? 'pt-2 top-12 bottom-0' : 'pt-14 top-0 h-full',
className,
)}
>
{children ? children : null}
</div>
);
},
);
MainContent.displayName = 'MainContent';

30
ui/src/helpers/captionOptions.ts
View File

@@ -13,6 +13,9 @@ export interface CaptionOption {
additionalSections?: AdditionalSections[];
name_or_path_options?: SelectOption[];
name_or_path2_options?: SelectOption[];
// named caption prompts the user can swap between; the picker only shows
// when there is more than one, and selecting one just fills caption_prompt
captionPrompts?: { [name: string]: string };
}
const defaultNameOrPath = '';
@@ -27,6 +30,27 @@ const defaultImageCaptionPrompt = "Caption this image as if you were going to tr
const defaultVideoCaptionPrompt = "Caption this video as if you were going to try to generate it with a video generator. Describe the visual content, how it moves and changes over time, and the camera work. Also describe the audio, including any speech, music, or sound effects, and transcribe spoken dialogue verbatim in quotes. Be decisive by stating things as they are. Do not say things like \"It appears that\" Or \"possibly\". No preamble. Just get to the point.";
// Captions videos as MiniMax T2VA training prompts, following the official
// video prompt writing guide (MiniMaxAI/MiniMax-H3 docs/VIDEO_PROMPT_WRITING_GUIDE_base_en.md):
// three fields, [Shot N] timeline with cut timestamps, controlled camera-motion
// vocabulary, (S1) speaker IDs with verbatim <d>[Language] ...</d> dialogue,
// quoted on-screen text, and separate soundscape / non-diegetic music fields.
const minimaxT2VCaptionPrompt = `Caption this video as a MiniMax text-to-video-audio (T2VA) training prompt. Watch the video and listen to the audio, then output exactly three fields in this order, each starting on its own line with these exact field names:
integrated_multimodal_description: [Shot 1] ...
overall_soundscape: ...
non_diegetic_music: ...
Rules for integrated_multimodal_description: Start with [Shot 1] (the first shot gets no timestamp) and state the overall visual style (Live-action, cinematic, 2D-animated, 3D CG, claymation, watercolor, or vintage film) and the initial framing. Then describe everything visible and audible in chronological order: subject appearance and position, scene and key props, actions and reactions, shot changes, speech, and the diegetic sounds that accompany them. Begin each later shot as "[Shot 2] At 00:03.500, the camera cuts to ..." with strictly increasing cut times within the video. Write camera motion as a natural action using this vocabulary: zoom in/out, push in, pull out, pan left/right, truck left/right, tilt up/down, pedestal up/down, arc shot, tracking shot, static shot, POV, roll clockwise/counterclockwise, shake slightly/strongly - adding "with small amplitude"/"with large amplitude" and "at slow speed"/"at fast speed" only when the range or pacing is notable. Give every person who speaks or sings a stable ID like (S1) or (S2); when a speaker first appears, identify them with type, age, gender, and voice quality (pitch, timbre, pace, accent). Transcribe all speech and lyrics verbatim in this form: The young woman with a quiet, breathy voice (S1) says: <d>[English] I get off at the next station.</d> - the language tag names the spoken language and only the exact spoken words go inside <d></d>. For narration with no visible lip movement, write "says in an off-screen voiceover:" before the <d> block and state afterwards that the character's lips remain completely closed. Put any legible on-screen text (signs, labels, subtitles) in double quotation marks verbatim without translating it.
overall_soundscape: 1-4 sentences in one paragraph summarizing the ambient sound, physical action sounds, and non-verbal human sounds across the whole video (wind, traffic, footsteps, fabric, impacts, breathing, laughter). Do not repeat dialogue, singing, or diegetic music here. Use N/A only if the video is completely silent.
non_diegetic_music: 1-3 sentences describing background music the characters cannot hear: instrumentation, tempo, rhythm, and dynamic changes, with no abstract mood words. Music coming from a source inside the scene belongs in the multimodal description instead. Use N/A if there is none.
Describe only what is actually seen and heard. Be decisive. No preamble and no extra text - output only the three fields.`;
// Editable ADDITIONAL INSTRUCTIONS block injected into the Ideogram system prompt.
// Users can tweak this for dataset-specific guidance without altering the fixed
// output contract, element/background rules, or bbox format.
@@ -90,13 +114,17 @@ export const captionerTypes: CaptionOption[] = [
'config.process[0].caption.extensions': [extensionsVideo, defaultExtensions],
'config.process[0].caption.caption_prompt': [defaultVideoCaptionPrompt, undefined],
'config.process[0].caption.max_res': [512, undefined],
'config.process[0].caption.max_new_tokens': [256, undefined],
'config.process[0].caption.max_new_tokens': [512, undefined],
'config.process[0].caption.batch_size': [1, undefined],
'config.process[0].caption.compile': [true, false],
},
name_or_path_options: [
{ value: 'ostris/Qwen3-Omni-30B-A3B-Instruct', label: 'ostris/Qwen3-Omni-30B-A3B-Instruct' },
],
captionPrompts: {
'General': defaultVideoCaptionPrompt,
'MiniMax H4 T2V': minimaxT2VCaptionPrompt,
},
additionalSections: [
'caption.caption_prompt',
'caption.max_res',