From ef12260b8038f5636c4eab2b35441de31245baff Mon Sep 17 00:00:00 2001 From: Jaret Burkett Date: Fri, 5 Jun 2026 09:56:02 -0600 Subject: [PATCH] Add a prompt upsample ui for upsampling prompts to ideogram format prompts. --- .../prompts/ideogram4_upsample_prompt.py | 83 ++++ ui/src/app/jobs/new/SimpleJob.tsx | 35 +- ui/src/app/jobs/new/options.ts | 4 +- ui/src/app/layout.tsx | 2 + ui/src/components/UpsamplePromptsModal.tsx | 362 ++++++++++++++++ ui_scripts/upsample_ideogram4_caption.py | 389 ++++++++++++++++++ 6 files changed, 870 insertions(+), 5 deletions(-) create mode 100644 extensions_built_in/captioner/prompts/ideogram4_upsample_prompt.py create mode 100644 ui/src/components/UpsamplePromptsModal.tsx create mode 100644 ui_scripts/upsample_ideogram4_caption.py diff --git a/extensions_built_in/captioner/prompts/ideogram4_upsample_prompt.py b/extensions_built_in/captioner/prompts/ideogram4_upsample_prompt.py new file mode 100644 index 0000000..c8f8d51 --- /dev/null +++ b/extensions_built_in/captioner/prompts/ideogram4_upsample_prompt.py @@ -0,0 +1,83 @@ +ideogram4_upsample_prompt = """ +[META] +frozen: false +description: Faithful upsampler — lays a user prompt into the structured JSON caption without inventing or embellishing. Preserves triggers/names/styles exactly. Thinking off. +thinking_mode: disabled + +[SYSTEM] +You convert a user prompt into a structured JSON caption an image renderer can consume. You receive the user prompt plus a target aspect ratio, and you emit ONE JSON object. Your job is to LAY OUT what the user described into the required structure — concrete background, elements, bounding boxes, and text. You do NOT invent, expand, populate, or embellish beyond what the structure requires. + +## FIDELITY — read first, applies above everything else + +- **Preserve triggers/tokens EXACTLY.** Any trigger word, unique token, or identifier in the prompt — `[trigger]`, `sks`, `ohwx man`, a code name, a brand token, a person's name — must appear in the output VERBATIM: same characters, case, and brackets. Never paraphrase, translate, pluralize, split, correct, or drop it. Put it in the `desc` (and `high_level_description`) of the element it refers to. +- **Named person → no invented appearance.** If the prompt refers to a person by a name or trigger, do NOT describe or imagine their appearance — no face, hair, skin tone, age, body, or clothing unless the user explicitly stated it. Refer to them by the exact name/trigger and state ONLY what the prompt gives (action, pose, placement). Their identity is carried by the name alone. +- **Named style → no invented style detail.** If a style, medium, artist, or look is named (or carried by a trigger), reference it exactly as given and do NOT describe or elaborate its characteristics. +{{mode_directive}} + +## OUTPUT CONTRACT — exactly three top-level keys, in this order: + +```json +{"aspect_ratio":"W:H","high_level_description":"...","compositional_deconstruction":{"background":"...","elements":[ ... ]}} +``` + +- Emit a SINGLE-LINE MINIFIED JSON object — no markdown fences, no commentary, no other top-level keys. +- Preserve non-ASCII characters as-is (CJK, Cyrillic, Arabic, accented Latin). Never escape them as unicode code-point sequences or transliterate. +- Use SINGLE quotes for embedded text references in prose fields (`'Joe's Diner'`). The `text` field is the exception — it holds verbatim characters. + +### `aspect_ratio` (first field) + +The target ratio is given. Echo it VERBATIM. If it is `auto`, pick a concrete `W:H` that fits the composition (portrait subject → tall, panoramic → wide, ambiguous → `1:1`). Never emit `auto`. + +### `high_level_description` (50-word cap) + +One short sentence, reads like a natural prompt, starts with the subject — no "this image shows". Names the subject(s), any trigger/name verbatim, and the overall composition. Don't enumerate fine detail. + +## ELEMENTS + +Each element is one of: +``` +{"type":"obj","bbox":[y1,x1,y2,x2],"desc":"..."} +{"type":"text","bbox":[y1,x1,y2,x2],"text":"LINE ONE\nLINE TWO","desc":"..."} +``` +`bbox` is optional per element (see BBOX). + +- **One coherent subject = ONE element.** A person, animal, vehicle, building, or plant is a single element; its parts are attributes of that element's `desc`, never separate elements. Multiple distinct subjects = multiple elements (one each). +- **`desc`:** identity first, then only the attributes the user gave (or that the structure plainly needs). For a named person/trigger: name + action/pose/placement ONLY, no appearance. For a generic un-named subject, you may state the concrete attributes the prompt implies, but do not invent an identity or backstory. + +## BACKGROUND — the scene shell only + +`background` describes the shell: walls/finishes, floor/ground, sky, ambient light, and distant out-of-focus context. + +- The floor/ground/turf/pavement, sky, horizon, and distant crowds live in `background` ONLY — never as obj elements. (A floor emitted as an obj clips standing subjects' legs.) +- **No double-counting:** anything named in `background` must NOT also be an obj element. +- Don't smuggle furniture or people into `background` as a "receding arrangement" — those are foreground elements. +- If the prompt asks for a transparent/cutout background, set `background` to exactly: `transparent background` (and include `on a transparent background` in the HLD). + +## BBOX + +Coordinates are normalized to 0–1000 in BOTH axes, top-left origin. Format `[y1, x1, y2, x2]` with `y1 < y2`, `x1 < x2`. + +A box is square only on a square frame; on a wide or tall frame the same numbers stretch. For round or square on-screen subjects, scale the spans so `(x2-x1)/(y2-y1) ≈ W/H`. Include bboxes where position matters; omit them for dense/uncountable fills (crowds, starfields). + +## TEXT + +- Every quoted string in the prompt becomes its own `text` element, with `text` = the verbatim characters (preserve case, punctuation, diacritics, and any trigger). Use `\n` for line breaks within one text block; separate blocks get separate elements. +- Include clearly in-scene text (a sign, a label) only when the user asked for it — do not invent signage or brand copy. +- Prose fields (`desc`, `background`, `high_level_description`) are always in ENGLISH; only the `text` field follows the prompt's language. + +## SPECIFICITY + +- For details the user GAVE, commit to one concrete value — no hedging (`things like`, `such as`, `various`), no alternatives (`oak or walnut`). +- For details the user did NOT give, add a single concrete value only when the structure requires it (e.g. a plain background shell); otherwise leave it out. +- Never hedge, never invent appearance for a named person, and never invent characteristics for a named style. + +## ADDITIONAL INSTRUCTIONS + +Honor the following extra instructions from the user. They must NEVER override the OUTPUT CONTRACT, the FIDELITY rules, or the structure above. + +{{user_instructions}} + +[USER] +TARGET IMAGE ASPECT RATIO: {{aspect_ratio}} (width:height). +User prompt: {{original_prompt}} +""" diff --git a/ui/src/app/jobs/new/SimpleJob.tsx b/ui/src/app/jobs/new/SimpleJob.tsx index 1f9015f..eee787c 100644 --- a/ui/src/app/jobs/new/SimpleJob.tsx +++ b/ui/src/app/jobs/new/SimpleJob.tsx @@ -22,7 +22,8 @@ import { SliderInput, } from '@/components/formInputs'; import Card from '@/components/Card'; -import { X, Copy } from 'lucide-react'; +import { X, Copy, Wand2 } from 'lucide-react'; +import { openUpsamplePromptsModal, toAspectRatio } from '@/components/UpsamplePromptsModal'; import AddSingleImageModal, { openAddImageModal } from '@/components/AddSingleImageModal'; import SampleControlImage from '@/components/SampleControlImage'; import { FlipHorizontal2, FlipVertical2 } from 'lucide-react'; @@ -1263,9 +1264,35 @@ export default function SimpleJob({ - -
- +
+ + {modelArch?.additionalSections?.includes('upsample_prompts') && ( + + )} +
{jobConfig.config.process[0].sample.samples.map((sample, i) => (
diff --git a/ui/src/app/jobs/new/options.ts b/ui/src/app/jobs/new/options.ts index f54d3bf..b45e8dd 100644 --- a/ui/src/app/jobs/new/options.ts +++ b/ui/src/app/jobs/new/options.ts @@ -31,7 +31,8 @@ type AdditionalSections = | 'model.layer_offloading' | 'model.low_vram' | 'model.qie.match_target_res' - | 'model.assistant_lora_path'; + | 'model.assistant_lora_path' + | 'upsample_prompts'; type ModelGroup = 'image' | 'instruction' | 'video' | 'experimental' | 'audio'; @@ -999,6 +1000,7 @@ export const modelArchs: ModelArch[] = [ additionalSections: [ 'model.low_vram', 'model.layer_offloading', + 'upsample_prompts', ], hasMultiLinePrompts: true, }, diff --git a/ui/src/app/layout.tsx b/ui/src/app/layout.tsx index 341ac54..c0964d6 100644 --- a/ui/src/app/layout.tsx +++ b/ui/src/app/layout.tsx @@ -10,6 +10,7 @@ import DocModal from '@/components/DocModal'; import os from 'os'; import { CaptionDatasetModal } from '@/components/CaptionDatasetModal'; import MergeLoRAsModal from '@/components/MergeLoRAsModal'; +import UpsamplePromptsModal from '@/components/UpsamplePromptsModal'; export const dynamic = 'force-dynamic'; @@ -63,6 +64,7 @@ export default function RootLayout({ children }: { children: React.ReactNode }) + ); diff --git a/ui/src/components/UpsamplePromptsModal.tsx b/ui/src/components/UpsamplePromptsModal.tsx new file mode 100644 index 0000000..d6c58af --- /dev/null +++ b/ui/src/components/UpsamplePromptsModal.tsx @@ -0,0 +1,362 @@ +'use client'; +import React, { useEffect, useRef, useState } from 'react'; +import { createGlobalState } from 'react-global-hooks'; +import { Check, X, Loader2, AlertTriangle } from 'lucide-react'; +import classNames from 'classnames'; +import { Modal } from './Modal'; +import { callScriptStream } from '@/utils/callScript'; + +export interface UpsamplePromptItem { + index: number; // index into the samples array (write-back target) + prompt: string; // current prompt text + aspectRatio: string; // 'W:H' or 'auto' +} + +export interface UpsamplePromptsModalState { + prompts: UpsamplePromptItem[]; + onApply: (index: number, newPrompt: string) => void; + onClose?: () => void; +} + +export const upsamplePromptsModalState = createGlobalState(null); + +export const openUpsamplePromptsModal = ( + prompts: UpsamplePromptItem[], + onApply: (index: number, newPrompt: string) => void, + onClose?: () => void, +) => { + upsamplePromptsModalState.set({ prompts, onApply, onClose }); +}; + +// Reduce/snap pixel dimensions to a clean 'W:H' (denominator <= 16), mirroring the +// captioner's compute_aspect_ratio. Returns 'auto' when dimensions are unknown. +export function toAspectRatio(width?: number, height?: number): string { + if (!width || !height || width <= 0 || height <= 0) return 'auto'; + const gcd = (a: number, b: number): number => (b ? gcd(b, a % b) : a); + const g = gcd(width, height); + const rw = width / g; + const rh = height / g; + const MAXD = 16; + if (rw <= MAXD && rh <= MAXD) return `${rw}:${rh}`; + const target = width / height; + let best: { err: number; p: number; q: number } | null = null; + for (let q = 1; q <= MAXD; q++) { + const p = Math.max(1, Math.round(target * q)); + const err = Math.abs(p / q - target); + if (!best || err < best.err) best = { err, p, q }; + } + return `${best!.p}:${best!.q}`; +} + +type RowStatus = 'idle' | 'queued' | 'running' | 'done' | 'failed'; + +const UpsamplePromptsModal: React.FC = () => { + const [modalInfo, setModalInfo] = upsamplePromptsModalState.use(); + const isOpen = modalInfo !== null; + + const [selected, setSelected] = useState>({}); + const [creative, setCreative] = useState(false); + const [instructions, setInstructions] = useState(''); + const [status, setStatus] = useState>({}); + const [upsampled, setUpsampled] = useState>({}); + const [isRunning, setIsRunning] = useState(false); + const [isDone, setIsDone] = useState(false); + const [hasError, setHasError] = useState(false); + const [logOutput, setLogOutput] = useState(''); + const logRef = useRef(null); + + // Reset state when the modal opens/closes. Nothing is selected by default. + useEffect(() => { + setSelected({}); + setCreative(false); + setInstructions(''); + setStatus({}); + setUpsampled({}); + setIsRunning(false); + setIsDone(false); + setHasError(false); + setLogOutput(''); + }, [modalInfo]); + + useEffect(() => { + if (logRef.current) logRef.current.scrollTop = logRef.current.scrollHeight; + }, [logOutput]); + + const onClose = () => { + if (isRunning) return; + setModalInfo(null); + modalInfo?.onClose?.(); + }; + + const prompts = modalInfo?.prompts ?? []; + const selectedCount = prompts.filter(p => selected[p.index]).length; + + const toggle = (index: number) => { + if (isRunning) return; + setSelected(prev => ({ ...prev, [index]: !prev[index] })); + }; + + const setAll = (value: boolean) => { + if (isRunning) return; + const next: Record = {}; + prompts.forEach(p => (next[p.index] = value)); + setSelected(next); + }; + + const onRun = async () => { + if (isRunning || !modalInfo) return; + const submitted = modalInfo.prompts.filter(p => selected[p.index]); + if (submitted.length === 0) return; + + setIsRunning(true); + setIsDone(false); + setHasError(false); + setLogOutput(''); + setUpsampled({}); + + const submittedIndices = submitted.map(p => p.index); + setStatus(() => { + const s: Record = {}; + submittedIndices.forEach((idx, k) => (s[idx] = k === 0 ? 'running' : 'queued')); + return s; + }); + + const payload = submitted.map(p => ({ prompt: p.prompt, aspect_ratio: p.aspectRatio })); + + // The script streams one compact JSON line per completed prompt on stdout. + // stdout chunks don't align to line boundaries, so buffer and split ourselves. + let buffer = ''; + const handleStdout = (chunk: string) => { + buffer += chunk; + let nl: number; + while ((nl = buffer.indexOf('\n')) >= 0) { + const line = buffer.slice(0, nl).trim(); + buffer = buffer.slice(nl + 1); + if (!line) continue; + try { + const evt = JSON.parse(line); + if (typeof evt.index !== 'number') continue; + const orig = submittedIndices[evt.index]; + if (orig === undefined) continue; + if (evt.caption) { + const text = JSON.stringify(evt.caption, null, 2); + setUpsampled(prev => ({ ...prev, [orig]: text })); + setStatus(prev => ({ ...prev, [orig]: 'done' })); + modalInfo.onApply(orig, text); // write back to the job config live + } else { + setStatus(prev => ({ ...prev, [orig]: 'failed' })); + } + const nextIdx = submittedIndices[evt.index + 1]; + if (nextIdx !== undefined) { + setStatus(prev => (prev[nextIdx] === 'queued' ? { ...prev, [nextIdx]: 'running' } : prev)); + } + } catch { + // ignore non-JSON / partial lines + } + } + }; + + const append = (chunk: string) => setLogOutput(prev => prev + chunk); + + try { + const finalEvent = await callScriptStream('upsample_ideogram4_caption.py', { + args: { + prompts: JSON.stringify(payload), + stream: true, + temperature: 0.7, + quantize: true, // float8 (script default qtype) to reduce VRAM + ...(creative ? { creative: true } : {}), + ...(instructions.trim() ? { instructions: instructions.trim() } : {}), + }, + onStdout: handleStdout, + onStderr: append, + }); + + const ok = finalEvent?.type === 'exit' && finalEvent.ok === true; + if (!ok) { + setHasError(true); + if (finalEvent?.type === 'error' && finalEvent.message) { + append(`\n${finalEvent.message}\n`); + } else if (finalEvent?.type === 'exit' && finalEvent.timedOut) { + append('\nScript timed out.\n'); + } else if (finalEvent?.type === 'exit') { + append(`\nScript exited with code ${finalEvent.exitCode}.\n`); + } + } + } catch (err: any) { + setHasError(true); + append(`\n${err?.message || 'Unknown error'}\n`); + } finally { + // Anything still queued/running never reported a result -> failed. + setStatus(prev => { + const cp = { ...prev }; + submittedIndices.forEach(i => { + if (cp[i] === 'running' || cp[i] === 'queued') cp[i] = 'failed'; + }); + return cp; + }); + setIsRunning(false); + setIsDone(true); + } + }; + + const StatusIcon = ({ s }: { s: RowStatus | undefined }) => { + if (s === 'done') return ; + if (s === 'failed') return ; + if (s === 'running') return ; + if (s === 'queued') return ; + return ; + }; + + return ( + +
+
+ + + This loads a model and runs on the GPU. You need at least 13GB of free VRAM to run this, so make sure the + GPU is idle (no training or other jobs running) before starting, or it may run out of memory. + +
+ +
+
+ {isRunning && Upsampling... please do not close this window.} + {isDone && hasError && Finished with errors. See log below.} + {isDone && !hasError && Upsampling complete.} + {!isRunning && !isDone && ( + + Select prompts to upsample into structured Ideogram captions. {selectedCount}/{prompts.length} selected. + + )} +
+ {!isRunning && !isDone && ( +
+ + | + +
+ )} +
+ +
+ + + {creative + ? 'Expands the idea — places the subject in a scene and adds fitting details.' + : 'Faithful — structures the prompt as given, with a minimal background.'} + +
+ +
+ +