Added an autocaptioner for Ideogram 4 captions
This commit is contained in:
Jaret Burkett
@@ -167,7 +167,11 @@ class BaseCaptioner(BaseExtensionProcess):
|
||||
caption_file_path = (
|
||||
f"{filename_no_ext}.{self.caption_config.caption_extension}"
|
||||
)
|
||||
if not os.path.exists(caption_file_path):
|
||||
has_caption = False
|
||||
if os.path.exists(caption_file_path):
|
||||
with open(caption_file_path, "r", encoding="utf-8") as f:
|
||||
has_caption = f.read().strip() != ""
|
||||
if not has_caption:
|
||||
filtered_file_paths.append(file_path)
|
||||
print(
|
||||
f"Found {len(self.file_paths)} files. {len(filtered_file_paths)} need captioning."
|
||||
|
||||
180
extensions_built_in/captioner/Ideogram4Captioner.py
Normal file
180
extensions_built_in/captioner/Ideogram4Captioner.py
Normal file
@@ -0,0 +1,180 @@
|
||||
import json
|
||||
import re
|
||||
from math import gcd
|
||||
from collections import OrderedDict
|
||||
from typing import Optional
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from .Qwen3VLCaptioner import Qwen3VLCaptioner
|
||||
from .prompts.ideogram4_caption_prompt import ideogram4_caption_prompt
|
||||
import transformers
|
||||
import logging
|
||||
import warnings
|
||||
|
||||
transformers.logging.set_verbosity_error()
|
||||
warnings.filterwarnings("ignore")
|
||||
logging.disable(logging.WARNING)
|
||||
|
||||
# The deconstruction JSON is long. 128 tokens (base default) truncates it badly,
|
||||
# so enforce a sane floor for this captioner unless the user asked for more.
|
||||
MIN_NEW_TOKENS = 3072
|
||||
|
||||
# Largest denominator allowed when snapping a real image's aspect ratio to a
|
||||
# clean W:H. Keeps captions in the same small-denominator ratio distribution the
|
||||
# generator was trained on, instead of ugly fractions like 1023:768.
|
||||
MAX_AR_DENOMINATOR = 16
|
||||
|
||||
|
||||
class Ideogram4Captioner(Qwen3VLCaptioner):
|
||||
def __init__(self, process_id: int, job, config: OrderedDict, **kwargs):
|
||||
super(Ideogram4Captioner, self).__init__(process_id, job, config, **kwargs)
|
||||
if self.caption_config.max_new_tokens < MIN_NEW_TOKENS:
|
||||
print(
|
||||
f"[Ideogram4Captioner] Raising max_new_tokens "
|
||||
f"{self.caption_config.max_new_tokens} -> {MIN_NEW_TOKENS} "
|
||||
f"(the deconstruction JSON is long)."
|
||||
)
|
||||
self.caption_config.max_new_tokens = MIN_NEW_TOKENS
|
||||
|
||||
def compute_aspect_ratio(self, width: int, height: int) -> str:
|
||||
"""Return a clean 'W:H' string for the image, snapped to a small
|
||||
denominator so it matches the generator's ratio distribution."""
|
||||
if width <= 0 or height <= 0:
|
||||
return "1:1"
|
||||
g = gcd(width, height)
|
||||
rw, rh = width // g, height // g
|
||||
# Already clean enough.
|
||||
if rw <= MAX_AR_DENOMINATOR and rh <= MAX_AR_DENOMINATOR:
|
||||
return f"{rw}:{rh}"
|
||||
# Otherwise find the closest p:q (q <= MAX_AR_DENOMINATOR) to the true ratio.
|
||||
target = width / height
|
||||
best = None
|
||||
for q in range(1, MAX_AR_DENOMINATOR + 1):
|
||||
p = max(1, round(target * q))
|
||||
err = abs(p / q - target)
|
||||
if best is None or err < best[0]:
|
||||
best = (err, p, q)
|
||||
return f"{best[1]}:{best[2]}"
|
||||
|
||||
def build_prompt(self, aspect_ratio: str) -> str:
|
||||
# caption_prompt is the user-editable ADDITIONAL INSTRUCTIONS block,
|
||||
# injected into the fixed system prompt (not the whole prompt).
|
||||
user_instructions = (self.caption_config.caption_prompt or "").strip()
|
||||
if not user_instructions:
|
||||
user_instructions = "None."
|
||||
prompt = ideogram4_caption_prompt.replace("{{aspect_ratio}}", aspect_ratio)
|
||||
prompt = prompt.replace("{{user_instructions}}", user_instructions)
|
||||
return prompt
|
||||
|
||||
def _extract_json(self, raw: str) -> Optional[dict]:
|
||||
"""Pull the JSON object out of the model output, tolerating fences and
|
||||
stray preamble. Returns the parsed dict or None."""
|
||||
text = raw.strip()
|
||||
# Strip ```json ... ``` fences if present.
|
||||
fence = re.search(r"```(?:json)?\s*(.*?)```", text, re.DOTALL)
|
||||
if fence:
|
||||
text = fence.group(1).strip()
|
||||
# Fall back to the outermost {...} span.
|
||||
start = text.find("{")
|
||||
end = text.rfind("}")
|
||||
if start == -1 or end == -1 or end <= start:
|
||||
return None
|
||||
candidate = text[start : end + 1]
|
||||
try:
|
||||
return json.loads(candidate)
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
|
||||
def _convert_bbox(self, bbox):
|
||||
"""Qwen3-VL emits NORMALIZED 0-1000 boxes in [x1,y1,x2,y2] order (verified
|
||||
empirically: coords are stable across input resolution). Our stored
|
||||
format is also 0-1000 but in [y1,x1,y2,x2] order, so this only reorders
|
||||
and clamps -- no pixel scaling. Returns the box or None to drop it."""
|
||||
if not isinstance(bbox, (list, tuple)) or len(bbox) != 4:
|
||||
return None
|
||||
try:
|
||||
x1, y1, x2, y2 = [float(v) for v in bbox]
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
x1, x2 = sorted((max(0, min(1000, round(x1))), max(0, min(1000, round(x2)))))
|
||||
y1, y2 = sorted((max(0, min(1000, round(y1))), max(0, min(1000, round(y2)))))
|
||||
if y2 <= y1 or x2 <= x1:
|
||||
return None
|
||||
# stored order is [y1, x1, y2, x2]
|
||||
return [y1, x1, y2, x2]
|
||||
|
||||
def _normalize_caption(self, data: dict, aspect_ratio: str) -> dict:
|
||||
"""Validate/cleanup the parsed caption before storage, reordering each
|
||||
bbox from model-native [x1,y1,x2,y2] to stored [y1,x1,y2,x2]."""
|
||||
# Force the aspect ratio we computed; the model is told to echo it but
|
||||
# we know the true value.
|
||||
data["aspect_ratio"] = aspect_ratio
|
||||
decon = data.get("compositional_deconstruction", {})
|
||||
elements = decon.get("elements", [])
|
||||
if isinstance(elements, list):
|
||||
for el in elements:
|
||||
if isinstance(el, dict) and "bbox" in el:
|
||||
cleaned = self._convert_bbox(el["bbox"])
|
||||
if cleaned is None:
|
||||
el.pop("bbox", None)
|
||||
else:
|
||||
el["bbox"] = cleaned
|
||||
return data
|
||||
|
||||
def get_caption_for_file(self, file_path: str) -> Optional[str]:
|
||||
try:
|
||||
# Read true dimensions before any resize so the aspect ratio is exact.
|
||||
with Image.open(file_path) as probe:
|
||||
width, height = probe.size
|
||||
aspect_ratio = self.compute_aspect_ratio(width, height)
|
||||
|
||||
img = self.load_pil_image(file_path, max_res=self.caption_config.max_res)
|
||||
prompt = self.build_prompt(aspect_ratio)
|
||||
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "image", "image": img},
|
||||
{"type": "text", "text": prompt},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
inputs = self.processor.apply_chat_template(
|
||||
messages,
|
||||
tokenize=True,
|
||||
add_generation_prompt=True,
|
||||
return_dict=True,
|
||||
return_tensors="pt",
|
||||
)
|
||||
inputs = inputs.to(self.device_torch)
|
||||
|
||||
generated_ids = self.model.generate(
|
||||
**inputs, max_new_tokens=self.caption_config.max_new_tokens
|
||||
)
|
||||
generated_ids_trimmed = [
|
||||
out_ids[len(in_ids) :]
|
||||
for in_ids, out_ids in zip(inputs.input_ids, generated_ids)
|
||||
]
|
||||
output_text = self.processor.batch_decode(
|
||||
generated_ids_trimmed,
|
||||
skip_special_tokens=True,
|
||||
clean_up_tokenization_spaces=False,
|
||||
)[0].strip()
|
||||
|
||||
data = self._extract_json(output_text)
|
||||
if data is None:
|
||||
print(
|
||||
f"[IdeogramCaptioner] Could not parse JSON for {file_path}; "
|
||||
f"saving raw output."
|
||||
)
|
||||
return output_text
|
||||
|
||||
data = self._normalize_caption(data, aspect_ratio)
|
||||
# Store pretty JSON for QC/editing; the dataloader minifies at load.
|
||||
return json.dumps(data, ensure_ascii=False, indent=2)
|
||||
except Exception as e:
|
||||
print(f"Error processing {file_path}: {e}")
|
||||
return None
|
||||
@@ -25,7 +25,20 @@ class Qwen3VLCaptionerExtension(Extension):
|
||||
return Qwen3VLCaptioner
|
||||
|
||||
|
||||
class Ideogram4CaptionerExtension(Extension):
|
||||
uid = "Ideogram4Captioner"
|
||||
name = "Ideogram4 Captioner"
|
||||
|
||||
@classmethod
|
||||
def get_process(cls):
|
||||
# import your process class here so it is only loaded when needed and return it
|
||||
from .Ideogram4Captioner import Ideogram4Captioner
|
||||
|
||||
return Ideogram4Captioner
|
||||
|
||||
|
||||
AI_TOOLKIT_EXTENSIONS = [
|
||||
AceStepCaptionerExtension,
|
||||
Qwen3VLCaptionerExtension,
|
||||
Ideogram4CaptionerExtension,
|
||||
]
|
||||
|
||||
0
extensions_built_in/captioner/prompts/__init__.py
Normal file
0
extensions_built_in/captioner/prompts/__init__.py
Normal file
@@ -0,0 +1,244 @@
|
||||
ideogram4_caption_prompt = """
|
||||
[META]
|
||||
frozen: false
|
||||
description: Image -> structured JSON caption. Inverted v15 magic-prompt: observe-only discipline, no invention, splatter-style compositional deconstruction with grounded bboxes. Thinking off.
|
||||
thinking_mode: disabled
|
||||
|
||||
[SYSTEM]
|
||||
You analyze a single provided IMAGE and emit one JSON object that decomposes what is ACTUALLY VISIBLE into a structured caption an image renderer can consume. You receive the image plus its exact target aspect ratio. You emit one JSON object.
|
||||
|
||||
## OBSERVE-ONLY — the cardinal rule
|
||||
|
||||
You are CAPTIONING a real image, not imagining one. Describe ONLY what is visibly present.
|
||||
- NEVER invent, populate, infer, or add subjects, props, text, background detail, or atmosphere that is not actually visible in the image.
|
||||
- NEVER guess at occluded or off-frame content. If you cannot see it, it does not exist for this caption.
|
||||
- Do NOT enrich sparse scenes. An empty room stays empty. A single subject on a plain backdrop stays single on a plain backdrop.
|
||||
- Do NOT invent brands, signage, or text that is not legibly present.
|
||||
- Specificity below means committing to the value you OBSERVE (the one color that is actually there), never inventing a value to fill a gap.
|
||||
|
||||
## OUTPUT CONTRACT — exactly three top-level keys, in this order:
|
||||
|
||||
```json
|
||||
{"aspect_ratio":"W:H","high_level_description":"...","compositional_deconstruction":{"background":"...","elements":[ ... ]}}
|
||||
```
|
||||
|
||||
- Emit a SINGLE-LINE MINIFIED JSON object — no markdown fences, no commentary, no other top-level keys.
|
||||
- Preserve non-ASCII characters as-is (CJK, Cyrillic, Devanagari, Arabic, accented Latin). Never escape with `\\uNNNN`, transliterate, or replace `café` with `cafe`.
|
||||
- Use SINGLE quotes for embedded text references in prose fields (`'Joe's Diner'`, not `\\"Joe's Diner\\"`). The `text` field of text elements is the exception — that field holds the verbatim characters visible in the image, may use any characters, and follows QUOTED SPAN FIDELITY below.
|
||||
|
||||
### `aspect_ratio` (first field, always required)
|
||||
|
||||
The exact target aspect ratio is GIVEN to you in the user message as `W:H`. Echo it VERBATIM. Never recompute it, never emit `auto`, never override it from your own reading of the frame. Every bbox you emit is normalized to THIS aspect ratio.
|
||||
|
||||
### `high_level_description` — observational summary (50-word hard cap)
|
||||
|
||||
- ONE long sentence preferred, never more than two.
|
||||
- Reads like a short natural-language prompt, not an analysis. Starts immediately with the subject — no "this image shows", "depicts", "captures".
|
||||
- Identifies subject(s), medium, and overall composition. Names recognized pop-culture entities by full name (`Nike Air Jordan 1`, `Eiffel Tower`, `Mario (Nintendo character)`) ONLY when you actually recognize them in the image.
|
||||
- Don't enumerate granular features (every color, every grid dimension, every typography choice). That detail belongs in element descs or `background`.
|
||||
- `various`, `multiple`, general categories ARE appropriate here. Specificity rule (below) applies to element descs and `background`, NOT this field.
|
||||
- For transparent/cutout backgrounds, include the literal phrase `on a transparent background`.
|
||||
|
||||
GOOD: `A full-action shot of a male soccer player in a red kit and black Adidas cleats kicking a soccer ball on a green turf field, with a blurred crowd in the stadium background.`
|
||||
BAD (over-specifies): `A male soccer player captured mid-kick on a bright green grass pitch, right leg fully extended through the follow-through at the precise moment his black-and-white studded boot makes contact with a white-and-black size-5 ball...`
|
||||
|
||||
## IDENTIFY THE MEDIUM
|
||||
|
||||
State the medium accurately in HLD/background prose as natural framing: `photograph | illustration | 3D render | graphic design` (and the specific style when obvious — `35mm film photograph`, `flat vector illustration`, `Pixar-style 3D render`, `watercolor illustration`). Read it from the image; do not impose a default. Name a recognizable style ONCE, briefly — do not append invented technique detail.
|
||||
|
||||
## ELEMENTS — what they are, what they're not
|
||||
|
||||
Each element is one of:
|
||||
```
|
||||
{"type":"obj","bbox":[x1,y1,x2,y2],"desc":"..."}
|
||||
{"type":"text","bbox":[x1,y1,x2,y2],"text":"LINE ONE\\nLINE TWO","desc":"..."}
|
||||
```
|
||||
|
||||
`bbox` is optional per-element (see BBOX section below).
|
||||
|
||||
### SINGLE SUBJECT = SINGLE ELEMENT
|
||||
|
||||
A coherent subject — one animal, person, vehicle, building, plant, instrument, machine — is exactly ONE `obj` element. Anatomical and structural parts are descriptive attributes inside that element's `desc`, NOT separate elements.
|
||||
|
||||
FORBIDDEN: a bee split into 8 elements (thorax/abdomen/wings/eyes/legs/...); a car split into 6 (body/wheels/windshield/...); a person split into 7 (head/torso/each limb/...); a building split into 5 (foundation/walls/windows/roof/door); a flower split into 3 (petals/stem/leaves).
|
||||
|
||||
When MULTIPLE distinct subjects are visible (a person AND a dog; two bees; three runners), use MULTIPLE elements — one per subject.
|
||||
|
||||
**Test:** part-of-one-thing → goes in that thing's desc. Separate thing → its own element.
|
||||
|
||||
**Transparent enclosure + featured contents = ONE element.** Display cases, snow globes, terrariums, aquariums, specimen jars, bell jars, vitrines containing a featured subject: name the enclosure + contents as a single unified desc.
|
||||
|
||||
**Configured parts + revealed interior = ONE element.** A car with an open door, a machine with raised hood, a building with drawn curtains: the open state and any revealed interior are attributes of the single subject's desc, not separate elements.
|
||||
|
||||
### Element desc — what to write (30–60 words, 60-word HARD CAP)
|
||||
|
||||
Identity first, then major attributes briefly, then one distinguishing detail if relevant. Each desc is a standalone catalog entry — open with the subject's identity, not a referring phrase like "the X" that assumes the reader has seen the scene.
|
||||
|
||||
GOOD (introduces from scratch):
|
||||
- `Woman walking on the platform, medium size. Shoulder-length dark wavy hair, medium skin tone, light blue button-down shirt and grey trousers. Small bag slung over the right shoulder.`
|
||||
- `Circular concrete tunnel entrance with glowing blue ring lights along the interior. Train tracks lead directly into the dark opening.`
|
||||
|
||||
**Major attributes — always name (when visible):**
|
||||
- People: skin tone, hair (color + style), each visible garment with color, expression/gaze, pose, distinguishing feature (mole, glasses, jewelry, held prop).
|
||||
- Objects: shape, material, color, distinctive parts (handle, label, logo, marking).
|
||||
- Scenes/structures: type, primary material, color, distinctive structural elements.
|
||||
|
||||
**Skip (eat word budget for marginal benefit):**
|
||||
- Surface-finish micro-prose (`finely granular matte texture with subtle sheen along the elytral ridges`). Pick one short descriptor (matte/glossy/metallic/textured) or omit.
|
||||
- Pose mechanics per-limb. Pick ONE summary action phrase plus the major attributes.
|
||||
- Camera/shadow/lighting micro-detail per element. Belongs in `background`.
|
||||
- Fabric weave, skin texture nuances, micro-anatomy.
|
||||
|
||||
### Element desc — what NOT to include
|
||||
|
||||
**No shadows.** Cast shadows, drop shadows, ground shadows, contact shadows, ambient occlusion — describe in `background` only when scene-wide, otherwise omit. Forbidden: `casts a thin hard shadow to the lower right`, `with a soft drop shadow beneath`.
|
||||
|
||||
**No camera or render language.** Depth of field, focus, sharpness, bokeh, exposure, motion blur, lens flare, chromatic aberration, film grain — render properties belong in `high_level_description` or `background` as natural prose. NEVER inside an obj desc.
|
||||
- EXCEPTION — viewpoint/angle (`from a low-angle perspective`, `bird's-eye view`, `eye-level`) IS allowed in obj descs. Place once, usually in the focal subject's desc or background.
|
||||
|
||||
**No describing impressions instead of physical reality.** Avoid `luminous`, `radiant`, `vibrant`, `lush`, `dynamic`, `glowing` (metaphorically), `gorgeous`, `stunning`, `breathtaking`, `mesmerizing`. Use observable properties: `cheekbone catches a small highlight`, not `luminous complexion`.
|
||||
|
||||
**No scene-context repetition per-element.** Lighting direction, ambient surface, mounting context, weather → describe ONCE in `background`. Each element's desc focuses on what's UNIQUE to that element.
|
||||
|
||||
### Anchor placements to named references
|
||||
|
||||
Specify body parts, surfaces, spatial landmarks.
|
||||
- CORRECT: `applied to the forehead near the hairline above the left eyebrow`.
|
||||
- INCORRECT: `pressed against the skin`.
|
||||
- CORRECT: `resting on the lower-right corner of the table directly in front of the laptop`.
|
||||
- INCORRECT: `sitting on the surface`.
|
||||
|
||||
## BACKGROUND — what goes here, what doesn't (CRITICAL)
|
||||
|
||||
`background` describes the scene SHELL: walls and finishes, floor/ground and surface state, ceiling and architectural fixtures, windows as architecture, atmospheric context (sky, clouds, fog, dust, mist), scene-wide ambient lighting, distant out-of-focus context (horizon, blurred crowds, distant scenery).
|
||||
|
||||
### No double-counting
|
||||
|
||||
Anything described in `background` CANNOT also appear as an obj element. Each scene component lives in EXACTLY ONE field. Decide once and commit. Before emitting an obj element, scan `background` — if the component is named there, omit the obj element.
|
||||
|
||||
### ALWAYS-BACKGROUND — these live in `background` only, never as obj elements:
|
||||
|
||||
- sky, clouds, atmospheric color
|
||||
- horizon
|
||||
- distant mountains, hills, tree lines
|
||||
- atmospheric weather (fog, haze, mist, smoke)
|
||||
- distant cityscape or stadium architecture
|
||||
- distant blurred or simplified crowds
|
||||
- the floor / ground / turf / paving surface the scene sits on
|
||||
- ambient walls or studio backdrop behind focal subjects
|
||||
|
||||
You cannot split these by region. `sky upper-left portion`, `sky behind the fortress`, `sky upper two-thirds` are the SAME component — describe in `background` once. Same for crowd, ground, horizon.
|
||||
|
||||
If a visible atmospheric component carries technique-level detail (watercolor wet-on-wet sky blooms, fog with directional density variation), put that detail in `background`. The `background` field is allowed to be long.
|
||||
|
||||
### Ground/floor/pavement is ALWAYS background — zero tolerance
|
||||
|
||||
The surface the scene sits on — floor, ground, turf, grass, dirt, sand, asphalt, pavement, road, sidewalk, deck, water surface, snow, tile floor, hardwood, marble — lives in `background` only.
|
||||
|
||||
**Surface character that belongs in background, not as a separate obj:** wet / rain-slicked / mud-streaked / dusty / cracked / polished / weathered surface state; reflective neon pools, fragmented color reflections, puddles, wet patches, mud patches, ice patches, frost, snow on the floor, water pooled on the ground, oil slicks, footprints, tire tracks; surface material (asphalt, cobblestone, hardwood, tile, marble, packed dirt); texture words for the floor (glassy, mirror-like, matte, polished, rough).
|
||||
|
||||
**Puddles, reflections, wet patches are part of the ground surface** — never separate obj elements, regardless of whether they reflect the hero's silhouette or carry visible content.
|
||||
|
||||
**Failure mode this prevents:** when a standing hero is the focal element and the floor is also emitted as an obj at the bottom of the frame, the renderer treats the floor obj as a 2D frame band rather than a perspectival receding plane, and clips the hero's legs into it.
|
||||
|
||||
**Discrete objects ON the floor are still elements:** broken glass shards, crushed cans, scattered debris, leaves, rocks, dropped tools, brick fragments, foreground litter remain obj elements. The rule applies to the SURFACE itself and any state of that surface (wet, frozen, muddy, puddled), never to solid objects resting on it.
|
||||
|
||||
### Background is the shell only — no individually-placeable things
|
||||
|
||||
Furniture, vehicles, equipment, people, animals, decor (artwork, signs, plants in pots, stacks of books), free-standing lamps → obj elements, never `background`.
|
||||
|
||||
### Shell-affixed prominent objects → DUAL MENTION
|
||||
|
||||
Some visible objects are simultaneously part of the shell AND focal elements that define the room's identity: a chalkboard covering the back wall of a classroom, a fireplace built into a living-room wall, a large mounted TV, a stage proscenium, a built-in altar, a built-in bookshelf, a large fixed reception desk, a fixed sign/banner.
|
||||
|
||||
For these, when visible, MANDATORY all three steps:
|
||||
1. **MENTION in `background`** as part of the shell — anchors the object to the wall.
|
||||
2. **EMIT as an obj element** with the qualifier `"the primary background element"` (or similar) at the start of its desc. The obj carries the detail (material, content, frame, mounting).
|
||||
3. **PLACE FIRST in the elements list** so painter's-algorithm draws it behind foreground items.
|
||||
|
||||
Skipping step 1 makes the renderer float the object in mid-room or render it in front of foreground subjects.
|
||||
|
||||
This is an EXCEPTION to the shell rule's "no individually placeable things". Applies ONLY to objects that genuinely define the room's architectural identity. Free-standing items (chairs, table lamps, plants in pots, framed pictures on a wall) get the normal treatment: elements only, no background mention.
|
||||
|
||||
### Recession/arrangement is not architecture
|
||||
|
||||
Do not smuggle furniture or people into `background` by describing them as a receding arrangement. Forbidden background phrasings: `rows of desks recede toward the back`, `a grid of desks fills the room`, `students seated at the desks`, `chairs arranged in front of the podium`, `cars parked along the street`, `customers seated at the tables`. The arrangement IS foreground content — emit elements (one per distinct visible subject, or omit bboxes for dense unenumerable groups per the bbox rules).
|
||||
|
||||
### No medium/post-processing effects in background
|
||||
|
||||
`background` describes WHAT is in the scene, not HOW it was made. Route medium/post-processing observations (film grain, lens flare, chromatic aberration, vignetting, bokeh quality, color cast, paper/canvas texture, brushstroke texture, halftone/screen-print/risograph texture) to HLD as natural prose, never to `background`.
|
||||
|
||||
**Test:** read `background` aloud. If you can picture the EMPTY room from the description — no furniture, no people, no equipment, no wall decor — you're in the shell. If anything disappears when you remove the room's contents, the background has leaked.
|
||||
|
||||
## BBOX STRATEGY
|
||||
|
||||
INCLUDE bboxes on elements where precise positioning matters and the element has a clear extent — portrait subjects, products on a surface, logos, signs on a wall, distinct individually-placeable objects.
|
||||
|
||||
OMIT bboxes on elements that represent dense or hard-to-enumerate visuals — crowds, fields of wildflowers, scattered particles, starry skies. Per-element judgment.
|
||||
|
||||
### Coordinate system
|
||||
|
||||
Coordinates are normalized to 0–1000 over the image: `x` runs left→right (0 = left edge, 1000 = right edge), `y` runs top→bottom (0 = top, 1000 = bottom). Top-left origin. Format `[x1, y1, x2, y2]` with `x1 < x2`, `y1 < y2`.
|
||||
|
||||
The bbox must tightly enclose the visible extent of the subject in the image. Trace the real bounds; do not round to convenient values.
|
||||
|
||||
## SPECIFICITY — commit to the observed value
|
||||
|
||||
This JSON feeds a diffusion model. State the value you OBSERVE; never hedge, never offer alternatives, never invent to fill a gap (if you cannot tell, describe what is actually visible at lower granularity rather than guessing a specific wrong value).
|
||||
|
||||
**Banned hedge phrasings** (in elements and background): `things like`, `such as`, `e.g.`, `for example`, `or similar`, `various`, `could include`, `might be`, `some kind of`, `style of`. Replace with the concrete noun, count, color, material, pose you see.
|
||||
|
||||
**Banned alternative listings for one property:** `pale institutional off-white or pale green`, `oak or walnut`, `cream or ivory`, `italic serif or italic sans-serif`, `bold or semibold`. Pick the ONE you observe. `or` is reserved for the loader's exclusive-choice idiom (`'YES' or 'NO'`), not captioner hedging.
|
||||
|
||||
**Typography specifically:** name ONE typeface category (serif OR sans-serif OR display OR script OR monospace), ONE weight (bold/regular/light/medium), ONE style (italic OR upright) — as observed.
|
||||
|
||||
**Banned "implied/suggested" hedges:** `a desk corner implied`, `a chair suggested beneath the figure`, `a shadow that reads as a person`. If it is visibly in the scene, describe it concretely. If it isn't, leave it out. Forbidden words: `implied, suggested, hinted, barely visible, possibly, perhaps, maybe, might be, could be, reads as, almost`.
|
||||
|
||||
**Exhaustive content preservation.** Every distinct visible subject MUST appear as its own element. When the image contains enumerable visible content — a schedule, a menu board, a list, a numbered set, a row of items — every legible item must appear in the output. Use as many text/obj elements as needed; never sacrifice completeness for layout.
|
||||
|
||||
**No placeholder enumeration.** When the image contains a sequentially-numbered, alphabetically-labeled, or otherwise individually-identified visible set (stones numbered 1–50, parking spaces A1–A20, place cards `1st`–`12th`, a calendar grid of dates, a team roster), EACH legible item is its own element. No `etc.`, no `and so on`, no single obj grouping them all. List ALL that are legible. (The dense-unenumerable exception — crowd of thousands, field of wildflowers, starry sky — does NOT apply to enumerable identified sets.)
|
||||
|
||||
**Don't invent visual concepts.** Do not add `glitch art`, `wireframe overlay`, `digital artifacts`, or any stylization not actually present in the image.
|
||||
|
||||
## TEXT HANDLING
|
||||
|
||||
For each piece of legibly visible text, emit a text element:
|
||||
- `text` — the literal characters AS THEY APPEAR in the image, verbatim. Preserve diacritics, capitalization, punctuation, line breaks. Never transliterate, translate, correct, or strip.
|
||||
- `bbox` — optional, same coordinate system as obj elements; box the text's visible extent.
|
||||
- `desc` — free-form prose covering size, location, font style, color, orientation, visual effects.
|
||||
|
||||
**Sources of text to include (only what is actually legible in the image):**
|
||||
1. Signage, labels, license plates, badges, jersey numbers, t-shirt prints, awnings, neon signs, name tags.
|
||||
2. Headlines, taglines, author names, dates, venues, CTA copy, brand names, publisher marks on designed artifacts.
|
||||
3. Numeric content — race numbers, jersey numbers, dates, prices, scores, time displays, address numbers. Numbers ARE text.
|
||||
4. Product brand text actually printed on visible packaging.
|
||||
|
||||
**Rules:**
|
||||
- Exhaustive: if a viewer could read it in the image, it goes in the list. If text is present but illegible/too small to read, do NOT invent its content — either omit it or, if it is a prominent block, note it as an obj with a desc like `a small block of illegible printed text`.
|
||||
- Each text element appears ONCE in the list. Do NOT also transcribe its characters in `desc` — refer by role/position instead.
|
||||
- Use `\\n` for line breaks WITHIN a single text element (multi-line sign, stacked headline). Use SEPARATE list items for visually distinct text blocks.
|
||||
- For stylized hero typography where each letter is a distinct visual unit, stack with `\\n` at natural word breaks. e.g., `"ENTRE\\nVERSOS E\\nCONTOS"`.
|
||||
- **Language scoping:** `background`/`desc`/position descriptors are always in ENGLISH regardless of the language of text in the image. Only the literal `text` field characters follow the image's language. A sign reading Portuguese → English prose + Portuguese `text:` content.
|
||||
|
||||
## POP CULTURE, BRANDS, NAMED REFERENCES
|
||||
|
||||
When the image clearly shows a recognizable brand, trademark, product (sneaker/car/device), public figure, athlete, musician, actor, fictional character, film, show, game, franchise, or team, name it explicitly in the relevant element `desc` rather than a generic stand-in.
|
||||
|
||||
Don't reduce a visible `Nike Dunk Low Panda` to `black and white retro sneakers`, or a visible `Spider-Man` to `a red-and-blue masked superhero`. Name the specific thing you recognize. But ONLY when you actually recognize it — never guess an identity you are unsure of; describe the appearance instead.
|
||||
|
||||
## TRANSPARENT BACKGROUND
|
||||
|
||||
If the image has a transparent/alpha background, or is an isolated cutout subject with no backdrop (sticker-style), the `background` field MUST be exactly this string, verbatim and nothing else: `transparent background`
|
||||
|
||||
Do not paraphrase (no `clear backdrop`, `empty alpha`, `no background`, `PNG transparency`). In `high_level_description`, include the literal phrase `on a transparent background`. (A plain solid-color studio backdrop is NOT transparent — describe it as a backdrop in `background`.)
|
||||
|
||||
## ADDITIONAL INSTRUCTIONS
|
||||
|
||||
Honor the following dataset-specific guidance. It must NEVER override the OUTPUT CONTRACT, the element/background structure, the bbox format, or the observe-only rule above — those are fixed.
|
||||
|
||||
{{user_instructions}}
|
||||
|
||||
[USER]
|
||||
TARGET IMAGE ASPECT RATIO: {{aspect_ratio}} (width:height).
|
||||
Analyze the provided image and emit the JSON caption.
|
||||
"""
|
||||
298
extensions_built_in/captioner/prompts/ideogram4_prompt.py
Normal file
298
extensions_built_in/captioner/prompts/ideogram4_prompt.py
Normal file
@@ -0,0 +1,298 @@
|
||||
ideogram4_prompt = """
|
||||
[META]
|
||||
frozen: false
|
||||
description: Slim single-shot magic prompt — splatter planning + v15 output discipline, deduped for faster inference. Thinking off.
|
||||
thinking_mode: disabled
|
||||
|
||||
[SYSTEM]
|
||||
You convert a natural-language user idea into a structured JSON caption an image renderer can consume. You receive the user idea plus a target aspect ratio, and you emit one JSON object.
|
||||
|
||||
## OUTPUT CONTRACT — exactly three top-level keys, in this order:
|
||||
|
||||
```json
|
||||
{"aspect_ratio":"W:H","high_level_description":"...","compositional_deconstruction":{"background":"...","elements":[ ... ]}}
|
||||
```
|
||||
|
||||
- Emit a SINGLE-LINE MINIFIED JSON object — no markdown fences, no commentary, no other top-level keys.
|
||||
- Preserve non-ASCII characters as-is (CJK, Cyrillic, Devanagari, Arabic, accented Latin). Never escape with `\uNNNN`, transliterate, or replace `café` with `cafe`.
|
||||
- Use SINGLE quotes for embedded text references in prose fields (`'Joe's Diner'`, not `\"Joe's Diner\"`). The `text` field of text elements is the exception — that field holds the user's verbatim characters, may use any characters, and follows QUOTED SPAN FIDELITY below.
|
||||
|
||||
### `aspect_ratio` (first field, always required)
|
||||
|
||||
A string in `W:H` form with positive integers (`1:1`, `16:9`, `9:16`, `4:5`, `3:1`, `2:3`, etc.).
|
||||
- If the user message gives a concrete `W:H`, echo it verbatim.
|
||||
- If the user message says `auto`, pick a concrete ratio that matches the medium and composition (panoramic subjects → wide ratios like `16:9` or `3:1`; portrait subjects → tall like `9:16` or `4:5`; designed artifacts → format conventions like `2:3` book cover, `3:4` poster; ambiguous → `1:1`). NEVER emit the literal string `auto`.
|
||||
- The aspect ratio you commit to drives every bbox decision. Pick it first.
|
||||
|
||||
### `high_level_description` — observational summary (50-word hard cap)
|
||||
|
||||
- ONE long sentence preferred, never more than two.
|
||||
- Reads like a short natural-language prompt, not an analysis. Starts immediately with the subject — no "this image shows", "depicts", "captures".
|
||||
- Identifies subject(s), medium, and overall composition. Names recognized pop-culture entities by full name (`Nike Air Jordan 1`, `Eiffel Tower`, `Mario (Nintendo character)`).
|
||||
- Don't enumerate granular features (every color, every grid dimension, every typography choice). That detail belongs in element descs or `background`.
|
||||
- `various`, `multiple`, general categories ARE appropriate here. Specificity rule (below) applies to element descs and `background`, NOT this field.
|
||||
- For transparent backgrounds, include the literal phrase `on a transparent background`.
|
||||
|
||||
GOOD: `A full-action shot of a male soccer player in a red kit and black Adidas cleats kicking a soccer ball on a green turf field, with a blurred crowd in the stadium background.`
|
||||
BAD (over-specifies): `A male soccer player captured mid-kick on a bright green grass pitch, right leg fully extended through the follow-through at the precise moment his black-and-white studded boot makes contact with a white-and-black size-5 ball...`
|
||||
|
||||
## ELEMENTS — what they are, what they're not
|
||||
|
||||
Each element is one of:
|
||||
```
|
||||
{"type":"obj","bbox":[y1,x1,y2,x2],"desc":"..."}
|
||||
{"type":"text","bbox":[y1,x1,y2,x2],"text":"LINE ONE\nLINE TWO","desc":"..."}
|
||||
```
|
||||
|
||||
`bbox` is optional per-element (see BBOX section below).
|
||||
|
||||
### SINGLE SUBJECT = SINGLE ELEMENT
|
||||
|
||||
A coherent subject — one animal, person, vehicle, building, plant, instrument, machine — is exactly ONE `obj` element. Anatomical and structural parts are descriptive attributes inside that element's `desc`, NOT separate elements.
|
||||
|
||||
FORBIDDEN: a bee split into 8 elements (thorax/abdomen/wings/eyes/legs/...); a car split into 6 (body/wheels/windshield/...); a person split into 7 (head/torso/each limb/...); a building split into 5 (foundation/walls/windows/roof/door); a flower split into 3 (petals/stem/leaves).
|
||||
|
||||
When MULTIPLE distinct subjects appear (a person AND a dog; two bees; three runners), use MULTIPLE elements — one per subject.
|
||||
|
||||
**Test:** part-of-one-thing → goes in that thing's desc. Separate thing → its own element.
|
||||
|
||||
**Transparent enclosure + featured contents = ONE element.** Display cases, snow globes, terrariums, aquariums, specimen jars, bell jars, vitrines containing a featured subject: name the enclosure + contents as a single unified desc.
|
||||
|
||||
**Configured parts + revealed interior = ONE element.** A car with an open door, a machine with raised hood, a building with drawn curtains: the open state and any revealed interior are attributes of the single subject's desc, not separate elements.
|
||||
|
||||
### Element desc — what to write (30–60 words, 60-word HARD CAP)
|
||||
|
||||
Identity first, then major attributes briefly, then one distinguishing detail if relevant. Each desc is a standalone catalog entry — open with the subject's identity, not a referring phrase like "the X" that assumes the reader has seen the scene.
|
||||
|
||||
GOOD (introduces from scratch):
|
||||
- `Woman walking on the platform, medium size. Shoulder-length dark wavy hair, medium skin tone, light blue button-down shirt and grey trousers. Small bag slung over the right shoulder.`
|
||||
- `Circular concrete tunnel entrance with glowing blue ring lights along the interior. Train tracks lead directly into the dark opening.`
|
||||
|
||||
**Major attributes — always name:**
|
||||
- People: skin tone, hair (color + style), each visible garment with color, expression/gaze, pose, distinguishing feature (mole, glasses, jewelry, held prop).
|
||||
- Objects: shape, material, color, distinctive parts (handle, label, logo, marking).
|
||||
- Scenes/structures: type, primary material, color, distinctive structural elements.
|
||||
|
||||
**Skip (eat word budget for marginal benefit):**
|
||||
- Surface-finish micro-prose (`finely granular matte texture with subtle sheen along the elytral ridges`). Pick one short descriptor (matte/glossy/metallic/textured) or omit.
|
||||
- Pose mechanics per-limb. Pick ONE summary action phrase plus the major attributes.
|
||||
- Camera/shadow/lighting micro-detail per element. Belongs in `background`.
|
||||
- Fabric weave, skin texture nuances, micro-anatomy.
|
||||
|
||||
### Element desc — what NOT to include
|
||||
|
||||
**No shadows.** Cast shadows, drop shadows, ground shadows, contact shadows, ambient occlusion — describe in `background` only when scene-wide, otherwise omit (the renderer infers them). Forbidden: `casts a thin hard shadow to the lower right`, `with a soft drop shadow beneath`.
|
||||
|
||||
**No camera or render language.** Depth of field, focus, sharpness, bokeh, exposure, motion blur, lens flare, chromatic aberration, film grain — render properties belong in `high_level_description` or `background` as natural prose ONLY when the user prompt explicitly named them. NEVER inside an obj desc.
|
||||
- EXCEPTION — viewpoint/angle (`from a low-angle perspective`, `bird's-eye view`, `eye-level`) IS allowed in obj descs when the prompt calls for it. Place once, usually in the focal subject's desc or background.
|
||||
|
||||
**No describing impressions instead of physical reality.** Avoid `luminous`, `radiant`, `vibrant`, `lush`, `dynamic`, `glowing` (metaphorically), `gorgeous`, `stunning`, `breathtaking`, `mesmerizing`. Use observable properties: `cheekbone catches a small highlight`, not `luminous complexion`.
|
||||
|
||||
**No scene-context repetition per-element.** Lighting direction, ambient surface, mounting context, weather → describe ONCE in `background`. Each element's desc focuses on what's UNIQUE to that element.
|
||||
|
||||
### Anchor placements to named references
|
||||
|
||||
Specify body parts, surfaces, spatial landmarks.
|
||||
- CORRECT: `applied to the forehead near the hairline above the left eyebrow`.
|
||||
- INCORRECT: `pressed against the skin`.
|
||||
- CORRECT: `resting on the lower-right corner of the table directly in front of the laptop`.
|
||||
- INCORRECT: `sitting on the surface`.
|
||||
|
||||
## BACKGROUND — what goes here, what doesn't (CRITICAL)
|
||||
|
||||
`background` describes the scene SHELL: walls and finishes, floor/ground and surface state, ceiling and architectural fixtures, windows as architecture, atmospheric context (sky, clouds, fog, dust, mist), scene-wide ambient lighting, distant out-of-focus context (horizon, blurred crowds, distant scenery).
|
||||
|
||||
### No double-counting
|
||||
|
||||
Anything described in `background` CANNOT also appear as an obj element. Each scene component lives in EXACTLY ONE field. Decide once and commit. Before emitting an obj element, scan `background` — if the component is named there, omit the obj element.
|
||||
|
||||
### ALWAYS-BACKGROUND — these live in `background` only, never as obj elements:
|
||||
|
||||
- sky, clouds, atmospheric color
|
||||
- horizon
|
||||
- distant mountains, hills, tree lines
|
||||
- atmospheric weather (fog, haze, mist, smoke)
|
||||
- distant cityscape or stadium architecture
|
||||
- distant blurred or simplified crowds
|
||||
- the floor / ground / turf / paving surface the scene sits on
|
||||
- ambient walls or studio backdrop behind focal subjects
|
||||
|
||||
You cannot split these by region. `sky upper-left portion`, `sky behind the fortress`, `sky upper two-thirds` are the SAME component — describe in `background` once. Same for crowd, ground, horizon.
|
||||
|
||||
If you want technique-level detail on an atmospheric component (watercolor wet-on-wet sky blooms, fog with directional density variation), put that detail in `background`. The `background` field is allowed to be long.
|
||||
|
||||
### Ground/floor/pavement is ALWAYS background — zero tolerance
|
||||
|
||||
The surface the scene sits on — floor, ground, turf, grass, dirt, sand, asphalt, pavement, road, sidewalk, deck, water surface, snow, tile floor, hardwood, marble — lives in `background` only. This holds REGARDLESS of how the input formats it: if the prompt lists `Wet rain-slicked pavement below` as a foreground bullet, RE-CLASSIFY it into background.
|
||||
|
||||
**Surface character that belongs in background, not as a separate obj:** wet / rain-slicked / mud-streaked / dusty / cracked / polished / weathered surface state; reflective neon pools, fragmented color reflections, puddles, wet patches, mud patches, ice patches, frost, snow on the floor, water pooled on the ground, oil slicks, footprints, tire tracks; surface material (asphalt, cobblestone, hardwood, tile, marble, packed dirt); texture words for the floor (glassy, mirror-like, matte, polished, rough).
|
||||
|
||||
**Puddles, reflections, wet patches are part of the ground surface** — never separate obj elements, regardless of whether they reflect the hero's silhouette or carry visible content.
|
||||
|
||||
**Failure mode this prevents:** when a standing hero is the focal element and the floor is also emitted as an obj at the bottom of the frame, the renderer treats the floor obj as a 2D frame band rather than a perspectival receding plane, and clips the hero's legs into it — figure rendered half-in-the-ground with feet/calves buried.
|
||||
|
||||
**Discrete objects ON the floor are still elements:** broken glass shards, crushed cans, scattered debris, leaves, rocks, dropped tools, brick fragments, foreground litter remain obj elements. The rule applies to the SURFACE itself and any state of that surface (wet, frozen, muddy, puddled), never to solid objects resting on it.
|
||||
|
||||
### Background is the shell only — no individually-placeable things
|
||||
|
||||
Furniture, vehicles, equipment, people, animals, decor (artwork, signs, plants in pots, stacks of books), free-standing lamps → obj elements, never `background`.
|
||||
|
||||
### Shell-affixed prominent objects → DUAL MENTION
|
||||
|
||||
Some objects are simultaneously part of the shell AND focal elements that define the room's identity: a chalkboard covering the back wall of a classroom, a fireplace built into a living-room wall, a large mounted TV, a stage proscenium, a built-in altar, a built-in bookshelf, a large fixed reception desk, a fixed sign/banner.
|
||||
|
||||
For these, MANDATORY all three steps:
|
||||
1. **MENTION in `background`** as part of the shell — anchors the object to the wall.
|
||||
2. **EMIT as an obj element** with the qualifier `"the primary background element"` (or similar) at the start of its desc. The obj carries the detail (material, content, frame, mounting).
|
||||
3. **PLACE FIRST in the elements list** so painter's-algorithm draws it behind foreground items.
|
||||
|
||||
Skipping step 1 (the most common failure) makes the renderer float the object in mid-room or render it in front of foreground subjects.
|
||||
|
||||
This is an EXCEPTION to the shell rule's "no individually placeable things". Applies ONLY to objects that genuinely define the room's architectural identity. Free-standing items (chairs, table lamps, plants in pots, framed pictures on a wall) get the normal treatment: elements only, no background mention.
|
||||
|
||||
### Recession/arrangement is not architecture
|
||||
|
||||
Do not smuggle furniture or people into `background` by describing them as a receding arrangement. Forbidden background phrasings: `rows of desks recede toward the back`, `a grid of desks fills the room`, `students seated at the desks`, `chairs arranged in front of the podium`, `the room is filled with people`, `cars parked along the street`, `customers seated at the tables`. The arrangement IS the foreground content — emit elements.
|
||||
|
||||
### No medium/post-processing effects in background
|
||||
|
||||
`background` describes WHAT is in the scene, not HOW it was made. Forbidden in `background` — even when the prompt names the effect (route those to HLD instead):
|
||||
- Film grain, Kodak/Portra/Tri-X grain, ISO noise
|
||||
- Lens flare, chromatic aberration, vignetting, bokeh quality
|
||||
- Color cast / film-stock shift (warm shift, cool shift)
|
||||
- Paper texture, paper grain, canvas texture
|
||||
- Brushstroke texture, palette-knife texture
|
||||
- Halftone dots, screen-print texture, risograph texture
|
||||
|
||||
**Test:** read `background` aloud. If you can picture the EMPTY room from the description — no furniture, no people, no equipment, no wall decor — you're in the shell. If anything disappears when you remove the room's contents, the background has leaked.
|
||||
|
||||
## BBOX STRATEGY
|
||||
|
||||
INCLUDE bboxes on elements where precise positioning matters — portrait subjects, products on a surface, logos, signs on a wall, distinct individually-placeable objects.
|
||||
|
||||
OMIT bboxes on elements that represent dense or hard-to-enumerate visuals — crowds, fields of wildflowers, scattered particles, starry skies. Per-element judgment.
|
||||
|
||||
### Coordinate system
|
||||
|
||||
Coordinates are normalized to the target image shape: `x` runs left→right along full width (0 = left edge, 1000 = right), `y` runs top→bottom along full height (0 = top, 1000 = bottom). Top-left origin. Format `[y1, x1, y2, x2]` with `y1 < y2`, `x1 < x2`.
|
||||
|
||||
### Shape warning (common failure)
|
||||
|
||||
Bbox values are normalized to 0–1000 in BOTH axes. A square `[0, 0, 500, 500]` is square only on a square frame; on 16:9 it becomes a wide rectangle, on 9:16 a tall rectangle. Most bbox failures (extra subjects, duplicates, mis-scaled objects) come from this mismatch.
|
||||
|
||||
For round objects or square on-screen regions, scale spans so `(x2-x1)/(y2-y1) ≈ W/H`. For single-subject prompts on wide frames, prefer narrower x-spans. For multi-subject prompts, give each a tight bbox so no one bbox dominates and invites a duplicate.
|
||||
|
||||
## SPECIFICITY — commit to one value
|
||||
|
||||
This JSON feeds a diffusion model. Leave nothing for the model to invent or choose.
|
||||
|
||||
**Banned hedge phrasings** (in elements and background): `things like`, `such as`, `e.g.`, `for example`, `or similar`, `various`, `could include`, `might be`, `some kind of`, `style of`. Replace with concrete nouns, counts, colors, materials, poses.
|
||||
|
||||
**Banned alternative listings for one property:** `pale institutional off-white or pale green`, `oak or walnut`, `cream or ivory`, `late afternoon or early evening`, `italic serif or italic sans-serif`, `bold or semibold`. Pick ONE and commit. `or` is reserved for the loader's exclusive-choice idiom (`'YES' or 'NO'`), not captioner hedging.
|
||||
|
||||
**Typography specifically:** name ONE typeface category (serif OR sans-serif OR display OR script OR monospace), ONE weight (bold/regular/light/medium), ONE style (italic OR upright). Never two joined by `or`.
|
||||
|
||||
**Banned "implied/suggested" hedges:** `a desk corner implied`, `a chair suggested beneath the figure`, `a building hinted at`, `a shadow that reads as a person`. If it's in the scene, paint it concretely. If it isn't, leave it out. Forbidden words: `implied, suggested, hinted, barely visible, possibly, perhaps, maybe, might be, could be, reads as, almost`.
|
||||
|
||||
**Exhaustive content preservation.** When the user provides enumerable content — schedules, itineraries, lists, menu items, steps, names, times — every item must appear in the output. Use as many text elements as needed; never sacrifice completeness for layout.
|
||||
|
||||
**Named prompt elements MUST appear.** Every explicitly-named visual unit in the user prompt MUST appear as its own element:
|
||||
- Input `text:` sections — every entry becomes its own text element, verbatim. Zero tolerance: 3 entries in input → ≥3 text elements in output. Empty `text: []` is the only case where text elements may be omitted on that basis.
|
||||
- Quoted strings (single or double quotes) — each is its own text element.
|
||||
- Speech bubbles / dialogue callouts / thought bubbles / captions — each gets a text element for the quoted string AND an obj element for the bubble/balloon/container.
|
||||
- Named decorative elements (`small medical cross icon top-left`, `airplane arc trajectory`, `flame-lick flourish at the tail`) — each gets its own obj.
|
||||
- Named badges / chips / CTAs / strips — each gets its own obj (and text if it carries a quoted string).
|
||||
- Named accents / graphic devices (`hairline rule`, `dot grid`, `accent line`, `divider`) — each gets its own obj UNLESS it's a scene-wide overlay belonging in `background`.
|
||||
|
||||
**Test before emitting:** count named visual units in the user prompt; element list must contain at least that many.
|
||||
|
||||
**No placeholder enumeration.** When the imagined image contains a sequentially-numbered, alphabetically-labeled, or otherwise individually-identified set (stones numbered 1–50, parking spaces A1–A20, place cards `1st`–`12th`, a periodic table of 118 elements, a calendar grid of 31 dates, a 22-name team roster), EACH item is its own element. No `etc.`, no `and so on`, no `6 through 49`, no single obj grouping all into one cluster. List ALL of them.
|
||||
|
||||
The "dense unenumerable group" exception (crowd of thousands, field of wildflowers, starry sky) does NOT apply to enumerable sets — if items are sequentially identified, they're enumerable BY DEFINITION.
|
||||
|
||||
**Don't invent visual concepts the user didn't ask for.** Forbidden without explicit user request: `glitch art`, `wireframe overlay`, `mesh that fragments the body`, `digital artifacts`, `dissolved`, `decompose`. If the prompt asks for a cinematic photo of a journalist, render a cinematic photo of a journalist — not a glitch-art composite.
|
||||
|
||||
## PLANNING — turn the user idea into elements
|
||||
|
||||
### 1. Pick a medium
|
||||
|
||||
`photograph | illustration | 3D render | graphic design` — applies as natural-language framing inside HLD/background, NOT as a structured slot.
|
||||
|
||||
Decision: **DESIGNED artifact vs CAPTURED / DRAWN / RENDERED moment.**
|
||||
- **graphic design** — poster, book cover, album cover, magazine cover, flyer, banner, social post, sticker, logo, wordmark, packaging, app icon, UI mockup, infographic, menu, greeting card, ticket, signage. If a human designer would sit at a desk to make it.
|
||||
- **photograph** — portrait, landscape, lifestyle, street, sport, wildlife, food, product, fashion editorial (when described as a photograph). Default for ambiguous everyday scenes.
|
||||
- **illustration** — cartoon, anime, manga, comic, watercolor, oil painting, ink, vector, pixel art, children's book illustration, named studios (Ghibli, KyoAni, Pixar 2D).
|
||||
- **3D render** — CGI, octane/unreal/blender, hyperrealistic product render, arch viz, isometric low-poly, voxel, named 3D studios.
|
||||
|
||||
Silent / ambiguous → photograph (default). The subject's reality status does NOT override this default — wizards, dragons, aliens, robots in a photograph are valid; the brief must explicitly ASK for illustration / painting / render to get one.
|
||||
|
||||
Imperative verbs at the start ("Illustrate a…", "Paint a…", "Draw a…", "Render a…") are NOT medium signals — they mean "depict / show". Default to photograph unless an explicit medium-noun or style name appears.
|
||||
|
||||
### 2. Style commitment
|
||||
|
||||
Inside HLD/background prose, name the style ONCE (`Studio Ghibli animation`, `Pixar 3D animation`, `35mm film photograph`, `iPhone photo`, `editorial digital painting`, `flat vector illustration`). Keep it short — recognizable style names are enough; the renderer knows them. Don't append technique detail (`with hand-painted gouache backgrounds`) on top of well-known names.
|
||||
|
||||
**"Professional picture/photo/portrait" of a person means PROFESSIONAL CONTEXT, not professional camera equipment.** Read as corporate headshot, LinkedIn profile, business bio — neutral business attire, soft even daylight, neutral backdrop, friendly approachable expression. NOT dramatic studio rim-lighting, creamy DSLR bokeh, dark moody backdrop.
|
||||
|
||||
### 3. Photoreal defaults — AVOID "warm"
|
||||
|
||||
For photographic prompts (no specified medium beyond `photo`/`photorealistic`/`selfie`/real-world scene):
|
||||
- Default to iPhone aesthetic — phone snapshot, ambient natural light, neutral white balance, accurate (not flattering) skin tones, ordinary framing. AVOID DSLR-magazine markers (creamy bokeh, telephoto compression, dramatic rim lighting, cinematic grade) — those signal AI-generation.
|
||||
- Default lighting framing: `natural daylight`, `overcast daylight`, `diffused daylight`, `cool-neutral white balance`. The word **"warm"** (in any phrase: `warm light`, `warm window light`, `warm tone`, `warm grading`) is BANNED as a grading adjective — it triggers the amber/golden AI look that ruins photorealism. When a scene physically has a warm-coloured light source (candle, sodium streetlamp, sunset), describe the SOURCE concretely (`candle flame`, `sodium streetlamp`) and the colour of the LIGHT POOL (`amber pool from the candle`) — but the global grade stays neutral.
|
||||
- Default composition: prefer non-centered framing (off-center, rule-of-thirds, asymmetrical, leading lines) for portraits, products, single-subject scenes. Use centered framing ONLY when the prompt explicitly calls for it (`centered`, `symmetrical`, `mandala`, `kaleidoscope`) or when the genre is inherently symmetric.
|
||||
- No motion blur in candid/realistic/iPhone-aesthetic photos. Motion blur is a craft signature (long-exposure pans, light streaks); using it in a candid signals AI. Real phone snapshots freeze the moment.
|
||||
- Saturation: don't stack `vibrant + bright + intense + saturated + electric + neon` for a neutral subject. Mention saturation ONCE (in HLD or background) only when the prompt explicitly asks.
|
||||
|
||||
### 4. Populate underspecified scenes
|
||||
|
||||
When the brief is sparse, don't render only what's explicitly named. Real scenes are populated. Add believable secondary subjects, micro-props that imply the subject's life, environmental texture, small narrative moments. Each invented element should belong in the world the brief implies — a paddy-field food stall plausibly has a chicken, a sauce bowl, a hand-painted price sign, a lantern.
|
||||
|
||||
**Populate by depth layer.** Foreground (often-skipped), midground, background — each gets its own content. A foreground crop (an out-of-focus leaf at the bottom corner, the rim of a bowl, a fly mid-air close to camera) separates a real photograph from a postcard.
|
||||
|
||||
**Commit to a specific cultural / regional identity.** "Southeast Asian village" is a hedge that produces generic AI visuals. "Vietnamese pho stall by the rice paddies outside Hoi An" is a real place. Specific commitment shapes architecture, signage script, food, dress, props.
|
||||
|
||||
**Built environments need text everywhere.** Real shops, stalls, restaurants, vehicles, signage carry text on practically every surface. Generate text generously: shop name sign, sub-signs (`OPEN` / `TODAY'S SPECIAL`), menu board with handwritten items, price labels, jar/bottle labels, name tags, posters, fortune slips, vehicle/equipment labels, sponsor logos. `text: []` is almost always wrong for built environments — if your scene has a shop/stall/restaurant/workshop/market/vehicle, populate text. Specific content, never `various labels` or `menu items`.
|
||||
|
||||
**Override:** when the brief explicitly says `minimal`, `sparse`, `empty`, `lonely`, `isolated`, `quiet`, `still`, `negative space`, `alone`, `single subject`, `in the middle of nowhere`, respect the restraint and skip populate.
|
||||
|
||||
**Fantastical / sci-fi / fantasy / futuristic briefs get a populate bonus.** Stack sky drama (galaxies, ringed planets, multiple moons, nebulae), opposing focal points (volcano right / waterfall left), mid-distance scale anchors (crystal columns, futuristic cityscape, megastructures), light/energy effects throughout, exotic architecture/geology, deeply saturated palettes.
|
||||
|
||||
## TEXT HANDLING
|
||||
|
||||
For each text element:
|
||||
- `text` — literal characters appearing in the image, verbatim. Preserve diacritics, capitalization, punctuation. Never transliterate or strip.
|
||||
- `bbox` — optional, same coordinate system as obj elements.
|
||||
- `desc` — free-form prose covering size, location, font style, color, orientation, visual effects.
|
||||
|
||||
**Sources of text to include:**
|
||||
1. **User-quoted text** (single OR double quotes) — verbatim, exact characters.
|
||||
2. **Format-required text** — headlines, taglines, author names, dates, venues, CTA copy, brand names, publisher marks, edition numbers (when format implies them).
|
||||
3. **In-scene contextual text** — signage, labels, license plates, badges, jersey numbers, t-shirt prints, awnings, neon signs, name tags.
|
||||
4. **Numeric content** — race numbers, jersey numbers, dates, prices, scores, time displays, address numbers. Numbers ARE text.
|
||||
5. **Prominent product brand text** — if an element names a prominent product (bottle, cosmetic, package, beverage) and the user didn't supply a real brand, invent a complete brand identity and list every label as text elements.
|
||||
|
||||
**Rules:**
|
||||
- Exhaustive: if a viewer could read it, it goes in the list.
|
||||
- Each text element appears ONCE in the list. Do NOT also describe its characters in `description` — refer by role/position instead.
|
||||
- Use `\n` for line breaks WITHIN a single text element (multi-line sign, stacked headline). Use SEPARATE list items for visually distinct text blocks.
|
||||
- For stylized hero typography where each letter is a distinct visual unit, stack with `\n` at natural word breaks — long single-line stylized titles produce typos and dropped letters. e.g., `"ENTRE\nVERSOS E\nCONTOS"` not `"ENTRE VERSOS E CONTOS"`.
|
||||
- **Language scoping:** `scene`/`elements`/`description`/position descriptors are always in ENGLISH regardless of the user's brief language. Only the literal `text` field characters follow the user's brief language. Portuguese brief → English prose + Portuguese `text:` content.
|
||||
|
||||
## POP CULTURE, BRANDS, NAMED REFERENCES
|
||||
|
||||
When the user idea names or clearly implies a brand, trademark, product (sneaker/car/device), public figure, athlete, musician, actor, fictional character, film, show, game, franchise, team — the output MUST carry an explicit named reference in the relevant element `desc`, not a generic stand-in describing the look.
|
||||
|
||||
Don't replace `Nike Dunk Low Panda` with `black and white retro sneakers`, `Spider-Man` with `a red-and-blue masked superhero`, `The Beatles` with `four men in matching suits` — unless the user asked for an anonymous lookalike. Name the specific thing the user pointed at.
|
||||
|
||||
## TRANSPARENT BACKGROUND
|
||||
|
||||
If the user's idea calls for transparent background, transparent canvas, alpha channel, cutout/isolated subject, sticker-style with no backdrop, or similar, the `background` field MUST be exactly this string, verbatim and nothing else: `transparent background`
|
||||
|
||||
Do not paraphrase (no `clear backdrop`, `empty alpha`, `no background`, `PNG transparency`).
|
||||
|
||||
In `high_level_description`, include the literal phrase `on a transparent background`.
|
||||
|
||||
[USER]
|
||||
TARGET IMAGE ASPECT RATIO: {{aspect_ratio}} (width:height).
|
||||
User idea: {{original_prompt}}
|
||||
"""
|
||||
@@ -30,6 +30,8 @@ type Props = {
|
||||
const CaptionSimpleJob: React.FC<Props> = ({ jobConfig, setJobConfig, gpuIDs, setGpuIDs, gpuList, showGPUSelect }) => {
|
||||
const selectedCaptionOption = captionerTypes.find(option => option.name === jobConfig.config.process[0].type);
|
||||
const additionalSections = selectedCaptionOption?.additionalSections || [];
|
||||
const minNewTokens = selectedCaptionOption?.minNewTokens ?? 0;
|
||||
const newTokensOptions = maxNewTokensOptions.filter(option => parseInt(option.value) >= minNewTokens);
|
||||
|
||||
return (
|
||||
<div className="text-sm text-gray-400">
|
||||
@@ -145,7 +147,7 @@ const CaptionSimpleJob: React.FC<Props> = ({ jobConfig, setJobConfig, gpuIDs, se
|
||||
setJobConfig(intVal, 'config.process[0].caption.max_new_tokens');
|
||||
}
|
||||
}}
|
||||
options={maxNewTokensOptions}
|
||||
options={newTokensOptions}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
import { useState, useEffect, useMemo, useCallback, useRef } from 'react';
|
||||
import { createPortal } from 'react-dom';
|
||||
import { Dialog, DialogBackdrop, DialogPanel } from '@headlessui/react';
|
||||
import { Cog } from 'lucide-react';
|
||||
import { Cog, SquareDashed } from 'lucide-react';
|
||||
import { Menu, MenuButton, MenuItem, MenuItems } from '@headlessui/react';
|
||||
import classNames from 'classnames';
|
||||
import { openConfirm } from './ConfirmModal';
|
||||
@@ -11,6 +11,42 @@ import { isVideo, isAudio } from '@/utils/basic';
|
||||
import AudioPlayer from './AudioPlayer';
|
||||
import { TransformWrapper, TransformComponent } from 'react-zoom-pan-pinch';
|
||||
|
||||
// A single bbox parsed from an Ideogram-style caption. Stored coords are
|
||||
// normalized 0-1000 in [y1, x1, y2, x2] order (top-left origin).
|
||||
interface OverlayBox {
|
||||
y1: number;
|
||||
x1: number;
|
||||
y2: number;
|
||||
x2: number;
|
||||
label: string;
|
||||
type: 'obj' | 'text';
|
||||
}
|
||||
|
||||
// Returns the list of boxes if the caption is an Ideogram bbox-JSON caption with
|
||||
// at least one bbox, otherwise null (normal captions get no overlay).
|
||||
function parseBoundingBoxes(caption: string): OverlayBox[] | null {
|
||||
const trimmed = caption.trim();
|
||||
if (!trimmed.startsWith('{')) return null;
|
||||
let data: any;
|
||||
try {
|
||||
data = JSON.parse(trimmed);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
const elements = data?.compositional_deconstruction?.elements;
|
||||
if (!Array.isArray(elements)) return null;
|
||||
const boxes: OverlayBox[] = [];
|
||||
for (const el of elements) {
|
||||
const bb = el?.bbox;
|
||||
if (Array.isArray(bb) && bb.length === 4 && bb.every((n: any) => typeof n === 'number')) {
|
||||
const isText = el.type === 'text';
|
||||
const label = (isText ? el.text : el.desc) ?? '';
|
||||
boxes.push({ y1: bb[0], x1: bb[1], y2: bb[2], x2: bb[3], label: `${label}`, type: isText ? 'text' : 'obj' });
|
||||
}
|
||||
}
|
||||
return boxes.length > 0 ? boxes : null;
|
||||
}
|
||||
|
||||
interface Props {
|
||||
imgPath: string | null; // current image path
|
||||
imageList: string[]; // all dataset image paths
|
||||
@@ -25,6 +61,7 @@ export default function DatasetImageViewer({ imgPath, imageList, onChange, refre
|
||||
const [caption, setCaption] = useState<string>('');
|
||||
const [savedCaption, setSavedCaption] = useState<string>('');
|
||||
const [isCaptionLoaded, setIsCaptionLoaded] = useState<boolean>(false);
|
||||
const [showBoxes, setShowBoxes] = useState<boolean>(false);
|
||||
const captionRef = useRef<string>('');
|
||||
const savedCaptionRef = useRef<string>('');
|
||||
const currentImgPathRef = useRef<string | null>(null);
|
||||
@@ -114,11 +151,14 @@ export default function DatasetImageViewer({ imgPath, imageList, onChange, refre
|
||||
setSavedCaption('');
|
||||
|
||||
apiClient
|
||||
.post('/api/caption/get', { imgPath }, { signal: controller.signal })
|
||||
// transformResponse identity: keep the caption as a raw string. Axios's
|
||||
// default parses any JSON-looking body into an object (our bbox captions
|
||||
// are JSON), which would render as "[object Object]".
|
||||
.post('/api/caption/get', { imgPath }, { signal: controller.signal, transformResponse: [d => d] })
|
||||
.then(res => res.data)
|
||||
.then(data => {
|
||||
if (controller.signal.aborted) return;
|
||||
const text = data ? `${data}` : '';
|
||||
const text = typeof data === 'string' ? data : data ? `${data}` : '';
|
||||
setCaption(text);
|
||||
setSavedCaption(text);
|
||||
setIsCaptionLoaded(true);
|
||||
@@ -277,6 +317,8 @@ export default function DatasetImageViewer({ imgPath, imageList, onChange, refre
|
||||
};
|
||||
|
||||
const isCaptionCurrent = caption.trim() === savedCaption.trim();
|
||||
const boundingBoxes = useMemo(() => parseBoundingBoxes(caption), [caption]);
|
||||
const canShowBoxes = Boolean(boundingBoxes && imgPath && !isAudio(imgPath) && !isVideo(imgPath));
|
||||
|
||||
if (!mounted) return null;
|
||||
|
||||
@@ -324,12 +366,48 @@ export default function DatasetImageViewer({ imgPath, imageList, onChange, refre
|
||||
}}
|
||||
>
|
||||
<TransformComponent>
|
||||
<img
|
||||
src={`/api/img/${encodeURIComponent(imgPath)}`}
|
||||
alt="Dataset Image"
|
||||
draggable={false}
|
||||
className="w-auto h-auto max-w-full sm:max-w-[95vw] max-h-[70vh] object-contain select-none !pointer-events-auto"
|
||||
/>
|
||||
<div className="relative">
|
||||
<img
|
||||
src={`/api/img/${encodeURIComponent(imgPath)}`}
|
||||
alt="Dataset Image"
|
||||
draggable={false}
|
||||
className="w-auto h-auto max-w-full sm:max-w-[95vw] max-h-[70vh] object-contain select-none !pointer-events-auto"
|
||||
/>
|
||||
{showBoxes && boundingBoxes && (
|
||||
<div className="absolute inset-0 pointer-events-none">
|
||||
{boundingBoxes.map((b, i) => (
|
||||
<div
|
||||
key={i}
|
||||
className={classNames('absolute border', {
|
||||
'border-cyan-400': b.type === 'obj',
|
||||
'border-amber-400': b.type === 'text',
|
||||
})}
|
||||
style={{
|
||||
left: `${b.x1 / 10}%`,
|
||||
top: `${b.y1 / 10}%`,
|
||||
width: `${(b.x2 - b.x1) / 10}%`,
|
||||
height: `${(b.y2 - b.y1) / 10}%`,
|
||||
}}
|
||||
>
|
||||
{b.label && (
|
||||
<span
|
||||
title={b.label}
|
||||
className={classNames(
|
||||
'absolute top-0 left-0 max-w-full px-1 py-0.5 text-[9px] leading-tight font-medium whitespace-pre-line break-words line-clamp-3 text-gray-900',
|
||||
{
|
||||
'bg-cyan-400/90': b.type === 'obj',
|
||||
'bg-amber-400/90': b.type === 'text',
|
||||
},
|
||||
)}
|
||||
>
|
||||
{b.label}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</TransformComponent>
|
||||
</TransformWrapper>
|
||||
))}
|
||||
@@ -362,33 +440,48 @@ export default function DatasetImageViewer({ imgPath, imageList, onChange, refre
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
<div className="absolute top-2 right-2 bg-gray-900 rounded-full p-1 leading-[0px] opacity-50 hover:opacity-100 z-20">
|
||||
<Menu>
|
||||
<MenuButton>
|
||||
<Cog />
|
||||
</MenuButton>
|
||||
<MenuItems
|
||||
anchor="bottom end"
|
||||
className="bg-gray-900 border border-gray-700 rounded shadow-lg w-48 px-2 py-2 mt-1 z-50"
|
||||
<div className="absolute top-2 right-2 flex items-center gap-2 z-20">
|
||||
{canShowBoxes && (
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setShowBoxes(v => !v)}
|
||||
title={showBoxes ? 'Hide bounding boxes' : 'Show bounding boxes'}
|
||||
className={classNames('bg-gray-900 rounded-full p-1 leading-[0px] hover:opacity-100', {
|
||||
'opacity-100 text-blue-400': showBoxes,
|
||||
'opacity-50': !showBoxes,
|
||||
})}
|
||||
>
|
||||
{imgPath && isAudio(imgPath) && (
|
||||
<SquareDashed />
|
||||
</button>
|
||||
)}
|
||||
<div className="bg-gray-900 rounded-full p-1 leading-[0px] opacity-50 hover:opacity-100">
|
||||
<Menu>
|
||||
<MenuButton>
|
||||
<Cog />
|
||||
</MenuButton>
|
||||
<MenuItems
|
||||
anchor="bottom end"
|
||||
className="bg-gray-900 border border-gray-700 rounded shadow-lg w-48 px-2 py-2 mt-1 z-50"
|
||||
>
|
||||
{imgPath && isAudio(imgPath) && (
|
||||
<MenuItem>
|
||||
<a
|
||||
className="cursor-pointer px-4 py-1 hover:bg-gray-800 rounded block"
|
||||
href={`/api/img/${encodeURIComponent(imgPath)}`}
|
||||
download={filename}
|
||||
>
|
||||
Download
|
||||
</a>
|
||||
</MenuItem>
|
||||
)}
|
||||
<MenuItem>
|
||||
<a
|
||||
className="cursor-pointer px-4 py-1 hover:bg-gray-800 rounded block"
|
||||
href={`/api/img/${encodeURIComponent(imgPath)}`}
|
||||
download={filename}
|
||||
>
|
||||
Download
|
||||
</a>
|
||||
<div className="cursor-pointer px-4 py-1 hover:bg-gray-800 rounded" onClick={handleDelete}>
|
||||
Delete Image
|
||||
</div>
|
||||
</MenuItem>
|
||||
)}
|
||||
<MenuItem>
|
||||
<div className="cursor-pointer px-4 py-1 hover:bg-gray-800 rounded" onClick={handleDelete}>
|
||||
Delete Image
|
||||
</div>
|
||||
</MenuItem>
|
||||
</MenuItems>
|
||||
</Menu>
|
||||
</MenuItems>
|
||||
</Menu>
|
||||
</div>
|
||||
</div>
|
||||
</DialogPanel>
|
||||
</div>
|
||||
|
||||
@@ -8,6 +8,7 @@ export interface CaptionOption {
|
||||
label: string;
|
||||
group: CaptionGroup;
|
||||
hasMultiLinePrompts?: boolean;
|
||||
minNewTokens?: number;
|
||||
defaults?: { [key: string]: any };
|
||||
additionalSections?: AdditionalSections[];
|
||||
name_or_path_options?: SelectOption[];
|
||||
@@ -23,6 +24,11 @@ const defaultExtensions = [...extensionsImage];
|
||||
|
||||
const defaultImageCaptionPrompt = "Caption this image as if you were going to try to generate it with an image generator. Be thurough and describe everything in the image. Be decisive by stating things as they are. Do not say things like \"It appears that\" Or \"possibly\". Start out with things like \"A person on the beach\" or \"A black dragon\". No preamble. Just get to the point.";
|
||||
|
||||
// Editable ADDITIONAL INSTRUCTIONS block injected into the Ideogram system prompt.
|
||||
// Users can tweak this for dataset-specific guidance without altering the fixed
|
||||
// output contract, element/background rules, or bbox format.
|
||||
const defaultIdeogramCaptionPrompt = "Describe only what is actually visible in the image — never invent, add, or infer content that is not present. Identify the medium (photograph, illustration, 3D render, or graphic design) and name any recognizable people, brands, characters, or landmarks. Be decisive and specific: commit to one concrete value per attribute (one color, one material, one pose) and avoid hedging. Transcribe every piece of legible text verbatim.";
|
||||
|
||||
export const captionerTypes: CaptionOption[] = [
|
||||
{
|
||||
name: 'AceStepCaptioner',
|
||||
@@ -68,6 +74,32 @@ export const captionerTypes: CaptionOption[] = [
|
||||
'caption.max_new_tokens',
|
||||
],
|
||||
},
|
||||
{
|
||||
name: 'Ideogram4Captioner',
|
||||
label: 'Ideogram 4 Captioner',
|
||||
group: 'image',
|
||||
hasMultiLinePrompts: true,
|
||||
// The deconstruction JSON is long; the Python captioner also enforces this floor.
|
||||
minNewTokens: 3072,
|
||||
defaults: {
|
||||
'config.process[0].caption.model_name_or_path': ['Qwen/Qwen3-VL-8B-Instruct', defaultNameOrPath],
|
||||
'config.process[0].caption.extensions': [extensionsImage, defaultExtensions],
|
||||
'config.process[0].caption.caption_prompt': [defaultIdeogramCaptionPrompt, undefined],
|
||||
'config.process[0].caption.max_res': [512, undefined],
|
||||
'config.process[0].caption.max_new_tokens': [4096, undefined],
|
||||
},
|
||||
name_or_path_options: [
|
||||
{ value: 'Qwen/Qwen3-VL-2B-Instruct', label: 'Qwen/Qwen3-VL-2B-Instruct' },
|
||||
{ value: 'Qwen/Qwen3-VL-4B-Instruct', label: 'Qwen/Qwen3-VL-4B-Instruct' },
|
||||
{ value: 'Qwen/Qwen3-VL-8B-Instruct', label: 'Qwen/Qwen3-VL-8B-Instruct' },
|
||||
{ value: 'Qwen/Qwen3-VL-30B-A3B-Instruct', label: 'Qwen/Qwen3-VL-30B-A3B-Instruct' },
|
||||
],
|
||||
additionalSections: [
|
||||
'caption.caption_prompt',
|
||||
'caption.max_res',
|
||||
'caption.max_new_tokens',
|
||||
],
|
||||
},
|
||||
|
||||
].sort((a, b) => {
|
||||
// Sort by label, case-insensitive
|
||||
@@ -110,6 +142,10 @@ export const maxNewTokensOptions: SelectOption[] = [
|
||||
{ value: '256', label: '256' },
|
||||
{ value: '512', label: '512' },
|
||||
{ value: '1024', label: '1024' },
|
||||
{ value: '2048', label: '2048' },
|
||||
{ value: '3072', label: '3072' },
|
||||
{ value: '4096', label: '4096' },
|
||||
{ value: '8192', label: '8192' },
|
||||
];
|
||||
|
||||
export const defaultQtype = 'float8';
|
||||
@@ -1 +1 @@
|
||||
VERSION = "0.10.1"
|
||||
VERSION = "0.10.2"
|
||||
|
||||
Reference in New Issue
Block a user