Update the Ideogram 4 prompt generation/parsing/ui to handle the updated format notes better.
This commit is contained in:
1 parent
fcccc0fbd2
commit
86b19589a0
10 files changed
+497
-155
No files matched your search
@@ -8,6 +8,7 @@ from PIL import Image
|
||||
|
||||
from .Qwen3VLCaptioner import Qwen3VLCaptioner
|
||||
from .prompts.ideogram4_caption_prompt import ideogram4_caption_prompt
|
||||
from toolkit.ideogram_caption import normalize_caption_dict
|
||||
import transformers
|
||||
import logging
|
||||
import warnings
|
||||
@@ -25,11 +26,6 @@ MIN_NEW_TOKENS = 3072
|
||||
# generator was trained on, instead of ugly fractions like 1023:768.
|
||||
MAX_AR_DENOMINATOR = 16
|
||||
|
||||
# color_palette caps: the model often ignores these, so we enforce them.
|
||||
MAX_IMAGE_PALETTE = 16 # style_description.color_palette
|
||||
MAX_ELEMENT_PALETTE = 5 # per-element color_palette
|
||||
HEX_COLOR_RE = re.compile(r"^#(?:[0-9a-fA-F]{3}|[0-9a-fA-F]{6})$")
|
||||
|
||||
|
||||
class Ideogram4Captioner(Qwen3VLCaptioner):
|
||||
def __init__(self, process_id: int, job, config: OrderedDict, **kwargs):
|
||||
@@ -109,65 +105,23 @@ class Ideogram4Captioner(Qwen3VLCaptioner):
|
||||
# stored order is [y1, x1, y2, x2]
|
||||
return [y1, x1, y2, x2]
|
||||
|
||||
def _sanitize_palette(self, palette, max_len):
|
||||
"""Keep unique, valid hex colors in order, capped to max_len. Returns the
|
||||
cleaned list, or None if nothing valid remains (drop the key)."""
|
||||
if not isinstance(palette, (list, tuple)):
|
||||
return None
|
||||
seen = set()
|
||||
out = []
|
||||
for c in palette:
|
||||
if not isinstance(c, str):
|
||||
continue
|
||||
c = c.strip()
|
||||
if not HEX_COLOR_RE.match(c):
|
||||
continue
|
||||
key = c.lower()
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
out.append(c)
|
||||
if len(out) >= max_len:
|
||||
break
|
||||
return out or None
|
||||
|
||||
def _normalize_caption(self, data: dict) -> dict:
|
||||
"""Validate/cleanup the parsed caption before storage: drop input-only
|
||||
aspect_ratio, reorder bboxes to [y1,x1,y2,x2], and cap color palettes
|
||||
(16 per image, 5 per element) since the model often exceeds them."""
|
||||
# aspect_ratio is input-only context, not part of the output. Drop it if
|
||||
# the model echoed it anyway.
|
||||
data.pop("aspect_ratio", None)
|
||||
|
||||
style = data.get("style_description")
|
||||
if isinstance(style, dict) and "color_palette" in style:
|
||||
pal = self._sanitize_palette(style["color_palette"], MAX_IMAGE_PALETTE)
|
||||
if pal is None:
|
||||
style.pop("color_palette", None)
|
||||
else:
|
||||
style["color_palette"] = pal
|
||||
|
||||
"""Cleanup the parsed caption before storage. The model emits bboxes in
|
||||
[x1,y1,x2,y2]; convert each to our stored [y1,x1,y2,x2] order, then hand off
|
||||
to the shared normalizer for the rest: drop aspect_ratio, enforce the
|
||||
photo/art_style branch and key order, canonicalize medium, and cap/uppercase
|
||||
color palettes (16 per image, 5 per element)."""
|
||||
decon = data.get("compositional_deconstruction", {})
|
||||
elements = decon.get("elements", [])
|
||||
elements = decon.get("elements", []) if isinstance(decon, dict) else []
|
||||
if isinstance(elements, list):
|
||||
for el in elements:
|
||||
if not isinstance(el, dict):
|
||||
continue
|
||||
if "bbox" in el:
|
||||
if isinstance(el, dict) and "bbox" in el:
|
||||
cleaned = self._convert_bbox(el["bbox"])
|
||||
if cleaned is None:
|
||||
el.pop("bbox", None)
|
||||
else:
|
||||
el["bbox"] = cleaned
|
||||
if "color_palette" in el:
|
||||
pal = self._sanitize_palette(
|
||||
el["color_palette"], MAX_ELEMENT_PALETTE
|
||||
)
|
||||
if pal is None:
|
||||
el.pop("color_palette", None)
|
||||
else:
|
||||
el["color_palette"] = pal
|
||||
return data
|
||||
return normalize_caption_dict(data)
|
||||
|
||||
def get_caption_for_file(self, file_path: str) -> Optional[str]:
|
||||
try:
|
||||
|
||||
@@ -19,7 +19,7 @@ You are CAPTIONING a real image, not imagining one. Describe ONLY what is visibl
|
||||
## OUTPUT CONTRACT — exactly three top-level keys, in this order:
|
||||
|
||||
```json
|
||||
{"high_level_description":"...","style_description":{"aesthetics":"...","lighting":"...","photo":"...","medium":"...","color_palette":["#RRGGBB"]},"compositional_deconstruction":{"background":"...","elements":[ ... ]}}
|
||||
{"high_level_description":"...","style_description":{ ...see STYLE DESCRIPTION... },"compositional_deconstruction":{"background":"...","elements":[ ... ]}}
|
||||
```
|
||||
|
||||
- Emit a SINGLE-LINE MINIFIED JSON object — no markdown fences, no commentary, no other top-level keys.
|
||||
@@ -44,22 +44,34 @@ BAD (over-specifies): `A male soccer player captured mid-kick on a bright green
|
||||
|
||||
## STYLE DESCRIPTION — the `style_description` block (always required)
|
||||
|
||||
A nested object capturing the image's overall look, OBSERVED from the image (never invented). Exactly these five keys:
|
||||
- `aesthetics` — the overall mood/aesthetic in a short phrase (`Cinematic, minimal, serene.` / `Bright, playful, high-energy.`).
|
||||
A nested object capturing the image's overall look, OBSERVED from the image (never invented). It carries EXACTLY ONE render key — `photo` for photographs, `art_style` for everything else (illustration / 3D render / painting / graphic design) — NEVER both. The key order is strict and depends on the branch:
|
||||
|
||||
- **Photograph** → keys in this order: `aesthetics`, `lighting`, `photo`, `medium`, `color_palette`
|
||||
```json
|
||||
{"aesthetics":"...","lighting":"...","photo":"...","medium":"photograph","color_palette":["#RRGGBB"]}
|
||||
```
|
||||
- **Non-photo** (illustration / 3D / painting / graphic design) → keys in this order: `aesthetics`, `lighting`, `medium`, `art_style`, `color_palette`
|
||||
```json
|
||||
{"aesthetics":"...","lighting":"...","medium":"illustration","art_style":"...","color_palette":["#RRGGBB"]}
|
||||
```
|
||||
|
||||
Field meanings:
|
||||
- `aesthetics` — the overall mood/aesthetic in a short phrase (`cinematic, minimal, serene` / `bright, playful, high-energy`).
|
||||
- `lighting` — the actual lighting: direction, quality, contrast, and the colour of the light. Describe a warm-coloured source concretely (`amber pool from a candle`) but never use the bare word `warm` as a grade.
|
||||
- `photo` — the medium-specific capture/render spec. Photograph → camera/film look, framing, grain, focus (`35mm film still, 16:9 framing, subtle grain, shallow depth of field`). Other media → the rendering technique (`flat vector, clean edges` / `octane 3D render, soft global illumination` / `loose watercolor on textured paper`).
|
||||
- `medium` — one short phrase: `Photograph.` / `Illustration.` / `3D render.` / `Graphic design.` Read it from the image; do not impose a default.
|
||||
- `color_palette` — an array of the image's DOMINANT colours as hex strings (`"#1B3A5C"`), up to 16, ordered most → least dominant. Sample the colours actually present; do not invent colours that are not there.
|
||||
- `photo` (photographs ONLY) — the camera/film capture spec: framing, grain, focus (`35mm film still, 16:9 framing, subtle grain, shallow depth of field`).
|
||||
- `art_style` (non-photo ONLY) — the rendering technique (`flat vector, clean edges` / `octane 3D render, soft global illumination` / `loose watercolor on textured paper`).
|
||||
- `medium` — exactly one token: `photograph` / `illustration` / `3d_render` / `painting` / `graphic_design`. Read it from the image; do not impose a default. Photograph ⇒ use `photo`; any other ⇒ use `art_style`.
|
||||
- `color_palette` — an array of the image's DOMINANT colours as UPPERCASE `#RRGGBB` hex strings (`"#1B3A5C"`), up to 16, ordered most → least dominant. Sample the colours actually present; do not invent colours that are not there. ALWAYS the last key.
|
||||
|
||||
## ELEMENTS — what they are, what they're not
|
||||
|
||||
Each element is one of:
|
||||
Each element is one of (keys in EXACTLY this order):
|
||||
```
|
||||
{"type":"obj","bbox":[x1,y1,x2,y2],"color_palette":["#RRGGBB"],"desc":"..."}
|
||||
{"type":"text","bbox":[x1,y1,x2,y2],"color_palette":["#RRGGBB"],"text":"LINE ONE\\nLINE TWO","desc":"..."}
|
||||
{"type":"obj","bbox":[x1,y1,x2,y2],"desc":"...","color_palette":["#RRGGBB"]}
|
||||
{"type":"text","bbox":[x1,y1,x2,y2],"text":"LINE ONE\\nLINE TWO","desc":"...","color_palette":["#RRGGBB"]}
|
||||
```
|
||||
|
||||
`bbox` and `color_palette` are both OPTIONAL per-element. `bbox`: see BBOX section below. `color_palette`: up to 5 hex strings of that element's own dominant colours — include it when the element has distinctive colours worth pinning (a red jacket, a brand logo, coloured text), omit it for colour-neutral elements.
|
||||
`bbox` and `color_palette` are both OPTIONAL per-element; when present they keep the order shown above (`color_palette` is always LAST). `bbox`: see BBOX section below. `color_palette`: up to 5 UPPERCASE `#RRGGBB` strings of that element's own dominant colours — include it when the element has distinctive colours worth pinning (a red jacket, a brand logo, coloured text), omit it for colour-neutral elements.
|
||||
|
||||
### SINGLE SUBJECT = SINGLE ELEMENT
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@ You convert a natural-language user idea into a structured JSON caption an image
|
||||
## OUTPUT CONTRACT — exactly three top-level keys, in this order:
|
||||
|
||||
```json
|
||||
{"high_level_description":"...","style_description":{"aesthetics":"...","lighting":"...","photo":"...","medium":"...","color_palette":["#RRGGBB"]},"compositional_deconstruction":{"background":"...","elements":[ ... ]}}
|
||||
{"high_level_description":"...","style_description":{ ...see style_description... },"compositional_deconstruction":{"background":"...","elements":[ ... ]}}
|
||||
```
|
||||
|
||||
- Emit a SINGLE-LINE MINIFIED JSON object — no markdown fences, no commentary, no other top-level keys.
|
||||
@@ -35,24 +35,29 @@ BAD (over-specifies): `A male soccer player captured mid-kick on a bright green
|
||||
|
||||
### `style_description` — the global look block (always required)
|
||||
|
||||
A nested object with exactly these five keys:
|
||||
- `aesthetics` — overall mood/aesthetic in a short phrase (`Cinematic, minimal, serene.`).
|
||||
A nested object carrying EXACTLY ONE render key — `photo` for photographs, `art_style` for everything else — NEVER both. Key order is strict and branch-dependent:
|
||||
|
||||
- **Photograph** → `aesthetics`, `lighting`, `photo`, `medium`, `color_palette`
|
||||
- **Non-photo** (illustration / 3D / painting / graphic design) → `aesthetics`, `lighting`, `medium`, `art_style`, `color_palette`
|
||||
|
||||
- `aesthetics` — overall mood/aesthetic in a short phrase (`cinematic, minimal, serene`).
|
||||
- `lighting` — direction, quality, contrast, and colour of the light. Describe a warm-coloured source concretely (`amber sun low at the horizon`); never use the bare word `warm` as a grade.
|
||||
- `photo` — the medium-specific capture/render spec. Photograph → camera/film look, framing, grain, focus (`35mm motion-picture film still, 16:9 framing, subtle grain`). Other media → the rendering technique (`flat vector, clean edges`; `octane 3D render`; `loose watercolor on textured paper`).
|
||||
- `medium` — one short phrase: `Photograph.` / `Illustration.` / `3D render.` / `Graphic design.`
|
||||
- `color_palette` — an array of the dominant colours as hex strings (`"#1B3A5C"`), up to 16, ordered most → least dominant. This conditions the image's colours directly, so commit to the actual hexes you intend.
|
||||
- `photo` (photographs ONLY) — the camera/film capture spec: framing, grain, focus (`35mm motion-picture film still, 16:9 framing, subtle grain`).
|
||||
- `art_style` (non-photo ONLY) — the rendering technique (`flat vector, clean edges`; `octane 3D render`; `loose watercolor on textured paper`).
|
||||
- `medium` — exactly one token: `photograph` / `illustration` / `3d_render` / `painting` / `graphic_design`. Photograph ⇒ use `photo`; any other ⇒ use `art_style`.
|
||||
- `color_palette` — an array of the dominant colours as UPPERCASE `#RRGGBB` hex strings (`"#1B3A5C"`), up to 16, ordered most → least dominant. This conditions the image's colours directly, so commit to the actual hexes you intend. ALWAYS the last key.
|
||||
|
||||
Name a recognized style ONCE here (see PLANNING → Style commitment); do not append invented technique detail on top of a well-known style name.
|
||||
|
||||
## ELEMENTS — what they are, what they're not
|
||||
|
||||
Each element is one of:
|
||||
Each element is one of (keys in EXACTLY this order):
|
||||
```
|
||||
{"type":"obj","bbox":[y1,x1,y2,x2],"color_palette":["#RRGGBB"],"desc":"..."}
|
||||
{"type":"text","bbox":[y1,x1,y2,x2],"color_palette":["#RRGGBB"],"text":"LINE ONE\nLINE TWO","desc":"..."}
|
||||
{"type":"obj","bbox":[y1,x1,y2,x2],"desc":"...","color_palette":["#RRGGBB"]}
|
||||
{"type":"text","bbox":[y1,x1,y2,x2],"text":"LINE ONE\nLINE TWO","desc":"...","color_palette":["#RRGGBB"]}
|
||||
```
|
||||
|
||||
`bbox` and `color_palette` are both OPTIONAL per-element. `bbox`: see BBOX section below. `color_palette`: up to 5 hex strings steering that element's own dominant colours — include it when the element has a distinctive colour (a red jacket, a brand logo, coloured text), omit it otherwise.
|
||||
`bbox` and `color_palette` are both OPTIONAL per-element; when present they keep the order shown (`color_palette` is always LAST). `bbox`: see BBOX section below. `color_palette`: up to 5 UPPERCASE `#RRGGBB` strings steering that element's own dominant colours — include it when the element has a distinctive colour (a red jacket, a brand logo, coloured text), omit it otherwise.
|
||||
|
||||
### SINGLE SUBJECT = SINGLE ELEMENT
|
||||
|
||||
@@ -224,13 +229,14 @@ The "dense unenumerable group" exception (crowd of thousands, field of wildflowe
|
||||
|
||||
### 1. Pick a medium
|
||||
|
||||
`photograph | illustration | 3D render | graphic design` — applies as natural-language framing inside HLD/background, NOT as a structured slot.
|
||||
`photograph | illustration | 3d_render | painting | graphic_design` — this is the `medium` token (photograph ⇒ `photo`, all others ⇒ `art_style`), and it also frames HLD/background prose naturally.
|
||||
|
||||
Decision: **DESIGNED artifact vs CAPTURED / DRAWN / RENDERED moment.**
|
||||
- **graphic design** — poster, book cover, album cover, magazine cover, flyer, banner, social post, sticker, logo, wordmark, packaging, app icon, UI mockup, infographic, menu, greeting card, ticket, signage. If a human designer would sit at a desk to make it.
|
||||
- **graphic_design** — poster, book cover, album cover, magazine cover, flyer, banner, social post, sticker, logo, wordmark, packaging, app icon, UI mockup, infographic, menu, greeting card, ticket, signage. If a human designer would sit at a desk to make it.
|
||||
- **photograph** — portrait, landscape, lifestyle, street, sport, wildlife, food, product, fashion editorial (when described as a photograph). Default for ambiguous everyday scenes.
|
||||
- **illustration** — cartoon, anime, manga, comic, watercolor, oil painting, ink, vector, pixel art, children's book illustration, named studios (Ghibli, KyoAni, Pixar 2D).
|
||||
- **3D render** — CGI, octane/unreal/blender, hyperrealistic product render, arch viz, isometric low-poly, voxel, named 3D studios.
|
||||
- **illustration** — cartoon, anime, manga, comic, ink, vector, pixel art, children's book illustration, named studios (Ghibli, KyoAni, Pixar 2D).
|
||||
- **painting** — watercolor, oil, gouache, acrylic, traditional painterly work.
|
||||
- **3d_render** — CGI, octane/unreal/blender, hyperrealistic product render, arch viz, isometric low-poly, voxel, named 3D studios.
|
||||
|
||||
Silent / ambiguous → photograph (default). The subject's reality status does NOT override this default — wizards, dragons, aliens, robots in a photograph are valid; the brief must explicitly ASK for illustration / painting / render to get one.
|
||||
|
||||
|
||||
@@ -17,7 +17,7 @@ You convert a user prompt into a structured JSON caption an image renderer can c
|
||||
## OUTPUT CONTRACT — exactly three top-level keys, in this order:
|
||||
|
||||
```json
|
||||
{"high_level_description":"...","style_description":{"aesthetics":"...","lighting":"...","photo":"...","medium":"...","color_palette":["#RRGGBB"]},"compositional_deconstruction":{"background":"...","elements":[ ... ]}}
|
||||
{"high_level_description":"...","style_description":{ ...see STYLE DESCRIPTION... },"compositional_deconstruction":{"background":"...","elements":[ ... ]}}
|
||||
```
|
||||
|
||||
- Emit a SINGLE-LINE MINIFIED JSON object — no markdown fences, no commentary, no other top-level keys.
|
||||
@@ -34,23 +34,29 @@ One short sentence, reads like a natural prompt, starts with the subject — no
|
||||
|
||||
## STYLE DESCRIPTION — the `style_description` block (always required)
|
||||
|
||||
A nested object with exactly these five keys, filled FROM the prompt:
|
||||
A nested object, filled FROM the prompt. It carries EXACTLY ONE render key — `photo` for photographs, `art_style` for everything else — NEVER both. Key order is strict and branch-dependent:
|
||||
|
||||
- **Photograph** → `aesthetics`, `lighting`, `photo`, `medium`, `color_palette`
|
||||
- **Non-photo** (illustration / 3D / painting / graphic design) → `aesthetics`, `lighting`, `medium`, `art_style`, `color_palette`
|
||||
|
||||
Fields:
|
||||
- `aesthetics` — the overall mood/aesthetic in a short phrase.
|
||||
- `lighting` — the lighting (direction, quality, colour). Describe a warm-coloured source concretely; never use the bare word `warm` as a grade.
|
||||
- `photo` — the medium-specific capture/render spec (photograph → camera/film look, framing, grain, focus; other media → the rendering technique).
|
||||
- `medium` — one short phrase: `Photograph.` / `Illustration.` / `3D render.` / `Graphic design.`
|
||||
- `color_palette` — an array of dominant colours as hex strings (`"#1B3A5C"`), up to 16, ordered most → least dominant.
|
||||
- `photo` (photographs ONLY) — the camera/film capture spec (framing, grain, focus).
|
||||
- `art_style` (non-photo ONLY) — the rendering technique (`flat vector, clean edges`; `octane 3D render`; `loose watercolor`).
|
||||
- `medium` — exactly one token: `photograph` / `illustration` / `3d_render` / `painting` / `graphic_design`. Photograph ⇒ use `photo`; any other ⇒ use `art_style`.
|
||||
- `color_palette` — an array of dominant colours as UPPERCASE `#RRGGBB` strings (`"#1B3A5C"`), up to 16, ordered most → least dominant. ALWAYS the last key.
|
||||
|
||||
Respect FIDELITY: if the prompt NAMES a style, medium, artist, or look, put it in these fields BY NAME (e.g. `medium`/`photo`/`aesthetics`) and do NOT invent its characteristics. Pull lighting and colours from what the prompt states. In faithful mode, only commit to a value the prompt implies, keeping the rest minimal; in creative mode you may infer fitting style values — but never elaborate a named style and never override what the user gave.
|
||||
Respect FIDELITY: if the prompt NAMES a style, medium, artist, or look, put it in these fields BY NAME (e.g. `medium`/`art_style`/`aesthetics`) and do NOT invent its characteristics. Pull lighting and colours from what the prompt states. In faithful mode, only commit to a value the prompt implies, keeping the rest minimal; in creative mode you may infer fitting style values — but never elaborate a named style and never override what the user gave.
|
||||
|
||||
## ELEMENTS
|
||||
|
||||
Each element is one of:
|
||||
Each element is one of (keys in EXACTLY this order):
|
||||
```
|
||||
{"type":"obj","bbox":[y1,x1,y2,x2],"color_palette":["#RRGGBB"],"desc":"..."}
|
||||
{"type":"text","bbox":[y1,x1,y2,x2],"color_palette":["#RRGGBB"],"text":"LINE ONE\nLINE TWO","desc":"..."}
|
||||
{"type":"obj","bbox":[y1,x1,y2,x2],"desc":"...","color_palette":["#RRGGBB"]}
|
||||
{"type":"text","bbox":[y1,x1,y2,x2],"text":"LINE ONE\nLINE TWO","desc":"...","color_palette":["#RRGGBB"]}
|
||||
```
|
||||
`bbox` and `color_palette` are both OPTIONAL per element. `bbox`: see BBOX. `color_palette`: up to 5 hex strings of that element's dominant colours — include it when the prompt gives the element a distinctive colour (a red jacket, coloured text), otherwise omit.
|
||||
`bbox` and `color_palette` are both OPTIONAL per element; when present they keep the order shown (`color_palette` is always LAST). `bbox`: see BBOX. `color_palette`: up to 5 UPPERCASE `#RRGGBB` strings of that element's dominant colours — include it when the prompt gives the element a distinctive colour (a red jacket, coloured text), otherwise omit.
|
||||
|
||||
- **One coherent subject = ONE element.** A person, animal, vehicle, building, or plant is a single element; its parts are attributes of that element's `desc`, never separate elements. Multiple distinct subjects = multiple elements (one each).
|
||||
- **`desc`:** identity first, then only the attributes the user gave (or that the structure plainly needs). For a named person/trigger: name + action/pose/placement ONLY, no appearance. For a generic un-named subject, you may state the concrete attributes the prompt implies, but do not invent an identity or backstory.
|
||||
|
||||
Reference in new issue
Block a user