From bb60f6d1d1438783ea78c35ae8770d74375e2e55 Mon Sep 17 00:00:00 2001 From: Jaret Burkett Date: Thu, 4 Jun 2026 15:59:02 -0600 Subject: [PATCH] Show bounding boxes on the sample image card if we have them in the prompt --- ui/src/components/BoundingBoxOverlay.tsx | 78 +++++++++++++++++++ ui/src/components/DatasetImageViewer.tsx | 72 +----------------- ui/src/components/SampleImageViewer.tsx | 97 ++++++++++++++++-------- 3 files changed, 146 insertions(+), 101 deletions(-) create mode 100644 ui/src/components/BoundingBoxOverlay.tsx diff --git a/ui/src/components/BoundingBoxOverlay.tsx b/ui/src/components/BoundingBoxOverlay.tsx new file mode 100644 index 0000000..669541b --- /dev/null +++ b/ui/src/components/BoundingBoxOverlay.tsx @@ -0,0 +1,78 @@ +import classNames from 'classnames'; + +// A single bbox parsed from an Ideogram-style caption/prompt. Stored coords are +// normalized 0-1000 in [y1, x1, y2, x2] order (top-left origin). +export interface OverlayBox { + y1: number; + x1: number; + y2: number; + x2: number; + label: string; + type: 'obj' | 'text'; +} + +// Returns the list of boxes if the text is an Ideogram bbox-JSON caption/prompt +// with at least one bbox, otherwise null (normal captions/prompts get no overlay). +export function parseBoundingBoxes(text: string): OverlayBox[] | null { + const trimmed = text.trim(); + if (!trimmed.startsWith('{')) return null; + let data: any; + try { + data = JSON.parse(trimmed); + } catch { + return null; + } + const elements = data?.compositional_deconstruction?.elements; + if (!Array.isArray(elements)) return null; + const boxes: OverlayBox[] = []; + for (const el of elements) { + const bb = el?.bbox; + if (Array.isArray(bb) && bb.length === 4 && bb.every((n: any) => typeof n === 'number')) { + const isText = el.type === 'text'; + const label = (isText ? el.text : el.desc) ?? ''; + boxes.push({ y1: bb[0], x1: bb[1], y2: bb[2], x2: bb[3], label: `${label}`, type: isText ? 'text' : 'obj' }); + } + } + return boxes.length > 0 ? boxes : null; +} + +// Absolute overlay layer of boxes. Place inside a `relative` container that wraps +// the image (e.g. inside react-zoom-pan-pinch's TransformComponent) so boxes +// track the image during zoom/pan. Coords are percentages of the frame, so this +// is resolution-independent. pointer-events-none keeps zoom/pan working through it. +export default function BoundingBoxOverlay({ boxes }: { boxes: OverlayBox[] }) { + return ( +
+ {boxes.map((b, i) => ( +
+ {b.label && ( + + {b.label} + + )} +
+ ))} +
+ ); +} diff --git a/ui/src/components/DatasetImageViewer.tsx b/ui/src/components/DatasetImageViewer.tsx index c185b0e..5619564 100644 --- a/ui/src/components/DatasetImageViewer.tsx +++ b/ui/src/components/DatasetImageViewer.tsx @@ -10,42 +10,7 @@ import { apiClient } from '@/utils/api'; import { isVideo, isAudio } from '@/utils/basic'; import AudioPlayer from './AudioPlayer'; import { TransformWrapper, TransformComponent } from 'react-zoom-pan-pinch'; - -// A single bbox parsed from an Ideogram-style caption. Stored coords are -// normalized 0-1000 in [y1, x1, y2, x2] order (top-left origin). -interface OverlayBox { - y1: number; - x1: number; - y2: number; - x2: number; - label: string; - type: 'obj' | 'text'; -} - -// Returns the list of boxes if the caption is an Ideogram bbox-JSON caption with -// at least one bbox, otherwise null (normal captions get no overlay). -function parseBoundingBoxes(caption: string): OverlayBox[] | null { - const trimmed = caption.trim(); - if (!trimmed.startsWith('{')) return null; - let data: any; - try { - data = JSON.parse(trimmed); - } catch { - return null; - } - const elements = data?.compositional_deconstruction?.elements; - if (!Array.isArray(elements)) return null; - const boxes: OverlayBox[] = []; - for (const el of elements) { - const bb = el?.bbox; - if (Array.isArray(bb) && bb.length === 4 && bb.every((n: any) => typeof n === 'number')) { - const isText = el.type === 'text'; - const label = (isText ? el.text : el.desc) ?? ''; - boxes.push({ y1: bb[0], x1: bb[1], y2: bb[2], x2: bb[3], label: `${label}`, type: isText ? 'text' : 'obj' }); - } - } - return boxes.length > 0 ? boxes : null; -} +import BoundingBoxOverlay, { parseBoundingBoxes } from './BoundingBoxOverlay'; interface Props { imgPath: string | null; // current image path @@ -373,40 +338,7 @@ export default function DatasetImageViewer({ imgPath, imageList, onChange, refre draggable={false} className="w-auto h-auto max-w-full sm:max-w-[95vw] max-h-[70vh] object-contain select-none !pointer-events-auto" /> - {showBoxes && boundingBoxes && ( -
- {boundingBoxes.map((b, i) => ( -
- {b.label && ( - - {b.label} - - )} -
- ))} -
- )} + {showBoxes && boundingBoxes && } diff --git a/ui/src/components/SampleImageViewer.tsx b/ui/src/components/SampleImageViewer.tsx index 886ca7e..9d973ca 100644 --- a/ui/src/components/SampleImageViewer.tsx +++ b/ui/src/components/SampleImageViewer.tsx @@ -3,13 +3,15 @@ import { useState, useEffect, useMemo, useCallback, useRef } from 'react'; import { createPortal } from 'react-dom'; import { Dialog, DialogBackdrop, DialogPanel } from '@headlessui/react'; import { SampleConfig, SampleItem } from '@/types'; -import { Cog } from 'lucide-react'; +import { Cog, SquareDashed } from 'lucide-react'; +import classNames from 'classnames'; import { Menu, MenuButton, MenuItem, MenuItems } from '@headlessui/react'; import { openConfirm } from './ConfirmModal'; import { apiClient } from '@/utils/api'; import { isVideo, isAudio } from '@/utils/basic'; import AudioPlayer from './AudioPlayer'; import { TransformWrapper, TransformComponent } from 'react-zoom-pan-pinch'; +import BoundingBoxOverlay, { parseBoundingBoxes } from './BoundingBoxOverlay'; interface Props { imgPath: string | null; // current image path @@ -31,6 +33,7 @@ export default function SampleImageViewer({ const [mounted, setMounted] = useState(false); const [isOpen, setIsOpen] = useState(Boolean(imgPath)); const [showingControlIdx, setShowingControlIdx] = useState(null); + const [showBoxes, setShowBoxes] = useState(false); useEffect(() => setMounted(true), []); @@ -196,6 +199,20 @@ export default function SampleImageViewer({ return imgPath; }, [showingControlIdx, controlImages, imgPath]); + // The sample's prompt is what generated it; if it's an Ideogram bbox-JSON we can + // overlay the boxes on the generated image. Only on the main image (not controls). + const boundingBoxes = useMemo( + () => (sampleItem?.prompt ? parseBoundingBoxes(sampleItem.prompt) : null), + [sampleItem], + ); + const canShowBoxes = Boolean( + boundingBoxes && + showingControlIdx === null && + displayedImgPath && + !isAudio(displayedImgPath) && + !isVideo(displayedImgPath), + ); + // keyboard events while open useEffect(() => { const handleKeyDown = (event: KeyboardEvent) => { @@ -326,12 +343,15 @@ export default function SampleImageViewer({ }} > - Sample Image +
+ Sample Image + {showBoxes && canShowBoxes && boundingBoxes && } +
))} @@ -386,33 +406,48 @@ export default function SampleImageViewer({ -
- - - - - + {canShowBoxes && ( + + )} +
+ + + + + + {imgPath && isAudio(imgPath) && ( + + + Download + + + )} - - Download - +
+ Delete Sample +
- )} - -
- Delete Sample -
- - - + +
+