Allow adjusting, adding, and deleting bounding boxes.

This commit is contained in:
Jaret Burkett committed 2026-06-04 20:29:31 -06:00
1 parent bb60f6d1d1
commit 90a2084f70
2 files changed
+502 -11

No files matched your search

+250 -11
View File
@@ -2,7 +2,7 @@
import { useState, useEffect, useMemo, useCallback, useRef } from 'react';
import { createPortal } from 'react-dom';
import { Dialog, DialogBackdrop, DialogPanel } from '@headlessui/react';
import { Cog, SquareDashed } from 'lucide-react';
import { Cog, SquareDashed, Pencil } from 'lucide-react';
import { Menu, MenuButton, MenuItem, MenuItems } from '@headlessui/react';
import classNames from 'classnames';
import { openConfirm } from './ConfirmModal';
@@ -10,7 +10,15 @@ import { apiClient } from '@/utils/api';
import { isVideo, isAudio } from '@/utils/basic';
import AudioPlayer from './AudioPlayer';
import { TransformWrapper, TransformComponent } from 'react-zoom-pan-pinch';
import BoundingBoxOverlay, { parseBoundingBoxes } from './BoundingBoxOverlay';
import BoundingBoxOverlay, { BoundingBoxEditor, parseBoundingBoxes, extractBoxes } from './BoundingBoxOverlay';
function safeParse(text: string): any {
try {
return JSON.parse(text);
} catch {
return null;
}
}
interface Props {
imgPath: string | null; // current image path
@@ -27,6 +35,9 @@ export default function DatasetImageViewer({ imgPath, imageList, onChange, refre
const [savedCaption, setSavedCaption] = useState<string>('');
const [isCaptionLoaded, setIsCaptionLoaded] = useState<boolean>(false);
const [showBoxes, setShowBoxes] = useState<boolean>(false);
const [isEditingBoxes, setIsEditingBoxes] = useState<boolean>(false);
const [selectedBoxIndex, setSelectedBoxIndex] = useState<number | null>(null);
const [isDrawing, setIsDrawing] = useState<boolean>(false);
const captionRef = useRef<string>('');
const savedCaptionRef = useRef<string>('');
const currentImgPathRef = useRef<string | null>(null);
@@ -34,6 +45,13 @@ export default function DatasetImageViewer({ imgPath, imageList, onChange, refre
useEffect(() => setMounted(true), []);
// Leave box-edit mode whenever the image changes, to avoid accidental edits.
useEffect(() => {
setIsEditingBoxes(false);
setSelectedBoxIndex(null);
setIsDrawing(false);
}, [imgPath]);
// open/close based on external value
useEffect(() => {
setIsOpen(Boolean(imgPath));
@@ -88,6 +106,14 @@ export default function DatasetImageViewer({ imgPath, imageList, onChange, refre
[onCaptionSaved],
);
// Stable handle to the latest saveCaptionForPath so the fetch effect doesn't
// re-run (and re-fetch, blanking the caption) every time a save changes its
// identity via the parent's onCaptionSaved.
const saveCaptionForPathRef = useRef(saveCaptionForPath);
useEffect(() => {
saveCaptionForPathRef.current = saveCaptionForPath;
}, [saveCaptionForPath]);
const saveCaption = useCallback(() => {
if (!imgPath) return;
saveCaptionForPath(imgPath, caption, savedCaption);
@@ -97,7 +123,7 @@ export default function DatasetImageViewer({ imgPath, imageList, onChange, refre
useEffect(() => {
const previousPath = currentImgPathRef.current;
if (previousPath && previousPath !== imgPath) {
saveCaptionForPath(previousPath, captionRef.current, savedCaptionRef.current);
saveCaptionForPathRef.current(previousPath, captionRef.current, savedCaptionRef.current);
}
currentImgPathRef.current = imgPath;
@@ -137,7 +163,7 @@ export default function DatasetImageViewer({ imgPath, imageList, onChange, refre
return () => {
controller.abort();
};
}, [imgPath, saveCaptionForPath]);
}, [imgPath]);
// Save any pending caption when the viewer fully unmounts
useEffect(() => {
@@ -194,6 +220,82 @@ export default function DatasetImageViewer({ imgPath, imageList, onChange, refre
});
}, [imgPath, onChange, refreshImages]);
// Mutate the caption JSON's element array, updating the textarea ONLY (no
// network). Persisting happens via the Save button / nav auto-save. Returns
// whatever the mutator returns.
const editCaption = useCallback(
(fn: (elements: any[], data: any) => any): any => {
let data: any;
try {
data = JSON.parse(caption);
} catch {
return undefined;
}
const elements = data?.compositional_deconstruction?.elements;
if (!Array.isArray(elements)) return undefined;
const result = fn(elements, data);
setCaption(JSON.stringify(data, null, 2));
return result;
},
[caption],
);
const handleBoxChange = useCallback(
(elementIndex: number, box: { y1: number; x1: number; y2: number; x2: number }) => {
editCaption(els => {
if (els[elementIndex]) els[elementIndex].bbox = [box.y1, box.x1, box.y2, box.x2];
});
},
[editCaption],
);
const handleDeleteBox = useCallback(
(elementIndex: number) => {
editCaption(els => {
els.splice(elementIndex, 1);
});
setSelectedBoxIndex(null);
},
[editCaption],
);
const handleCreateBox = useCallback(
(box: { y1: number; x1: number; y2: number; x2: number }) => {
const newIndex = editCaption(els => {
els.push({ type: 'obj', bbox: [box.y1, box.x1, box.y2, box.x2], desc: '' });
return els.length - 1;
});
setSelectedBoxIndex(typeof newIndex === 'number' ? newIndex : null);
setIsDrawing(false);
},
[editCaption],
);
const handleFieldChange = useCallback(
(field: 'desc' | 'text', value: string) => {
editCaption(els => {
if (selectedBoxIndex != null && els[selectedBoxIndex]) els[selectedBoxIndex][field] = value;
});
},
[editCaption, selectedBoxIndex],
);
const handleTypeChange = useCallback(
(type: 'obj' | 'text') => {
editCaption(els => {
const el = selectedBoxIndex != null ? els[selectedBoxIndex] : null;
if (!el) return;
el.type = type;
if (type === 'text') {
if (el.text == null) el.text = '';
} else {
delete el.text;
}
});
},
[editCaption, selectedBoxIndex],
);
// keyboard events while open — skip nav while caption textarea is focused
useEffect(() => {
const handleKeyDown = (event: KeyboardEvent) => {
@@ -219,7 +321,12 @@ export default function DatasetImageViewer({ imgPath, imageList, onChange, refre
break;
case 'Delete':
case 'Backspace':
handleDelete();
// While editing boxes, Delete removes the selected box (never the image).
if (isEditingBoxes) {
if (selectedBoxIndex != null) handleDeleteBox(selectedBoxIndex);
} else {
handleDelete();
}
break;
default:
break;
@@ -227,7 +334,7 @@ export default function DatasetImageViewer({ imgPath, imageList, onChange, refre
};
window.addEventListener('keydown', handleKeyDown);
return () => window.removeEventListener('keydown', handleKeyDown);
}, [isOpen, onCancel, handlePrev, handleNext, handleDelete]);
}, [isOpen, onCancel, handlePrev, handleNext, handleDelete, isEditingBoxes, selectedBoxIndex, handleDeleteBox]);
// Touch swipe navigation
const touchStartRef = useRef<{ x: number; y: number } | null>(null);
@@ -285,6 +392,13 @@ export default function DatasetImageViewer({ imgPath, imageList, onChange, refre
const boundingBoxes = useMemo(() => parseBoundingBoxes(caption), [caption]);
const canShowBoxes = Boolean(boundingBoxes && imgPath && !isAudio(imgPath) && !isVideo(imgPath));
// Boxes and the selected element are derived from the (locally edited) caption.
const editBoxes = useMemo(() => extractBoxes(safeParse(caption)), [caption]);
const selectedElement = useMemo(() => {
if (selectedBoxIndex == null) return null;
return safeParse(caption)?.compositional_deconstruction?.elements?.[selectedBoxIndex] ?? null;
}, [caption, selectedBoxIndex]);
if (!mounted) return null;
return createPortal(
@@ -323,9 +437,9 @@ export default function DatasetImageViewer({ imgPath, imageList, onChange, refre
initialScale={1}
minScale={1}
maxScale={6}
doubleClick={{ mode: 'toggle', step: 2 }}
wheel={{ step: 0.2 }}
panning={{ disabled: false, allowRightClickPan: false }}
doubleClick={{ mode: 'toggle', step: 2, disabled: isEditingBoxes }}
wheel={{ step: 0.2, disabled: isEditingBoxes }}
panning={{ disabled: isEditingBoxes, allowRightClickPan: false }}
onTransform={(_ref, state) => {
zoomedRef.current = state.scale > 1.01;
}}
@@ -338,7 +452,18 @@ export default function DatasetImageViewer({ imgPath, imageList, onChange, refre
draggable={false}
className="w-auto h-auto max-w-full sm:max-w-[95vw] max-h-[70vh] object-contain select-none !pointer-events-auto"
/>
{showBoxes && boundingBoxes && <BoundingBoxOverlay boxes={boundingBoxes} />}
{isEditingBoxes ? (
<BoundingBoxEditor
boxes={editBoxes}
selectedIndex={selectedBoxIndex}
drawing={isDrawing}
onSelect={setSelectedBoxIndex}
onChangeBox={handleBoxChange}
onCreateBox={handleCreateBox}
/>
) : (
showBoxes && boundingBoxes && <BoundingBoxOverlay boxes={boundingBoxes} />
)}
</div>
</TransformComponent>
</TransformWrapper>
@@ -371,12 +496,106 @@ export default function DatasetImageViewer({ imgPath, imageList, onChange, refre
disabled={!isCaptionLoaded}
/>
</div>
{isEditingBoxes && (
<div className="rounded border border-gray-700 bg-gray-900 p-2 flex flex-col gap-2 text-xs">
<div className="flex items-center gap-2">
<button
type="button"
onClick={() => setIsDrawing(d => !d)}
className={classNames('px-2 py-1 rounded border', {
'bg-blue-600 border-blue-500 text-white': isDrawing,
'border-gray-600 text-gray-300 hover:bg-gray-800': !isDrawing,
})}
>
{isDrawing ? 'Cancel' : '+ Add Box'}
</button>
<span className="text-gray-500">
{isDrawing
? 'Drag on the image to draw a new box'
: 'Click a box to select; drag to move, handles to resize'}
</span>
<button
type="button"
onClick={saveCaption}
disabled={isCaptionCurrent}
className={classNames('ml-auto px-3 py-1 rounded border', {
'bg-green-600 border-green-500 text-white hover:bg-green-500': !isCaptionCurrent,
'border-gray-700 text-gray-500 cursor-default': isCaptionCurrent,
})}
>
{isCaptionCurrent ? 'Saved' : 'Save'}
</button>
</div>
{selectedElement && (
<div className="flex flex-col gap-2 border-t border-gray-700 pt-2">
<div className="flex items-center gap-2">
<span className="text-gray-400">Type:</span>
<button
type="button"
onClick={() => handleTypeChange('obj')}
className={classNames('px-2 py-0.5 rounded border', {
'bg-cyan-600 border-cyan-500 text-white': selectedElement.type !== 'text',
'border-gray-600 text-gray-300 hover:bg-gray-800': selectedElement.type === 'text',
})}
>
Object
</button>
<button
type="button"
onClick={() => handleTypeChange('text')}
className={classNames('px-2 py-0.5 rounded border', {
'bg-amber-600 border-amber-500 text-white': selectedElement.type === 'text',
'border-gray-600 text-gray-300 hover:bg-gray-800': selectedElement.type !== 'text',
})}
>
Text
</button>
<button
type="button"
onClick={() => handleDeleteBox(selectedBoxIndex!)}
className="ml-auto px-2 py-0.5 rounded border border-red-700 text-red-400 hover:bg-red-900/40"
>
Delete
</button>
</div>
{selectedElement.type === 'text' && (
<label className="flex flex-col gap-1">
<span className="text-gray-400">Text (shown in image)</span>
<textarea
className="w-full bg-gray-950 text-gray-100 rounded border border-gray-700 p-1 resize-none outline-none focus:border-blue-500"
rows={2}
value={selectedElement.text ?? ''}
onChange={e => handleFieldChange('text', e.target.value)}
/>
</label>
)}
<label className="flex flex-col gap-1">
<span className="text-gray-400">Description</span>
<textarea
className="w-full bg-gray-950 text-gray-100 rounded border border-gray-700 p-1 resize-none outline-none focus:border-blue-500"
rows={2}
value={selectedElement.desc ?? ''}
onChange={e => handleFieldChange('desc', e.target.value)}
/>
</label>
</div>
)}
</div>
)}
</div>
<div className="absolute top-2 right-2 flex items-center gap-2 z-20">
{canShowBoxes && (
<button
type="button"
onClick={() => setShowBoxes(v => !v)}
onClick={() => {
const next = !showBoxes;
setShowBoxes(next);
if (!next) {
setIsEditingBoxes(false);
setSelectedBoxIndex(null);
setIsDrawing(false);
}
}}
title={showBoxes ? 'Hide bounding boxes' : 'Show bounding boxes'}
className={classNames('bg-gray-900 rounded-full p-1 leading-[0px] hover:opacity-100', {
'opacity-100 text-blue-400': showBoxes,
@@ -386,6 +605,26 @@ export default function DatasetImageViewer({ imgPath, imageList, onChange, refre
<SquareDashed />
</button>
)}
{((canShowBoxes && showBoxes) || isEditingBoxes) && (
<button
type="button"
onClick={() => {
const next = !isEditingBoxes;
setIsEditingBoxes(next);
if (!next) {
setSelectedBoxIndex(null);
setIsDrawing(false);
}
}}
title={isEditingBoxes ? 'Done editing boxes' : 'Edit bounding boxes'}
className={classNames('bg-gray-900 rounded-full p-1 leading-[0px] hover:opacity-100', {
'opacity-100 text-blue-400': isEditingBoxes,
'opacity-50': !isEditingBoxes,
})}
>
<Pencil />
</button>
)}
<div className="bg-gray-900 rounded-full p-1 leading-[0px] opacity-50 hover:opacity-100">
<Menu>
<MenuButton>