Add ability to delete the loss logs from a selected range on the loss graph
This commit is contained in:
@@ -96,3 +96,72 @@ export async function GET(request: NextRequest, { params }: { params: { jobID: s
|
|||||||
await closeDb(db);
|
await closeDb(db);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function run(db: sqlite3.Database, sql: string, params: any[] = []) {
|
||||||
|
return new Promise<void>((resolve, reject) => {
|
||||||
|
db.run(sql, params, (err) => (err ? reject(err) : resolve()));
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Delete every logged step in [min_step, max_step] (inclusive) across all
|
||||||
|
// metric keys. Used by the loss graph's "Delete Selected Range" action.
|
||||||
|
export async function DELETE(request: NextRequest, { params }: { params: { jobID: string } }) {
|
||||||
|
const { jobID } = await params;
|
||||||
|
|
||||||
|
const job = await prisma.job.findUnique({ where: { id: jobID } });
|
||||||
|
if (!job) return NextResponse.json({ error: 'Job not found' }, { status: 404 });
|
||||||
|
|
||||||
|
let body: { min_step?: unknown; max_step?: unknown } = {};
|
||||||
|
try {
|
||||||
|
body = await request.json();
|
||||||
|
} catch {
|
||||||
|
// fall through to validation below
|
||||||
|
}
|
||||||
|
const minStep = Number(body.min_step);
|
||||||
|
const maxStep = Number(body.max_step);
|
||||||
|
if (!Number.isFinite(minStep) || !Number.isFinite(maxStep) || minStep > maxStep) {
|
||||||
|
return NextResponse.json({ error: 'min_step and max_step must be numbers with min_step <= max_step' }, { status: 400 });
|
||||||
|
}
|
||||||
|
|
||||||
|
const trainingFolder = await getTrainingFolder();
|
||||||
|
const jobFolder = path.join(trainingFolder, job.name);
|
||||||
|
const logPath = path.join(jobFolder, 'loss_log.db');
|
||||||
|
|
||||||
|
try {
|
||||||
|
await fs.promises.access(logPath);
|
||||||
|
} catch {
|
||||||
|
return NextResponse.json({ error: 'No loss log for this job' }, { status: 404 });
|
||||||
|
}
|
||||||
|
|
||||||
|
const db = openDb(logPath);
|
||||||
|
|
||||||
|
try {
|
||||||
|
await run(db, 'BEGIN;');
|
||||||
|
try {
|
||||||
|
// The FK cascade on metrics only fires with PRAGMA foreign_keys=ON, so
|
||||||
|
// delete metrics explicitly rather than relying on it.
|
||||||
|
await run(db, `DELETE FROM metrics WHERE step >= ? AND step <= ?;`, [minStep, maxStep]);
|
||||||
|
await run(db, `DELETE FROM steps WHERE step >= ? AND step <= ?;`, [minStep, maxStep]);
|
||||||
|
await run(
|
||||||
|
db,
|
||||||
|
`DELETE FROM metric_keys WHERE NOT EXISTS (SELECT 1 FROM metrics WHERE metrics.key = metric_keys.key);`
|
||||||
|
);
|
||||||
|
await run(
|
||||||
|
db,
|
||||||
|
`UPDATE metric_keys SET
|
||||||
|
first_seen_step = (SELECT MIN(step) FROM metrics WHERE metrics.key = metric_keys.key),
|
||||||
|
last_seen_step = (SELECT MAX(step) FROM metrics WHERE metrics.key = metric_keys.key);`
|
||||||
|
);
|
||||||
|
await run(db, 'COMMIT;');
|
||||||
|
} catch (e) {
|
||||||
|
await run(db, 'ROLLBACK;').catch(() => { });
|
||||||
|
throw e;
|
||||||
|
}
|
||||||
|
return NextResponse.json({ ok: true, min_step: minStep, max_step: maxStep });
|
||||||
|
} catch (e: any) {
|
||||||
|
console.error('Error deleting loss range:', e);
|
||||||
|
return NextResponse.json({ error: e?.message ?? 'Failed to delete range' }, { status: 500 });
|
||||||
|
} finally {
|
||||||
|
await closeDb(db);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import useJobLossLog, { LossPoint } from '@/hooks/useJobLossLog';
|
|||||||
import { useMemo, useState, useEffect, useRef, useCallback } from 'react';
|
import { useMemo, useState, useEffect, useRef, useCallback } from 'react';
|
||||||
import uPlot from 'uplot';
|
import uPlot from 'uplot';
|
||||||
import 'uplot/dist/uPlot.min.css';
|
import 'uplot/dist/uPlot.min.css';
|
||||||
|
import { openConfirm } from '@/components/ConfirmModal';
|
||||||
|
|
||||||
interface Props {
|
interface Props {
|
||||||
job: Job;
|
job: Job;
|
||||||
@@ -138,7 +139,7 @@ function dulledColor(rgba: string): string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export default function JobLossGraph({ job }: Props) {
|
export default function JobLossGraph({ job }: Props) {
|
||||||
const { series, lossKeys, status, refreshLoss } = useJobLossLog(job.id, 2000);
|
const { series, lossKeys, status, refreshLoss, deleteRange } = useJobLossLog(job.id, 2000);
|
||||||
|
|
||||||
// Controls
|
// Controls
|
||||||
const [useLogScale, setUseLogScale] = useState(false);
|
const [useLogScale, setUseLogScale] = useState(false);
|
||||||
@@ -222,7 +223,8 @@ export default function JobLossGraph({ job }: Props) {
|
|||||||
setEnabled(prev => {
|
setEnabled(prev => {
|
||||||
const next = { ...prev };
|
const next = { ...prev };
|
||||||
for (const k of lossKeys) {
|
for (const k of lossKeys) {
|
||||||
if (next[k] === undefined) next[k] = persistedEnabledRef.current?.[k] ?? (k === 'loss/loss' || k === 'val/loss');
|
if (next[k] === undefined)
|
||||||
|
next[k] = persistedEnabledRef.current?.[k] ?? (k === 'loss/loss' || k === 'val/loss');
|
||||||
}
|
}
|
||||||
for (const k of Object.keys(next)) {
|
for (const k of Object.keys(next)) {
|
||||||
if (!lossKeys.includes(k)) delete next[k];
|
if (!lossKeys.includes(k)) delete next[k];
|
||||||
@@ -529,6 +531,46 @@ export default function JobLossGraph({ job }: Props) {
|
|||||||
u.setScale('x', { min: xs[0], max: xs[xs.length - 1] });
|
u.setScale('x', { min: xs[0], max: xs[xs.length - 1] });
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
|
const [deleting, setDeleting] = useState(false);
|
||||||
|
|
||||||
|
// Delete every logged step inside the current zoom window, then zoom back out.
|
||||||
|
const handleDeleteSelectedRange = useCallback(() => {
|
||||||
|
const u = uplotRef.current;
|
||||||
|
if (!u) return;
|
||||||
|
const xs = u.data[0] as number[];
|
||||||
|
if (!xs || !xs.length) return;
|
||||||
|
const sx = u.scales.x;
|
||||||
|
if (sx.min == null || sx.max == null) return;
|
||||||
|
// Snap the visible window to whole steps that actually fall inside it.
|
||||||
|
const minStep = Math.ceil(sx.min);
|
||||||
|
const maxStep = Math.floor(sx.max);
|
||||||
|
if (minStep > maxStep) return;
|
||||||
|
const count = xs.filter(x => x >= minStep && x <= maxStep).length;
|
||||||
|
|
||||||
|
openConfirm({
|
||||||
|
title: 'Delete Selected Range',
|
||||||
|
message: `Permanently delete steps ${minStep.toLocaleString()}–${maxStep.toLocaleString()} (${count.toLocaleString()} plotted points) from the loss log for all metrics? This cannot be undone.`,
|
||||||
|
type: 'danger',
|
||||||
|
confirmText: 'Delete',
|
||||||
|
onConfirm: async () => {
|
||||||
|
setDeleting(true);
|
||||||
|
try {
|
||||||
|
await deleteRange(minStep, maxStep);
|
||||||
|
// Zoom out: clear the zoom flag first so the data-update effect
|
||||||
|
// refits the x-scale to the remaining data instead of holding the
|
||||||
|
// old window.
|
||||||
|
isZoomedRef.current = false;
|
||||||
|
setIsZoomed(false);
|
||||||
|
handleResetZoom();
|
||||||
|
} catch (e) {
|
||||||
|
console.error('Error deleting loss range:', e);
|
||||||
|
} finally {
|
||||||
|
setDeleting(false);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}, [deleteRange, handleResetZoom]);
|
||||||
|
|
||||||
const totalPoints = built.data[0]?.length ?? 0;
|
const totalPoints = built.data[0]?.length ?? 0;
|
||||||
|
|
||||||
return (
|
return (
|
||||||
@@ -568,13 +610,23 @@ export default function JobLossGraph({ job }: Props) {
|
|||||||
) : (
|
) : (
|
||||||
<>
|
<>
|
||||||
{isZoomed && (
|
{isZoomed && (
|
||||||
<button
|
<div className="absolute top-2 right-2 z-10 flex items-center gap-2">
|
||||||
type="button"
|
<button
|
||||||
onClick={handleResetZoom}
|
type="button"
|
||||||
className="absolute top-2 right-2 z-10 px-2 py-1 rounded text-xs bg-blue-600/80 hover:bg-blue-600 text-white border border-blue-500/50"
|
onClick={handleDeleteSelectedRange}
|
||||||
>
|
disabled={deleting}
|
||||||
Reset zoom
|
className="px-2 py-1 rounded text-xs bg-red-600/80 hover:bg-red-600 disabled:opacity-50 disabled:cursor-not-allowed text-white border border-red-500/50"
|
||||||
</button>
|
>
|
||||||
|
{deleting ? 'Deleting...' : 'Delete Selected Range'}
|
||||||
|
</button>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
onClick={handleResetZoom}
|
||||||
|
className="px-2 py-1 rounded text-xs bg-blue-600/80 hover:bg-blue-600 text-white border border-blue-500/50"
|
||||||
|
>
|
||||||
|
Reset zoom
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
)}
|
)}
|
||||||
<div ref={chartHostRef} className="absolute top-0 left-0 right-0 bottom-2 overflow-hidden">
|
<div ref={chartHostRef} className="absolute top-0 left-0 right-0 bottom-2 overflow-hidden">
|
||||||
<div ref={containerRef} />
|
<div ref={containerRef} />
|
||||||
|
|||||||
@@ -116,6 +116,27 @@ export default function useJobLossLog(jobID: string, reloadInterval: null | numb
|
|||||||
}
|
}
|
||||||
}, [jobID, reloadInterval]);
|
}, [jobID, reloadInterval]);
|
||||||
|
|
||||||
|
// Delete every logged step in [minStep, maxStep] from the on-disk log, then
|
||||||
|
// drop those points locally so the chart updates without a full reload.
|
||||||
|
const deleteRange = useCallback(
|
||||||
|
async (minStep: number, maxStep: number) => {
|
||||||
|
if (!jobID) return;
|
||||||
|
await apiClient.delete(`/api/jobs/${jobID}/loss`, { data: { min_step: minStep, max_step: maxStep } });
|
||||||
|
setSeries(prev => {
|
||||||
|
const next: SeriesMap = {};
|
||||||
|
for (const k of Object.keys(prev)) {
|
||||||
|
const kept = prev[k].filter(p => p.step < minStep || p.step > maxStep);
|
||||||
|
next[k] = kept;
|
||||||
|
// Re-anchor incremental polling to the new tail so a deleted tail
|
||||||
|
// isn't treated as already-fetched.
|
||||||
|
lastStepByKeyRef.current[k] = kept.length ? kept[kept.length - 1].step : null;
|
||||||
|
}
|
||||||
|
return next;
|
||||||
|
});
|
||||||
|
},
|
||||||
|
[jobID],
|
||||||
|
);
|
||||||
|
|
||||||
// reset when job changes. Declared before the poll loop so the reset runs
|
// reset when job changes. Declared before the poll loop so the reset runs
|
||||||
// before the first fetch when jobID changes.
|
// before the first fetch when jobID changes.
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
@@ -128,5 +149,5 @@ export default function useJobLossLog(jobID: string, reloadInterval: null | numb
|
|||||||
|
|
||||||
usePollLoop(refreshLoss, reloadInterval, [jobID]);
|
usePollLoop(refreshLoss, reloadInterval, [jobID]);
|
||||||
|
|
||||||
return { series, keys, lossKeys, status, refreshLoss, setSeries };
|
return { series, keys, lossKeys, status, refreshLoss, deleteRange, setSeries };
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user