Save job pid to the database and sing sigint to kill it when stopping so it stops immediatly.

This commit is contained in:
Jaret Burkett
2026-03-26 09:10:37 -06:00
parent 89d2090962
commit 4df3b0463f
4 changed files with 49 additions and 8 deletions

View File

@@ -125,18 +125,25 @@ const startAndWatchJob = (job: Job) => {
});
}
// Save the PID to the database and a file for future management (stop/inspect)
const pid = subprocess.pid ?? null;
if (pid != null) {
await prisma.job.update({
where: { id: jobID },
data: { pid },
});
}
try {
fs.writeFileSync(path.join(trainingFolder, 'pid.txt'), String(pid ?? ''), { flag: 'w' });
} catch (e) {
console.error('Error writing pid file:', e);
}
// Important: let the child run independently of this Node process.
if (subprocess.unref) {
subprocess.unref();
}
// Optionally write a pid file for future management (stop/inspect) without keeping streams open
try {
fs.writeFileSync(path.join(trainingFolder, 'pid.txt'), String(subprocess.pid ?? ''), { flag: 'w' });
} catch (e) {
console.error('Error writing pid file:', e);
}
// (No stdout/stderr listeners — logging should go to --log handled by your Python)
// (No monitoring loop — the whole point is to let it live past this worker)
} catch (error: any) {

View File

@@ -35,6 +35,7 @@ model Job {
info String @default("")
speed_string String @default("")
queue_position Int @default(0)
pid Int?
@@index([status])
@@index([gpu_ids])

View File

@@ -17,6 +17,7 @@ export async function GET(request: NextRequest, { params }: { params: { jobID: s
stop: true,
status: 'stopped',
info: 'Job stopped',
pid: null,
},
});

View File

@@ -2,6 +2,7 @@ import { NextRequest, NextResponse } from 'next/server';
import { PrismaClient } from '@prisma/client';
const prisma = new PrismaClient();
const isWindows = process.platform === 'win32';
export async function GET(request: NextRequest, { params }: { params: { jobID: string } }) {
const { jobID } = await params;
@@ -10,7 +11,10 @@ export async function GET(request: NextRequest, { params }: { params: { jobID: s
where: { id: jobID },
});
// update job status to 'running'
if (!job) {
return NextResponse.json({ error: 'Job not found' }, { status: 404 });
}
await prisma.job.update({
where: { id: jobID },
data: {
@@ -19,5 +23,33 @@ export async function GET(request: NextRequest, { params }: { params: { jobID: s
},
});
// Send SIGINT to the process if we have a PID
if (job.pid != null) {
console.log(`Attempting to stop job ${jobID} with PID ${job.pid}`);
try {
if (isWindows) {
// Windows doesn't support SIGINT for arbitrary processes.
// Use taskkill with /T (tree) to send a CTRL+C-like termination.
const { execSync } = require('child_process');
execSync(`taskkill /PID ${job.pid} /T /F`, { stdio: 'ignore' });
} else {
process.kill(job.pid, 'SIGINT');
}
// if it killed it, mark it stopped in the database
await prisma.job.update({
where: { id: jobID },
data: {
status: 'stopped',
info: 'Job stopped',
},
});
} catch (e) {
// Process may have already exited — that's fine
console.error('Error sending signal to process:', e);
}
} else {
console.warn(`No PID found for job ${jobID}, cannot send stop signal`);
}
return NextResponse.json(job);
}