Save job pid to the database and sing sigint to kill it when stopping so it stops immediatly.
This commit is contained in:
@@ -125,18 +125,25 @@ const startAndWatchJob = (job: Job) => {
|
||||
});
|
||||
}
|
||||
|
||||
// Save the PID to the database and a file for future management (stop/inspect)
|
||||
const pid = subprocess.pid ?? null;
|
||||
if (pid != null) {
|
||||
await prisma.job.update({
|
||||
where: { id: jobID },
|
||||
data: { pid },
|
||||
});
|
||||
}
|
||||
try {
|
||||
fs.writeFileSync(path.join(trainingFolder, 'pid.txt'), String(pid ?? ''), { flag: 'w' });
|
||||
} catch (e) {
|
||||
console.error('Error writing pid file:', e);
|
||||
}
|
||||
|
||||
// Important: let the child run independently of this Node process.
|
||||
if (subprocess.unref) {
|
||||
subprocess.unref();
|
||||
}
|
||||
|
||||
// Optionally write a pid file for future management (stop/inspect) without keeping streams open
|
||||
try {
|
||||
fs.writeFileSync(path.join(trainingFolder, 'pid.txt'), String(subprocess.pid ?? ''), { flag: 'w' });
|
||||
} catch (e) {
|
||||
console.error('Error writing pid file:', e);
|
||||
}
|
||||
|
||||
// (No stdout/stderr listeners — logging should go to --log handled by your Python)
|
||||
// (No monitoring loop — the whole point is to let it live past this worker)
|
||||
} catch (error: any) {
|
||||
|
||||
@@ -35,6 +35,7 @@ model Job {
|
||||
info String @default("")
|
||||
speed_string String @default("")
|
||||
queue_position Int @default(0)
|
||||
pid Int?
|
||||
|
||||
@@index([status])
|
||||
@@index([gpu_ids])
|
||||
|
||||
@@ -17,6 +17,7 @@ export async function GET(request: NextRequest, { params }: { params: { jobID: s
|
||||
stop: true,
|
||||
status: 'stopped',
|
||||
info: 'Job stopped',
|
||||
pid: null,
|
||||
},
|
||||
});
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ import { NextRequest, NextResponse } from 'next/server';
|
||||
import { PrismaClient } from '@prisma/client';
|
||||
|
||||
const prisma = new PrismaClient();
|
||||
const isWindows = process.platform === 'win32';
|
||||
|
||||
export async function GET(request: NextRequest, { params }: { params: { jobID: string } }) {
|
||||
const { jobID } = await params;
|
||||
@@ -10,7 +11,10 @@ export async function GET(request: NextRequest, { params }: { params: { jobID: s
|
||||
where: { id: jobID },
|
||||
});
|
||||
|
||||
// update job status to 'running'
|
||||
if (!job) {
|
||||
return NextResponse.json({ error: 'Job not found' }, { status: 404 });
|
||||
}
|
||||
|
||||
await prisma.job.update({
|
||||
where: { id: jobID },
|
||||
data: {
|
||||
@@ -19,5 +23,33 @@ export async function GET(request: NextRequest, { params }: { params: { jobID: s
|
||||
},
|
||||
});
|
||||
|
||||
// Send SIGINT to the process if we have a PID
|
||||
if (job.pid != null) {
|
||||
console.log(`Attempting to stop job ${jobID} with PID ${job.pid}`);
|
||||
try {
|
||||
if (isWindows) {
|
||||
// Windows doesn't support SIGINT for arbitrary processes.
|
||||
// Use taskkill with /T (tree) to send a CTRL+C-like termination.
|
||||
const { execSync } = require('child_process');
|
||||
execSync(`taskkill /PID ${job.pid} /T /F`, { stdio: 'ignore' });
|
||||
} else {
|
||||
process.kill(job.pid, 'SIGINT');
|
||||
}
|
||||
// if it killed it, mark it stopped in the database
|
||||
await prisma.job.update({
|
||||
where: { id: jobID },
|
||||
data: {
|
||||
status: 'stopped',
|
||||
info: 'Job stopped',
|
||||
},
|
||||
});
|
||||
} catch (e) {
|
||||
// Process may have already exited — that's fine
|
||||
console.error('Error sending signal to process:', e);
|
||||
}
|
||||
} else {
|
||||
console.warn(`No PID found for job ${jobID}, cannot send stop signal`);
|
||||
}
|
||||
|
||||
return NextResponse.json(job);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user