Save job pid to the database and sing sigint to kill it when stopping so it stops immediatly.
This commit is contained in:
@@ -125,18 +125,25 @@ const startAndWatchJob = (job: Job) => {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Save the PID to the database and a file for future management (stop/inspect)
|
||||||
|
const pid = subprocess.pid ?? null;
|
||||||
|
if (pid != null) {
|
||||||
|
await prisma.job.update({
|
||||||
|
where: { id: jobID },
|
||||||
|
data: { pid },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
fs.writeFileSync(path.join(trainingFolder, 'pid.txt'), String(pid ?? ''), { flag: 'w' });
|
||||||
|
} catch (e) {
|
||||||
|
console.error('Error writing pid file:', e);
|
||||||
|
}
|
||||||
|
|
||||||
// Important: let the child run independently of this Node process.
|
// Important: let the child run independently of this Node process.
|
||||||
if (subprocess.unref) {
|
if (subprocess.unref) {
|
||||||
subprocess.unref();
|
subprocess.unref();
|
||||||
}
|
}
|
||||||
|
|
||||||
// Optionally write a pid file for future management (stop/inspect) without keeping streams open
|
|
||||||
try {
|
|
||||||
fs.writeFileSync(path.join(trainingFolder, 'pid.txt'), String(subprocess.pid ?? ''), { flag: 'w' });
|
|
||||||
} catch (e) {
|
|
||||||
console.error('Error writing pid file:', e);
|
|
||||||
}
|
|
||||||
|
|
||||||
// (No stdout/stderr listeners — logging should go to --log handled by your Python)
|
// (No stdout/stderr listeners — logging should go to --log handled by your Python)
|
||||||
// (No monitoring loop — the whole point is to let it live past this worker)
|
// (No monitoring loop — the whole point is to let it live past this worker)
|
||||||
} catch (error: any) {
|
} catch (error: any) {
|
||||||
|
|||||||
@@ -35,6 +35,7 @@ model Job {
|
|||||||
info String @default("")
|
info String @default("")
|
||||||
speed_string String @default("")
|
speed_string String @default("")
|
||||||
queue_position Int @default(0)
|
queue_position Int @default(0)
|
||||||
|
pid Int?
|
||||||
|
|
||||||
@@index([status])
|
@@index([status])
|
||||||
@@index([gpu_ids])
|
@@index([gpu_ids])
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ export async function GET(request: NextRequest, { params }: { params: { jobID: s
|
|||||||
stop: true,
|
stop: true,
|
||||||
status: 'stopped',
|
status: 'stopped',
|
||||||
info: 'Job stopped',
|
info: 'Job stopped',
|
||||||
|
pid: null,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ import { NextRequest, NextResponse } from 'next/server';
|
|||||||
import { PrismaClient } from '@prisma/client';
|
import { PrismaClient } from '@prisma/client';
|
||||||
|
|
||||||
const prisma = new PrismaClient();
|
const prisma = new PrismaClient();
|
||||||
|
const isWindows = process.platform === 'win32';
|
||||||
|
|
||||||
export async function GET(request: NextRequest, { params }: { params: { jobID: string } }) {
|
export async function GET(request: NextRequest, { params }: { params: { jobID: string } }) {
|
||||||
const { jobID } = await params;
|
const { jobID } = await params;
|
||||||
@@ -10,7 +11,10 @@ export async function GET(request: NextRequest, { params }: { params: { jobID: s
|
|||||||
where: { id: jobID },
|
where: { id: jobID },
|
||||||
});
|
});
|
||||||
|
|
||||||
// update job status to 'running'
|
if (!job) {
|
||||||
|
return NextResponse.json({ error: 'Job not found' }, { status: 404 });
|
||||||
|
}
|
||||||
|
|
||||||
await prisma.job.update({
|
await prisma.job.update({
|
||||||
where: { id: jobID },
|
where: { id: jobID },
|
||||||
data: {
|
data: {
|
||||||
@@ -19,5 +23,33 @@ export async function GET(request: NextRequest, { params }: { params: { jobID: s
|
|||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// Send SIGINT to the process if we have a PID
|
||||||
|
if (job.pid != null) {
|
||||||
|
console.log(`Attempting to stop job ${jobID} with PID ${job.pid}`);
|
||||||
|
try {
|
||||||
|
if (isWindows) {
|
||||||
|
// Windows doesn't support SIGINT for arbitrary processes.
|
||||||
|
// Use taskkill with /T (tree) to send a CTRL+C-like termination.
|
||||||
|
const { execSync } = require('child_process');
|
||||||
|
execSync(`taskkill /PID ${job.pid} /T /F`, { stdio: 'ignore' });
|
||||||
|
} else {
|
||||||
|
process.kill(job.pid, 'SIGINT');
|
||||||
|
}
|
||||||
|
// if it killed it, mark it stopped in the database
|
||||||
|
await prisma.job.update({
|
||||||
|
where: { id: jobID },
|
||||||
|
data: {
|
||||||
|
status: 'stopped',
|
||||||
|
info: 'Job stopped',
|
||||||
|
},
|
||||||
|
});
|
||||||
|
} catch (e) {
|
||||||
|
// Process may have already exited — that's fine
|
||||||
|
console.error('Error sending signal to process:', e);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
console.warn(`No PID found for job ${jobID}, cannot send stop signal`);
|
||||||
|
}
|
||||||
|
|
||||||
return NextResponse.json(job);
|
return NextResponse.json(job);
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user