Fix several cases of silent crashing on UI (#922)
Co-authored-by: Jaret Burkett <jaretburkett@gmail.com>
This commit is contained in:
@@ -7,6 +7,12 @@ import { TOOLKIT_ROOT, getTrainingFolder, getHFToken } from '../paths';
|
|||||||
import { resolvePythonPath } from '../pythonPath';
|
import { resolvePythonPath } from '../pythonPath';
|
||||||
const isWindows = process.platform === 'win32';
|
const isWindows = process.platform === 'win32';
|
||||||
|
|
||||||
|
const appendJobLog = (logPath: string, message: string) => {
|
||||||
|
fs.appendFile(logPath, message, error => {
|
||||||
|
if (error) console.error('Error writing to job log:', error);
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
const startAndWatchJob = (job: Job) => {
|
const startAndWatchJob = (job: Job) => {
|
||||||
// starts and watches the job asynchronously
|
// starts and watches the job asynchronously
|
||||||
return new Promise<void>(async (resolve, reject) => {
|
return new Promise<void>(async (resolve, reject) => {
|
||||||
@@ -73,6 +79,7 @@ const startAndWatchJob = (job: Job) => {
|
|||||||
CUDA_DEVICE_ORDER: 'PCI_BUS_ID',
|
CUDA_DEVICE_ORDER: 'PCI_BUS_ID',
|
||||||
CUDA_VISIBLE_DEVICES: `${job.gpu_ids}`,
|
CUDA_VISIBLE_DEVICES: `${job.gpu_ids}`,
|
||||||
IS_AI_TOOLKIT_UI: '1',
|
IS_AI_TOOLKIT_UI: '1',
|
||||||
|
PYTHONUNBUFFERED: '1', // write Python output immediately so it is not lost on a crash
|
||||||
};
|
};
|
||||||
|
|
||||||
// HF_TOKEN
|
// HF_TOKEN
|
||||||
@@ -81,10 +88,12 @@ const startAndWatchJob = (job: Job) => {
|
|||||||
additionalEnv.HF_TOKEN = hfToken;
|
additionalEnv.HF_TOKEN = hfToken;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Add the --log argument to the command
|
const args = [runFilePath, configPath];
|
||||||
const args = [runFilePath, configPath, '--log', logPath];
|
|
||||||
|
|
||||||
|
let logFd: number | null = null;
|
||||||
try {
|
try {
|
||||||
|
// Capture errors that occur before run.py can initialize file logging.
|
||||||
|
logFd = fs.openSync(logPath, 'a');
|
||||||
let subprocess;
|
let subprocess;
|
||||||
|
|
||||||
if (isWindows) {
|
if (isWindows) {
|
||||||
@@ -97,13 +106,13 @@ const startAndWatchJob = (job: Job) => {
|
|||||||
cwd: TOOLKIT_ROOT,
|
cwd: TOOLKIT_ROOT,
|
||||||
detached: true,
|
detached: true,
|
||||||
windowsHide: true,
|
windowsHide: true,
|
||||||
stdio: 'ignore', // don't tie stdio to parent
|
stdio: ['ignore', logFd, logFd], // don't tie stdio to parent; log fd passed as stdout and stderr
|
||||||
});
|
});
|
||||||
} else {
|
} else {
|
||||||
// For non-Windows platforms, fully detach and ignore stdio so it survives daemon-like
|
// For non-Windows platforms, fully detach and ignore stdio so it survives daemon-like
|
||||||
subprocess = spawn(pythonPath, args, {
|
subprocess = spawn(pythonPath, args, {
|
||||||
detached: true,
|
detached: true,
|
||||||
stdio: 'ignore',
|
stdio: ['ignore', logFd, logFd], // don't tie stdio to parent; log fd passed as stdout and stderr
|
||||||
env: {
|
env: {
|
||||||
...process.env,
|
...process.env,
|
||||||
...additionalEnv,
|
...additionalEnv,
|
||||||
@@ -112,6 +121,38 @@ const startAndWatchJob = (job: Job) => {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Handle failures where the child process could not be started.
|
||||||
|
subprocess.once('error', error => {
|
||||||
|
const message = `Error launching job process: ${error.message}`;
|
||||||
|
console.error(message);
|
||||||
|
appendJobLog(logPath, `${message}\n`);
|
||||||
|
void prisma.job
|
||||||
|
.update({
|
||||||
|
where: { id: jobID },
|
||||||
|
data: { status: 'error', info: message, pid: null },
|
||||||
|
})
|
||||||
|
.catch(updateError => {
|
||||||
|
console.error('Error updating job after process launch failure:', updateError);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// Record abnormal termination and repair jobs Python could not update itself.
|
||||||
|
subprocess.once('exit', (code, signal) => {
|
||||||
|
if (code === 0) return;
|
||||||
|
|
||||||
|
const result = signal ? `signal ${signal}` : `exit code ${code}`;
|
||||||
|
const message = `Job process terminated with ${result}.`;
|
||||||
|
appendJobLog(logPath, `\n${message}\n`);
|
||||||
|
void prisma.job
|
||||||
|
.updateMany({
|
||||||
|
where: { id: jobID, status: 'running' },
|
||||||
|
data: { status: 'error', info: message, pid: null },
|
||||||
|
})
|
||||||
|
.catch(updateError => {
|
||||||
|
console.error('Error updating job after abnormal process exit:', updateError);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
// Save the PID to the database and a file for future management (stop/inspect)
|
// Save the PID to the database and a file for future management (stop/inspect)
|
||||||
const pid = subprocess.pid ?? null;
|
const pid = subprocess.pid ?? null;
|
||||||
if (pid != null) {
|
if (pid != null) {
|
||||||
@@ -131,11 +172,12 @@ const startAndWatchJob = (job: Job) => {
|
|||||||
subprocess.unref();
|
subprocess.unref();
|
||||||
}
|
}
|
||||||
|
|
||||||
// (No stdout/stderr listeners — logging should go to --log handled by your Python)
|
// The child remains independent; these listeners only record failures
|
||||||
// (No monitoring loop — the whole point is to let it live past this worker)
|
// while the worker is alive.
|
||||||
} catch (error: any) {
|
} catch (error: any) {
|
||||||
// Handle any exceptions during process launch
|
// Handle any exceptions during process launch
|
||||||
console.error('Error launching process:', error);
|
console.error('Error launching process:', error);
|
||||||
|
appendJobLog(logPath, `Error launching job process: ${error?.message || 'Unknown error'}\n`);
|
||||||
|
|
||||||
await prisma.job.update({
|
await prisma.job.update({
|
||||||
where: { id: jobID },
|
where: { id: jobID },
|
||||||
@@ -145,6 +187,10 @@ const startAndWatchJob = (job: Job) => {
|
|||||||
},
|
},
|
||||||
});
|
});
|
||||||
return;
|
return;
|
||||||
|
} finally {
|
||||||
|
if (logFd !== null) {
|
||||||
|
fs.closeSync(logFd);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
// Resolve the promise immediately after starting the process
|
// Resolve the promise immediately after starting the process
|
||||||
resolve();
|
resolve();
|
||||||
|
|||||||
Reference in New Issue
Block a user