Replace all sync functions with async to allow more parallel api calls
This commit is contained in:
@@ -46,15 +46,18 @@ export async function POST(request: NextRequest) {
|
|||||||
return new NextResponse('Access denied', { status: 403 });
|
return new NextResponse('Access denied', { status: 403 });
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check if file exists
|
// Read caption file; a missing file just means no caption yet
|
||||||
if (!fs.existsSync(captionPath)) {
|
let caption: string;
|
||||||
// send back blank string if caption file does not exist
|
try {
|
||||||
return new NextResponse('');
|
caption = await fs.promises.readFile(captionPath, 'utf-8');
|
||||||
|
} catch (err: any) {
|
||||||
|
if (err?.code === 'ENOENT') {
|
||||||
|
// send back blank string if caption file does not exist
|
||||||
|
return new NextResponse('');
|
||||||
|
}
|
||||||
|
throw err;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Read caption file
|
|
||||||
const caption = fs.readFileSync(captionPath, 'utf-8');
|
|
||||||
|
|
||||||
// Return caption
|
// Return caption
|
||||||
return new NextResponse(caption);
|
return new NextResponse(caption);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
|||||||
@@ -30,17 +30,21 @@ export async function POST(request: NextRequest) {
|
|||||||
const allowedDir = await getDatasetsRoot();
|
const allowedDir = await getDatasetsRoot();
|
||||||
const captions: Record<string, string> = {};
|
const captions: Record<string, string> = {};
|
||||||
|
|
||||||
for (const imgPath of imgPaths) {
|
// Read every caption file concurrently instead of blocking on each one in turn.
|
||||||
if (typeof imgPath !== 'string') continue;
|
await Promise.all(
|
||||||
if (!isUnderRoot(imgPath, allowedDir)) continue;
|
imgPaths.map(async imgPath => {
|
||||||
|
if (typeof imgPath !== 'string') return;
|
||||||
|
if (!isUnderRoot(imgPath, allowedDir)) return;
|
||||||
|
|
||||||
const captionPath = imgPath.replace(/\.[^/.]+$/, '') + '.' + captionExt;
|
const captionPath = imgPath.replace(/\.[^/.]+$/, '') + '.' + captionExt;
|
||||||
try {
|
try {
|
||||||
captions[imgPath] = fs.existsSync(captionPath) ? fs.readFileSync(captionPath, 'utf-8') : '';
|
// Missing file (ENOENT) or any read error falls back to an empty caption.
|
||||||
} catch {
|
captions[imgPath] = await fs.promises.readFile(captionPath, 'utf-8');
|
||||||
captions[imgPath] = '';
|
} catch {
|
||||||
}
|
captions[imgPath] = '';
|
||||||
}
|
}
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
return NextResponse.json({ captions });
|
return NextResponse.json({ captions });
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -7,13 +7,14 @@ export async function GET() {
|
|||||||
let datasetsPath = await getDatasetsRoot();
|
let datasetsPath = await getDatasetsRoot();
|
||||||
|
|
||||||
// if folder doesnt exist, create it
|
// if folder doesnt exist, create it
|
||||||
if (!fs.existsSync(datasetsPath)) {
|
try {
|
||||||
fs.mkdirSync(datasetsPath);
|
await fs.promises.access(datasetsPath);
|
||||||
|
} catch {
|
||||||
|
await fs.promises.mkdir(datasetsPath);
|
||||||
}
|
}
|
||||||
|
|
||||||
// find all the folders in the datasets folder
|
// find all the folders in the datasets folder
|
||||||
let folders = fs
|
let folders = (await fs.promises.readdir(datasetsPath, { withFileTypes: true }))
|
||||||
.readdirSync(datasetsPath, { withFileTypes: true })
|
|
||||||
.filter(dirent => dirent.isDirectory())
|
.filter(dirent => dirent.isDirectory())
|
||||||
.filter(dirent => !dirent.name.startsWith('.'))
|
.filter(dirent => !dirent.name.startsWith('.'))
|
||||||
.map(dirent => dirent.name);
|
.map(dirent => dirent.name);
|
||||||
|
|||||||
@@ -16,12 +16,14 @@ export async function POST(request: Request) {
|
|||||||
|
|
||||||
try {
|
try {
|
||||||
// Check if folder exists
|
// Check if folder exists
|
||||||
if (!fs.existsSync(datasetFolder)) {
|
try {
|
||||||
|
await fs.promises.access(datasetFolder);
|
||||||
|
} catch {
|
||||||
return NextResponse.json({ error: `Folder '${datasetName}' not found` }, { status: 404 });
|
return NextResponse.json({ error: `Folder '${datasetName}' not found` }, { status: 404 });
|
||||||
}
|
}
|
||||||
|
|
||||||
// Find all images recursively
|
// Find all images recursively
|
||||||
const imageFiles = findImagesRecursively(datasetFolder);
|
const imageFiles = await findImagesRecursively(datasetFolder);
|
||||||
|
|
||||||
// Sort server-side so the client doesn't have to sort large lists
|
// Sort server-side so the client doesn't have to sort large lists
|
||||||
imageFiles.sort((a, b) => a.localeCompare(b));
|
imageFiles.sort((a, b) => a.localeCompare(b));
|
||||||
@@ -64,13 +66,15 @@ export async function POST(request: Request) {
|
|||||||
* @param dir Directory to search
|
* @param dir Directory to search
|
||||||
* @returns Array of absolute paths to image files
|
* @returns Array of absolute paths to image files
|
||||||
*/
|
*/
|
||||||
function findImagesRecursively(dir: string): string[] {
|
async function findImagesRecursively(dir: string): Promise<string[]> {
|
||||||
const imageExtensions = ['.png', '.jpg', '.jpeg', '.webp', '.mp4', '.avi', '.mov', '.mkv', '.wmv', '.m4v', '.flv', '.mp3', '.wav', '.flac', '.ogg'];
|
const imageExtensions = ['.png', '.jpg', '.jpeg', '.webp', '.mp4', '.avi', '.mov', '.mkv', '.wmv', '.m4v', '.flv', '.mp3', '.wav', '.flac', '.ogg'];
|
||||||
let results: string[] = [];
|
let results: string[] = [];
|
||||||
|
|
||||||
// withFileTypes avoids a separate statSync per entry — a big win on large datasets
|
// withFileTypes avoids a separate stat per entry — a big win on large datasets.
|
||||||
const entries = fs.readdirSync(dir, { withFileTypes: true });
|
// Async readdir yields between directories so other requests aren't blocked.
|
||||||
|
const entries = await fs.promises.readdir(dir, { withFileTypes: true });
|
||||||
|
|
||||||
|
const subdirs: string[] = [];
|
||||||
for (const entry of entries) {
|
for (const entry of entries) {
|
||||||
const name = entry.name;
|
const name = entry.name;
|
||||||
if (name.startsWith('.')) continue;
|
if (name.startsWith('.')) continue;
|
||||||
@@ -78,7 +82,7 @@ function findImagesRecursively(dir: string): string[] {
|
|||||||
|
|
||||||
if (entry.isDirectory()) {
|
if (entry.isDirectory()) {
|
||||||
if (name === '_controls') continue;
|
if (name === '_controls') continue;
|
||||||
results = results.concat(findImagesRecursively(itemPath));
|
subdirs.push(itemPath);
|
||||||
} else if (entry.isFile()) {
|
} else if (entry.isFile()) {
|
||||||
const ext = path.extname(name).toLowerCase();
|
const ext = path.extname(name).toLowerCase();
|
||||||
if (imageExtensions.includes(ext)) {
|
if (imageExtensions.includes(ext)) {
|
||||||
@@ -87,5 +91,11 @@ function findImagesRecursively(dir: string): string[] {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Recurse into subdirectories concurrently.
|
||||||
|
const nested = await Promise.all(subdirs.map(subdir => findImagesRecursively(subdir)));
|
||||||
|
for (const list of nested) {
|
||||||
|
results = results.concat(list);
|
||||||
|
}
|
||||||
|
|
||||||
return results;
|
return results;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -28,14 +28,14 @@ export async function GET(request: NextRequest, { params }: { params: { filePath
|
|||||||
return new NextResponse('Access denied', { status: 403 });
|
return new NextResponse('Access denied', { status: 403 });
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check if file exists
|
// Check it exists and grab file info in one stat
|
||||||
if (!fs.existsSync(resolvedFilePath)) {
|
let stat;
|
||||||
|
try {
|
||||||
|
stat = await fs.promises.stat(resolvedFilePath);
|
||||||
|
} catch {
|
||||||
console.warn(`File not found: ${resolvedFilePath}`);
|
console.warn(`File not found: ${resolvedFilePath}`);
|
||||||
return new NextResponse('File not found', { status: 404 });
|
return new NextResponse('File not found', { status: 404 });
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get file info
|
|
||||||
const stat = fs.statSync(resolvedFilePath);
|
|
||||||
if (!stat.isFile()) {
|
if (!stat.isFile()) {
|
||||||
return new NextResponse('Not a file', { status: 400 });
|
return new NextResponse('Not a file', { status: 400 });
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -34,19 +34,19 @@ export async function POST(request: NextRequest) {
|
|||||||
return new NextResponse('Access denied', { status: 403 });
|
return new NextResponse('Access denied', { status: 403 });
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check if file exists
|
// Check if file exists and grab file info in one stat
|
||||||
if (!fs.existsSync(resolvedFilePath)) {
|
let stat;
|
||||||
|
try {
|
||||||
|
stat = await fs.promises.stat(resolvedFilePath);
|
||||||
|
} catch {
|
||||||
console.warn(`File not found: ${resolvedFilePath}`);
|
console.warn(`File not found: ${resolvedFilePath}`);
|
||||||
return new NextResponse('File not found', { status: 404 });
|
return new NextResponse('File not found', { status: 404 });
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get file info
|
|
||||||
const stat = fs.statSync(resolvedFilePath);
|
|
||||||
if (!stat.isFile()) {
|
if (!stat.isFile()) {
|
||||||
return new NextResponse('Not a file', { status: 400 });
|
return new NextResponse('Not a file', { status: 400 });
|
||||||
}
|
}
|
||||||
|
|
||||||
fs.unlinkSync(resolvedFilePath);
|
await fs.promises.unlink(resolvedFilePath);
|
||||||
|
|
||||||
return NextResponse.json({ success: true });
|
return NextResponse.json({ success: true });
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import { NextResponse } from 'next/server';
|
import { NextResponse } from 'next/server';
|
||||||
import { exec, execSync } from 'child_process';
|
import { exec } from 'child_process';
|
||||||
import { promisify } from 'util';
|
import { promisify } from 'util';
|
||||||
import { createRequire } from 'module';
|
import { createRequire } from 'module';
|
||||||
import os from 'os';
|
import os from 'os';
|
||||||
@@ -23,7 +23,7 @@ async function getMacGpuInfo(): Promise<MacGpuResult | null> {
|
|||||||
// Get GPU name and core count from system_profiler
|
// Get GPU name and core count from system_profiler
|
||||||
let gpuName = 'Apple GPU';
|
let gpuName = 'Apple GPU';
|
||||||
try {
|
try {
|
||||||
const spOut = execSync(
|
const { stdout: spOut } = await execAsync(
|
||||||
'system_profiler SPDisplaysDataType 2>/dev/null | grep -E "Chipset Model|Total Number of Cores"',
|
'system_profiler SPDisplaysDataType 2>/dev/null | grep -E "Chipset Model|Total Number of Cores"',
|
||||||
{ encoding: 'utf-8', timeout: 5000 },
|
{ encoding: 'utf-8', timeout: 5000 },
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -13,7 +13,9 @@ export async function POST(request: Request) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// if img doesnt exist, ignore
|
// if img doesnt exist, ignore
|
||||||
if (!fs.existsSync(imgPath)) {
|
try {
|
||||||
|
await fs.promises.access(imgPath);
|
||||||
|
} catch {
|
||||||
return NextResponse.json({ error: 'Image does not exist' }, { status: 404 });
|
return NextResponse.json({ error: 'Image does not exist' }, { status: 404 });
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -21,7 +23,7 @@ export async function POST(request: Request) {
|
|||||||
const captionExt = ((ext || 'txt') as string).replace(/^\.+/, '').trim() || 'txt';
|
const captionExt = ((ext || 'txt') as string).replace(/^\.+/, '').trim() || 'txt';
|
||||||
const captionPath = imgPath.replace(/\.[^/.]+$/, '') + '.' + captionExt;
|
const captionPath = imgPath.replace(/\.[^/.]+$/, '') + '.' + captionExt;
|
||||||
// save caption to file
|
// save caption to file
|
||||||
fs.writeFileSync(captionPath, caption);
|
await fs.promises.writeFile(captionPath, caption);
|
||||||
|
|
||||||
return NextResponse.json({ success: true });
|
return NextResponse.json({ success: true });
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
|||||||
@@ -2,6 +2,8 @@ import { NextResponse } from 'next/server';
|
|||||||
import fs from 'fs';
|
import fs from 'fs';
|
||||||
import { getDatasetsRoot, getTrainingFolder } from '@/server/settings';
|
import { getDatasetsRoot, getTrainingFolder } from '@/server/settings';
|
||||||
|
|
||||||
|
const fileExists = (p: string) => fs.promises.access(p).then(() => true).catch(() => false);
|
||||||
|
|
||||||
export async function POST(request: Request) {
|
export async function POST(request: Request) {
|
||||||
try {
|
try {
|
||||||
const body = await request.json();
|
const body = await request.json();
|
||||||
@@ -20,18 +22,18 @@ export async function POST(request: Request) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// if img doesnt exist, ignore
|
// if img doesnt exist, ignore
|
||||||
if (!fs.existsSync(imgPath)) {
|
if (!(await fileExists(imgPath))) {
|
||||||
return NextResponse.json({ success: true });
|
return NextResponse.json({ success: true });
|
||||||
}
|
}
|
||||||
|
|
||||||
// delete it and return success
|
// delete it and return success
|
||||||
fs.unlinkSync(imgPath);
|
await fs.promises.unlink(imgPath);
|
||||||
|
|
||||||
// check for caption
|
// check for caption
|
||||||
const captionPath = imgPath.replace(/\.[^/.]+$/, '') + '.txt';
|
const captionPath = imgPath.replace(/\.[^/.]+$/, '') + '.txt';
|
||||||
if (fs.existsSync(captionPath)) {
|
if (await fileExists(captionPath)) {
|
||||||
// delete caption file
|
// delete caption file
|
||||||
fs.unlinkSync(captionPath);
|
await fs.promises.unlink(captionPath);
|
||||||
}
|
}
|
||||||
|
|
||||||
return NextResponse.json({ success: true });
|
return NextResponse.json({ success: true });
|
||||||
|
|||||||
@@ -20,9 +20,8 @@ export async function GET(request: NextRequest, { params }: { params: { jobID: s
|
|||||||
const trainingRoot = await getTrainingFolder();
|
const trainingRoot = await getTrainingFolder();
|
||||||
const trainingFolder = path.join(trainingRoot, job.name);
|
const trainingFolder = path.join(trainingRoot, job.name);
|
||||||
|
|
||||||
if (fs.existsSync(trainingFolder)) {
|
// force:true makes this a no-op if the folder is already gone
|
||||||
fs.rmSync(trainingFolder, { recursive: true, force: true });
|
await fs.promises.rm(trainingFolder, { recursive: true, force: true });
|
||||||
}
|
|
||||||
|
|
||||||
await prisma.job.delete({
|
await prisma.job.delete({
|
||||||
where: { id: jobID },
|
where: { id: jobID },
|
||||||
|
|||||||
@@ -20,13 +20,14 @@ export async function GET(request: NextRequest, { params }: { params: { jobID: s
|
|||||||
const trainingFolder = await getTrainingFolder();
|
const trainingFolder = await getTrainingFolder();
|
||||||
const jobFolder = path.join(trainingFolder, job.name);
|
const jobFolder = path.join(trainingFolder, job.name);
|
||||||
|
|
||||||
if (!fs.existsSync(jobFolder)) {
|
try {
|
||||||
|
await fs.promises.access(jobFolder);
|
||||||
|
} catch {
|
||||||
return NextResponse.json({ files: [] });
|
return NextResponse.json({ files: [] });
|
||||||
}
|
}
|
||||||
|
|
||||||
// find all safetensors files in the job folder
|
// find all safetensors files in the job folder
|
||||||
let files = fs
|
let files = (await fs.promises.readdir(jobFolder))
|
||||||
.readdirSync(jobFolder)
|
|
||||||
.filter(file => {
|
.filter(file => {
|
||||||
return file.endsWith('.safetensors');
|
return file.endsWith('.safetensors');
|
||||||
})
|
})
|
||||||
@@ -35,23 +36,27 @@ export async function GET(request: NextRequest, { params }: { params: { jobID: s
|
|||||||
})
|
})
|
||||||
.sort();
|
.sort();
|
||||||
|
|
||||||
// get the file size for each file
|
// get the file size for each file (stat all in parallel)
|
||||||
const fileObjects = files.map(file => {
|
const fileObjects = await Promise.all(
|
||||||
const stats = fs.statSync(file);
|
files.map(async file => {
|
||||||
return {
|
const stats = await fs.promises.stat(file);
|
||||||
path: file,
|
return {
|
||||||
size: stats.size,
|
path: file,
|
||||||
};
|
size: stats.size,
|
||||||
});
|
};
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
// include the optimizer state if it exists
|
// include the optimizer state if it exists
|
||||||
const optimizerPath = path.join(jobFolder, 'optimizer.pt');
|
const optimizerPath = path.join(jobFolder, 'optimizer.pt');
|
||||||
if (fs.existsSync(optimizerPath)) {
|
try {
|
||||||
const stats = fs.statSync(optimizerPath);
|
const stats = await fs.promises.stat(optimizerPath);
|
||||||
fileObjects.push({
|
fileObjects.push({
|
||||||
path: optimizerPath,
|
path: optimizerPath,
|
||||||
size: stats.size,
|
size: stats.size,
|
||||||
});
|
});
|
||||||
|
} catch {
|
||||||
|
// no optimizer state present, skip it
|
||||||
}
|
}
|
||||||
|
|
||||||
return NextResponse.json({ files: fileObjects });
|
return NextResponse.json({ files: fileObjects });
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ import { NextRequest, NextResponse } from 'next/server';
|
|||||||
import { PrismaClient } from '@prisma/client';
|
import { PrismaClient } from '@prisma/client';
|
||||||
import path from 'path';
|
import path from 'path';
|
||||||
import fs from 'fs';
|
import fs from 'fs';
|
||||||
|
import type { FileHandle } from 'fs/promises';
|
||||||
import { getTrainingFolder } from '@/server/settings';
|
import { getTrainingFolder } from '@/server/settings';
|
||||||
|
|
||||||
const prisma = new PrismaClient();
|
const prisma = new PrismaClient();
|
||||||
@@ -21,7 +22,9 @@ export async function GET(request: NextRequest, { params }: { params: { jobID: s
|
|||||||
const jobFolder = path.join(trainingFolder, job.name);
|
const jobFolder = path.join(trainingFolder, job.name);
|
||||||
const logPath = path.join(jobFolder, 'log.txt');
|
const logPath = path.join(jobFolder, 'log.txt');
|
||||||
|
|
||||||
if (!fs.existsSync(logPath)) {
|
try {
|
||||||
|
await fs.promises.access(logPath);
|
||||||
|
} catch {
|
||||||
return NextResponse.json({ log: '', offset: 0, reset: true });
|
return NextResponse.json({ log: '', offset: 0, reset: true });
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -31,28 +34,28 @@ export async function GET(request: NextRequest, { params }: { params: { jobID: s
|
|||||||
const offsetParam = request.nextUrl.searchParams.get('offset');
|
const offsetParam = request.nextUrl.searchParams.get('offset');
|
||||||
const offset = offsetParam === null ? NaN : parseInt(offsetParam, 10);
|
const offset = offsetParam === null ? NaN : parseInt(offsetParam, 10);
|
||||||
|
|
||||||
const readRange = (fd: number, start: number, end: number): string => {
|
const readRange = async (fh: FileHandle, start: number, end: number): Promise<string> => {
|
||||||
const length = end - start;
|
const length = end - start;
|
||||||
if (length <= 0) return '';
|
if (length <= 0) return '';
|
||||||
const buffer = Buffer.alloc(length);
|
const buffer = Buffer.alloc(length);
|
||||||
fs.readSync(fd, buffer, 0, length, start);
|
await fh.read(buffer, 0, length, start);
|
||||||
return buffer.toString('utf-8');
|
return buffer.toString('utf-8');
|
||||||
};
|
};
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const stats = fs.statSync(logPath);
|
const stats = await fs.promises.stat(logPath);
|
||||||
const size = stats.size;
|
const size = stats.size;
|
||||||
// If the client's offset is past the current end, the log was reset/truncated
|
// If the client's offset is past the current end, the log was reset/truncated
|
||||||
// (e.g. a fresh run overwrote it) — fall back to a fresh tail load.
|
// (e.g. a fresh run overwrote it) — fall back to a fresh tail load.
|
||||||
const isReset = Number.isNaN(offset) || offset > size;
|
const isReset = Number.isNaN(offset) || offset > size;
|
||||||
|
|
||||||
const fd = fs.openSync(logPath, 'r');
|
const fh = await fs.promises.open(logPath, 'r');
|
||||||
try {
|
try {
|
||||||
if (isReset) {
|
if (isReset) {
|
||||||
// Read only the tail of the file to avoid loading huge logs into memory.
|
// Read only the tail of the file to avoid loading huge logs into memory.
|
||||||
// Assume an average line length so we grab enough bytes to cover MAX_LINES.
|
// Assume an average line length so we grab enough bytes to cover MAX_LINES.
|
||||||
const start = Math.max(0, size - MAX_LINES * 512);
|
const start = Math.max(0, size - MAX_LINES * 512);
|
||||||
let log = readRange(fd, start, size);
|
let log = await readRange(fh, start, size);
|
||||||
// Drop a partial first line if we started mid-file.
|
// Drop a partial first line if we started mid-file.
|
||||||
if (start > 0) {
|
if (start > 0) {
|
||||||
const newlineIdx = log.indexOf('\n');
|
const newlineIdx = log.indexOf('\n');
|
||||||
@@ -67,10 +70,10 @@ export async function GET(request: NextRequest, { params }: { params: { jobID: s
|
|||||||
return NextResponse.json({ log, offset: size, reset: true });
|
return NextResponse.json({ log, offset: size, reset: true });
|
||||||
}
|
}
|
||||||
// Incremental: return only the bytes appended since the last offset.
|
// Incremental: return only the bytes appended since the last offset.
|
||||||
const log = readRange(fd, offset, size);
|
const log = await readRange(fh, offset, size);
|
||||||
return NextResponse.json({ log, offset: size, reset: false });
|
return NextResponse.json({ log, offset: size, reset: false });
|
||||||
} finally {
|
} finally {
|
||||||
fs.closeSync(fd);
|
await fh.close();
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('Error reading log file:', error);
|
console.error('Error reading log file:', error);
|
||||||
|
|||||||
@@ -42,7 +42,9 @@ export async function GET(request: NextRequest, { params }: { params: { jobID: s
|
|||||||
const jobFolder = path.join(trainingFolder, job.name);
|
const jobFolder = path.join(trainingFolder, job.name);
|
||||||
const logPath = path.join(jobFolder, 'loss_log.db');
|
const logPath = path.join(jobFolder, 'loss_log.db');
|
||||||
|
|
||||||
if (!fs.existsSync(logPath)) {
|
try {
|
||||||
|
await fs.promises.access(logPath);
|
||||||
|
} catch {
|
||||||
return NextResponse.json({ keys: [], key: 'loss', points: [] });
|
return NextResponse.json({ keys: [], key: 'loss', points: [] });
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -21,7 +21,9 @@ export async function GET(request: NextRequest, { params }: { params: { jobID: s
|
|||||||
const jobFolder = path.join(trainingFolder, job.name);
|
const jobFolder = path.join(trainingFolder, job.name);
|
||||||
const pluginPath = path.join(jobFolder, 'plugin.html');
|
const pluginPath = path.join(jobFolder, 'plugin.html');
|
||||||
|
|
||||||
if (!fs.existsSync(pluginPath)) {
|
try {
|
||||||
|
await fs.promises.access(pluginPath);
|
||||||
|
} catch {
|
||||||
return NextResponse.json({ exists: false, html: null });
|
return NextResponse.json({ exists: false, html: null });
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -33,7 +35,7 @@ export async function GET(request: NextRequest, { params }: { params: { jobID: s
|
|||||||
// serve the raw html so it can be loaded directly as an iframe src
|
// serve the raw html so it can be loaded directly as an iframe src
|
||||||
let html = '';
|
let html = '';
|
||||||
try {
|
try {
|
||||||
html = fs.readFileSync(pluginPath, 'utf-8');
|
html = await fs.promises.readFile(pluginPath, 'utf-8');
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('Error reading plugin file:', error);
|
console.error('Error reading plugin file:', error);
|
||||||
return NextResponse.json({ error: 'Error reading plugin file' }, { status: 500 });
|
return NextResponse.json({ error: 'Error reading plugin file' }, { status: 500 });
|
||||||
|
|||||||
@@ -21,13 +21,14 @@ export async function GET(request: NextRequest, { params }: { params: { jobID: s
|
|||||||
const trainingFolder = await getTrainingFolder();
|
const trainingFolder = await getTrainingFolder();
|
||||||
|
|
||||||
const samplesFolder = path.join(trainingFolder, job.name, 'samples');
|
const samplesFolder = path.join(trainingFolder, job.name, 'samples');
|
||||||
if (!fs.existsSync(samplesFolder)) {
|
try {
|
||||||
|
await fs.promises.access(samplesFolder);
|
||||||
|
} catch {
|
||||||
return NextResponse.json({ samples: [] });
|
return NextResponse.json({ samples: [] });
|
||||||
}
|
}
|
||||||
|
|
||||||
// find all img (png, jpg, jpeg) files in the samples folder
|
// find all img (png, jpg, jpeg) files in the samples folder
|
||||||
const samples = fs
|
const samples = (await fs.promises.readdir(samplesFolder))
|
||||||
.readdirSync(samplesFolder)
|
|
||||||
.filter(file => {
|
.filter(file => {
|
||||||
return file.endsWith('.png') || file.endsWith('.jpg') || file.endsWith('.jpeg') || file.endsWith('.webp') || file.endsWith('.mp4') || file.endsWith('mp3') || file.endsWith('wav') || file.endsWith('flac') || file.endsWith('ogg');
|
return file.endsWith('.png') || file.endsWith('.jpg') || file.endsWith('.jpeg') || file.endsWith('.webp') || file.endsWith('.mp4') || file.endsWith('mp3') || file.endsWith('wav') || file.endsWith('flac') || file.endsWith('ogg');
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -1,6 +1,9 @@
|
|||||||
import { NextRequest, NextResponse } from 'next/server';
|
import { NextRequest, NextResponse } from 'next/server';
|
||||||
import { PrismaClient } from '@prisma/client';
|
import { PrismaClient } from '@prisma/client';
|
||||||
|
import { exec } from 'child_process';
|
||||||
|
import { promisify } from 'util';
|
||||||
|
|
||||||
|
const execAsync = promisify(exec);
|
||||||
const prisma = new PrismaClient();
|
const prisma = new PrismaClient();
|
||||||
const isWindows = process.platform === 'win32';
|
const isWindows = process.platform === 'win32';
|
||||||
|
|
||||||
@@ -30,8 +33,7 @@ export async function GET(request: NextRequest, { params }: { params: { jobID: s
|
|||||||
if (isWindows) {
|
if (isWindows) {
|
||||||
// Windows doesn't support SIGINT for arbitrary processes.
|
// Windows doesn't support SIGINT for arbitrary processes.
|
||||||
// Use taskkill with /T (tree) to send a CTRL+C-like termination.
|
// Use taskkill with /T (tree) to send a CTRL+C-like termination.
|
||||||
const { execSync } = require('child_process');
|
await execAsync(`taskkill /PID ${job.pid} /T /F`, { windowsHide: true });
|
||||||
execSync(`taskkill /PID ${job.pid} /T /F`, { stdio: 'ignore' });
|
|
||||||
} else {
|
} else {
|
||||||
process.kill(job.pid, 'SIGINT');
|
process.kill(job.pid, 'SIGINT');
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ const UI_SCRIPTS_ROOT = path.join(TOOLKIT_ROOT, 'ui_scripts');
|
|||||||
// Only allow flat script names (no path separators, no traversal).
|
// Only allow flat script names (no path separators, no traversal).
|
||||||
const SCRIPT_NAME_RE = /^[A-Za-z0-9_][A-Za-z0-9_.-]*\.py$/;
|
const SCRIPT_NAME_RE = /^[A-Za-z0-9_][A-Za-z0-9_.-]*\.py$/;
|
||||||
|
|
||||||
const resolveScriptPath = (rawName: unknown): string | null => {
|
const resolveScriptPath = async (rawName: unknown): Promise<string | null> => {
|
||||||
if (typeof rawName !== 'string') return null;
|
if (typeof rawName !== 'string') return null;
|
||||||
const name = rawName.trim();
|
const name = rawName.trim();
|
||||||
if (!SCRIPT_NAME_RE.test(name)) return null;
|
if (!SCRIPT_NAME_RE.test(name)) return null;
|
||||||
@@ -23,7 +23,11 @@ const resolveScriptPath = (rawName: unknown): string | null => {
|
|||||||
const target = path.resolve(UI_SCRIPTS_ROOT, name);
|
const target = path.resolve(UI_SCRIPTS_ROOT, name);
|
||||||
const rootWithSep = UI_SCRIPTS_ROOT.endsWith(path.sep) ? UI_SCRIPTS_ROOT : UI_SCRIPTS_ROOT + path.sep;
|
const rootWithSep = UI_SCRIPTS_ROOT.endsWith(path.sep) ? UI_SCRIPTS_ROOT : UI_SCRIPTS_ROOT + path.sep;
|
||||||
if (!target.startsWith(rootWithSep)) return null;
|
if (!target.startsWith(rootWithSep)) return null;
|
||||||
if (!fs.existsSync(target) || !fs.statSync(target).isFile()) return null;
|
try {
|
||||||
|
if (!(await fs.promises.stat(target)).isFile()) return null;
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
return target;
|
return target;
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -225,7 +229,7 @@ export async function POST(request: Request) {
|
|||||||
return NextResponse.json({ error: 'Invalid JSON body' }, { status: 400 });
|
return NextResponse.json({ error: 'Invalid JSON body' }, { status: 400 });
|
||||||
}
|
}
|
||||||
|
|
||||||
const scriptPath = resolveScriptPath(body?.script);
|
const scriptPath = await resolveScriptPath(body?.script);
|
||||||
if (!scriptPath) {
|
if (!scriptPath) {
|
||||||
return NextResponse.json(
|
return NextResponse.json(
|
||||||
{ error: 'Invalid or unknown script. Must be a *.py file inside ui_scripts/.' },
|
{ error: 'Invalid or unknown script. Must be a *.py file inside ui_scripts/.' },
|
||||||
|
|||||||
@@ -41,10 +41,8 @@ export async function POST(request: NextRequest) {
|
|||||||
return new NextResponse('Not a directory', { status: 400 });
|
return new NextResponse('Not a directory', { status: 400 });
|
||||||
}
|
}
|
||||||
|
|
||||||
// delete current one if it exists
|
// delete current one if it exists (force:true => no error if missing)
|
||||||
if (fs.existsSync(outputPath)) {
|
await fsp.rm(outputPath, { force: true });
|
||||||
await fsp.unlink(outputPath);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Create write stream & archive
|
// Create write stream & archive
|
||||||
await new Promise<void>((resolve, reject) => {
|
await new Promise<void>((resolve, reject) => {
|
||||||
|
|||||||
Reference in New Issue
Block a user