Only pull the new log bytes from the ui to improve performance
This commit is contained in:
1 parent
733e14cb58
commit
b36a8e9c4b
2 files changed
+65
-8
No files matched your search
@@ -22,14 +22,58 @@ export async function GET(request: NextRequest, { params }: { params: { jobID: s
|
|||||||
const logPath = path.join(jobFolder, 'log.txt');
|
const logPath = path.join(jobFolder, 'log.txt');
|
||||||
|
|
||||||
if (!fs.existsSync(logPath)) {
|
if (!fs.existsSync(logPath)) {
|
||||||
return NextResponse.json({ log: '' });
|
return NextResponse.json({ log: '', offset: 0, reset: true });
|
||||||
}
|
}
|
||||||
let log = '';
|
|
||||||
|
const MAX_LINES = 2000;
|
||||||
|
// Client sends the byte offset it has already consumed so we only return new
|
||||||
|
// content. `offset` omitted (or NaN) => initial load / full tail.
|
||||||
|
const offsetParam = request.nextUrl.searchParams.get('offset');
|
||||||
|
const offset = offsetParam === null ? NaN : parseInt(offsetParam, 10);
|
||||||
|
|
||||||
|
const readRange = (fd: number, start: number, end: number): string => {
|
||||||
|
const length = end - start;
|
||||||
|
if (length <= 0) return '';
|
||||||
|
const buffer = Buffer.alloc(length);
|
||||||
|
fs.readSync(fd, buffer, 0, length, start);
|
||||||
|
return buffer.toString('utf-8');
|
||||||
|
};
|
||||||
|
|
||||||
try {
|
try {
|
||||||
log = fs.readFileSync(logPath, 'utf-8');
|
const stats = fs.statSync(logPath);
|
||||||
|
const size = stats.size;
|
||||||
|
// If the client's offset is past the current end, the log was reset/truncated
|
||||||
|
// (e.g. a fresh run overwrote it) — fall back to a fresh tail load.
|
||||||
|
const isReset = Number.isNaN(offset) || offset > size;
|
||||||
|
|
||||||
|
const fd = fs.openSync(logPath, 'r');
|
||||||
|
try {
|
||||||
|
if (isReset) {
|
||||||
|
// Read only the tail of the file to avoid loading huge logs into memory.
|
||||||
|
// Assume an average line length so we grab enough bytes to cover MAX_LINES.
|
||||||
|
const start = Math.max(0, size - MAX_LINES * 512);
|
||||||
|
let log = readRange(fd, start, size);
|
||||||
|
// Drop a partial first line if we started mid-file.
|
||||||
|
if (start > 0) {
|
||||||
|
const newlineIdx = log.indexOf('\n');
|
||||||
|
if (newlineIdx !== -1) {
|
||||||
|
log = log.slice(newlineIdx + 1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const lines = log.split('\n');
|
||||||
|
if (lines.length > MAX_LINES) {
|
||||||
|
log = lines.slice(-MAX_LINES).join('\n');
|
||||||
|
}
|
||||||
|
return NextResponse.json({ log, offset: size, reset: true });
|
||||||
|
}
|
||||||
|
// Incremental: return only the bytes appended since the last offset.
|
||||||
|
const log = readRange(fd, offset, size);
|
||||||
|
return NextResponse.json({ log, offset: size, reset: false });
|
||||||
|
} finally {
|
||||||
|
fs.closeSync(fd);
|
||||||
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('Error reading log file:', error);
|
console.error('Error reading log file:', error);
|
||||||
log = 'Error reading log file';
|
return NextResponse.json({ log: 'Error reading log file', offset: 0, reset: true });
|
||||||
}
|
}
|
||||||
return NextResponse.json({ log: log });
|
|
||||||
}
|
}
|
||||||
@@ -17,6 +17,9 @@ const clean = (text: string): string => {
|
|||||||
export default function useJobLog(jobID: string, reloadInterval: null | number = null) {
|
export default function useJobLog(jobID: string, reloadInterval: null | number = null) {
|
||||||
const [log, setLog] = useState<string>('');
|
const [log, setLog] = useState<string>('');
|
||||||
const didInitialLoadRef = useRef(false);
|
const didInitialLoadRef = useRef(false);
|
||||||
|
// Byte offset into the log file that we've already consumed. Sent to the
|
||||||
|
// server so it only returns newly appended content.
|
||||||
|
const offsetRef = useRef<number | null>(null);
|
||||||
const [status, setStatus] = useState<'idle' | 'loading' | 'success' | 'error' | 'refreshing'>('idle');
|
const [status, setStatus] = useState<'idle' | 'loading' | 'success' | 'error' | 'refreshing'>('idle');
|
||||||
|
|
||||||
const refresh = () => {
|
const refresh = () => {
|
||||||
@@ -25,13 +28,19 @@ export default function useJobLog(jobID: string, reloadInterval: null | number =
|
|||||||
loadStatus = 'refreshing';
|
loadStatus = 'refreshing';
|
||||||
}
|
}
|
||||||
setStatus(loadStatus);
|
setStatus(loadStatus);
|
||||||
|
const offset = offsetRef.current;
|
||||||
apiClient
|
apiClient
|
||||||
.get(`/api/jobs/${jobID}/log`)
|
.get(`/api/jobs/${jobID}/log`, offset !== null ? { params: { offset } } : undefined)
|
||||||
.then(res => res.data)
|
.then(res => res.data)
|
||||||
.then(data => {
|
.then(data => {
|
||||||
if (data.log) {
|
offsetRef.current = data.offset ?? null;
|
||||||
let cleanLog = clean(data.log);
|
const cleanLog = clean(data.log ?? '');
|
||||||
|
if (data.reset) {
|
||||||
|
// Log was reset/truncated (or initial load) — replace everything.
|
||||||
setLog(cleanLog);
|
setLog(cleanLog);
|
||||||
|
} else if (cleanLog) {
|
||||||
|
// Incremental — append only the new content.
|
||||||
|
setLog(prev => prev + cleanLog);
|
||||||
}
|
}
|
||||||
setStatus('success');
|
setStatus('success');
|
||||||
didInitialLoadRef.current = true;
|
didInitialLoadRef.current = true;
|
||||||
@@ -43,6 +52,10 @@ export default function useJobLog(jobID: string, reloadInterval: null | number =
|
|||||||
};
|
};
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
|
// New job — start fresh.
|
||||||
|
offsetRef.current = null;
|
||||||
|
didInitialLoadRef.current = false;
|
||||||
|
setLog('');
|
||||||
refresh();
|
refresh();
|
||||||
|
|
||||||
if (reloadInterval) {
|
if (reloadInterval) {
|
||||||
|
|||||||
Reference in new issue
Block a user