Improvements to file transfer speed when downloading loras from cloud
This commit is contained in:
1 parent
d5612dd35c
commit
1086bd0b3e
1 file changed
+15
-7
@@ -2,6 +2,7 @@
|
|||||||
import { NextRequest, NextResponse } from 'next/server';
|
import { NextRequest, NextResponse } from 'next/server';
|
||||||
import fs from 'fs';
|
import fs from 'fs';
|
||||||
import path from 'path';
|
import path from 'path';
|
||||||
|
import { Readable } from 'stream';
|
||||||
import { getDatasetsRoot, getTrainingFolder } from '@/server/settings';
|
import { getDatasetsRoot, getTrainingFolder } from '@/server/settings';
|
||||||
|
|
||||||
export async function GET(request: NextRequest, { params }: { params: { filePath: string } }) {
|
export async function GET(request: NextRequest, { params }: { params: { filePath: string } }) {
|
||||||
@@ -85,19 +86,27 @@ export async function GET(request: NextRequest, { params }: { params: { filePath
|
|||||||
};
|
};
|
||||||
|
|
||||||
if (range) {
|
if (range) {
|
||||||
// Parse range header
|
// Parse range header. An open-ended range (`bytes=0-`) must serve to EOF —
|
||||||
|
// capping it forces clients into serial re-requests and RTT-bound throughput.
|
||||||
const parts = range.replace(/bytes=/, '').split('-');
|
const parts = range.replace(/bytes=/, '').split('-');
|
||||||
const start = parseInt(parts[0], 10);
|
const start = parseInt(parts[0], 10);
|
||||||
const end = parts[1] ? parseInt(parts[1], 10) : Math.min(start + 10 * 1024 * 1024, stat.size - 1); // 10MB chunks
|
const end = parts[1] ? parseInt(parts[1], 10) : stat.size - 1;
|
||||||
|
|
||||||
|
if (isNaN(start) || isNaN(end) || start > end || start >= stat.size) {
|
||||||
|
return new NextResponse('Range Not Satisfiable', {
|
||||||
|
status: 416,
|
||||||
|
headers: { 'Content-Range': `bytes */${stat.size}` },
|
||||||
|
});
|
||||||
|
}
|
||||||
const chunkSize = end - start + 1;
|
const chunkSize = end - start + 1;
|
||||||
|
|
||||||
const fileStream = fs.createReadStream(resolvedFilePath, {
|
const fileStream = fs.createReadStream(resolvedFilePath, {
|
||||||
start,
|
start,
|
||||||
end,
|
end,
|
||||||
highWaterMark: 64 * 1024, // 64KB buffer
|
highWaterMark: 4 * 1024 * 1024, // large buffer to minimize per-chunk overhead
|
||||||
});
|
});
|
||||||
|
|
||||||
return new NextResponse(fileStream as any, {
|
return new NextResponse(Readable.toWeb(fileStream) as any, {
|
||||||
status: 206,
|
status: 206,
|
||||||
headers: {
|
headers: {
|
||||||
...commonHeaders,
|
...commonHeaders,
|
||||||
@@ -106,12 +115,11 @@ export async function GET(request: NextRequest, { params }: { params: { filePath
|
|||||||
},
|
},
|
||||||
});
|
});
|
||||||
} else {
|
} else {
|
||||||
// For full file download, read directly without streaming wrapper
|
|
||||||
const fileStream = fs.createReadStream(resolvedFilePath, {
|
const fileStream = fs.createReadStream(resolvedFilePath, {
|
||||||
highWaterMark: 64 * 1024, // 64KB buffer
|
highWaterMark: 4 * 1024 * 1024, // large buffer to minimize per-chunk overhead
|
||||||
});
|
});
|
||||||
|
|
||||||
return new NextResponse(fileStream as any, {
|
return new NextResponse(Readable.toWeb(fileStream) as any, {
|
||||||
headers: {
|
headers: {
|
||||||
...commonHeaders,
|
...commonHeaders,
|
||||||
'Content-Length': String(stat.size),
|
'Content-Length': String(stat.size),
|
||||||
|
|||||||
Reference in new issue
Block a user