From 48781f900bb1afc19d4b25e712aded16eb6fa62a Mon Sep 17 00:00:00 2001
From: Jaret Burkett <jaretburkett@gmail.com>
Date: Wed, 1 Jul 2026 16:40:01 -0600
Subject: [PATCH] Improve the loading and transfer speed of the dataset file
 lists

---
 ui/src/app/api/datasets/listImages/route.ts | 36 ++++++++++++++++++---
 ui/src/app/datasets/[datasetName]/page.tsx  | 16 +++++++--
 2 files changed, 44 insertions(+), 8 deletions(-)

diff --git a/ui/src/app/api/datasets/listImages/route.ts b/ui/src/app/api/datasets/listImages/route.ts
index 92e666a..b1e70db 100644
--- a/ui/src/app/api/datasets/listImages/route.ts
+++ b/ui/src/app/api/datasets/listImages/route.ts
@@ -1,8 +1,13 @@
 import { NextResponse } from 'next/server';
 import fs from 'fs';
 import path from 'path';
+import zlib from 'zlib';
+import { promisify } from 'util';
 import { getDatasetsRoot } from '@/server/settings';
 
+const brotliCompress = promisify(zlib.brotliCompress);
+const gzipCompress = promisify(zlib.gzip);
+
 export async function POST(request: Request) {
   const datasetsPath = await getDatasetsRoot();
   const body = await request.json();
@@ -21,12 +26,33 @@ export async function POST(request: Request) {
     // Sort server-side so the client doesn't have to sort large lists
     imageFiles.sort((a, b) => a.localeCompare(b));
 
-    // Format response
-    const result = imageFiles.map(imgPath => ({
-      img_path: imgPath,
-    }));
+    // Send a single shared root plus each file's sub-path, rather than repeating the full
+    // absolute path (and an "img_path" key) on every entry. The root carries the trailing
+    // OS separator so the client rebuilds the native path with a plain concat (root + subPath),
+    // keeping correct paths on Windows, macOS, and Linux without any separator logic client-side.
+    const root = datasetFolder + path.sep;
+    const result = imageFiles.map(imgPath => imgPath.slice(root.length));
 
-    return NextResponse.json({ images: result });
+    // Compress the payload explicitly. Even after stripping the shared prefix these lists
+    // still gzip/brotli down substantially — a big win on slow connections and huge datasets.
+    // The browser decompresses transparently via Content-Encoding, so the client just parses JSON.
+    const json = JSON.stringify({ root, images: result });
+    const acceptEncoding = request.headers.get('accept-encoding') ?? '';
+
+    if (/\bbr\b/.test(acceptEncoding)) {
+      const body = await brotliCompress(json);
+      return new NextResponse(body as any, {
+        headers: { 'Content-Type': 'application/json', 'Content-Encoding': 'br' },
+      });
+    }
+    if (/\bgzip\b/.test(acceptEncoding)) {
+      const body = await gzipCompress(json);
+      return new NextResponse(body as any, {
+        headers: { 'Content-Type': 'application/json', 'Content-Encoding': 'gzip' },
+      });
+    }
+
+    return new NextResponse(json, { headers: { 'Content-Type': 'application/json' } });
   } catch (error) {
     console.error('Error finding images:', error);
     return NextResponse.json({ error: 'Failed to process request' }, { status: 500 });
diff --git a/ui/src/app/datasets/[datasetName]/page.tsx b/ui/src/app/datasets/[datasetName]/page.tsx
index 147f9c1..94a0700 100644
--- a/ui/src/app/datasets/[datasetName]/page.tsx
+++ b/ui/src/app/datasets/[datasetName]/page.tsx
@@ -1,6 +1,6 @@
 'use client';
 
-import { useEffect, useState, use, useMemo, useCallback } from 'react';
+import { useEffect, useState, use, useMemo, useCallback, useRef } from 'react';
 import { LuImageOff, LuLoader, LuBan } from 'react-icons/lu';
 import { FaChevronLeft } from 'react-icons/fa';
 import { VirtuosoGrid } from 'react-virtuoso';
@@ -29,20 +29,30 @@ export default function DatasetPage({ params }: { params: { datasetName: string
   const [scrollParent, setScrollParent] = useState<HTMLDivElement | null>(null);
   const [captionBarHeight, setCaptionBarHeight] = useState(0);
   const scrollParentCallback = useCallback((el: HTMLDivElement | null) => setScrollParent(el), []);
+  const isRefreshingRef = useRef(false);
 
   const refreshImageList = (dbName: string) => {
+    // Only allow one listImages request in flight at a time.
+    if (isRefreshingRef.current) return;
+    isRefreshingRef.current = true;
     setStatus('loading');
     apiClient
       .post('/api/datasets/listImages', { datasetName: dbName })
       .then((res: any) => {
         const data = res.data;
-        // Server already sorts; avoid the client-side sort that's expensive on large lists.
-        setImgList(data.images);
+        // Server sends a shared root (with trailing OS separator) + each file's sub-path to
+        // keep the payload small. Plain concat rebuilds the native absolute path on any OS.
+        // Server already sorts; avoid a client-side sort on large lists.
+        const root = data.root;
+        setImgList(data.images.map((subPath: string) => ({ img_path: root + subPath })));
         setStatus('success');
       })
       .catch(error => {
         console.error('Error fetching images:', error);
         setStatus('error');
+      })
+      .finally(() => {
+        isRefreshingRef.current = false;
       });
   };
   useOpenImagesModalOnDrag(datasetName, () => refreshImageList(datasetName));