Improve dataset uploader. Upload the files one at a time instead of one huge chunk. Show progress for each file.

ui/src/components/AddImagesModal.tsx

@@ -2,9 +2,7 @@
 import { createGlobalState } from 'react-global-hooks';
 import { Dialog, DialogBackdrop, DialogPanel, DialogTitle } from '@headlessui/react';
 import { FaUpload } from 'react-icons/fa';
-import { useCallback, useState } from 'react';
+import { useEffect, useRef } from 'react';
-import { useDropzone } from 'react-dropzone';
-import { apiClient } from '@/utils/api';
 
 export interface AddImagesModalState {
   datasetName: string;
@@ -19,69 +17,34 @@ export const openImagesModal = (datasetName: string, onComplete: () => void) =>
 
 export default function AddImagesModal() {
   const [addImagesModalInfo, setAddImagesModalInfo] = addImagesModalState.use();
-  const [uploadProgress, setUploadProgress] = useState<number>(0);
-  const [isUploading, setIsUploading] = useState<boolean>(false);
   const open = addImagesModalInfo !== null;
+  const panelRef = useRef<HTMLDivElement>(null);
 
   const onCancel = () => {
-    if (!isUploading) {
+    setAddImagesModalInfo(null);
-      setAddImagesModalInfo(null);
-    }
   };
 
   const onDone = () => {
-    if (addImagesModalInfo?.onComplete && !isUploading) {
+    if (addImagesModalInfo?.onComplete) {
       addImagesModalInfo.onComplete();
-      setAddImagesModalInfo(null);
     }
+    setAddImagesModalInfo(null);
   };
 
-  const onDrop = useCallback(
+  // Close modal as soon as files are dragged in so the FullscreenDropOverlay can handle the drop
-    async (acceptedFiles: File[]) => {
+  useEffect(() => {
-      if (acceptedFiles.length === 0) return;
+    if (!open) return;
 
-      setIsUploading(true);
+    const handleDragEnter = (e: DragEvent) => {
-      setUploadProgress(0);
+      const types = e?.dataTransfer?.types;
-
+      if (types && Array.from(types).includes('Files')) {
-      const formData = new FormData();
+        setAddImagesModalInfo(null);
-      acceptedFiles.forEach(file => {
-        formData.append('files', file);
-      });
-      formData.append('datasetName', addImagesModalInfo?.datasetName || '');
-
-      try {
-        await apiClient.post(`/api/datasets/upload`, formData, {
-          headers: {
-            'Content-Type': 'multipart/form-data',
-          },
-          onUploadProgress: progressEvent => {
-            const percentCompleted = Math.round((progressEvent.loaded * 100) / (progressEvent.total || 100));
-            setUploadProgress(percentCompleted);
-          },
-          timeout: 0, // Disable timeout
-        });
-
-        onDone();
-      } catch (error) {
-        console.error('Upload failed:', error);
-      } finally {
-        setIsUploading(false);
-        setUploadProgress(0);
       }
-    },
+    };
-    [addImagesModalInfo],
-  );
 
-  const { getRootProps, getInputProps, isDragActive } = useDropzone({
+    window.addEventListener('dragenter', handleDragEnter);
-    onDrop,
+    return () => window.removeEventListener('dragenter', handleDragEnter);
-    accept: {
+  }, [open, setAddImagesModalInfo]);
-      'image/*': ['.png', '.jpg', '.jpeg', '.gif', '.bmp', '.webp'],
-      'video/*': ['.mp4', '.avi', '.mov', '.mkv', '.wmv', '.m4v', '.flv'],
-      'audio/*': ['.mp3', '.wav'],
-      'text/*': ['.txt'],
-    },
-    multiple: true,
-  });
 
   return (
     <Dialog open={open} onClose={onCancel} className="relative z-10">
@@ -93,6 +56,7 @@ export default function AddImagesModal() {
       <div className="fixed inset-0 z-10 w-screen overflow-y-auto">
         <div className="flex min-h-full items-end justify-center p-4 text-center sm:items-center sm:p-0">
           <DialogPanel
+            ref={panelRef}
             transition
             className="relative transform overflow-hidden rounded-lg bg-gray-800 text-left shadow-xl transition-all data-closed:translate-y-4 data-closed:opacity-0 data-enter:duration-300 data-enter:ease-out data-leave:duration-200 data-leave:ease-in sm:my-8 sm:w-full sm:max-w-lg data-closed:sm:translate-y-0 data-closed:sm:scale-95"
           >
@@ -103,24 +67,13 @@ export default function AddImagesModal() {
                 </DialogTitle>
                 <div className="w-full">
                   <div
-                    {...getRootProps()}
+                    className="h-40 w-full flex flex-col items-center justify-center border-2 border-dashed rounded-lg border-gray-600"
-                    className={`h-40 w-full flex flex-col items-center justify-center border-2 border-dashed rounded-lg cursor-pointer transition-colors duration-200
-                      ${isDragActive ? 'border-blue-500 bg-blue-50/10' : 'border-gray-600'}`}
                   >
-                    <input {...getInputProps()} />
                     <FaUpload className="size-8 mb-3 text-gray-400" />
                     <p className="text-sm text-gray-200 text-center">
-                      {isDragActive ? 'Drop the files here...' : 'Drag & drop files here, or click to select files'}
+                      Drag & drop files anywhere on the page to upload
                     </p>
                   </div>
-                  {isUploading && (
-                    <div className="mt-4">
-                      <div className="w-full bg-gray-700 rounded-full h-2.5">
-                        <div className="bg-blue-600 h-2.5 rounded-full" style={{ width: `${uploadProgress}%` }}></div>
-                      </div>
-                      <p className="text-sm text-gray-300 mt-2 text-center">Uploading... {uploadProgress}%</p>
-                    </div>
-                  )}
                 </div>
               </div>
             </div>
@@ -128,9 +81,7 @@ export default function AddImagesModal() {
               <button
                 type="button"
                 onClick={onDone}
-                disabled={isUploading}
+                className="inline-flex w-full justify-center rounded-md bg-slate-600 px-3 py-2 text-sm font-semibold text-white shadow-xs sm:ml-3 sm:w-auto"
-                className={`inline-flex w-full justify-center rounded-md bg-slate-600 px-3 py-2 text-sm font-semibold text-white shadow-xs sm:ml-3 sm:w-auto
-                  ${isUploading ? 'opacity-50 cursor-not-allowed' : ''}`}
               >
                 Done
               </button>
@@ -138,9 +89,7 @@ export default function AddImagesModal() {
                 type="button"
                 data-autofocus
                 onClick={onCancel}
-                disabled={isUploading}
+                className="mt-3 inline-flex w-full justify-center rounded-md bg-gray-800 px-3 py-2 text-sm font-semibold text-gray-200 hover:bg-gray-800 sm:mt-0 sm:w-auto ring-0"
-                className={`mt-3 inline-flex w-full justify-center rounded-md bg-gray-800 px-3 py-2 text-sm font-semibold text-gray-200 hover:bg-gray-800 sm:mt-0 sm:w-auto ring-0
-                  ${isUploading ? 'opacity-50 cursor-not-allowed' : ''}`}
               >
                 Cancel
               </button>

ui/src/components/FullscreenDropOverlay.tsx

@@ -2,7 +2,7 @@
 
 import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
 import { useDropzone } from 'react-dropzone';
-import { FaUpload } from 'react-icons/fa';
+import { FaUpload, FaTimesCircle, FaSpinner } from 'react-icons/fa';
 import { apiClient } from '@/utils/api';
 
 type AcceptMap = {
@@ -10,12 +10,28 @@ type AcceptMap = {
 };
 
 interface FullscreenDropOverlayProps {
-  datasetName: string; // where to upload
+  datasetName: string;
-  onComplete?: () => void; // called after successful upload
+  onComplete?: () => void;
-  accept?: AcceptMap; // optional override
+  accept?: AcceptMap;
-  multiple?: boolean; // default true
+  multiple?: boolean;
 }
 
+type FileStatus = 'pending' | 'uploading' | 'error';
+
+interface FileEntry {
+  id: number;
+  file: File;
+  status: FileStatus;
+  progress: number;
+  error?: string;
+}
+
+const MAX_CONCURRENT = 3;
+const ROW_HEIGHT = 32;
+const VISIBLE_ROWS = 8;
+
+let nextId = 0;
+
 export default function FullscreenDropOverlay({
   datasetName,
   onComplete,
@@ -24,16 +40,18 @@ export default function FullscreenDropOverlay({
 }: FullscreenDropOverlayProps) {
   const [visible, setVisible] = useState(false);
   const [isUploading, setIsUploading] = useState(false);
-  const [uploadProgress, setUploadProgress] = useState(0);
+  const [fileEntries, setFileEntries] = useState<FileEntry[]>([]);
-  const dragDepthRef = useRef(0); // drag-enter/leave tracking
+  const [totalCount, setTotalCount] = useState(0);
+  const [doneCount, setDoneCount] = useState(0);
+  const [errorCount, setErrorCount] = useState(0);
+  const dragDepthRef = useRef(0);
+  const abortRef = useRef(false);
 
-  // Only show the overlay for real file drags (not text, images from page, etc)
   const isFileDrag = (e: DragEvent) => {
     const types = e?.dataTransfer?.types;
     return !!types && Array.from(types).includes('Files');
   };
 
-  // Window-level drag listeners to toggle visibility
   useEffect(() => {
     const onDragEnter = (e: DragEvent) => {
       if (!isFileDrag(e)) return;
@@ -43,7 +61,6 @@ export default function FullscreenDropOverlay({
     };
     const onDragOver = (e: DragEvent) => {
       if (!isFileDrag(e)) return;
-      // Must preventDefault to allow dropping in the browser
       e.preventDefault();
       if (!visible) setVisible(true);
     };
@@ -54,63 +71,131 @@ export default function FullscreenDropOverlay({
         setVisible(false);
       }
     };
-    const onDrop = (e: DragEvent) => {
+    const onWindowDrop = (e: DragEvent) => {
       if (!isFileDrag(e)) return;
-      // Prevent browser from opening the file
       e.preventDefault();
       dragDepthRef.current = 0;
-      // We do NOT hide here; the dropzone onDrop will handle workflow visibility.
     };
 
     window.addEventListener('dragenter', onDragEnter);
     window.addEventListener('dragover', onDragOver);
     window.addEventListener('dragleave', onDragLeave);
-    window.addEventListener('drop', onDrop);
+    window.addEventListener('drop', onWindowDrop);
 
     return () => {
       window.removeEventListener('dragenter', onDragEnter);
       window.removeEventListener('dragover', onDragOver);
       window.removeEventListener('dragleave', onDragLeave);
-      window.removeEventListener('drop', onDrop);
+      window.removeEventListener('drop', onWindowDrop);
     };
   }, [visible, isUploading]);
 
+  const uploadSingleFile = useCallback(
+    async (entry: FileEntry): Promise<'done' | 'error'> => {
+      if (abortRef.current) return 'error';
+
+      const id = entry.id;
+
+      setFileEntries(prev =>
+        prev.map(e => (e.id === id ? { ...e, status: 'uploading' as FileStatus, progress: 0 } : e)),
+      );
+
+      const formData = new FormData();
+      formData.append('files', entry.file);
+      formData.append('datasetName', datasetName || '');
+
+      try {
+        await apiClient.post('/api/datasets/upload', formData, {
+          headers: { 'Content-Type': 'multipart/form-data' },
+          onUploadProgress: pe => {
+            const percent = Math.round(((pe.loaded || 0) * 100) / (pe.total || pe.loaded || 1));
+            setFileEntries(prev => prev.map(e => (e.id === id ? { ...e, progress: percent } : e)));
+          },
+          timeout: 0,
+        });
+        // Remove from list on success
+        setFileEntries(prev => prev.filter(e => e.id !== id));
+        setDoneCount(prev => prev + 1);
+        return 'done';
+      } catch (err) {
+        console.error(`Upload failed for ${entry.file.name}:`, err);
+        setFileEntries(prev =>
+          prev.map(e =>
+            e.id === id
+              ? { ...e, status: 'error' as FileStatus, error: err instanceof Error ? err.message : 'Upload failed' }
+              : e,
+          ),
+        );
+        setErrorCount(prev => prev + 1);
+        return 'error';
+      }
+    },
+    [datasetName],
+  );
+
+  const processQueue = useCallback(
+    async (entries: FileEntry[]) => {
+      setIsUploading(true);
+      abortRef.current = false;
+
+      let nextIndex = 0;
+
+      const runNext = async (): Promise<void> => {
+        while (nextIndex < entries.length) {
+          if (abortRef.current) return;
+          const idx = nextIndex++;
+          await uploadSingleFile(entries[idx]);
+        }
+      };
+
+      const workers = Array.from({ length: Math.min(MAX_CONCURRENT, entries.length) }, () => runNext());
+      await Promise.all(workers);
+
+      setIsUploading(false);
+      if (!abortRef.current) {
+        onComplete?.();
+        setFileEntries([]);
+        setVisible(false);
+        setTotalCount(0);
+        setDoneCount(0);
+        setErrorCount(0);
+      }
+    },
+    [uploadSingleFile, onComplete],
+  );
+
   const onDrop = useCallback(
-    async (acceptedFiles: File[]) => {
+    (acceptedFiles: File[]) => {
       if (acceptedFiles.length === 0) {
-        // no accepted files; hide overlay cleanly
         setVisible(false);
         return;
       }
 
-      setIsUploading(true);
+      const entries: FileEntry[] = acceptedFiles.map(file => ({
-      setUploadProgress(0);
+        id: nextId++,
-
+        file,
-      const formData = new FormData();
+        status: 'pending' as FileStatus,
-      acceptedFiles.forEach(file => formData.append('files', file));
+        progress: 0,
-      formData.append('datasetName', datasetName || '');
+      }));
-
+      setFileEntries(entries);
-      try {
+      setTotalCount(entries.length);
-        await apiClient.post(`/api/datasets/upload`, formData, {
+      setDoneCount(0);
-          headers: { 'Content-Type': 'multipart/form-data' },
+      setErrorCount(0);
-          onUploadProgress: pe => {
+      processQueue(entries);
-            const percent = Math.round(((pe.loaded || 0) * 100) / (pe.total || pe.loaded || 1));
-            setUploadProgress(percent);
-          },
-          timeout: 0,
-        });
-        onComplete?.();
-      } catch (err) {
-        console.error('Upload failed:', err);
-      } finally {
-        setIsUploading(false);
-        setUploadProgress(0);
-        setVisible(false);
-      }
     },
-    [datasetName, onComplete],
+    [processQueue],
   );
 
+  const handleClose = useCallback(() => {
+    abortRef.current = true;
+    setIsUploading(false);
+    setFileEntries([]);
+    setVisible(false);
+    setTotalCount(0);
+    setDoneCount(0);
+    setErrorCount(0);
+  }, []);
+
   const dropAccept = useMemo<AcceptMap>(
     () =>
       accept || {
@@ -128,56 +213,134 @@ export default function FullscreenDropOverlay({
     multiple,
     noClick: true,
     noKeyboard: true,
-    // Prevent "folder opens" by browser if someone drags outside the overlay mid-drop:
     preventDropOnDocument: true,
   });
 
+  const overallPercent = totalCount > 0 ? Math.round(((doneCount + errorCount) / totalCount) * 100) : 0;
+
   return (
     <div
-      // When hidden: opacity-0 + pointer-events-none so the page is fully interactive
-      // When visible or uploading: fade in and capture the drop
       className={`fixed inset-0 z-[9999] transition-opacity duration-200 ${
         visible || isUploading ? 'opacity-100 pointer-events-auto' : 'opacity-0 pointer-events-none'
       }`}
       aria-hidden={!visible && !isUploading}
       {...getRootProps()}
     >
-      {/* Fullscreen capture layer */}
       <input {...getInputProps()} />
-
-      {/* Backdrop: keep it subtle so context remains visible */}
       <div className={`absolute inset-0 ${isUploading ? 'bg-gray-900/70' : 'bg-gray-900/40'}`} />
 
-      {/* Center drop target UI */}
       <div className="absolute inset-0 flex items-center justify-center p-6">
-        <div
+        <div className="w-full max-w-2xl flex flex-col gap-3">
-          className={`w-full max-w-2xl rounded-2xl border-2 border-dashed px-8 py-10 text-center shadow-2xl backdrop-blur-sm
+          {/* Drop target / status box */}
-          ${isDragActive ? 'border-blue-400 bg-white/10' : 'border-white/30 bg-white/5'}`}
+          <div
-        >
+            className={`rounded-2xl border-2 border-dashed px-8 py-10 text-center shadow-2xl backdrop-blur-sm
-          <div className="flex flex-col items-center gap-4">
+            ${isDragActive ? 'border-blue-400 bg-white/10' : 'border-white/30 bg-white/5'}`}
-            <FaUpload className="size-10 opacity-80" />
+          >
-            {!isUploading ? (
+            <div className="flex flex-col items-center gap-4">
-              <>
+              <FaUpload className="size-10 opacity-80" />
-                <p className="text-lg font-semibold">Drop files to upload</p>
+              {!isUploading ? (
-                <p className="text-sm opacity-80">
+                <>
-                  Destination:&nbsp;<span className="font-mono">{datasetName || 'unknown'}</span>
+                  <p className="text-lg font-semibold">Drop files to upload</p>
-                </p>
+                  <p className="text-sm opacity-80">
-                <p className="text-xs opacity-70 mt-1">Images, videos, or .txt supported</p>
+                    Destination:&nbsp;<span className="font-mono">{datasetName || 'unknown'}</span>
-              </>
+                  </p>
-            ) : (
+                  <p className="text-xs opacity-70 mt-1">Images, videos, or .txt supported</p>
-              <>
+                </>
-                <p className="text-lg font-semibold">Uploading… {uploadProgress}%</p>
+              ) : (
-                <div className="w-full h-2.5 bg-white/20 rounded-full overflow-hidden">
+                <>
-                  <div
+                  <p className="text-lg font-semibold">
-                    className="h-2.5 bg-blue-500 rounded-full transition-[width] duration-150 ease-linear"
+                    Uploading… {doneCount + errorCount} / {totalCount}
-                    style={{ width: `${uploadProgress}%` }}
+                  </p>
-                  />
+                  <div className="w-full h-2.5 bg-white/20 rounded-full overflow-hidden">
-                </div>
+                    <div
-              </>
+                      className="h-2.5 bg-blue-500 rounded-full transition-[width] duration-150 ease-linear"
-            )}
+                      style={{ width: `${overallPercent}%` }}
+                    />
+                  </div>
+                  {errorCount > 0 && (
+                    <p className="text-xs text-red-400">
+                      {errorCount} file{errorCount !== 1 ? 's' : ''} failed
+                    </p>
+                  )}
+                  <button
+                    type="button"
+                    onClick={e => {
+                      e.stopPropagation();
+                      handleClose();
+                    }}
+                    className="mt-2 px-4 py-1.5 text-sm rounded-lg bg-white/10 hover:bg-white/20 transition-colors"
+                  >
+                    Cancel
+                  </button>
+                </>
+              )}
+            </div>
           </div>
+
+          {/* File progress list — only shows pending, uploading, and errored files */}
+          {fileEntries.length > 0 && <FileProgressList entries={fileEntries} />}
         </div>
       </div>
     </div>
   );
 }
+
+/** Virtualized file progress list for handling thousands of files */
+function FileProgressList({ entries }: { entries: FileEntry[] }) {
+  const containerRef = useRef<HTMLDivElement>(null);
+  const [scrollTop, setScrollTop] = useState(0);
+
+  const totalHeight = entries.length * ROW_HEIGHT;
+  const containerHeight = Math.min(entries.length, VISIBLE_ROWS) * ROW_HEIGHT;
+
+  const startIdx = Math.floor(scrollTop / ROW_HEIGHT);
+  const endIdx = Math.min(entries.length, startIdx + VISIBLE_ROWS + 2);
+  const visibleEntries = entries.slice(startIdx, endIdx);
+  const offsetY = startIdx * ROW_HEIGHT;
+
+  const onScroll = useCallback(() => {
+    if (containerRef.current) {
+      setScrollTop(containerRef.current.scrollTop);
+    }
+  }, []);
+
+  return (
+    <div
+      ref={containerRef}
+      onScroll={onScroll}
+      onClick={e => e.stopPropagation()}
+      className="rounded-xl bg-black/60 backdrop-blur-sm border border-white/10 overflow-y-auto"
+      style={{ height: containerHeight + 2 }}
+    >
+      <div style={{ height: totalHeight, position: 'relative' }}>
+        <div style={{ position: 'absolute', top: offsetY, left: 0, right: 0 }}>
+          {visibleEntries.map(entry => (
+            <FileRow key={entry.id} entry={entry} />
+          ))}
+        </div>
+      </div>
+    </div>
+  );
+}
+
+function FileRow({ entry }: { entry: FileEntry }) {
+  return (
+    <div className="flex items-center gap-2 px-3 text-xs font-mono" style={{ height: ROW_HEIGHT }}>
+      <span className="flex-shrink-0 w-4 text-center">
+        {entry.status === 'error' && <FaTimesCircle className="text-red-400 inline" />}
+        {entry.status === 'uploading' && <FaSpinner className="text-blue-400 inline animate-spin" />}
+        {entry.status === 'pending' && <span className="inline-block w-2 h-2 rounded-full bg-white/30" />}
+      </span>
+
+      <span className="truncate flex-1 opacity-80" title={entry.file.name}>
+        {entry.file.name}
+      </span>
+
+      <span className="flex-shrink-0 w-16 text-right">
+        {entry.status === 'uploading' && <span className="text-blue-300">{entry.progress}%</span>}
+        {entry.status === 'error' && <span className="text-red-400">Failed</span>}
+        {entry.status === 'pending' && <span className="text-white/30">Queued</span>}
+      </span>
+    </div>
+  );
+}