reworked upload system to be more robust

next.config.ts

@@ -4,7 +4,6 @@ const nextConfig: NextConfig = {
   /* config options here */
   experimental: {
     serverActions: {
-      bodySizeLimit: '10000mb'
     },
   },
   output: "standalone",

src/app/actions/upload.ts

@@ -4,82 +4,15 @@ import db from '@/lib/db';
 import { dataset_entry } from '@/lib/model/dataset_entry';
 import { revalidatePath } from 'next/cache';
 import { eq } from 'drizzle-orm';
-import { writeFile, mkdir, readFile, rm, readdir } from 'fs/promises';
+import { writeFile, mkdir, readFile, rm } from 'fs/promises';
 import { join } from 'path';
 import { existsSync } from 'fs';
-import extract from 'extract-zip';
-import { line } from 'drizzle-orm/pg-core';
 import { requireAdmin } from '@/lib/auth';
 
-interface DatasetEntryInput {
-  id: string;
-  audio_file: string;
-  duration_ms: string;
-  speaker: string;
-  model: string;
-  dialect: string;
-  iteration: string;
-}
-
-export async function uploadDatasetEntries(
-  datasetId: number,
-  formData: FormData
-) {
-  const result = await requireAdmin();
-  if (!result.authenticated || !result.admin) {
-    throw new Error('Unauthorized');
-  }
-
-  const file = formData.get('file') as File;
-
-  if (!file) {
-    throw new Error('No file provided');
-  }
-
-  if (!file.name.endsWith('.zip')) {
-    throw new Error('File must be a ZIP archive');
-  }
-
-  const tempDir = join(process.cwd(), 'tmp', `upload-${Date.now()}`);
-  const datasetDir = join(process.cwd(), 'public', 'datasets', datasetId.toString());
-
-  try {
-    // Create temp directory
-    await mkdir(tempDir, { recursive: true });
-
-    // Save uploaded file to temp location
-    const tempZipPath = join(tempDir, file.name);
-    const arrayBuffer = await file.arrayBuffer();
-    const buffer = Buffer.from(arrayBuffer);
-    await writeFile(tempZipPath, buffer);
-
-    // Extract ZIP file
-    await extract( tempZipPath, { dir: tempDir } );
-
-    // Read metadata.csv
-    const metadataPath = join(tempDir, 'metadata.csv');
-    if (!existsSync(metadataPath)) {
-      throw new Error('metadata.csv not found in ZIP');
-    }
-
-    const metadataContent = await readFile(metadataPath, 'utf-8');
-    const lines = metadataContent.split('\n').filter(line => line.trim());
-
-    if (lines.length < 2) {
-      throw new Error('metadata.csv is empty or invalid');
-    }
-
-    // Parse CSV header
-    const headers = lines[0].split(',').map(h => h.trim());
-
-    // Create dataset directory
-    await mkdir(datasetDir, { recursive: true });
-
-    // Parse and insert entries
-    const entries: typeof dataset_entry.$inferInsert[] = [];
-
-    // CSV parser that handles quoted fields
-    const parseCSVLine = (line: string): string[] => {
+/**
+ * CSV parser that handles quoted fields
+ */
+function parseCSVLine(line: string): string[] {
   const values: string[] = [];
   let current = '';
   let inQuotes = false;
@@ -109,8 +42,50 @@ export async function uploadDatasetEntries(
   // Add the last field
   values.push(current.trim());
   return values;
-    };
+}
 
+/**
+ * Step 2: Process the extracted ZIP data and insert into database
+ * Copies audio files and creates database entries
+ */
+export async function processDatasetEntries(
+  datasetId: number,
+  tempDir: string
+) {
+  const result = await requireAdmin();
+  if (!result.authenticated || !result.admin) {
+    throw new Error('Unauthorized');
+  }
+
+  const datasetDir = join(process.cwd(), 'public', 'datasets', datasetId.toString());
+
+  try {
+    // Validate temp directory exists
+    if (!existsSync(tempDir)) {
+      throw new Error('Temporary extraction directory not found');
+    }
+
+    // Read metadata.csv again
+    const metadataPath = join(tempDir, 'metadata.csv');
+    if (!existsSync(metadataPath)) {
+      throw new Error('metadata.csv not found in extracted files');
+    }
+
+    const metadataContent = await readFile(metadataPath, 'utf-8');
+    const lines = metadataContent.split('\n').filter(line => line.trim());
+
+    if (lines.length < 2) {
+      throw new Error('metadata.csv is empty or invalid');
+    }
+
+    // Parse CSV
+    const headers = lines[0].split(',').map(h => h.trim());
+    const entries: typeof dataset_entry.$inferInsert[] = [];
+
+    // Create dataset directory
+    await mkdir(datasetDir, { recursive: true });
+
+    // Process each row
     for (let i = 1; i < lines.length; i++) {
       const values = parseCSVLine(lines[i]);
 
@@ -121,11 +96,10 @@ export async function uploadDatasetEntries(
       const row: Record<string, string> = {};
       headers.forEach((header, index) => {
         row[header] = values[index];
-      })
+      });
 
       const audioFile = row.audio_file as string;
-      // Try different path separators and remove /output prefix
-      const relativePath = audioFile.replace(/\\/g, '/')
+      const relativePath = audioFile.replace(/\\/g, '/');
       const audioPath = join(tempDir, relativePath);
 
       if (existsSync(audioPath)) {
@@ -149,7 +123,7 @@ export async function uploadDatasetEntries(
     }
 
     if (entries.length === 0) {
-      throw new Error('No valid entries found in metadata.csv');
+      throw new Error('No valid audio files found for entries');
     }
 
     // Check for existing entries to avoid duplicates
@@ -174,12 +148,13 @@ export async function uploadDatasetEntries(
 
     return {
       success: true,
-      entriesCreated: entries.length,
+      entriesCreated: newEntries.length,
+      message: `Successfully processed and inserted ${newEntries.length} entries.`,
     };
   } catch (error) {
-    console.error('Error uploading dataset entries:', error);
+    console.error('Error processing dataset entries:', error);
     throw new Error(
-      error instanceof Error ? error.message : 'Failed to upload dataset entries'
+      error instanceof Error ? error.message : 'Failed to process dataset entries'
     );
   } finally {
     // Clean up temp directory

src/app/api/upload/route.ts

@@ -0,0 +1,135 @@
+import { writeFile, mkdir, readFile, rm } from 'fs/promises';
+import { join } from 'path';
+import { existsSync } from 'fs';
+import extract from 'extract-zip';
+import { NextRequest, NextResponse } from 'next/server';
+import { requireAdmin } from '@/lib/auth';
+
+function parseCSVLine(line: string): string[] {
+  const values: string[] = [];
+  let current = '';
+  let inQuotes = false;
+
+  for (let i = 0; i < line.length; i++) {
+    const char = line[i];
+    const nextChar = line[i + 1];
+
+    if (char === '"') {
+      if (inQuotes && nextChar === '"') {
+        current += '"';
+        i++;
+      } else {
+        inQuotes = !inQuotes;
+      }
+    } else if (char === ',' && !inQuotes) {
+      values.push(current.trim());
+      current = '';
+    } else {
+      current += char;
+    }
+  }
+
+  values.push(current.trim());
+  return values;
+}
+
+export async function POST(request: NextRequest) {
+  try {
+    const result = await requireAdmin();
+    if (!result.authenticated || !result.admin) {
+      return NextResponse.json(
+        { error: 'Unauthorized' },
+        { status: 401 }
+      );
+    }
+
+    const formData = await request.formData();
+    const file = formData.get('file') as File;
+    const datasetId = formData.get('datasetId') as string;
+
+    if (!file) {
+      return NextResponse.json(
+        { error: 'No file provided' },
+        { status: 400 }
+      );
+    }
+
+    if (!file.name.endsWith('.zip')) {
+      return NextResponse.json(
+        { error: 'File must be a ZIP archive' },
+        { status: 400 }
+      );
+    }
+
+    const tempDir = join(process.cwd(), 'tmp', `upload-${Date.now()}`);
+
+    try {
+      // Create temp directory
+      await mkdir(tempDir, { recursive: true });
+
+      // Save uploaded file to temp location
+      const tempZipPath = join(tempDir, file.name);
+      const arrayBuffer = await file.arrayBuffer();
+      const buffer = Buffer.from(arrayBuffer);
+      await writeFile(tempZipPath, buffer);
+
+      // Extract ZIP file
+      await extract(tempZipPath, { dir: tempDir });
+
+      // Read and validate metadata.csv
+      const metadataPath = join(tempDir, 'metadata.csv');
+      if (!existsSync(metadataPath)) {
+        throw new Error('metadata.csv not found in ZIP');
+      }
+
+      const metadataContent = await readFile(metadataPath, 'utf-8');
+      const lines = metadataContent.split('\n').filter(line => line.trim());
+
+      if (lines.length < 2) {
+        throw new Error('metadata.csv is empty or invalid');
+      }
+
+      // Parse CSV header
+      const headers = lines[0].split(',').map(h => h.trim());
+
+      // Parse and count rows
+      let validRowCount = 0;
+      for (let i = 1; i < lines.length; i++) {
+        const values = parseCSVLine(lines[i]);
+        if (values.length === headers.length) {
+          validRowCount++;
+        }
+      }
+
+      if (validRowCount === 0) {
+        throw new Error('No valid entries found in metadata.csv');
+      }
+
+      return NextResponse.json({
+        success: true,
+        tempDir,
+        metadataCount: validRowCount,
+        message: `Successfully extracted ${validRowCount} entries. Ready for processing.`,
+      });
+    } catch (error) {
+      // Clean up on error
+      try {
+        if (existsSync(tempDir)) {
+          await rm(tempDir, { recursive: true, force: true });
+        }
+      } catch (cleanupError) {
+        console.error('Error cleaning up temp directory:', cleanupError);
+      }
+
+      throw error;
+    }
+  } catch (error) {
+    console.error('Error uploading dataset ZIP:', error);
+    const errorMsg =
+      error instanceof Error ? error.message : 'Failed to upload dataset ZIP';
+    return NextResponse.json(
+      { error: errorMsg },
+      { status: 500 }
+    );
+  }
+}

src/components/UploadDatasetEntriesModal.tsx

@@ -1,7 +1,7 @@
 'use client';
 
 import { useState } from 'react';
-import { uploadDatasetEntries } from '@/app/actions/upload';
+import { processDatasetEntries } from '@/app/actions/upload';
 import Modal from '@/components/Modal';
 
 interface UploadDatasetEntriesModalProps {
@@ -19,6 +19,7 @@ export default function UploadDatasetEntriesModal({
   const [entriesCreated, setEntriesCreated] = useState(0);
   const [status, setStatus] = useState<string>('');
   const [isDragging, setIsDragging] = useState(false);
+  const [tempDir, setTempDir] = useState<string | null>(null);
 
   function handleFileChange(e: React.ChangeEvent<HTMLInputElement>) {
     const selectedFile = e.target.files?.[0];
@@ -39,18 +40,42 @@ export default function UploadDatasetEntriesModal({
     }
 
     setLoading(true);
-    setStatus('Preparing upload...');
     
     try {
+      // Step 1: Upload and extract ZIP via API route
+      setStatus('Uploading and extracting ZIP file...');
       const formData = new FormData();
       formData.append('file', file);
+      formData.append('datasetId', datasetId.toString());
 
-      const result = await uploadDatasetEntries(datasetId, formData);
+      const uploadResponse = await fetch('/api/upload', {
+        method: 'POST',
+        body: formData,
+      });
+
+      if (!uploadResponse.ok) {
+        const errorData = await uploadResponse.json();
+        throw new Error(errorData.error || 'Failed to upload ZIP file');
+      }
+
+      const uploadResult = await uploadResponse.json();
+      
+      if (!uploadResult.success) {
+        throw new Error('Failed to upload ZIP file');
+      }
+
+      setTempDir(uploadResult.tempDir);
+      setStatus(`${uploadResult.message} Now processing...`);
+
+      // Step 2: Process the extracted data (server action)
+      setStatus('Processing entries and copying audio files...');
+      const processResult = await processDatasetEntries(datasetId, uploadResult.tempDir);
 
       setSuccess(true);
-      setEntriesCreated(result.entriesCreated);
-      setStatus(`Successfully created ${result.entriesCreated} entries!`);
+      setEntriesCreated(processResult.entriesCreated);
+      setStatus(processResult.message);
       setFile(null);
+      setTempDir(null);
 
       setTimeout(() => {
         setSuccess(false);
@@ -60,7 +85,7 @@ export default function UploadDatasetEntriesModal({
       }, 2500);
     } catch (err) {
       const errorMsg =
-        err instanceof Error ? err.message : 'Failed to upload dataset entries';
+        err instanceof Error ? err.message : 'Failed to upload and process dataset entries';
       setError(errorMsg);
       setStatus('');
     } finally {
@@ -75,6 +100,7 @@ export default function UploadDatasetEntriesModal({
       setError(null);
       setSuccess(false);
       setStatus('');
+      setTempDir(null);
     }
   }
 
@@ -99,7 +125,7 @@ export default function UploadDatasetEntriesModal({
             disabled: loading,
           },
           {
-            label: loading ? 'Uploading...' : 'Upload',
+            label: loading ? 'Uploading & Processing...' : 'Upload',
             onClick: () => {
               const form = document.getElementById(
                 'uploadForm'
@@ -165,8 +191,8 @@ export default function UploadDatasetEntriesModal({
             </div>
           )}
 
-          {success && (
-            <div className="p-3 bg-green-100 border border-green-400 text-green-700 rounded text-sm">
+          {status && (
+            <div className={`p-3 rounded text-sm ${success ? 'bg-green-100 border border-green-400 text-green-700' : 'bg-blue-100 border border-blue-400 text-blue-700'}`}>
               {status}
             </div>
           )}