feat: add functionality to upload and update dataset metadata via CSV files

src/app/actions/upload.ts

@@ -10,6 +10,74 @@ import { join } from 'path';
 import { existsSync } from 'fs';
 import { requireAdmin } from '@/lib/auth';
 
+/**
+ * Helper: Parse metadata.csv from extracted directory
+ * Returns structured data for both insert and update operations
+ */
+async function parseMetadataFromDirectory(
+  tempDir: string
+): Promise<{
+  headers: string[];
+  rows: Record<string, string>[];
+}> {
+  const metadataPath = join(tempDir, 'metadata.csv');
+  if (!existsSync(metadataPath)) {
+    throw new Error('metadata.csv not found in extracted files');
+  }
+
+  const metadataContent = await readFile(metadataPath, 'utf-8');
+  const lines = metadataContent.split('\n').filter((line) => line.trim());
+
+  if (lines.length < 2) {
+    throw new Error('metadata.csv is empty or invalid');
+  }
+
+  const headers = lines[0].split(',').map((h) => h.trim());
+  const rows: Record<string, string>[] = [];
+
+  for (let i = 1; i < lines.length; i++) {
+    const values = parseCSVLine(lines[i]);
+
+    if (values.length !== headers.length) {
+      continue; // Skip malformed lines
+    }
+
+    const row: Record<string, string> = {};
+    headers.forEach((header, index) => {
+      row[header] = values[index];
+    });
+
+    rows.push(row);
+  }
+
+  return { headers, rows };
+}
+
+/**
+ * Convert metadata row to dataset_entry insert object
+ */
+function rowToDatasetEntry(
+  row: Record<string, string>,
+  datasetId: number
+): typeof dataset_entry.$inferInsert {
+  return {
+    datasetId,
+    externalId: row.id,
+    speakerId: row.speaker,
+    modelName: row.model,
+    utteranceId: row.utt_id,
+    utteranceText: row.text,
+    fileName: row.audio_file,
+    dialect: row.dialect,
+    iteration: parseInt(row.iteration, 10),
+    durationMs: row.duration_ms ? parseInt(row.duration_ms, 10) : undefined,
+    rmsValue: row.rms_value ? parseFloat(row.rms_value) : undefined,
+    longestPause: row.longest_pause ? parseFloat(row.longest_pause) : undefined,
+    utmosScore: row.utmos_score ? parseFloat(row.utmos_score) : undefined,
+    werScore: row.wer_score ? parseFloat(row.wer_score) : undefined,
+  };
+}
+
 /**
  * Step 2: Process the extracted ZIP data and insert into database
  * Copies audio files and creates database entries
@@ -39,47 +107,16 @@ export async function processDatasetEntries(
       };
     }
 
-    // Read metadata.csv again
+    // Parse metadata from directory
-    const metadataPath = join(tempDir, 'metadata.csv');
+    const { rows } = await parseMetadataFromDirectory(tempDir);
-    if (!existsSync(metadataPath)) {
-      return {
-        success: false,
-        error: 'metadata.csv not found in extracted files',
-        entriesCreated: 0,
-      };
-    }
 
-    const metadataContent = await readFile(metadataPath, 'utf-8');
-    const lines = metadataContent.split('\n').filter(line => line.trim());
-
-    if (lines.length < 2) {
-      return {
-        success: false,
-        error: 'metadata.csv is empty or invalid',
-        entriesCreated: 0,
-      };
-    }
-
-    // Parse CSV
-    const headers = lines[0].split(',').map(h => h.trim());
     const entries: typeof dataset_entry.$inferInsert[] = [];
 
     // Create dataset directory
     await mkdir(datasetDir, { recursive: true });
 
     // Process each row
-    for (let i = 1; i < lines.length; i++) {
+    for (const row of rows) {
-      const values = parseCSVLine(lines[i]);
-
-      if (values.length !== headers.length) {
-        continue; // Skip malformed lines
-      }
-
-      const row: Record<string, string> = {};
-      headers.forEach((header, index) => {
-        row[header] = values[index];
-      });
-
       const audioFile = row.audio_file as string;
       const relativePath = audioFile.replace(/\\/g, '/');
       const audioPath = join(tempDir, relativePath);
@@ -92,22 +129,7 @@ export async function processDatasetEntries(
 
         await writeFile(destPath, audioBuffer);
 
-        entries.push({
+        entries.push(rowToDatasetEntry(row, datasetId));
-          datasetId,
-          externalId: row.id,
-          speakerId: row.speaker,
-          modelName: row.model,
-          utteranceId: row.utt_id,
-          utteranceText: row.text,
-          fileName: relativePath,
-          dialect: row.dialect,
-          iteration: parseInt(row.iteration, 10),
-          durationMs: row.duration_ms ? parseInt(row.duration_ms, 10) : undefined,
-          rmsValue: row.rms_value ? parseFloat(row.rms_value) : undefined,
-          longestPause: row.longest_pause ? parseFloat(row.longest_pause) : undefined,
-          utmosScore: row.utmos_score ? parseFloat(row.utmos_score) : undefined,
-          werScore: row.wer_score ? parseFloat(row.wer_score) : undefined,
-        });
       }
     }
 
@@ -167,3 +189,116 @@ export async function processDatasetEntries(
     }
   }
 }
+
+/**
+ * Update metadata for existing dataset entries
+ * Uses the same metadata.csv parsing as processDatasetEntries
+ * Matches entries by externalId and updates their metadata fields
+ */
+export async function updateDatasetEntriesMetadata(
+  datasetId: number,
+  tempDir: string
+) {
+  try {
+    const result = await requireAdmin();
+    if (!result.authenticated || !result.admin) {
+      return {
+        success: false,
+        error: 'Unauthorized',
+        entriesUpdated: 0,
+      };
+    }
+
+    // Validate temp directory exists
+    if (!existsSync(tempDir)) {
+      return {
+        success: false,
+        error: 'Temporary extraction directory not found',
+        entriesUpdated: 0,
+      };
+    }
+
+    // Parse metadata from directory (reusing same logic as processDatasetEntries)
+    const { rows } = await parseMetadataFromDirectory(tempDir);
+
+    if (rows.length === 0) {
+      return {
+        success: false,
+        error: 'No valid entries found in metadata.csv',
+        entriesUpdated: 0,
+      };
+    }
+
+    // Get all existing entries for this dataset
+    const existingEntries = await db
+      .select()
+      .from(dataset_entry)
+      .where(eq(dataset_entry.datasetId, datasetId));
+
+    const existingByExternalId = new Map(
+      existingEntries.map(entry => [entry.externalId, entry])
+    );
+
+    let entriesUpdated = 0;
+    let entriesNotFound = 0;
+
+    // Update each row that has a matching entry
+    for (const row of rows) {
+      const externalId = row.id;
+      const existingEntry = existingByExternalId.get(externalId);
+
+      if (!existingEntry) {
+        entriesNotFound++;
+        continue;
+      }
+
+      // Update only the metadata fields, don't change the ID or dataset reference
+      await db
+        .update(dataset_entry)
+        .set({
+          speakerId: row.speaker,
+          modelName: row.model,
+          utteranceId: row.utt_id,
+          utteranceText: row.text,
+          dialect: row.dialect,
+          iteration: parseInt(row.iteration, 10),
+          durationMs: row.duration_ms ? parseInt(row.duration_ms, 10) : undefined,
+          rmsValue: row.rms_value ? parseFloat(row.rms_value) : undefined,
+          longestPause: row.longest_pause ? parseFloat(row.longest_pause) : undefined,
+          utmosScore: row.utmos_score ? parseFloat(row.utmos_score) : undefined,
+          werScore: row.wer_score ? parseFloat(row.wer_score) : undefined,
+        })
+        .where(eq(dataset_entry.id, existingEntry.id));
+
+      entriesUpdated++;
+    }
+
+    revalidatePath(`/admin/datasets/${datasetId}`);
+
+    return {
+      success: true,
+      entriesUpdated,
+      entriesNotFound,
+      message: entriesUpdated > 0
+        ? `Successfully updated ${entriesUpdated} entries${entriesNotFound > 0 ? ` (${entriesNotFound} not found)` : ''}.`
+        : `No entries were updated (${entriesNotFound} not found).`,
+    };
+  } catch (error) {
+    console.error('Error updating dataset entries metadata:', error);
+    const errorMessage = error instanceof Error ? error.message : 'Failed to update dataset entries metadata';
+    return {
+      success: false,
+      error: errorMessage,
+      entriesUpdated: 0,
+    };
+  } finally {
+    // Clean up temp directory
+    try {
+      if (existsSync(tempDir)) {
+        await rm(tempDir, { recursive: true, force: true });
+      }
+    } catch (cleanupError) {
+      console.error('Error cleaning up temp directory:', cleanupError);
+    }
+  }
+}

src/app/admin/datasets/[id]/page.tsx

@@ -5,6 +5,7 @@ import { eq } from 'drizzle-orm';
 import EditableDatasetHeader from '@/components/EditableDatasetHeader';
 import DatasetEntriesList from '@/components/DatasetEntriesList';
 import UploadDatasetEntriesModal from '@/components/UploadDatasetEntriesModal';
+import UpdateDatasetMetadataModal from '@/components/UpdateDatasetMetadataModal';
 import RemoveAllEntriesButton from '@/components/RemoveAllEntriesButton';
 import DeleteDatasetButton from '@/components/DeleteDatasetButton';
 import { requireAdmin } from '@/lib/auth';
@@ -69,6 +70,7 @@ export default async function DatasetPage({ params }: DatasetPageProps) {
 
         <div className="flex gap-3 mb-6">
           <UploadDatasetEntriesModal datasetId={datasetId} />
+          <UpdateDatasetMetadataModal datasetId={datasetId} />
           <RemoveAllEntriesButton datasetId={datasetId} />
           <DeleteDatasetButton datasetId={datasetId} datasetName={ds.name} />
         </div>

src/app/api/update-metadata/route.ts

@@ -0,0 +1,74 @@
+import { writeFile, mkdir, rm } from 'fs/promises';
+import { join } from 'path';
+import { existsSync } from 'fs';
+import { NextRequest, NextResponse } from 'next/server';
+import { requireAdmin } from '@/lib/auth';
+
+export async function POST(request: NextRequest) {
+  try {
+    const result = await requireAdmin();
+    if (!result.authenticated || !result.admin) {
+      return NextResponse.json(
+        { error: 'Unauthorized' },
+        { status: 401 }
+      );
+    }
+
+    const formData = await request.formData();
+    const file = formData.get('file') as File;
+    const datasetId = formData.get('datasetId') as string;
+
+    if (!file) {
+      return NextResponse.json(
+        { error: 'No file provided' },
+        { status: 400 }
+      );
+    }
+
+    if (!file.name.endsWith('.csv')) {
+      return NextResponse.json(
+        { error: 'File must be a CSV file' },
+        { status: 400 }
+      );
+    }
+
+    const tempDir = join(process.cwd(), 'tmp', `update-metadata-${Date.now()}`);
+
+    try {
+      // Create temp directory
+      await mkdir(tempDir, { recursive: true });
+
+      // Save uploaded CSV file as metadata.csv
+      const arrayBuffer = await file.arrayBuffer();
+      const buffer = Buffer.from(arrayBuffer);
+      const metadataPath = join(tempDir, 'metadata.csv');
+      await writeFile(metadataPath, buffer);
+
+      return NextResponse.json({
+        success: true,
+        tempDir,
+        message: `Successfully uploaded metadata.csv. Ready for processing.`,
+      });
+    } catch (error) {
+      // Clean up on error
+      try {
+        if (existsSync(tempDir)) {
+          await rm(tempDir, { recursive: true, force: true });
+        }
+      } catch (cleanupError) {
+        console.error('Error cleaning up temp directory:', cleanupError);
+      }
+
+      throw error;
+    }
+  } catch (error) {
+    console.error('Error uploading metadata CSV:', error);
+    const errorMsg =
+      error instanceof Error ? error.message : 'Failed to upload metadata CSV';
+    return NextResponse.json(
+      { error: errorMsg },
+      { status: 500 }
+    );
+  }
+}
+

src/components/UpdateDatasetMetadataModal.tsx

@@ -0,0 +1,365 @@
+'use client';
+
+import { useState } from 'react';
+import { updateDatasetEntriesMetadata } from '@/app/actions/upload';
+import Modal from '@/components/Modal';
+
+interface UpdateDatasetMetadataModalProps {
+  datasetId: number;
+}
+
+export default function UpdateDatasetMetadataModal({
+  datasetId,
+}: UpdateDatasetMetadataModalProps) {
+  const [isOpen, setIsOpen] = useState(false);
+  const [files, setFiles] = useState<File[]>([]);
+  const [loading, setLoading] = useState(false);
+  const [error, setError] = useState<string | null>(null);
+  const [success, setSuccess] = useState(false);
+  const [uploadQueue, setUploadQueue] = useState<Array<{ file: File; status: 'pending' | 'uploading' | 'completed' | 'failed'; message?: string; progress: number }>>([]);
+  const [status, setStatus] = useState<string>('');
+
+  function handleFileChange(e: React.ChangeEvent<HTMLInputElement>) {
+    const selectedFiles = e.target.files ? Array.from(e.target.files) : [];
+    if (selectedFiles.length > 0) {
+      setFiles(selectedFiles);
+      setError(null);
+      // Initialize queue with pending status and 0 progress
+      setUploadQueue(selectedFiles.map(file => ({ file, status: 'pending', progress: 0 })));
+    }
+  }
+
+  async function handleSubmit(e: React.FormEvent) {
+    e.preventDefault();
+    setError(null);
+    setSuccess(false);
+
+    if (files.length === 0) {
+      setError('Please select at least one CSV file');
+      return;
+    }
+
+    setLoading(true);
+    
+    try {
+      // Process each file in the queue sequentially
+      const newQueue = [...uploadQueue];
+      
+      for (let i = 0; i < newQueue.length; i++) {
+        const item = newQueue[i];
+        item.status = 'uploading';
+        item.progress = 0;
+        setUploadQueue([...newQueue]);
+        setStatus(`Processing file ${i + 1} of ${newQueue.length}: ${item.file.name}`);
+
+        try {
+          // Upload with progress tracking
+          const uploadResult = await uploadFileWithProgress(item.file, i, newQueue);
+
+          // Update the metadata
+          const updateResult = await updateDatasetEntriesMetadata(datasetId, uploadResult.tempDir);
+
+          if (!updateResult.success) {
+            item.status = 'failed';
+            item.progress = 0;
+            item.message = `✗ ${updateResult.error || 'Update failed'}`;
+          } else {
+            item.status = 'completed';
+            item.progress = 100;
+            item.message = `✓ Updated (${updateResult.entriesUpdated} entries)`;
+          }
+        } catch (err) {
+          item.status = 'failed';
+          item.progress = 0;
+          item.message = `✗ ${err instanceof Error ? err.message : 'Network error'}`;
+        }
+
+        setUploadQueue([...newQueue]);
+      }
+
+      const failedCount = newQueue.filter(item => item.status === 'failed').length;
+      const completedCount = newQueue.filter(item => item.status === 'completed').length;
+
+      if (failedCount === 0) {
+        setSuccess(true);
+        setStatus(`All files processed! (${completedCount}/${newQueue.length} completed)`);
+      } else if (completedCount > 0) {
+        setStatus(`Completed with errors: ${completedCount} succeeded, ${failedCount} failed`);
+      } else {
+        setError(`All files failed to process`);
+      }
+
+      setFiles([]);
+
+      if (failedCount === 0) {
+        setTimeout(() => {
+          setSuccess(false);
+          setIsOpen(false);
+          setStatus('');
+          setUploadQueue([]);
+          window.location.reload();
+        }, 2500);
+      }
+    } catch (err) {
+      const errorMsg = err instanceof Error ? err.message : 'Failed to process updates';
+      setError(errorMsg);
+      setStatus('');
+    } finally {
+      setLoading(false);
+    }
+  }
+
+  function uploadFileWithProgress(
+    file: File,
+    index: number,
+    queue: Array<{ file: File; status: 'pending' | 'uploading' | 'completed' | 'failed'; progress: number; message?: string }>
+  ): Promise<{ tempDir: string }> {
+    const MAX_RETRIES = 3;
+
+    return (async () => {
+      let lastError: Error | null = null;
+
+      for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) {
+        try {
+          const result = await uploadAttempt(file, index, queue, attempt, MAX_RETRIES);
+          return result;
+        } catch (error) {
+          lastError = error instanceof Error ? error : new Error('Unknown error');
+
+          if (attempt < MAX_RETRIES) {
+            // Exponential backoff: 1s, 2s, 4s
+            const delay = Math.pow(2, attempt - 1) * 1000;
+            queue[index].message = `Retrying in ${delay / 1000}s... (attempt ${attempt}/${MAX_RETRIES})`;
+            setUploadQueue([...queue]);
+
+            await new Promise(resolve => setTimeout(resolve, delay));
+            queue[index].progress = 0; // Reset progress for next attempt
+            setUploadQueue([...queue]);
+          }
+        }
+      }
+
+      throw lastError || new Error('Upload failed after all retries');
+    })();
+  }
+
+  function uploadAttempt(
+    file: File,
+    index: number,
+    queue: Array<{ file: File; status: 'pending' | 'uploading' | 'completed' | 'failed'; progress: number; message?: string }>,
+    attempt: number,
+    maxRetries: number
+  ): Promise<{ tempDir: string }> {
+    return new Promise((resolve, reject) => {
+      const formData = new FormData();
+      formData.append('file', file);
+      formData.append('datasetId', datasetId.toString());
+
+      const xhr = new XMLHttpRequest();
+
+      // Track upload progress
+      xhr.upload.addEventListener('progress', (event) => {
+        if (event.lengthComputable) {
+          const progress = Math.round((event.loaded / event.total) * 100);
+          queue[index].progress = progress;
+          if (attempt > 1) {
+            queue[index].message = `Uploading (retry ${attempt}/${maxRetries})`;
+          }
+          setUploadQueue([...queue]);
+        }
+      });
+
+      xhr.addEventListener('load', () => {
+        if (xhr.status === 200) {
+          try {
+            const result = JSON.parse(xhr.responseText);
+            if (result.success) {
+              resolve(result);
+            } else {
+              reject(new Error(result.error || 'Upload failed'));
+            }
+          } catch (error) {
+            reject(new Error('Failed to parse response'));
+          }
+        } else {
+          try {
+            const error = JSON.parse(xhr.responseText);
+            reject(new Error(error.error || `Upload failed with status ${xhr.status}`));
+          } catch {
+            reject(new Error(`Upload failed with status ${xhr.status}`));
+          }
+        }
+      });
+
+      xhr.addEventListener('error', () => {
+        reject(new Error('Network error'));
+      });
+
+      xhr.addEventListener('abort', () => {
+        reject(new Error('Upload aborted'));
+      });
+
+      xhr.open('POST', '/api/update-metadata');
+      xhr.send(formData);
+    });
+  }
+
+  function handleClose() {
+    if (!loading) {
+      setIsOpen(false);
+      setFiles([]);
+      setError(null);
+      setSuccess(false);
+      setStatus('');
+      setUploadQueue([]);
+    }
+  }
+
+  return (
+    <>
+      <button
+        onClick={() => setIsOpen(true)}
+        className="mb-6 px-4 py-2 bg-blue-500 text-white rounded-md hover:bg-blue-600 transition-colors"
+      >
+        ⟳ Update Metadata
+      </button>
+
+      <Modal
+        isOpen={isOpen}
+        onClose={handleClose}
+        title="Update Dataset Metadata"
+        actions={[
+          {
+            label: 'Cancel',
+            onClick: handleClose,
+            variant: 'secondary',
+            disabled: loading,
+          },
+          {
+            label: loading ? `Processing ${uploadQueue.filter(f => f.status !== 'pending').length}/${uploadQueue.length}...` : 'Upload & Update',
+            onClick: () => {
+              const form = document.getElementById(
+                'updateMetadataForm'
+              ) as HTMLFormElement;
+              form?.dispatchEvent(new Event('submit', { bubbles: true }));
+            },
+            variant: 'primary',
+            disabled: loading || files.length === 0,
+          },
+        ]}
+      >
+        <form id="updateMetadataForm" onSubmit={handleSubmit} className="space-y-4">
+          <div>
+            <label htmlFor="file" className="block text-sm font-medium mb-2">
+              CSV File
+            </label>
+            <div className="border-2 border-dashed border-gray-300 rounded-lg p-6 text-center">
+              <input
+                id="file"
+                type="file"
+                accept=".csv"
+                multiple
+                onChange={handleFileChange}
+                disabled={loading}
+                className="hidden"
+              />
+              <label htmlFor="file" className="cursor-pointer block">
+                {files.length > 0 ? (
+                  <div>
+                    <p className="text-sm font-medium text-blue-600">
+                      {files.length} file{files.length !== 1 ? 's' : ''} selected
+                    </p>
+                    <div className="mt-2 text-xs text-gray-600 max-h-24 overflow-y-auto">
+                      {files.map((f, idx) => (
+                        <div key={idx} className="py-1">
+                          {f.name} ({(f.size / 1024).toFixed(2)} KB)
+                        </div>
+                      ))}
+                    </div>
+                  </div>
+                ) : (
+                  <div>
+                    <p className="text-sm font-medium text-gray-700">
+                      Click to select or drag and drop
+                    </p>
+                    <p className="text-xs text-gray-500 mt-1">CSV files only (multiple allowed)</p>
+                  </div>
+                )}
+              </label>
+            </div>
+          </div>
+
+          <div className="bg-blue-50 border border-blue-200 rounded-lg p-4 text-sm text-blue-800">
+            <p className="font-semibold mb-2">CSV file format:</p>
+            <ul className="list-disc list-inside space-y-1 text-xs">
+              <li>
+                Columns: <code className="bg-blue-100 px-1 rounded">id, speaker, model, utt_id, text, dialect, iteration, duration_ms, rms_value, longest_pause, utmos_score, wer_score</code>
+              </li>
+              <li>Entry IDs in the CSV must match existing dataset entries to update</li>
+            </ul>
+          </div>
+
+          {uploadQueue.length > 0 && (
+            <div className="bg-gray-50 border border-gray-200 rounded-lg p-4 space-y-3">
+              <p className="font-semibold text-sm text-gray-700">Upload Queue:</p>
+              <div className="space-y-2 max-h-48 overflow-y-auto">
+                {uploadQueue.map((item, idx) => (
+                  <div key={idx} className="space-y-1">
+                    <div className="flex items-center gap-2 text-xs">
+                      <span className={`font-medium ${
+                        item.status === 'completed' ? 'text-green-600' :
+                        item.status === 'failed' ? 'text-red-600' :
+                        item.status === 'uploading' ? 'text-blue-600' :
+                        'text-gray-600'
+                      }`}>
+                        {item.status === 'completed' ? '✓' :
+                         item.status === 'failed' ? '✗' :
+                         item.status === 'uploading' ? '⟳' :
+                         '○'}
+                      </span>
+                      <span className="flex-1 truncate">{item.file.name}</span>
+                      {item.progress > 0 && item.progress < 100 && (
+                        <span className="text-gray-500">{item.progress}%</span>
+                      )}
+                      {item.message && <span className="text-gray-500 text-xs">{item.message}</span>}
+                    </div>
+                    {item.status === 'uploading' && (
+                      <div className="w-full bg-gray-200 rounded-full h-2 overflow-hidden">
+                        <div
+                          className="bg-blue-500 h-full transition-all duration-300"
+                          style={{ width: `${item.progress}%` }}
+                        />
+                      </div>
+                    )}
+                    {item.status === 'completed' && (
+                      <div className="w-full bg-green-200 rounded-full h-2 overflow-hidden">
+                        <div className="bg-green-500 h-full w-full" />
+                      </div>
+                    )}
+                    {item.status === 'failed' && (
+                      <div className="w-full bg-red-200 rounded-full h-2 overflow-hidden">
+                        <div className="bg-red-500 h-full w-full" />
+                      </div>
+                    )}
+                  </div>
+                ))}
+              </div>
+            </div>
+          )}
+
+          {error && (
+            <div className="p-3 bg-red-100 border border-red-400 text-red-700 rounded text-sm">
+              {error}
+            </div>
+          )}
+
+          {status && (
+            <div className={`p-3 rounded text-sm ${success ? 'bg-green-100 border border-green-400 text-green-700' : 'bg-blue-100 border border-blue-400 text-blue-700'}`}>
+              {status}
+            </div>
+          )}
+        </form>
+      </Modal>
+    </>
+  );
+}