From e9df095f067a5c7ac94189f5cbe04054d320b5d2 Mon Sep 17 00:00:00 2001 From: averel10 Date: Fri, 13 Mar 2026 14:10:12 +0100 Subject: [PATCH] feat: add functionality to upload and update dataset metadata via CSV files --- src/app/actions/upload.ts | 235 ++++++++--- src/app/admin/datasets/[id]/page.tsx | 2 + src/app/api/update-metadata/route.ts | 74 ++++ src/components/UpdateDatasetMetadataModal.tsx | 365 ++++++++++++++++++ 4 files changed, 626 insertions(+), 50 deletions(-) create mode 100644 src/app/api/update-metadata/route.ts create mode 100644 src/components/UpdateDatasetMetadataModal.tsx diff --git a/src/app/actions/upload.ts b/src/app/actions/upload.ts index 1201bad..47d5003 100644 --- a/src/app/actions/upload.ts +++ b/src/app/actions/upload.ts @@ -10,6 +10,74 @@ import { join } from 'path'; import { existsSync } from 'fs'; import { requireAdmin } from '@/lib/auth'; +/** + * Helper: Parse metadata.csv from extracted directory + * Returns structured data for both insert and update operations + */ +async function parseMetadataFromDirectory( + tempDir: string +): Promise<{ + headers: string[]; + rows: Record[]; +}> { + const metadataPath = join(tempDir, 'metadata.csv'); + if (!existsSync(metadataPath)) { + throw new Error('metadata.csv not found in extracted files'); + } + + const metadataContent = await readFile(metadataPath, 'utf-8'); + const lines = metadataContent.split('\n').filter((line) => line.trim()); + + if (lines.length < 2) { + throw new Error('metadata.csv is empty or invalid'); + } + + const headers = lines[0].split(',').map((h) => h.trim()); + const rows: Record[] = []; + + for (let i = 1; i < lines.length; i++) { + const values = parseCSVLine(lines[i]); + + if (values.length !== headers.length) { + continue; // Skip malformed lines + } + + const row: Record = {}; + headers.forEach((header, index) => { + row[header] = values[index]; + }); + + rows.push(row); + } + + return { headers, rows }; +} + +/** + * Convert metadata row to dataset_entry insert object + */ +function rowToDatasetEntry( + row: Record, + datasetId: number +): typeof dataset_entry.$inferInsert { + return { + datasetId, + externalId: row.id, + speakerId: row.speaker, + modelName: row.model, + utteranceId: row.utt_id, + utteranceText: row.text, + fileName: row.audio_file, + dialect: row.dialect, + iteration: parseInt(row.iteration, 10), + durationMs: row.duration_ms ? parseInt(row.duration_ms, 10) : undefined, + rmsValue: row.rms_value ? parseFloat(row.rms_value) : undefined, + longestPause: row.longest_pause ? parseFloat(row.longest_pause) : undefined, + utmosScore: row.utmos_score ? parseFloat(row.utmos_score) : undefined, + werScore: row.wer_score ? parseFloat(row.wer_score) : undefined, + }; +} + /** * Step 2: Process the extracted ZIP data and insert into database * Copies audio files and creates database entries @@ -39,47 +107,16 @@ export async function processDatasetEntries( }; } - // Read metadata.csv again - const metadataPath = join(tempDir, 'metadata.csv'); - if (!existsSync(metadataPath)) { - return { - success: false, - error: 'metadata.csv not found in extracted files', - entriesCreated: 0, - }; - } + // Parse metadata from directory + const { rows } = await parseMetadataFromDirectory(tempDir); - const metadataContent = await readFile(metadataPath, 'utf-8'); - const lines = metadataContent.split('\n').filter(line => line.trim()); - - if (lines.length < 2) { - return { - success: false, - error: 'metadata.csv is empty or invalid', - entriesCreated: 0, - }; - } - - // Parse CSV - const headers = lines[0].split(',').map(h => h.trim()); const entries: typeof dataset_entry.$inferInsert[] = []; // Create dataset directory await mkdir(datasetDir, { recursive: true }); // Process each row - for (let i = 1; i < lines.length; i++) { - const values = parseCSVLine(lines[i]); - - if (values.length !== headers.length) { - continue; // Skip malformed lines - } - - const row: Record = {}; - headers.forEach((header, index) => { - row[header] = values[index]; - }); - + for (const row of rows) { const audioFile = row.audio_file as string; const relativePath = audioFile.replace(/\\/g, '/'); const audioPath = join(tempDir, relativePath); @@ -92,22 +129,7 @@ export async function processDatasetEntries( await writeFile(destPath, audioBuffer); - entries.push({ - datasetId, - externalId: row.id, - speakerId: row.speaker, - modelName: row.model, - utteranceId: row.utt_id, - utteranceText: row.text, - fileName: relativePath, - dialect: row.dialect, - iteration: parseInt(row.iteration, 10), - durationMs: row.duration_ms ? parseInt(row.duration_ms, 10) : undefined, - rmsValue: row.rms_value ? parseFloat(row.rms_value) : undefined, - longestPause: row.longest_pause ? parseFloat(row.longest_pause) : undefined, - utmosScore: row.utmos_score ? parseFloat(row.utmos_score) : undefined, - werScore: row.wer_score ? parseFloat(row.wer_score) : undefined, - }); + entries.push(rowToDatasetEntry(row, datasetId)); } } @@ -167,3 +189,116 @@ export async function processDatasetEntries( } } } + +/** + * Update metadata for existing dataset entries + * Uses the same metadata.csv parsing as processDatasetEntries + * Matches entries by externalId and updates their metadata fields + */ +export async function updateDatasetEntriesMetadata( + datasetId: number, + tempDir: string +) { + try { + const result = await requireAdmin(); + if (!result.authenticated || !result.admin) { + return { + success: false, + error: 'Unauthorized', + entriesUpdated: 0, + }; + } + + // Validate temp directory exists + if (!existsSync(tempDir)) { + return { + success: false, + error: 'Temporary extraction directory not found', + entriesUpdated: 0, + }; + } + + // Parse metadata from directory (reusing same logic as processDatasetEntries) + const { rows } = await parseMetadataFromDirectory(tempDir); + + if (rows.length === 0) { + return { + success: false, + error: 'No valid entries found in metadata.csv', + entriesUpdated: 0, + }; + } + + // Get all existing entries for this dataset + const existingEntries = await db + .select() + .from(dataset_entry) + .where(eq(dataset_entry.datasetId, datasetId)); + + const existingByExternalId = new Map( + existingEntries.map(entry => [entry.externalId, entry]) + ); + + let entriesUpdated = 0; + let entriesNotFound = 0; + + // Update each row that has a matching entry + for (const row of rows) { + const externalId = row.id; + const existingEntry = existingByExternalId.get(externalId); + + if (!existingEntry) { + entriesNotFound++; + continue; + } + + // Update only the metadata fields, don't change the ID or dataset reference + await db + .update(dataset_entry) + .set({ + speakerId: row.speaker, + modelName: row.model, + utteranceId: row.utt_id, + utteranceText: row.text, + dialect: row.dialect, + iteration: parseInt(row.iteration, 10), + durationMs: row.duration_ms ? parseInt(row.duration_ms, 10) : undefined, + rmsValue: row.rms_value ? parseFloat(row.rms_value) : undefined, + longestPause: row.longest_pause ? parseFloat(row.longest_pause) : undefined, + utmosScore: row.utmos_score ? parseFloat(row.utmos_score) : undefined, + werScore: row.wer_score ? parseFloat(row.wer_score) : undefined, + }) + .where(eq(dataset_entry.id, existingEntry.id)); + + entriesUpdated++; + } + + revalidatePath(`/admin/datasets/${datasetId}`); + + return { + success: true, + entriesUpdated, + entriesNotFound, + message: entriesUpdated > 0 + ? `Successfully updated ${entriesUpdated} entries${entriesNotFound > 0 ? ` (${entriesNotFound} not found)` : ''}.` + : `No entries were updated (${entriesNotFound} not found).`, + }; + } catch (error) { + console.error('Error updating dataset entries metadata:', error); + const errorMessage = error instanceof Error ? error.message : 'Failed to update dataset entries metadata'; + return { + success: false, + error: errorMessage, + entriesUpdated: 0, + }; + } finally { + // Clean up temp directory + try { + if (existsSync(tempDir)) { + await rm(tempDir, { recursive: true, force: true }); + } + } catch (cleanupError) { + console.error('Error cleaning up temp directory:', cleanupError); + } + } +} diff --git a/src/app/admin/datasets/[id]/page.tsx b/src/app/admin/datasets/[id]/page.tsx index 21fd494..6c8b7af 100644 --- a/src/app/admin/datasets/[id]/page.tsx +++ b/src/app/admin/datasets/[id]/page.tsx @@ -5,6 +5,7 @@ import { eq } from 'drizzle-orm'; import EditableDatasetHeader from '@/components/EditableDatasetHeader'; import DatasetEntriesList from '@/components/DatasetEntriesList'; import UploadDatasetEntriesModal from '@/components/UploadDatasetEntriesModal'; +import UpdateDatasetMetadataModal from '@/components/UpdateDatasetMetadataModal'; import RemoveAllEntriesButton from '@/components/RemoveAllEntriesButton'; import DeleteDatasetButton from '@/components/DeleteDatasetButton'; import { requireAdmin } from '@/lib/auth'; @@ -69,6 +70,7 @@ export default async function DatasetPage({ params }: DatasetPageProps) {
+
diff --git a/src/app/api/update-metadata/route.ts b/src/app/api/update-metadata/route.ts new file mode 100644 index 0000000..b9f1b9c --- /dev/null +++ b/src/app/api/update-metadata/route.ts @@ -0,0 +1,74 @@ +import { writeFile, mkdir, rm } from 'fs/promises'; +import { join } from 'path'; +import { existsSync } from 'fs'; +import { NextRequest, NextResponse } from 'next/server'; +import { requireAdmin } from '@/lib/auth'; + +export async function POST(request: NextRequest) { + try { + const result = await requireAdmin(); + if (!result.authenticated || !result.admin) { + return NextResponse.json( + { error: 'Unauthorized' }, + { status: 401 } + ); + } + + const formData = await request.formData(); + const file = formData.get('file') as File; + const datasetId = formData.get('datasetId') as string; + + if (!file) { + return NextResponse.json( + { error: 'No file provided' }, + { status: 400 } + ); + } + + if (!file.name.endsWith('.csv')) { + return NextResponse.json( + { error: 'File must be a CSV file' }, + { status: 400 } + ); + } + + const tempDir = join(process.cwd(), 'tmp', `update-metadata-${Date.now()}`); + + try { + // Create temp directory + await mkdir(tempDir, { recursive: true }); + + // Save uploaded CSV file as metadata.csv + const arrayBuffer = await file.arrayBuffer(); + const buffer = Buffer.from(arrayBuffer); + const metadataPath = join(tempDir, 'metadata.csv'); + await writeFile(metadataPath, buffer); + + return NextResponse.json({ + success: true, + tempDir, + message: `Successfully uploaded metadata.csv. Ready for processing.`, + }); + } catch (error) { + // Clean up on error + try { + if (existsSync(tempDir)) { + await rm(tempDir, { recursive: true, force: true }); + } + } catch (cleanupError) { + console.error('Error cleaning up temp directory:', cleanupError); + } + + throw error; + } + } catch (error) { + console.error('Error uploading metadata CSV:', error); + const errorMsg = + error instanceof Error ? error.message : 'Failed to upload metadata CSV'; + return NextResponse.json( + { error: errorMsg }, + { status: 500 } + ); + } +} + diff --git a/src/components/UpdateDatasetMetadataModal.tsx b/src/components/UpdateDatasetMetadataModal.tsx new file mode 100644 index 0000000..6ed4e14 --- /dev/null +++ b/src/components/UpdateDatasetMetadataModal.tsx @@ -0,0 +1,365 @@ +'use client'; + +import { useState } from 'react'; +import { updateDatasetEntriesMetadata } from '@/app/actions/upload'; +import Modal from '@/components/Modal'; + +interface UpdateDatasetMetadataModalProps { + datasetId: number; +} + +export default function UpdateDatasetMetadataModal({ + datasetId, +}: UpdateDatasetMetadataModalProps) { + const [isOpen, setIsOpen] = useState(false); + const [files, setFiles] = useState([]); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); + const [success, setSuccess] = useState(false); + const [uploadQueue, setUploadQueue] = useState>([]); + const [status, setStatus] = useState(''); + + function handleFileChange(e: React.ChangeEvent) { + const selectedFiles = e.target.files ? Array.from(e.target.files) : []; + if (selectedFiles.length > 0) { + setFiles(selectedFiles); + setError(null); + // Initialize queue with pending status and 0 progress + setUploadQueue(selectedFiles.map(file => ({ file, status: 'pending', progress: 0 }))); + } + } + + async function handleSubmit(e: React.FormEvent) { + e.preventDefault(); + setError(null); + setSuccess(false); + + if (files.length === 0) { + setError('Please select at least one CSV file'); + return; + } + + setLoading(true); + + try { + // Process each file in the queue sequentially + const newQueue = [...uploadQueue]; + + for (let i = 0; i < newQueue.length; i++) { + const item = newQueue[i]; + item.status = 'uploading'; + item.progress = 0; + setUploadQueue([...newQueue]); + setStatus(`Processing file ${i + 1} of ${newQueue.length}: ${item.file.name}`); + + try { + // Upload with progress tracking + const uploadResult = await uploadFileWithProgress(item.file, i, newQueue); + + // Update the metadata + const updateResult = await updateDatasetEntriesMetadata(datasetId, uploadResult.tempDir); + + if (!updateResult.success) { + item.status = 'failed'; + item.progress = 0; + item.message = `✗ ${updateResult.error || 'Update failed'}`; + } else { + item.status = 'completed'; + item.progress = 100; + item.message = `✓ Updated (${updateResult.entriesUpdated} entries)`; + } + } catch (err) { + item.status = 'failed'; + item.progress = 0; + item.message = `✗ ${err instanceof Error ? err.message : 'Network error'}`; + } + + setUploadQueue([...newQueue]); + } + + const failedCount = newQueue.filter(item => item.status === 'failed').length; + const completedCount = newQueue.filter(item => item.status === 'completed').length; + + if (failedCount === 0) { + setSuccess(true); + setStatus(`All files processed! (${completedCount}/${newQueue.length} completed)`); + } else if (completedCount > 0) { + setStatus(`Completed with errors: ${completedCount} succeeded, ${failedCount} failed`); + } else { + setError(`All files failed to process`); + } + + setFiles([]); + + if (failedCount === 0) { + setTimeout(() => { + setSuccess(false); + setIsOpen(false); + setStatus(''); + setUploadQueue([]); + window.location.reload(); + }, 2500); + } + } catch (err) { + const errorMsg = err instanceof Error ? err.message : 'Failed to process updates'; + setError(errorMsg); + setStatus(''); + } finally { + setLoading(false); + } + } + + function uploadFileWithProgress( + file: File, + index: number, + queue: Array<{ file: File; status: 'pending' | 'uploading' | 'completed' | 'failed'; progress: number; message?: string }> + ): Promise<{ tempDir: string }> { + const MAX_RETRIES = 3; + + return (async () => { + let lastError: Error | null = null; + + for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) { + try { + const result = await uploadAttempt(file, index, queue, attempt, MAX_RETRIES); + return result; + } catch (error) { + lastError = error instanceof Error ? error : new Error('Unknown error'); + + if (attempt < MAX_RETRIES) { + // Exponential backoff: 1s, 2s, 4s + const delay = Math.pow(2, attempt - 1) * 1000; + queue[index].message = `Retrying in ${delay / 1000}s... (attempt ${attempt}/${MAX_RETRIES})`; + setUploadQueue([...queue]); + + await new Promise(resolve => setTimeout(resolve, delay)); + queue[index].progress = 0; // Reset progress for next attempt + setUploadQueue([...queue]); + } + } + } + + throw lastError || new Error('Upload failed after all retries'); + })(); + } + + function uploadAttempt( + file: File, + index: number, + queue: Array<{ file: File; status: 'pending' | 'uploading' | 'completed' | 'failed'; progress: number; message?: string }>, + attempt: number, + maxRetries: number + ): Promise<{ tempDir: string }> { + return new Promise((resolve, reject) => { + const formData = new FormData(); + formData.append('file', file); + formData.append('datasetId', datasetId.toString()); + + const xhr = new XMLHttpRequest(); + + // Track upload progress + xhr.upload.addEventListener('progress', (event) => { + if (event.lengthComputable) { + const progress = Math.round((event.loaded / event.total) * 100); + queue[index].progress = progress; + if (attempt > 1) { + queue[index].message = `Uploading (retry ${attempt}/${maxRetries})`; + } + setUploadQueue([...queue]); + } + }); + + xhr.addEventListener('load', () => { + if (xhr.status === 200) { + try { + const result = JSON.parse(xhr.responseText); + if (result.success) { + resolve(result); + } else { + reject(new Error(result.error || 'Upload failed')); + } + } catch (error) { + reject(new Error('Failed to parse response')); + } + } else { + try { + const error = JSON.parse(xhr.responseText); + reject(new Error(error.error || `Upload failed with status ${xhr.status}`)); + } catch { + reject(new Error(`Upload failed with status ${xhr.status}`)); + } + } + }); + + xhr.addEventListener('error', () => { + reject(new Error('Network error')); + }); + + xhr.addEventListener('abort', () => { + reject(new Error('Upload aborted')); + }); + + xhr.open('POST', '/api/update-metadata'); + xhr.send(formData); + }); + } + + function handleClose() { + if (!loading) { + setIsOpen(false); + setFiles([]); + setError(null); + setSuccess(false); + setStatus(''); + setUploadQueue([]); + } + } + + return ( + <> + + + f.status !== 'pending').length}/${uploadQueue.length}...` : 'Upload & Update', + onClick: () => { + const form = document.getElementById( + 'updateMetadataForm' + ) as HTMLFormElement; + form?.dispatchEvent(new Event('submit', { bubbles: true })); + }, + variant: 'primary', + disabled: loading || files.length === 0, + }, + ]} + > +
+
+ +
+ + +
+
+ +
+

CSV file format:

+
    +
  • + Columns: id, speaker, model, utt_id, text, dialect, iteration, duration_ms, rms_value, longest_pause, utmos_score, wer_score +
    • +
  • Entry IDs in the CSV must match existing dataset entries to update
    • +
+
+ + {uploadQueue.length > 0 && ( +
+

Upload Queue:

+
+ {uploadQueue.map((item, idx) => ( +
+
+ + {item.status === 'completed' ? '✓' : + item.status === 'failed' ? '✗' : + item.status === 'uploading' ? '⟳' : + '○'} + + {item.file.name} + {item.progress > 0 && item.progress < 100 && ( + {item.progress}% + )} + {item.message && {item.message}} +
+ {item.status === 'uploading' && ( +
+
+
+ )} + {item.status === 'completed' && ( +
+
+
+ )} + {item.status === 'failed' && ( +
+
+
+ )} +
+ ))} +
+
+ )} + + {error && ( +
+ {error} +
+ )} + + {status && ( +
+ {status} +
+ )} + + + + ); +}