diff --git a/next.config.ts b/next.config.ts index 9ae1f64..d9e47a8 100644 --- a/next.config.ts +++ b/next.config.ts @@ -4,7 +4,6 @@ const nextConfig: NextConfig = { /* config options here */ experimental: { serverActions: { - bodySizeLimit: '10000mb' }, }, output: "standalone", diff --git a/src/app/actions/upload.ts b/src/app/actions/upload.ts index 5429f9e..04c1bd9 100644 --- a/src/app/actions/upload.ts +++ b/src/app/actions/upload.ts @@ -4,62 +4,71 @@ import db from '@/lib/db'; import { dataset_entry } from '@/lib/model/dataset_entry'; import { revalidatePath } from 'next/cache'; import { eq } from 'drizzle-orm'; -import { writeFile, mkdir, readFile, rm, readdir } from 'fs/promises'; +import { writeFile, mkdir, readFile, rm } from 'fs/promises'; import { join } from 'path'; import { existsSync } from 'fs'; -import extract from 'extract-zip'; -import { line } from 'drizzle-orm/pg-core'; import { requireAdmin } from '@/lib/auth'; -interface DatasetEntryInput { - id: string; - audio_file: string; - duration_ms: string; - speaker: string; - model: string; - dialect: string; - iteration: string; +/** + * CSV parser that handles quoted fields + */ +function parseCSVLine(line: string): string[] { + const values: string[] = []; + let current = ''; + let inQuotes = false; + + for (let i = 0; i < line.length; i++) { + const char = line[i]; + const nextChar = line[i + 1]; + + if (char === '"') { + if (inQuotes && nextChar === '"') { + // Escaped quote + current += '"'; + i++; + } else { + // Toggle quote state + inQuotes = !inQuotes; + } + } else if (char === ',' && !inQuotes) { + // Field separator + values.push(current.trim()); + current = ''; + } else { + current += char; + } + } + + // Add the last field + values.push(current.trim()); + return values; } -export async function uploadDatasetEntries( +/** + * Step 2: Process the extracted ZIP data and insert into database + * Copies audio files and creates database entries + */ +export async function processDatasetEntries( datasetId: number, - formData: FormData + tempDir: string ) { const result = await requireAdmin(); if (!result.authenticated || !result.admin) { throw new Error('Unauthorized'); } - const file = formData.get('file') as File; - - if (!file) { - throw new Error('No file provided'); - } - - if (!file.name.endsWith('.zip')) { - throw new Error('File must be a ZIP archive'); - } - - const tempDir = join(process.cwd(), 'tmp', `upload-${Date.now()}`); const datasetDir = join(process.cwd(), 'public', 'datasets', datasetId.toString()); try { - // Create temp directory - await mkdir(tempDir, { recursive: true }); + // Validate temp directory exists + if (!existsSync(tempDir)) { + throw new Error('Temporary extraction directory not found'); + } - // Save uploaded file to temp location - const tempZipPath = join(tempDir, file.name); - const arrayBuffer = await file.arrayBuffer(); - const buffer = Buffer.from(arrayBuffer); - await writeFile(tempZipPath, buffer); - - // Extract ZIP file - await extract( tempZipPath, { dir: tempDir } ); - - // Read metadata.csv + // Read metadata.csv again const metadataPath = join(tempDir, 'metadata.csv'); if (!existsSync(metadataPath)) { - throw new Error('metadata.csv not found in ZIP'); + throw new Error('metadata.csv not found in extracted files'); } const metadataContent = await readFile(metadataPath, 'utf-8'); @@ -69,48 +78,14 @@ export async function uploadDatasetEntries( throw new Error('metadata.csv is empty or invalid'); } - // Parse CSV header + // Parse CSV const headers = lines[0].split(',').map(h => h.trim()); + const entries: typeof dataset_entry.$inferInsert[] = []; // Create dataset directory await mkdir(datasetDir, { recursive: true }); - // Parse and insert entries - const entries: typeof dataset_entry.$inferInsert[] = []; - - // CSV parser that handles quoted fields - const parseCSVLine = (line: string): string[] => { - const values: string[] = []; - let current = ''; - let inQuotes = false; - - for (let i = 0; i < line.length; i++) { - const char = line[i]; - const nextChar = line[i + 1]; - - if (char === '"') { - if (inQuotes && nextChar === '"') { - // Escaped quote - current += '"'; - i++; - } else { - // Toggle quote state - inQuotes = !inQuotes; - } - } else if (char === ',' && !inQuotes) { - // Field separator - values.push(current.trim()); - current = ''; - } else { - current += char; - } - } - - // Add the last field - values.push(current.trim()); - return values; - }; - + // Process each row for (let i = 1; i < lines.length; i++) { const values = parseCSVLine(lines[i]); @@ -121,11 +96,10 @@ export async function uploadDatasetEntries( const row: Record = {}; headers.forEach((header, index) => { row[header] = values[index]; - }) + }); const audioFile = row.audio_file as string; - // Try different path separators and remove /output prefix - const relativePath = audioFile.replace(/\\/g, '/') + const relativePath = audioFile.replace(/\\/g, '/'); const audioPath = join(tempDir, relativePath); if (existsSync(audioPath)) { @@ -149,7 +123,7 @@ export async function uploadDatasetEntries( } if (entries.length === 0) { - throw new Error('No valid entries found in metadata.csv'); + throw new Error('No valid audio files found for entries'); } // Check for existing entries to avoid duplicates @@ -174,12 +148,13 @@ export async function uploadDatasetEntries( return { success: true, - entriesCreated: entries.length, + entriesCreated: newEntries.length, + message: `Successfully processed and inserted ${newEntries.length} entries.`, }; } catch (error) { - console.error('Error uploading dataset entries:', error); + console.error('Error processing dataset entries:', error); throw new Error( - error instanceof Error ? error.message : 'Failed to upload dataset entries' + error instanceof Error ? error.message : 'Failed to process dataset entries' ); } finally { // Clean up temp directory diff --git a/src/app/api/upload/route.ts b/src/app/api/upload/route.ts new file mode 100644 index 0000000..2452349 --- /dev/null +++ b/src/app/api/upload/route.ts @@ -0,0 +1,135 @@ +import { writeFile, mkdir, readFile, rm } from 'fs/promises'; +import { join } from 'path'; +import { existsSync } from 'fs'; +import extract from 'extract-zip'; +import { NextRequest, NextResponse } from 'next/server'; +import { requireAdmin } from '@/lib/auth'; + +function parseCSVLine(line: string): string[] { + const values: string[] = []; + let current = ''; + let inQuotes = false; + + for (let i = 0; i < line.length; i++) { + const char = line[i]; + const nextChar = line[i + 1]; + + if (char === '"') { + if (inQuotes && nextChar === '"') { + current += '"'; + i++; + } else { + inQuotes = !inQuotes; + } + } else if (char === ',' && !inQuotes) { + values.push(current.trim()); + current = ''; + } else { + current += char; + } + } + + values.push(current.trim()); + return values; +} + +export async function POST(request: NextRequest) { + try { + const result = await requireAdmin(); + if (!result.authenticated || !result.admin) { + return NextResponse.json( + { error: 'Unauthorized' }, + { status: 401 } + ); + } + + const formData = await request.formData(); + const file = formData.get('file') as File; + const datasetId = formData.get('datasetId') as string; + + if (!file) { + return NextResponse.json( + { error: 'No file provided' }, + { status: 400 } + ); + } + + if (!file.name.endsWith('.zip')) { + return NextResponse.json( + { error: 'File must be a ZIP archive' }, + { status: 400 } + ); + } + + const tempDir = join(process.cwd(), 'tmp', `upload-${Date.now()}`); + + try { + // Create temp directory + await mkdir(tempDir, { recursive: true }); + + // Save uploaded file to temp location + const tempZipPath = join(tempDir, file.name); + const arrayBuffer = await file.arrayBuffer(); + const buffer = Buffer.from(arrayBuffer); + await writeFile(tempZipPath, buffer); + + // Extract ZIP file + await extract(tempZipPath, { dir: tempDir }); + + // Read and validate metadata.csv + const metadataPath = join(tempDir, 'metadata.csv'); + if (!existsSync(metadataPath)) { + throw new Error('metadata.csv not found in ZIP'); + } + + const metadataContent = await readFile(metadataPath, 'utf-8'); + const lines = metadataContent.split('\n').filter(line => line.trim()); + + if (lines.length < 2) { + throw new Error('metadata.csv is empty or invalid'); + } + + // Parse CSV header + const headers = lines[0].split(',').map(h => h.trim()); + + // Parse and count rows + let validRowCount = 0; + for (let i = 1; i < lines.length; i++) { + const values = parseCSVLine(lines[i]); + if (values.length === headers.length) { + validRowCount++; + } + } + + if (validRowCount === 0) { + throw new Error('No valid entries found in metadata.csv'); + } + + return NextResponse.json({ + success: true, + tempDir, + metadataCount: validRowCount, + message: `Successfully extracted ${validRowCount} entries. Ready for processing.`, + }); + } catch (error) { + // Clean up on error + try { + if (existsSync(tempDir)) { + await rm(tempDir, { recursive: true, force: true }); + } + } catch (cleanupError) { + console.error('Error cleaning up temp directory:', cleanupError); + } + + throw error; + } + } catch (error) { + console.error('Error uploading dataset ZIP:', error); + const errorMsg = + error instanceof Error ? error.message : 'Failed to upload dataset ZIP'; + return NextResponse.json( + { error: errorMsg }, + { status: 500 } + ); + } +} diff --git a/src/components/UploadDatasetEntriesModal.tsx b/src/components/UploadDatasetEntriesModal.tsx index 4f5c324..9551690 100644 --- a/src/components/UploadDatasetEntriesModal.tsx +++ b/src/components/UploadDatasetEntriesModal.tsx @@ -1,7 +1,7 @@ 'use client'; import { useState } from 'react'; -import { uploadDatasetEntries } from '@/app/actions/upload'; +import { processDatasetEntries } from '@/app/actions/upload'; import Modal from '@/components/Modal'; interface UploadDatasetEntriesModalProps { @@ -19,6 +19,7 @@ export default function UploadDatasetEntriesModal({ const [entriesCreated, setEntriesCreated] = useState(0); const [status, setStatus] = useState(''); const [isDragging, setIsDragging] = useState(false); + const [tempDir, setTempDir] = useState(null); function handleFileChange(e: React.ChangeEvent) { const selectedFile = e.target.files?.[0]; @@ -39,18 +40,42 @@ export default function UploadDatasetEntriesModal({ } setLoading(true); - setStatus('Preparing upload...'); try { + // Step 1: Upload and extract ZIP via API route + setStatus('Uploading and extracting ZIP file...'); const formData = new FormData(); formData.append('file', file); + formData.append('datasetId', datasetId.toString()); - const result = await uploadDatasetEntries(datasetId, formData); + const uploadResponse = await fetch('/api/upload', { + method: 'POST', + body: formData, + }); + + if (!uploadResponse.ok) { + const errorData = await uploadResponse.json(); + throw new Error(errorData.error || 'Failed to upload ZIP file'); + } + + const uploadResult = await uploadResponse.json(); + + if (!uploadResult.success) { + throw new Error('Failed to upload ZIP file'); + } + + setTempDir(uploadResult.tempDir); + setStatus(`${uploadResult.message} Now processing...`); + + // Step 2: Process the extracted data (server action) + setStatus('Processing entries and copying audio files...'); + const processResult = await processDatasetEntries(datasetId, uploadResult.tempDir); setSuccess(true); - setEntriesCreated(result.entriesCreated); - setStatus(`Successfully created ${result.entriesCreated} entries!`); + setEntriesCreated(processResult.entriesCreated); + setStatus(processResult.message); setFile(null); + setTempDir(null); setTimeout(() => { setSuccess(false); @@ -60,7 +85,7 @@ export default function UploadDatasetEntriesModal({ }, 2500); } catch (err) { const errorMsg = - err instanceof Error ? err.message : 'Failed to upload dataset entries'; + err instanceof Error ? err.message : 'Failed to upload and process dataset entries'; setError(errorMsg); setStatus(''); } finally { @@ -75,6 +100,7 @@ export default function UploadDatasetEntriesModal({ setError(null); setSuccess(false); setStatus(''); + setTempDir(null); } } @@ -99,7 +125,7 @@ export default function UploadDatasetEntriesModal({ disabled: loading, }, { - label: loading ? 'Uploading...' : 'Upload', + label: loading ? 'Uploading & Processing...' : 'Upload', onClick: () => { const form = document.getElementById( 'uploadForm' @@ -165,8 +191,8 @@ export default function UploadDatasetEntriesModal({ )} - {success && ( -
+ {status && ( +
{status}
)} diff --git a/src/lib/model/upload_status.ts b/src/lib/model/upload_status.ts new file mode 100644 index 0000000..e69de29