'use server'; import db from '@/lib/db'; import { dataset_entry } from '@/lib/model/dataset_entry'; import { revalidatePath } from 'next/cache'; import { eq } from 'drizzle-orm'; import { writeFile, mkdir, readFile, rm } from 'fs/promises'; import { join } from 'path'; import { existsSync } from 'fs'; import { requireAdmin } from '@/lib/auth'; /** * CSV parser that handles quoted fields */ function parseCSVLine(line: string): string[] { const values: string[] = []; let current = ''; let inQuotes = false; for (let i = 0; i < line.length; i++) { const char = line[i]; const nextChar = line[i + 1]; if (char === '"') { if (inQuotes && nextChar === '"') { // Escaped quote current += '"'; i++; } else { // Toggle quote state inQuotes = !inQuotes; } } else if (char === ',' && !inQuotes) { // Field separator values.push(current.trim()); current = ''; } else { current += char; } } // Add the last field values.push(current.trim()); return values; } /** * Step 2: Process the extracted ZIP data and insert into database * Copies audio files and creates database entries */ export async function processDatasetEntries( datasetId: number, tempDir: string ) { const result = await requireAdmin(); if (!result.authenticated || !result.admin) { throw new Error('Unauthorized'); } const datasetDir = join(process.cwd(), 'public', 'datasets', datasetId.toString()); try { // Validate temp directory exists if (!existsSync(tempDir)) { throw new Error('Temporary extraction directory not found'); } // Read metadata.csv again const metadataPath = join(tempDir, 'metadata.csv'); if (!existsSync(metadataPath)) { throw new Error('metadata.csv not found in extracted files'); } const metadataContent = await readFile(metadataPath, 'utf-8'); const lines = metadataContent.split('\n').filter(line => line.trim()); if (lines.length < 2) { throw new Error('metadata.csv is empty or invalid'); } // Parse CSV const headers = lines[0].split(',').map(h => h.trim()); const entries: typeof dataset_entry.$inferInsert[] = []; // Create dataset directory await mkdir(datasetDir, { recursive: true }); // Process each row for (let i = 1; i < lines.length; i++) { const values = parseCSVLine(lines[i]); if (values.length !== headers.length) { continue; // Skip malformed lines } const row: Record = {}; headers.forEach((header, index) => { row[header] = values[index]; }); const audioFile = row.audio_file as string; const relativePath = audioFile.replace(/\\/g, '/'); const audioPath = join(tempDir, relativePath); if (existsSync(audioPath)) { // Copy audio file to dataset directory with external ID as filename const audioBuffer = await readFile(audioPath); const fileExtension = audioFile.substring(audioFile.lastIndexOf('.')); const destPath = join(datasetDir, `${row.id}${fileExtension}`); await writeFile(destPath, audioBuffer); entries.push({ datasetId, externalId: row.id, speakerId: row.speaker, modelName: row.model, fileName: relativePath, dialect: row.dialect, iteration: parseInt(row.iteration, 10), }); } } if (entries.length === 0) { throw new Error('No valid audio files found for entries'); } // Check for existing entries to avoid duplicates const existingEntries = await db .select({ externalId: dataset_entry.externalId }) .from(dataset_entry) .where(eq(dataset_entry.datasetId, datasetId)); const existingExternalIds = new Set(existingEntries.map(e => e.externalId)); // Filter out duplicates const newEntries = entries.filter(entry => !existingExternalIds.has(entry.externalId)); if (newEntries.length === 0) { throw new Error('All entries already exist in the dataset'); } // Insert new entries into database await db.insert(dataset_entry).values(newEntries); revalidatePath(`/admin/datasets/${datasetId}`); return { success: true, entriesCreated: newEntries.length, message: `Successfully processed and inserted ${newEntries.length} entries.`, }; } catch (error) { console.error('Error processing dataset entries:', error); throw new Error( error instanceof Error ? error.message : 'Failed to process dataset entries' ); } finally { // Clean up temp directory try { if (existsSync(tempDir)) { await rm(tempDir, { recursive: true, force: true }); } } catch (cleanupError) { console.error('Error cleaning up temp directory:', cleanupError); } } }