Merge branch 'main' into prototyp-2-fabio

src/app/actions/download-filtered-entries.ts

@@ -53,6 +53,10 @@ export async function downloadFilteredEntries(
     'id',    
     'audio_file',
     'duration_ms',
+    'rms_value',
+    'longest_pause',
+    'utmos_score',
+    'wer_score',
     'utt_id',
     'text',
     'speaker',
@@ -67,6 +71,10 @@ export async function downloadFilteredEntries(
       `"${entry.externalId}"`,
       entry.fileName,
       entry.durationMs?.toString() || '',
+      entry.rmsValue?.toString() || '',
+      entry.longestPause?.toString() || '',
+      entry.utmosScore?.toString() || '',
+      entry.werScore?.toString() || '',
       entry.utteranceId || '',
       `"${(entry.utteranceText || '').replace(/"/g, '""')}"`,
       entry.speakerId,

src/app/actions/get-dataset-entries.ts

@@ -2,9 +2,10 @@
 
 import db from '@/lib/db';
 import { dataset_entry } from '@/lib/model/dataset_entry';
-import { count, eq, and, like } from 'drizzle-orm';
+import { count, eq, and, like, asc, desc, getTableColumns } from 'drizzle-orm';
 
 const ENTRIES_PER_PAGE = 20;
+const SORT_COLUMNS = getTableColumns(dataset_entry);
 
 interface FilterParams {
   speakerId?: string;
@@ -14,10 +15,16 @@ interface FilterParams {
   utteranceId?: string;
 }
 
+interface SortParams {
+  sortBy?: keyof typeof SORT_COLUMNS;
+  sortOrder?: 'asc' | 'desc';
+}
+
 export async function getDatasetEntries(
   datasetId: number,
   page: number = 1,
-  filters?: FilterParams
+  filters?: FilterParams,
+  sort?: SortParams
 ) {
   const offset = (page - 1) * ENTRIES_PER_PAGE;
 
@@ -46,10 +53,17 @@ export async function getDatasetEntries(
 
   const whereClause = and(...conditions);
 
+  // Build sort clause
+  const sortFn = sort?.sortOrder === 'desc' ? desc : asc;
+  const sortColumn = sort?.sortBy && sort.sortBy in SORT_COLUMNS
+    ? sortFn(SORT_COLUMNS[sort.sortBy as keyof typeof SORT_COLUMNS])
+    : asc(dataset_entry.externalId);
+
   const entries = await db
     .select()
     .from(dataset_entry)
     .where(whereClause)
+    .orderBy(sortColumn)
     .limit(ENTRIES_PER_PAGE)
     .offset(offset);
 

src/app/actions/upload.ts

@@ -2,6 +2,7 @@
 
 import db from '@/lib/db';
 import { dataset_entry } from '@/lib/model/dataset_entry';
+import { parseCSVLine } from '@/lib/csv-parser';
 import { revalidatePath } from 'next/cache';
 import { eq } from 'drizzle-orm';
 import { writeFile, mkdir, readFile, rm } from 'fs/promises';
@@ -10,38 +11,71 @@ import { existsSync } from 'fs';
 import { requireAdmin } from '@/lib/auth';
 
 /**
- * CSV parser that handles quoted fields
+ * Helper: Parse metadata.csv from extracted directory
+ * Returns structured data for both insert and update operations
  */
-function parseCSVLine(line: string): string[] {
-  const values: string[] = [];
-  let current = '';
-  let inQuotes = false;
-
-  for (let i = 0; i < line.length; i++) {
-    const char = line[i];
-    const nextChar = line[i + 1];
-
-    if (char === '"') {
-      if (inQuotes && nextChar === '"') {
-        // Escaped quote
-        current += '"';
-        i++;
-      } else {
-        // Toggle quote state
-        inQuotes = !inQuotes;
-      }
-    } else if (char === ',' && !inQuotes) {
-      // Field separator
-      values.push(current.trim());
-      current = '';
-    } else {
-      current += char;
-    }
+async function parseMetadataFromDirectory(
+  tempDir: string
+): Promise<{
+  headers: string[];
+  rows: Record<string, string>[];
+}> {
+  const metadataPath = join(tempDir, 'metadata.csv');
+  if (!existsSync(metadataPath)) {
+    throw new Error('metadata.csv not found in extracted files');
   }
 
-  // Add the last field
-  values.push(current.trim());
-  return values;
+  const metadataContent = await readFile(metadataPath, 'utf-8');
+  const lines = metadataContent.split('\n').filter((line) => line.trim());
+
+  if (lines.length < 2) {
+    throw new Error('metadata.csv is empty or invalid');
+  }
+
+  const headers = lines[0].split(',').map((h) => h.trim());
+  const rows: Record<string, string>[] = [];
+
+  for (let i = 1; i < lines.length; i++) {
+    const values = parseCSVLine(lines[i]);
+
+    if (values.length !== headers.length) {
+      continue; // Skip malformed lines
+    }
+
+    const row: Record<string, string> = {};
+    headers.forEach((header, index) => {
+      row[header] = values[index];
+    });
+
+    rows.push(row);
+  }
+
+  return { headers, rows };
+}
+
+/**
+ * Convert metadata row to dataset_entry insert object
+ */
+function rowToDatasetEntry(
+  row: Record<string, string>,
+  datasetId: number
+): typeof dataset_entry.$inferInsert {
+  return {
+    datasetId,
+    externalId: row.id,
+    speakerId: row.speaker,
+    modelName: row.model,
+    utteranceId: row.utt_id,
+    utteranceText: row.text,
+    fileName: row.audio_file,
+    dialect: row.dialect,
+    iteration: parseInt(row.iteration, 10),
+    durationMs: row.duration_ms ? parseInt(row.duration_ms, 10) : undefined,
+    rmsValue: row.rms_value ? parseFloat(row.rms_value) : undefined,
+    longestPause: row.longest_pause ? parseFloat(row.longest_pause) : undefined,
+    utmosScore: row.utmos_score ? parseFloat(row.utmos_score) : undefined,
+    werScore: row.wer_score ? parseFloat(row.wer_score) : undefined,
+  };
 }
 
 /**
@@ -73,47 +107,16 @@ export async function processDatasetEntries(
       };
     }
 
-    // Read metadata.csv again
-    const metadataPath = join(tempDir, 'metadata.csv');
-    if (!existsSync(metadataPath)) {
-      return {
-        success: false,
-        error: 'metadata.csv not found in extracted files',
-        entriesCreated: 0,
-      };
-    }
+    // Parse metadata from directory
+    const { rows } = await parseMetadataFromDirectory(tempDir);
 
-    const metadataContent = await readFile(metadataPath, 'utf-8');
-    const lines = metadataContent.split('\n').filter(line => line.trim());
-
-    if (lines.length < 2) {
-      return {
-        success: false,
-        error: 'metadata.csv is empty or invalid',
-        entriesCreated: 0,
-      };
-    }
-
-    // Parse CSV
-    const headers = lines[0].split(',').map(h => h.trim());
     const entries: typeof dataset_entry.$inferInsert[] = [];
 
     // Create dataset directory
     await mkdir(datasetDir, { recursive: true });
 
     // Process each row
-    for (let i = 1; i < lines.length; i++) {
-      const values = parseCSVLine(lines[i]);
-
-      if (values.length !== headers.length) {
-        continue; // Skip malformed lines
-      }
-
-      const row: Record<string, string> = {};
-      headers.forEach((header, index) => {
-        row[header] = values[index];
-      });
-
+    for (const row of rows) {
       const audioFile = row.audio_file as string;
       const relativePath = audioFile.replace(/\\/g, '/');
       const audioPath = join(tempDir, relativePath);
@@ -126,18 +129,7 @@ export async function processDatasetEntries(
 
         await writeFile(destPath, audioBuffer);
 
-        entries.push({
-          datasetId,
-          externalId: row.id,
-          speakerId: row.speaker,
-          modelName: row.model,
-          utteranceId: row.utt_id,
-          utteranceText: row.text,
-          fileName: relativePath,
-          dialect: row.dialect,
-          iteration: parseInt(row.iteration, 10),
-          durationMs: row.duration_ms ? parseInt(row.duration_ms, 10) : undefined,
-        });
+        entries.push(rowToDatasetEntry(row, datasetId));
       }
     }
 
@@ -197,3 +189,116 @@ export async function processDatasetEntries(
     }
   }
 }
+
+/**
+ * Update metadata for existing dataset entries
+ * Uses the same metadata.csv parsing as processDatasetEntries
+ * Matches entries by externalId and updates their metadata fields
+ */
+export async function updateDatasetEntriesMetadata(
+  datasetId: number,
+  tempDir: string
+) {
+  try {
+    const result = await requireAdmin();
+    if (!result.authenticated || !result.admin) {
+      return {
+        success: false,
+        error: 'Unauthorized',
+        entriesUpdated: 0,
+      };
+    }
+
+    // Validate temp directory exists
+    if (!existsSync(tempDir)) {
+      return {
+        success: false,
+        error: 'Temporary extraction directory not found',
+        entriesUpdated: 0,
+      };
+    }
+
+    // Parse metadata from directory (reusing same logic as processDatasetEntries)
+    const { rows } = await parseMetadataFromDirectory(tempDir);
+
+    if (rows.length === 0) {
+      return {
+        success: false,
+        error: 'No valid entries found in metadata.csv',
+        entriesUpdated: 0,
+      };
+    }
+
+    // Get all existing entries for this dataset
+    const existingEntries = await db
+      .select()
+      .from(dataset_entry)
+      .where(eq(dataset_entry.datasetId, datasetId));
+
+    const existingByExternalId = new Map(
+      existingEntries.map(entry => [entry.externalId, entry])
+    );
+
+    let entriesUpdated = 0;
+    let entriesNotFound = 0;
+
+    // Update each row that has a matching entry
+    for (const row of rows) {
+      const externalId = row.id;
+      const existingEntry = existingByExternalId.get(externalId);
+
+      if (!existingEntry) {
+        entriesNotFound++;
+        continue;
+      }
+
+      // Update only the metadata fields, don't change the ID or dataset reference
+      await db
+        .update(dataset_entry)
+        .set({
+          speakerId: row.speaker,
+          modelName: row.model,
+          utteranceId: row.utt_id,
+          utteranceText: row.text,
+          dialect: row.dialect,
+          iteration: parseInt(row.iteration, 10),
+          durationMs: row.duration_ms ? parseInt(row.duration_ms, 10) : undefined,
+          rmsValue: row.rms_value ? parseFloat(row.rms_value) : undefined,
+          longestPause: row.longest_pause ? parseFloat(row.longest_pause) : undefined,
+          utmosScore: row.utmos_score ? parseFloat(row.utmos_score) : undefined,
+          werScore: row.wer_score ? parseFloat(row.wer_score) : undefined,
+        })
+        .where(eq(dataset_entry.id, existingEntry.id));
+
+      entriesUpdated++;
+    }
+
+    revalidatePath(`/admin/datasets/${datasetId}`);
+
+    return {
+      success: true,
+      entriesUpdated,
+      entriesNotFound,
+      message: entriesUpdated > 0
+        ? `Successfully updated ${entriesUpdated} entries${entriesNotFound > 0 ? ` (${entriesNotFound} not found)` : ''}.`
+        : `No entries were updated (${entriesNotFound} not found).`,
+    };
+  } catch (error) {
+    console.error('Error updating dataset entries metadata:', error);
+    const errorMessage = error instanceof Error ? error.message : 'Failed to update dataset entries metadata';
+    return {
+      success: false,
+      error: errorMessage,
+      entriesUpdated: 0,
+    };
+  } finally {
+    // Clean up temp directory
+    try {
+      if (existsSync(tempDir)) {
+        await rm(tempDir, { recursive: true, force: true });
+      }
+    } catch (cleanupError) {
+      console.error('Error cleaning up temp directory:', cleanupError);
+    }
+  }
+}