Merge branch 'main' into prototyp-2-fabio

2026-03-13 14:58:02 +01:00
parent a823e968a2 3c3176b746
commit 97b8422bbd
18 changed files with 819 additions and 177 deletions
--- a/src/app/actions/download-filtered-entries.ts
+++ b/src/app/actions/download-filtered-entries.ts
@@ -53,6 +53,10 @@ export async function downloadFilteredEntries(
    'id',    
    'audio_file',
    'duration_ms',
+    'rms_value',
+    'longest_pause',
+    'utmos_score',
+    'wer_score',
    'utt_id',
    'text',
    'speaker',
@@ -67,6 +71,10 @@ export async function downloadFilteredEntries(
      `"${entry.externalId}"`,
      entry.fileName,
      entry.durationMs?.toString() || '',
+      entry.rmsValue?.toString() || '',
+      entry.longestPause?.toString() || '',
+      entry.utmosScore?.toString() || '',
+      entry.werScore?.toString() || '',
      entry.utteranceId || '',
      `"${(entry.utteranceText || '').replace(/"/g, '""')}"`,
      entry.speakerId,
--- a/src/app/actions/get-dataset-entries.ts
+++ b/src/app/actions/get-dataset-entries.ts
@@ -2,9 +2,10 @@

 import db from '@/lib/db';
 import { dataset_entry } from '@/lib/model/dataset_entry';
-import { count, eq, and, like } from 'drizzle-orm';
+import { count, eq, and, like, asc, desc, getTableColumns } from 'drizzle-orm';

 const ENTRIES_PER_PAGE = 20;
+const SORT_COLUMNS = getTableColumns(dataset_entry);

 interface FilterParams {
  speakerId?: string;
@@ -14,10 +15,16 @@ interface FilterParams {
  utteranceId?: string;
 }

+interface SortParams {
+  sortBy?: keyof typeof SORT_COLUMNS;
+  sortOrder?: 'asc' | 'desc';
+}
+
 export async function getDatasetEntries(
  datasetId: number,
  page: number = 1,
-  filters?: FilterParams
+  filters?: FilterParams,
+  sort?: SortParams
 ) {
  const offset = (page - 1) * ENTRIES_PER_PAGE;

@@ -46,10 +53,17 @@ export async function getDatasetEntries(

  const whereClause = and(...conditions);

+  // Build sort clause
+  const sortFn = sort?.sortOrder === 'desc' ? desc : asc;
+  const sortColumn = sort?.sortBy && sort.sortBy in SORT_COLUMNS
+    ? sortFn(SORT_COLUMNS[sort.sortBy as keyof typeof SORT_COLUMNS])
+    : asc(dataset_entry.externalId);
+
  const entries = await db
    .select()
    .from(dataset_entry)
    .where(whereClause)
+    .orderBy(sortColumn)
    .limit(ENTRIES_PER_PAGE)
    .offset(offset);

--- a/src/app/actions/upload.ts
+++ b/src/app/actions/upload.ts
@@ -2,6 +2,7 @@

 import db from '@/lib/db';
 import { dataset_entry } from '@/lib/model/dataset_entry';
+import { parseCSVLine } from '@/lib/csv-parser';
 import { revalidatePath } from 'next/cache';
 import { eq } from 'drizzle-orm';
 import { writeFile, mkdir, readFile, rm } from 'fs/promises';
@@ -10,38 +11,71 @@ import { existsSync } from 'fs';
 import { requireAdmin } from '@/lib/auth';

 /**
- * CSV parser that handles quoted fields
+ * Helper: Parse metadata.csv from extracted directory
+ * Returns structured data for both insert and update operations
 */
-function parseCSVLine(line: string): string[] {
-  const values: string[] = [];
-  let current = '';
-  let inQuotes = false;
-
-  for (let i = 0; i < line.length; i++) {
-    const char = line[i];
-    const nextChar = line[i + 1];
-
-    if (char === '"') {
-      if (inQuotes && nextChar === '"') {
-        // Escaped quote
-        current += '"';
-        i++;
-      } else {
-        // Toggle quote state
-        inQuotes = !inQuotes;
-      }
-    } else if (char === ',' && !inQuotes) {
-      // Field separator
-      values.push(current.trim());
-      current = '';
-    } else {
-      current += char;
-    }
+async function parseMetadataFromDirectory(
+  tempDir: string
+): Promise<{
+  headers: string[];
+  rows: Record<string, string>[];
+}> {
+  const metadataPath = join(tempDir, 'metadata.csv');
+  if (!existsSync(metadataPath)) {
+    throw new Error('metadata.csv not found in extracted files');
  }

-  // Add the last field
-  values.push(current.trim());
-  return values;
+  const metadataContent = await readFile(metadataPath, 'utf-8');
+  const lines = metadataContent.split('\n').filter((line) => line.trim());
+
+  if (lines.length < 2) {
+    throw new Error('metadata.csv is empty or invalid');
+  }
+
+  const headers = lines[0].split(',').map((h) => h.trim());
+  const rows: Record<string, string>[] = [];
+
+  for (let i = 1; i < lines.length; i++) {
+    const values = parseCSVLine(lines[i]);
+
+    if (values.length !== headers.length) {
+      continue; // Skip malformed lines
+    }
+
+    const row: Record<string, string> = {};
+    headers.forEach((header, index) => {
+      row[header] = values[index];
+    });
+
+    rows.push(row);
+  }
+
+  return { headers, rows };
+}
+
+/**
+ * Convert metadata row to dataset_entry insert object
+ */
+function rowToDatasetEntry(
+  row: Record<string, string>,
+  datasetId: number
+): typeof dataset_entry.$inferInsert {
+  return {
+    datasetId,
+    externalId: row.id,
+    speakerId: row.speaker,
+    modelName: row.model,
+    utteranceId: row.utt_id,
+    utteranceText: row.text,
+    fileName: row.audio_file,
+    dialect: row.dialect,
+    iteration: parseInt(row.iteration, 10),
+    durationMs: row.duration_ms ? parseInt(row.duration_ms, 10) : undefined,
+    rmsValue: row.rms_value ? parseFloat(row.rms_value) : undefined,
+    longestPause: row.longest_pause ? parseFloat(row.longest_pause) : undefined,
+    utmosScore: row.utmos_score ? parseFloat(row.utmos_score) : undefined,
+    werScore: row.wer_score ? parseFloat(row.wer_score) : undefined,
+  };
 }

 /**
@@ -73,47 +107,16 @@ export async function processDatasetEntries(
      };
    }

-    // Read metadata.csv again
-    const metadataPath = join(tempDir, 'metadata.csv');
-    if (!existsSync(metadataPath)) {
-      return {
-        success: false,
-        error: 'metadata.csv not found in extracted files',
-        entriesCreated: 0,
-      };
-    }
+    // Parse metadata from directory
+    const { rows } = await parseMetadataFromDirectory(tempDir);

-    const metadataContent = await readFile(metadataPath, 'utf-8');
-    const lines = metadataContent.split('\n').filter(line => line.trim());
-
-    if (lines.length < 2) {
-      return {
-        success: false,
-        error: 'metadata.csv is empty or invalid',
-        entriesCreated: 0,
-      };
-    }
-
-    // Parse CSV
-    const headers = lines[0].split(',').map(h => h.trim());
    const entries: typeof dataset_entry.$inferInsert[] = [];

    // Create dataset directory
    await mkdir(datasetDir, { recursive: true });

    // Process each row
-    for (let i = 1; i < lines.length; i++) {
-      const values = parseCSVLine(lines[i]);
-
-      if (values.length !== headers.length) {
-        continue; // Skip malformed lines
-      }
-
-      const row: Record<string, string> = {};
-      headers.forEach((header, index) => {
-        row[header] = values[index];
-      });
-
+    for (const row of rows) {
      const audioFile = row.audio_file as string;
      const relativePath = audioFile.replace(/\\/g, '/');
      const audioPath = join(tempDir, relativePath);
@@ -126,18 +129,7 @@ export async function processDatasetEntries(

        await writeFile(destPath, audioBuffer);

-        entries.push({
-          datasetId,
-          externalId: row.id,
-          speakerId: row.speaker,
-          modelName: row.model,
-          utteranceId: row.utt_id,
-          utteranceText: row.text,
-          fileName: relativePath,
-          dialect: row.dialect,
-          iteration: parseInt(row.iteration, 10),
-          durationMs: row.duration_ms ? parseInt(row.duration_ms, 10) : undefined,
-        });
+        entries.push(rowToDatasetEntry(row, datasetId));
      }
    }

@@ -197,3 +189,116 @@ export async function processDatasetEntries(
    }
  }
 }
+
+/**
+ * Update metadata for existing dataset entries
+ * Uses the same metadata.csv parsing as processDatasetEntries
+ * Matches entries by externalId and updates their metadata fields
+ */
+export async function updateDatasetEntriesMetadata(
+  datasetId: number,
+  tempDir: string
+) {
+  try {
+    const result = await requireAdmin();
+    if (!result.authenticated || !result.admin) {
+      return {
+        success: false,
+        error: 'Unauthorized',
+        entriesUpdated: 0,
+      };
+    }
+
+    // Validate temp directory exists
+    if (!existsSync(tempDir)) {
+      return {
+        success: false,
+        error: 'Temporary extraction directory not found',
+        entriesUpdated: 0,
+      };
+    }
+
+    // Parse metadata from directory (reusing same logic as processDatasetEntries)
+    const { rows } = await parseMetadataFromDirectory(tempDir);
+
+    if (rows.length === 0) {
+      return {
+        success: false,
+        error: 'No valid entries found in metadata.csv',
+        entriesUpdated: 0,
+      };
+    }
+
+    // Get all existing entries for this dataset
+    const existingEntries = await db
+      .select()
+      .from(dataset_entry)
+      .where(eq(dataset_entry.datasetId, datasetId));
+
+    const existingByExternalId = new Map(
+      existingEntries.map(entry => [entry.externalId, entry])
+    );
+
+    let entriesUpdated = 0;
+    let entriesNotFound = 0;
+
+    // Update each row that has a matching entry
+    for (const row of rows) {
+      const externalId = row.id;
+      const existingEntry = existingByExternalId.get(externalId);
+
+      if (!existingEntry) {
+        entriesNotFound++;
+        continue;
+      }
+
+      // Update only the metadata fields, don't change the ID or dataset reference
+      await db
+        .update(dataset_entry)
+        .set({
+          speakerId: row.speaker,
+          modelName: row.model,
+          utteranceId: row.utt_id,
+          utteranceText: row.text,
+          dialect: row.dialect,
+          iteration: parseInt(row.iteration, 10),
+          durationMs: row.duration_ms ? parseInt(row.duration_ms, 10) : undefined,
+          rmsValue: row.rms_value ? parseFloat(row.rms_value) : undefined,
+          longestPause: row.longest_pause ? parseFloat(row.longest_pause) : undefined,
+          utmosScore: row.utmos_score ? parseFloat(row.utmos_score) : undefined,
+          werScore: row.wer_score ? parseFloat(row.wer_score) : undefined,
+        })
+        .where(eq(dataset_entry.id, existingEntry.id));
+
+      entriesUpdated++;
+    }
+
+    revalidatePath(`/admin/datasets/${datasetId}`);
+
+    return {
+      success: true,
+      entriesUpdated,
+      entriesNotFound,
+      message: entriesUpdated > 0
+        ? `Successfully updated ${entriesUpdated} entries${entriesNotFound > 0 ? ` (${entriesNotFound} not found)` : ''}.`
+        : `No entries were updated (${entriesNotFound} not found).`,
+    };
+  } catch (error) {
+    console.error('Error updating dataset entries metadata:', error);
+    const errorMessage = error instanceof Error ? error.message : 'Failed to update dataset entries metadata';
+    return {
+      success: false,
+      error: errorMessage,
+      entriesUpdated: 0,
+    };
+  } finally {
+    // Clean up temp directory
+    try {
+      if (existsSync(tempDir)) {
+        await rm(tempDir, { recursive: true, force: true });
+      }
+    } catch (cleanupError) {
+      console.error('Error cleaning up temp directory:', cleanupError);
+    }
+  }
+}
--- a/src/app/admin/datasets/[id]/entries/[entryId]/page.tsx
+++ b/src/app/admin/datasets/[id]/entries/[entryId]/page.tsx
@@ -36,7 +36,7 @@ export default async function DatasetEntryPage({ params }: DatasetEntryPageProps
  if (entries.length === 0 || entries[0].datasetId !== datasetIdNum) {
    return (
      <div className="min-h-screen">
-        <div className="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8 py-8">
+        <div className="px-4 sm:px-6 lg:px-8 py-8">
          <Link
            href={`/admin/datasets/${datasetId}`}
            className="text-blue-500 hover:text-blue-600 mb-6 inline-block"
@@ -55,7 +55,7 @@ export default async function DatasetEntryPage({ params }: DatasetEntryPageProps

  return (
    <div>
-      <div className="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8 py-8">
+      <div className="px-4 sm:px-6 lg:px-8 py-8">
        <Link
          href={`/admin/datasets/${datasetId}`}
          className="text-blue-500 hover:text-blue-600 mb-6 inline-block"
@@ -122,6 +122,26 @@ export default async function DatasetEntryPage({ params }: DatasetEntryPageProps
              <p className="text-lg font-semibold">{entry.durationMs?.toLocaleString() || '-'}</p>
            </div>

+            <div className="bg-white border border-gray-200 p-4 rounded-lg">
+              <p className="text-sm text-gray-600 mb-1">RMS Value</p>
+              <p className="text-lg font-semibold">{entry.rmsValue?.toFixed(3) || '-'}</p>
+            </div>
+
+            <div className="bg-white border border-gray-200 p-4 rounded-lg">
+              <p className="text-sm text-gray-600 mb-1">Longest Pause (s)</p>
+              <p className="text-lg font-semibold">{entry.longestPause?.toFixed(3) || '-'}</p>
+            </div>
+
+            <div className="bg-white border border-gray-200 p-4 rounded-lg">
+              <p className="text-sm text-gray-600 mb-1">UTMOS Score</p>
+              <p className="text-lg font-semibold">{entry.utmosScore?.toFixed(3) || '-'}</p>
+            </div>
+
+            <div className="bg-white border border-gray-200 p-4 rounded-lg">
+              <p className="text-sm text-gray-600 mb-1">WER Score</p>
+              <p className="text-lg font-semibold">{entry.werScore?.toFixed(3) || '-'}</p>
+            </div>
+
            <div className="bg-white border border-gray-200 p-4 rounded-lg">
              <p className="text-sm text-gray-600 mb-1">Created</p>
              <p className="text-lg font-semibold">
--- a/src/app/admin/datasets/[id]/page.tsx
+++ b/src/app/admin/datasets/[id]/page.tsx
@@ -5,6 +5,7 @@ import { eq } from 'drizzle-orm';
 import EditableDatasetHeader from '@/components/EditableDatasetHeader';
 import DatasetEntriesList from '@/components/DatasetEntriesList';
 import UploadDatasetEntriesModal from '@/components/UploadDatasetEntriesModal';
+import UpdateDatasetMetadataModal from '@/components/UpdateDatasetMetadataModal';
 import RemoveAllEntriesButton from '@/components/RemoveAllEntriesButton';
 import DeleteDatasetButton from '@/components/DeleteDatasetButton';
 import { requireAdmin } from '@/lib/auth';
@@ -38,7 +39,7 @@ export default async function DatasetPage({ params }: DatasetPageProps) {
  if (datasets.length === 0) {
    return (
      <div className="min-h-screen">
-        <div className="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8 py-8">
+        <div className="px-4 sm:px-6 lg:px-8 py-8">
          <Link
            href="/admin"
            className="text-blue-500 hover:text-blue-600 mb-6 inline-block"
@@ -57,7 +58,7 @@ export default async function DatasetPage({ params }: DatasetPageProps) {

  return (
    <div>
-      <div className="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8 py-8">
+      <div className="px-4 sm:px-6 lg:px-8 py-8">
        <Link
          href="/admin"
          className="text-blue-500 hover:text-blue-600 mb-6 inline-block"
@@ -69,6 +70,7 @@ export default async function DatasetPage({ params }: DatasetPageProps) {

        <div className="flex gap-3 mb-6">
          <UploadDatasetEntriesModal datasetId={datasetId} />
+          <UpdateDatasetMetadataModal datasetId={datasetId} />
          <RemoveAllEntriesButton datasetId={datasetId} />
          <DeleteDatasetButton datasetId={datasetId} datasetName={ds.name} />
        </div>
--- a/src/app/admin/page.tsx
+++ b/src/app/admin/page.tsx
@@ -18,7 +18,7 @@ export default async function AdminPage() {
    }

  return (
-    <div className="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8 py-8">
+    <div className="px-4 sm:px-6 lg:px-8 py-8">
      <div className="mb-8">
        <h1 className="text-3xl font-bold mb-4">Admin Panel</h1>
        <div className="flex gap-4">
--- a/src/app/admin/users/page.tsx
+++ b/src/app/admin/users/page.tsx
@@ -18,7 +18,7 @@ export default async function UsersPage() {
  const users = await getAllUsers();

  return (
-    <div className="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8 py-8">
+    <div className="px-4 sm:px-6 lg:px-8 py-8">
      <div className="mb-6">
        <Link
          href="/admin"
--- a/src/app/api/update-metadata/route.ts
+++ b/src/app/api/update-metadata/route.ts
@@ -0,0 +1,74 @@
+import { writeFile, mkdir, rm } from 'fs/promises';
+import { join } from 'path';
+import { existsSync } from 'fs';
+import { NextRequest, NextResponse } from 'next/server';
+import { requireAdmin } from '@/lib/auth';
+
+export async function POST(request: NextRequest) {
+  try {
+    const result = await requireAdmin();
+    if (!result.authenticated || !result.admin) {
+      return NextResponse.json(
+        { error: 'Unauthorized' },
+        { status: 401 }
+      );
+    }
+
+    const formData = await request.formData();
+    const file = formData.get('file') as File;
+    const datasetId = formData.get('datasetId') as string;
+
+    if (!file) {
+      return NextResponse.json(
+        { error: 'No file provided' },
+        { status: 400 }
+      );
+    }
+
+    if (!file.name.endsWith('.csv')) {
+      return NextResponse.json(
+        { error: 'File must be a CSV file' },
+        { status: 400 }
+      );
+    }
+
+    const tempDir = join(process.cwd(), 'tmp', `update-metadata-${Date.now()}`);
+
+    try {
+      // Create temp directory
+      await mkdir(tempDir, { recursive: true });
+
+      // Save uploaded CSV file as metadata.csv
+      const arrayBuffer = await file.arrayBuffer();
+      const buffer = Buffer.from(arrayBuffer);
+      const metadataPath = join(tempDir, 'metadata.csv');
+      await writeFile(metadataPath, buffer);
+
+      return NextResponse.json({
+        success: true,
+        tempDir,
+        message: `Successfully uploaded metadata.csv. Ready for processing.`,
+      });
+    } catch (error) {
+      // Clean up on error
+      try {
+        if (existsSync(tempDir)) {
+          await rm(tempDir, { recursive: true, force: true });
+        }
+      } catch (cleanupError) {
+        console.error('Error cleaning up temp directory:', cleanupError);
+      }
+
+      throw error;
+    }
+  } catch (error) {
+    console.error('Error uploading metadata CSV:', error);
+    const errorMsg =
+      error instanceof Error ? error.message : 'Failed to upload metadata CSV';
+    return NextResponse.json(
+      { error: errorMsg },
+      { status: 500 }
+    );
+  }
+}
+
--- a/src/app/layout.tsx
+++ b/src/app/layout.tsx
@@ -13,7 +13,7 @@ export default function RootLayout({
      <body className="flex flex-col min-h-screen">
        <AudioProvider>
          <Header />
-          <main className="flex-grow mt-4 mb-4 mr-8 ml-8">
+          <main className="flex-grow">
            {children}
          </main>
          <Footer />