From 4c9b4734e7220e82a3ae8cc306d090da6f6f8470 Mon Sep 17 00:00:00 2001 From: averel10 Date: Fri, 13 Mar 2026 14:01:25 +0100 Subject: [PATCH 1/5] feat: add new fields to dataset_entry and update related components for processing and display --- drizzle/0006_special_junta.sql | 4 + drizzle/meta/0006_snapshot.json | 570 ++++++++++++++++++ drizzle/meta/_journal.json | 7 + src/app/actions/download-filtered-entries.ts | 8 + src/app/actions/upload.ts | 40 +- .../datasets/[id]/entries/[entryId]/page.tsx | 20 + src/components/DatasetEntriesList.tsx | 15 + src/lib/csv-parser.ts | 34 ++ src/lib/model/dataset_entry.ts | 4 + 9 files changed, 667 insertions(+), 35 deletions(-) create mode 100644 drizzle/0006_special_junta.sql create mode 100644 drizzle/meta/0006_snapshot.json create mode 100644 src/lib/csv-parser.ts diff --git a/drizzle/0006_special_junta.sql b/drizzle/0006_special_junta.sql new file mode 100644 index 0000000..b7624fa --- /dev/null +++ b/drizzle/0006_special_junta.sql @@ -0,0 +1,4 @@ +ALTER TABLE `dataset_entry` ADD `rms_value` real;--> statement-breakpoint +ALTER TABLE `dataset_entry` ADD `longest_pause` real;--> statement-breakpoint +ALTER TABLE `dataset_entry` ADD `utmos_score` real;--> statement-breakpoint +ALTER TABLE `dataset_entry` ADD `wer_score` real; \ No newline at end of file diff --git a/drizzle/meta/0006_snapshot.json b/drizzle/meta/0006_snapshot.json new file mode 100644 index 0000000..f9c5f13 --- /dev/null +++ b/drizzle/meta/0006_snapshot.json @@ -0,0 +1,570 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "4e72f679-ef9b-47d5-a6fb-a1c45d5cbe74", + "prevId": "e6165973-fcea-4b1b-8004-60db966b8106", + "tables": { + "account": { + "name": "account", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "account_userId_idx": { + "name": "account_userId_idx", + "columns": [ + "user_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "session": { + "name": "session", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "session_token_unique": { + "name": "session_token_unique", + "columns": [ + "token" + ], + "isUnique": true + }, + "session_userId_idx": { + "name": "session_userId_idx", + "columns": [ + "user_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "user": { + "name": "user", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email_verified": { + "name": "email_verified", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "admin": { + "name": "admin", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "user_email_unique": { + "name": "user_email_unique", + "columns": [ + "email" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "verification": { + "name": "verification", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "verification_identifier_idx": { + "name": "verification_identifier_idx", + "columns": [ + "identifier" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "dataset_entry": { + "name": "dataset_entry", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "external_id": { + "name": "external_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "speaker_id": { + "name": "speaker_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model_name": { + "name": "model_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "utterance_id": { + "name": "utterance_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "utterance_text": { + "name": "utterance_text", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "dataset_id": { + "name": "dataset_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "file_name": { + "name": "file_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "dialect": { + "name": "dialect", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "iteration": { + "name": "iteration", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "rms_value": { + "name": "rms_value", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "longest_pause": { + "name": "longest_pause", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "utmos_score": { + "name": "utmos_score", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "wer_score": { + "name": "wer_score", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + } + }, + "indexes": {}, + "foreignKeys": { + "dataset_entry_dataset_id_dataset_id_fk": { + "name": "dataset_entry_dataset_id_dataset_id_fk", + "tableFrom": "dataset_entry", + "tableTo": "dataset", + "columnsFrom": [ + "dataset_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "dataset": { + "name": "dataset", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json index 97bf443..0455de6 100644 --- a/drizzle/meta/_journal.json +++ b/drizzle/meta/_journal.json @@ -43,6 +43,13 @@ "when": 1773395728082, "tag": "0005_flashy_piledriver", "breakpoints": true + }, + { + "idx": 6, + "version": "6", + "when": 1773406006276, + "tag": "0006_special_junta", + "breakpoints": true } ] } \ No newline at end of file diff --git a/src/app/actions/download-filtered-entries.ts b/src/app/actions/download-filtered-entries.ts index 7556474..8a01847 100644 --- a/src/app/actions/download-filtered-entries.ts +++ b/src/app/actions/download-filtered-entries.ts @@ -53,6 +53,10 @@ export async function downloadFilteredEntries( 'id', 'audio_file', 'duration_ms', + 'rms_value', + 'longest_pause', + 'utmos_score', + 'wer_score', 'utt_id', 'text', 'speaker', @@ -67,6 +71,10 @@ export async function downloadFilteredEntries( `"${entry.externalId}"`, entry.fileName, entry.durationMs?.toString() || '', + entry.rmsValue?.toString() || '', + entry.longestPause?.toString() || '', + entry.utmosScore?.toString() || '', + entry.werScore?.toString() || '', entry.utteranceId || '', `"${(entry.utteranceText || '').replace(/"/g, '""')}"`, entry.speakerId, diff --git a/src/app/actions/upload.ts b/src/app/actions/upload.ts index d3d52c1..1201bad 100644 --- a/src/app/actions/upload.ts +++ b/src/app/actions/upload.ts @@ -2,6 +2,7 @@ import db from '@/lib/db'; import { dataset_entry } from '@/lib/model/dataset_entry'; +import { parseCSVLine } from '@/lib/csv-parser'; import { revalidatePath } from 'next/cache'; import { eq } from 'drizzle-orm'; import { writeFile, mkdir, readFile, rm } from 'fs/promises'; @@ -9,41 +10,6 @@ import { join } from 'path'; import { existsSync } from 'fs'; import { requireAdmin } from '@/lib/auth'; -/** - * CSV parser that handles quoted fields - */ -function parseCSVLine(line: string): string[] { - const values: string[] = []; - let current = ''; - let inQuotes = false; - - for (let i = 0; i < line.length; i++) { - const char = line[i]; - const nextChar = line[i + 1]; - - if (char === '"') { - if (inQuotes && nextChar === '"') { - // Escaped quote - current += '"'; - i++; - } else { - // Toggle quote state - inQuotes = !inQuotes; - } - } else if (char === ',' && !inQuotes) { - // Field separator - values.push(current.trim()); - current = ''; - } else { - current += char; - } - } - - // Add the last field - values.push(current.trim()); - return values; -} - /** * Step 2: Process the extracted ZIP data and insert into database * Copies audio files and creates database entries @@ -137,6 +103,10 @@ export async function processDatasetEntries( dialect: row.dialect, iteration: parseInt(row.iteration, 10), durationMs: row.duration_ms ? parseInt(row.duration_ms, 10) : undefined, + rmsValue: row.rms_value ? parseFloat(row.rms_value) : undefined, + longestPause: row.longest_pause ? parseFloat(row.longest_pause) : undefined, + utmosScore: row.utmos_score ? parseFloat(row.utmos_score) : undefined, + werScore: row.wer_score ? parseFloat(row.wer_score) : undefined, }); } } diff --git a/src/app/admin/datasets/[id]/entries/[entryId]/page.tsx b/src/app/admin/datasets/[id]/entries/[entryId]/page.tsx index b9c80d0..c3f8f0b 100644 --- a/src/app/admin/datasets/[id]/entries/[entryId]/page.tsx +++ b/src/app/admin/datasets/[id]/entries/[entryId]/page.tsx @@ -122,6 +122,26 @@ export default async function DatasetEntryPage({ params }: DatasetEntryPageProps

{entry.durationMs?.toLocaleString() || '-'}

+
+

RMS Value

+

{entry.rmsValue?.toFixed(3) || '-'}

+
+ +
+

Longest Pause (s)

+

{entry.longestPause?.toFixed(3) || '-'}

+
+ +
+

UTMOS Score

+

{entry.utmosScore?.toFixed(3) || '-'}

+
+ +
+

WER Score

+

{entry.werScore?.toFixed(3) || '-'}

+
+

Created

diff --git a/src/components/DatasetEntriesList.tsx b/src/components/DatasetEntriesList.tsx index 029d53f..041a339 100644 --- a/src/components/DatasetEntriesList.tsx +++ b/src/components/DatasetEntriesList.tsx @@ -22,6 +22,11 @@ interface DatasetEntry { fileName: string; dialect: string; iteration: number; + durationMs: number | null; + rmsValue: number | null; + longestPause: number | null; + utmosScore: number | null; + werScore: number | null; createdAt: Date; updatedAt: Date; } @@ -417,6 +422,11 @@ export default function DatasetEntriesList({ Dialect File Name Iteration + Duration (ms) + RMS Value + Longest Pause (s) + UTMOS Score + WER Score @@ -439,6 +449,11 @@ export default function DatasetEntriesList({ {entry.dialect} {entry.fileName} {entry.iteration} + {entry.durationMs?.toLocaleString() || '-'} + {entry.rmsValue?.toFixed(3) || '-'} + {entry.longestPause?.toFixed(3) || '-'} + {entry.utmosScore?.toFixed(3) || '-'} + {entry.werScore?.toFixed(3) || '-'} ))} diff --git a/src/lib/csv-parser.ts b/src/lib/csv-parser.ts new file mode 100644 index 0000000..1b90869 --- /dev/null +++ b/src/lib/csv-parser.ts @@ -0,0 +1,34 @@ +/** + * CSV parser that handles quoted fields + */ +export function parseCSVLine(line: string): string[] { + const values: string[] = []; + let current = ''; + let inQuotes = false; + + for (let i = 0; i < line.length; i++) { + const char = line[i]; + const nextChar = line[i + 1]; + + if (char === '"') { + if (inQuotes && nextChar === '"') { + // Escaped quote + current += '"'; + i++; + } else { + // Toggle quote state + inQuotes = !inQuotes; + } + } else if (char === ',' && !inQuotes) { + // Field separator + values.push(current.trim()); + current = ''; + } else { + current += char; + } + } + + // Add the last field + values.push(current.trim()); + return values; +} diff --git a/src/lib/model/dataset_entry.ts b/src/lib/model/dataset_entry.ts index 482aa56..900450c 100644 --- a/src/lib/model/dataset_entry.ts +++ b/src/lib/model/dataset_entry.ts @@ -16,6 +16,10 @@ export const dataset_entry = sqliteTable('dataset_entry', { dialect: text('dialect').notNull(), iteration: integer('iteration').notNull(), durationMs: integer('duration_ms'), + rmsValue: real('rms_value'), + longestPause: real('longest_pause'), + utmosScore: real('utmos_score'), + werScore: real('wer_score'), createdAt: integer('created_at', { mode: 'timestamp' }) .notNull() .default(sql`(unixepoch())`), From e9df095f067a5c7ac94189f5cbe04054d320b5d2 Mon Sep 17 00:00:00 2001 From: averel10 Date: Fri, 13 Mar 2026 14:10:12 +0100 Subject: [PATCH 2/5] feat: add functionality to upload and update dataset metadata via CSV files --- src/app/actions/upload.ts | 235 ++++++++--- src/app/admin/datasets/[id]/page.tsx | 2 + src/app/api/update-metadata/route.ts | 74 ++++ src/components/UpdateDatasetMetadataModal.tsx | 365 ++++++++++++++++++ 4 files changed, 626 insertions(+), 50 deletions(-) create mode 100644 src/app/api/update-metadata/route.ts create mode 100644 src/components/UpdateDatasetMetadataModal.tsx diff --git a/src/app/actions/upload.ts b/src/app/actions/upload.ts index 1201bad..47d5003 100644 --- a/src/app/actions/upload.ts +++ b/src/app/actions/upload.ts @@ -10,6 +10,74 @@ import { join } from 'path'; import { existsSync } from 'fs'; import { requireAdmin } from '@/lib/auth'; +/** + * Helper: Parse metadata.csv from extracted directory + * Returns structured data for both insert and update operations + */ +async function parseMetadataFromDirectory( + tempDir: string +): Promise<{ + headers: string[]; + rows: Record[]; +}> { + const metadataPath = join(tempDir, 'metadata.csv'); + if (!existsSync(metadataPath)) { + throw new Error('metadata.csv not found in extracted files'); + } + + const metadataContent = await readFile(metadataPath, 'utf-8'); + const lines = metadataContent.split('\n').filter((line) => line.trim()); + + if (lines.length < 2) { + throw new Error('metadata.csv is empty or invalid'); + } + + const headers = lines[0].split(',').map((h) => h.trim()); + const rows: Record[] = []; + + for (let i = 1; i < lines.length; i++) { + const values = parseCSVLine(lines[i]); + + if (values.length !== headers.length) { + continue; // Skip malformed lines + } + + const row: Record = {}; + headers.forEach((header, index) => { + row[header] = values[index]; + }); + + rows.push(row); + } + + return { headers, rows }; +} + +/** + * Convert metadata row to dataset_entry insert object + */ +function rowToDatasetEntry( + row: Record, + datasetId: number +): typeof dataset_entry.$inferInsert { + return { + datasetId, + externalId: row.id, + speakerId: row.speaker, + modelName: row.model, + utteranceId: row.utt_id, + utteranceText: row.text, + fileName: row.audio_file, + dialect: row.dialect, + iteration: parseInt(row.iteration, 10), + durationMs: row.duration_ms ? parseInt(row.duration_ms, 10) : undefined, + rmsValue: row.rms_value ? parseFloat(row.rms_value) : undefined, + longestPause: row.longest_pause ? parseFloat(row.longest_pause) : undefined, + utmosScore: row.utmos_score ? parseFloat(row.utmos_score) : undefined, + werScore: row.wer_score ? parseFloat(row.wer_score) : undefined, + }; +} + /** * Step 2: Process the extracted ZIP data and insert into database * Copies audio files and creates database entries @@ -39,47 +107,16 @@ export async function processDatasetEntries( }; } - // Read metadata.csv again - const metadataPath = join(tempDir, 'metadata.csv'); - if (!existsSync(metadataPath)) { - return { - success: false, - error: 'metadata.csv not found in extracted files', - entriesCreated: 0, - }; - } + // Parse metadata from directory + const { rows } = await parseMetadataFromDirectory(tempDir); - const metadataContent = await readFile(metadataPath, 'utf-8'); - const lines = metadataContent.split('\n').filter(line => line.trim()); - - if (lines.length < 2) { - return { - success: false, - error: 'metadata.csv is empty or invalid', - entriesCreated: 0, - }; - } - - // Parse CSV - const headers = lines[0].split(',').map(h => h.trim()); const entries: typeof dataset_entry.$inferInsert[] = []; // Create dataset directory await mkdir(datasetDir, { recursive: true }); // Process each row - for (let i = 1; i < lines.length; i++) { - const values = parseCSVLine(lines[i]); - - if (values.length !== headers.length) { - continue; // Skip malformed lines - } - - const row: Record = {}; - headers.forEach((header, index) => { - row[header] = values[index]; - }); - + for (const row of rows) { const audioFile = row.audio_file as string; const relativePath = audioFile.replace(/\\/g, '/'); const audioPath = join(tempDir, relativePath); @@ -92,22 +129,7 @@ export async function processDatasetEntries( await writeFile(destPath, audioBuffer); - entries.push({ - datasetId, - externalId: row.id, - speakerId: row.speaker, - modelName: row.model, - utteranceId: row.utt_id, - utteranceText: row.text, - fileName: relativePath, - dialect: row.dialect, - iteration: parseInt(row.iteration, 10), - durationMs: row.duration_ms ? parseInt(row.duration_ms, 10) : undefined, - rmsValue: row.rms_value ? parseFloat(row.rms_value) : undefined, - longestPause: row.longest_pause ? parseFloat(row.longest_pause) : undefined, - utmosScore: row.utmos_score ? parseFloat(row.utmos_score) : undefined, - werScore: row.wer_score ? parseFloat(row.wer_score) : undefined, - }); + entries.push(rowToDatasetEntry(row, datasetId)); } } @@ -167,3 +189,116 @@ export async function processDatasetEntries( } } } + +/** + * Update metadata for existing dataset entries + * Uses the same metadata.csv parsing as processDatasetEntries + * Matches entries by externalId and updates their metadata fields + */ +export async function updateDatasetEntriesMetadata( + datasetId: number, + tempDir: string +) { + try { + const result = await requireAdmin(); + if (!result.authenticated || !result.admin) { + return { + success: false, + error: 'Unauthorized', + entriesUpdated: 0, + }; + } + + // Validate temp directory exists + if (!existsSync(tempDir)) { + return { + success: false, + error: 'Temporary extraction directory not found', + entriesUpdated: 0, + }; + } + + // Parse metadata from directory (reusing same logic as processDatasetEntries) + const { rows } = await parseMetadataFromDirectory(tempDir); + + if (rows.length === 0) { + return { + success: false, + error: 'No valid entries found in metadata.csv', + entriesUpdated: 0, + }; + } + + // Get all existing entries for this dataset + const existingEntries = await db + .select() + .from(dataset_entry) + .where(eq(dataset_entry.datasetId, datasetId)); + + const existingByExternalId = new Map( + existingEntries.map(entry => [entry.externalId, entry]) + ); + + let entriesUpdated = 0; + let entriesNotFound = 0; + + // Update each row that has a matching entry + for (const row of rows) { + const externalId = row.id; + const existingEntry = existingByExternalId.get(externalId); + + if (!existingEntry) { + entriesNotFound++; + continue; + } + + // Update only the metadata fields, don't change the ID or dataset reference + await db + .update(dataset_entry) + .set({ + speakerId: row.speaker, + modelName: row.model, + utteranceId: row.utt_id, + utteranceText: row.text, + dialect: row.dialect, + iteration: parseInt(row.iteration, 10), + durationMs: row.duration_ms ? parseInt(row.duration_ms, 10) : undefined, + rmsValue: row.rms_value ? parseFloat(row.rms_value) : undefined, + longestPause: row.longest_pause ? parseFloat(row.longest_pause) : undefined, + utmosScore: row.utmos_score ? parseFloat(row.utmos_score) : undefined, + werScore: row.wer_score ? parseFloat(row.wer_score) : undefined, + }) + .where(eq(dataset_entry.id, existingEntry.id)); + + entriesUpdated++; + } + + revalidatePath(`/admin/datasets/${datasetId}`); + + return { + success: true, + entriesUpdated, + entriesNotFound, + message: entriesUpdated > 0 + ? `Successfully updated ${entriesUpdated} entries${entriesNotFound > 0 ? ` (${entriesNotFound} not found)` : ''}.` + : `No entries were updated (${entriesNotFound} not found).`, + }; + } catch (error) { + console.error('Error updating dataset entries metadata:', error); + const errorMessage = error instanceof Error ? error.message : 'Failed to update dataset entries metadata'; + return { + success: false, + error: errorMessage, + entriesUpdated: 0, + }; + } finally { + // Clean up temp directory + try { + if (existsSync(tempDir)) { + await rm(tempDir, { recursive: true, force: true }); + } + } catch (cleanupError) { + console.error('Error cleaning up temp directory:', cleanupError); + } + } +} diff --git a/src/app/admin/datasets/[id]/page.tsx b/src/app/admin/datasets/[id]/page.tsx index 21fd494..6c8b7af 100644 --- a/src/app/admin/datasets/[id]/page.tsx +++ b/src/app/admin/datasets/[id]/page.tsx @@ -5,6 +5,7 @@ import { eq } from 'drizzle-orm'; import EditableDatasetHeader from '@/components/EditableDatasetHeader'; import DatasetEntriesList from '@/components/DatasetEntriesList'; import UploadDatasetEntriesModal from '@/components/UploadDatasetEntriesModal'; +import UpdateDatasetMetadataModal from '@/components/UpdateDatasetMetadataModal'; import RemoveAllEntriesButton from '@/components/RemoveAllEntriesButton'; import DeleteDatasetButton from '@/components/DeleteDatasetButton'; import { requireAdmin } from '@/lib/auth'; @@ -69,6 +70,7 @@ export default async function DatasetPage({ params }: DatasetPageProps) {

+
diff --git a/src/app/api/update-metadata/route.ts b/src/app/api/update-metadata/route.ts new file mode 100644 index 0000000..b9f1b9c --- /dev/null +++ b/src/app/api/update-metadata/route.ts @@ -0,0 +1,74 @@ +import { writeFile, mkdir, rm } from 'fs/promises'; +import { join } from 'path'; +import { existsSync } from 'fs'; +import { NextRequest, NextResponse } from 'next/server'; +import { requireAdmin } from '@/lib/auth'; + +export async function POST(request: NextRequest) { + try { + const result = await requireAdmin(); + if (!result.authenticated || !result.admin) { + return NextResponse.json( + { error: 'Unauthorized' }, + { status: 401 } + ); + } + + const formData = await request.formData(); + const file = formData.get('file') as File; + const datasetId = formData.get('datasetId') as string; + + if (!file) { + return NextResponse.json( + { error: 'No file provided' }, + { status: 400 } + ); + } + + if (!file.name.endsWith('.csv')) { + return NextResponse.json( + { error: 'File must be a CSV file' }, + { status: 400 } + ); + } + + const tempDir = join(process.cwd(), 'tmp', `update-metadata-${Date.now()}`); + + try { + // Create temp directory + await mkdir(tempDir, { recursive: true }); + + // Save uploaded CSV file as metadata.csv + const arrayBuffer = await file.arrayBuffer(); + const buffer = Buffer.from(arrayBuffer); + const metadataPath = join(tempDir, 'metadata.csv'); + await writeFile(metadataPath, buffer); + + return NextResponse.json({ + success: true, + tempDir, + message: `Successfully uploaded metadata.csv. Ready for processing.`, + }); + } catch (error) { + // Clean up on error + try { + if (existsSync(tempDir)) { + await rm(tempDir, { recursive: true, force: true }); + } + } catch (cleanupError) { + console.error('Error cleaning up temp directory:', cleanupError); + } + + throw error; + } + } catch (error) { + console.error('Error uploading metadata CSV:', error); + const errorMsg = + error instanceof Error ? error.message : 'Failed to upload metadata CSV'; + return NextResponse.json( + { error: errorMsg }, + { status: 500 } + ); + } +} + diff --git a/src/components/UpdateDatasetMetadataModal.tsx b/src/components/UpdateDatasetMetadataModal.tsx new file mode 100644 index 0000000..6ed4e14 --- /dev/null +++ b/src/components/UpdateDatasetMetadataModal.tsx @@ -0,0 +1,365 @@ +'use client'; + +import { useState } from 'react'; +import { updateDatasetEntriesMetadata } from '@/app/actions/upload'; +import Modal from '@/components/Modal'; + +interface UpdateDatasetMetadataModalProps { + datasetId: number; +} + +export default function UpdateDatasetMetadataModal({ + datasetId, +}: UpdateDatasetMetadataModalProps) { + const [isOpen, setIsOpen] = useState(false); + const [files, setFiles] = useState([]); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); + const [success, setSuccess] = useState(false); + const [uploadQueue, setUploadQueue] = useState>([]); + const [status, setStatus] = useState(''); + + function handleFileChange(e: React.ChangeEvent) { + const selectedFiles = e.target.files ? Array.from(e.target.files) : []; + if (selectedFiles.length > 0) { + setFiles(selectedFiles); + setError(null); + // Initialize queue with pending status and 0 progress + setUploadQueue(selectedFiles.map(file => ({ file, status: 'pending', progress: 0 }))); + } + } + + async function handleSubmit(e: React.FormEvent) { + e.preventDefault(); + setError(null); + setSuccess(false); + + if (files.length === 0) { + setError('Please select at least one CSV file'); + return; + } + + setLoading(true); + + try { + // Process each file in the queue sequentially + const newQueue = [...uploadQueue]; + + for (let i = 0; i < newQueue.length; i++) { + const item = newQueue[i]; + item.status = 'uploading'; + item.progress = 0; + setUploadQueue([...newQueue]); + setStatus(`Processing file ${i + 1} of ${newQueue.length}: ${item.file.name}`); + + try { + // Upload with progress tracking + const uploadResult = await uploadFileWithProgress(item.file, i, newQueue); + + // Update the metadata + const updateResult = await updateDatasetEntriesMetadata(datasetId, uploadResult.tempDir); + + if (!updateResult.success) { + item.status = 'failed'; + item.progress = 0; + item.message = `✗ ${updateResult.error || 'Update failed'}`; + } else { + item.status = 'completed'; + item.progress = 100; + item.message = `✓ Updated (${updateResult.entriesUpdated} entries)`; + } + } catch (err) { + item.status = 'failed'; + item.progress = 0; + item.message = `✗ ${err instanceof Error ? err.message : 'Network error'}`; + } + + setUploadQueue([...newQueue]); + } + + const failedCount = newQueue.filter(item => item.status === 'failed').length; + const completedCount = newQueue.filter(item => item.status === 'completed').length; + + if (failedCount === 0) { + setSuccess(true); + setStatus(`All files processed! (${completedCount}/${newQueue.length} completed)`); + } else if (completedCount > 0) { + setStatus(`Completed with errors: ${completedCount} succeeded, ${failedCount} failed`); + } else { + setError(`All files failed to process`); + } + + setFiles([]); + + if (failedCount === 0) { + setTimeout(() => { + setSuccess(false); + setIsOpen(false); + setStatus(''); + setUploadQueue([]); + window.location.reload(); + }, 2500); + } + } catch (err) { + const errorMsg = err instanceof Error ? err.message : 'Failed to process updates'; + setError(errorMsg); + setStatus(''); + } finally { + setLoading(false); + } + } + + function uploadFileWithProgress( + file: File, + index: number, + queue: Array<{ file: File; status: 'pending' | 'uploading' | 'completed' | 'failed'; progress: number; message?: string }> + ): Promise<{ tempDir: string }> { + const MAX_RETRIES = 3; + + return (async () => { + let lastError: Error | null = null; + + for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) { + try { + const result = await uploadAttempt(file, index, queue, attempt, MAX_RETRIES); + return result; + } catch (error) { + lastError = error instanceof Error ? error : new Error('Unknown error'); + + if (attempt < MAX_RETRIES) { + // Exponential backoff: 1s, 2s, 4s + const delay = Math.pow(2, attempt - 1) * 1000; + queue[index].message = `Retrying in ${delay / 1000}s... (attempt ${attempt}/${MAX_RETRIES})`; + setUploadQueue([...queue]); + + await new Promise(resolve => setTimeout(resolve, delay)); + queue[index].progress = 0; // Reset progress for next attempt + setUploadQueue([...queue]); + } + } + } + + throw lastError || new Error('Upload failed after all retries'); + })(); + } + + function uploadAttempt( + file: File, + index: number, + queue: Array<{ file: File; status: 'pending' | 'uploading' | 'completed' | 'failed'; progress: number; message?: string }>, + attempt: number, + maxRetries: number + ): Promise<{ tempDir: string }> { + return new Promise((resolve, reject) => { + const formData = new FormData(); + formData.append('file', file); + formData.append('datasetId', datasetId.toString()); + + const xhr = new XMLHttpRequest(); + + // Track upload progress + xhr.upload.addEventListener('progress', (event) => { + if (event.lengthComputable) { + const progress = Math.round((event.loaded / event.total) * 100); + queue[index].progress = progress; + if (attempt > 1) { + queue[index].message = `Uploading (retry ${attempt}/${maxRetries})`; + } + setUploadQueue([...queue]); + } + }); + + xhr.addEventListener('load', () => { + if (xhr.status === 200) { + try { + const result = JSON.parse(xhr.responseText); + if (result.success) { + resolve(result); + } else { + reject(new Error(result.error || 'Upload failed')); + } + } catch (error) { + reject(new Error('Failed to parse response')); + } + } else { + try { + const error = JSON.parse(xhr.responseText); + reject(new Error(error.error || `Upload failed with status ${xhr.status}`)); + } catch { + reject(new Error(`Upload failed with status ${xhr.status}`)); + } + } + }); + + xhr.addEventListener('error', () => { + reject(new Error('Network error')); + }); + + xhr.addEventListener('abort', () => { + reject(new Error('Upload aborted')); + }); + + xhr.open('POST', '/api/update-metadata'); + xhr.send(formData); + }); + } + + function handleClose() { + if (!loading) { + setIsOpen(false); + setFiles([]); + setError(null); + setSuccess(false); + setStatus(''); + setUploadQueue([]); + } + } + + return ( + <> + + + f.status !== 'pending').length}/${uploadQueue.length}...` : 'Upload & Update', + onClick: () => { + const form = document.getElementById( + 'updateMetadataForm' + ) as HTMLFormElement; + form?.dispatchEvent(new Event('submit', { bubbles: true })); + }, + variant: 'primary', + disabled: loading || files.length === 0, + }, + ]} + > +
+
+ +
+ + +
+
+ +
+

CSV file format:

+
    +
  • + Columns: id, speaker, model, utt_id, text, dialect, iteration, duration_ms, rms_value, longest_pause, utmos_score, wer_score +
    • +
  • Entry IDs in the CSV must match existing dataset entries to update
    • +
+
+ + {uploadQueue.length > 0 && ( +
+

Upload Queue:

+
+ {uploadQueue.map((item, idx) => ( +
+
+ + {item.status === 'completed' ? '✓' : + item.status === 'failed' ? '✗' : + item.status === 'uploading' ? '⟳' : + '○'} + + {item.file.name} + {item.progress > 0 && item.progress < 100 && ( + {item.progress}% + )} + {item.message && {item.message}} +
+ {item.status === 'uploading' && ( +
+
+
+ )} + {item.status === 'completed' && ( +
+
+
+ )} + {item.status === 'failed' && ( +
+
+
+ )} +
+ ))} +
+
+ )} + + {error && ( +
+ {error} +
+ )} + + {status && ( +
+ {status} +
+ )} + + + + ); +} From c66835555d1f4ddf21a6fef5f12c2868f64d4624 Mon Sep 17 00:00:00 2001 From: averel10 Date: Fri, 13 Mar 2026 14:16:59 +0100 Subject: [PATCH 3/5] cleanup margin/width/padding --- src/app/admin/datasets/[id]/entries/[entryId]/page.tsx | 4 ++-- src/app/admin/datasets/[id]/page.tsx | 4 ++-- src/app/admin/page.tsx | 2 +- src/app/admin/users/page.tsx | 2 +- src/app/layout.tsx | 2 +- src/components/DatasetEntriesList.tsx | 4 +--- src/components/Footer.tsx | 2 +- src/components/Header.tsx | 2 +- 8 files changed, 10 insertions(+), 12 deletions(-) diff --git a/src/app/admin/datasets/[id]/entries/[entryId]/page.tsx b/src/app/admin/datasets/[id]/entries/[entryId]/page.tsx index c3f8f0b..f680b10 100644 --- a/src/app/admin/datasets/[id]/entries/[entryId]/page.tsx +++ b/src/app/admin/datasets/[id]/entries/[entryId]/page.tsx @@ -36,7 +36,7 @@ export default async function DatasetEntryPage({ params }: DatasetEntryPageProps if (entries.length === 0 || entries[0].datasetId !== datasetIdNum) { return (
-
+
-
+
-
+
-
+
+

Admin Panel

diff --git a/src/app/admin/users/page.tsx b/src/app/admin/users/page.tsx index 51e4cc9..1858289 100644 --- a/src/app/admin/users/page.tsx +++ b/src/app/admin/users/page.tsx @@ -18,7 +18,7 @@ export default async function UsersPage() { const users = await getAllUsers(); return ( -
+
-
+
{children}