diff --git a/drizzle/0006_special_junta.sql b/drizzle/0006_special_junta.sql new file mode 100644 index 0000000..b7624fa --- /dev/null +++ b/drizzle/0006_special_junta.sql @@ -0,0 +1,4 @@ +ALTER TABLE `dataset_entry` ADD `rms_value` real;--> statement-breakpoint +ALTER TABLE `dataset_entry` ADD `longest_pause` real;--> statement-breakpoint +ALTER TABLE `dataset_entry` ADD `utmos_score` real;--> statement-breakpoint +ALTER TABLE `dataset_entry` ADD `wer_score` real; \ No newline at end of file diff --git a/drizzle/meta/0006_snapshot.json b/drizzle/meta/0006_snapshot.json index 9ccf987..f9c5f13 100644 --- a/drizzle/meta/0006_snapshot.json +++ b/drizzle/meta/0006_snapshot.json @@ -1,78 +1,9 @@ { "version": "6", "dialect": "sqlite", - "id": "59929a2a-9ed3-4f73-8ef7-7af2b8d436b6", + "id": "4e72f679-ef9b-47d5-a6fb-a1c45d5cbe74", "prevId": "e6165973-fcea-4b1b-8004-60db966b8106", "tables": { - "annotation": { - "name": "annotation", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "dataset_entry_id": { - "name": "dataset_entry_id", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "user_id": { - "name": "user_id", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "rating": { - "name": "rating", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "created_at": { - "name": "created_at", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(unixepoch())" - } - }, - "indexes": { - "annotation_user_entry_idx": { - "name": "annotation_user_entry_idx", - "columns": [ - "user_id", - "dataset_entry_id" - ], - "isUnique": true - } - }, - "foreignKeys": { - "annotation_dataset_entry_id_dataset_entry_id_fk": { - "name": "annotation_dataset_entry_id_dataset_entry_id_fk", - "tableFrom": "annotation", - "tableTo": "dataset_entry", - "columnsFrom": [ - "dataset_entry_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, "account": { "name": "account", "columns": { @@ -513,6 +444,34 @@ "notNull": false, "autoincrement": false }, + "rms_value": { + "name": "rms_value", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "longest_pause": { + "name": "longest_pause", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "utmos_score": { + "name": "utmos_score", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "wer_score": { + "name": "wer_score", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, "created_at": { "name": "created_at", "type": "integer", diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json index 99cd494..0455de6 100644 --- a/drizzle/meta/_journal.json +++ b/drizzle/meta/_journal.json @@ -47,15 +47,8 @@ { "idx": 6, "version": "6", - "when": 1773406795226, - "tag": "0006_polite_moondragon", - "breakpoints": true - }, - { - "idx": 7, - "version": "6", - "when": 1773407494321, - "tag": "0007_flawless_wonder_man", + "when": 1773406006276, + "tag": "0006_special_junta", "breakpoints": true } ] diff --git a/src/app/actions/download-filtered-entries.ts b/src/app/actions/download-filtered-entries.ts index 7556474..8a01847 100644 --- a/src/app/actions/download-filtered-entries.ts +++ b/src/app/actions/download-filtered-entries.ts @@ -53,6 +53,10 @@ export async function downloadFilteredEntries( 'id', 'audio_file', 'duration_ms', + 'rms_value', + 'longest_pause', + 'utmos_score', + 'wer_score', 'utt_id', 'text', 'speaker', @@ -67,6 +71,10 @@ export async function downloadFilteredEntries( `"${entry.externalId}"`, entry.fileName, entry.durationMs?.toString() || '', + entry.rmsValue?.toString() || '', + entry.longestPause?.toString() || '', + entry.utmosScore?.toString() || '', + entry.werScore?.toString() || '', entry.utteranceId || '', `"${(entry.utteranceText || '').replace(/"/g, '""')}"`, entry.speakerId, diff --git a/src/app/actions/get-dataset-entries.ts b/src/app/actions/get-dataset-entries.ts index 7104e98..4dc8618 100644 --- a/src/app/actions/get-dataset-entries.ts +++ b/src/app/actions/get-dataset-entries.ts @@ -2,9 +2,10 @@ import db from '@/lib/db'; import { dataset_entry } from '@/lib/model/dataset_entry'; -import { count, eq, and, like } from 'drizzle-orm'; +import { count, eq, and, like, asc, desc, getTableColumns } from 'drizzle-orm'; const ENTRIES_PER_PAGE = 20; +const SORT_COLUMNS = getTableColumns(dataset_entry); interface FilterParams { speakerId?: string; @@ -14,10 +15,16 @@ interface FilterParams { utteranceId?: string; } +interface SortParams { + sortBy?: keyof typeof SORT_COLUMNS; + sortOrder?: 'asc' | 'desc'; +} + export async function getDatasetEntries( datasetId: number, page: number = 1, - filters?: FilterParams + filters?: FilterParams, + sort?: SortParams ) { const offset = (page - 1) * ENTRIES_PER_PAGE; @@ -46,10 +53,17 @@ export async function getDatasetEntries( const whereClause = and(...conditions); + // Build sort clause + const sortFn = sort?.sortOrder === 'desc' ? desc : asc; + const sortColumn = sort?.sortBy && sort.sortBy in SORT_COLUMNS + ? sortFn(SORT_COLUMNS[sort.sortBy as keyof typeof SORT_COLUMNS]) + : asc(dataset_entry.externalId); + const entries = await db .select() .from(dataset_entry) .where(whereClause) + .orderBy(sortColumn) .limit(ENTRIES_PER_PAGE) .offset(offset); diff --git a/src/app/actions/upload.ts b/src/app/actions/upload.ts index d3d52c1..47d5003 100644 --- a/src/app/actions/upload.ts +++ b/src/app/actions/upload.ts @@ -2,6 +2,7 @@ import db from '@/lib/db'; import { dataset_entry } from '@/lib/model/dataset_entry'; +import { parseCSVLine } from '@/lib/csv-parser'; import { revalidatePath } from 'next/cache'; import { eq } from 'drizzle-orm'; import { writeFile, mkdir, readFile, rm } from 'fs/promises'; @@ -10,38 +11,71 @@ import { existsSync } from 'fs'; import { requireAdmin } from '@/lib/auth'; /** - * CSV parser that handles quoted fields + * Helper: Parse metadata.csv from extracted directory + * Returns structured data for both insert and update operations */ -function parseCSVLine(line: string): string[] { - const values: string[] = []; - let current = ''; - let inQuotes = false; - - for (let i = 0; i < line.length; i++) { - const char = line[i]; - const nextChar = line[i + 1]; - - if (char === '"') { - if (inQuotes && nextChar === '"') { - // Escaped quote - current += '"'; - i++; - } else { - // Toggle quote state - inQuotes = !inQuotes; - } - } else if (char === ',' && !inQuotes) { - // Field separator - values.push(current.trim()); - current = ''; - } else { - current += char; - } +async function parseMetadataFromDirectory( + tempDir: string +): Promise<{ + headers: string[]; + rows: Record[]; +}> { + const metadataPath = join(tempDir, 'metadata.csv'); + if (!existsSync(metadataPath)) { + throw new Error('metadata.csv not found in extracted files'); } - // Add the last field - values.push(current.trim()); - return values; + const metadataContent = await readFile(metadataPath, 'utf-8'); + const lines = metadataContent.split('\n').filter((line) => line.trim()); + + if (lines.length < 2) { + throw new Error('metadata.csv is empty or invalid'); + } + + const headers = lines[0].split(',').map((h) => h.trim()); + const rows: Record[] = []; + + for (let i = 1; i < lines.length; i++) { + const values = parseCSVLine(lines[i]); + + if (values.length !== headers.length) { + continue; // Skip malformed lines + } + + const row: Record = {}; + headers.forEach((header, index) => { + row[header] = values[index]; + }); + + rows.push(row); + } + + return { headers, rows }; +} + +/** + * Convert metadata row to dataset_entry insert object + */ +function rowToDatasetEntry( + row: Record, + datasetId: number +): typeof dataset_entry.$inferInsert { + return { + datasetId, + externalId: row.id, + speakerId: row.speaker, + modelName: row.model, + utteranceId: row.utt_id, + utteranceText: row.text, + fileName: row.audio_file, + dialect: row.dialect, + iteration: parseInt(row.iteration, 10), + durationMs: row.duration_ms ? parseInt(row.duration_ms, 10) : undefined, + rmsValue: row.rms_value ? parseFloat(row.rms_value) : undefined, + longestPause: row.longest_pause ? parseFloat(row.longest_pause) : undefined, + utmosScore: row.utmos_score ? parseFloat(row.utmos_score) : undefined, + werScore: row.wer_score ? parseFloat(row.wer_score) : undefined, + }; } /** @@ -73,47 +107,16 @@ export async function processDatasetEntries( }; } - // Read metadata.csv again - const metadataPath = join(tempDir, 'metadata.csv'); - if (!existsSync(metadataPath)) { - return { - success: false, - error: 'metadata.csv not found in extracted files', - entriesCreated: 0, - }; - } + // Parse metadata from directory + const { rows } = await parseMetadataFromDirectory(tempDir); - const metadataContent = await readFile(metadataPath, 'utf-8'); - const lines = metadataContent.split('\n').filter(line => line.trim()); - - if (lines.length < 2) { - return { - success: false, - error: 'metadata.csv is empty or invalid', - entriesCreated: 0, - }; - } - - // Parse CSV - const headers = lines[0].split(',').map(h => h.trim()); const entries: typeof dataset_entry.$inferInsert[] = []; // Create dataset directory await mkdir(datasetDir, { recursive: true }); // Process each row - for (let i = 1; i < lines.length; i++) { - const values = parseCSVLine(lines[i]); - - if (values.length !== headers.length) { - continue; // Skip malformed lines - } - - const row: Record = {}; - headers.forEach((header, index) => { - row[header] = values[index]; - }); - + for (const row of rows) { const audioFile = row.audio_file as string; const relativePath = audioFile.replace(/\\/g, '/'); const audioPath = join(tempDir, relativePath); @@ -126,18 +129,7 @@ export async function processDatasetEntries( await writeFile(destPath, audioBuffer); - entries.push({ - datasetId, - externalId: row.id, - speakerId: row.speaker, - modelName: row.model, - utteranceId: row.utt_id, - utteranceText: row.text, - fileName: relativePath, - dialect: row.dialect, - iteration: parseInt(row.iteration, 10), - durationMs: row.duration_ms ? parseInt(row.duration_ms, 10) : undefined, - }); + entries.push(rowToDatasetEntry(row, datasetId)); } } @@ -197,3 +189,116 @@ export async function processDatasetEntries( } } } + +/** + * Update metadata for existing dataset entries + * Uses the same metadata.csv parsing as processDatasetEntries + * Matches entries by externalId and updates their metadata fields + */ +export async function updateDatasetEntriesMetadata( + datasetId: number, + tempDir: string +) { + try { + const result = await requireAdmin(); + if (!result.authenticated || !result.admin) { + return { + success: false, + error: 'Unauthorized', + entriesUpdated: 0, + }; + } + + // Validate temp directory exists + if (!existsSync(tempDir)) { + return { + success: false, + error: 'Temporary extraction directory not found', + entriesUpdated: 0, + }; + } + + // Parse metadata from directory (reusing same logic as processDatasetEntries) + const { rows } = await parseMetadataFromDirectory(tempDir); + + if (rows.length === 0) { + return { + success: false, + error: 'No valid entries found in metadata.csv', + entriesUpdated: 0, + }; + } + + // Get all existing entries for this dataset + const existingEntries = await db + .select() + .from(dataset_entry) + .where(eq(dataset_entry.datasetId, datasetId)); + + const existingByExternalId = new Map( + existingEntries.map(entry => [entry.externalId, entry]) + ); + + let entriesUpdated = 0; + let entriesNotFound = 0; + + // Update each row that has a matching entry + for (const row of rows) { + const externalId = row.id; + const existingEntry = existingByExternalId.get(externalId); + + if (!existingEntry) { + entriesNotFound++; + continue; + } + + // Update only the metadata fields, don't change the ID or dataset reference + await db + .update(dataset_entry) + .set({ + speakerId: row.speaker, + modelName: row.model, + utteranceId: row.utt_id, + utteranceText: row.text, + dialect: row.dialect, + iteration: parseInt(row.iteration, 10), + durationMs: row.duration_ms ? parseInt(row.duration_ms, 10) : undefined, + rmsValue: row.rms_value ? parseFloat(row.rms_value) : undefined, + longestPause: row.longest_pause ? parseFloat(row.longest_pause) : undefined, + utmosScore: row.utmos_score ? parseFloat(row.utmos_score) : undefined, + werScore: row.wer_score ? parseFloat(row.wer_score) : undefined, + }) + .where(eq(dataset_entry.id, existingEntry.id)); + + entriesUpdated++; + } + + revalidatePath(`/admin/datasets/${datasetId}`); + + return { + success: true, + entriesUpdated, + entriesNotFound, + message: entriesUpdated > 0 + ? `Successfully updated ${entriesUpdated} entries${entriesNotFound > 0 ? ` (${entriesNotFound} not found)` : ''}.` + : `No entries were updated (${entriesNotFound} not found).`, + }; + } catch (error) { + console.error('Error updating dataset entries metadata:', error); + const errorMessage = error instanceof Error ? error.message : 'Failed to update dataset entries metadata'; + return { + success: false, + error: errorMessage, + entriesUpdated: 0, + }; + } finally { + // Clean up temp directory + try { + if (existsSync(tempDir)) { + await rm(tempDir, { recursive: true, force: true }); + } + } catch (cleanupError) { + console.error('Error cleaning up temp directory:', cleanupError); + } + } +} diff --git a/src/app/admin/datasets/[id]/entries/[entryId]/page.tsx b/src/app/admin/datasets/[id]/entries/[entryId]/page.tsx index b9c80d0..f680b10 100644 --- a/src/app/admin/datasets/[id]/entries/[entryId]/page.tsx +++ b/src/app/admin/datasets/[id]/entries/[entryId]/page.tsx @@ -36,7 +36,7 @@ export default async function DatasetEntryPage({ params }: DatasetEntryPageProps if (entries.length === 0 || entries[0].datasetId !== datasetIdNum) { return (
-
+
-
+
{entry.durationMs?.toLocaleString() || '-'}

+
+

RMS Value

+

{entry.rmsValue?.toFixed(3) || '-'}

+
+ +
+

Longest Pause (s)

+

{entry.longestPause?.toFixed(3) || '-'}

+
+ +
+

UTMOS Score

+

{entry.utmosScore?.toFixed(3) || '-'}

+
+ +
+

WER Score

+

{entry.werScore?.toFixed(3) || '-'}

+
+

Created

diff --git a/src/app/admin/datasets/[id]/page.tsx b/src/app/admin/datasets/[id]/page.tsx index 21fd494..ef61a11 100644 --- a/src/app/admin/datasets/[id]/page.tsx +++ b/src/app/admin/datasets/[id]/page.tsx @@ -5,6 +5,7 @@ import { eq } from 'drizzle-orm'; import EditableDatasetHeader from '@/components/EditableDatasetHeader'; import DatasetEntriesList from '@/components/DatasetEntriesList'; import UploadDatasetEntriesModal from '@/components/UploadDatasetEntriesModal'; +import UpdateDatasetMetadataModal from '@/components/UpdateDatasetMetadataModal'; import RemoveAllEntriesButton from '@/components/RemoveAllEntriesButton'; import DeleteDatasetButton from '@/components/DeleteDatasetButton'; import { requireAdmin } from '@/lib/auth'; @@ -38,7 +39,7 @@ export default async function DatasetPage({ params }: DatasetPageProps) { if (datasets.length === 0) { return (

-
+
-
+
+
diff --git a/src/app/admin/page.tsx b/src/app/admin/page.tsx index bd9ac4e..1623152 100644 --- a/src/app/admin/page.tsx +++ b/src/app/admin/page.tsx @@ -18,7 +18,7 @@ export default async function AdminPage() { } return ( -
+

Admin Panel

diff --git a/src/app/admin/users/page.tsx b/src/app/admin/users/page.tsx index 51e4cc9..1858289 100644 --- a/src/app/admin/users/page.tsx +++ b/src/app/admin/users/page.tsx @@ -18,7 +18,7 @@ export default async function UsersPage() { const users = await getAllUsers(); return ( -
+
-
+
{children}