diff --git a/drizzle/0006_special_junta.sql b/drizzle/0006_special_junta.sql new file mode 100644 index 0000000..b7624fa --- /dev/null +++ b/drizzle/0006_special_junta.sql @@ -0,0 +1,4 @@ +ALTER TABLE `dataset_entry` ADD `rms_value` real;--> statement-breakpoint +ALTER TABLE `dataset_entry` ADD `longest_pause` real;--> statement-breakpoint +ALTER TABLE `dataset_entry` ADD `utmos_score` real;--> statement-breakpoint +ALTER TABLE `dataset_entry` ADD `wer_score` real; \ No newline at end of file diff --git a/drizzle/meta/0006_snapshot.json b/drizzle/meta/0006_snapshot.json new file mode 100644 index 0000000..f9c5f13 --- /dev/null +++ b/drizzle/meta/0006_snapshot.json @@ -0,0 +1,570 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "4e72f679-ef9b-47d5-a6fb-a1c45d5cbe74", + "prevId": "e6165973-fcea-4b1b-8004-60db966b8106", + "tables": { + "account": { + "name": "account", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "account_userId_idx": { + "name": "account_userId_idx", + "columns": [ + "user_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "session": { + "name": "session", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "session_token_unique": { + "name": "session_token_unique", + "columns": [ + "token" + ], + "isUnique": true + }, + "session_userId_idx": { + "name": "session_userId_idx", + "columns": [ + "user_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "user": { + "name": "user", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email_verified": { + "name": "email_verified", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "admin": { + "name": "admin", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "user_email_unique": { + "name": "user_email_unique", + "columns": [ + "email" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "verification": { + "name": "verification", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "verification_identifier_idx": { + "name": "verification_identifier_idx", + "columns": [ + "identifier" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "dataset_entry": { + "name": "dataset_entry", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "external_id": { + "name": "external_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "speaker_id": { + "name": "speaker_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model_name": { + "name": "model_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "utterance_id": { + "name": "utterance_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "utterance_text": { + "name": "utterance_text", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "dataset_id": { + "name": "dataset_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "file_name": { + "name": "file_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "dialect": { + "name": "dialect", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "iteration": { + "name": "iteration", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "rms_value": { + "name": "rms_value", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "longest_pause": { + "name": "longest_pause", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "utmos_score": { + "name": "utmos_score", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "wer_score": { + "name": "wer_score", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + } + }, + "indexes": {}, + "foreignKeys": { + "dataset_entry_dataset_id_dataset_id_fk": { + "name": "dataset_entry_dataset_id_dataset_id_fk", + "tableFrom": "dataset_entry", + "tableTo": "dataset", + "columnsFrom": [ + "dataset_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "dataset": { + "name": "dataset", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json index 97bf443..0455de6 100644 --- a/drizzle/meta/_journal.json +++ b/drizzle/meta/_journal.json @@ -43,6 +43,13 @@ "when": 1773395728082, "tag": "0005_flashy_piledriver", "breakpoints": true + }, + { + "idx": 6, + "version": "6", + "when": 1773406006276, + "tag": "0006_special_junta", + "breakpoints": true } ] } \ No newline at end of file diff --git a/src/app/actions/download-filtered-entries.ts b/src/app/actions/download-filtered-entries.ts index 7556474..8a01847 100644 --- a/src/app/actions/download-filtered-entries.ts +++ b/src/app/actions/download-filtered-entries.ts @@ -53,6 +53,10 @@ export async function downloadFilteredEntries( 'id', 'audio_file', 'duration_ms', + 'rms_value', + 'longest_pause', + 'utmos_score', + 'wer_score', 'utt_id', 'text', 'speaker', @@ -67,6 +71,10 @@ export async function downloadFilteredEntries( `"${entry.externalId}"`, entry.fileName, entry.durationMs?.toString() || '', + entry.rmsValue?.toString() || '', + entry.longestPause?.toString() || '', + entry.utmosScore?.toString() || '', + entry.werScore?.toString() || '', entry.utteranceId || '', `"${(entry.utteranceText || '').replace(/"/g, '""')}"`, entry.speakerId, diff --git a/src/app/actions/upload.ts b/src/app/actions/upload.ts index d3d52c1..1201bad 100644 --- a/src/app/actions/upload.ts +++ b/src/app/actions/upload.ts @@ -2,6 +2,7 @@ import db from '@/lib/db'; import { dataset_entry } from '@/lib/model/dataset_entry'; +import { parseCSVLine } from '@/lib/csv-parser'; import { revalidatePath } from 'next/cache'; import { eq } from 'drizzle-orm'; import { writeFile, mkdir, readFile, rm } from 'fs/promises'; @@ -9,41 +10,6 @@ import { join } from 'path'; import { existsSync } from 'fs'; import { requireAdmin } from '@/lib/auth'; -/** - * CSV parser that handles quoted fields - */ -function parseCSVLine(line: string): string[] { - const values: string[] = []; - let current = ''; - let inQuotes = false; - - for (let i = 0; i < line.length; i++) { - const char = line[i]; - const nextChar = line[i + 1]; - - if (char === '"') { - if (inQuotes && nextChar === '"') { - // Escaped quote - current += '"'; - i++; - } else { - // Toggle quote state - inQuotes = !inQuotes; - } - } else if (char === ',' && !inQuotes) { - // Field separator - values.push(current.trim()); - current = ''; - } else { - current += char; - } - } - - // Add the last field - values.push(current.trim()); - return values; -} - /** * Step 2: Process the extracted ZIP data and insert into database * Copies audio files and creates database entries @@ -137,6 +103,10 @@ export async function processDatasetEntries( dialect: row.dialect, iteration: parseInt(row.iteration, 10), durationMs: row.duration_ms ? parseInt(row.duration_ms, 10) : undefined, + rmsValue: row.rms_value ? parseFloat(row.rms_value) : undefined, + longestPause: row.longest_pause ? parseFloat(row.longest_pause) : undefined, + utmosScore: row.utmos_score ? parseFloat(row.utmos_score) : undefined, + werScore: row.wer_score ? parseFloat(row.wer_score) : undefined, }); } } diff --git a/src/app/admin/datasets/[id]/entries/[entryId]/page.tsx b/src/app/admin/datasets/[id]/entries/[entryId]/page.tsx index b9c80d0..c3f8f0b 100644 --- a/src/app/admin/datasets/[id]/entries/[entryId]/page.tsx +++ b/src/app/admin/datasets/[id]/entries/[entryId]/page.tsx @@ -122,6 +122,26 @@ export default async function DatasetEntryPage({ params }: DatasetEntryPageProps

{entry.durationMs?.toLocaleString() || '-'}

+
+

RMS Value

+

{entry.rmsValue?.toFixed(3) || '-'}

+
+ +
+

Longest Pause (s)

+

{entry.longestPause?.toFixed(3) || '-'}

+
+ +
+

UTMOS Score

+

{entry.utmosScore?.toFixed(3) || '-'}

+
+ +
+

WER Score

+

{entry.werScore?.toFixed(3) || '-'}

+
+

Created

diff --git a/src/components/DatasetEntriesList.tsx b/src/components/DatasetEntriesList.tsx index 029d53f..041a339 100644 --- a/src/components/DatasetEntriesList.tsx +++ b/src/components/DatasetEntriesList.tsx @@ -22,6 +22,11 @@ interface DatasetEntry { fileName: string; dialect: string; iteration: number; + durationMs: number | null; + rmsValue: number | null; + longestPause: number | null; + utmosScore: number | null; + werScore: number | null; createdAt: Date; updatedAt: Date; } @@ -417,6 +422,11 @@ export default function DatasetEntriesList({ Dialect File Name Iteration + Duration (ms) + RMS Value + Longest Pause (s) + UTMOS Score + WER Score @@ -439,6 +449,11 @@ export default function DatasetEntriesList({ {entry.dialect} {entry.fileName} {entry.iteration} + {entry.durationMs?.toLocaleString() || '-'} + {entry.rmsValue?.toFixed(3) || '-'} + {entry.longestPause?.toFixed(3) || '-'} + {entry.utmosScore?.toFixed(3) || '-'} + {entry.werScore?.toFixed(3) || '-'} ))} diff --git a/src/lib/csv-parser.ts b/src/lib/csv-parser.ts new file mode 100644 index 0000000..1b90869 --- /dev/null +++ b/src/lib/csv-parser.ts @@ -0,0 +1,34 @@ +/** + * CSV parser that handles quoted fields + */ +export function parseCSVLine(line: string): string[] { + const values: string[] = []; + let current = ''; + let inQuotes = false; + + for (let i = 0; i < line.length; i++) { + const char = line[i]; + const nextChar = line[i + 1]; + + if (char === '"') { + if (inQuotes && nextChar === '"') { + // Escaped quote + current += '"'; + i++; + } else { + // Toggle quote state + inQuotes = !inQuotes; + } + } else if (char === ',' && !inQuotes) { + // Field separator + values.push(current.trim()); + current = ''; + } else { + current += char; + } + } + + // Add the last field + values.push(current.trim()); + return values; +} diff --git a/src/lib/model/dataset_entry.ts b/src/lib/model/dataset_entry.ts index 482aa56..900450c 100644 --- a/src/lib/model/dataset_entry.ts +++ b/src/lib/model/dataset_entry.ts @@ -16,6 +16,10 @@ export const dataset_entry = sqliteTable('dataset_entry', { dialect: text('dialect').notNull(), iteration: integer('iteration').notNull(), durationMs: integer('duration_ms'), + rmsValue: real('rms_value'), + longestPause: real('longest_pause'), + utmosScore: real('utmos_score'), + werScore: real('wer_score'), createdAt: integer('created_at', { mode: 'timestamp' }) .notNull() .default(sql`(unixepoch())`),