ZHAW/BA_TTS_Annotation_Plattform
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat: add new fields to dataset_entry and update related components for processing and display

Browse Source
This commit is contained in:
averel10
2026-03-13 14:01:25 +01:00
parent 2b6d23c406
commit 4c9b4734e7
9 changed files with 667 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
drizzle/0006_special_junta.sql Normal file
View File

@@ -0,0 +1,4 @@
ALTER TABLE `dataset_entry` ADD `rms_value` real;--> statement-breakpoint
ALTER TABLE `dataset_entry` ADD `longest_pause` real;--> statement-breakpoint
ALTER TABLE `dataset_entry` ADD `utmos_score` real;--> statement-breakpoint
ALTER TABLE `dataset_entry` ADD `wer_score` real;

570
drizzle/meta/0006_snapshot.json Normal file
View File

@@ -0,0 +1,570 @@
{
"version": "6",
"dialect": "sqlite",
"id": "4e72f679-ef9b-47d5-a6fb-a1c45d5cbe74",
"prevId": "e6165973-fcea-4b1b-8004-60db966b8106",
"tables": {
"account": {
"name": "account",
"columns": {
"id": {
"name": "id",
"type": "text",
"primaryKey": true,
"notNull": true,
"autoincrement": false
},
"account_id": {
"name": "account_id",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"provider_id": {
"name": "provider_id",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"user_id": {
"name": "user_id",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"access_token": {
"name": "access_token",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"refresh_token": {
"name": "refresh_token",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"id_token": {
"name": "id_token",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"access_token_expires_at": {
"name": "access_token_expires_at",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"refresh_token_expires_at": {
"name": "refresh_token_expires_at",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"scope": {
"name": "scope",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"password": {
"name": "password",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"created_at": {
"name": "created_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": "(cast(unixepoch('subsecond') * 1000 as integer))"
},
"updated_at": {
"name": "updated_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
}
},
"indexes": {
"account_userId_idx": {
"name": "account_userId_idx",
"columns": [
"user_id"
],
"isUnique": false
}
},
"foreignKeys": {
"account_user_id_user_id_fk": {
"name": "account_user_id_user_id_fk",
"tableFrom": "account",
"tableTo": "user",
"columnsFrom": [
"user_id"
],
"columnsTo": [
"id"
],
"onDelete": "cascade",
"onUpdate": "no action"
}
},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"checkConstraints": {}
},
"session": {
"name": "session",
"columns": {
"id": {
"name": "id",
"type": "text",
"primaryKey": true,
"notNull": true,
"autoincrement": false
},
"expires_at": {
"name": "expires_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"token": {
"name": "token",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"created_at": {
"name": "created_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": "(cast(unixepoch('subsecond') * 1000 as integer))"
},
"updated_at": {
"name": "updated_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"ip_address": {
"name": "ip_address",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"user_agent": {
"name": "user_agent",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"user_id": {
"name": "user_id",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
}
},
"indexes": {
"session_token_unique": {
"name": "session_token_unique",
"columns": [
"token"
],
"isUnique": true
},
"session_userId_idx": {
"name": "session_userId_idx",
"columns": [
"user_id"
],
"isUnique": false
}
},
"foreignKeys": {
"session_user_id_user_id_fk": {
"name": "session_user_id_user_id_fk",
"tableFrom": "session",
"tableTo": "user",
"columnsFrom": [
"user_id"
],
"columnsTo": [
"id"
],
"onDelete": "cascade",
"onUpdate": "no action"
}
},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"checkConstraints": {}
},
"user": {
"name": "user",
"columns": {
"id": {
"name": "id",
"type": "text",
"primaryKey": true,
"notNull": true,
"autoincrement": false
},
"name": {
"name": "name",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"email": {
"name": "email",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"email_verified": {
"name": "email_verified",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": false
},
"image": {
"name": "image",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"admin": {
"name": "admin",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": false
},
"created_at": {
"name": "created_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": "(cast(unixepoch('subsecond') * 1000 as integer))"
},
"updated_at": {
"name": "updated_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": "(cast(unixepoch('subsecond') * 1000 as integer))"
}
},
"indexes": {
"user_email_unique": {
"name": "user_email_unique",
"columns": [
"email"
],
"isUnique": true
}
},
"foreignKeys": {},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"checkConstraints": {}
},
"verification": {
"name": "verification",
"columns": {
"id": {
"name": "id",
"type": "text",
"primaryKey": true,
"notNull": true,
"autoincrement": false
},
"identifier": {
"name": "identifier",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"value": {
"name": "value",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"expires_at": {
"name": "expires_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"created_at": {
"name": "created_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": "(cast(unixepoch('subsecond') * 1000 as integer))"
},
"updated_at": {
"name": "updated_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": "(cast(unixepoch('subsecond') * 1000 as integer))"
}
},
"indexes": {
"verification_identifier_idx": {
"name": "verification_identifier_idx",
"columns": [
"identifier"
],
"isUnique": false
}
},
"foreignKeys": {},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"checkConstraints": {}
},
"dataset_entry": {
"name": "dataset_entry",
"columns": {
"id": {
"name": "id",
"type": "integer",
"primaryKey": true,
"notNull": true,
"autoincrement": true
},
"external_id": {
"name": "external_id",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"speaker_id": {
"name": "speaker_id",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"model_name": {
"name": "model_name",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"utterance_id": {
"name": "utterance_id",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"utterance_text": {
"name": "utterance_text",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"dataset_id": {
"name": "dataset_id",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"file_name": {
"name": "file_name",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"dialect": {
"name": "dialect",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"iteration": {
"name": "iteration",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"duration_ms": {
"name": "duration_ms",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"rms_value": {
"name": "rms_value",
"type": "real",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"longest_pause": {
"name": "longest_pause",
"type": "real",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"utmos_score": {
"name": "utmos_score",
"type": "real",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"wer_score": {
"name": "wer_score",
"type": "real",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"created_at": {
"name": "created_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": "(unixepoch())"
},
"updated_at": {
"name": "updated_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": "(unixepoch())"
}
},
"indexes": {},
"foreignKeys": {
"dataset_entry_dataset_id_dataset_id_fk": {
"name": "dataset_entry_dataset_id_dataset_id_fk",
"tableFrom": "dataset_entry",
"tableTo": "dataset",
"columnsFrom": [
"dataset_id"
],
"columnsTo": [
"id"
],
"onDelete": "no action",
"onUpdate": "no action"
}
},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"checkConstraints": {}
},
"dataset": {
"name": "dataset",
"columns": {
"id": {
"name": "id",
"type": "integer",
"primaryKey": true,
"notNull": true,
"autoincrement": true
},
"name": {
"name": "name",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"description": {
"name": "description",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"created_at": {
"name": "created_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": "(unixepoch())"
},
"updated_at": {
"name": "updated_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": "(unixepoch())"
}
},
"indexes": {},
"foreignKeys": {},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"checkConstraints": {}
}
},
"views": {},
"enums": {},
"_meta": {
"schemas": {},
"tables": {},
"columns": {}
},
"internal": {
"indexes": {}
}
}

7
drizzle/meta/_journal.json
View File

@@ -43,6 +43,13 @@
"when": 1773395728082,
"tag": "0005_flashy_piledriver",
"breakpoints": true
},
{
"idx": 6,
"version": "6",
"when": 1773406006276,
"tag": "0006_special_junta",
"breakpoints": true
}
]
}

8
src/app/actions/download-filtered-entries.ts
View File

@@ -53,6 +53,10 @@ export async function downloadFilteredEntries(
'id',
'audio_file',
'duration_ms',
'rms_value',
'longest_pause',
'utmos_score',
'wer_score',
'utt_id',
'text',
'speaker',
@@ -67,6 +71,10 @@ export async function downloadFilteredEntries(
`"${entry.externalId}"`,
entry.fileName,
entry.durationMs?.toString() || '',
entry.rmsValue?.toString() || '',
entry.longestPause?.toString() || '',
entry.utmosScore?.toString() || '',
entry.werScore?.toString() || '',
entry.utteranceId || '',
`"${(entry.utteranceText || '').replace(/"/g, '""')}"`,
entry.speakerId,

40
src/app/actions/upload.ts
View File

@@ -2,6 +2,7 @@
import db from '@/lib/db';
import { dataset_entry } from '@/lib/model/dataset_entry';
import { parseCSVLine } from '@/lib/csv-parser';
import { revalidatePath } from 'next/cache';
import { eq } from 'drizzle-orm';
import { writeFile, mkdir, readFile, rm } from 'fs/promises';
@@ -9,41 +10,6 @@ import { join } from 'path';
import { existsSync } from 'fs';
import { requireAdmin } from '@/lib/auth';
/**
* CSV parser that handles quoted fields
*/
function parseCSVLine(line: string): string[] {
const values: string[] = [];
let current = '';
let inQuotes = false;
for (let i = 0; i < line.length; i++) {
const char = line[i];
const nextChar = line[i + 1];
if (char === '"') {
if (inQuotes && nextChar === '"') {
// Escaped quote
current += '"';
i++;
} else {
// Toggle quote state
inQuotes = !inQuotes;
}
} else if (char === ',' && !inQuotes) {
// Field separator
values.push(current.trim());
current = '';
} else {
current += char;
}
}
// Add the last field
values.push(current.trim());
return values;
}
/**
* Step 2: Process the extracted ZIP data and insert into database
* Copies audio files and creates database entries
@@ -137,6 +103,10 @@ export async function processDatasetEntries(
dialect: row.dialect,
iteration: parseInt(row.iteration, 10),
durationMs: row.duration_ms ? parseInt(row.duration_ms, 10) : undefined,
rmsValue: row.rms_value ? parseFloat(row.rms_value) : undefined,
longestPause: row.longest_pause ? parseFloat(row.longest_pause) : undefined,
utmosScore: row.utmos_score ? parseFloat(row.utmos_score) : undefined,
werScore: row.wer_score ? parseFloat(row.wer_score) : undefined,
});
}
}

20
src/app/admin/datasets/[id]/entries/[entryId]/page.tsx
View File

@@ -122,6 +122,26 @@ export default async function DatasetEntryPage({ params }: DatasetEntryPageProps
<p className="text-lg font-semibold">{entry.durationMs?.toLocaleString() || '-'}</p>
</div>
<div className="bg-white border border-gray-200 p-4 rounded-lg">
<p className="text-sm text-gray-600 mb-1">RMS Value</p>
<p className="text-lg font-semibold">{entry.rmsValue?.toFixed(3) || '-'}</p>
</div>
<div className="bg-white border border-gray-200 p-4 rounded-lg">
<p className="text-sm text-gray-600 mb-1">Longest Pause (s)</p>
<p className="text-lg font-semibold">{entry.longestPause?.toFixed(3) || '-'}</p>
</div>
<div className="bg-white border border-gray-200 p-4 rounded-lg">
<p className="text-sm text-gray-600 mb-1">UTMOS Score</p>
<p className="text-lg font-semibold">{entry.utmosScore?.toFixed(3) || '-'}</p>
</div>
<div className="bg-white border border-gray-200 p-4 rounded-lg">
<p className="text-sm text-gray-600 mb-1">WER Score</p>
<p className="text-lg font-semibold">{entry.werScore?.toFixed(3) || '-'}</p>
</div>
<div className="bg-white border border-gray-200 p-4 rounded-lg">
<p className="text-sm text-gray-600 mb-1">Created</p>
<p className="text-lg font-semibold">

15
src/components/DatasetEntriesList.tsx
View File

@@ -22,6 +22,11 @@ interface DatasetEntry {
fileName: string;
dialect: string;
iteration: number;
durationMs: number | null;
rmsValue: number | null;
longestPause: number | null;
utmosScore: number | null;
werScore: number | null;
createdAt: Date;
updatedAt: Date;
}
@@ -417,6 +422,11 @@ export default function DatasetEntriesList({
<th className="text-left py-3 px-4 font-semibold text-gray-700">Dialect</th>
<th className="text-left py-3 px-4 font-semibold text-gray-700">File Name</th>
<th className="text-left py-3 px-4 font-semibold text-gray-700">Iteration</th>
<th className="text-left py-3 px-4 font-semibold text-gray-700">Duration (ms)</th>
<th className="text-left py-3 px-4 font-semibold text-gray-700">RMS Value</th>
<th className="text-left py-3 px-4 font-semibold text-gray-700">Longest Pause (s)</th>
<th className="text-left py-3 px-4 font-semibold text-gray-700">UTMOS Score</th>
<th className="text-left py-3 px-4 font-semibold text-gray-700">WER Score</th>
</tr>
</thead>
<tbody>
@@ -439,6 +449,11 @@ export default function DatasetEntriesList({
<td className="py-3 px-4">{entry.dialect}</td>
<td className="py-3 px-4 truncate">{entry.fileName}</td>
<td className="py-3 px-4">{entry.iteration}</td>
<td className="py-3 px-4">{entry.durationMs?.toLocaleString() || '-'}</td>
<td className="py-3 px-4">{entry.rmsValue?.toFixed(3) || '-'}</td>
<td className="py-3 px-4">{entry.longestPause?.toFixed(3) || '-'}</td>
<td className="py-3 px-4">{entry.utmosScore?.toFixed(3) || '-'}</td>
<td className="py-3 px-4">{entry.werScore?.toFixed(3) || '-'}</td>
</tr>
))}
</tbody>

34
src/lib/csv-parser.ts Normal file
View File

@@ -0,0 +1,34 @@
/**
* CSV parser that handles quoted fields
*/
export function parseCSVLine(line: string): string[] {
const values: string[] = [];
let current = '';
let inQuotes = false;
for (let i = 0; i < line.length; i++) {
const char = line[i];
const nextChar = line[i + 1];
if (char === '"') {
if (inQuotes && nextChar === '"') {
// Escaped quote
current += '"';
i++;
} else {
// Toggle quote state
inQuotes = !inQuotes;
}
} else if (char === ',' && !inQuotes) {
// Field separator
values.push(current.trim());
current = '';
} else {
current += char;
}
}
// Add the last field
values.push(current.trim());
return values;
}

4
src/lib/model/dataset_entry.ts
View File

@@ -16,6 +16,10 @@ export const dataset_entry = sqliteTable('dataset_entry', {
dialect: text('dialect').notNull(),
iteration: integer('iteration').notNull(),
durationMs: integer('duration_ms'),
rmsValue: real('rms_value'),
longestPause: real('longest_pause'),
utmosScore: real('utmos_score'),
werScore: real('wer_score'),
createdAt: integer('created_at', { mode: 'timestamp' })
.notNull()
.default(sql`(unixepoch())`),