Merge branch 'main' into prototyp-2-fabio

This commit is contained in:
smaubio
2026-03-13 14:58:02 +01:00
18 changed files with 819 additions and 177 deletions

View File

@@ -0,0 +1,4 @@
ALTER TABLE `dataset_entry` ADD `rms_value` real;--> statement-breakpoint
ALTER TABLE `dataset_entry` ADD `longest_pause` real;--> statement-breakpoint
ALTER TABLE `dataset_entry` ADD `utmos_score` real;--> statement-breakpoint
ALTER TABLE `dataset_entry` ADD `wer_score` real;

View File

@@ -1,78 +1,9 @@
{ {
"version": "6", "version": "6",
"dialect": "sqlite", "dialect": "sqlite",
"id": "59929a2a-9ed3-4f73-8ef7-7af2b8d436b6", "id": "4e72f679-ef9b-47d5-a6fb-a1c45d5cbe74",
"prevId": "e6165973-fcea-4b1b-8004-60db966b8106", "prevId": "e6165973-fcea-4b1b-8004-60db966b8106",
"tables": { "tables": {
"annotation": {
"name": "annotation",
"columns": {
"id": {
"name": "id",
"type": "integer",
"primaryKey": true,
"notNull": true,
"autoincrement": true
},
"dataset_entry_id": {
"name": "dataset_entry_id",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"user_id": {
"name": "user_id",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"rating": {
"name": "rating",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"created_at": {
"name": "created_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": "(unixepoch())"
}
},
"indexes": {
"annotation_user_entry_idx": {
"name": "annotation_user_entry_idx",
"columns": [
"user_id",
"dataset_entry_id"
],
"isUnique": true
}
},
"foreignKeys": {
"annotation_dataset_entry_id_dataset_entry_id_fk": {
"name": "annotation_dataset_entry_id_dataset_entry_id_fk",
"tableFrom": "annotation",
"tableTo": "dataset_entry",
"columnsFrom": [
"dataset_entry_id"
],
"columnsTo": [
"id"
],
"onDelete": "no action",
"onUpdate": "no action"
}
},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"checkConstraints": {}
},
"account": { "account": {
"name": "account", "name": "account",
"columns": { "columns": {
@@ -513,6 +444,34 @@
"notNull": false, "notNull": false,
"autoincrement": false "autoincrement": false
}, },
"rms_value": {
"name": "rms_value",
"type": "real",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"longest_pause": {
"name": "longest_pause",
"type": "real",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"utmos_score": {
"name": "utmos_score",
"type": "real",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"wer_score": {
"name": "wer_score",
"type": "real",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"created_at": { "created_at": {
"name": "created_at", "name": "created_at",
"type": "integer", "type": "integer",

View File

@@ -47,15 +47,8 @@
{ {
"idx": 6, "idx": 6,
"version": "6", "version": "6",
"when": 1773406795226, "when": 1773406006276,
"tag": "0006_polite_moondragon", "tag": "0006_special_junta",
"breakpoints": true
},
{
"idx": 7,
"version": "6",
"when": 1773407494321,
"tag": "0007_flawless_wonder_man",
"breakpoints": true "breakpoints": true
} }
] ]

View File

@@ -53,6 +53,10 @@ export async function downloadFilteredEntries(
'id', 'id',
'audio_file', 'audio_file',
'duration_ms', 'duration_ms',
'rms_value',
'longest_pause',
'utmos_score',
'wer_score',
'utt_id', 'utt_id',
'text', 'text',
'speaker', 'speaker',
@@ -67,6 +71,10 @@ export async function downloadFilteredEntries(
`"${entry.externalId}"`, `"${entry.externalId}"`,
entry.fileName, entry.fileName,
entry.durationMs?.toString() || '', entry.durationMs?.toString() || '',
entry.rmsValue?.toString() || '',
entry.longestPause?.toString() || '',
entry.utmosScore?.toString() || '',
entry.werScore?.toString() || '',
entry.utteranceId || '', entry.utteranceId || '',
`"${(entry.utteranceText || '').replace(/"/g, '""')}"`, `"${(entry.utteranceText || '').replace(/"/g, '""')}"`,
entry.speakerId, entry.speakerId,

View File

@@ -2,9 +2,10 @@
import db from '@/lib/db'; import db from '@/lib/db';
import { dataset_entry } from '@/lib/model/dataset_entry'; import { dataset_entry } from '@/lib/model/dataset_entry';
import { count, eq, and, like } from 'drizzle-orm'; import { count, eq, and, like, asc, desc, getTableColumns } from 'drizzle-orm';
const ENTRIES_PER_PAGE = 20; const ENTRIES_PER_PAGE = 20;
const SORT_COLUMNS = getTableColumns(dataset_entry);
interface FilterParams { interface FilterParams {
speakerId?: string; speakerId?: string;
@@ -14,10 +15,16 @@ interface FilterParams {
utteranceId?: string; utteranceId?: string;
} }
interface SortParams {
sortBy?: keyof typeof SORT_COLUMNS;
sortOrder?: 'asc' | 'desc';
}
export async function getDatasetEntries( export async function getDatasetEntries(
datasetId: number, datasetId: number,
page: number = 1, page: number = 1,
filters?: FilterParams filters?: FilterParams,
sort?: SortParams
) { ) {
const offset = (page - 1) * ENTRIES_PER_PAGE; const offset = (page - 1) * ENTRIES_PER_PAGE;
@@ -46,10 +53,17 @@ export async function getDatasetEntries(
const whereClause = and(...conditions); const whereClause = and(...conditions);
// Build sort clause
const sortFn = sort?.sortOrder === 'desc' ? desc : asc;
const sortColumn = sort?.sortBy && sort.sortBy in SORT_COLUMNS
? sortFn(SORT_COLUMNS[sort.sortBy as keyof typeof SORT_COLUMNS])
: asc(dataset_entry.externalId);
const entries = await db const entries = await db
.select() .select()
.from(dataset_entry) .from(dataset_entry)
.where(whereClause) .where(whereClause)
.orderBy(sortColumn)
.limit(ENTRIES_PER_PAGE) .limit(ENTRIES_PER_PAGE)
.offset(offset); .offset(offset);

View File

@@ -2,6 +2,7 @@
import db from '@/lib/db'; import db from '@/lib/db';
import { dataset_entry } from '@/lib/model/dataset_entry'; import { dataset_entry } from '@/lib/model/dataset_entry';
import { parseCSVLine } from '@/lib/csv-parser';
import { revalidatePath } from 'next/cache'; import { revalidatePath } from 'next/cache';
import { eq } from 'drizzle-orm'; import { eq } from 'drizzle-orm';
import { writeFile, mkdir, readFile, rm } from 'fs/promises'; import { writeFile, mkdir, readFile, rm } from 'fs/promises';
@@ -10,38 +11,71 @@ import { existsSync } from 'fs';
import { requireAdmin } from '@/lib/auth'; import { requireAdmin } from '@/lib/auth';
/** /**
* CSV parser that handles quoted fields * Helper: Parse metadata.csv from extracted directory
* Returns structured data for both insert and update operations
*/ */
function parseCSVLine(line: string): string[] { async function parseMetadataFromDirectory(
const values: string[] = []; tempDir: string
let current = ''; ): Promise<{
let inQuotes = false; headers: string[];
rows: Record<string, string>[];
for (let i = 0; i < line.length; i++) { }> {
const char = line[i]; const metadataPath = join(tempDir, 'metadata.csv');
const nextChar = line[i + 1]; if (!existsSync(metadataPath)) {
throw new Error('metadata.csv not found in extracted files');
if (char === '"') {
if (inQuotes && nextChar === '"') {
// Escaped quote
current += '"';
i++;
} else {
// Toggle quote state
inQuotes = !inQuotes;
}
} else if (char === ',' && !inQuotes) {
// Field separator
values.push(current.trim());
current = '';
} else {
current += char;
}
} }
// Add the last field const metadataContent = await readFile(metadataPath, 'utf-8');
values.push(current.trim()); const lines = metadataContent.split('\n').filter((line) => line.trim());
return values;
if (lines.length < 2) {
throw new Error('metadata.csv is empty or invalid');
}
const headers = lines[0].split(',').map((h) => h.trim());
const rows: Record<string, string>[] = [];
for (let i = 1; i < lines.length; i++) {
const values = parseCSVLine(lines[i]);
if (values.length !== headers.length) {
continue; // Skip malformed lines
}
const row: Record<string, string> = {};
headers.forEach((header, index) => {
row[header] = values[index];
});
rows.push(row);
}
return { headers, rows };
}
/**
* Convert metadata row to dataset_entry insert object
*/
function rowToDatasetEntry(
row: Record<string, string>,
datasetId: number
): typeof dataset_entry.$inferInsert {
return {
datasetId,
externalId: row.id,
speakerId: row.speaker,
modelName: row.model,
utteranceId: row.utt_id,
utteranceText: row.text,
fileName: row.audio_file,
dialect: row.dialect,
iteration: parseInt(row.iteration, 10),
durationMs: row.duration_ms ? parseInt(row.duration_ms, 10) : undefined,
rmsValue: row.rms_value ? parseFloat(row.rms_value) : undefined,
longestPause: row.longest_pause ? parseFloat(row.longest_pause) : undefined,
utmosScore: row.utmos_score ? parseFloat(row.utmos_score) : undefined,
werScore: row.wer_score ? parseFloat(row.wer_score) : undefined,
};
} }
/** /**
@@ -73,47 +107,16 @@ export async function processDatasetEntries(
}; };
} }
// Read metadata.csv again // Parse metadata from directory
const metadataPath = join(tempDir, 'metadata.csv'); const { rows } = await parseMetadataFromDirectory(tempDir);
if (!existsSync(metadataPath)) {
return {
success: false,
error: 'metadata.csv not found in extracted files',
entriesCreated: 0,
};
}
const metadataContent = await readFile(metadataPath, 'utf-8');
const lines = metadataContent.split('\n').filter(line => line.trim());
if (lines.length < 2) {
return {
success: false,
error: 'metadata.csv is empty or invalid',
entriesCreated: 0,
};
}
// Parse CSV
const headers = lines[0].split(',').map(h => h.trim());
const entries: typeof dataset_entry.$inferInsert[] = []; const entries: typeof dataset_entry.$inferInsert[] = [];
// Create dataset directory // Create dataset directory
await mkdir(datasetDir, { recursive: true }); await mkdir(datasetDir, { recursive: true });
// Process each row // Process each row
for (let i = 1; i < lines.length; i++) { for (const row of rows) {
const values = parseCSVLine(lines[i]);
if (values.length !== headers.length) {
continue; // Skip malformed lines
}
const row: Record<string, string> = {};
headers.forEach((header, index) => {
row[header] = values[index];
});
const audioFile = row.audio_file as string; const audioFile = row.audio_file as string;
const relativePath = audioFile.replace(/\\/g, '/'); const relativePath = audioFile.replace(/\\/g, '/');
const audioPath = join(tempDir, relativePath); const audioPath = join(tempDir, relativePath);
@@ -126,18 +129,7 @@ export async function processDatasetEntries(
await writeFile(destPath, audioBuffer); await writeFile(destPath, audioBuffer);
entries.push({ entries.push(rowToDatasetEntry(row, datasetId));
datasetId,
externalId: row.id,
speakerId: row.speaker,
modelName: row.model,
utteranceId: row.utt_id,
utteranceText: row.text,
fileName: relativePath,
dialect: row.dialect,
iteration: parseInt(row.iteration, 10),
durationMs: row.duration_ms ? parseInt(row.duration_ms, 10) : undefined,
});
} }
} }
@@ -197,3 +189,116 @@ export async function processDatasetEntries(
} }
} }
} }
/**
* Update metadata for existing dataset entries
* Uses the same metadata.csv parsing as processDatasetEntries
* Matches entries by externalId and updates their metadata fields
*/
export async function updateDatasetEntriesMetadata(
datasetId: number,
tempDir: string
) {
try {
const result = await requireAdmin();
if (!result.authenticated || !result.admin) {
return {
success: false,
error: 'Unauthorized',
entriesUpdated: 0,
};
}
// Validate temp directory exists
if (!existsSync(tempDir)) {
return {
success: false,
error: 'Temporary extraction directory not found',
entriesUpdated: 0,
};
}
// Parse metadata from directory (reusing same logic as processDatasetEntries)
const { rows } = await parseMetadataFromDirectory(tempDir);
if (rows.length === 0) {
return {
success: false,
error: 'No valid entries found in metadata.csv',
entriesUpdated: 0,
};
}
// Get all existing entries for this dataset
const existingEntries = await db
.select()
.from(dataset_entry)
.where(eq(dataset_entry.datasetId, datasetId));
const existingByExternalId = new Map(
existingEntries.map(entry => [entry.externalId, entry])
);
let entriesUpdated = 0;
let entriesNotFound = 0;
// Update each row that has a matching entry
for (const row of rows) {
const externalId = row.id;
const existingEntry = existingByExternalId.get(externalId);
if (!existingEntry) {
entriesNotFound++;
continue;
}
// Update only the metadata fields, don't change the ID or dataset reference
await db
.update(dataset_entry)
.set({
speakerId: row.speaker,
modelName: row.model,
utteranceId: row.utt_id,
utteranceText: row.text,
dialect: row.dialect,
iteration: parseInt(row.iteration, 10),
durationMs: row.duration_ms ? parseInt(row.duration_ms, 10) : undefined,
rmsValue: row.rms_value ? parseFloat(row.rms_value) : undefined,
longestPause: row.longest_pause ? parseFloat(row.longest_pause) : undefined,
utmosScore: row.utmos_score ? parseFloat(row.utmos_score) : undefined,
werScore: row.wer_score ? parseFloat(row.wer_score) : undefined,
})
.where(eq(dataset_entry.id, existingEntry.id));
entriesUpdated++;
}
revalidatePath(`/admin/datasets/${datasetId}`);
return {
success: true,
entriesUpdated,
entriesNotFound,
message: entriesUpdated > 0
? `Successfully updated ${entriesUpdated} entries${entriesNotFound > 0 ? ` (${entriesNotFound} not found)` : ''}.`
: `No entries were updated (${entriesNotFound} not found).`,
};
} catch (error) {
console.error('Error updating dataset entries metadata:', error);
const errorMessage = error instanceof Error ? error.message : 'Failed to update dataset entries metadata';
return {
success: false,
error: errorMessage,
entriesUpdated: 0,
};
} finally {
// Clean up temp directory
try {
if (existsSync(tempDir)) {
await rm(tempDir, { recursive: true, force: true });
}
} catch (cleanupError) {
console.error('Error cleaning up temp directory:', cleanupError);
}
}
}

View File

@@ -36,7 +36,7 @@ export default async function DatasetEntryPage({ params }: DatasetEntryPageProps
if (entries.length === 0 || entries[0].datasetId !== datasetIdNum) { if (entries.length === 0 || entries[0].datasetId !== datasetIdNum) {
return ( return (
<div className="min-h-screen"> <div className="min-h-screen">
<div className="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8 py-8"> <div className="px-4 sm:px-6 lg:px-8 py-8">
<Link <Link
href={`/admin/datasets/${datasetId}`} href={`/admin/datasets/${datasetId}`}
className="text-blue-500 hover:text-blue-600 mb-6 inline-block" className="text-blue-500 hover:text-blue-600 mb-6 inline-block"
@@ -55,7 +55,7 @@ export default async function DatasetEntryPage({ params }: DatasetEntryPageProps
return ( return (
<div> <div>
<div className="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8 py-8"> <div className="px-4 sm:px-6 lg:px-8 py-8">
<Link <Link
href={`/admin/datasets/${datasetId}`} href={`/admin/datasets/${datasetId}`}
className="text-blue-500 hover:text-blue-600 mb-6 inline-block" className="text-blue-500 hover:text-blue-600 mb-6 inline-block"
@@ -122,6 +122,26 @@ export default async function DatasetEntryPage({ params }: DatasetEntryPageProps
<p className="text-lg font-semibold">{entry.durationMs?.toLocaleString() || '-'}</p> <p className="text-lg font-semibold">{entry.durationMs?.toLocaleString() || '-'}</p>
</div> </div>
<div className="bg-white border border-gray-200 p-4 rounded-lg">
<p className="text-sm text-gray-600 mb-1">RMS Value</p>
<p className="text-lg font-semibold">{entry.rmsValue?.toFixed(3) || '-'}</p>
</div>
<div className="bg-white border border-gray-200 p-4 rounded-lg">
<p className="text-sm text-gray-600 mb-1">Longest Pause (s)</p>
<p className="text-lg font-semibold">{entry.longestPause?.toFixed(3) || '-'}</p>
</div>
<div className="bg-white border border-gray-200 p-4 rounded-lg">
<p className="text-sm text-gray-600 mb-1">UTMOS Score</p>
<p className="text-lg font-semibold">{entry.utmosScore?.toFixed(3) || '-'}</p>
</div>
<div className="bg-white border border-gray-200 p-4 rounded-lg">
<p className="text-sm text-gray-600 mb-1">WER Score</p>
<p className="text-lg font-semibold">{entry.werScore?.toFixed(3) || '-'}</p>
</div>
<div className="bg-white border border-gray-200 p-4 rounded-lg"> <div className="bg-white border border-gray-200 p-4 rounded-lg">
<p className="text-sm text-gray-600 mb-1">Created</p> <p className="text-sm text-gray-600 mb-1">Created</p>
<p className="text-lg font-semibold"> <p className="text-lg font-semibold">

View File

@@ -5,6 +5,7 @@ import { eq } from 'drizzle-orm';
import EditableDatasetHeader from '@/components/EditableDatasetHeader'; import EditableDatasetHeader from '@/components/EditableDatasetHeader';
import DatasetEntriesList from '@/components/DatasetEntriesList'; import DatasetEntriesList from '@/components/DatasetEntriesList';
import UploadDatasetEntriesModal from '@/components/UploadDatasetEntriesModal'; import UploadDatasetEntriesModal from '@/components/UploadDatasetEntriesModal';
import UpdateDatasetMetadataModal from '@/components/UpdateDatasetMetadataModal';
import RemoveAllEntriesButton from '@/components/RemoveAllEntriesButton'; import RemoveAllEntriesButton from '@/components/RemoveAllEntriesButton';
import DeleteDatasetButton from '@/components/DeleteDatasetButton'; import DeleteDatasetButton from '@/components/DeleteDatasetButton';
import { requireAdmin } from '@/lib/auth'; import { requireAdmin } from '@/lib/auth';
@@ -38,7 +39,7 @@ export default async function DatasetPage({ params }: DatasetPageProps) {
if (datasets.length === 0) { if (datasets.length === 0) {
return ( return (
<div className="min-h-screen"> <div className="min-h-screen">
<div className="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8 py-8"> <div className="px-4 sm:px-6 lg:px-8 py-8">
<Link <Link
href="/admin" href="/admin"
className="text-blue-500 hover:text-blue-600 mb-6 inline-block" className="text-blue-500 hover:text-blue-600 mb-6 inline-block"
@@ -57,7 +58,7 @@ export default async function DatasetPage({ params }: DatasetPageProps) {
return ( return (
<div> <div>
<div className="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8 py-8"> <div className="px-4 sm:px-6 lg:px-8 py-8">
<Link <Link
href="/admin" href="/admin"
className="text-blue-500 hover:text-blue-600 mb-6 inline-block" className="text-blue-500 hover:text-blue-600 mb-6 inline-block"
@@ -69,6 +70,7 @@ export default async function DatasetPage({ params }: DatasetPageProps) {
<div className="flex gap-3 mb-6"> <div className="flex gap-3 mb-6">
<UploadDatasetEntriesModal datasetId={datasetId} /> <UploadDatasetEntriesModal datasetId={datasetId} />
<UpdateDatasetMetadataModal datasetId={datasetId} />
<RemoveAllEntriesButton datasetId={datasetId} /> <RemoveAllEntriesButton datasetId={datasetId} />
<DeleteDatasetButton datasetId={datasetId} datasetName={ds.name} /> <DeleteDatasetButton datasetId={datasetId} datasetName={ds.name} />
</div> </div>

View File

@@ -18,7 +18,7 @@ export default async function AdminPage() {
} }
return ( return (
<div className="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8 py-8"> <div className="px-4 sm:px-6 lg:px-8 py-8">
<div className="mb-8"> <div className="mb-8">
<h1 className="text-3xl font-bold mb-4">Admin Panel</h1> <h1 className="text-3xl font-bold mb-4">Admin Panel</h1>
<div className="flex gap-4"> <div className="flex gap-4">

View File

@@ -18,7 +18,7 @@ export default async function UsersPage() {
const users = await getAllUsers(); const users = await getAllUsers();
return ( return (
<div className="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8 py-8"> <div className="px-4 sm:px-6 lg:px-8 py-8">
<div className="mb-6"> <div className="mb-6">
<Link <Link
href="/admin" href="/admin"

View File

@@ -0,0 +1,74 @@
import { writeFile, mkdir, rm } from 'fs/promises';
import { join } from 'path';
import { existsSync } from 'fs';
import { NextRequest, NextResponse } from 'next/server';
import { requireAdmin } from '@/lib/auth';
export async function POST(request: NextRequest) {
try {
const result = await requireAdmin();
if (!result.authenticated || !result.admin) {
return NextResponse.json(
{ error: 'Unauthorized' },
{ status: 401 }
);
}
const formData = await request.formData();
const file = formData.get('file') as File;
const datasetId = formData.get('datasetId') as string;
if (!file) {
return NextResponse.json(
{ error: 'No file provided' },
{ status: 400 }
);
}
if (!file.name.endsWith('.csv')) {
return NextResponse.json(
{ error: 'File must be a CSV file' },
{ status: 400 }
);
}
const tempDir = join(process.cwd(), 'tmp', `update-metadata-${Date.now()}`);
try {
// Create temp directory
await mkdir(tempDir, { recursive: true });
// Save uploaded CSV file as metadata.csv
const arrayBuffer = await file.arrayBuffer();
const buffer = Buffer.from(arrayBuffer);
const metadataPath = join(tempDir, 'metadata.csv');
await writeFile(metadataPath, buffer);
return NextResponse.json({
success: true,
tempDir,
message: `Successfully uploaded metadata.csv. Ready for processing.`,
});
} catch (error) {
// Clean up on error
try {
if (existsSync(tempDir)) {
await rm(tempDir, { recursive: true, force: true });
}
} catch (cleanupError) {
console.error('Error cleaning up temp directory:', cleanupError);
}
throw error;
}
} catch (error) {
console.error('Error uploading metadata CSV:', error);
const errorMsg =
error instanceof Error ? error.message : 'Failed to upload metadata CSV';
return NextResponse.json(
{ error: errorMsg },
{ status: 500 }
);
}
}

View File

@@ -13,7 +13,7 @@ export default function RootLayout({
<body className="flex flex-col min-h-screen"> <body className="flex flex-col min-h-screen">
<AudioProvider> <AudioProvider>
<Header /> <Header />
<main className="flex-grow mt-4 mb-4 mr-8 ml-8"> <main className="flex-grow">
{children} {children}
</main> </main>
<Footer /> <Footer />

View File

@@ -22,6 +22,11 @@ interface DatasetEntry {
fileName: string; fileName: string;
dialect: string; dialect: string;
iteration: number; iteration: number;
durationMs: number | null;
rmsValue: number | null;
longestPause: number | null;
utmosScore: number | null;
werScore: number | null;
createdAt: Date; createdAt: Date;
updatedAt: Date; updatedAt: Date;
} }
@@ -62,6 +67,10 @@ export default function DatasetEntriesList({
utteranceIds: [], utteranceIds: [],
}); });
// Sort state
const [sortBy, setSortBy] = useState<keyof DatasetEntry>('externalId');
const [sortOrder, setSortOrder] = useState<'asc' | 'desc'>('asc');
// Load filter options on mount // Load filter options on mount
useEffect(() => { useEffect(() => {
async function loadFilterOptions() { async function loadFilterOptions() {
@@ -78,7 +87,7 @@ export default function DatasetEntriesList({
useEffect(() => { useEffect(() => {
setPage(1); setPage(1);
loadEntries(1); loadEntries(1);
}, [filters]); }, [filters, sortBy, sortOrder]);
async function loadEntries(pageNum: number = page) { async function loadEntries(pageNum: number = page) {
setLoading(true); setLoading(true);
@@ -91,7 +100,12 @@ export default function DatasetEntriesList({
utteranceId: filters.utteranceId || undefined, utteranceId: filters.utteranceId || undefined,
}; };
const result = await getDatasetEntries(datasetId, pageNum, filterParams); const sortParams = {
sortBy: sortBy as keyof DatasetEntry,
sortOrder: sortOrder,
};
const result = await getDatasetEntries(datasetId, pageNum, filterParams, sortParams);
if (pageNum === 1) { if (pageNum === 1) {
setEntries(result.entries); setEntries(result.entries);
} else { } else {
@@ -118,7 +132,12 @@ export default function DatasetEntriesList({
utteranceId: filters.utteranceId || undefined, utteranceId: filters.utteranceId || undefined,
}; };
const result = await getDatasetEntries(datasetId, page + 1, filterParams); const sortParams = {
sortBy: sortBy as keyof DatasetEntry,
sortOrder: sortOrder,
};
const result = await getDatasetEntries(datasetId, page + 1, filterParams, sortParams);
setEntries((prev) => [...prev, ...result.entries]); setEntries((prev) => [...prev, ...result.entries]);
setHasMore(result.hasMore); setHasMore(result.hasMore);
setTotal(result.total); setTotal(result.total);
@@ -147,6 +166,17 @@ export default function DatasetEntriesList({
}); });
} }
function handleSort(column: keyof DatasetEntry) {
if (sortBy === column) {
// Toggle sort order if clicking the same column
setSortOrder(sortOrder === 'asc' ? 'desc' : 'asc');
} else {
// Set new column to sort by, default to ascending
setSortBy(column);
setSortOrder('asc');
}
}
async function handleDownloadFiltered() { async function handleDownloadFiltered() {
setDownloading(true); setDownloading(true);
try { try {
@@ -410,13 +440,39 @@ export default function DatasetEntriesList({
<thead> <thead>
<tr className="border-b border-gray-200"> <tr className="border-b border-gray-200">
<th className="text-left py-3 px-4 font-semibold text-gray-700">Play</th> <th className="text-left py-3 px-4 font-semibold text-gray-700">Play</th>
<th className="text-left py-3 px-4 font-semibold text-gray-700">External ID</th> <th className="text-left py-3 px-4 font-semibold text-gray-700 cursor-pointer hover:bg-gray-100" onClick={() => handleSort('externalId')}>
<th className="text-left py-3 px-4 font-semibold text-gray-700">Speaker ID</th> External ID {sortBy === 'externalId' && (sortOrder === 'asc' ? '↑' : '↓')}
<th className="text-left py-3 px-4 font-semibold text-gray-700">Model Name</th> </th>
<th className="text-left py-3 px-4 font-semibold text-gray-700">Utterance ID</th> <th className="text-left py-3 px-4 font-semibold text-gray-700 cursor-pointer hover:bg-gray-100" onClick={() => handleSort('speakerId')}>
<th className="text-left py-3 px-4 font-semibold text-gray-700">Dialect</th> Speaker ID {sortBy === 'speakerId' && (sortOrder === 'asc' ? '↑' : '↓')}
<th className="text-left py-3 px-4 font-semibold text-gray-700">File Name</th> </th>
<th className="text-left py-3 px-4 font-semibold text-gray-700">Iteration</th> <th className="text-left py-3 px-4 font-semibold text-gray-700 cursor-pointer hover:bg-gray-100" onClick={() => handleSort('modelName')}>
Model Name {sortBy === 'modelName' && (sortOrder === 'asc' ? '↑' : '↓')}
</th>
<th className="text-left py-3 px-4 font-semibold text-gray-700 cursor-pointer hover:bg-gray-100" onClick={() => handleSort('utteranceId')}>
Utterance ID {sortBy === 'utteranceId' && (sortOrder === 'asc' ? '↑' : '↓')}
</th>
<th className="text-left py-3 px-4 font-semibold text-gray-700 cursor-pointer hover:bg-gray-100" onClick={() => handleSort('dialect')}>
Dialect {sortBy === 'dialect' && (sortOrder === 'asc' ? '↑' : '↓')}
</th>
<th className="text-left py-3 px-4 font-semibold text-gray-700 cursor-pointer hover:bg-gray-100" onClick={() => handleSort('iteration')}>
Iteration {sortBy === 'iteration' && (sortOrder === 'asc' ? '↑' : '↓')}
</th>
<th className="text-left py-3 px-4 font-semibold text-gray-700 cursor-pointer hover:bg-gray-100" onClick={() => handleSort('durationMs')}>
Duration (ms) {sortBy === 'durationMs' && (sortOrder === 'asc' ? '↑' : '↓')}
</th>
<th className="text-left py-3 px-4 font-semibold text-gray-700 cursor-pointer hover:bg-gray-100" onClick={() => handleSort('rmsValue')}>
RMS Value {sortBy === 'rmsValue' && (sortOrder === 'asc' ? '↑' : '↓')}
</th>
<th className="text-left py-3 px-4 font-semibold text-gray-700 cursor-pointer hover:bg-gray-100" onClick={() => handleSort('longestPause')}>
Longest Pause (s) {sortBy === 'longestPause' && (sortOrder === 'asc' ? '↑' : '↓')}
</th>
<th className="text-left py-3 px-4 font-semibold text-gray-700 cursor-pointer hover:bg-gray-100" onClick={() => handleSort('utmosScore')}>
UTMOS Score {sortBy === 'utmosScore' && (sortOrder === 'asc' ? '↑' : '↓')}
</th>
<th className="text-left py-3 px-4 font-semibold text-gray-700 cursor-pointer hover:bg-gray-100" onClick={() => handleSort('werScore')}>
WER Score {sortBy === 'werScore' && (sortOrder === 'asc' ? '↑' : '↓')}
</th>
</tr> </tr>
</thead> </thead>
<tbody> <tbody>
@@ -435,10 +491,14 @@ export default function DatasetEntriesList({
</td> </td>
<td className="py-3 px-4">{entry.speakerId}</td> <td className="py-3 px-4">{entry.speakerId}</td>
<td className="py-3 px-4">{entry.modelName}</td> <td className="py-3 px-4">{entry.modelName}</td>
<td className="py-3 px-4 truncate">{entry.utteranceId || '-'}</td> <td className="py-3 px-4">{entry.utteranceId || '-'}</td>
<td className="py-3 px-4">{entry.dialect}</td> <td className="py-3 px-4">{entry.dialect}</td>
<td className="py-3 px-4 truncate">{entry.fileName}</td>
<td className="py-3 px-4">{entry.iteration}</td> <td className="py-3 px-4">{entry.iteration}</td>
<td className="py-3 px-4">{entry.durationMs?.toLocaleString() || '-'}</td>
<td className="py-3 px-4">{entry.rmsValue?.toFixed(3) || '-'}</td>
<td className="py-3 px-4">{entry.longestPause?.toFixed(3) || '-'}</td>
<td className="py-3 px-4">{entry.utmosScore?.toFixed(3) || '-'}</td>
<td className="py-3 px-4">{entry.werScore?.toFixed(3) || '-'}</td>
</tr> </tr>
))} ))}
</tbody> </tbody>

View File

@@ -3,7 +3,7 @@ export function Footer() {
return ( return (
<footer className="bg-gray-800 text-gray-300 mt-12"> <footer className="bg-gray-800 text-gray-300 mt-12">
<div className="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8 py-8"> <div className="px-4 sm:px-6 lg:px-8 py-8">
<div className="grid grid-cols-1 md:grid-cols-3 gap-8 mb-8"> <div className="grid grid-cols-1 md:grid-cols-3 gap-8 mb-8">
<div> <div>
<h3 className="text-white font-semibold mb-4">Über das Projekt</h3> <h3 className="text-white font-semibold mb-4">Über das Projekt</h3>

View File

@@ -16,7 +16,7 @@ export function Header() {
return ( return (
<header className="sticky top-0 z-50 bg-gradient-to-r from-blue-600 to-blue-700 text-white shadow-lg"> <header className="sticky top-0 z-50 bg-gradient-to-r from-blue-600 to-blue-700 text-white shadow-lg">
<div className="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8"> <div className="px-4 sm:px-6 lg:px-8">
<div className="flex justify-between items-center py-5"> <div className="flex justify-between items-center py-5">
<Link href="/" className="text-2xl font-bold hover:text-blue-100 transition-colors"> <Link href="/" className="text-2xl font-bold hover:text-blue-100 transition-colors">
TTS Dialektannotations-Plattform TTS Dialektannotations-Plattform

View File

@@ -0,0 +1,365 @@
'use client';
import { useState } from 'react';
import { updateDatasetEntriesMetadata } from '@/app/actions/upload';
import Modal from '@/components/Modal';
interface UpdateDatasetMetadataModalProps {
datasetId: number;
}
export default function UpdateDatasetMetadataModal({
datasetId,
}: UpdateDatasetMetadataModalProps) {
const [isOpen, setIsOpen] = useState(false);
const [files, setFiles] = useState<File[]>([]);
const [loading, setLoading] = useState(false);
const [error, setError] = useState<string | null>(null);
const [success, setSuccess] = useState(false);
const [uploadQueue, setUploadQueue] = useState<Array<{ file: File; status: 'pending' | 'uploading' | 'completed' | 'failed'; message?: string; progress: number }>>([]);
const [status, setStatus] = useState<string>('');
function handleFileChange(e: React.ChangeEvent<HTMLInputElement>) {
const selectedFiles = e.target.files ? Array.from(e.target.files) : [];
if (selectedFiles.length > 0) {
setFiles(selectedFiles);
setError(null);
// Initialize queue with pending status and 0 progress
setUploadQueue(selectedFiles.map(file => ({ file, status: 'pending', progress: 0 })));
}
}
async function handleSubmit(e: React.FormEvent) {
e.preventDefault();
setError(null);
setSuccess(false);
if (files.length === 0) {
setError('Please select at least one CSV file');
return;
}
setLoading(true);
try {
// Process each file in the queue sequentially
const newQueue = [...uploadQueue];
for (let i = 0; i < newQueue.length; i++) {
const item = newQueue[i];
item.status = 'uploading';
item.progress = 0;
setUploadQueue([...newQueue]);
setStatus(`Processing file ${i + 1} of ${newQueue.length}: ${item.file.name}`);
try {
// Upload with progress tracking
const uploadResult = await uploadFileWithProgress(item.file, i, newQueue);
// Update the metadata
const updateResult = await updateDatasetEntriesMetadata(datasetId, uploadResult.tempDir);
if (!updateResult.success) {
item.status = 'failed';
item.progress = 0;
item.message = `${updateResult.error || 'Update failed'}`;
} else {
item.status = 'completed';
item.progress = 100;
item.message = `✓ Updated (${updateResult.entriesUpdated} entries)`;
}
} catch (err) {
item.status = 'failed';
item.progress = 0;
item.message = `${err instanceof Error ? err.message : 'Network error'}`;
}
setUploadQueue([...newQueue]);
}
const failedCount = newQueue.filter(item => item.status === 'failed').length;
const completedCount = newQueue.filter(item => item.status === 'completed').length;
if (failedCount === 0) {
setSuccess(true);
setStatus(`All files processed! (${completedCount}/${newQueue.length} completed)`);
} else if (completedCount > 0) {
setStatus(`Completed with errors: ${completedCount} succeeded, ${failedCount} failed`);
} else {
setError(`All files failed to process`);
}
setFiles([]);
if (failedCount === 0) {
setTimeout(() => {
setSuccess(false);
setIsOpen(false);
setStatus('');
setUploadQueue([]);
window.location.reload();
}, 2500);
}
} catch (err) {
const errorMsg = err instanceof Error ? err.message : 'Failed to process updates';
setError(errorMsg);
setStatus('');
} finally {
setLoading(false);
}
}
function uploadFileWithProgress(
file: File,
index: number,
queue: Array<{ file: File; status: 'pending' | 'uploading' | 'completed' | 'failed'; progress: number; message?: string }>
): Promise<{ tempDir: string }> {
const MAX_RETRIES = 3;
return (async () => {
let lastError: Error | null = null;
for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) {
try {
const result = await uploadAttempt(file, index, queue, attempt, MAX_RETRIES);
return result;
} catch (error) {
lastError = error instanceof Error ? error : new Error('Unknown error');
if (attempt < MAX_RETRIES) {
// Exponential backoff: 1s, 2s, 4s
const delay = Math.pow(2, attempt - 1) * 1000;
queue[index].message = `Retrying in ${delay / 1000}s... (attempt ${attempt}/${MAX_RETRIES})`;
setUploadQueue([...queue]);
await new Promise(resolve => setTimeout(resolve, delay));
queue[index].progress = 0; // Reset progress for next attempt
setUploadQueue([...queue]);
}
}
}
throw lastError || new Error('Upload failed after all retries');
})();
}
function uploadAttempt(
file: File,
index: number,
queue: Array<{ file: File; status: 'pending' | 'uploading' | 'completed' | 'failed'; progress: number; message?: string }>,
attempt: number,
maxRetries: number
): Promise<{ tempDir: string }> {
return new Promise((resolve, reject) => {
const formData = new FormData();
formData.append('file', file);
formData.append('datasetId', datasetId.toString());
const xhr = new XMLHttpRequest();
// Track upload progress
xhr.upload.addEventListener('progress', (event) => {
if (event.lengthComputable) {
const progress = Math.round((event.loaded / event.total) * 100);
queue[index].progress = progress;
if (attempt > 1) {
queue[index].message = `Uploading (retry ${attempt}/${maxRetries})`;
}
setUploadQueue([...queue]);
}
});
xhr.addEventListener('load', () => {
if (xhr.status === 200) {
try {
const result = JSON.parse(xhr.responseText);
if (result.success) {
resolve(result);
} else {
reject(new Error(result.error || 'Upload failed'));
}
} catch (error) {
reject(new Error('Failed to parse response'));
}
} else {
try {
const error = JSON.parse(xhr.responseText);
reject(new Error(error.error || `Upload failed with status ${xhr.status}`));
} catch {
reject(new Error(`Upload failed with status ${xhr.status}`));
}
}
});
xhr.addEventListener('error', () => {
reject(new Error('Network error'));
});
xhr.addEventListener('abort', () => {
reject(new Error('Upload aborted'));
});
xhr.open('POST', '/api/update-metadata');
xhr.send(formData);
});
}
function handleClose() {
if (!loading) {
setIsOpen(false);
setFiles([]);
setError(null);
setSuccess(false);
setStatus('');
setUploadQueue([]);
}
}
return (
<>
<button
onClick={() => setIsOpen(true)}
className="mb-6 px-4 py-2 bg-blue-500 text-white rounded-md hover:bg-blue-600 transition-colors"
>
Update Metadata
</button>
<Modal
isOpen={isOpen}
onClose={handleClose}
title="Update Dataset Metadata"
actions={[
{
label: 'Cancel',
onClick: handleClose,
variant: 'secondary',
disabled: loading,
},
{
label: loading ? `Processing ${uploadQueue.filter(f => f.status !== 'pending').length}/${uploadQueue.length}...` : 'Upload & Update',
onClick: () => {
const form = document.getElementById(
'updateMetadataForm'
) as HTMLFormElement;
form?.dispatchEvent(new Event('submit', { bubbles: true }));
},
variant: 'primary',
disabled: loading || files.length === 0,
},
]}
>
<form id="updateMetadataForm" onSubmit={handleSubmit} className="space-y-4">
<div>
<label htmlFor="file" className="block text-sm font-medium mb-2">
CSV File
</label>
<div className="border-2 border-dashed border-gray-300 rounded-lg p-6 text-center">
<input
id="file"
type="file"
accept=".csv"
multiple
onChange={handleFileChange}
disabled={loading}
className="hidden"
/>
<label htmlFor="file" className="cursor-pointer block">
{files.length > 0 ? (
<div>
<p className="text-sm font-medium text-blue-600">
{files.length} file{files.length !== 1 ? 's' : ''} selected
</p>
<div className="mt-2 text-xs text-gray-600 max-h-24 overflow-y-auto">
{files.map((f, idx) => (
<div key={idx} className="py-1">
{f.name} ({(f.size / 1024).toFixed(2)} KB)
</div>
))}
</div>
</div>
) : (
<div>
<p className="text-sm font-medium text-gray-700">
Click to select or drag and drop
</p>
<p className="text-xs text-gray-500 mt-1">CSV files only (multiple allowed)</p>
</div>
)}
</label>
</div>
</div>
<div className="bg-blue-50 border border-blue-200 rounded-lg p-4 text-sm text-blue-800">
<p className="font-semibold mb-2">CSV file format:</p>
<ul className="list-disc list-inside space-y-1 text-xs">
<li>
Columns: <code className="bg-blue-100 px-1 rounded">id, speaker, model, utt_id, text, dialect, iteration, duration_ms, rms_value, longest_pause, utmos_score, wer_score</code>
</li>
<li>Entry IDs in the CSV must match existing dataset entries to update</li>
</ul>
</div>
{uploadQueue.length > 0 && (
<div className="bg-gray-50 border border-gray-200 rounded-lg p-4 space-y-3">
<p className="font-semibold text-sm text-gray-700">Upload Queue:</p>
<div className="space-y-2 max-h-48 overflow-y-auto">
{uploadQueue.map((item, idx) => (
<div key={idx} className="space-y-1">
<div className="flex items-center gap-2 text-xs">
<span className={`font-medium ${
item.status === 'completed' ? 'text-green-600' :
item.status === 'failed' ? 'text-red-600' :
item.status === 'uploading' ? 'text-blue-600' :
'text-gray-600'
}`}>
{item.status === 'completed' ? '✓' :
item.status === 'failed' ? '✗' :
item.status === 'uploading' ? '⟳' :
'○'}
</span>
<span className="flex-1 truncate">{item.file.name}</span>
{item.progress > 0 && item.progress < 100 && (
<span className="text-gray-500">{item.progress}%</span>
)}
{item.message && <span className="text-gray-500 text-xs">{item.message}</span>}
</div>
{item.status === 'uploading' && (
<div className="w-full bg-gray-200 rounded-full h-2 overflow-hidden">
<div
className="bg-blue-500 h-full transition-all duration-300"
style={{ width: `${item.progress}%` }}
/>
</div>
)}
{item.status === 'completed' && (
<div className="w-full bg-green-200 rounded-full h-2 overflow-hidden">
<div className="bg-green-500 h-full w-full" />
</div>
)}
{item.status === 'failed' && (
<div className="w-full bg-red-200 rounded-full h-2 overflow-hidden">
<div className="bg-red-500 h-full w-full" />
</div>
)}
</div>
))}
</div>
</div>
)}
{error && (
<div className="p-3 bg-red-100 border border-red-400 text-red-700 rounded text-sm">
{error}
</div>
)}
{status && (
<div className={`p-3 rounded text-sm ${success ? 'bg-green-100 border border-green-400 text-green-700' : 'bg-blue-100 border border-blue-400 text-blue-700'}`}>
{status}
</div>
)}
</form>
</Modal>
</>
);
}

34
src/lib/csv-parser.ts Normal file
View File

@@ -0,0 +1,34 @@
/**
* CSV parser that handles quoted fields
*/
export function parseCSVLine(line: string): string[] {
const values: string[] = [];
let current = '';
let inQuotes = false;
for (let i = 0; i < line.length; i++) {
const char = line[i];
const nextChar = line[i + 1];
if (char === '"') {
if (inQuotes && nextChar === '"') {
// Escaped quote
current += '"';
i++;
} else {
// Toggle quote state
inQuotes = !inQuotes;
}
} else if (char === ',' && !inQuotes) {
// Field separator
values.push(current.trim());
current = '';
} else {
current += char;
}
}
// Add the last field
values.push(current.trim());
return values;
}

View File

@@ -16,6 +16,10 @@ export const dataset_entry = sqliteTable('dataset_entry', {
dialect: text('dialect').notNull(), dialect: text('dialect').notNull(),
iteration: integer('iteration').notNull(), iteration: integer('iteration').notNull(),
durationMs: integer('duration_ms'), durationMs: integer('duration_ms'),
rmsValue: real('rms_value'),
longestPause: real('longest_pause'),
utmosScore: real('utmos_score'),
werScore: real('wer_score'),
createdAt: integer('created_at', { mode: 'timestamp' }) createdAt: integer('created_at', { mode: 'timestamp' })
.notNull() .notNull()
.default(sql`(unixepoch())`), .default(sql`(unixepoch())`),