Merge branch 'main' into prototyp-2-fabio
This commit is contained in:
4
drizzle/0006_special_junta.sql
Normal file
4
drizzle/0006_special_junta.sql
Normal file
@@ -0,0 +1,4 @@
|
|||||||
|
ALTER TABLE `dataset_entry` ADD `rms_value` real;--> statement-breakpoint
|
||||||
|
ALTER TABLE `dataset_entry` ADD `longest_pause` real;--> statement-breakpoint
|
||||||
|
ALTER TABLE `dataset_entry` ADD `utmos_score` real;--> statement-breakpoint
|
||||||
|
ALTER TABLE `dataset_entry` ADD `wer_score` real;
|
||||||
@@ -1,78 +1,9 @@
|
|||||||
{
|
{
|
||||||
"version": "6",
|
"version": "6",
|
||||||
"dialect": "sqlite",
|
"dialect": "sqlite",
|
||||||
"id": "59929a2a-9ed3-4f73-8ef7-7af2b8d436b6",
|
"id": "4e72f679-ef9b-47d5-a6fb-a1c45d5cbe74",
|
||||||
"prevId": "e6165973-fcea-4b1b-8004-60db966b8106",
|
"prevId": "e6165973-fcea-4b1b-8004-60db966b8106",
|
||||||
"tables": {
|
"tables": {
|
||||||
"annotation": {
|
|
||||||
"name": "annotation",
|
|
||||||
"columns": {
|
|
||||||
"id": {
|
|
||||||
"name": "id",
|
|
||||||
"type": "integer",
|
|
||||||
"primaryKey": true,
|
|
||||||
"notNull": true,
|
|
||||||
"autoincrement": true
|
|
||||||
},
|
|
||||||
"dataset_entry_id": {
|
|
||||||
"name": "dataset_entry_id",
|
|
||||||
"type": "integer",
|
|
||||||
"primaryKey": false,
|
|
||||||
"notNull": true,
|
|
||||||
"autoincrement": false
|
|
||||||
},
|
|
||||||
"user_id": {
|
|
||||||
"name": "user_id",
|
|
||||||
"type": "text",
|
|
||||||
"primaryKey": false,
|
|
||||||
"notNull": true,
|
|
||||||
"autoincrement": false
|
|
||||||
},
|
|
||||||
"rating": {
|
|
||||||
"name": "rating",
|
|
||||||
"type": "integer",
|
|
||||||
"primaryKey": false,
|
|
||||||
"notNull": true,
|
|
||||||
"autoincrement": false
|
|
||||||
},
|
|
||||||
"created_at": {
|
|
||||||
"name": "created_at",
|
|
||||||
"type": "integer",
|
|
||||||
"primaryKey": false,
|
|
||||||
"notNull": true,
|
|
||||||
"autoincrement": false,
|
|
||||||
"default": "(unixepoch())"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"indexes": {
|
|
||||||
"annotation_user_entry_idx": {
|
|
||||||
"name": "annotation_user_entry_idx",
|
|
||||||
"columns": [
|
|
||||||
"user_id",
|
|
||||||
"dataset_entry_id"
|
|
||||||
],
|
|
||||||
"isUnique": true
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"foreignKeys": {
|
|
||||||
"annotation_dataset_entry_id_dataset_entry_id_fk": {
|
|
||||||
"name": "annotation_dataset_entry_id_dataset_entry_id_fk",
|
|
||||||
"tableFrom": "annotation",
|
|
||||||
"tableTo": "dataset_entry",
|
|
||||||
"columnsFrom": [
|
|
||||||
"dataset_entry_id"
|
|
||||||
],
|
|
||||||
"columnsTo": [
|
|
||||||
"id"
|
|
||||||
],
|
|
||||||
"onDelete": "no action",
|
|
||||||
"onUpdate": "no action"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"compositePrimaryKeys": {},
|
|
||||||
"uniqueConstraints": {},
|
|
||||||
"checkConstraints": {}
|
|
||||||
},
|
|
||||||
"account": {
|
"account": {
|
||||||
"name": "account",
|
"name": "account",
|
||||||
"columns": {
|
"columns": {
|
||||||
@@ -513,6 +444,34 @@
|
|||||||
"notNull": false,
|
"notNull": false,
|
||||||
"autoincrement": false
|
"autoincrement": false
|
||||||
},
|
},
|
||||||
|
"rms_value": {
|
||||||
|
"name": "rms_value",
|
||||||
|
"type": "real",
|
||||||
|
"primaryKey": false,
|
||||||
|
"notNull": false,
|
||||||
|
"autoincrement": false
|
||||||
|
},
|
||||||
|
"longest_pause": {
|
||||||
|
"name": "longest_pause",
|
||||||
|
"type": "real",
|
||||||
|
"primaryKey": false,
|
||||||
|
"notNull": false,
|
||||||
|
"autoincrement": false
|
||||||
|
},
|
||||||
|
"utmos_score": {
|
||||||
|
"name": "utmos_score",
|
||||||
|
"type": "real",
|
||||||
|
"primaryKey": false,
|
||||||
|
"notNull": false,
|
||||||
|
"autoincrement": false
|
||||||
|
},
|
||||||
|
"wer_score": {
|
||||||
|
"name": "wer_score",
|
||||||
|
"type": "real",
|
||||||
|
"primaryKey": false,
|
||||||
|
"notNull": false,
|
||||||
|
"autoincrement": false
|
||||||
|
},
|
||||||
"created_at": {
|
"created_at": {
|
||||||
"name": "created_at",
|
"name": "created_at",
|
||||||
"type": "integer",
|
"type": "integer",
|
||||||
|
|||||||
@@ -47,15 +47,8 @@
|
|||||||
{
|
{
|
||||||
"idx": 6,
|
"idx": 6,
|
||||||
"version": "6",
|
"version": "6",
|
||||||
"when": 1773406795226,
|
"when": 1773406006276,
|
||||||
"tag": "0006_polite_moondragon",
|
"tag": "0006_special_junta",
|
||||||
"breakpoints": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"idx": 7,
|
|
||||||
"version": "6",
|
|
||||||
"when": 1773407494321,
|
|
||||||
"tag": "0007_flawless_wonder_man",
|
|
||||||
"breakpoints": true
|
"breakpoints": true
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -53,6 +53,10 @@ export async function downloadFilteredEntries(
|
|||||||
'id',
|
'id',
|
||||||
'audio_file',
|
'audio_file',
|
||||||
'duration_ms',
|
'duration_ms',
|
||||||
|
'rms_value',
|
||||||
|
'longest_pause',
|
||||||
|
'utmos_score',
|
||||||
|
'wer_score',
|
||||||
'utt_id',
|
'utt_id',
|
||||||
'text',
|
'text',
|
||||||
'speaker',
|
'speaker',
|
||||||
@@ -67,6 +71,10 @@ export async function downloadFilteredEntries(
|
|||||||
`"${entry.externalId}"`,
|
`"${entry.externalId}"`,
|
||||||
entry.fileName,
|
entry.fileName,
|
||||||
entry.durationMs?.toString() || '',
|
entry.durationMs?.toString() || '',
|
||||||
|
entry.rmsValue?.toString() || '',
|
||||||
|
entry.longestPause?.toString() || '',
|
||||||
|
entry.utmosScore?.toString() || '',
|
||||||
|
entry.werScore?.toString() || '',
|
||||||
entry.utteranceId || '',
|
entry.utteranceId || '',
|
||||||
`"${(entry.utteranceText || '').replace(/"/g, '""')}"`,
|
`"${(entry.utteranceText || '').replace(/"/g, '""')}"`,
|
||||||
entry.speakerId,
|
entry.speakerId,
|
||||||
|
|||||||
@@ -2,9 +2,10 @@
|
|||||||
|
|
||||||
import db from '@/lib/db';
|
import db from '@/lib/db';
|
||||||
import { dataset_entry } from '@/lib/model/dataset_entry';
|
import { dataset_entry } from '@/lib/model/dataset_entry';
|
||||||
import { count, eq, and, like } from 'drizzle-orm';
|
import { count, eq, and, like, asc, desc, getTableColumns } from 'drizzle-orm';
|
||||||
|
|
||||||
const ENTRIES_PER_PAGE = 20;
|
const ENTRIES_PER_PAGE = 20;
|
||||||
|
const SORT_COLUMNS = getTableColumns(dataset_entry);
|
||||||
|
|
||||||
interface FilterParams {
|
interface FilterParams {
|
||||||
speakerId?: string;
|
speakerId?: string;
|
||||||
@@ -14,10 +15,16 @@ interface FilterParams {
|
|||||||
utteranceId?: string;
|
utteranceId?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
interface SortParams {
|
||||||
|
sortBy?: keyof typeof SORT_COLUMNS;
|
||||||
|
sortOrder?: 'asc' | 'desc';
|
||||||
|
}
|
||||||
|
|
||||||
export async function getDatasetEntries(
|
export async function getDatasetEntries(
|
||||||
datasetId: number,
|
datasetId: number,
|
||||||
page: number = 1,
|
page: number = 1,
|
||||||
filters?: FilterParams
|
filters?: FilterParams,
|
||||||
|
sort?: SortParams
|
||||||
) {
|
) {
|
||||||
const offset = (page - 1) * ENTRIES_PER_PAGE;
|
const offset = (page - 1) * ENTRIES_PER_PAGE;
|
||||||
|
|
||||||
@@ -46,10 +53,17 @@ export async function getDatasetEntries(
|
|||||||
|
|
||||||
const whereClause = and(...conditions);
|
const whereClause = and(...conditions);
|
||||||
|
|
||||||
|
// Build sort clause
|
||||||
|
const sortFn = sort?.sortOrder === 'desc' ? desc : asc;
|
||||||
|
const sortColumn = sort?.sortBy && sort.sortBy in SORT_COLUMNS
|
||||||
|
? sortFn(SORT_COLUMNS[sort.sortBy as keyof typeof SORT_COLUMNS])
|
||||||
|
: asc(dataset_entry.externalId);
|
||||||
|
|
||||||
const entries = await db
|
const entries = await db
|
||||||
.select()
|
.select()
|
||||||
.from(dataset_entry)
|
.from(dataset_entry)
|
||||||
.where(whereClause)
|
.where(whereClause)
|
||||||
|
.orderBy(sortColumn)
|
||||||
.limit(ENTRIES_PER_PAGE)
|
.limit(ENTRIES_PER_PAGE)
|
||||||
.offset(offset);
|
.offset(offset);
|
||||||
|
|
||||||
|
|||||||
@@ -2,6 +2,7 @@
|
|||||||
|
|
||||||
import db from '@/lib/db';
|
import db from '@/lib/db';
|
||||||
import { dataset_entry } from '@/lib/model/dataset_entry';
|
import { dataset_entry } from '@/lib/model/dataset_entry';
|
||||||
|
import { parseCSVLine } from '@/lib/csv-parser';
|
||||||
import { revalidatePath } from 'next/cache';
|
import { revalidatePath } from 'next/cache';
|
||||||
import { eq } from 'drizzle-orm';
|
import { eq } from 'drizzle-orm';
|
||||||
import { writeFile, mkdir, readFile, rm } from 'fs/promises';
|
import { writeFile, mkdir, readFile, rm } from 'fs/promises';
|
||||||
@@ -10,38 +11,71 @@ import { existsSync } from 'fs';
|
|||||||
import { requireAdmin } from '@/lib/auth';
|
import { requireAdmin } from '@/lib/auth';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* CSV parser that handles quoted fields
|
* Helper: Parse metadata.csv from extracted directory
|
||||||
|
* Returns structured data for both insert and update operations
|
||||||
*/
|
*/
|
||||||
function parseCSVLine(line: string): string[] {
|
async function parseMetadataFromDirectory(
|
||||||
const values: string[] = [];
|
tempDir: string
|
||||||
let current = '';
|
): Promise<{
|
||||||
let inQuotes = false;
|
headers: string[];
|
||||||
|
rows: Record<string, string>[];
|
||||||
for (let i = 0; i < line.length; i++) {
|
}> {
|
||||||
const char = line[i];
|
const metadataPath = join(tempDir, 'metadata.csv');
|
||||||
const nextChar = line[i + 1];
|
if (!existsSync(metadataPath)) {
|
||||||
|
throw new Error('metadata.csv not found in extracted files');
|
||||||
if (char === '"') {
|
|
||||||
if (inQuotes && nextChar === '"') {
|
|
||||||
// Escaped quote
|
|
||||||
current += '"';
|
|
||||||
i++;
|
|
||||||
} else {
|
|
||||||
// Toggle quote state
|
|
||||||
inQuotes = !inQuotes;
|
|
||||||
}
|
|
||||||
} else if (char === ',' && !inQuotes) {
|
|
||||||
// Field separator
|
|
||||||
values.push(current.trim());
|
|
||||||
current = '';
|
|
||||||
} else {
|
|
||||||
current += char;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Add the last field
|
const metadataContent = await readFile(metadataPath, 'utf-8');
|
||||||
values.push(current.trim());
|
const lines = metadataContent.split('\n').filter((line) => line.trim());
|
||||||
return values;
|
|
||||||
|
if (lines.length < 2) {
|
||||||
|
throw new Error('metadata.csv is empty or invalid');
|
||||||
|
}
|
||||||
|
|
||||||
|
const headers = lines[0].split(',').map((h) => h.trim());
|
||||||
|
const rows: Record<string, string>[] = [];
|
||||||
|
|
||||||
|
for (let i = 1; i < lines.length; i++) {
|
||||||
|
const values = parseCSVLine(lines[i]);
|
||||||
|
|
||||||
|
if (values.length !== headers.length) {
|
||||||
|
continue; // Skip malformed lines
|
||||||
|
}
|
||||||
|
|
||||||
|
const row: Record<string, string> = {};
|
||||||
|
headers.forEach((header, index) => {
|
||||||
|
row[header] = values[index];
|
||||||
|
});
|
||||||
|
|
||||||
|
rows.push(row);
|
||||||
|
}
|
||||||
|
|
||||||
|
return { headers, rows };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Convert metadata row to dataset_entry insert object
|
||||||
|
*/
|
||||||
|
function rowToDatasetEntry(
|
||||||
|
row: Record<string, string>,
|
||||||
|
datasetId: number
|
||||||
|
): typeof dataset_entry.$inferInsert {
|
||||||
|
return {
|
||||||
|
datasetId,
|
||||||
|
externalId: row.id,
|
||||||
|
speakerId: row.speaker,
|
||||||
|
modelName: row.model,
|
||||||
|
utteranceId: row.utt_id,
|
||||||
|
utteranceText: row.text,
|
||||||
|
fileName: row.audio_file,
|
||||||
|
dialect: row.dialect,
|
||||||
|
iteration: parseInt(row.iteration, 10),
|
||||||
|
durationMs: row.duration_ms ? parseInt(row.duration_ms, 10) : undefined,
|
||||||
|
rmsValue: row.rms_value ? parseFloat(row.rms_value) : undefined,
|
||||||
|
longestPause: row.longest_pause ? parseFloat(row.longest_pause) : undefined,
|
||||||
|
utmosScore: row.utmos_score ? parseFloat(row.utmos_score) : undefined,
|
||||||
|
werScore: row.wer_score ? parseFloat(row.wer_score) : undefined,
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -73,47 +107,16 @@ export async function processDatasetEntries(
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
// Read metadata.csv again
|
// Parse metadata from directory
|
||||||
const metadataPath = join(tempDir, 'metadata.csv');
|
const { rows } = await parseMetadataFromDirectory(tempDir);
|
||||||
if (!existsSync(metadataPath)) {
|
|
||||||
return {
|
|
||||||
success: false,
|
|
||||||
error: 'metadata.csv not found in extracted files',
|
|
||||||
entriesCreated: 0,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
const metadataContent = await readFile(metadataPath, 'utf-8');
|
|
||||||
const lines = metadataContent.split('\n').filter(line => line.trim());
|
|
||||||
|
|
||||||
if (lines.length < 2) {
|
|
||||||
return {
|
|
||||||
success: false,
|
|
||||||
error: 'metadata.csv is empty or invalid',
|
|
||||||
entriesCreated: 0,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
// Parse CSV
|
|
||||||
const headers = lines[0].split(',').map(h => h.trim());
|
|
||||||
const entries: typeof dataset_entry.$inferInsert[] = [];
|
const entries: typeof dataset_entry.$inferInsert[] = [];
|
||||||
|
|
||||||
// Create dataset directory
|
// Create dataset directory
|
||||||
await mkdir(datasetDir, { recursive: true });
|
await mkdir(datasetDir, { recursive: true });
|
||||||
|
|
||||||
// Process each row
|
// Process each row
|
||||||
for (let i = 1; i < lines.length; i++) {
|
for (const row of rows) {
|
||||||
const values = parseCSVLine(lines[i]);
|
|
||||||
|
|
||||||
if (values.length !== headers.length) {
|
|
||||||
continue; // Skip malformed lines
|
|
||||||
}
|
|
||||||
|
|
||||||
const row: Record<string, string> = {};
|
|
||||||
headers.forEach((header, index) => {
|
|
||||||
row[header] = values[index];
|
|
||||||
});
|
|
||||||
|
|
||||||
const audioFile = row.audio_file as string;
|
const audioFile = row.audio_file as string;
|
||||||
const relativePath = audioFile.replace(/\\/g, '/');
|
const relativePath = audioFile.replace(/\\/g, '/');
|
||||||
const audioPath = join(tempDir, relativePath);
|
const audioPath = join(tempDir, relativePath);
|
||||||
@@ -126,18 +129,7 @@ export async function processDatasetEntries(
|
|||||||
|
|
||||||
await writeFile(destPath, audioBuffer);
|
await writeFile(destPath, audioBuffer);
|
||||||
|
|
||||||
entries.push({
|
entries.push(rowToDatasetEntry(row, datasetId));
|
||||||
datasetId,
|
|
||||||
externalId: row.id,
|
|
||||||
speakerId: row.speaker,
|
|
||||||
modelName: row.model,
|
|
||||||
utteranceId: row.utt_id,
|
|
||||||
utteranceText: row.text,
|
|
||||||
fileName: relativePath,
|
|
||||||
dialect: row.dialect,
|
|
||||||
iteration: parseInt(row.iteration, 10),
|
|
||||||
durationMs: row.duration_ms ? parseInt(row.duration_ms, 10) : undefined,
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -197,3 +189,116 @@ export async function processDatasetEntries(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Update metadata for existing dataset entries
|
||||||
|
* Uses the same metadata.csv parsing as processDatasetEntries
|
||||||
|
* Matches entries by externalId and updates their metadata fields
|
||||||
|
*/
|
||||||
|
export async function updateDatasetEntriesMetadata(
|
||||||
|
datasetId: number,
|
||||||
|
tempDir: string
|
||||||
|
) {
|
||||||
|
try {
|
||||||
|
const result = await requireAdmin();
|
||||||
|
if (!result.authenticated || !result.admin) {
|
||||||
|
return {
|
||||||
|
success: false,
|
||||||
|
error: 'Unauthorized',
|
||||||
|
entriesUpdated: 0,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// Validate temp directory exists
|
||||||
|
if (!existsSync(tempDir)) {
|
||||||
|
return {
|
||||||
|
success: false,
|
||||||
|
error: 'Temporary extraction directory not found',
|
||||||
|
entriesUpdated: 0,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parse metadata from directory (reusing same logic as processDatasetEntries)
|
||||||
|
const { rows } = await parseMetadataFromDirectory(tempDir);
|
||||||
|
|
||||||
|
if (rows.length === 0) {
|
||||||
|
return {
|
||||||
|
success: false,
|
||||||
|
error: 'No valid entries found in metadata.csv',
|
||||||
|
entriesUpdated: 0,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get all existing entries for this dataset
|
||||||
|
const existingEntries = await db
|
||||||
|
.select()
|
||||||
|
.from(dataset_entry)
|
||||||
|
.where(eq(dataset_entry.datasetId, datasetId));
|
||||||
|
|
||||||
|
const existingByExternalId = new Map(
|
||||||
|
existingEntries.map(entry => [entry.externalId, entry])
|
||||||
|
);
|
||||||
|
|
||||||
|
let entriesUpdated = 0;
|
||||||
|
let entriesNotFound = 0;
|
||||||
|
|
||||||
|
// Update each row that has a matching entry
|
||||||
|
for (const row of rows) {
|
||||||
|
const externalId = row.id;
|
||||||
|
const existingEntry = existingByExternalId.get(externalId);
|
||||||
|
|
||||||
|
if (!existingEntry) {
|
||||||
|
entriesNotFound++;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Update only the metadata fields, don't change the ID or dataset reference
|
||||||
|
await db
|
||||||
|
.update(dataset_entry)
|
||||||
|
.set({
|
||||||
|
speakerId: row.speaker,
|
||||||
|
modelName: row.model,
|
||||||
|
utteranceId: row.utt_id,
|
||||||
|
utteranceText: row.text,
|
||||||
|
dialect: row.dialect,
|
||||||
|
iteration: parseInt(row.iteration, 10),
|
||||||
|
durationMs: row.duration_ms ? parseInt(row.duration_ms, 10) : undefined,
|
||||||
|
rmsValue: row.rms_value ? parseFloat(row.rms_value) : undefined,
|
||||||
|
longestPause: row.longest_pause ? parseFloat(row.longest_pause) : undefined,
|
||||||
|
utmosScore: row.utmos_score ? parseFloat(row.utmos_score) : undefined,
|
||||||
|
werScore: row.wer_score ? parseFloat(row.wer_score) : undefined,
|
||||||
|
})
|
||||||
|
.where(eq(dataset_entry.id, existingEntry.id));
|
||||||
|
|
||||||
|
entriesUpdated++;
|
||||||
|
}
|
||||||
|
|
||||||
|
revalidatePath(`/admin/datasets/${datasetId}`);
|
||||||
|
|
||||||
|
return {
|
||||||
|
success: true,
|
||||||
|
entriesUpdated,
|
||||||
|
entriesNotFound,
|
||||||
|
message: entriesUpdated > 0
|
||||||
|
? `Successfully updated ${entriesUpdated} entries${entriesNotFound > 0 ? ` (${entriesNotFound} not found)` : ''}.`
|
||||||
|
: `No entries were updated (${entriesNotFound} not found).`,
|
||||||
|
};
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Error updating dataset entries metadata:', error);
|
||||||
|
const errorMessage = error instanceof Error ? error.message : 'Failed to update dataset entries metadata';
|
||||||
|
return {
|
||||||
|
success: false,
|
||||||
|
error: errorMessage,
|
||||||
|
entriesUpdated: 0,
|
||||||
|
};
|
||||||
|
} finally {
|
||||||
|
// Clean up temp directory
|
||||||
|
try {
|
||||||
|
if (existsSync(tempDir)) {
|
||||||
|
await rm(tempDir, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
} catch (cleanupError) {
|
||||||
|
console.error('Error cleaning up temp directory:', cleanupError);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -36,7 +36,7 @@ export default async function DatasetEntryPage({ params }: DatasetEntryPageProps
|
|||||||
if (entries.length === 0 || entries[0].datasetId !== datasetIdNum) {
|
if (entries.length === 0 || entries[0].datasetId !== datasetIdNum) {
|
||||||
return (
|
return (
|
||||||
<div className="min-h-screen">
|
<div className="min-h-screen">
|
||||||
<div className="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8 py-8">
|
<div className="px-4 sm:px-6 lg:px-8 py-8">
|
||||||
<Link
|
<Link
|
||||||
href={`/admin/datasets/${datasetId}`}
|
href={`/admin/datasets/${datasetId}`}
|
||||||
className="text-blue-500 hover:text-blue-600 mb-6 inline-block"
|
className="text-blue-500 hover:text-blue-600 mb-6 inline-block"
|
||||||
@@ -55,7 +55,7 @@ export default async function DatasetEntryPage({ params }: DatasetEntryPageProps
|
|||||||
|
|
||||||
return (
|
return (
|
||||||
<div>
|
<div>
|
||||||
<div className="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8 py-8">
|
<div className="px-4 sm:px-6 lg:px-8 py-8">
|
||||||
<Link
|
<Link
|
||||||
href={`/admin/datasets/${datasetId}`}
|
href={`/admin/datasets/${datasetId}`}
|
||||||
className="text-blue-500 hover:text-blue-600 mb-6 inline-block"
|
className="text-blue-500 hover:text-blue-600 mb-6 inline-block"
|
||||||
@@ -122,6 +122,26 @@ export default async function DatasetEntryPage({ params }: DatasetEntryPageProps
|
|||||||
<p className="text-lg font-semibold">{entry.durationMs?.toLocaleString() || '-'}</p>
|
<p className="text-lg font-semibold">{entry.durationMs?.toLocaleString() || '-'}</p>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<div className="bg-white border border-gray-200 p-4 rounded-lg">
|
||||||
|
<p className="text-sm text-gray-600 mb-1">RMS Value</p>
|
||||||
|
<p className="text-lg font-semibold">{entry.rmsValue?.toFixed(3) || '-'}</p>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="bg-white border border-gray-200 p-4 rounded-lg">
|
||||||
|
<p className="text-sm text-gray-600 mb-1">Longest Pause (s)</p>
|
||||||
|
<p className="text-lg font-semibold">{entry.longestPause?.toFixed(3) || '-'}</p>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="bg-white border border-gray-200 p-4 rounded-lg">
|
||||||
|
<p className="text-sm text-gray-600 mb-1">UTMOS Score</p>
|
||||||
|
<p className="text-lg font-semibold">{entry.utmosScore?.toFixed(3) || '-'}</p>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="bg-white border border-gray-200 p-4 rounded-lg">
|
||||||
|
<p className="text-sm text-gray-600 mb-1">WER Score</p>
|
||||||
|
<p className="text-lg font-semibold">{entry.werScore?.toFixed(3) || '-'}</p>
|
||||||
|
</div>
|
||||||
|
|
||||||
<div className="bg-white border border-gray-200 p-4 rounded-lg">
|
<div className="bg-white border border-gray-200 p-4 rounded-lg">
|
||||||
<p className="text-sm text-gray-600 mb-1">Created</p>
|
<p className="text-sm text-gray-600 mb-1">Created</p>
|
||||||
<p className="text-lg font-semibold">
|
<p className="text-lg font-semibold">
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import { eq } from 'drizzle-orm';
|
|||||||
import EditableDatasetHeader from '@/components/EditableDatasetHeader';
|
import EditableDatasetHeader from '@/components/EditableDatasetHeader';
|
||||||
import DatasetEntriesList from '@/components/DatasetEntriesList';
|
import DatasetEntriesList from '@/components/DatasetEntriesList';
|
||||||
import UploadDatasetEntriesModal from '@/components/UploadDatasetEntriesModal';
|
import UploadDatasetEntriesModal from '@/components/UploadDatasetEntriesModal';
|
||||||
|
import UpdateDatasetMetadataModal from '@/components/UpdateDatasetMetadataModal';
|
||||||
import RemoveAllEntriesButton from '@/components/RemoveAllEntriesButton';
|
import RemoveAllEntriesButton from '@/components/RemoveAllEntriesButton';
|
||||||
import DeleteDatasetButton from '@/components/DeleteDatasetButton';
|
import DeleteDatasetButton from '@/components/DeleteDatasetButton';
|
||||||
import { requireAdmin } from '@/lib/auth';
|
import { requireAdmin } from '@/lib/auth';
|
||||||
@@ -38,7 +39,7 @@ export default async function DatasetPage({ params }: DatasetPageProps) {
|
|||||||
if (datasets.length === 0) {
|
if (datasets.length === 0) {
|
||||||
return (
|
return (
|
||||||
<div className="min-h-screen">
|
<div className="min-h-screen">
|
||||||
<div className="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8 py-8">
|
<div className="px-4 sm:px-6 lg:px-8 py-8">
|
||||||
<Link
|
<Link
|
||||||
href="/admin"
|
href="/admin"
|
||||||
className="text-blue-500 hover:text-blue-600 mb-6 inline-block"
|
className="text-blue-500 hover:text-blue-600 mb-6 inline-block"
|
||||||
@@ -57,7 +58,7 @@ export default async function DatasetPage({ params }: DatasetPageProps) {
|
|||||||
|
|
||||||
return (
|
return (
|
||||||
<div>
|
<div>
|
||||||
<div className="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8 py-8">
|
<div className="px-4 sm:px-6 lg:px-8 py-8">
|
||||||
<Link
|
<Link
|
||||||
href="/admin"
|
href="/admin"
|
||||||
className="text-blue-500 hover:text-blue-600 mb-6 inline-block"
|
className="text-blue-500 hover:text-blue-600 mb-6 inline-block"
|
||||||
@@ -69,6 +70,7 @@ export default async function DatasetPage({ params }: DatasetPageProps) {
|
|||||||
|
|
||||||
<div className="flex gap-3 mb-6">
|
<div className="flex gap-3 mb-6">
|
||||||
<UploadDatasetEntriesModal datasetId={datasetId} />
|
<UploadDatasetEntriesModal datasetId={datasetId} />
|
||||||
|
<UpdateDatasetMetadataModal datasetId={datasetId} />
|
||||||
<RemoveAllEntriesButton datasetId={datasetId} />
|
<RemoveAllEntriesButton datasetId={datasetId} />
|
||||||
<DeleteDatasetButton datasetId={datasetId} datasetName={ds.name} />
|
<DeleteDatasetButton datasetId={datasetId} datasetName={ds.name} />
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ export default async function AdminPage() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8 py-8">
|
<div className="px-4 sm:px-6 lg:px-8 py-8">
|
||||||
<div className="mb-8">
|
<div className="mb-8">
|
||||||
<h1 className="text-3xl font-bold mb-4">Admin Panel</h1>
|
<h1 className="text-3xl font-bold mb-4">Admin Panel</h1>
|
||||||
<div className="flex gap-4">
|
<div className="flex gap-4">
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ export default async function UsersPage() {
|
|||||||
const users = await getAllUsers();
|
const users = await getAllUsers();
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8 py-8">
|
<div className="px-4 sm:px-6 lg:px-8 py-8">
|
||||||
<div className="mb-6">
|
<div className="mb-6">
|
||||||
<Link
|
<Link
|
||||||
href="/admin"
|
href="/admin"
|
||||||
|
|||||||
74
src/app/api/update-metadata/route.ts
Normal file
74
src/app/api/update-metadata/route.ts
Normal file
@@ -0,0 +1,74 @@
|
|||||||
|
import { writeFile, mkdir, rm } from 'fs/promises';
|
||||||
|
import { join } from 'path';
|
||||||
|
import { existsSync } from 'fs';
|
||||||
|
import { NextRequest, NextResponse } from 'next/server';
|
||||||
|
import { requireAdmin } from '@/lib/auth';
|
||||||
|
|
||||||
|
export async function POST(request: NextRequest) {
|
||||||
|
try {
|
||||||
|
const result = await requireAdmin();
|
||||||
|
if (!result.authenticated || !result.admin) {
|
||||||
|
return NextResponse.json(
|
||||||
|
{ error: 'Unauthorized' },
|
||||||
|
{ status: 401 }
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const formData = await request.formData();
|
||||||
|
const file = formData.get('file') as File;
|
||||||
|
const datasetId = formData.get('datasetId') as string;
|
||||||
|
|
||||||
|
if (!file) {
|
||||||
|
return NextResponse.json(
|
||||||
|
{ error: 'No file provided' },
|
||||||
|
{ status: 400 }
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!file.name.endsWith('.csv')) {
|
||||||
|
return NextResponse.json(
|
||||||
|
{ error: 'File must be a CSV file' },
|
||||||
|
{ status: 400 }
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const tempDir = join(process.cwd(), 'tmp', `update-metadata-${Date.now()}`);
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Create temp directory
|
||||||
|
await mkdir(tempDir, { recursive: true });
|
||||||
|
|
||||||
|
// Save uploaded CSV file as metadata.csv
|
||||||
|
const arrayBuffer = await file.arrayBuffer();
|
||||||
|
const buffer = Buffer.from(arrayBuffer);
|
||||||
|
const metadataPath = join(tempDir, 'metadata.csv');
|
||||||
|
await writeFile(metadataPath, buffer);
|
||||||
|
|
||||||
|
return NextResponse.json({
|
||||||
|
success: true,
|
||||||
|
tempDir,
|
||||||
|
message: `Successfully uploaded metadata.csv. Ready for processing.`,
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
// Clean up on error
|
||||||
|
try {
|
||||||
|
if (existsSync(tempDir)) {
|
||||||
|
await rm(tempDir, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
} catch (cleanupError) {
|
||||||
|
console.error('Error cleaning up temp directory:', cleanupError);
|
||||||
|
}
|
||||||
|
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Error uploading metadata CSV:', error);
|
||||||
|
const errorMsg =
|
||||||
|
error instanceof Error ? error.message : 'Failed to upload metadata CSV';
|
||||||
|
return NextResponse.json(
|
||||||
|
{ error: errorMsg },
|
||||||
|
{ status: 500 }
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@@ -13,7 +13,7 @@ export default function RootLayout({
|
|||||||
<body className="flex flex-col min-h-screen">
|
<body className="flex flex-col min-h-screen">
|
||||||
<AudioProvider>
|
<AudioProvider>
|
||||||
<Header />
|
<Header />
|
||||||
<main className="flex-grow mt-4 mb-4 mr-8 ml-8">
|
<main className="flex-grow">
|
||||||
{children}
|
{children}
|
||||||
</main>
|
</main>
|
||||||
<Footer />
|
<Footer />
|
||||||
|
|||||||
@@ -22,6 +22,11 @@ interface DatasetEntry {
|
|||||||
fileName: string;
|
fileName: string;
|
||||||
dialect: string;
|
dialect: string;
|
||||||
iteration: number;
|
iteration: number;
|
||||||
|
durationMs: number | null;
|
||||||
|
rmsValue: number | null;
|
||||||
|
longestPause: number | null;
|
||||||
|
utmosScore: number | null;
|
||||||
|
werScore: number | null;
|
||||||
createdAt: Date;
|
createdAt: Date;
|
||||||
updatedAt: Date;
|
updatedAt: Date;
|
||||||
}
|
}
|
||||||
@@ -62,6 +67,10 @@ export default function DatasetEntriesList({
|
|||||||
utteranceIds: [],
|
utteranceIds: [],
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// Sort state
|
||||||
|
const [sortBy, setSortBy] = useState<keyof DatasetEntry>('externalId');
|
||||||
|
const [sortOrder, setSortOrder] = useState<'asc' | 'desc'>('asc');
|
||||||
|
|
||||||
// Load filter options on mount
|
// Load filter options on mount
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
async function loadFilterOptions() {
|
async function loadFilterOptions() {
|
||||||
@@ -78,7 +87,7 @@ export default function DatasetEntriesList({
|
|||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
setPage(1);
|
setPage(1);
|
||||||
loadEntries(1);
|
loadEntries(1);
|
||||||
}, [filters]);
|
}, [filters, sortBy, sortOrder]);
|
||||||
|
|
||||||
async function loadEntries(pageNum: number = page) {
|
async function loadEntries(pageNum: number = page) {
|
||||||
setLoading(true);
|
setLoading(true);
|
||||||
@@ -91,7 +100,12 @@ export default function DatasetEntriesList({
|
|||||||
utteranceId: filters.utteranceId || undefined,
|
utteranceId: filters.utteranceId || undefined,
|
||||||
};
|
};
|
||||||
|
|
||||||
const result = await getDatasetEntries(datasetId, pageNum, filterParams);
|
const sortParams = {
|
||||||
|
sortBy: sortBy as keyof DatasetEntry,
|
||||||
|
sortOrder: sortOrder,
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = await getDatasetEntries(datasetId, pageNum, filterParams, sortParams);
|
||||||
if (pageNum === 1) {
|
if (pageNum === 1) {
|
||||||
setEntries(result.entries);
|
setEntries(result.entries);
|
||||||
} else {
|
} else {
|
||||||
@@ -118,7 +132,12 @@ export default function DatasetEntriesList({
|
|||||||
utteranceId: filters.utteranceId || undefined,
|
utteranceId: filters.utteranceId || undefined,
|
||||||
};
|
};
|
||||||
|
|
||||||
const result = await getDatasetEntries(datasetId, page + 1, filterParams);
|
const sortParams = {
|
||||||
|
sortBy: sortBy as keyof DatasetEntry,
|
||||||
|
sortOrder: sortOrder,
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = await getDatasetEntries(datasetId, page + 1, filterParams, sortParams);
|
||||||
setEntries((prev) => [...prev, ...result.entries]);
|
setEntries((prev) => [...prev, ...result.entries]);
|
||||||
setHasMore(result.hasMore);
|
setHasMore(result.hasMore);
|
||||||
setTotal(result.total);
|
setTotal(result.total);
|
||||||
@@ -147,6 +166,17 @@ export default function DatasetEntriesList({
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function handleSort(column: keyof DatasetEntry) {
|
||||||
|
if (sortBy === column) {
|
||||||
|
// Toggle sort order if clicking the same column
|
||||||
|
setSortOrder(sortOrder === 'asc' ? 'desc' : 'asc');
|
||||||
|
} else {
|
||||||
|
// Set new column to sort by, default to ascending
|
||||||
|
setSortBy(column);
|
||||||
|
setSortOrder('asc');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async function handleDownloadFiltered() {
|
async function handleDownloadFiltered() {
|
||||||
setDownloading(true);
|
setDownloading(true);
|
||||||
try {
|
try {
|
||||||
@@ -410,13 +440,39 @@ export default function DatasetEntriesList({
|
|||||||
<thead>
|
<thead>
|
||||||
<tr className="border-b border-gray-200">
|
<tr className="border-b border-gray-200">
|
||||||
<th className="text-left py-3 px-4 font-semibold text-gray-700">Play</th>
|
<th className="text-left py-3 px-4 font-semibold text-gray-700">Play</th>
|
||||||
<th className="text-left py-3 px-4 font-semibold text-gray-700">External ID</th>
|
<th className="text-left py-3 px-4 font-semibold text-gray-700 cursor-pointer hover:bg-gray-100" onClick={() => handleSort('externalId')}>
|
||||||
<th className="text-left py-3 px-4 font-semibold text-gray-700">Speaker ID</th>
|
External ID {sortBy === 'externalId' && (sortOrder === 'asc' ? '↑' : '↓')}
|
||||||
<th className="text-left py-3 px-4 font-semibold text-gray-700">Model Name</th>
|
</th>
|
||||||
<th className="text-left py-3 px-4 font-semibold text-gray-700">Utterance ID</th>
|
<th className="text-left py-3 px-4 font-semibold text-gray-700 cursor-pointer hover:bg-gray-100" onClick={() => handleSort('speakerId')}>
|
||||||
<th className="text-left py-3 px-4 font-semibold text-gray-700">Dialect</th>
|
Speaker ID {sortBy === 'speakerId' && (sortOrder === 'asc' ? '↑' : '↓')}
|
||||||
<th className="text-left py-3 px-4 font-semibold text-gray-700">File Name</th>
|
</th>
|
||||||
<th className="text-left py-3 px-4 font-semibold text-gray-700">Iteration</th>
|
<th className="text-left py-3 px-4 font-semibold text-gray-700 cursor-pointer hover:bg-gray-100" onClick={() => handleSort('modelName')}>
|
||||||
|
Model Name {sortBy === 'modelName' && (sortOrder === 'asc' ? '↑' : '↓')}
|
||||||
|
</th>
|
||||||
|
<th className="text-left py-3 px-4 font-semibold text-gray-700 cursor-pointer hover:bg-gray-100" onClick={() => handleSort('utteranceId')}>
|
||||||
|
Utterance ID {sortBy === 'utteranceId' && (sortOrder === 'asc' ? '↑' : '↓')}
|
||||||
|
</th>
|
||||||
|
<th className="text-left py-3 px-4 font-semibold text-gray-700 cursor-pointer hover:bg-gray-100" onClick={() => handleSort('dialect')}>
|
||||||
|
Dialect {sortBy === 'dialect' && (sortOrder === 'asc' ? '↑' : '↓')}
|
||||||
|
</th>
|
||||||
|
<th className="text-left py-3 px-4 font-semibold text-gray-700 cursor-pointer hover:bg-gray-100" onClick={() => handleSort('iteration')}>
|
||||||
|
Iteration {sortBy === 'iteration' && (sortOrder === 'asc' ? '↑' : '↓')}
|
||||||
|
</th>
|
||||||
|
<th className="text-left py-3 px-4 font-semibold text-gray-700 cursor-pointer hover:bg-gray-100" onClick={() => handleSort('durationMs')}>
|
||||||
|
Duration (ms) {sortBy === 'durationMs' && (sortOrder === 'asc' ? '↑' : '↓')}
|
||||||
|
</th>
|
||||||
|
<th className="text-left py-3 px-4 font-semibold text-gray-700 cursor-pointer hover:bg-gray-100" onClick={() => handleSort('rmsValue')}>
|
||||||
|
RMS Value {sortBy === 'rmsValue' && (sortOrder === 'asc' ? '↑' : '↓')}
|
||||||
|
</th>
|
||||||
|
<th className="text-left py-3 px-4 font-semibold text-gray-700 cursor-pointer hover:bg-gray-100" onClick={() => handleSort('longestPause')}>
|
||||||
|
Longest Pause (s) {sortBy === 'longestPause' && (sortOrder === 'asc' ? '↑' : '↓')}
|
||||||
|
</th>
|
||||||
|
<th className="text-left py-3 px-4 font-semibold text-gray-700 cursor-pointer hover:bg-gray-100" onClick={() => handleSort('utmosScore')}>
|
||||||
|
UTMOS Score {sortBy === 'utmosScore' && (sortOrder === 'asc' ? '↑' : '↓')}
|
||||||
|
</th>
|
||||||
|
<th className="text-left py-3 px-4 font-semibold text-gray-700 cursor-pointer hover:bg-gray-100" onClick={() => handleSort('werScore')}>
|
||||||
|
WER Score {sortBy === 'werScore' && (sortOrder === 'asc' ? '↑' : '↓')}
|
||||||
|
</th>
|
||||||
</tr>
|
</tr>
|
||||||
</thead>
|
</thead>
|
||||||
<tbody>
|
<tbody>
|
||||||
@@ -435,10 +491,14 @@ export default function DatasetEntriesList({
|
|||||||
</td>
|
</td>
|
||||||
<td className="py-3 px-4">{entry.speakerId}</td>
|
<td className="py-3 px-4">{entry.speakerId}</td>
|
||||||
<td className="py-3 px-4">{entry.modelName}</td>
|
<td className="py-3 px-4">{entry.modelName}</td>
|
||||||
<td className="py-3 px-4 truncate">{entry.utteranceId || '-'}</td>
|
<td className="py-3 px-4">{entry.utteranceId || '-'}</td>
|
||||||
<td className="py-3 px-4">{entry.dialect}</td>
|
<td className="py-3 px-4">{entry.dialect}</td>
|
||||||
<td className="py-3 px-4 truncate">{entry.fileName}</td>
|
|
||||||
<td className="py-3 px-4">{entry.iteration}</td>
|
<td className="py-3 px-4">{entry.iteration}</td>
|
||||||
|
<td className="py-3 px-4">{entry.durationMs?.toLocaleString() || '-'}</td>
|
||||||
|
<td className="py-3 px-4">{entry.rmsValue?.toFixed(3) || '-'}</td>
|
||||||
|
<td className="py-3 px-4">{entry.longestPause?.toFixed(3) || '-'}</td>
|
||||||
|
<td className="py-3 px-4">{entry.utmosScore?.toFixed(3) || '-'}</td>
|
||||||
|
<td className="py-3 px-4">{entry.werScore?.toFixed(3) || '-'}</td>
|
||||||
</tr>
|
</tr>
|
||||||
))}
|
))}
|
||||||
</tbody>
|
</tbody>
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ export function Footer() {
|
|||||||
|
|
||||||
return (
|
return (
|
||||||
<footer className="bg-gray-800 text-gray-300 mt-12">
|
<footer className="bg-gray-800 text-gray-300 mt-12">
|
||||||
<div className="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8 py-8">
|
<div className="px-4 sm:px-6 lg:px-8 py-8">
|
||||||
<div className="grid grid-cols-1 md:grid-cols-3 gap-8 mb-8">
|
<div className="grid grid-cols-1 md:grid-cols-3 gap-8 mb-8">
|
||||||
<div>
|
<div>
|
||||||
<h3 className="text-white font-semibold mb-4">Über das Projekt</h3>
|
<h3 className="text-white font-semibold mb-4">Über das Projekt</h3>
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ export function Header() {
|
|||||||
|
|
||||||
return (
|
return (
|
||||||
<header className="sticky top-0 z-50 bg-gradient-to-r from-blue-600 to-blue-700 text-white shadow-lg">
|
<header className="sticky top-0 z-50 bg-gradient-to-r from-blue-600 to-blue-700 text-white shadow-lg">
|
||||||
<div className="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8">
|
<div className="px-4 sm:px-6 lg:px-8">
|
||||||
<div className="flex justify-between items-center py-5">
|
<div className="flex justify-between items-center py-5">
|
||||||
<Link href="/" className="text-2xl font-bold hover:text-blue-100 transition-colors">
|
<Link href="/" className="text-2xl font-bold hover:text-blue-100 transition-colors">
|
||||||
TTS Dialektannotations-Plattform
|
TTS Dialektannotations-Plattform
|
||||||
|
|||||||
365
src/components/UpdateDatasetMetadataModal.tsx
Normal file
365
src/components/UpdateDatasetMetadataModal.tsx
Normal file
@@ -0,0 +1,365 @@
|
|||||||
|
'use client';
|
||||||
|
|
||||||
|
import { useState } from 'react';
|
||||||
|
import { updateDatasetEntriesMetadata } from '@/app/actions/upload';
|
||||||
|
import Modal from '@/components/Modal';
|
||||||
|
|
||||||
|
interface UpdateDatasetMetadataModalProps {
|
||||||
|
datasetId: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export default function UpdateDatasetMetadataModal({
|
||||||
|
datasetId,
|
||||||
|
}: UpdateDatasetMetadataModalProps) {
|
||||||
|
const [isOpen, setIsOpen] = useState(false);
|
||||||
|
const [files, setFiles] = useState<File[]>([]);
|
||||||
|
const [loading, setLoading] = useState(false);
|
||||||
|
const [error, setError] = useState<string | null>(null);
|
||||||
|
const [success, setSuccess] = useState(false);
|
||||||
|
const [uploadQueue, setUploadQueue] = useState<Array<{ file: File; status: 'pending' | 'uploading' | 'completed' | 'failed'; message?: string; progress: number }>>([]);
|
||||||
|
const [status, setStatus] = useState<string>('');
|
||||||
|
|
||||||
|
function handleFileChange(e: React.ChangeEvent<HTMLInputElement>) {
|
||||||
|
const selectedFiles = e.target.files ? Array.from(e.target.files) : [];
|
||||||
|
if (selectedFiles.length > 0) {
|
||||||
|
setFiles(selectedFiles);
|
||||||
|
setError(null);
|
||||||
|
// Initialize queue with pending status and 0 progress
|
||||||
|
setUploadQueue(selectedFiles.map(file => ({ file, status: 'pending', progress: 0 })));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function handleSubmit(e: React.FormEvent) {
|
||||||
|
e.preventDefault();
|
||||||
|
setError(null);
|
||||||
|
setSuccess(false);
|
||||||
|
|
||||||
|
if (files.length === 0) {
|
||||||
|
setError('Please select at least one CSV file');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
setLoading(true);
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Process each file in the queue sequentially
|
||||||
|
const newQueue = [...uploadQueue];
|
||||||
|
|
||||||
|
for (let i = 0; i < newQueue.length; i++) {
|
||||||
|
const item = newQueue[i];
|
||||||
|
item.status = 'uploading';
|
||||||
|
item.progress = 0;
|
||||||
|
setUploadQueue([...newQueue]);
|
||||||
|
setStatus(`Processing file ${i + 1} of ${newQueue.length}: ${item.file.name}`);
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Upload with progress tracking
|
||||||
|
const uploadResult = await uploadFileWithProgress(item.file, i, newQueue);
|
||||||
|
|
||||||
|
// Update the metadata
|
||||||
|
const updateResult = await updateDatasetEntriesMetadata(datasetId, uploadResult.tempDir);
|
||||||
|
|
||||||
|
if (!updateResult.success) {
|
||||||
|
item.status = 'failed';
|
||||||
|
item.progress = 0;
|
||||||
|
item.message = `✗ ${updateResult.error || 'Update failed'}`;
|
||||||
|
} else {
|
||||||
|
item.status = 'completed';
|
||||||
|
item.progress = 100;
|
||||||
|
item.message = `✓ Updated (${updateResult.entriesUpdated} entries)`;
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
item.status = 'failed';
|
||||||
|
item.progress = 0;
|
||||||
|
item.message = `✗ ${err instanceof Error ? err.message : 'Network error'}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
setUploadQueue([...newQueue]);
|
||||||
|
}
|
||||||
|
|
||||||
|
const failedCount = newQueue.filter(item => item.status === 'failed').length;
|
||||||
|
const completedCount = newQueue.filter(item => item.status === 'completed').length;
|
||||||
|
|
||||||
|
if (failedCount === 0) {
|
||||||
|
setSuccess(true);
|
||||||
|
setStatus(`All files processed! (${completedCount}/${newQueue.length} completed)`);
|
||||||
|
} else if (completedCount > 0) {
|
||||||
|
setStatus(`Completed with errors: ${completedCount} succeeded, ${failedCount} failed`);
|
||||||
|
} else {
|
||||||
|
setError(`All files failed to process`);
|
||||||
|
}
|
||||||
|
|
||||||
|
setFiles([]);
|
||||||
|
|
||||||
|
if (failedCount === 0) {
|
||||||
|
setTimeout(() => {
|
||||||
|
setSuccess(false);
|
||||||
|
setIsOpen(false);
|
||||||
|
setStatus('');
|
||||||
|
setUploadQueue([]);
|
||||||
|
window.location.reload();
|
||||||
|
}, 2500);
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
const errorMsg = err instanceof Error ? err.message : 'Failed to process updates';
|
||||||
|
setError(errorMsg);
|
||||||
|
setStatus('');
|
||||||
|
} finally {
|
||||||
|
setLoading(false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function uploadFileWithProgress(
|
||||||
|
file: File,
|
||||||
|
index: number,
|
||||||
|
queue: Array<{ file: File; status: 'pending' | 'uploading' | 'completed' | 'failed'; progress: number; message?: string }>
|
||||||
|
): Promise<{ tempDir: string }> {
|
||||||
|
const MAX_RETRIES = 3;
|
||||||
|
|
||||||
|
return (async () => {
|
||||||
|
let lastError: Error | null = null;
|
||||||
|
|
||||||
|
for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) {
|
||||||
|
try {
|
||||||
|
const result = await uploadAttempt(file, index, queue, attempt, MAX_RETRIES);
|
||||||
|
return result;
|
||||||
|
} catch (error) {
|
||||||
|
lastError = error instanceof Error ? error : new Error('Unknown error');
|
||||||
|
|
||||||
|
if (attempt < MAX_RETRIES) {
|
||||||
|
// Exponential backoff: 1s, 2s, 4s
|
||||||
|
const delay = Math.pow(2, attempt - 1) * 1000;
|
||||||
|
queue[index].message = `Retrying in ${delay / 1000}s... (attempt ${attempt}/${MAX_RETRIES})`;
|
||||||
|
setUploadQueue([...queue]);
|
||||||
|
|
||||||
|
await new Promise(resolve => setTimeout(resolve, delay));
|
||||||
|
queue[index].progress = 0; // Reset progress for next attempt
|
||||||
|
setUploadQueue([...queue]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
throw lastError || new Error('Upload failed after all retries');
|
||||||
|
})();
|
||||||
|
}
|
||||||
|
|
||||||
|
function uploadAttempt(
|
||||||
|
file: File,
|
||||||
|
index: number,
|
||||||
|
queue: Array<{ file: File; status: 'pending' | 'uploading' | 'completed' | 'failed'; progress: number; message?: string }>,
|
||||||
|
attempt: number,
|
||||||
|
maxRetries: number
|
||||||
|
): Promise<{ tempDir: string }> {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
const formData = new FormData();
|
||||||
|
formData.append('file', file);
|
||||||
|
formData.append('datasetId', datasetId.toString());
|
||||||
|
|
||||||
|
const xhr = new XMLHttpRequest();
|
||||||
|
|
||||||
|
// Track upload progress
|
||||||
|
xhr.upload.addEventListener('progress', (event) => {
|
||||||
|
if (event.lengthComputable) {
|
||||||
|
const progress = Math.round((event.loaded / event.total) * 100);
|
||||||
|
queue[index].progress = progress;
|
||||||
|
if (attempt > 1) {
|
||||||
|
queue[index].message = `Uploading (retry ${attempt}/${maxRetries})`;
|
||||||
|
}
|
||||||
|
setUploadQueue([...queue]);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
xhr.addEventListener('load', () => {
|
||||||
|
if (xhr.status === 200) {
|
||||||
|
try {
|
||||||
|
const result = JSON.parse(xhr.responseText);
|
||||||
|
if (result.success) {
|
||||||
|
resolve(result);
|
||||||
|
} else {
|
||||||
|
reject(new Error(result.error || 'Upload failed'));
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
reject(new Error('Failed to parse response'));
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
try {
|
||||||
|
const error = JSON.parse(xhr.responseText);
|
||||||
|
reject(new Error(error.error || `Upload failed with status ${xhr.status}`));
|
||||||
|
} catch {
|
||||||
|
reject(new Error(`Upload failed with status ${xhr.status}`));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
xhr.addEventListener('error', () => {
|
||||||
|
reject(new Error('Network error'));
|
||||||
|
});
|
||||||
|
|
||||||
|
xhr.addEventListener('abort', () => {
|
||||||
|
reject(new Error('Upload aborted'));
|
||||||
|
});
|
||||||
|
|
||||||
|
xhr.open('POST', '/api/update-metadata');
|
||||||
|
xhr.send(formData);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function handleClose() {
|
||||||
|
if (!loading) {
|
||||||
|
setIsOpen(false);
|
||||||
|
setFiles([]);
|
||||||
|
setError(null);
|
||||||
|
setSuccess(false);
|
||||||
|
setStatus('');
|
||||||
|
setUploadQueue([]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return (
|
||||||
|
<>
|
||||||
|
<button
|
||||||
|
onClick={() => setIsOpen(true)}
|
||||||
|
className="mb-6 px-4 py-2 bg-blue-500 text-white rounded-md hover:bg-blue-600 transition-colors"
|
||||||
|
>
|
||||||
|
⟳ Update Metadata
|
||||||
|
</button>
|
||||||
|
|
||||||
|
<Modal
|
||||||
|
isOpen={isOpen}
|
||||||
|
onClose={handleClose}
|
||||||
|
title="Update Dataset Metadata"
|
||||||
|
actions={[
|
||||||
|
{
|
||||||
|
label: 'Cancel',
|
||||||
|
onClick: handleClose,
|
||||||
|
variant: 'secondary',
|
||||||
|
disabled: loading,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
label: loading ? `Processing ${uploadQueue.filter(f => f.status !== 'pending').length}/${uploadQueue.length}...` : 'Upload & Update',
|
||||||
|
onClick: () => {
|
||||||
|
const form = document.getElementById(
|
||||||
|
'updateMetadataForm'
|
||||||
|
) as HTMLFormElement;
|
||||||
|
form?.dispatchEvent(new Event('submit', { bubbles: true }));
|
||||||
|
},
|
||||||
|
variant: 'primary',
|
||||||
|
disabled: loading || files.length === 0,
|
||||||
|
},
|
||||||
|
]}
|
||||||
|
>
|
||||||
|
<form id="updateMetadataForm" onSubmit={handleSubmit} className="space-y-4">
|
||||||
|
<div>
|
||||||
|
<label htmlFor="file" className="block text-sm font-medium mb-2">
|
||||||
|
CSV File
|
||||||
|
</label>
|
||||||
|
<div className="border-2 border-dashed border-gray-300 rounded-lg p-6 text-center">
|
||||||
|
<input
|
||||||
|
id="file"
|
||||||
|
type="file"
|
||||||
|
accept=".csv"
|
||||||
|
multiple
|
||||||
|
onChange={handleFileChange}
|
||||||
|
disabled={loading}
|
||||||
|
className="hidden"
|
||||||
|
/>
|
||||||
|
<label htmlFor="file" className="cursor-pointer block">
|
||||||
|
{files.length > 0 ? (
|
||||||
|
<div>
|
||||||
|
<p className="text-sm font-medium text-blue-600">
|
||||||
|
{files.length} file{files.length !== 1 ? 's' : ''} selected
|
||||||
|
</p>
|
||||||
|
<div className="mt-2 text-xs text-gray-600 max-h-24 overflow-y-auto">
|
||||||
|
{files.map((f, idx) => (
|
||||||
|
<div key={idx} className="py-1">
|
||||||
|
{f.name} ({(f.size / 1024).toFixed(2)} KB)
|
||||||
|
</div>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
) : (
|
||||||
|
<div>
|
||||||
|
<p className="text-sm font-medium text-gray-700">
|
||||||
|
Click to select or drag and drop
|
||||||
|
</p>
|
||||||
|
<p className="text-xs text-gray-500 mt-1">CSV files only (multiple allowed)</p>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</label>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="bg-blue-50 border border-blue-200 rounded-lg p-4 text-sm text-blue-800">
|
||||||
|
<p className="font-semibold mb-2">CSV file format:</p>
|
||||||
|
<ul className="list-disc list-inside space-y-1 text-xs">
|
||||||
|
<li>
|
||||||
|
Columns: <code className="bg-blue-100 px-1 rounded">id, speaker, model, utt_id, text, dialect, iteration, duration_ms, rms_value, longest_pause, utmos_score, wer_score</code>
|
||||||
|
</li>
|
||||||
|
<li>Entry IDs in the CSV must match existing dataset entries to update</li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{uploadQueue.length > 0 && (
|
||||||
|
<div className="bg-gray-50 border border-gray-200 rounded-lg p-4 space-y-3">
|
||||||
|
<p className="font-semibold text-sm text-gray-700">Upload Queue:</p>
|
||||||
|
<div className="space-y-2 max-h-48 overflow-y-auto">
|
||||||
|
{uploadQueue.map((item, idx) => (
|
||||||
|
<div key={idx} className="space-y-1">
|
||||||
|
<div className="flex items-center gap-2 text-xs">
|
||||||
|
<span className={`font-medium ${
|
||||||
|
item.status === 'completed' ? 'text-green-600' :
|
||||||
|
item.status === 'failed' ? 'text-red-600' :
|
||||||
|
item.status === 'uploading' ? 'text-blue-600' :
|
||||||
|
'text-gray-600'
|
||||||
|
}`}>
|
||||||
|
{item.status === 'completed' ? '✓' :
|
||||||
|
item.status === 'failed' ? '✗' :
|
||||||
|
item.status === 'uploading' ? '⟳' :
|
||||||
|
'○'}
|
||||||
|
</span>
|
||||||
|
<span className="flex-1 truncate">{item.file.name}</span>
|
||||||
|
{item.progress > 0 && item.progress < 100 && (
|
||||||
|
<span className="text-gray-500">{item.progress}%</span>
|
||||||
|
)}
|
||||||
|
{item.message && <span className="text-gray-500 text-xs">{item.message}</span>}
|
||||||
|
</div>
|
||||||
|
{item.status === 'uploading' && (
|
||||||
|
<div className="w-full bg-gray-200 rounded-full h-2 overflow-hidden">
|
||||||
|
<div
|
||||||
|
className="bg-blue-500 h-full transition-all duration-300"
|
||||||
|
style={{ width: `${item.progress}%` }}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
{item.status === 'completed' && (
|
||||||
|
<div className="w-full bg-green-200 rounded-full h-2 overflow-hidden">
|
||||||
|
<div className="bg-green-500 h-full w-full" />
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
{item.status === 'failed' && (
|
||||||
|
<div className="w-full bg-red-200 rounded-full h-2 overflow-hidden">
|
||||||
|
<div className="bg-red-500 h-full w-full" />
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{error && (
|
||||||
|
<div className="p-3 bg-red-100 border border-red-400 text-red-700 rounded text-sm">
|
||||||
|
{error}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{status && (
|
||||||
|
<div className={`p-3 rounded text-sm ${success ? 'bg-green-100 border border-green-400 text-green-700' : 'bg-blue-100 border border-blue-400 text-blue-700'}`}>
|
||||||
|
{status}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</form>
|
||||||
|
</Modal>
|
||||||
|
</>
|
||||||
|
);
|
||||||
|
}
|
||||||
34
src/lib/csv-parser.ts
Normal file
34
src/lib/csv-parser.ts
Normal file
@@ -0,0 +1,34 @@
|
|||||||
|
/**
|
||||||
|
* CSV parser that handles quoted fields
|
||||||
|
*/
|
||||||
|
export function parseCSVLine(line: string): string[] {
|
||||||
|
const values: string[] = [];
|
||||||
|
let current = '';
|
||||||
|
let inQuotes = false;
|
||||||
|
|
||||||
|
for (let i = 0; i < line.length; i++) {
|
||||||
|
const char = line[i];
|
||||||
|
const nextChar = line[i + 1];
|
||||||
|
|
||||||
|
if (char === '"') {
|
||||||
|
if (inQuotes && nextChar === '"') {
|
||||||
|
// Escaped quote
|
||||||
|
current += '"';
|
||||||
|
i++;
|
||||||
|
} else {
|
||||||
|
// Toggle quote state
|
||||||
|
inQuotes = !inQuotes;
|
||||||
|
}
|
||||||
|
} else if (char === ',' && !inQuotes) {
|
||||||
|
// Field separator
|
||||||
|
values.push(current.trim());
|
||||||
|
current = '';
|
||||||
|
} else {
|
||||||
|
current += char;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add the last field
|
||||||
|
values.push(current.trim());
|
||||||
|
return values;
|
||||||
|
}
|
||||||
@@ -16,6 +16,10 @@ export const dataset_entry = sqliteTable('dataset_entry', {
|
|||||||
dialect: text('dialect').notNull(),
|
dialect: text('dialect').notNull(),
|
||||||
iteration: integer('iteration').notNull(),
|
iteration: integer('iteration').notNull(),
|
||||||
durationMs: integer('duration_ms'),
|
durationMs: integer('duration_ms'),
|
||||||
|
rmsValue: real('rms_value'),
|
||||||
|
longestPause: real('longest_pause'),
|
||||||
|
utmosScore: real('utmos_score'),
|
||||||
|
werScore: real('wer_score'),
|
||||||
createdAt: integer('created_at', { mode: 'timestamp' })
|
createdAt: integer('created_at', { mode: 'timestamp' })
|
||||||
.notNull()
|
.notNull()
|
||||||
.default(sql`(unixepoch())`),
|
.default(sql`(unixepoch())`),
|
||||||
|
|||||||
Reference in New Issue
Block a user