Merge branch 'main' into prototyp-2-fabio
This commit is contained in:
4
drizzle/0006_special_junta.sql
Normal file
4
drizzle/0006_special_junta.sql
Normal file
@@ -0,0 +1,4 @@
|
||||
ALTER TABLE `dataset_entry` ADD `rms_value` real;--> statement-breakpoint
|
||||
ALTER TABLE `dataset_entry` ADD `longest_pause` real;--> statement-breakpoint
|
||||
ALTER TABLE `dataset_entry` ADD `utmos_score` real;--> statement-breakpoint
|
||||
ALTER TABLE `dataset_entry` ADD `wer_score` real;
|
||||
@@ -1,78 +1,9 @@
|
||||
{
|
||||
"version": "6",
|
||||
"dialect": "sqlite",
|
||||
"id": "59929a2a-9ed3-4f73-8ef7-7af2b8d436b6",
|
||||
"id": "4e72f679-ef9b-47d5-a6fb-a1c45d5cbe74",
|
||||
"prevId": "e6165973-fcea-4b1b-8004-60db966b8106",
|
||||
"tables": {
|
||||
"annotation": {
|
||||
"name": "annotation",
|
||||
"columns": {
|
||||
"id": {
|
||||
"name": "id",
|
||||
"type": "integer",
|
||||
"primaryKey": true,
|
||||
"notNull": true,
|
||||
"autoincrement": true
|
||||
},
|
||||
"dataset_entry_id": {
|
||||
"name": "dataset_entry_id",
|
||||
"type": "integer",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"user_id": {
|
||||
"name": "user_id",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"rating": {
|
||||
"name": "rating",
|
||||
"type": "integer",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"created_at": {
|
||||
"name": "created_at",
|
||||
"type": "integer",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "(unixepoch())"
|
||||
}
|
||||
},
|
||||
"indexes": {
|
||||
"annotation_user_entry_idx": {
|
||||
"name": "annotation_user_entry_idx",
|
||||
"columns": [
|
||||
"user_id",
|
||||
"dataset_entry_id"
|
||||
],
|
||||
"isUnique": true
|
||||
}
|
||||
},
|
||||
"foreignKeys": {
|
||||
"annotation_dataset_entry_id_dataset_entry_id_fk": {
|
||||
"name": "annotation_dataset_entry_id_dataset_entry_id_fk",
|
||||
"tableFrom": "annotation",
|
||||
"tableTo": "dataset_entry",
|
||||
"columnsFrom": [
|
||||
"dataset_entry_id"
|
||||
],
|
||||
"columnsTo": [
|
||||
"id"
|
||||
],
|
||||
"onDelete": "no action",
|
||||
"onUpdate": "no action"
|
||||
}
|
||||
},
|
||||
"compositePrimaryKeys": {},
|
||||
"uniqueConstraints": {},
|
||||
"checkConstraints": {}
|
||||
},
|
||||
"account": {
|
||||
"name": "account",
|
||||
"columns": {
|
||||
@@ -513,6 +444,34 @@
|
||||
"notNull": false,
|
||||
"autoincrement": false
|
||||
},
|
||||
"rms_value": {
|
||||
"name": "rms_value",
|
||||
"type": "real",
|
||||
"primaryKey": false,
|
||||
"notNull": false,
|
||||
"autoincrement": false
|
||||
},
|
||||
"longest_pause": {
|
||||
"name": "longest_pause",
|
||||
"type": "real",
|
||||
"primaryKey": false,
|
||||
"notNull": false,
|
||||
"autoincrement": false
|
||||
},
|
||||
"utmos_score": {
|
||||
"name": "utmos_score",
|
||||
"type": "real",
|
||||
"primaryKey": false,
|
||||
"notNull": false,
|
||||
"autoincrement": false
|
||||
},
|
||||
"wer_score": {
|
||||
"name": "wer_score",
|
||||
"type": "real",
|
||||
"primaryKey": false,
|
||||
"notNull": false,
|
||||
"autoincrement": false
|
||||
},
|
||||
"created_at": {
|
||||
"name": "created_at",
|
||||
"type": "integer",
|
||||
|
||||
@@ -47,15 +47,8 @@
|
||||
{
|
||||
"idx": 6,
|
||||
"version": "6",
|
||||
"when": 1773406795226,
|
||||
"tag": "0006_polite_moondragon",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 7,
|
||||
"version": "6",
|
||||
"when": 1773407494321,
|
||||
"tag": "0007_flawless_wonder_man",
|
||||
"when": 1773406006276,
|
||||
"tag": "0006_special_junta",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
|
||||
@@ -53,6 +53,10 @@ export async function downloadFilteredEntries(
|
||||
'id',
|
||||
'audio_file',
|
||||
'duration_ms',
|
||||
'rms_value',
|
||||
'longest_pause',
|
||||
'utmos_score',
|
||||
'wer_score',
|
||||
'utt_id',
|
||||
'text',
|
||||
'speaker',
|
||||
@@ -67,6 +71,10 @@ export async function downloadFilteredEntries(
|
||||
`"${entry.externalId}"`,
|
||||
entry.fileName,
|
||||
entry.durationMs?.toString() || '',
|
||||
entry.rmsValue?.toString() || '',
|
||||
entry.longestPause?.toString() || '',
|
||||
entry.utmosScore?.toString() || '',
|
||||
entry.werScore?.toString() || '',
|
||||
entry.utteranceId || '',
|
||||
`"${(entry.utteranceText || '').replace(/"/g, '""')}"`,
|
||||
entry.speakerId,
|
||||
|
||||
@@ -2,9 +2,10 @@
|
||||
|
||||
import db from '@/lib/db';
|
||||
import { dataset_entry } from '@/lib/model/dataset_entry';
|
||||
import { count, eq, and, like } from 'drizzle-orm';
|
||||
import { count, eq, and, like, asc, desc, getTableColumns } from 'drizzle-orm';
|
||||
|
||||
const ENTRIES_PER_PAGE = 20;
|
||||
const SORT_COLUMNS = getTableColumns(dataset_entry);
|
||||
|
||||
interface FilterParams {
|
||||
speakerId?: string;
|
||||
@@ -14,10 +15,16 @@ interface FilterParams {
|
||||
utteranceId?: string;
|
||||
}
|
||||
|
||||
interface SortParams {
|
||||
sortBy?: keyof typeof SORT_COLUMNS;
|
||||
sortOrder?: 'asc' | 'desc';
|
||||
}
|
||||
|
||||
export async function getDatasetEntries(
|
||||
datasetId: number,
|
||||
page: number = 1,
|
||||
filters?: FilterParams
|
||||
filters?: FilterParams,
|
||||
sort?: SortParams
|
||||
) {
|
||||
const offset = (page - 1) * ENTRIES_PER_PAGE;
|
||||
|
||||
@@ -46,10 +53,17 @@ export async function getDatasetEntries(
|
||||
|
||||
const whereClause = and(...conditions);
|
||||
|
||||
// Build sort clause
|
||||
const sortFn = sort?.sortOrder === 'desc' ? desc : asc;
|
||||
const sortColumn = sort?.sortBy && sort.sortBy in SORT_COLUMNS
|
||||
? sortFn(SORT_COLUMNS[sort.sortBy as keyof typeof SORT_COLUMNS])
|
||||
: asc(dataset_entry.externalId);
|
||||
|
||||
const entries = await db
|
||||
.select()
|
||||
.from(dataset_entry)
|
||||
.where(whereClause)
|
||||
.orderBy(sortColumn)
|
||||
.limit(ENTRIES_PER_PAGE)
|
||||
.offset(offset);
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
import db from '@/lib/db';
|
||||
import { dataset_entry } from '@/lib/model/dataset_entry';
|
||||
import { parseCSVLine } from '@/lib/csv-parser';
|
||||
import { revalidatePath } from 'next/cache';
|
||||
import { eq } from 'drizzle-orm';
|
||||
import { writeFile, mkdir, readFile, rm } from 'fs/promises';
|
||||
@@ -10,38 +11,71 @@ import { existsSync } from 'fs';
|
||||
import { requireAdmin } from '@/lib/auth';
|
||||
|
||||
/**
|
||||
* CSV parser that handles quoted fields
|
||||
* Helper: Parse metadata.csv from extracted directory
|
||||
* Returns structured data for both insert and update operations
|
||||
*/
|
||||
function parseCSVLine(line: string): string[] {
|
||||
const values: string[] = [];
|
||||
let current = '';
|
||||
let inQuotes = false;
|
||||
|
||||
for (let i = 0; i < line.length; i++) {
|
||||
const char = line[i];
|
||||
const nextChar = line[i + 1];
|
||||
|
||||
if (char === '"') {
|
||||
if (inQuotes && nextChar === '"') {
|
||||
// Escaped quote
|
||||
current += '"';
|
||||
i++;
|
||||
} else {
|
||||
// Toggle quote state
|
||||
inQuotes = !inQuotes;
|
||||
}
|
||||
} else if (char === ',' && !inQuotes) {
|
||||
// Field separator
|
||||
values.push(current.trim());
|
||||
current = '';
|
||||
} else {
|
||||
current += char;
|
||||
}
|
||||
async function parseMetadataFromDirectory(
|
||||
tempDir: string
|
||||
): Promise<{
|
||||
headers: string[];
|
||||
rows: Record<string, string>[];
|
||||
}> {
|
||||
const metadataPath = join(tempDir, 'metadata.csv');
|
||||
if (!existsSync(metadataPath)) {
|
||||
throw new Error('metadata.csv not found in extracted files');
|
||||
}
|
||||
|
||||
// Add the last field
|
||||
values.push(current.trim());
|
||||
return values;
|
||||
const metadataContent = await readFile(metadataPath, 'utf-8');
|
||||
const lines = metadataContent.split('\n').filter((line) => line.trim());
|
||||
|
||||
if (lines.length < 2) {
|
||||
throw new Error('metadata.csv is empty or invalid');
|
||||
}
|
||||
|
||||
const headers = lines[0].split(',').map((h) => h.trim());
|
||||
const rows: Record<string, string>[] = [];
|
||||
|
||||
for (let i = 1; i < lines.length; i++) {
|
||||
const values = parseCSVLine(lines[i]);
|
||||
|
||||
if (values.length !== headers.length) {
|
||||
continue; // Skip malformed lines
|
||||
}
|
||||
|
||||
const row: Record<string, string> = {};
|
||||
headers.forEach((header, index) => {
|
||||
row[header] = values[index];
|
||||
});
|
||||
|
||||
rows.push(row);
|
||||
}
|
||||
|
||||
return { headers, rows };
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert metadata row to dataset_entry insert object
|
||||
*/
|
||||
function rowToDatasetEntry(
|
||||
row: Record<string, string>,
|
||||
datasetId: number
|
||||
): typeof dataset_entry.$inferInsert {
|
||||
return {
|
||||
datasetId,
|
||||
externalId: row.id,
|
||||
speakerId: row.speaker,
|
||||
modelName: row.model,
|
||||
utteranceId: row.utt_id,
|
||||
utteranceText: row.text,
|
||||
fileName: row.audio_file,
|
||||
dialect: row.dialect,
|
||||
iteration: parseInt(row.iteration, 10),
|
||||
durationMs: row.duration_ms ? parseInt(row.duration_ms, 10) : undefined,
|
||||
rmsValue: row.rms_value ? parseFloat(row.rms_value) : undefined,
|
||||
longestPause: row.longest_pause ? parseFloat(row.longest_pause) : undefined,
|
||||
utmosScore: row.utmos_score ? parseFloat(row.utmos_score) : undefined,
|
||||
werScore: row.wer_score ? parseFloat(row.wer_score) : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -73,47 +107,16 @@ export async function processDatasetEntries(
|
||||
};
|
||||
}
|
||||
|
||||
// Read metadata.csv again
|
||||
const metadataPath = join(tempDir, 'metadata.csv');
|
||||
if (!existsSync(metadataPath)) {
|
||||
return {
|
||||
success: false,
|
||||
error: 'metadata.csv not found in extracted files',
|
||||
entriesCreated: 0,
|
||||
};
|
||||
}
|
||||
// Parse metadata from directory
|
||||
const { rows } = await parseMetadataFromDirectory(tempDir);
|
||||
|
||||
const metadataContent = await readFile(metadataPath, 'utf-8');
|
||||
const lines = metadataContent.split('\n').filter(line => line.trim());
|
||||
|
||||
if (lines.length < 2) {
|
||||
return {
|
||||
success: false,
|
||||
error: 'metadata.csv is empty or invalid',
|
||||
entriesCreated: 0,
|
||||
};
|
||||
}
|
||||
|
||||
// Parse CSV
|
||||
const headers = lines[0].split(',').map(h => h.trim());
|
||||
const entries: typeof dataset_entry.$inferInsert[] = [];
|
||||
|
||||
// Create dataset directory
|
||||
await mkdir(datasetDir, { recursive: true });
|
||||
|
||||
// Process each row
|
||||
for (let i = 1; i < lines.length; i++) {
|
||||
const values = parseCSVLine(lines[i]);
|
||||
|
||||
if (values.length !== headers.length) {
|
||||
continue; // Skip malformed lines
|
||||
}
|
||||
|
||||
const row: Record<string, string> = {};
|
||||
headers.forEach((header, index) => {
|
||||
row[header] = values[index];
|
||||
});
|
||||
|
||||
for (const row of rows) {
|
||||
const audioFile = row.audio_file as string;
|
||||
const relativePath = audioFile.replace(/\\/g, '/');
|
||||
const audioPath = join(tempDir, relativePath);
|
||||
@@ -126,18 +129,7 @@ export async function processDatasetEntries(
|
||||
|
||||
await writeFile(destPath, audioBuffer);
|
||||
|
||||
entries.push({
|
||||
datasetId,
|
||||
externalId: row.id,
|
||||
speakerId: row.speaker,
|
||||
modelName: row.model,
|
||||
utteranceId: row.utt_id,
|
||||
utteranceText: row.text,
|
||||
fileName: relativePath,
|
||||
dialect: row.dialect,
|
||||
iteration: parseInt(row.iteration, 10),
|
||||
durationMs: row.duration_ms ? parseInt(row.duration_ms, 10) : undefined,
|
||||
});
|
||||
entries.push(rowToDatasetEntry(row, datasetId));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -197,3 +189,116 @@ export async function processDatasetEntries(
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update metadata for existing dataset entries
|
||||
* Uses the same metadata.csv parsing as processDatasetEntries
|
||||
* Matches entries by externalId and updates their metadata fields
|
||||
*/
|
||||
export async function updateDatasetEntriesMetadata(
|
||||
datasetId: number,
|
||||
tempDir: string
|
||||
) {
|
||||
try {
|
||||
const result = await requireAdmin();
|
||||
if (!result.authenticated || !result.admin) {
|
||||
return {
|
||||
success: false,
|
||||
error: 'Unauthorized',
|
||||
entriesUpdated: 0,
|
||||
};
|
||||
}
|
||||
|
||||
// Validate temp directory exists
|
||||
if (!existsSync(tempDir)) {
|
||||
return {
|
||||
success: false,
|
||||
error: 'Temporary extraction directory not found',
|
||||
entriesUpdated: 0,
|
||||
};
|
||||
}
|
||||
|
||||
// Parse metadata from directory (reusing same logic as processDatasetEntries)
|
||||
const { rows } = await parseMetadataFromDirectory(tempDir);
|
||||
|
||||
if (rows.length === 0) {
|
||||
return {
|
||||
success: false,
|
||||
error: 'No valid entries found in metadata.csv',
|
||||
entriesUpdated: 0,
|
||||
};
|
||||
}
|
||||
|
||||
// Get all existing entries for this dataset
|
||||
const existingEntries = await db
|
||||
.select()
|
||||
.from(dataset_entry)
|
||||
.where(eq(dataset_entry.datasetId, datasetId));
|
||||
|
||||
const existingByExternalId = new Map(
|
||||
existingEntries.map(entry => [entry.externalId, entry])
|
||||
);
|
||||
|
||||
let entriesUpdated = 0;
|
||||
let entriesNotFound = 0;
|
||||
|
||||
// Update each row that has a matching entry
|
||||
for (const row of rows) {
|
||||
const externalId = row.id;
|
||||
const existingEntry = existingByExternalId.get(externalId);
|
||||
|
||||
if (!existingEntry) {
|
||||
entriesNotFound++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Update only the metadata fields, don't change the ID or dataset reference
|
||||
await db
|
||||
.update(dataset_entry)
|
||||
.set({
|
||||
speakerId: row.speaker,
|
||||
modelName: row.model,
|
||||
utteranceId: row.utt_id,
|
||||
utteranceText: row.text,
|
||||
dialect: row.dialect,
|
||||
iteration: parseInt(row.iteration, 10),
|
||||
durationMs: row.duration_ms ? parseInt(row.duration_ms, 10) : undefined,
|
||||
rmsValue: row.rms_value ? parseFloat(row.rms_value) : undefined,
|
||||
longestPause: row.longest_pause ? parseFloat(row.longest_pause) : undefined,
|
||||
utmosScore: row.utmos_score ? parseFloat(row.utmos_score) : undefined,
|
||||
werScore: row.wer_score ? parseFloat(row.wer_score) : undefined,
|
||||
})
|
||||
.where(eq(dataset_entry.id, existingEntry.id));
|
||||
|
||||
entriesUpdated++;
|
||||
}
|
||||
|
||||
revalidatePath(`/admin/datasets/${datasetId}`);
|
||||
|
||||
return {
|
||||
success: true,
|
||||
entriesUpdated,
|
||||
entriesNotFound,
|
||||
message: entriesUpdated > 0
|
||||
? `Successfully updated ${entriesUpdated} entries${entriesNotFound > 0 ? ` (${entriesNotFound} not found)` : ''}.`
|
||||
: `No entries were updated (${entriesNotFound} not found).`,
|
||||
};
|
||||
} catch (error) {
|
||||
console.error('Error updating dataset entries metadata:', error);
|
||||
const errorMessage = error instanceof Error ? error.message : 'Failed to update dataset entries metadata';
|
||||
return {
|
||||
success: false,
|
||||
error: errorMessage,
|
||||
entriesUpdated: 0,
|
||||
};
|
||||
} finally {
|
||||
// Clean up temp directory
|
||||
try {
|
||||
if (existsSync(tempDir)) {
|
||||
await rm(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
} catch (cleanupError) {
|
||||
console.error('Error cleaning up temp directory:', cleanupError);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -36,7 +36,7 @@ export default async function DatasetEntryPage({ params }: DatasetEntryPageProps
|
||||
if (entries.length === 0 || entries[0].datasetId !== datasetIdNum) {
|
||||
return (
|
||||
<div className="min-h-screen">
|
||||
<div className="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8 py-8">
|
||||
<div className="px-4 sm:px-6 lg:px-8 py-8">
|
||||
<Link
|
||||
href={`/admin/datasets/${datasetId}`}
|
||||
className="text-blue-500 hover:text-blue-600 mb-6 inline-block"
|
||||
@@ -55,7 +55,7 @@ export default async function DatasetEntryPage({ params }: DatasetEntryPageProps
|
||||
|
||||
return (
|
||||
<div>
|
||||
<div className="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8 py-8">
|
||||
<div className="px-4 sm:px-6 lg:px-8 py-8">
|
||||
<Link
|
||||
href={`/admin/datasets/${datasetId}`}
|
||||
className="text-blue-500 hover:text-blue-600 mb-6 inline-block"
|
||||
@@ -122,6 +122,26 @@ export default async function DatasetEntryPage({ params }: DatasetEntryPageProps
|
||||
<p className="text-lg font-semibold">{entry.durationMs?.toLocaleString() || '-'}</p>
|
||||
</div>
|
||||
|
||||
<div className="bg-white border border-gray-200 p-4 rounded-lg">
|
||||
<p className="text-sm text-gray-600 mb-1">RMS Value</p>
|
||||
<p className="text-lg font-semibold">{entry.rmsValue?.toFixed(3) || '-'}</p>
|
||||
</div>
|
||||
|
||||
<div className="bg-white border border-gray-200 p-4 rounded-lg">
|
||||
<p className="text-sm text-gray-600 mb-1">Longest Pause (s)</p>
|
||||
<p className="text-lg font-semibold">{entry.longestPause?.toFixed(3) || '-'}</p>
|
||||
</div>
|
||||
|
||||
<div className="bg-white border border-gray-200 p-4 rounded-lg">
|
||||
<p className="text-sm text-gray-600 mb-1">UTMOS Score</p>
|
||||
<p className="text-lg font-semibold">{entry.utmosScore?.toFixed(3) || '-'}</p>
|
||||
</div>
|
||||
|
||||
<div className="bg-white border border-gray-200 p-4 rounded-lg">
|
||||
<p className="text-sm text-gray-600 mb-1">WER Score</p>
|
||||
<p className="text-lg font-semibold">{entry.werScore?.toFixed(3) || '-'}</p>
|
||||
</div>
|
||||
|
||||
<div className="bg-white border border-gray-200 p-4 rounded-lg">
|
||||
<p className="text-sm text-gray-600 mb-1">Created</p>
|
||||
<p className="text-lg font-semibold">
|
||||
|
||||
@@ -5,6 +5,7 @@ import { eq } from 'drizzle-orm';
|
||||
import EditableDatasetHeader from '@/components/EditableDatasetHeader';
|
||||
import DatasetEntriesList from '@/components/DatasetEntriesList';
|
||||
import UploadDatasetEntriesModal from '@/components/UploadDatasetEntriesModal';
|
||||
import UpdateDatasetMetadataModal from '@/components/UpdateDatasetMetadataModal';
|
||||
import RemoveAllEntriesButton from '@/components/RemoveAllEntriesButton';
|
||||
import DeleteDatasetButton from '@/components/DeleteDatasetButton';
|
||||
import { requireAdmin } from '@/lib/auth';
|
||||
@@ -38,7 +39,7 @@ export default async function DatasetPage({ params }: DatasetPageProps) {
|
||||
if (datasets.length === 0) {
|
||||
return (
|
||||
<div className="min-h-screen">
|
||||
<div className="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8 py-8">
|
||||
<div className="px-4 sm:px-6 lg:px-8 py-8">
|
||||
<Link
|
||||
href="/admin"
|
||||
className="text-blue-500 hover:text-blue-600 mb-6 inline-block"
|
||||
@@ -57,7 +58,7 @@ export default async function DatasetPage({ params }: DatasetPageProps) {
|
||||
|
||||
return (
|
||||
<div>
|
||||
<div className="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8 py-8">
|
||||
<div className="px-4 sm:px-6 lg:px-8 py-8">
|
||||
<Link
|
||||
href="/admin"
|
||||
className="text-blue-500 hover:text-blue-600 mb-6 inline-block"
|
||||
@@ -69,6 +70,7 @@ export default async function DatasetPage({ params }: DatasetPageProps) {
|
||||
|
||||
<div className="flex gap-3 mb-6">
|
||||
<UploadDatasetEntriesModal datasetId={datasetId} />
|
||||
<UpdateDatasetMetadataModal datasetId={datasetId} />
|
||||
<RemoveAllEntriesButton datasetId={datasetId} />
|
||||
<DeleteDatasetButton datasetId={datasetId} datasetName={ds.name} />
|
||||
</div>
|
||||
|
||||
@@ -18,7 +18,7 @@ export default async function AdminPage() {
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8 py-8">
|
||||
<div className="px-4 sm:px-6 lg:px-8 py-8">
|
||||
<div className="mb-8">
|
||||
<h1 className="text-3xl font-bold mb-4">Admin Panel</h1>
|
||||
<div className="flex gap-4">
|
||||
|
||||
@@ -18,7 +18,7 @@ export default async function UsersPage() {
|
||||
const users = await getAllUsers();
|
||||
|
||||
return (
|
||||
<div className="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8 py-8">
|
||||
<div className="px-4 sm:px-6 lg:px-8 py-8">
|
||||
<div className="mb-6">
|
||||
<Link
|
||||
href="/admin"
|
||||
|
||||
74
src/app/api/update-metadata/route.ts
Normal file
74
src/app/api/update-metadata/route.ts
Normal file
@@ -0,0 +1,74 @@
|
||||
import { writeFile, mkdir, rm } from 'fs/promises';
|
||||
import { join } from 'path';
|
||||
import { existsSync } from 'fs';
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { requireAdmin } from '@/lib/auth';
|
||||
|
||||
export async function POST(request: NextRequest) {
|
||||
try {
|
||||
const result = await requireAdmin();
|
||||
if (!result.authenticated || !result.admin) {
|
||||
return NextResponse.json(
|
||||
{ error: 'Unauthorized' },
|
||||
{ status: 401 }
|
||||
);
|
||||
}
|
||||
|
||||
const formData = await request.formData();
|
||||
const file = formData.get('file') as File;
|
||||
const datasetId = formData.get('datasetId') as string;
|
||||
|
||||
if (!file) {
|
||||
return NextResponse.json(
|
||||
{ error: 'No file provided' },
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
|
||||
if (!file.name.endsWith('.csv')) {
|
||||
return NextResponse.json(
|
||||
{ error: 'File must be a CSV file' },
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
|
||||
const tempDir = join(process.cwd(), 'tmp', `update-metadata-${Date.now()}`);
|
||||
|
||||
try {
|
||||
// Create temp directory
|
||||
await mkdir(tempDir, { recursive: true });
|
||||
|
||||
// Save uploaded CSV file as metadata.csv
|
||||
const arrayBuffer = await file.arrayBuffer();
|
||||
const buffer = Buffer.from(arrayBuffer);
|
||||
const metadataPath = join(tempDir, 'metadata.csv');
|
||||
await writeFile(metadataPath, buffer);
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
tempDir,
|
||||
message: `Successfully uploaded metadata.csv. Ready for processing.`,
|
||||
});
|
||||
} catch (error) {
|
||||
// Clean up on error
|
||||
try {
|
||||
if (existsSync(tempDir)) {
|
||||
await rm(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
} catch (cleanupError) {
|
||||
console.error('Error cleaning up temp directory:', cleanupError);
|
||||
}
|
||||
|
||||
throw error;
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error uploading metadata CSV:', error);
|
||||
const errorMsg =
|
||||
error instanceof Error ? error.message : 'Failed to upload metadata CSV';
|
||||
return NextResponse.json(
|
||||
{ error: errorMsg },
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,7 +13,7 @@ export default function RootLayout({
|
||||
<body className="flex flex-col min-h-screen">
|
||||
<AudioProvider>
|
||||
<Header />
|
||||
<main className="flex-grow mt-4 mb-4 mr-8 ml-8">
|
||||
<main className="flex-grow">
|
||||
{children}
|
||||
</main>
|
||||
<Footer />
|
||||
|
||||
@@ -22,6 +22,11 @@ interface DatasetEntry {
|
||||
fileName: string;
|
||||
dialect: string;
|
||||
iteration: number;
|
||||
durationMs: number | null;
|
||||
rmsValue: number | null;
|
||||
longestPause: number | null;
|
||||
utmosScore: number | null;
|
||||
werScore: number | null;
|
||||
createdAt: Date;
|
||||
updatedAt: Date;
|
||||
}
|
||||
@@ -62,6 +67,10 @@ export default function DatasetEntriesList({
|
||||
utteranceIds: [],
|
||||
});
|
||||
|
||||
// Sort state
|
||||
const [sortBy, setSortBy] = useState<keyof DatasetEntry>('externalId');
|
||||
const [sortOrder, setSortOrder] = useState<'asc' | 'desc'>('asc');
|
||||
|
||||
// Load filter options on mount
|
||||
useEffect(() => {
|
||||
async function loadFilterOptions() {
|
||||
@@ -78,7 +87,7 @@ export default function DatasetEntriesList({
|
||||
useEffect(() => {
|
||||
setPage(1);
|
||||
loadEntries(1);
|
||||
}, [filters]);
|
||||
}, [filters, sortBy, sortOrder]);
|
||||
|
||||
async function loadEntries(pageNum: number = page) {
|
||||
setLoading(true);
|
||||
@@ -91,7 +100,12 @@ export default function DatasetEntriesList({
|
||||
utteranceId: filters.utteranceId || undefined,
|
||||
};
|
||||
|
||||
const result = await getDatasetEntries(datasetId, pageNum, filterParams);
|
||||
const sortParams = {
|
||||
sortBy: sortBy as keyof DatasetEntry,
|
||||
sortOrder: sortOrder,
|
||||
};
|
||||
|
||||
const result = await getDatasetEntries(datasetId, pageNum, filterParams, sortParams);
|
||||
if (pageNum === 1) {
|
||||
setEntries(result.entries);
|
||||
} else {
|
||||
@@ -118,7 +132,12 @@ export default function DatasetEntriesList({
|
||||
utteranceId: filters.utteranceId || undefined,
|
||||
};
|
||||
|
||||
const result = await getDatasetEntries(datasetId, page + 1, filterParams);
|
||||
const sortParams = {
|
||||
sortBy: sortBy as keyof DatasetEntry,
|
||||
sortOrder: sortOrder,
|
||||
};
|
||||
|
||||
const result = await getDatasetEntries(datasetId, page + 1, filterParams, sortParams);
|
||||
setEntries((prev) => [...prev, ...result.entries]);
|
||||
setHasMore(result.hasMore);
|
||||
setTotal(result.total);
|
||||
@@ -147,6 +166,17 @@ export default function DatasetEntriesList({
|
||||
});
|
||||
}
|
||||
|
||||
function handleSort(column: keyof DatasetEntry) {
|
||||
if (sortBy === column) {
|
||||
// Toggle sort order if clicking the same column
|
||||
setSortOrder(sortOrder === 'asc' ? 'desc' : 'asc');
|
||||
} else {
|
||||
// Set new column to sort by, default to ascending
|
||||
setSortBy(column);
|
||||
setSortOrder('asc');
|
||||
}
|
||||
}
|
||||
|
||||
async function handleDownloadFiltered() {
|
||||
setDownloading(true);
|
||||
try {
|
||||
@@ -410,13 +440,39 @@ export default function DatasetEntriesList({
|
||||
<thead>
|
||||
<tr className="border-b border-gray-200">
|
||||
<th className="text-left py-3 px-4 font-semibold text-gray-700">Play</th>
|
||||
<th className="text-left py-3 px-4 font-semibold text-gray-700">External ID</th>
|
||||
<th className="text-left py-3 px-4 font-semibold text-gray-700">Speaker ID</th>
|
||||
<th className="text-left py-3 px-4 font-semibold text-gray-700">Model Name</th>
|
||||
<th className="text-left py-3 px-4 font-semibold text-gray-700">Utterance ID</th>
|
||||
<th className="text-left py-3 px-4 font-semibold text-gray-700">Dialect</th>
|
||||
<th className="text-left py-3 px-4 font-semibold text-gray-700">File Name</th>
|
||||
<th className="text-left py-3 px-4 font-semibold text-gray-700">Iteration</th>
|
||||
<th className="text-left py-3 px-4 font-semibold text-gray-700 cursor-pointer hover:bg-gray-100" onClick={() => handleSort('externalId')}>
|
||||
External ID {sortBy === 'externalId' && (sortOrder === 'asc' ? '↑' : '↓')}
|
||||
</th>
|
||||
<th className="text-left py-3 px-4 font-semibold text-gray-700 cursor-pointer hover:bg-gray-100" onClick={() => handleSort('speakerId')}>
|
||||
Speaker ID {sortBy === 'speakerId' && (sortOrder === 'asc' ? '↑' : '↓')}
|
||||
</th>
|
||||
<th className="text-left py-3 px-4 font-semibold text-gray-700 cursor-pointer hover:bg-gray-100" onClick={() => handleSort('modelName')}>
|
||||
Model Name {sortBy === 'modelName' && (sortOrder === 'asc' ? '↑' : '↓')}
|
||||
</th>
|
||||
<th className="text-left py-3 px-4 font-semibold text-gray-700 cursor-pointer hover:bg-gray-100" onClick={() => handleSort('utteranceId')}>
|
||||
Utterance ID {sortBy === 'utteranceId' && (sortOrder === 'asc' ? '↑' : '↓')}
|
||||
</th>
|
||||
<th className="text-left py-3 px-4 font-semibold text-gray-700 cursor-pointer hover:bg-gray-100" onClick={() => handleSort('dialect')}>
|
||||
Dialect {sortBy === 'dialect' && (sortOrder === 'asc' ? '↑' : '↓')}
|
||||
</th>
|
||||
<th className="text-left py-3 px-4 font-semibold text-gray-700 cursor-pointer hover:bg-gray-100" onClick={() => handleSort('iteration')}>
|
||||
Iteration {sortBy === 'iteration' && (sortOrder === 'asc' ? '↑' : '↓')}
|
||||
</th>
|
||||
<th className="text-left py-3 px-4 font-semibold text-gray-700 cursor-pointer hover:bg-gray-100" onClick={() => handleSort('durationMs')}>
|
||||
Duration (ms) {sortBy === 'durationMs' && (sortOrder === 'asc' ? '↑' : '↓')}
|
||||
</th>
|
||||
<th className="text-left py-3 px-4 font-semibold text-gray-700 cursor-pointer hover:bg-gray-100" onClick={() => handleSort('rmsValue')}>
|
||||
RMS Value {sortBy === 'rmsValue' && (sortOrder === 'asc' ? '↑' : '↓')}
|
||||
</th>
|
||||
<th className="text-left py-3 px-4 font-semibold text-gray-700 cursor-pointer hover:bg-gray-100" onClick={() => handleSort('longestPause')}>
|
||||
Longest Pause (s) {sortBy === 'longestPause' && (sortOrder === 'asc' ? '↑' : '↓')}
|
||||
</th>
|
||||
<th className="text-left py-3 px-4 font-semibold text-gray-700 cursor-pointer hover:bg-gray-100" onClick={() => handleSort('utmosScore')}>
|
||||
UTMOS Score {sortBy === 'utmosScore' && (sortOrder === 'asc' ? '↑' : '↓')}
|
||||
</th>
|
||||
<th className="text-left py-3 px-4 font-semibold text-gray-700 cursor-pointer hover:bg-gray-100" onClick={() => handleSort('werScore')}>
|
||||
WER Score {sortBy === 'werScore' && (sortOrder === 'asc' ? '↑' : '↓')}
|
||||
</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
@@ -435,10 +491,14 @@ export default function DatasetEntriesList({
|
||||
</td>
|
||||
<td className="py-3 px-4">{entry.speakerId}</td>
|
||||
<td className="py-3 px-4">{entry.modelName}</td>
|
||||
<td className="py-3 px-4 truncate">{entry.utteranceId || '-'}</td>
|
||||
<td className="py-3 px-4">{entry.utteranceId || '-'}</td>
|
||||
<td className="py-3 px-4">{entry.dialect}</td>
|
||||
<td className="py-3 px-4 truncate">{entry.fileName}</td>
|
||||
<td className="py-3 px-4">{entry.iteration}</td>
|
||||
<td className="py-3 px-4">{entry.durationMs?.toLocaleString() || '-'}</td>
|
||||
<td className="py-3 px-4">{entry.rmsValue?.toFixed(3) || '-'}</td>
|
||||
<td className="py-3 px-4">{entry.longestPause?.toFixed(3) || '-'}</td>
|
||||
<td className="py-3 px-4">{entry.utmosScore?.toFixed(3) || '-'}</td>
|
||||
<td className="py-3 px-4">{entry.werScore?.toFixed(3) || '-'}</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
|
||||
@@ -3,7 +3,7 @@ export function Footer() {
|
||||
|
||||
return (
|
||||
<footer className="bg-gray-800 text-gray-300 mt-12">
|
||||
<div className="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8 py-8">
|
||||
<div className="px-4 sm:px-6 lg:px-8 py-8">
|
||||
<div className="grid grid-cols-1 md:grid-cols-3 gap-8 mb-8">
|
||||
<div>
|
||||
<h3 className="text-white font-semibold mb-4">Über das Projekt</h3>
|
||||
|
||||
@@ -16,7 +16,7 @@ export function Header() {
|
||||
|
||||
return (
|
||||
<header className="sticky top-0 z-50 bg-gradient-to-r from-blue-600 to-blue-700 text-white shadow-lg">
|
||||
<div className="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8">
|
||||
<div className="px-4 sm:px-6 lg:px-8">
|
||||
<div className="flex justify-between items-center py-5">
|
||||
<Link href="/" className="text-2xl font-bold hover:text-blue-100 transition-colors">
|
||||
TTS Dialektannotations-Plattform
|
||||
|
||||
365
src/components/UpdateDatasetMetadataModal.tsx
Normal file
365
src/components/UpdateDatasetMetadataModal.tsx
Normal file
@@ -0,0 +1,365 @@
|
||||
'use client';
|
||||
|
||||
import { useState } from 'react';
|
||||
import { updateDatasetEntriesMetadata } from '@/app/actions/upload';
|
||||
import Modal from '@/components/Modal';
|
||||
|
||||
interface UpdateDatasetMetadataModalProps {
|
||||
datasetId: number;
|
||||
}
|
||||
|
||||
export default function UpdateDatasetMetadataModal({
|
||||
datasetId,
|
||||
}: UpdateDatasetMetadataModalProps) {
|
||||
const [isOpen, setIsOpen] = useState(false);
|
||||
const [files, setFiles] = useState<File[]>([]);
|
||||
const [loading, setLoading] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [success, setSuccess] = useState(false);
|
||||
const [uploadQueue, setUploadQueue] = useState<Array<{ file: File; status: 'pending' | 'uploading' | 'completed' | 'failed'; message?: string; progress: number }>>([]);
|
||||
const [status, setStatus] = useState<string>('');
|
||||
|
||||
function handleFileChange(e: React.ChangeEvent<HTMLInputElement>) {
|
||||
const selectedFiles = e.target.files ? Array.from(e.target.files) : [];
|
||||
if (selectedFiles.length > 0) {
|
||||
setFiles(selectedFiles);
|
||||
setError(null);
|
||||
// Initialize queue with pending status and 0 progress
|
||||
setUploadQueue(selectedFiles.map(file => ({ file, status: 'pending', progress: 0 })));
|
||||
}
|
||||
}
|
||||
|
||||
async function handleSubmit(e: React.FormEvent) {
|
||||
e.preventDefault();
|
||||
setError(null);
|
||||
setSuccess(false);
|
||||
|
||||
if (files.length === 0) {
|
||||
setError('Please select at least one CSV file');
|
||||
return;
|
||||
}
|
||||
|
||||
setLoading(true);
|
||||
|
||||
try {
|
||||
// Process each file in the queue sequentially
|
||||
const newQueue = [...uploadQueue];
|
||||
|
||||
for (let i = 0; i < newQueue.length; i++) {
|
||||
const item = newQueue[i];
|
||||
item.status = 'uploading';
|
||||
item.progress = 0;
|
||||
setUploadQueue([...newQueue]);
|
||||
setStatus(`Processing file ${i + 1} of ${newQueue.length}: ${item.file.name}`);
|
||||
|
||||
try {
|
||||
// Upload with progress tracking
|
||||
const uploadResult = await uploadFileWithProgress(item.file, i, newQueue);
|
||||
|
||||
// Update the metadata
|
||||
const updateResult = await updateDatasetEntriesMetadata(datasetId, uploadResult.tempDir);
|
||||
|
||||
if (!updateResult.success) {
|
||||
item.status = 'failed';
|
||||
item.progress = 0;
|
||||
item.message = `✗ ${updateResult.error || 'Update failed'}`;
|
||||
} else {
|
||||
item.status = 'completed';
|
||||
item.progress = 100;
|
||||
item.message = `✓ Updated (${updateResult.entriesUpdated} entries)`;
|
||||
}
|
||||
} catch (err) {
|
||||
item.status = 'failed';
|
||||
item.progress = 0;
|
||||
item.message = `✗ ${err instanceof Error ? err.message : 'Network error'}`;
|
||||
}
|
||||
|
||||
setUploadQueue([...newQueue]);
|
||||
}
|
||||
|
||||
const failedCount = newQueue.filter(item => item.status === 'failed').length;
|
||||
const completedCount = newQueue.filter(item => item.status === 'completed').length;
|
||||
|
||||
if (failedCount === 0) {
|
||||
setSuccess(true);
|
||||
setStatus(`All files processed! (${completedCount}/${newQueue.length} completed)`);
|
||||
} else if (completedCount > 0) {
|
||||
setStatus(`Completed with errors: ${completedCount} succeeded, ${failedCount} failed`);
|
||||
} else {
|
||||
setError(`All files failed to process`);
|
||||
}
|
||||
|
||||
setFiles([]);
|
||||
|
||||
if (failedCount === 0) {
|
||||
setTimeout(() => {
|
||||
setSuccess(false);
|
||||
setIsOpen(false);
|
||||
setStatus('');
|
||||
setUploadQueue([]);
|
||||
window.location.reload();
|
||||
}, 2500);
|
||||
}
|
||||
} catch (err) {
|
||||
const errorMsg = err instanceof Error ? err.message : 'Failed to process updates';
|
||||
setError(errorMsg);
|
||||
setStatus('');
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
}
|
||||
|
||||
function uploadFileWithProgress(
|
||||
file: File,
|
||||
index: number,
|
||||
queue: Array<{ file: File; status: 'pending' | 'uploading' | 'completed' | 'failed'; progress: number; message?: string }>
|
||||
): Promise<{ tempDir: string }> {
|
||||
const MAX_RETRIES = 3;
|
||||
|
||||
return (async () => {
|
||||
let lastError: Error | null = null;
|
||||
|
||||
for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) {
|
||||
try {
|
||||
const result = await uploadAttempt(file, index, queue, attempt, MAX_RETRIES);
|
||||
return result;
|
||||
} catch (error) {
|
||||
lastError = error instanceof Error ? error : new Error('Unknown error');
|
||||
|
||||
if (attempt < MAX_RETRIES) {
|
||||
// Exponential backoff: 1s, 2s, 4s
|
||||
const delay = Math.pow(2, attempt - 1) * 1000;
|
||||
queue[index].message = `Retrying in ${delay / 1000}s... (attempt ${attempt}/${MAX_RETRIES})`;
|
||||
setUploadQueue([...queue]);
|
||||
|
||||
await new Promise(resolve => setTimeout(resolve, delay));
|
||||
queue[index].progress = 0; // Reset progress for next attempt
|
||||
setUploadQueue([...queue]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
throw lastError || new Error('Upload failed after all retries');
|
||||
})();
|
||||
}
|
||||
|
||||
function uploadAttempt(
|
||||
file: File,
|
||||
index: number,
|
||||
queue: Array<{ file: File; status: 'pending' | 'uploading' | 'completed' | 'failed'; progress: number; message?: string }>,
|
||||
attempt: number,
|
||||
maxRetries: number
|
||||
): Promise<{ tempDir: string }> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const formData = new FormData();
|
||||
formData.append('file', file);
|
||||
formData.append('datasetId', datasetId.toString());
|
||||
|
||||
const xhr = new XMLHttpRequest();
|
||||
|
||||
// Track upload progress
|
||||
xhr.upload.addEventListener('progress', (event) => {
|
||||
if (event.lengthComputable) {
|
||||
const progress = Math.round((event.loaded / event.total) * 100);
|
||||
queue[index].progress = progress;
|
||||
if (attempt > 1) {
|
||||
queue[index].message = `Uploading (retry ${attempt}/${maxRetries})`;
|
||||
}
|
||||
setUploadQueue([...queue]);
|
||||
}
|
||||
});
|
||||
|
||||
xhr.addEventListener('load', () => {
|
||||
if (xhr.status === 200) {
|
||||
try {
|
||||
const result = JSON.parse(xhr.responseText);
|
||||
if (result.success) {
|
||||
resolve(result);
|
||||
} else {
|
||||
reject(new Error(result.error || 'Upload failed'));
|
||||
}
|
||||
} catch (error) {
|
||||
reject(new Error('Failed to parse response'));
|
||||
}
|
||||
} else {
|
||||
try {
|
||||
const error = JSON.parse(xhr.responseText);
|
||||
reject(new Error(error.error || `Upload failed with status ${xhr.status}`));
|
||||
} catch {
|
||||
reject(new Error(`Upload failed with status ${xhr.status}`));
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
xhr.addEventListener('error', () => {
|
||||
reject(new Error('Network error'));
|
||||
});
|
||||
|
||||
xhr.addEventListener('abort', () => {
|
||||
reject(new Error('Upload aborted'));
|
||||
});
|
||||
|
||||
xhr.open('POST', '/api/update-metadata');
|
||||
xhr.send(formData);
|
||||
});
|
||||
}
|
||||
|
||||
function handleClose() {
|
||||
if (!loading) {
|
||||
setIsOpen(false);
|
||||
setFiles([]);
|
||||
setError(null);
|
||||
setSuccess(false);
|
||||
setStatus('');
|
||||
setUploadQueue([]);
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<>
|
||||
<button
|
||||
onClick={() => setIsOpen(true)}
|
||||
className="mb-6 px-4 py-2 bg-blue-500 text-white rounded-md hover:bg-blue-600 transition-colors"
|
||||
>
|
||||
⟳ Update Metadata
|
||||
</button>
|
||||
|
||||
<Modal
|
||||
isOpen={isOpen}
|
||||
onClose={handleClose}
|
||||
title="Update Dataset Metadata"
|
||||
actions={[
|
||||
{
|
||||
label: 'Cancel',
|
||||
onClick: handleClose,
|
||||
variant: 'secondary',
|
||||
disabled: loading,
|
||||
},
|
||||
{
|
||||
label: loading ? `Processing ${uploadQueue.filter(f => f.status !== 'pending').length}/${uploadQueue.length}...` : 'Upload & Update',
|
||||
onClick: () => {
|
||||
const form = document.getElementById(
|
||||
'updateMetadataForm'
|
||||
) as HTMLFormElement;
|
||||
form?.dispatchEvent(new Event('submit', { bubbles: true }));
|
||||
},
|
||||
variant: 'primary',
|
||||
disabled: loading || files.length === 0,
|
||||
},
|
||||
]}
|
||||
>
|
||||
<form id="updateMetadataForm" onSubmit={handleSubmit} className="space-y-4">
|
||||
<div>
|
||||
<label htmlFor="file" className="block text-sm font-medium mb-2">
|
||||
CSV File
|
||||
</label>
|
||||
<div className="border-2 border-dashed border-gray-300 rounded-lg p-6 text-center">
|
||||
<input
|
||||
id="file"
|
||||
type="file"
|
||||
accept=".csv"
|
||||
multiple
|
||||
onChange={handleFileChange}
|
||||
disabled={loading}
|
||||
className="hidden"
|
||||
/>
|
||||
<label htmlFor="file" className="cursor-pointer block">
|
||||
{files.length > 0 ? (
|
||||
<div>
|
||||
<p className="text-sm font-medium text-blue-600">
|
||||
{files.length} file{files.length !== 1 ? 's' : ''} selected
|
||||
</p>
|
||||
<div className="mt-2 text-xs text-gray-600 max-h-24 overflow-y-auto">
|
||||
{files.map((f, idx) => (
|
||||
<div key={idx} className="py-1">
|
||||
{f.name} ({(f.size / 1024).toFixed(2)} KB)
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
) : (
|
||||
<div>
|
||||
<p className="text-sm font-medium text-gray-700">
|
||||
Click to select or drag and drop
|
||||
</p>
|
||||
<p className="text-xs text-gray-500 mt-1">CSV files only (multiple allowed)</p>
|
||||
</div>
|
||||
)}
|
||||
</label>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="bg-blue-50 border border-blue-200 rounded-lg p-4 text-sm text-blue-800">
|
||||
<p className="font-semibold mb-2">CSV file format:</p>
|
||||
<ul className="list-disc list-inside space-y-1 text-xs">
|
||||
<li>
|
||||
Columns: <code className="bg-blue-100 px-1 rounded">id, speaker, model, utt_id, text, dialect, iteration, duration_ms, rms_value, longest_pause, utmos_score, wer_score</code>
|
||||
</li>
|
||||
<li>Entry IDs in the CSV must match existing dataset entries to update</li>
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
{uploadQueue.length > 0 && (
|
||||
<div className="bg-gray-50 border border-gray-200 rounded-lg p-4 space-y-3">
|
||||
<p className="font-semibold text-sm text-gray-700">Upload Queue:</p>
|
||||
<div className="space-y-2 max-h-48 overflow-y-auto">
|
||||
{uploadQueue.map((item, idx) => (
|
||||
<div key={idx} className="space-y-1">
|
||||
<div className="flex items-center gap-2 text-xs">
|
||||
<span className={`font-medium ${
|
||||
item.status === 'completed' ? 'text-green-600' :
|
||||
item.status === 'failed' ? 'text-red-600' :
|
||||
item.status === 'uploading' ? 'text-blue-600' :
|
||||
'text-gray-600'
|
||||
}`}>
|
||||
{item.status === 'completed' ? '✓' :
|
||||
item.status === 'failed' ? '✗' :
|
||||
item.status === 'uploading' ? '⟳' :
|
||||
'○'}
|
||||
</span>
|
||||
<span className="flex-1 truncate">{item.file.name}</span>
|
||||
{item.progress > 0 && item.progress < 100 && (
|
||||
<span className="text-gray-500">{item.progress}%</span>
|
||||
)}
|
||||
{item.message && <span className="text-gray-500 text-xs">{item.message}</span>}
|
||||
</div>
|
||||
{item.status === 'uploading' && (
|
||||
<div className="w-full bg-gray-200 rounded-full h-2 overflow-hidden">
|
||||
<div
|
||||
className="bg-blue-500 h-full transition-all duration-300"
|
||||
style={{ width: `${item.progress}%` }}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
{item.status === 'completed' && (
|
||||
<div className="w-full bg-green-200 rounded-full h-2 overflow-hidden">
|
||||
<div className="bg-green-500 h-full w-full" />
|
||||
</div>
|
||||
)}
|
||||
{item.status === 'failed' && (
|
||||
<div className="w-full bg-red-200 rounded-full h-2 overflow-hidden">
|
||||
<div className="bg-red-500 h-full w-full" />
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{error && (
|
||||
<div className="p-3 bg-red-100 border border-red-400 text-red-700 rounded text-sm">
|
||||
{error}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{status && (
|
||||
<div className={`p-3 rounded text-sm ${success ? 'bg-green-100 border border-green-400 text-green-700' : 'bg-blue-100 border border-blue-400 text-blue-700'}`}>
|
||||
{status}
|
||||
</div>
|
||||
)}
|
||||
</form>
|
||||
</Modal>
|
||||
</>
|
||||
);
|
||||
}
|
||||
34
src/lib/csv-parser.ts
Normal file
34
src/lib/csv-parser.ts
Normal file
@@ -0,0 +1,34 @@
|
||||
/**
|
||||
* CSV parser that handles quoted fields
|
||||
*/
|
||||
export function parseCSVLine(line: string): string[] {
|
||||
const values: string[] = [];
|
||||
let current = '';
|
||||
let inQuotes = false;
|
||||
|
||||
for (let i = 0; i < line.length; i++) {
|
||||
const char = line[i];
|
||||
const nextChar = line[i + 1];
|
||||
|
||||
if (char === '"') {
|
||||
if (inQuotes && nextChar === '"') {
|
||||
// Escaped quote
|
||||
current += '"';
|
||||
i++;
|
||||
} else {
|
||||
// Toggle quote state
|
||||
inQuotes = !inQuotes;
|
||||
}
|
||||
} else if (char === ',' && !inQuotes) {
|
||||
// Field separator
|
||||
values.push(current.trim());
|
||||
current = '';
|
||||
} else {
|
||||
current += char;
|
||||
}
|
||||
}
|
||||
|
||||
// Add the last field
|
||||
values.push(current.trim());
|
||||
return values;
|
||||
}
|
||||
@@ -16,6 +16,10 @@ export const dataset_entry = sqliteTable('dataset_entry', {
|
||||
dialect: text('dialect').notNull(),
|
||||
iteration: integer('iteration').notNull(),
|
||||
durationMs: integer('duration_ms'),
|
||||
rmsValue: real('rms_value'),
|
||||
longestPause: real('longest_pause'),
|
||||
utmosScore: real('utmos_score'),
|
||||
werScore: real('wer_score'),
|
||||
createdAt: integer('created_at', { mode: 'timestamp' })
|
||||
.notNull()
|
||||
.default(sql`(unixepoch())`),
|
||||
|
||||
Reference in New Issue
Block a user