ZHAW/BA_TTS_Annotation_Plattform
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

removed utterance system atm

Browse Source
This commit is contained in:
averel10
2026-03-08 09:47:12 +01:00
parent c08dc46305
commit 194396940e
16 changed files with 220 additions and 516 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/app/actions/get-dataset-entries.ts
View File

@@ -11,7 +11,6 @@ interface FilterParams {
modelName?: string;
dialect?: string;
iteration?: number;
utteranceId?: string;
}
export async function getDatasetEntries(
@@ -40,10 +39,6 @@ export async function getDatasetEntries(
conditions.push(eq(dataset_entry.iteration, filters.iteration));
}
if (filters?.utteranceId) {
conditions.push(eq(dataset_entry.utteranceId, filters.utteranceId));
}
const whereClause = and(...conditions);
const entries = await db

2
src/app/actions/get-filter-options.ts
View File

@@ -15,13 +15,11 @@ export async function getFilterOptions(datasetId: number) {
const modelNames = [...new Set(entries.map(e => e.modelName))].sort();
const dialects = [...new Set(entries.map(e => e.dialect))].sort();
const iterations = [...new Set(entries.map(e => e.iteration))].sort((a, b) => a - b);
const utteranceIds = [...new Set(entries.map(e => e.utteranceId).filter(Boolean))].sort();
return {
speakerIds,
modelNames,
dialects,
iterations,
utteranceIds,
};
}

119
src/app/actions/upload-utterances.ts
View File

@@ -1,119 +0,0 @@
'use server';
import db from '@/lib/db';
import { dataset_utterance } from '@/lib/model/utterance';
import { eq, and } from 'drizzle-orm';
import { revalidatePath } from 'next/cache';
export async function uploadDatasetUtterances(
datasetId: number,
formData: FormData
) {
const file = formData.get('file') as File;
if (!file) {
throw new Error('No file provided');
}
if (!file.name.endsWith('.csv')) {
throw new Error('File must be a CSV file');
}
try {
const csvContent = await file.text();
const lines = csvContent.split('\n').filter((line) => line.trim());
if (lines.length < 2) {
throw new Error('CSV is empty or invalid');
}
// Parse CSV header
const headers = lines[0].split(',').map((h) => h.trim());
if (!headers.includes('id') || !headers.includes('text')) {
throw new Error('CSV must have "id" and "text" columns');
}
// Get indices of id and text columns
const idIndex = headers.indexOf('id');
const textIndex = headers.indexOf('text');
// Parse utterances
const utterances: typeof dataset_utterance.$inferInsert[] = [];
for (let i = 1; i < lines.length; i++) {
// Handle CSV parsing with quotes
const line = lines[i];
const parts: string[] = [];
let current = '';
let inQuotes = false;
for (let j = 0; j < line.length; j++) {
const char = line[j];
const nextChar = line[j + 1];
if (char === '"') {
if (inQuotes && nextChar === '"') {
current += '"';
j++; // Skip next quote
} else {
inQuotes = !inQuotes;
}
} else if (char === ',' && !inQuotes) {
parts.push(current.trim());
current = '';
} else {
current += char;
}
}
parts.push(current.trim());
if (parts.length > Math.max(idIndex, textIndex)) {
const id = parts[idIndex]?.trim();
const text = parts[textIndex]?.trim();
if (id && text) {
utterances.push({
datasetId,
id,
text,
});
}
}
}
if (utterances.length === 0) {
throw new Error('No valid utterances found in CSV');
}
// Check for existing utterances to avoid duplicates
const existingUtterances = await db
.select({ id: dataset_utterance.id })
.from(dataset_utterance)
.where(eq(dataset_utterance.datasetId, datasetId));
const existingIds = new Set(existingUtterances.map((u) => u.id));
// Filter out duplicates
const newUtterances = utterances.filter((u) => !existingIds.has(u.id));
if (newUtterances.length === 0) {
throw new Error('All utterances already exist in the dataset');
}
// Insert new utterances into database
await db.insert(dataset_utterance).values(newUtterances);
revalidatePath(`/admin/datasets/${datasetId}`);
return {
success: true,
utterancesCreated: newUtterances.length,
};
} catch (error) {
console.error('Error uploading utterances:', error);
throw new Error(
error instanceof Error ? error.message : 'Failed to upload utterances'
);
}
}

3
src/app/actions/upload.ts
View File

@@ -2,7 +2,6 @@
import db from '@/lib/db';
import { dataset_entry } from '@/lib/model/dataset_entry';
import { dataset_utterance } from '@/lib/model/utterance';
import { revalidatePath } from 'next/cache';
import { eq } from 'drizzle-orm';
import { writeFile, mkdir, readFile, rm, readdir } from 'fs/promises';
@@ -14,7 +13,6 @@ interface DatasetEntryInput {
id: string;
audio_file: string;
duration_ms: string;
utt_id: string;
speaker: string;
model: string;
dialect: string;
@@ -104,7 +102,6 @@ export async function uploadDatasetEntries(
speakerId: row.speaker,
modelName: row.model,
fileName: relativePath,
utteranceId: row.utt_id || undefined,
dialect: row.dialect,
iteration: parseInt(row.iteration, 10),
});

19
src/app/admin/datasets/[id]/entries/[entryId]/page.tsx
View File

@@ -1,7 +1,6 @@
import Link from 'next/link';
import db from '@/lib/db';
import { dataset_entry } from '@/lib/model/dataset_entry';
import { dataset_utterance } from '@/lib/model/utterance';
import { eq } from 'drizzle-orm';
import AudioPlayer from '@/components/AudioPlayer';
@@ -42,17 +41,6 @@ export default async function DatasetEntryPage({ params }: DatasetEntryPageProps
const entry = entries[0];
// Fetch utterance if utteranceId is present
let utterance = null;
if (entry.utteranceId) {
const utterances = await db
.select()
.from(dataset_utterance)
.where(eq(dataset_utterance.id, entry.utteranceId));
utterance = utterances.length > 0 ? utterances[0] : null;
}
return (
<div>
<div className="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8 py-8">
@@ -107,13 +95,6 @@ export default async function DatasetEntryPage({ params }: DatasetEntryPageProps
<AudioPlayer datasetId={datasetIdNum} fileName={entry.fileName} externalId={entry.externalId} />
</div>
{utterance && (
<div className="col-span-2 bg-blue-50 p-4 rounded-lg border border-blue-200">
<p className="text-sm font-semibold text-gray-700 mb-2">Utterance {entry.utteranceId}</p>
<p className="text-lg text-gray-800 leading-relaxed">{utterance.text}</p>
</div>
)}
<div className="bg-white border border-gray-200 p-4 rounded-lg">
<p className="text-sm text-gray-600 mb-1">Created</p>
<p className="text-lg font-semibold">

2
src/app/admin/datasets/[id]/page.tsx
View File

@@ -5,7 +5,6 @@ import { eq } from 'drizzle-orm';
import EditableDatasetHeader from '@/components/EditableDatasetHeader';
import DatasetEntriesList from '@/components/DatasetEntriesList';
import UploadDatasetEntriesModal from '@/components/UploadDatasetEntriesModal';
import UploadUtterancesModal from '@/components/UploadUtterancesModal';
import RemoveAllEntriesButton from '@/components/RemoveAllEntriesButton';
interface DatasetPageProps {
@@ -57,7 +56,6 @@ export default async function DatasetPage({ params }: DatasetPageProps) {
<div className="flex gap-3 mb-6">
<UploadDatasetEntriesModal datasetId={datasetId} />
<UploadUtterancesModal datasetId={datasetId} />
<RemoveAllEntriesButton datasetId={datasetId} />
</div>

43
src/components/DatasetEntriesList.tsx
View File

@@ -17,7 +17,6 @@ interface DatasetEntry {
speakerId: string;
modelName: string;
fileName: string;
utteranceId: string | null;
dialect: string;
iteration: number;
createdAt: Date;
@@ -29,7 +28,6 @@ interface FilterOptions {
modelNames: string[];
dialects: string[];
iterations: number[];
utteranceIds: string[];
}
export default function DatasetEntriesList({
@@ -47,7 +45,6 @@ export default function DatasetEntriesList({
modelName: '',
dialect: '',
iteration: '',
utteranceId: '',
});
// Filter options
@@ -56,7 +53,6 @@ export default function DatasetEntriesList({
modelNames: [],
dialects: [],
iterations: [],
utteranceIds: [],
});
// Load filter options on mount
@@ -85,7 +81,6 @@ export default function DatasetEntriesList({
modelName: filters.modelName || undefined,
dialect: filters.dialect || undefined,
iteration: filters.iteration ? parseInt(filters.iteration, 10) : undefined,
utteranceId: filters.utteranceId || undefined,
};
const result = await getDatasetEntries(datasetId, pageNum, filterParams);
@@ -112,7 +107,6 @@ export default function DatasetEntriesList({
modelName: filters.modelName || undefined,
dialect: filters.dialect || undefined,
iteration: filters.iteration ? parseInt(filters.iteration, 10) : undefined,
utteranceId: filters.utteranceId || undefined,
};
const result = await getDatasetEntries(datasetId, page + 1, filterParams);
@@ -140,7 +134,6 @@ export default function DatasetEntriesList({
modelName: '',
dialect: '',
iteration: '',
utteranceId: '',
});
}
@@ -229,23 +222,6 @@ export default function DatasetEntriesList({
))}
</select>
</div>
<div>
<label className="block text-sm font-medium text-gray-700 mb-1">
Utterance ID
</label>
<select
value={filters.utteranceId}
onChange={(e) => handleFilterChange('utteranceId', e.target.value)}
className="w-full px-3 py-2 border border-gray-300 rounded-md text-sm focus:outline-none focus:ring-2 focus:ring-blue-500"
>
<option value="">All utterances</option>
{filterOptions.utteranceIds.map((id) => (
<option key={id} value={id}>
{id}
</option>
))}
</select>
</div>
</div>
</div>
@@ -338,23 +314,6 @@ export default function DatasetEntriesList({
))}
</select>
</div>
<div>
<label className="block text-sm font-medium text-gray-700 mb-1">
Utterance ID
</label>
<select
value={filters.utteranceId}
onChange={(e) => handleFilterChange('utteranceId', e.target.value)}
className="w-full px-3 py-2 border border-gray-300 rounded-md text-sm focus:outline-none focus:ring-2 focus:ring-blue-500"
>
<option value="">All utterances</option>
{filterOptions.utteranceIds.map((id) => (
<option key={id} value={id}>
{id}
</option>
))}
</select>
</div>
</div>
</div>
@@ -373,7 +332,6 @@ export default function DatasetEntriesList({
<th className="text-left py-3 px-4 font-semibold text-gray-700">Speaker ID</th>
<th className="text-left py-3 px-4 font-semibold text-gray-700">Model Name</th>
<th className="text-left py-3 px-4 font-semibold text-gray-700">Dialect</th>
<th className="text-left py-3 px-4 font-semibold text-gray-700">Utterance ID</th>
<th className="text-left py-3 px-4 font-semibold text-gray-700">File Name</th>
<th className="text-left py-3 px-4 font-semibold text-gray-700">Iteration</th>
</tr>
@@ -395,7 +353,6 @@ export default function DatasetEntriesList({
<td className="py-3 px-4">{entry.speakerId}</td>
<td className="py-3 px-4">{entry.modelName}</td>
<td className="py-3 px-4">{entry.dialect}</td>
<td className="py-3 px-4">{entry.utteranceId || '-'}</td>
<td className="py-3 px-4 truncate">{entry.fileName}</td>
<td className="py-3 px-4">{entry.iteration}</td>
</tr>

215
src/components/UploadUtterancesModal.tsx
View File

@@ -1,215 +0,0 @@
'use client';
import { useState } from 'react';
import { uploadDatasetUtterances } from '@/app/actions/upload-utterances';
import Modal from '@/components/Modal';
interface UploadUtterancesModalProps {
datasetId: number;
}
export default function UploadUtterancesModal({
datasetId,
}: UploadUtterancesModalProps) {
const [isOpen, setIsOpen] = useState(false);
const [file, setFile] = useState<File | null>(null);
const [loading, setLoading] = useState(false);
const [error, setError] = useState<string | null>(null);
const [success, setSuccess] = useState(false);
const [utterancesCreated, setUtterancesCreated] = useState(0);
const [status, setStatus] = useState<string>('');
const [isDragging, setIsDragging] = useState(false);
function handleFileChange(e: React.ChangeEvent<HTMLInputElement>) {
const selectedFile = e.target.files?.[0];
if (selectedFile) {
setFile(selectedFile);
setError(null);
}
}
function handleDragOver(e: React.DragEvent<HTMLDivElement>) {
e.preventDefault();
e.stopPropagation();
setIsDragging(true);
}
function handleDragLeave(e: React.DragEvent<HTMLDivElement>) {
e.preventDefault();
e.stopPropagation();
setIsDragging(false);
}
function handleDrop(e: React.DragEvent<HTMLDivElement>) {
e.preventDefault();
e.stopPropagation();
setIsDragging(false);
const droppedFiles = e.dataTransfer.files;
if (droppedFiles.length > 0) {
const droppedFile = droppedFiles[0];
if (droppedFile.name.endsWith('.csv')) {
setFile(droppedFile);
setError(null);
} else {
setError('Please upload a CSV file');
}
}
}
async function handleSubmit(e: React.FormEvent) {
e.preventDefault();
setError(null);
setSuccess(false);
if (!file) {
setError('Please select a CSV file');
return;
}
setLoading(true);
setStatus('Preparing upload...');
try {
const formData = new FormData();
formData.append('file', file);
const result = await uploadDatasetUtterances(datasetId, formData);
setSuccess(true);
setUtterancesCreated(result.utterancesCreated);
setStatus(`Successfully created ${result.utterancesCreated} utterances!`);
setFile(null);
setTimeout(() => {
setSuccess(false);
setIsOpen(false);
setStatus('');
window.location.reload();
}, 2500);
} catch (err) {
const errorMsg =
err instanceof Error ? err.message : 'Failed to upload utterances';
setError(errorMsg);
setStatus('');
} finally {
setLoading(false);
}
}
function handleClose() {
if (!loading) {
setIsOpen(false);
setFile(null);
setError(null);
setSuccess(false);
setStatus('');
}
}
return (
<>
<button
onClick={() => setIsOpen(true)}
className="px-4 py-2 bg-purple-500 text-white rounded-md hover:bg-purple-600 transition-colors"
>
📝 Upload Utterances
</button>
<Modal
isOpen={isOpen}
onClose={handleClose}
title="Upload Utterances CSV"
actions={[
{
label: 'Cancel',
onClick: handleClose,
variant: 'secondary',
disabled: loading,
},
{
label: loading ? 'Uploading...' : 'Upload',
onClick: () => {
const form = document.getElementById(
'uploadUtterancesForm'
) as HTMLFormElement;
form?.dispatchEvent(new Event('submit', { bubbles: true }));
},
variant: 'primary',
disabled: loading || !file,
},
]}
>
<form id="uploadUtterancesForm" onSubmit={handleSubmit} className="space-y-4">
<div>
<label htmlFor="utterances-file" className="block text-sm font-medium mb-2">
CSV File
</label>
<div
className={`border-2 border-dashed rounded-lg p-6 text-center transition-colors ${
isDragging ? 'border-blue-400 bg-blue-50' : 'border-gray-300'
}`}
onDragOver={handleDragOver}
onDragLeave={handleDragLeave}
onDrop={handleDrop}
>
<input
id="utterances-file"
type="file"
accept=".csv"
onChange={handleFileChange}
disabled={loading}
className="hidden"
/>
<label htmlFor="utterances-file" className="cursor-pointer block">
{file ? (
<div>
<p className="text-sm font-medium text-blue-600">
{file.name}
</p>
<p className="text-xs text-gray-500 mt-1">
{(file.size / 1024).toFixed(2)} KB
</p>
</div>
) : (
<div>
<p className="text-sm font-medium text-gray-700">
Click to select or drag and drop
</p>
<p className="text-xs text-gray-500 mt-1">CSV file only</p>
</div>
)}
</label>
</div>
</div>
<div className="bg-blue-50 border border-blue-200 rounded-lg p-4 text-sm text-blue-800">
<p className="font-semibold mb-2">CSV file should contain:</p>
<ul className="list-disc list-inside space-y-1 text-xs">
<li>
<code className="bg-blue-100 px-1 rounded">id</code> column with
utterance IDs (e.g., utt_001)
</li>
<li>
<code className="bg-blue-100 px-1 rounded">text</code> column with the
utterance text
</li>
</ul>
</div>
{error && (
<div className="p-3 bg-red-100 border border-red-400 text-red-700 rounded text-sm">
{error}
</div>
)}
{success && (
<div className="p-3 bg-green-100 border border-green-400 text-green-700 rounded text-sm">
{status}
</div>
)}
</form>
</Modal>
</>
);
}

7
src/lib/auth.ts
View File

@@ -1,10 +1,13 @@
import { betterAuth } from "better-auth";
import Database from "better-sqlite3";
import { localization } from "better-auth-localization";
import { drizzleAdapter } from "@better-auth/drizzle-adapter";
import db from "./db";
export const auth = betterAuth({
database: new Database("./sqlite_users.db"),
database: drizzleAdapter(db, {
provider: "sqlite",
}),
emailAndPassword: {
enabled: true,
},

107
src/lib/model/auth-schema.ts Normal file
View File

@@ -0,0 +1,107 @@
import { relations, sql } from "drizzle-orm";
import { sqliteTable, text, integer, index } from "drizzle-orm/sqlite-core";
export const user = sqliteTable("user", {
id: text("id").primaryKey(),
name: text("name").notNull(),
email: text("email").notNull().unique(),
emailVerified: integer("email_verified", { mode: "boolean" })
.default(false)
.notNull(),
image: text("image"),
createdAt: integer("created_at", { mode: "timestamp_ms" })
.default(sql`(cast(unixepoch('subsecond') * 1000 as integer))`)
.notNull(),
updatedAt: integer("updated_at", { mode: "timestamp_ms" })
.default(sql`(cast(unixepoch('subsecond') * 1000 as integer))`)
.$onUpdate(() => /* @__PURE__ */ new Date())
.notNull(),
});
export const session = sqliteTable(
"session",
{
id: text("id").primaryKey(),
expiresAt: integer("expires_at", { mode: "timestamp_ms" }).notNull(),
token: text("token").notNull().unique(),
createdAt: integer("created_at", { mode: "timestamp_ms" })
.default(sql`(cast(unixepoch('subsecond') * 1000 as integer))`)
.notNull(),
updatedAt: integer("updated_at", { mode: "timestamp_ms" })
.$onUpdate(() => /* @__PURE__ */ new Date())
.notNull(),
ipAddress: text("ip_address"),
userAgent: text("user_agent"),
userId: text("user_id")
.notNull()
.references(() => user.id, { onDelete: "cascade" }),
},
(table) => [index("session_userId_idx").on(table.userId)],
);
export const account = sqliteTable(
"account",
{
id: text("id").primaryKey(),
accountId: text("account_id").notNull(),
providerId: text("provider_id").notNull(),
userId: text("user_id")
.notNull()
.references(() => user.id, { onDelete: "cascade" }),
accessToken: text("access_token"),
refreshToken: text("refresh_token"),
idToken: text("id_token"),
accessTokenExpiresAt: integer("access_token_expires_at", {
mode: "timestamp_ms",
}),
refreshTokenExpiresAt: integer("refresh_token_expires_at", {
mode: "timestamp_ms",
}),
scope: text("scope"),
password: text("password"),
createdAt: integer("created_at", { mode: "timestamp_ms" })
.default(sql`(cast(unixepoch('subsecond') * 1000 as integer))`)
.notNull(),
updatedAt: integer("updated_at", { mode: "timestamp_ms" })
.$onUpdate(() => /* @__PURE__ */ new Date())
.notNull(),
},
(table) => [index("account_userId_idx").on(table.userId)],
);
export const verification = sqliteTable(
"verification",
{
id: text("id").primaryKey(),
identifier: text("identifier").notNull(),
value: text("value").notNull(),
expiresAt: integer("expires_at", { mode: "timestamp_ms" }).notNull(),
createdAt: integer("created_at", { mode: "timestamp_ms" })
.default(sql`(cast(unixepoch('subsecond') * 1000 as integer))`)
.notNull(),
updatedAt: integer("updated_at", { mode: "timestamp_ms" })
.default(sql`(cast(unixepoch('subsecond') * 1000 as integer))`)
.$onUpdate(() => /* @__PURE__ */ new Date())
.notNull(),
},
(table) => [index("verification_identifier_idx").on(table.identifier)],
);
export const userRelations = relations(user, ({ many }) => ({
sessions: many(session),
accounts: many(account),
}));
export const sessionRelations = relations(session, ({ one }) => ({
user: one(user, {
fields: [session.userId],
references: [user.id],
}),
}));
export const accountRelations = relations(account, ({ one }) => ({
user: one(user, {
fields: [account.userId],
references: [user.id],
}),
}));

7
src/lib/model/dataset_entry.ts
View File

@@ -1,9 +1,7 @@
import { sqliteTable, text, integer, real } from 'drizzle-orm/sqlite-core';
import { relations } from 'drizzle-orm';
import { dataset } from './dataset';
import { dataset_utterance } from './utterance';
// Define the dataset table schema
export const dataset_entry = sqliteTable('dataset_entry', {
id: integer('id').primaryKey({ autoIncrement: true }),
externalId: text('external_id').notNull(),
@@ -12,7 +10,6 @@ export const dataset_entry = sqliteTable('dataset_entry', {
datasetId: integer('dataset_id')
.notNull()
.references(() => dataset.id),
utteranceId: text('utterance_id').references(() => dataset_utterance.id),
fileName: text('file_name').notNull(),
dialect: text('dialect').notNull(),
iteration: integer('iteration').notNull(),
@@ -28,10 +25,6 @@ export const dataset_entryRelations = relations(dataset_entry, ({ one }) => ({
dataset: one(dataset, {
fields: [dataset_entry.datasetId],
references: [dataset.id]
}),
utterance: one(dataset_utterance, {
fields: [dataset_entry.utteranceId],
references: [dataset_utterance.id]
})
}));

2
src/lib/model/index.ts
View File

@@ -1,3 +1,3 @@
export * from "./dataset";
export * from "./dataset_entry";
export * from "./utterance";
export * from "./auth-schema";

29
src/lib/model/utterance.ts
View File

@@ -1,29 +0,0 @@
import { sqliteTable, text, integer } from 'drizzle-orm/sqlite-core';
import { relations } from 'drizzle-orm';
import { dataset } from './dataset';
// Define the dataset_utterance table schema
export const dataset_utterance = sqliteTable('dataset_utterance', {
id: text('id').primaryKey(),
datasetId: integer('dataset_id')
.notNull()
.references(() => dataset.id),
text: text('text').notNull(),
createdAt: integer('created_at', { mode: 'timestamp' })
.notNull()
.default(new Date()),
updatedAt: integer('updated_at', { mode: 'timestamp' })
.notNull()
.default(new Date()),
});
export const dataset_utteranceRelations = relations(dataset_utterance, ({ one }) => ({
dataset: one(dataset, {
fields: [dataset_utterance.datasetId],
references: [dataset.id],
}),
}));
// Create types for dataset_utterance records based on the schema
export type DatasetUtterance = typeof dataset_utterance.$inferSelect;
export type NewDatasetUtterance = typeof dataset_utterance.$inferInsert;