From 44d3eeb6f784704b227d4fea06f881a8a42a53a5 Mon Sep 17 00:00:00 2001 From: averel10 Date: Thu, 2 Apr 2026 09:25:35 +0200 Subject: [PATCH] feat: added dataset duplication function (with iteration filtering) --- src/app/actions/datasets.ts | 198 +++++++++++++++ src/components/DatasetsList.tsx | 118 ++++++--- src/components/DuplicateDatasetModal.tsx | 294 +++++++++++++++++++++++ 3 files changed, 581 insertions(+), 29 deletions(-) create mode 100644 src/components/DuplicateDatasetModal.tsx diff --git a/src/app/actions/datasets.ts b/src/app/actions/datasets.ts index 79b532e..fe82844 100644 --- a/src/app/actions/datasets.ts +++ b/src/app/actions/datasets.ts @@ -2,10 +2,14 @@ import db from '@/lib/db'; import { dataset } from '@/lib/model/dataset'; +import { dataset_entry } from '@/lib/model/dataset_entry'; import { eq } from 'drizzle-orm'; import { revalidatePath } from 'next/cache'; import { removeAllDatasetEntries } from './remove-dataset-entries'; import { requireAdmin } from '@/lib/auth'; +import { writeFile, mkdir, readFile } from 'fs/promises'; +import { join } from 'path'; +import { existsSync } from 'fs'; export async function getAllDatasets() { const result = await requireAdmin(); @@ -104,3 +108,197 @@ export async function deleteDataset(id: number) { throw new Error('Failed to delete dataset'); } } + +/** + * Duplicates a dataset with filtered and prioritized entries. + * + * @param sourceDatasetId - ID of the source dataset to duplicate from + * @param newDatasetName - Name for the new duplicated dataset + * @param newDatasetDescription - Description for the new dataset + * @param iterations - Number of times to replicate selected entries + * @param prioritizationMetric - Metric to prioritize entries by ('utmos' or 'wer') + * @param filters - Filtering criteria (all AND-ed together) + * @returns Object with success status and details + */ +export async function duplicateDatasetWithCriteria( + sourceDatasetId: number, + newDatasetName: string, + newDatasetDescription: string | undefined, + iterations: number, + prioritizationMetric: 'utmos' | 'wer', + filters?: { + utmosMin?: number; + utmosMax?: number; + werMin?: number; + werMax?: number; + durationMsMin?: number; + durationMsMax?: number; + } +) { + const authResult = await requireAdmin(); + if (!authResult.authenticated || !authResult.admin) { + throw new Error('Unauthorized'); + } + + if (!newDatasetName || !newDatasetName.trim()) { + throw new Error('Dataset name is required'); + } + + if (iterations < 1) { + throw new Error('Iterations must be at least 1'); + } + + try { + // Step 1: Fetch all entries from source dataset + const sourceEntries = await db + .select() + .from(dataset_entry) + .where(eq(dataset_entry.datasetId, sourceDatasetId)); + + if (sourceEntries.length === 0) { + throw new Error('Source dataset has no entries'); + } + + // Step 2: Apply filters (all AND-ed together) + const filteredEntries = sourceEntries.filter((entry) => { + // UTMOS filter + if (filters?.utmosMin !== undefined && entry.utmosScore !== null) { + if (entry.utmosScore < filters.utmosMin) return false; + } + if (filters?.utmosMax !== undefined && entry.utmosScore !== null) { + if (entry.utmosScore > filters.utmosMax) return false; + } + + // WER filter + if (filters?.werMin !== undefined && entry.werScore !== null) { + if (entry.werScore < filters.werMin) return false; + } + if (filters?.werMax !== undefined && entry.werScore !== null) { + if (entry.werScore > filters.werMax) return false; + } + + // Duration filter + if (filters?.durationMsMin !== undefined && entry.durationMs !== null) { + if (entry.durationMs < filters.durationMsMin) return false; + } + if (filters?.durationMsMax !== undefined && entry.durationMs !== null) { + if (entry.durationMs > filters.durationMsMax) return false; + } + + return true; + }); + + if (filteredEntries.length === 0) { + throw new Error('No entries match the specified filter criteria'); + } + + // Step 3: Group entries by speaker, utterance, dialect, and model + const groupedEntries = new Map(); + for (const entry of filteredEntries) { + const groupKey = `${entry.speakerId}|${entry.utteranceId}|${entry.dialect}|${entry.modelName}`; + if (!groupedEntries.has(groupKey)) { + groupedEntries.set(groupKey, []); + } + groupedEntries.get(groupKey)!.push(entry); + } + + // Step 4: Sort each group by prioritization metric and select top iteration count entries + const selectedEntries: typeof sourceEntries = []; + for (const [, groupEntries] of groupedEntries) { + // Sort by the prioritization metric + const sorted = groupEntries.sort((a, b) => { + if (prioritizationMetric === 'utmos') { + // Higher UTMOS is better + const aScore = a.utmosScore ?? Number.NEGATIVE_INFINITY; + const bScore = b.utmosScore ?? Number.NEGATIVE_INFINITY; + return bScore - aScore; + } else { + // Lower WER is better + const aScore = a.werScore ?? Number.POSITIVE_INFINITY; + const bScore = b.werScore ?? Number.POSITIVE_INFINITY; + return aScore - bScore; + } + }); + + // Take the top entries from the group + selectedEntries.push(...sorted.slice(0, iterations)); + } + + // Step 5: Create new dataset + const newDatasetResult = await db.insert(dataset).values({ + name: newDatasetName.trim(), + description: newDatasetDescription?.trim() || null, + }); + + if (!newDatasetResult.lastInsertRowid) { + throw new Error('Failed to create new dataset'); + } + + const newDatasetId = Number(newDatasetResult.lastInsertRowid); + + // Step 6: Create dataset directory for new dataset + const sourceDatasetDir = join(process.cwd(), 'public', 'datasets', sourceDatasetId.toString()); + const newDatasetDir = join(process.cwd(), 'public', 'datasets', newDatasetId.toString()); + + await mkdir(newDatasetDir, { recursive: true }); + + // Step 7: Copy audio files and create entries for each iteration + const newEntries: typeof dataset_entry.$inferInsert[] = []; + + for (const entry of selectedEntries) { + // Copy audio file + const audioFileExtension = entry.fileName.substring(entry.fileName.lastIndexOf('.')); + const sourceAudioPath = join(sourceDatasetDir, `${entry.externalId}${audioFileExtension}`); + const newAudioPath = join(newDatasetDir, `${entry.externalId}${audioFileExtension}`); + + // Only copy if file exists + if (existsSync(sourceAudioPath)) { + const audioBuffer = await readFile(sourceAudioPath); + await writeFile(newAudioPath, audioBuffer); + } + + // Create entry with updated iteration number + newEntries.push({ + datasetId: newDatasetId, + externalId: entry.externalId, + speakerId: entry.speakerId, + modelName: entry.modelName, + utteranceId: entry.utteranceId, + utteranceText: entry.utteranceText, + fileName: entry.fileName, + dialect: entry.dialect, + iteration: entry.iteration, + durationMs: entry.durationMs, + rmsValue: entry.rmsValue, + longestPause: entry.longestPause, + utmosScore: entry.utmosScore, + werScore: entry.werScore, + }); + } + + + // Step 8: Insert all entries into new dataset + await db.insert(dataset_entry).values(newEntries); + + revalidatePath('/admin'); + revalidatePath('/admin/datasets'); + + return { + success: true, + newDatasetId, + newDatasetName: newDatasetName.trim(), + sourceEntriesCount: sourceEntries.length, + filteredEntriesCount: filteredEntries.length, + selectedEntriesCount: selectedEntries.length, + totalEntriesCreated: newEntries.length, + message: `Successfully created new dataset '${newDatasetName.trim()}' with ${selectedEntries.length} unique entries replicated ${iterations} time(s) (total: ${newEntries.length} entries).`, + }; + } catch (error) { + console.error('Error duplicating dataset:', error); + const errorMessage = error instanceof Error ? error.message : 'Failed to duplicate dataset'; + return { + success: false, + error: errorMessage, + }; + } +} diff --git a/src/components/DatasetsList.tsx b/src/components/DatasetsList.tsx index 7a6d253..815b265 100644 --- a/src/components/DatasetsList.tsx +++ b/src/components/DatasetsList.tsx @@ -1,37 +1,97 @@ -'use server'; +'use client'; -import db from '@/lib/db'; -import { dataset } from '@/lib/model/dataset'; +import { useState, useEffect } from 'react'; +import type { Dataset } from '@/lib/model/dataset'; import Link from 'next/link'; +import { getAllDatasets } from '@/app/actions/datasets'; +import DuplicateDatasetModal from '@/components/DuplicateDatasetModal'; -export default async function DatasetsList() { - const datasets = await db.select().from(dataset).orderBy(dataset.createdAt); +export default function DatasetsList() { + const [datasets, setDatasets] = useState([]); + const [loading, setLoading] = useState(true); + const [selectedDatasetId, setSelectedDatasetId] = useState(null); + const [selectedDatasetName, setSelectedDatasetName] = useState(''); + + useEffect(() => { + async function loadDatasets() { + try { + const result = await getAllDatasets(); + setDatasets(result); + } catch (error) { + console.error('Failed to load datasets:', error); + } finally { + setLoading(false); + } + } + loadDatasets(); + }, []); + + function handleOpenDuplicateModal(id: number, name: string) { + setSelectedDatasetId(id); + setSelectedDatasetName(name); + } + + function handleCloseDuplicateModal() { + setSelectedDatasetId(null); + setSelectedDatasetName(''); + } + + if (loading) { + return ( +
+

Datasets

+

Loading datasets...

+
+ ); + } return ( -
-

Datasets

- {datasets.length === 0 ? ( -

No datasets found.

- ) : ( -
- {datasets.map((ds) => ( - -

{ds.name}

-

{ds.description}

-

- ID: {ds.id} -

-

- Created: {new Date(ds.createdAt).toLocaleDateString()} -

- - ))} -
+ <> +
+

Datasets

+ {datasets.length === 0 ? ( +

No datasets found.

+ ) : ( +
+ {datasets.map((ds) => ( +
+
+ +

{ds.name}

+ + +
+

{ds.description}

+

ID: {ds.id}

+

+ Created: {new Date(ds.createdAt).toLocaleDateString()} +

+
+ ))} +
+ )} +
+ + {selectedDatasetId && ( + )} -
+ ); } diff --git a/src/components/DuplicateDatasetModal.tsx b/src/components/DuplicateDatasetModal.tsx new file mode 100644 index 0000000..fc7d091 --- /dev/null +++ b/src/components/DuplicateDatasetModal.tsx @@ -0,0 +1,294 @@ +'use client'; + +import { useState } from 'react'; +import { duplicateDatasetWithCriteria } from '@/app/actions/datasets'; +import Modal from '@/components/Modal'; + +interface DuplicateDatasetModalProps { + datasetId: number; + datasetName: string; + isOpen: boolean; + onClose: () => void; +} + +export default function DuplicateDatasetModal({ + datasetId, + datasetName, + isOpen, + onClose, +}: DuplicateDatasetModalProps) { + const [newName, setNewName] = useState(`${datasetName} - Copy`); + const [description, setDescription] = useState(''); + const [iterations, setIterations] = useState(1); + const [prioritizationMetric, setPrioritizationMetric] = useState<'utmos' | 'wer'>('utmos'); + + // Filter states + const [showFilters, setShowFilters] = useState(false); + const [utmosMin, setUtmosMin] = useState(''); + const [utmosMax, setUtmosMax] = useState(''); + const [werMin, setWerMin] = useState(''); + const [werMax, setWerMax] = useState(''); + const [durationMsMin, setDurationMsMin] = useState(''); + const [durationMsMax, setDurationMsMax] = useState(''); + + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); + const [success, setSuccess] = useState(false); + const [result, setResult] = useState(null); + + async function handleSubmit(e: React.FormEvent) { + e.preventDefault(); + setError(null); + setSuccess(false); + + if (!newName.trim()) { + setError('Dataset name is required'); + return; + } + + if (iterations < 1) { + setError('Iterations must be at least 1'); + return; + } + + setLoading(true); + try { + const filters: any = {}; + if (utmosMin !== '') filters.utmosMin = Number(utmosMin); + if (utmosMax !== '') filters.utmosMax = Number(utmosMax); + if (werMin !== '') filters.werMin = Number(werMin); + if (werMax !== '') filters.werMax = Number(werMax); + if (durationMsMin !== '') filters.durationMsMin = Number(durationMsMin); + if (durationMsMax !== '') filters.durationMsMax = Number(durationMsMax); + + const res = await duplicateDatasetWithCriteria( + datasetId, + newName.trim(), + description.trim() || undefined, + iterations, + prioritizationMetric, + Object.keys(filters).length > 0 ? filters : undefined + ); + + if (res.success) { + setResult(res); + setSuccess(true); + setTimeout(() => { + setSuccess(false); + handleClose(); + // Refresh the page to show the new dataset + window.location.reload(); + }, 2000); + } else { + setError((res as any).error || 'Failed to duplicate dataset'); + } + } catch (err) { + setError(err instanceof Error ? err.message : 'Failed to duplicate dataset'); + } finally { + setLoading(false); + } + } + + function handleClose() { + if (!loading) { + onClose(); + setNewName(`${datasetName} - Copy`); + setDescription(''); + setIterations(1); + setPrioritizationMetric('utmos'); + setUtmosMin(''); + setUtmosMax(''); + setWerMin(''); + setWerMax(''); + setDurationMsMin(''); + setDurationMsMax(''); + setError(null); + setSuccess(false); + setResult(null); + } + } + + return ( + { + const form = document.getElementById('duplicateDatasetForm') as HTMLFormElement; + form?.dispatchEvent(new Event('submit', { bubbles: true })); + }, + variant: 'primary', + disabled: loading || !newName.trim(), + }, + ]} + > + {error && ( +
+ {error} +
+ )} + + {success && result && ( +
+

{result.message}

+

+ Created {result.totalEntriesCreated} entries ({result.selectedEntriesCount} unique × {iterations}) +

+
+ )} + +
+ {/* Dataset Info */} +
+ + setNewName(e.target.value)} + disabled={loading || success} + className="w-full px-3 py-2 border border-gray-300 rounded-md text-sm focus:outline-none focus:ring-2 focus:ring-blue-500 disabled:bg-gray-100" + placeholder="Enter dataset name" + /> +
+ +
+ +