feat: added dataset duplication function (with iteration filtering)

src/app/actions/datasets.ts

@@ -2,10 +2,14 @@
 
 import db from '@/lib/db';
 import { dataset } from '@/lib/model/dataset';
+import { dataset_entry } from '@/lib/model/dataset_entry';
 import { eq } from 'drizzle-orm';
 import { revalidatePath } from 'next/cache';
 import { removeAllDatasetEntries } from './remove-dataset-entries';
 import { requireAdmin } from '@/lib/auth';
+import { writeFile, mkdir, readFile } from 'fs/promises';
+import { join } from 'path';
+import { existsSync } from 'fs';
 
 export async function getAllDatasets() {
   const result = await requireAdmin();
@@ -104,3 +108,197 @@ export async function deleteDataset(id: number) {
     throw new Error('Failed to delete dataset');
   }
 }
+
+/**
+ * Duplicates a dataset with filtered and prioritized entries.
+ * 
+ * @param sourceDatasetId - ID of the source dataset to duplicate from
+ * @param newDatasetName - Name for the new duplicated dataset
+ * @param newDatasetDescription - Description for the new dataset
+ * @param iterations - Number of times to replicate selected entries
+ * @param prioritizationMetric - Metric to prioritize entries by ('utmos' or 'wer')
+ * @param filters - Filtering criteria (all AND-ed together)
+ * @returns Object with success status and details
+ */
+export async function duplicateDatasetWithCriteria(
+  sourceDatasetId: number,
+  newDatasetName: string,
+  newDatasetDescription: string | undefined,
+  iterations: number,
+  prioritizationMetric: 'utmos' | 'wer',
+  filters?: {
+    utmosMin?: number;
+    utmosMax?: number;
+    werMin?: number;
+    werMax?: number;
+    durationMsMin?: number;
+    durationMsMax?: number;
+  }
+) {
+  const authResult = await requireAdmin();
+  if (!authResult.authenticated || !authResult.admin) {
+    throw new Error('Unauthorized');
+  }
+
+  if (!newDatasetName || !newDatasetName.trim()) {
+    throw new Error('Dataset name is required');
+  }
+
+  if (iterations < 1) {
+    throw new Error('Iterations must be at least 1');
+  }
+
+  try {
+    // Step 1: Fetch all entries from source dataset
+    const sourceEntries = await db
+      .select()
+      .from(dataset_entry)
+      .where(eq(dataset_entry.datasetId, sourceDatasetId));
+
+    if (sourceEntries.length === 0) {
+      throw new Error('Source dataset has no entries');
+    }
+
+    // Step 2: Apply filters (all AND-ed together)
+    const filteredEntries = sourceEntries.filter((entry) => {
+      // UTMOS filter
+      if (filters?.utmosMin !== undefined && entry.utmosScore !== null) {
+        if (entry.utmosScore < filters.utmosMin) return false;
+      }
+      if (filters?.utmosMax !== undefined && entry.utmosScore !== null) {
+        if (entry.utmosScore > filters.utmosMax) return false;
+      }
+
+      // WER filter
+      if (filters?.werMin !== undefined && entry.werScore !== null) {
+        if (entry.werScore < filters.werMin) return false;
+      }
+      if (filters?.werMax !== undefined && entry.werScore !== null) {
+        if (entry.werScore > filters.werMax) return false;
+      }
+
+      // Duration filter
+      if (filters?.durationMsMin !== undefined && entry.durationMs !== null) {
+        if (entry.durationMs < filters.durationMsMin) return false;
+      }
+      if (filters?.durationMsMax !== undefined && entry.durationMs !== null) {
+        if (entry.durationMs > filters.durationMsMax) return false;
+      }
+
+      return true;
+    });
+
+    if (filteredEntries.length === 0) {
+      throw new Error('No entries match the specified filter criteria');
+    }
+
+    // Step 3: Group entries by speaker, utterance, dialect, and model
+    const groupedEntries = new Map<string, typeof sourceEntries[0][]>();
+    for (const entry of filteredEntries) {
+      const groupKey = `${entry.speakerId}|${entry.utteranceId}|${entry.dialect}|${entry.modelName}`;
+      if (!groupedEntries.has(groupKey)) {
+        groupedEntries.set(groupKey, []);
+      }
+      groupedEntries.get(groupKey)!.push(entry);
+    }
+
+    // Step 4: Sort each group by prioritization metric and select top iteration count entries
+    const selectedEntries: typeof sourceEntries = [];
+    for (const [, groupEntries] of groupedEntries) {
+      // Sort by the prioritization metric
+      const sorted = groupEntries.sort((a, b) => {
+        if (prioritizationMetric === 'utmos') {
+          // Higher UTMOS is better
+          const aScore = a.utmosScore ?? Number.NEGATIVE_INFINITY;
+          const bScore = b.utmosScore ?? Number.NEGATIVE_INFINITY;
+          return bScore - aScore;
+        } else {
+          // Lower WER is better
+          const aScore = a.werScore ?? Number.POSITIVE_INFINITY;
+          const bScore = b.werScore ?? Number.POSITIVE_INFINITY;
+          return aScore - bScore;
+        }
+      });
+
+      // Take the top entries from the group
+      selectedEntries.push(...sorted.slice(0, iterations));
+    }
+
+    // Step 5: Create new dataset
+    const newDatasetResult = await db.insert(dataset).values({
+      name: newDatasetName.trim(),
+      description: newDatasetDescription?.trim() || null,
+    });
+
+    if (!newDatasetResult.lastInsertRowid) {
+      throw new Error('Failed to create new dataset');
+    }
+
+    const newDatasetId = Number(newDatasetResult.lastInsertRowid);
+
+    // Step 6: Create dataset directory for new dataset
+    const sourceDatasetDir = join(process.cwd(), 'public', 'datasets', sourceDatasetId.toString());
+    const newDatasetDir = join(process.cwd(), 'public', 'datasets', newDatasetId.toString());
+
+    await mkdir(newDatasetDir, { recursive: true });
+
+    // Step 7: Copy audio files and create entries for each iteration
+    const newEntries: typeof dataset_entry.$inferInsert[] = [];
+
+      for (const entry of selectedEntries) {
+        // Copy audio file
+        const audioFileExtension = entry.fileName.substring(entry.fileName.lastIndexOf('.'));
+        const sourceAudioPath = join(sourceDatasetDir, `${entry.externalId}${audioFileExtension}`);
+        const newAudioPath = join(newDatasetDir, `${entry.externalId}${audioFileExtension}`);
+
+        // Only copy if file exists
+        if (existsSync(sourceAudioPath)) {
+          const audioBuffer = await readFile(sourceAudioPath);
+          await writeFile(newAudioPath, audioBuffer);
+        }
+
+        // Create entry with updated iteration number
+        newEntries.push({
+          datasetId: newDatasetId,
+          externalId: entry.externalId,
+          speakerId: entry.speakerId,
+          modelName: entry.modelName,
+          utteranceId: entry.utteranceId,
+          utteranceText: entry.utteranceText,
+          fileName: entry.fileName,
+          dialect: entry.dialect,
+          iteration: entry.iteration,
+          durationMs: entry.durationMs,
+          rmsValue: entry.rmsValue,
+          longestPause: entry.longestPause,
+          utmosScore: entry.utmosScore,
+          werScore: entry.werScore,
+        });
+      }
+    
+
+    // Step 8: Insert all entries into new dataset
+    await db.insert(dataset_entry).values(newEntries);
+
+    revalidatePath('/admin');
+    revalidatePath('/admin/datasets');
+
+    return {
+      success: true,
+      newDatasetId,
+      newDatasetName: newDatasetName.trim(),
+      sourceEntriesCount: sourceEntries.length,
+      filteredEntriesCount: filteredEntries.length,
+      selectedEntriesCount: selectedEntries.length,
+      totalEntriesCreated: newEntries.length,
+      message: `Successfully created new dataset '${newDatasetName.trim()}' with ${selectedEntries.length} unique entries replicated ${iterations} time(s) (total: ${newEntries.length} entries).`,
+    };
+  } catch (error) {
+    console.error('Error duplicating dataset:', error);
+    const errorMessage = error instanceof Error ? error.message : 'Failed to duplicate dataset';
+    return {
+      success: false,
+      error: errorMessage,
+    };
+  }
+}

src/components/DatasetsList.tsx

@@ -1,37 +1,97 @@
-'use server';
+'use client';
 
-import db from '@/lib/db';
+import { useState, useEffect } from 'react';
-import { dataset } from '@/lib/model/dataset';
+import type { Dataset } from '@/lib/model/dataset';
 import Link from 'next/link';
+import { getAllDatasets } from '@/app/actions/datasets';
+import DuplicateDatasetModal from '@/components/DuplicateDatasetModal';
 
-export default async function DatasetsList() {
+export default function DatasetsList() {
-  const datasets = await db.select().from(dataset).orderBy(dataset.createdAt);
+  const [datasets, setDatasets] = useState<Dataset[]>([]);
+  const [loading, setLoading] = useState(true);
+  const [selectedDatasetId, setSelectedDatasetId] = useState<number | null>(null);
+  const [selectedDatasetName, setSelectedDatasetName] = useState('');
+
+  useEffect(() => {
+    async function loadDatasets() {
+      try {
+        const result = await getAllDatasets();
+        setDatasets(result);
+      } catch (error) {
+        console.error('Failed to load datasets:', error);
+      } finally {
+        setLoading(false);
+      }
+    }
+    loadDatasets();
+  }, []);
+
+  function handleOpenDuplicateModal(id: number, name: string) {
+    setSelectedDatasetId(id);
+    setSelectedDatasetName(name);
+  }
+
+  function handleCloseDuplicateModal() {
+    setSelectedDatasetId(null);
+    setSelectedDatasetName('');
+  }
+
+  if (loading) {
+    return (
+      <div className="mt-8">
+        <h2 className="text-2xl font-bold mb-4">Datasets</h2>
+        <p className="text-gray-500">Loading datasets...</p>
+      </div>
+    );
+  }
 
   return (
-    <div className="mt-8">
+    <>
-      <h2 className="text-2xl font-bold mb-4">Datasets</h2>
+      <div className="mt-8">
-      {datasets.length === 0 ? (
+        <h2 className="text-2xl font-bold mb-4">Datasets</h2>
-        <p className="text-gray-500">No datasets found.</p>
+        {datasets.length === 0 ? (
-      ) : (
+          <p className="text-gray-500">No datasets found.</p>
-        <div className="grid gap-4">
+        ) : (
-          {datasets.map((ds) => (
+          <div className="grid gap-4">
-            <Link
+            {datasets.map((ds) => (
-              key={ds.id}
+              <div
-              href={`/admin/datasets/${ds.id}`}
+                key={ds.id}
-              className="border border-gray-300 rounded-lg p-4 bg-white shadow-sm hover:shadow-md transition-shadow block cursor-pointer"
+                className="border border-gray-300 rounded-lg p-4 bg-white shadow-sm hover:shadow-md transition-shadow"
-            >
+              >
-              <h3 className="text-lg font-semibold">{ds.name}</h3>
+                <div className="flex justify-between items-start mb-2">
-              <p className="text-md text-gray-600">{ds.description}</p>
+                  <Link
-              <p className="text-sm text-gray-600 mt-2">
+                    href={`/admin/datasets/${ds.id}`}
-                ID: {ds.id}
+                    className="flex-1 hover:text-blue-600 transition-colors"
-              </p>
+                  >
-              <p className="text-sm text-gray-500 mt-1">
+                    <h3 className="text-lg font-semibold">{ds.name}</h3>
-                Created: {new Date(ds.createdAt).toLocaleDateString()}
+                  </Link>
-              </p>
+                  <button
-            </Link>
+                    onClick={() => handleOpenDuplicateModal(ds.id, ds.name)}
-          ))}
+                    className="ml-2 px-3 py-1 bg-blue-500 text-white text-sm rounded hover:bg-blue-600 transition-colors whitespace-nowrap"
-        </div>
+                    title="Duplicate this dataset with filtering and replication"
+                  >
+                    ⎘ Duplicate
+                  </button>
+                </div>
+                <p className="text-md text-gray-600">{ds.description}</p>
+                <p className="text-sm text-gray-600 mt-2">ID: {ds.id}</p>
+                <p className="text-sm text-gray-500 mt-1">
+                  Created: {new Date(ds.createdAt).toLocaleDateString()}
+                </p>
+              </div>
+            ))}
+          </div>
+        )}
+      </div>
+
+      {selectedDatasetId && (
+        <DuplicateDatasetModal
+          datasetId={selectedDatasetId}
+          datasetName={selectedDatasetName}
+          isOpen={selectedDatasetId !== null}
+          onClose={handleCloseDuplicateModal}
+        />
       )}
-    </div>
+    </>
   );
 }

src/components/DuplicateDatasetModal.tsx

@@ -0,0 +1,294 @@
+'use client';
+
+import { useState } from 'react';
+import { duplicateDatasetWithCriteria } from '@/app/actions/datasets';
+import Modal from '@/components/Modal';
+
+interface DuplicateDatasetModalProps {
+  datasetId: number;
+  datasetName: string;
+  isOpen: boolean;
+  onClose: () => void;
+}
+
+export default function DuplicateDatasetModal({
+  datasetId,
+  datasetName,
+  isOpen,
+  onClose,
+}: DuplicateDatasetModalProps) {
+  const [newName, setNewName] = useState(`${datasetName} - Copy`);
+  const [description, setDescription] = useState('');
+  const [iterations, setIterations] = useState(1);
+  const [prioritizationMetric, setPrioritizationMetric] = useState<'utmos' | 'wer'>('utmos');
+
+  // Filter states
+  const [showFilters, setShowFilters] = useState(false);
+  const [utmosMin, setUtmosMin] = useState<number | ''>('');
+  const [utmosMax, setUtmosMax] = useState<number | ''>('');
+  const [werMin, setWerMin] = useState<number | ''>('');
+  const [werMax, setWerMax] = useState<number | ''>('');
+  const [durationMsMin, setDurationMsMin] = useState<number | ''>('');
+  const [durationMsMax, setDurationMsMax] = useState<number | ''>('');
+
+  const [loading, setLoading] = useState(false);
+  const [error, setError] = useState<string | null>(null);
+  const [success, setSuccess] = useState(false);
+  const [result, setResult] = useState<any>(null);
+
+  async function handleSubmit(e: React.FormEvent) {
+    e.preventDefault();
+    setError(null);
+    setSuccess(false);
+
+    if (!newName.trim()) {
+      setError('Dataset name is required');
+      return;
+    }
+
+    if (iterations < 1) {
+      setError('Iterations must be at least 1');
+      return;
+    }
+
+    setLoading(true);
+    try {
+      const filters: any = {};
+      if (utmosMin !== '') filters.utmosMin = Number(utmosMin);
+      if (utmosMax !== '') filters.utmosMax = Number(utmosMax);
+      if (werMin !== '') filters.werMin = Number(werMin);
+      if (werMax !== '') filters.werMax = Number(werMax);
+      if (durationMsMin !== '') filters.durationMsMin = Number(durationMsMin);
+      if (durationMsMax !== '') filters.durationMsMax = Number(durationMsMax);
+
+      const res = await duplicateDatasetWithCriteria(
+        datasetId,
+        newName.trim(),
+        description.trim() || undefined,
+        iterations,
+        prioritizationMetric,
+        Object.keys(filters).length > 0 ? filters : undefined
+      );
+
+      if (res.success) {
+        setResult(res);
+        setSuccess(true);
+        setTimeout(() => {
+          setSuccess(false);
+          handleClose();
+          // Refresh the page to show the new dataset
+          window.location.reload();
+        }, 2000);
+      } else {
+        setError((res as any).error || 'Failed to duplicate dataset');
+      }
+    } catch (err) {
+      setError(err instanceof Error ? err.message : 'Failed to duplicate dataset');
+    } finally {
+      setLoading(false);
+    }
+  }
+
+  function handleClose() {
+    if (!loading) {
+      onClose();
+      setNewName(`${datasetName} - Copy`);
+      setDescription('');
+      setIterations(1);
+      setPrioritizationMetric('utmos');
+      setUtmosMin('');
+      setUtmosMax('');
+      setWerMin('');
+      setWerMax('');
+      setDurationMsMin('');
+      setDurationMsMax('');
+      setError(null);
+      setSuccess(false);
+      setResult(null);
+    }
+  }
+
+  return (
+    <Modal
+      isOpen={isOpen}
+      onClose={handleClose}
+      title={`Duplicate Dataset: ${datasetName}`}
+      actions={[
+        {
+          label: 'Cancel',
+          onClick: handleClose,
+          variant: 'secondary',
+          disabled: loading,
+        },
+        {
+          label: loading ? 'Duplicating...' : 'Duplicate',
+          onClick: () => {
+            const form = document.getElementById('duplicateDatasetForm') as HTMLFormElement;
+            form?.dispatchEvent(new Event('submit', { bubbles: true }));
+          },
+          variant: 'primary',
+          disabled: loading || !newName.trim(),
+        },
+      ]}
+    >
+      {error && (
+        <div className="mb-4 p-3 bg-red-100 border border-red-400 text-red-700 rounded">
+          {error}
+        </div>
+      )}
+
+      {success && result && (
+        <div className="mb-4 p-3 bg-green-100 border border-green-400 text-green-700 rounded">
+          <p className="font-semibold">{result.message}</p>
+          <p className="text-sm mt-2">
+            Created {result.totalEntriesCreated} entries ({result.selectedEntriesCount} unique × {iterations})
+          </p>
+        </div>
+      )}
+
+      <form id="duplicateDatasetForm" onSubmit={handleSubmit} className="space-y-4">
+        {/* Dataset Info */}
+        <div>
+          <label className="block text-sm font-medium text-gray-700 mb-1">New Dataset Name *</label>
+          <input
+            type="text"
+            value={newName}
+            onChange={(e) => setNewName(e.target.value)}
+            disabled={loading || success}
+            className="w-full px-3 py-2 border border-gray-300 rounded-md text-sm focus:outline-none focus:ring-2 focus:ring-blue-500 disabled:bg-gray-100"
+            placeholder="Enter dataset name"
+          />
+        </div>
+
+        <div>
+          <label className="block text-sm font-medium text-gray-700 mb-1">Description</label>
+          <textarea
+            value={description}
+            onChange={(e) => setDescription(e.target.value)}
+            disabled={loading || success}
+            rows={2}
+            className="w-full px-3 py-2 border border-gray-300 rounded-md text-sm focus:outline-none focus:ring-2 focus:ring-blue-500 disabled:bg-gray-100"
+            placeholder="Enter dataset description (optional)"
+          />
+        </div>
+
+        {/* Replication & Prioritization */}
+        <div className="grid grid-cols-2 gap-3">
+          <div>
+            <label className="block text-sm font-medium text-gray-700 mb-1">Iterations *</label>
+            <input
+              type="number"
+              min="1"
+              value={iterations}
+              onChange={(e) => setIterations(Math.max(1, parseInt(e.target.value) || 1))}
+              disabled={loading || success}
+              className="w-full px-3 py-2 border border-gray-300 rounded-md text-sm focus:outline-none focus:ring-2 focus:ring-blue-500 disabled:bg-gray-100"
+            />
+          </div>
+
+          <div>
+            <label className="block text-sm font-medium text-gray-700 mb-1">Prioritization</label>
+            <select
+              value={prioritizationMetric}
+              onChange={(e) => setPrioritizationMetric(e.target.value as 'utmos' | 'wer')}
+              disabled={loading || success}
+              className="w-full px-3 py-2 border border-gray-300 rounded-md text-sm focus:outline-none focus:ring-2 focus:ring-blue-500 disabled:bg-gray-100"
+            >
+              <option value="utmos">UTMOS (higher better)</option>
+              <option value="wer">WER (lower better)</option>
+            </select>
+          </div>
+        </div>
+
+        {/* Filters Section */}
+        <div className="border-t border-gray-200 pt-4">
+          <button
+            type="button"
+            onClick={() => setShowFilters(!showFilters)}
+            disabled={loading || success}
+            className="text-sm font-medium text-blue-600 hover:text-blue-800 underline disabled:text-gray-400"
+          >
+            {showFilters ? '▼ Hide' : '▶ Show'} Filters (optional)
+          </button>
+
+          {showFilters && (
+            <div className="mt-4 space-y-4 bg-gray-50 p-3 rounded-md">
+              {/* UTMOS Filter */}
+              <div>
+                <label className="block text-xs font-semibold text-gray-700 mb-2">UTMOS Score Range</label>
+                <div className="grid grid-cols-2 gap-2">
+                  <input
+                    type="number"
+                    step="0.1"
+                    placeholder="Min"
+                    value={utmosMin}
+                    onChange={(e) => setUtmosMin(e.target.value ? Number(e.target.value) : '')}
+                    disabled={loading || success}
+                    className="px-2 py-1 border border-gray-300 rounded text-sm focus:outline-none focus:ring-2 focus:ring-blue-500 disabled:bg-gray-100"
+                  />
+                  <input
+                    type="number"
+                    step="0.1"
+                    placeholder="Max"
+                    value={utmosMax}
+                    onChange={(e) => setUtmosMax(e.target.value ? Number(e.target.value) : '')}
+                    disabled={loading || success}
+                    className="px-2 py-1 border border-gray-300 rounded text-sm focus:outline-none focus:ring-2 focus:ring-blue-500 disabled:bg-gray-100"
+                  />
+                </div>
+              </div>
+
+              {/* WER Filter */}
+              <div>
+                <label className="block text-xs font-semibold text-gray-700 mb-2">WER Score Range</label>
+                <div className="grid grid-cols-2 gap-2">
+                  <input
+                    type="number"
+                    step="0.1"
+                    placeholder="Min"
+                    value={werMin}
+                    onChange={(e) => setWerMin(e.target.value ? Number(e.target.value) : '')}
+                    disabled={loading || success}
+                    className="px-2 py-1 border border-gray-300 rounded text-sm focus:outline-none focus:ring-2 focus:ring-blue-500 disabled:bg-gray-100"
+                  />
+                  <input
+                    type="number"
+                    step="0.1"
+                    placeholder="Max"
+                    value={werMax}
+                    onChange={(e) => setWerMax(e.target.value ? Number(e.target.value) : '')}
+                    disabled={loading || success}
+                    className="px-2 py-1 border border-gray-300 rounded text-sm focus:outline-none focus:ring-2 focus:ring-blue-500 disabled:bg-gray-100"
+                  />
+                </div>
+              </div>
+
+              {/* Duration Filter */}
+              <div>
+                <label className="block text-xs font-semibold text-gray-700 mb-2">Duration Range (ms)</label>
+                <div className="grid grid-cols-2 gap-2">
+                  <input
+                    type="number"
+                    placeholder="Min"
+                    value={durationMsMin}
+                    onChange={(e) => setDurationMsMin(e.target.value ? Number(e.target.value) : '')}
+                    disabled={loading || success}
+                    className="px-2 py-1 border border-gray-300 rounded text-sm focus:outline-none focus:ring-2 focus:ring-blue-500 disabled:bg-gray-100"
+                  />
+                  <input
+                    type="number"
+                    placeholder="Max"
+                    value={durationMsMax}
+                    onChange={(e) => setDurationMsMax(e.target.value ? Number(e.target.value) : '')}
+                    disabled={loading || success}
+                    className="px-2 py-1 border border-gray-300 rounded text-sm focus:outline-none focus:ring-2 focus:ring-blue-500 disabled:bg-gray-100"
+                  />
+                </div>
+              </div>
+            </div>
+          )}
+        </div>
+      </form>
+    </Modal>
+  );
+}