feat: implement download functionality for filtered dataset entries as ZIP

2026-03-13 10:53:55 +01:00
parent e107765752
commit 03118c468a
7 changed files with 402 additions and 14 deletions
--- a/package-lock.json
+++ b/package-lock.json
@@ -18,6 +18,7 @@
        "drizzle-kit": "^0.31.9",
        "drizzle-orm": "^0.45.1",
        "extract-zip": "^2.0.1",
+        "jszip": "^3.10.1",
        "next": "16.1.6",
        "postcss": "^8.5.6",
        "tailwindcss": "^4.2.1"
@@ -3535,6 +3536,12 @@
      "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==",
      "license": "MIT"
    },
+    "node_modules/core-util-is": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz",
+      "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==",
+      "license": "MIT"
+    },
    "node_modules/cross-fetch": {
      "version": "4.1.0",
      "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-4.1.0.tgz",
@@ -4100,6 +4107,12 @@
      ],
      "license": "BSD-3-Clause"
    },
+    "node_modules/immediate": {
+      "version": "3.0.6",
+      "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz",
+      "integrity": "sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==",
+      "license": "MIT"
+    },
    "node_modules/inherits": {
      "version": "2.0.4",
      "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
@@ -4160,6 +4173,12 @@
        "url": "https://github.com/sponsors/sindresorhus"
      }
    },
+    "node_modules/isarray": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
+      "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==",
+      "license": "MIT"
+    },
    "node_modules/jiti": {
      "version": "2.6.1",
      "resolved": "https://registry.npmjs.org/jiti/-/jiti-2.6.1.tgz",
@@ -4214,6 +4233,48 @@
        "node": ">=6"
      }
    },
+    "node_modules/jszip": {
+      "version": "3.10.1",
+      "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.10.1.tgz",
+      "integrity": "sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g==",
+      "license": "(MIT OR GPL-3.0-or-later)",
+      "dependencies": {
+        "lie": "~3.3.0",
+        "pako": "~1.0.2",
+        "readable-stream": "~2.3.6",
+        "setimmediate": "^1.0.5"
+      }
+    },
+    "node_modules/jszip/node_modules/readable-stream": {
+      "version": "2.3.8",
+      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz",
+      "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==",
+      "license": "MIT",
+      "dependencies": {
+        "core-util-is": "~1.0.0",
+        "inherits": "~2.0.3",
+        "isarray": "~1.0.0",
+        "process-nextick-args": "~2.0.0",
+        "safe-buffer": "~5.1.1",
+        "string_decoder": "~1.1.1",
+        "util-deprecate": "~1.0.1"
+      }
+    },
+    "node_modules/jszip/node_modules/safe-buffer": {
+      "version": "5.1.2",
+      "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
+      "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==",
+      "license": "MIT"
+    },
+    "node_modules/jszip/node_modules/string_decoder": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
+      "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
+      "license": "MIT",
+      "dependencies": {
+        "safe-buffer": "~5.1.0"
+      }
+    },
    "node_modules/kleur": {
      "version": "3.0.3",
      "resolved": "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz",
@@ -4273,6 +4334,15 @@
        "node": ">=8"
      }
    },
+    "node_modules/lie": {
+      "version": "3.3.0",
+      "resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz",
+      "integrity": "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==",
+      "license": "MIT",
+      "dependencies": {
+        "immediate": "~3.0.5"
+      }
+    },
    "node_modules/lightningcss": {
      "version": "1.31.1",
      "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.31.1.tgz",
@@ -4826,6 +4896,12 @@
        "url": "https://github.com/sponsors/sindresorhus"
      }
    },
+    "node_modules/pako": {
+      "version": "1.0.11",
+      "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz",
+      "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==",
+      "license": "(MIT AND Zlib)"
+    },
    "node_modules/pathe": {
      "version": "2.0.3",
      "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz",
@@ -5059,6 +5135,12 @@
        "url": "https://github.com/prettier/prettier?sponsor=1"
      }
    },
+    "node_modules/process-nextick-args": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
+      "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==",
+      "license": "MIT"
+    },
    "node_modules/promise-limit": {
      "version": "2.7.0",
      "resolved": "https://registry.npmjs.org/promise-limit/-/promise-limit-2.7.0.tgz",
@@ -5241,6 +5323,12 @@
      "integrity": "sha512-n7Z7dXZhJbwuAHhNzkTti6Aw9QDDjZtm3JTpTGATIdNzdQz5GuFs22w90BcvF4INfnrL5xrX3oGsuqO5Dx3A1Q==",
      "license": "MIT"
    },
+    "node_modules/setimmediate": {
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz",
+      "integrity": "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==",
+      "license": "MIT"
+    },
    "node_modules/sharp": {
      "version": "0.34.5",
      "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.34.5.tgz",
--- a/package.json
+++ b/package.json
@@ -17,12 +17,13 @@
    "better-auth": "^1.4.19",
    "better-auth-localization": "^2.3.1",
    "better-sqlite3": "^12.6.2",
+    "drizzle-kit": "^0.31.9",
    "drizzle-orm": "^0.45.1",
    "extract-zip": "^2.0.1",
+    "jszip": "^3.10.1",
    "next": "16.1.6",
    "postcss": "^8.5.6",
-    "tailwindcss": "^4.2.1",
-    "drizzle-kit": "^0.31.9"
+    "tailwindcss": "^4.2.1"
  },
  "devDependencies": {
    "@types/better-sqlite3": "^7.6.13",
--- a/src/app/actions/download-filtered-entries.ts
+++ b/src/app/actions/download-filtered-entries.ts
@@ -0,0 +1,102 @@
+'use server';
+
+import { requireAdmin } from '@/lib/auth';
+import db from '@/lib/db';
+import { dataset_entry } from '@/lib/model/dataset_entry';
+import { eq, and } from 'drizzle-orm';
+import * as fs from 'fs';
+import * as path from 'path';
+
+interface FilterParams {
+  speakerId?: string;
+  modelName?: string;
+  dialect?: string;
+  iteration?: number;
+  utteranceId?: string;
+}
+
+export async function downloadFilteredEntries(
+  datasetId: number,
+  filters: FilterParams
+) {
+  const result = await requireAdmin();
+  if (!result.authenticated || !result.admin) {
+    throw new Error('Unauthorized');
+  }
+  // Build the where clause based on active filters
+  const conditions = [eq(dataset_entry.datasetId, datasetId)];
+
+  if (filters.speakerId) {
+    conditions.push(eq(dataset_entry.speakerId, filters.speakerId));
+  }
+  if (filters.modelName) {
+    conditions.push(eq(dataset_entry.modelName, filters.modelName));
+  }
+  if (filters.dialect) {
+    conditions.push(eq(dataset_entry.dialect, filters.dialect));
+  }
+  if (filters.iteration !== undefined) {
+    conditions.push(eq(dataset_entry.iteration, filters.iteration));
+  }
+  if (filters.utteranceId) {
+    conditions.push(eq(dataset_entry.utteranceId, filters.utteranceId));
+  }
+
+  // Fetch all matching entries
+  const entries = await db
+    .select()
+    .from(dataset_entry)
+    .where(conditions.length > 1 ? and(...conditions) : conditions[0]);
+
+  // Create CSV content
+  const headers = [
+    'id',    
+    'audio_file',
+    'duration_ms',
+    'utt_id',
+    'text',
+    'speaker',
+    'model',
+    'dialect',
+    'iteration',
+  ];
+
+  const csvLines = [headers.join(',')];
+  entries.forEach((entry) => {
+    const row = [
+      `"${entry.externalId}"`,
+      entry.fileName,
+      '',
+      entry.utteranceId || '',
+      `"${(entry.utteranceText || '').replace(/"/g, '""')}"`,
+      entry.speakerId,
+      entry.modelName,
+      entry.dialect,
+      entry.iteration,
+    ];
+    csvLines.push(row.join(','));
+  });
+
+  const csvContent = csvLines.join('\n');
+
+  // Create filter metadata JSON
+  const filterMetadata = {
+    exportedAt: new Date().toISOString(),
+    datasetId,
+    filters,
+    entriesCount: entries.length,
+  };
+
+  const metadataContent = JSON.stringify(filterMetadata, null, 2);
+
+  // Return the data that will be zipped on the client
+  return {
+    csvContent,
+    metadataContent,
+    entriesCount: entries.length,
+    entries: entries.map(entry => ({
+      id: entry.externalId,
+      fileName: entry.fileName,
+    })),
+  };
+}
--- a/src/app/actions/remove-dataset-entries.ts
+++ b/src/app/actions/remove-dataset-entries.ts
@@ -7,9 +7,14 @@ import { revalidatePath } from 'next/cache';
 import { rm } from 'fs/promises';
 import { join } from 'path';
 import { existsSync } from 'fs';
+import { requireAdmin } from '@/lib/auth';

 export async function removeAllDatasetEntries(datasetId: number) {
  try {
+    const result = await requireAdmin();
+    if (!result.authenticated || !result.admin) {
+      throw new Error('Unauthorized');
+    }
    // Delete all entries from database
    await db.delete(dataset_entry).where(eq(dataset_entry.datasetId, datasetId));

--- a/src/app/api/download-filtered-entries/route.ts
+++ b/src/app/api/download-filtered-entries/route.ts
@@ -0,0 +1,106 @@
+import { NextRequest, NextResponse } from 'next/server';
+import { downloadFilteredEntries } from '@/app/actions/download-filtered-entries';
+import JSZip from 'jszip';
+import { join } from 'path';
+import { readFile, existsSync } from 'fs';
+import { promisify } from 'util';
+import { requireAdmin } from '@/lib/auth';
+
+const readFileAsync = promisify(readFile);
+
+interface FilterParams {
+  speakerId?: string;
+  modelName?: string;
+  dialect?: string;
+  iteration?: number;
+  utteranceId?: string;
+}
+
+export async function POST(request: NextRequest) {
+  try {
+    const resultAdmin = await requireAdmin();
+    if (!resultAdmin.authenticated || !resultAdmin.admin) {
+      return NextResponse.json(
+        { error: 'Unauthorized' },
+        { status: 401 }
+      );
+    }
+    const body = await request.json();
+    const { datasetId, filters } = body;
+
+    if (!datasetId) {
+      return NextResponse.json(
+        { error: 'datasetId is required' },
+        { status: 400 }
+      );
+    }
+
+    // Get filtered entries data from server action
+    const result = await downloadFilteredEntries(datasetId, filters || {});
+
+    // Create ZIP on server
+    const zip = new JSZip();
+
+    // Add CSV file
+    zip.file('metadata.csv', result.csvContent);
+
+    // Add metadata JSON
+    zip.file('filter-metadata.json', result.metadataContent);
+
+    // Add audio files to 'audio' folder
+    if (result.entries && result.entries.length > 0) {
+      const audioFolder = zip.folder('wavs');
+      if (audioFolder) {
+        let filesAdded = 0;
+        for (const entry of result.entries) {
+          try {
+            const fileExtension = entry.fileName.substring(entry.fileName.lastIndexOf('.'));
+            const audioPath = join(
+              process.cwd(),
+              'public',
+              'datasets',
+              datasetId.toString(),
+              `${entry.id}${fileExtension}`
+            );
+
+            if (existsSync(audioPath)) {
+              const fileBuffer = await readFileAsync(audioPath);
+              const fileName = `${entry.id}${fileExtension}`;
+              audioFolder.file(fileName, fileBuffer);
+              filesAdded++;
+            } else {
+              console.warn(`Audio file not found: ${audioPath}`);
+            }
+          } catch (error) {
+            console.error(`Failed to add audio file for entry ${entry.id}:`, error);
+            // Continue with other files
+          }
+        }
+        console.log(`Added ${filesAdded}/${result.entries.length} audio files to ZIP`);
+      }
+    } else {
+      console.warn('No entries to add audio files for');
+    }
+
+    // Generate ZIP blob
+    const zipBuffer = await zip.generateAsync({ type: 'nodebuffer' });
+
+    // Create response with ZIP file
+    const fileName = `dataset-${datasetId}-export-${new Date().toISOString().split('T')[0]}.zip`;
+
+    return new NextResponse(new Uint8Array(zipBuffer), {
+      status: 200,
+      headers: {
+        'Content-Type': 'application/zip',
+        'Content-Disposition': `attachment; filename="${fileName}"`,
+        'Content-Length': zipBuffer.length.toString(),
+      },
+    });
+  } catch (error) {
+    console.error('Error creating ZIP:', error);
+    return NextResponse.json(
+      { error: 'Failed to create ZIP file' },
+      { status: 500 }
+    );
+  }
+}
--- a/src/components/DatasetEntriesList.tsx
+++ b/src/components/DatasetEntriesList.tsx
@@ -5,6 +5,7 @@ import Link from 'next/link';
 import AudioPlayer from './AudioPlayer';
 import { getDatasetEntries } from '@/app/actions/get-dataset-entries';
 import { getFilterOptions } from '@/app/actions/get-filter-options';
+import { downloadFilteredEntriesAsZip } from '@/lib/download-utils';

 interface DatasetEntriesListProps {
  datasetId: number;
@@ -38,6 +39,7 @@ export default function DatasetEntriesList({
 }: DatasetEntriesListProps) {
  const [entries, setEntries] = useState<DatasetEntry[]>([]);
  const [loading, setLoading] = useState(false);
+  const [downloading, setDownloading] = useState(false);
  const [hasMore, setHasMore] = useState(true);
  const [page, setPage] = useState(1);
  const [total, setTotal] = useState(0);
@@ -145,6 +147,26 @@ export default function DatasetEntriesList({
    });
  }

+  async function handleDownloadFiltered() {
+    setDownloading(true);
+    try {
+      const filterParams = {
+        speakerId: filters.speakerId || undefined,
+        modelName: filters.modelName || undefined,
+        dialect: filters.dialect || undefined,
+        iteration: filters.iteration ? parseInt(filters.iteration, 10) : undefined,
+        utteranceId: filters.utteranceId || undefined,
+      };
+
+      await downloadFilteredEntriesAsZip(datasetId, filterParams);
+    } catch (error) {
+      console.error('Error downloading entries:', error);
+      alert('Failed to download entries. Please try again.');
+    } finally {
+      setDownloading(false);
+    }
+  }
+
  if (entries.length === 0 && !loading) {
    return (
      <div className="bg-white rounded-lg shadow p-6">
@@ -154,6 +176,14 @@ export default function DatasetEntriesList({
        <div className="mb-6 p-4 bg-gray-50 rounded-lg border border-gray-200">
          <div className="flex items-center justify-between mb-4">
            <h3 className="text-lg font-semibold text-gray-700">Filters</h3>
+            <div className="flex gap-3">
+              <button
+                onClick={handleDownloadFiltered}
+                disabled={downloading || total === 0}
+                className="text-sm px-3 py-1 bg-green-500 text-white rounded hover:bg-green-600 disabled:bg-gray-400 disabled:cursor-not-allowed transition-colors"
+              >
+                {downloading ? 'Downloading...' : 'Download as ZIP'}
+              </button>
              <button
                onClick={handleClearFilters}
                className="text-sm text-blue-500 hover:text-blue-600"
@@ -161,6 +191,7 @@ export default function DatasetEntriesList({
                Clear all
              </button>
            </div>
+          </div>
          <div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-6 gap-4">
          <div>
            <label className="block text-sm font-medium text-gray-700 mb-1">
@@ -263,6 +294,14 @@ export default function DatasetEntriesList({
      <div className="mb-6 p-4 bg-gray-50 rounded-lg border border-gray-200">
        <div className="flex items-center justify-between mb-4">
          <h3 className="text-lg font-semibold text-gray-700">Filters</h3>
+          <div className="flex gap-3">
+            <button
+              onClick={handleDownloadFiltered}
+              disabled={downloading || total === 0}
+              className="text-sm px-3 py-1 bg-green-500 text-white rounded hover:bg-green-600 disabled:bg-gray-400 disabled:cursor-not-allowed transition-colors"
+            >
+              {downloading ? 'Downloading...' : 'Download as ZIP'}
+            </button>
            <button
              onClick={handleClearFilters}
              className="text-sm text-blue-500 hover:text-blue-600"
@@ -270,6 +309,7 @@ export default function DatasetEntriesList({
              Clear all
            </button>
          </div>
+        </div>
        <div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-6 gap-4">
          <div>
            <label className="block text-sm font-medium text-gray-700 mb-1">
--- a/src/lib/download-utils.ts
+++ b/src/lib/download-utils.ts
@@ -0,0 +1,46 @@
+interface FilterParams {
+  speakerId?: string;
+  modelName?: string;
+  dialect?: string;
+  iteration?: number | string;
+  utteranceId?: string;
+}
+
+export async function downloadFilteredEntriesAsZip(
+  datasetId: number,
+  filters: FilterParams
+) {
+  try {
+    const response = await fetch('/api/download-filtered-entries', {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify({
+        datasetId,
+        filters,
+      }),
+    });
+
+    if (!response.ok) {
+      const errorData = await response.json();
+      throw new Error(errorData.error || 'Failed to download ZIP');
+    }
+
+    // Get the ZIP blob from response
+    const blob = await response.blob();
+
+    // Trigger download
+    const url = URL.createObjectURL(blob);
+    const a = document.createElement('a');
+    a.href = url;
+    a.download = `dataset-${datasetId}-export-${new Date().toISOString().split('T')[0]}.zip`;
+    document.body.appendChild(a);
+    a.click();
+    document.body.removeChild(a);
+    URL.revokeObjectURL(url);
+  } catch (error) {
+    console.error('Error downloading ZIP:', error);
+    throw error;
+  }
+}