From 03118c468adbe104ff01fa7c269ba6b86fa72600 Mon Sep 17 00:00:00 2001 From: averel10 Date: Fri, 13 Mar 2026 10:53:55 +0100 Subject: [PATCH] feat: implement download functionality for filtered dataset entries as ZIP --- package-lock.json | 88 +++++++++++++++ package.json | 5 +- src/app/actions/download-filtered-entries.ts | 102 +++++++++++++++++ src/app/actions/remove-dataset-entries.ts | 5 + .../api/download-filtered-entries/route.ts | 106 ++++++++++++++++++ src/components/DatasetEntriesList.tsx | 64 +++++++++-- src/lib/download-utils.ts | 46 ++++++++ 7 files changed, 402 insertions(+), 14 deletions(-) create mode 100644 src/app/actions/download-filtered-entries.ts create mode 100644 src/app/api/download-filtered-entries/route.ts create mode 100644 src/lib/download-utils.ts diff --git a/package-lock.json b/package-lock.json index fe88edf..733bc30 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,6 +18,7 @@ "drizzle-kit": "^0.31.9", "drizzle-orm": "^0.45.1", "extract-zip": "^2.0.1", + "jszip": "^3.10.1", "next": "16.1.6", "postcss": "^8.5.6", "tailwindcss": "^4.2.1" @@ -3535,6 +3536,12 @@ "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", "license": "MIT" }, + "node_modules/core-util-is": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", + "license": "MIT" + }, "node_modules/cross-fetch": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-4.1.0.tgz", @@ -4100,6 +4107,12 @@ ], "license": "BSD-3-Clause" }, + "node_modules/immediate": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz", + "integrity": "sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==", + "license": "MIT" + }, "node_modules/inherits": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", @@ -4160,6 +4173,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "license": "MIT" + }, "node_modules/jiti": { "version": "2.6.1", "resolved": "https://registry.npmjs.org/jiti/-/jiti-2.6.1.tgz", @@ -4214,6 +4233,48 @@ "node": ">=6" } }, + "node_modules/jszip": { + "version": "3.10.1", + "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.10.1.tgz", + "integrity": "sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g==", + "license": "(MIT OR GPL-3.0-or-later)", + "dependencies": { + "lie": "~3.3.0", + "pako": "~1.0.2", + "readable-stream": "~2.3.6", + "setimmediate": "^1.0.5" + } + }, + "node_modules/jszip/node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "license": "MIT", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/jszip/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "license": "MIT" + }, + "node_modules/jszip/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, "node_modules/kleur": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz", @@ -4273,6 +4334,15 @@ "node": ">=8" } }, + "node_modules/lie": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz", + "integrity": "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==", + "license": "MIT", + "dependencies": { + "immediate": "~3.0.5" + } + }, "node_modules/lightningcss": { "version": "1.31.1", "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.31.1.tgz", @@ -4826,6 +4896,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/pako": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", + "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==", + "license": "(MIT AND Zlib)" + }, "node_modules/pathe": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", @@ -5059,6 +5135,12 @@ "url": "https://github.com/prettier/prettier?sponsor=1" } }, + "node_modules/process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", + "license": "MIT" + }, "node_modules/promise-limit": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/promise-limit/-/promise-limit-2.7.0.tgz", @@ -5241,6 +5323,12 @@ "integrity": "sha512-n7Z7dXZhJbwuAHhNzkTti6Aw9QDDjZtm3JTpTGATIdNzdQz5GuFs22w90BcvF4INfnrL5xrX3oGsuqO5Dx3A1Q==", "license": "MIT" }, + "node_modules/setimmediate": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", + "integrity": "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==", + "license": "MIT" + }, "node_modules/sharp": { "version": "0.34.5", "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.34.5.tgz", diff --git a/package.json b/package.json index 108f631..5d753f3 100644 --- a/package.json +++ b/package.json @@ -17,12 +17,13 @@ "better-auth": "^1.4.19", "better-auth-localization": "^2.3.1", "better-sqlite3": "^12.6.2", + "drizzle-kit": "^0.31.9", "drizzle-orm": "^0.45.1", "extract-zip": "^2.0.1", + "jszip": "^3.10.1", "next": "16.1.6", "postcss": "^8.5.6", - "tailwindcss": "^4.2.1", - "drizzle-kit": "^0.31.9" + "tailwindcss": "^4.2.1" }, "devDependencies": { "@types/better-sqlite3": "^7.6.13", diff --git a/src/app/actions/download-filtered-entries.ts b/src/app/actions/download-filtered-entries.ts new file mode 100644 index 0000000..d650cee --- /dev/null +++ b/src/app/actions/download-filtered-entries.ts @@ -0,0 +1,102 @@ +'use server'; + +import { requireAdmin } from '@/lib/auth'; +import db from '@/lib/db'; +import { dataset_entry } from '@/lib/model/dataset_entry'; +import { eq, and } from 'drizzle-orm'; +import * as fs from 'fs'; +import * as path from 'path'; + +interface FilterParams { + speakerId?: string; + modelName?: string; + dialect?: string; + iteration?: number; + utteranceId?: string; +} + +export async function downloadFilteredEntries( + datasetId: number, + filters: FilterParams +) { + const result = await requireAdmin(); + if (!result.authenticated || !result.admin) { + throw new Error('Unauthorized'); + } + // Build the where clause based on active filters + const conditions = [eq(dataset_entry.datasetId, datasetId)]; + + if (filters.speakerId) { + conditions.push(eq(dataset_entry.speakerId, filters.speakerId)); + } + if (filters.modelName) { + conditions.push(eq(dataset_entry.modelName, filters.modelName)); + } + if (filters.dialect) { + conditions.push(eq(dataset_entry.dialect, filters.dialect)); + } + if (filters.iteration !== undefined) { + conditions.push(eq(dataset_entry.iteration, filters.iteration)); + } + if (filters.utteranceId) { + conditions.push(eq(dataset_entry.utteranceId, filters.utteranceId)); + } + + // Fetch all matching entries + const entries = await db + .select() + .from(dataset_entry) + .where(conditions.length > 1 ? and(...conditions) : conditions[0]); + + // Create CSV content + const headers = [ + 'id', + 'audio_file', + 'duration_ms', + 'utt_id', + 'text', + 'speaker', + 'model', + 'dialect', + 'iteration', + ]; + + const csvLines = [headers.join(',')]; + entries.forEach((entry) => { + const row = [ + `"${entry.externalId}"`, + entry.fileName, + '', + entry.utteranceId || '', + `"${(entry.utteranceText || '').replace(/"/g, '""')}"`, + entry.speakerId, + entry.modelName, + entry.dialect, + entry.iteration, + ]; + csvLines.push(row.join(',')); + }); + + const csvContent = csvLines.join('\n'); + + // Create filter metadata JSON + const filterMetadata = { + exportedAt: new Date().toISOString(), + datasetId, + filters, + entriesCount: entries.length, + }; + + const metadataContent = JSON.stringify(filterMetadata, null, 2); + + // Return the data that will be zipped on the client + return { + csvContent, + metadataContent, + entriesCount: entries.length, + entries: entries.map(entry => ({ + id: entry.externalId, + fileName: entry.fileName, + })), + }; +} diff --git a/src/app/actions/remove-dataset-entries.ts b/src/app/actions/remove-dataset-entries.ts index 47ae780..9d799f8 100644 --- a/src/app/actions/remove-dataset-entries.ts +++ b/src/app/actions/remove-dataset-entries.ts @@ -7,9 +7,14 @@ import { revalidatePath } from 'next/cache'; import { rm } from 'fs/promises'; import { join } from 'path'; import { existsSync } from 'fs'; +import { requireAdmin } from '@/lib/auth'; export async function removeAllDatasetEntries(datasetId: number) { try { + const result = await requireAdmin(); + if (!result.authenticated || !result.admin) { + throw new Error('Unauthorized'); + } // Delete all entries from database await db.delete(dataset_entry).where(eq(dataset_entry.datasetId, datasetId)); diff --git a/src/app/api/download-filtered-entries/route.ts b/src/app/api/download-filtered-entries/route.ts new file mode 100644 index 0000000..f3f5549 --- /dev/null +++ b/src/app/api/download-filtered-entries/route.ts @@ -0,0 +1,106 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { downloadFilteredEntries } from '@/app/actions/download-filtered-entries'; +import JSZip from 'jszip'; +import { join } from 'path'; +import { readFile, existsSync } from 'fs'; +import { promisify } from 'util'; +import { requireAdmin } from '@/lib/auth'; + +const readFileAsync = promisify(readFile); + +interface FilterParams { + speakerId?: string; + modelName?: string; + dialect?: string; + iteration?: number; + utteranceId?: string; +} + +export async function POST(request: NextRequest) { + try { + const resultAdmin = await requireAdmin(); + if (!resultAdmin.authenticated || !resultAdmin.admin) { + return NextResponse.json( + { error: 'Unauthorized' }, + { status: 401 } + ); + } + const body = await request.json(); + const { datasetId, filters } = body; + + if (!datasetId) { + return NextResponse.json( + { error: 'datasetId is required' }, + { status: 400 } + ); + } + + // Get filtered entries data from server action + const result = await downloadFilteredEntries(datasetId, filters || {}); + + // Create ZIP on server + const zip = new JSZip(); + + // Add CSV file + zip.file('metadata.csv', result.csvContent); + + // Add metadata JSON + zip.file('filter-metadata.json', result.metadataContent); + + // Add audio files to 'audio' folder + if (result.entries && result.entries.length > 0) { + const audioFolder = zip.folder('wavs'); + if (audioFolder) { + let filesAdded = 0; + for (const entry of result.entries) { + try { + const fileExtension = entry.fileName.substring(entry.fileName.lastIndexOf('.')); + const audioPath = join( + process.cwd(), + 'public', + 'datasets', + datasetId.toString(), + `${entry.id}${fileExtension}` + ); + + if (existsSync(audioPath)) { + const fileBuffer = await readFileAsync(audioPath); + const fileName = `${entry.id}${fileExtension}`; + audioFolder.file(fileName, fileBuffer); + filesAdded++; + } else { + console.warn(`Audio file not found: ${audioPath}`); + } + } catch (error) { + console.error(`Failed to add audio file for entry ${entry.id}:`, error); + // Continue with other files + } + } + console.log(`Added ${filesAdded}/${result.entries.length} audio files to ZIP`); + } + } else { + console.warn('No entries to add audio files for'); + } + + // Generate ZIP blob + const zipBuffer = await zip.generateAsync({ type: 'nodebuffer' }); + + // Create response with ZIP file + const fileName = `dataset-${datasetId}-export-${new Date().toISOString().split('T')[0]}.zip`; + + return new NextResponse(new Uint8Array(zipBuffer), { + status: 200, + headers: { + 'Content-Type': 'application/zip', + 'Content-Disposition': `attachment; filename="${fileName}"`, + 'Content-Length': zipBuffer.length.toString(), + }, + }); + } catch (error) { + console.error('Error creating ZIP:', error); + return NextResponse.json( + { error: 'Failed to create ZIP file' }, + { status: 500 } + ); + } +} diff --git a/src/components/DatasetEntriesList.tsx b/src/components/DatasetEntriesList.tsx index 1ff8b54..029d53f 100644 --- a/src/components/DatasetEntriesList.tsx +++ b/src/components/DatasetEntriesList.tsx @@ -5,6 +5,7 @@ import Link from 'next/link'; import AudioPlayer from './AudioPlayer'; import { getDatasetEntries } from '@/app/actions/get-dataset-entries'; import { getFilterOptions } from '@/app/actions/get-filter-options'; +import { downloadFilteredEntriesAsZip } from '@/lib/download-utils'; interface DatasetEntriesListProps { datasetId: number; @@ -38,6 +39,7 @@ export default function DatasetEntriesList({ }: DatasetEntriesListProps) { const [entries, setEntries] = useState([]); const [loading, setLoading] = useState(false); + const [downloading, setDownloading] = useState(false); const [hasMore, setHasMore] = useState(true); const [page, setPage] = useState(1); const [total, setTotal] = useState(0); @@ -145,6 +147,26 @@ export default function DatasetEntriesList({ }); } + async function handleDownloadFiltered() { + setDownloading(true); + try { + const filterParams = { + speakerId: filters.speakerId || undefined, + modelName: filters.modelName || undefined, + dialect: filters.dialect || undefined, + iteration: filters.iteration ? parseInt(filters.iteration, 10) : undefined, + utteranceId: filters.utteranceId || undefined, + }; + + await downloadFilteredEntriesAsZip(datasetId, filterParams); + } catch (error) { + console.error('Error downloading entries:', error); + alert('Failed to download entries. Please try again.'); + } finally { + setDownloading(false); + } + } + if (entries.length === 0 && !loading) { return (
@@ -154,12 +176,21 @@ export default function DatasetEntriesList({

Filters

- +
+ + +
@@ -263,12 +294,21 @@ export default function DatasetEntriesList({

Filters

- +
+ + +
diff --git a/src/lib/download-utils.ts b/src/lib/download-utils.ts new file mode 100644 index 0000000..84c3937 --- /dev/null +++ b/src/lib/download-utils.ts @@ -0,0 +1,46 @@ +interface FilterParams { + speakerId?: string; + modelName?: string; + dialect?: string; + iteration?: number | string; + utteranceId?: string; +} + +export async function downloadFilteredEntriesAsZip( + datasetId: number, + filters: FilterParams +) { + try { + const response = await fetch('/api/download-filtered-entries', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + datasetId, + filters, + }), + }); + + if (!response.ok) { + const errorData = await response.json(); + throw new Error(errorData.error || 'Failed to download ZIP'); + } + + // Get the ZIP blob from response + const blob = await response.blob(); + + // Trigger download + const url = URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = `dataset-${datasetId}-export-${new Date().toISOString().split('T')[0]}.zip`; + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + URL.revokeObjectURL(url); + } catch (error) { + console.error('Error downloading ZIP:', error); + throw error; + } +}