fixed issues with upload
This commit is contained in:
@@ -8,6 +8,7 @@ import { writeFile, mkdir, readFile, rm, readdir } from 'fs/promises';
|
|||||||
import { join } from 'path';
|
import { join } from 'path';
|
||||||
import { existsSync } from 'fs';
|
import { existsSync } from 'fs';
|
||||||
import extract from 'extract-zip';
|
import extract from 'extract-zip';
|
||||||
|
import { line } from 'drizzle-orm/pg-core';
|
||||||
|
|
||||||
interface DatasetEntryInput {
|
interface DatasetEntryInput {
|
||||||
id: string;
|
id: string;
|
||||||
@@ -71,8 +72,41 @@ export async function uploadDatasetEntries(
|
|||||||
// Parse and insert entries
|
// Parse and insert entries
|
||||||
const entries: typeof dataset_entry.$inferInsert[] = [];
|
const entries: typeof dataset_entry.$inferInsert[] = [];
|
||||||
|
|
||||||
|
// CSV parser that handles quoted fields
|
||||||
|
const parseCSVLine = (line: string): string[] => {
|
||||||
|
const values: string[] = [];
|
||||||
|
let current = '';
|
||||||
|
let inQuotes = false;
|
||||||
|
|
||||||
|
for (let i = 0; i < line.length; i++) {
|
||||||
|
const char = line[i];
|
||||||
|
const nextChar = line[i + 1];
|
||||||
|
|
||||||
|
if (char === '"') {
|
||||||
|
if (inQuotes && nextChar === '"') {
|
||||||
|
// Escaped quote
|
||||||
|
current += '"';
|
||||||
|
i++;
|
||||||
|
} else {
|
||||||
|
// Toggle quote state
|
||||||
|
inQuotes = !inQuotes;
|
||||||
|
}
|
||||||
|
} else if (char === ',' && !inQuotes) {
|
||||||
|
// Field separator
|
||||||
|
values.push(current.trim());
|
||||||
|
current = '';
|
||||||
|
} else {
|
||||||
|
current += char;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add the last field
|
||||||
|
values.push(current.trim());
|
||||||
|
return values;
|
||||||
|
};
|
||||||
|
|
||||||
for (let i = 1; i < lines.length; i++) {
|
for (let i = 1; i < lines.length; i++) {
|
||||||
const values = lines[i].split(',').map(v => v.trim());
|
const values = parseCSVLine(lines[i]);
|
||||||
|
|
||||||
if (values.length !== headers.length) {
|
if (values.length !== headers.length) {
|
||||||
continue; // Skip malformed lines
|
continue; // Skip malformed lines
|
||||||
@@ -81,11 +115,11 @@ export async function uploadDatasetEntries(
|
|||||||
const row: Record<string, string> = {};
|
const row: Record<string, string> = {};
|
||||||
headers.forEach((header, index) => {
|
headers.forEach((header, index) => {
|
||||||
row[header] = values[index];
|
row[header] = values[index];
|
||||||
});
|
})
|
||||||
|
|
||||||
const audioFile = row.audio_file as string;
|
const audioFile = row.audio_file as string;
|
||||||
// Try different path separators and remove /output prefix
|
// Try different path separators and remove /output prefix
|
||||||
const relativePath = audioFile.replace(/\\/g, '/').replace(/^output\//, '');
|
const relativePath = audioFile.replace(/\\/g, '/')
|
||||||
const audioPath = join(tempDir, relativePath);
|
const audioPath = join(tempDir, relativePath);
|
||||||
|
|
||||||
if (existsSync(audioPath)) {
|
if (existsSync(audioPath)) {
|
||||||
|
|||||||
Reference in New Issue
Block a user