diff --git a/src/app/actions/annotations.ts b/src/app/actions/annotations.ts index 76a6c2b..e609591 100644 --- a/src/app/actions/annotations.ts +++ b/src/app/actions/annotations.ts @@ -9,11 +9,63 @@ import { auth } from '@/lib/auth'; import { headers } from 'next/headers'; import type { DatasetEntryForAnnotation } from '@/lib/dialects'; import { experiment } from '@/lib/model/experiment'; -import { experiment_calibration } from '@/lib/model/experiment_calibration'; -import { participant } from '@/lib/model/participant'; +import { getDialectScoresFromCalibration, isCalibrationDone } from '@/app/actions/calibration-scoring'; + + +/** + * Sorts entries by weighted dialect score with fuzzy interleaving (deterministic round-robin). + * Higher-scoring dialects appear more frequently based on their relative performance. + */ +function sortEntriesByWeightedDialectScore( + entries: DatasetEntryForAnnotation[], + dialectScores: Record +): DatasetEntryForAnnotation[] { + const dialectGroups = new Map(); + + // Group entries by dialect + for (const entry of entries) { + if (!dialectGroups.has(entry.dialect)) { + dialectGroups.set(entry.dialect, []); + } + dialectGroups.get(entry.dialect)!.push(entry); + } + + // Sort groups by dialect score (highest first) + const sortedGroups = Array.from(dialectGroups.entries()) + .sort((a, b) => { + const aScore = dialectScores[a[0]] || 0; + const bScore = dialectScores[b[0]] || 0; + return bScore - aScore; + }); + + // Calculate weight multipliers based on scores (higher score = more samples per cycle) + const maxScore = Math.max(...sortedGroups.map(g => dialectScores[g[0]] || 0), 0.1); + const groupWeights = sortedGroups.map(([dialect, entries]) => { + const score = dialectScores[dialect] || 0; + const weight = Math.max(1, Math.round((score / maxScore) * 5)); // 1-5x multiplier based on score + return { dialect, entries, weight, index: 0 }; // Track current position in group + }); + + // Interleave entries with weighted round-robin for fuzzy distribution + const result: DatasetEntryForAnnotation[] = []; + const totalEntries = entries.length; + + while (result.length < totalEntries) { + for (const group of groupWeights) { + // Add 'weight' entries from this group (or until we run out) + for (let w = 0; w < group.weight && group.index < group.entries.length && result.length < totalEntries; w++) { + result.push(group.entries[group.index]); + group.index++; + } + } + } + + return result; +} /** * Returns unannotated entries for the given dataset and authenticated user. + * Sorts entries by calibration performance - dialects the user identified correctly and confidently appear first. */ export async function getAnnotationEntries(experimentId: number): Promise { const session = await auth.api.getSession({ headers: await headers() }); @@ -52,6 +104,9 @@ export async function getAnnotationEntries(experimentId: number): Promise ({ id: e.id, externalId: e.externalId, @@ -63,7 +118,7 @@ export async function getAnnotationEntries(experimentId: number): Promise a.datasetEntryId === e.id)?.rating || null, })); - return mapped; + return sortEntriesByWeightedDialectScore(mapped, dialectScores); } /** @@ -135,60 +190,3 @@ export async function getAnnotationProgress( return { total: allEntries.length, done: annotated.length }; } -/** - * Checks if calibration is required and completed for the current user in an experiment. - * Returns true if calibration is completed or not required, false if calibration is pending. - */ -export async function isCalibrationDone(experimentId: number): Promise { - console.log('Checking calibration status for experimentId:', experimentId); - const session = await auth.api.getSession({ headers: await headers() }); - if (!session) throw new Error('Nicht angemeldet'); - - // Get calibration items for this experiment - const calibrationItems = await db - .select() - .from(experiment_calibration) - .where(eq(experiment_calibration.experimentId, experimentId)); - - // If no calibration items are defined, calibration is not required - if (calibrationItems.length === 0) { - return true; - } - - // Check if participant exists for this user and experiment - const participantRecord = await db - .select() - .from(participant) - .where( - and( - eq(participant.experimentId, experimentId), - eq(participant.userId, session.user.id) - ) - ) - .limit(1); - - // If no participant record exists, calibration is not done - if (participantRecord.length === 0) { - return false; - } - - // Check if calibration answers are filled - const calibrationAnswers = participantRecord[0].calibrationAnswers as Record | null; - - if (!calibrationAnswers) { - return false; - } - - // Verify that all calibration items have complete answers - for (const item of calibrationItems) { - const answer = calibrationAnswers[item.id]; - - // Check if answer exists and has both required fields - if (!answer || !answer.dialectLabel || !answer.confidence) { - return false; - } - } - - return true; -} - diff --git a/src/app/actions/calibration-scoring.ts b/src/app/actions/calibration-scoring.ts new file mode 100644 index 0000000..ed41998 --- /dev/null +++ b/src/app/actions/calibration-scoring.ts @@ -0,0 +1,137 @@ +'use server'; + +import db from '@/lib/db'; +import { experiment_calibration } from '@/lib/model/experiment_calibration'; +import { participant } from '@/lib/model/participant'; +import { eq, and } from 'drizzle-orm'; +import { auth } from '@/lib/auth'; +import { headers } from 'next/headers'; + +/** + * Calculates dialect performance scores based on the participant's calibration answers. + * Score = (correctness × confidence) averaged per dialect + * Range: -1 to 1, where: + * 1 = perfect identification with full confidence + * 0 = neutral (correct but unsure, or incorrect and unsure) + * -1 = completely wrong with full confidence (penalizes false confidence) + */ +export async function getDialectScoresFromCalibration( + experimentId: number +): Promise> { + const session = await auth.api.getSession({ headers: await headers() }); + if (!session) throw new Error('Nicht angemeldet'); + + // Fetch calibration items and participant's answers + const calibrationItems = await db + .select() + .from(experiment_calibration) + .where(eq(experiment_calibration.experimentId, experimentId)); + + const participantRecord = await db + .select() + .from(participant) + .where( + and( + eq(participant.experimentId, experimentId), + eq(participant.userId, session.user.id) + ) + ) + .limit(1); + + // Calculate dialect scores based on calibration performance + const dialectScores: Record = {}; + + if (participantRecord.length > 0 && participantRecord[0].calibrationAnswers) { + const calibrationAnswers = participantRecord[0].calibrationAnswers as Record< + number, + { dialectLabel: string; confidence: number } + >; + + // For each dialect, calculate accuracy and average confidence + for (const item of calibrationItems) { + const answer = calibrationAnswers[item.id]; + if (!answer) continue; + + const isCorrect = answer.dialectLabel === item.dialectLabel ? 1 : -1; + const confidence = answer.confidence / 5; // Normalize to 0-1 + + // Score = correctness * confidence + // Correct answers: +confidence (0 to 1) + // Incorrect answers: -confidence (-1 to 0) - penalizes false confidence + const itemScore = isCorrect * confidence; + + if (!dialectScores[item.dialectLabel]) { + dialectScores[item.dialectLabel] = 0; + } + dialectScores[item.dialectLabel] += itemScore; + } + + // Average the scores by number of items per dialect + const dialectCounts: Record = {}; + for (const item of calibrationItems) { + dialectCounts[item.dialectLabel] = (dialectCounts[item.dialectLabel] || 0) + 1; + } + for (const dialect in dialectScores) { + dialectScores[dialect] = dialectScores[dialect] / (dialectCounts[dialect] || 1); + } + } + + return dialectScores; +} + +/** + * Checks if calibration is required and completed for the current user in an experiment. + * Returns true if calibration is completed or not required, false if calibration is pending. + */ +export async function isCalibrationDone(experimentId: number): Promise { + console.log('Checking calibration status for experimentId:', experimentId); + const session = await auth.api.getSession({ headers: await headers() }); + if (!session) throw new Error('Nicht angemeldet'); + + // Get calibration items for this experiment + const calibrationItems = await db + .select() + .from(experiment_calibration) + .where(eq(experiment_calibration.experimentId, experimentId)); + + // If no calibration items are defined, calibration is not required + if (calibrationItems.length === 0) { + return true; + } + + // Check if participant exists for this user and experiment + const participantRecord = await db + .select() + .from(participant) + .where( + and( + eq(participant.experimentId, experimentId), + eq(participant.userId, session.user.id) + ) + ) + .limit(1); + + // If no participant record exists, calibration is not done + if (participantRecord.length === 0) { + return false; + } + + // Check if calibration answers are filled + const calibrationAnswers = participantRecord[0].calibrationAnswers as Record | null; + + if (!calibrationAnswers) { + return false; + } + + // Verify that all calibration items have complete answers + for (const item of calibrationItems) { + const answer = calibrationAnswers[item.id]; + + // Check if answer exists and has both required fields + if (!answer || !answer.dialectLabel || !answer.confidence) { + return false; + } + } + + return true; +} diff --git a/src/app/annotate/[experimentId]/page.tsx b/src/app/annotate/[experimentId]/page.tsx index 79a162d..ae6654a 100644 --- a/src/app/annotate/[experimentId]/page.tsx +++ b/src/app/annotate/[experimentId]/page.tsx @@ -2,7 +2,8 @@ import { redirect, notFound } from 'next/navigation'; import { headers } from 'next/headers'; import Link from 'next/link'; import { auth } from '@/lib/auth'; -import { getAnnotationEntries, isCalibrationDone } from '@/app/actions/annotations'; +import { getAnnotationEntries} from '@/app/actions/annotations'; +import { isCalibrationDone } from '@/app/actions/calibration-scoring'; import { getExperimentById } from '@/app/actions/experiment'; import AnnotationPageView from '@/components/AnnotationViews/AnnotationPageView'; import CalibrationPhase from '@/components/CalibrationViews/CalibrationPhase'; diff --git a/src/components/AnnotationViews/AnnotationPageView.tsx b/src/components/AnnotationViews/AnnotationPageView.tsx index 91888a8..ce26735 100644 --- a/src/components/AnnotationViews/AnnotationPageView.tsx +++ b/src/components/AnnotationViews/AnnotationPageView.tsx @@ -1,12 +1,14 @@ 'use client'; -import { useState, useMemo, useTransition, JSX } from 'react'; +import { useState, useMemo, useTransition, JSX, useEffect } from 'react'; import Link from 'next/link'; import { saveAnnotations } from '@/app/actions/annotations'; +import { getDialectScoresFromCalibration } from '@/app/actions/calibration-scoring'; import { DIALECT_LABELS, type DatasetEntryForAnnotation } from '@/lib/dialects'; import SingleChoiceEntryView from './SingleChoiceEntryView'; import SingleChoiceBinaryEntryView from './SingleChoiceBinaryEntryView'; import AnnotationSidebarNavigation from './AnnotationSidebarNavigation'; +import CalibrationScoresModal from '@/components/CalibrationScoresModal'; export interface EntryViewProps { entry: DatasetEntryForAnnotation; @@ -29,6 +31,21 @@ export default function AnnotationPageView({ }: AnnotationPageViewProps) { const [isPending, startTransition] = useTransition(); + const [isCalibrationModalOpen, setIsCalibrationModalOpen] = useState(false); + const [dialectScores, setDialectScores] = useState>({}); + + // Load calibration scores on component mount + useEffect(() => { + const loadScores = async () => { + try { + const scores = await getDialectScoresFromCalibration(experimentId); + setDialectScores(scores); + } catch (error) { + console.error('Error loading calibration scores:', error); + } + }; + loadScores(); + }, [experimentId]); const getAnnotationConfig = (dialectLabel: string) => { switch (viewType) { @@ -245,6 +262,15 @@ export default function AnnotationPageView({ > ← Startseite + {Object.keys(dialectScores).length > 0 && ( + + )}
+ + {/* Calibration Scores Modal */} + setIsCalibrationModalOpen(false)} + dialectScores={dialectScores} + /> ); } diff --git a/src/components/AnnotationViews/AnnotationSidebarNavigation.tsx b/src/components/AnnotationViews/AnnotationSidebarNavigation.tsx index 1f68b8f..4479cb9 100644 --- a/src/components/AnnotationViews/AnnotationSidebarNavigation.tsx +++ b/src/components/AnnotationViews/AnnotationSidebarNavigation.tsx @@ -1,6 +1,6 @@ 'use client'; -import { DatasetEntryForAnnotation } from '@/lib/dialects'; +import { DatasetEntryForAnnotation, DIALECT_LABELS } from '@/lib/dialects'; import { useEffect, useRef } from 'react'; interface AnnotationSidebarNavigationProps { @@ -77,7 +77,7 @@ export default function AnnotationSidebarNavigation({ /> {/* Entry number and filename */} - {`Sample ${index + 1}`} + {`Sample ${index + 1} - ${DIALECT_LABELS[entry.dialect]}`} diff --git a/src/components/CalibrationScoresDisplay.tsx b/src/components/CalibrationScoresDisplay.tsx new file mode 100644 index 0000000..8399a15 --- /dev/null +++ b/src/components/CalibrationScoresDisplay.tsx @@ -0,0 +1,53 @@ +'use client'; + +import { DIALECT_LABELS } from '@/lib/dialects'; + +interface CalibrationScoresDisplayProps { + dialectScores: Record; +} + +export default function CalibrationScoresDisplay({ + dialectScores, +}: CalibrationScoresDisplayProps) { + if (Object.keys(dialectScores).length === 0) return null; + + const sortedScores = Object.entries(dialectScores).sort((a, b) => b[1] - a[1]); + + return ( +
+
+ {sortedScores.map(([dialect, score]) => { + // Normalize score from -1 to 1 range to 0 to 100 percentage + const percentage = ((score + 1) / 2) * 100; + const isGood = score > 0.5; + const isFair = score > -0.5; + + return ( +
+
+ {DIALECT_LABELS[dialect]} + + {(score * 100).toFixed(0)}% + +
+
+
+
+
+ ); + })} +
+

+ Die Scores basieren auf der Genauigkeit und dem Vertrauen in Ihre Kalibrierergebnisse. + Dialekte mit höheren Scores werden zuerst in der Annotation angezeigt. +

+
+ ); +} diff --git a/src/components/CalibrationScoresModal.tsx b/src/components/CalibrationScoresModal.tsx new file mode 100644 index 0000000..9851406 --- /dev/null +++ b/src/components/CalibrationScoresModal.tsx @@ -0,0 +1,39 @@ +'use client'; + +import Modal from '@/components/Modal'; +import { DIALECT_LABELS } from '@/lib/dialects'; +import CalibrationScoresDisplay from './CalibrationScoresDisplay'; + +interface CalibrationScoresModalProps { + isOpen: boolean; + onClose: () => void; + dialectScores: Record; +} + +export default function CalibrationScoresModal({ + isOpen, + onClose, + dialectScores, +}: CalibrationScoresModalProps) { + if (!isOpen || Object.keys(dialectScores).length === 0) return null; + + const sortedScores = Object.entries(dialectScores).sort((a, b) => b[1] - a[1]); + + return ( +
+
+
+

Ihre Kalibrierungsergebnisse

+ +
+ + +
+
+ ); +} diff --git a/src/components/CalibrationViews/CalibrationInfoView.tsx b/src/components/CalibrationViews/CalibrationInfoView.tsx new file mode 100644 index 0000000..ea464aa --- /dev/null +++ b/src/components/CalibrationViews/CalibrationInfoView.tsx @@ -0,0 +1,100 @@ +'use client'; + +interface CalibrationInfoViewProps { + onContinue: () => void; + hasExistingAnswers: boolean; +} + +export default function CalibrationInfoView({ + onContinue, + hasExistingAnswers, +}: CalibrationInfoViewProps) { + const handleContinue = () => { + onContinue(); + }; + + return ( +
+
+
+ {/* Header */} +
+

Kalibrierungsphase

+
+ + {/* Info Section */} +
+

Was ist die Kalibrierungsphase?

+

+ Die Kalibrierungsphase dient dazu, Ihre individuellen Stärken und Schwächen bei der Dialekterkennung zu verstehen. Sie hören verschiedene Audiosamples und geben an, welchen Dialekt Sie hören und wie sicher Sie sich dabei sind. +

+

+ Dies hilft uns, Ihre Annotationsergebnisse später besser zu bewerten und zu + verstehen. Die Annotationen werden auf Basis Ihrer Kalibrierungsergebnisse gewichtet, sodass Ihre Stärken stärker berücksichtigt werden. +

+
+ + {/* Instructions Section */} +
+

Wie funktioniert es?

+
    +
  1. + + 1 + + Sie hören ein Audiosample +
    2. +
  2. + + 2 + + Sie wählen den Dialekt aus, den Sie gehört haben +
    3. +
  3. + + 3 + + Sie bewerten Ihr Vertrauen in die Identifikation (von 1-5) +
    4. +
  4. + + 4 + + Sie wiederholen dies für alle Audiosamples +
    5. +
+
+ + {/* Important Note */} +
+

ℹ️ Wichtig

+

+ Sie können diese Seite jederzeit verlassen und später zurückkehren, um Ihre + Kalibrierung fortzusetzen. Ihre Antworten werden automatisch gespeichert. +

+
+ + {/* Status */} + {hasExistingAnswers && ( +
+

+ ✓ Sie haben diese Einführung bereits gesehen. Sie können Ihre Kalibrierung + fortsetzen. +

+
+ )} + + {/* Action Button */} +
+ +
+
+
+
+ ); +} diff --git a/src/components/CalibrationViews/CalibrationPageView.tsx b/src/components/CalibrationViews/CalibrationPageView.tsx index b9c5e7a..7c22925 100644 --- a/src/components/CalibrationViews/CalibrationPageView.tsx +++ b/src/components/CalibrationViews/CalibrationPageView.tsx @@ -3,8 +3,11 @@ import { useState, useTransition, useMemo, useEffect } from 'react'; import Link from 'next/link'; import { saveCalibrationAnswers, getCalibrationAnswers } from '@/app/actions/calibration'; +import { getDialectScoresFromCalibration } from '@/app/actions/calibration-scoring'; import { ExperimentCalibration } from '@/lib/model/experiment_calibration'; import CalibrationEntryView from './CalibrationEntryView'; +import CalibrationScoresDisplay from '@/components/CalibrationScoresDisplay'; +import { DIALECT_LABELS } from '@/lib/dialects'; interface CalibrationPageViewProps { entries: ExperimentCalibration[]; @@ -27,6 +30,7 @@ export default function CalibrationPageView({ const [answers, setAnswers] = useState({}); const [isLoading, setIsLoading] = useState(true); const [currentIndex, setCurrentIndex] = useState(0); + const [dialectScores, setDialectScores] = useState>({}); // Load existing answers on component mount useEffect(() => { @@ -64,6 +68,21 @@ export default function CalibrationPageView({ ); const isComplete = entries.length > 0 && completeAnswers.length === entries.length; const progressPct = Math.round((completeAnswers.length / entries.length) * 100); + + // Load dialect scores when calibration is complete + useEffect(() => { + if (isComplete && !isLoading) { + const loadScores = async () => { + try { + const scores = await getDialectScoresFromCalibration(experimentId); + setDialectScores(scores); + } catch (error) { + console.error('Error loading dialect scores:', error); + } + }; + loadScores(); + } + }, [isComplete, isLoading, experimentId]); // Check if current entry has complete answers const currentEntryComplete = currentEntry && @@ -142,40 +161,44 @@ export default function CalibrationPageView({ }); }; - if (isLoading) { - return ( -
-
-
-
⏳
-

Laden Sie Ihre vorherigen Antworten...

-
-
-
- ); - } - if (isComplete) { + // Sort dialect scores from highest to lowest + const sortedScores = Object.entries(dialectScores).sort((a, b) => b[1] - a[1]); + return (
-
-
🎉
-

- Kalibrierung abgeschlossen! -

-

- Sie haben alle Kalibrierungssamples bewertet. Vielen Dank! Sie können jetzt mit der - Annotation beginnen. -

- +
+
+
🎉
+

+ Kalibrierung abgeschlossen! +

+

+ Sie haben alle Kalibrierungssamples bewertet. Vielen Dank! Sie können jetzt mit der + Annotation beginnen. +

+
+ + {/* Dialect Scores Summary */} + {sortedScores.length > 0 && ( +
+

Ihre Kalibrierungsergebnisse

+ + +
+ )} + +
+ +
diff --git a/src/components/CalibrationViews/CalibrationPhase.tsx b/src/components/CalibrationViews/CalibrationPhase.tsx index 1f2297a..536f04d 100644 --- a/src/components/CalibrationViews/CalibrationPhase.tsx +++ b/src/components/CalibrationViews/CalibrationPhase.tsx @@ -1,30 +1,41 @@ 'use client'; import { useEffect, useState } from 'react'; -import { getCalibrationEntries } from '@/app/actions/calibration'; +import { getCalibrationEntries, getCalibrationAnswers } from '@/app/actions/calibration'; import { ExperimentCalibration } from '@/lib/model/experiment_calibration'; import CalibrationPageView from './CalibrationPageView'; +import CalibrationInfoView from './CalibrationInfoView'; interface CalibrationPhaseProps { experimentId: number; } export default function CalibrationPhase({ experimentId }: CalibrationPhaseProps) { - const [entries, setEntries] = useState(null); - const [error, setError] = useState(null); + const [entries, setEntries] = useState([]); + const [showInfoPage, setShowInfoPage] = useState(true); + const [hasExistingAnswers, setHasExistingAnswers] = useState(false); const [isLoading, setIsLoading] = useState(true); useEffect(() => { const load = async () => { try { - const result = await getCalibrationEntries(experimentId); - if (result.success && result.data) { + // Load calibration entries + const entriesResult = await getCalibrationEntries(experimentId); + if (entriesResult.success && entriesResult.data) { // Sort by order - const sorted = [...result.data].sort((a, b) => a.order - b.order); + const sorted = [...entriesResult.data].sort((a, b) => a.order - b.order); setEntries(sorted); } + + // Load existing answers to check if user has seen the info page before + const answersResult = await getCalibrationAnswers(experimentId); + if (answersResult.success && answersResult.data) { + const answers = answersResult.data as Record | null; + if (answers && Object.keys(answers).length > 0) { + setHasExistingAnswers(true); + } + } } catch (err) { - setError('Fehler beim Laden der Kalibrierungssamples'); console.error(err); } finally { setIsLoading(false); @@ -34,27 +45,28 @@ export default function CalibrationPhase({ experimentId }: CalibrationPhaseProps load(); }, [experimentId]); + // Show loading view if (isLoading) { return ( -
-
⏳
-

Lädt Kalibrierungssamples...

-

Bitte warten Sie...

+
+
+
+

Loading calibration...

+
); } - if (error || !entries || entries.length === 0) { + // Show info page first + if (showInfoPage) { return ( -
-
❌
-

Kalibrierung nicht verfügbar

-

- {error || 'Für dieses Experiment sind keine Kalibrierungssamples vorhanden.'} -

-
+ setShowInfoPage(false)} + hasExistingAnswers={hasExistingAnswers} + /> ); } + // Show calibration page return ; }