/**
* Audio Identification Component for SamCloud Music App
* For identifying unknown songs and maintaining identification history
*/
import React, { useState, useRef, useCallback, useEffect } from 'react';
import { toast } from 'sonner';
import { Play, Mic, MicOff, Search, Download, X, History, Clock, Volume2 } from 'lucide-react';
import './AudioIdentifier.css';
interface AudioMatch {
song_id: string;
title: string;
artist: string;
album: string;
confidence_score: number;
time_offset: number;
spotify_id?: string;
jellyfin_id?: string;
}
interface IdentificationResult {
identified: boolean;
matches: AudioMatch[];
top_match?: AudioMatch;
found_in_library: boolean;
library_track?: any;
}
interface HistoryEntry {
id: string;
timestamp: Date;
result: IdentificationResult;
user_action?: 'played' | 'downloaded' | 'ignored';
audio_duration: number;
}
interface AudioIdentifierProps {
mode?: 'icon' | 'full'; // Icon mode for header/sidebar, full mode for search page
onIdentified?: (result: IdentificationResult) => void;
}
// Sub-components for different states
const RecordingIndicator: React.FC<{ onStop: () => void }> = ({ onStop }) => (
🎵 Listening for music...
Recording up to 20 seconds
💡 Tips for better results:
- Hold device closer to the music source
- Avoid talking during recording
- Make sure music volume is adequate
- Wait for a clear part of the song (vocals/instruments)
);
const IdentifyingIndicator: React.FC = () => (
Identifying audio...
Matching against our database
);
const IdentificationStart: React.FC<{ onFileUpload: (e: React.ChangeEvent) => void }> = ({ onFileUpload }) => (
);
const IdentificationResults: React.FC<{
result: IdentificationResult;
onPlay: (id: string) => void;
onDownload: (spotify_id: string, title: string, artist: string) => void;
onIgnore: () => void;
inline?: boolean;
}> = ({ result, onPlay, onDownload, onIgnore, inline = false }) => (
{result.identified && result.top_match ? (
<>
{result.top_match.title}
{result.top_match.artist}
{result.top_match.album && (
{result.top_match.album}
)}
Confidence: {result.top_match.confidence_score.toFixed(1)}%
{result.found_in_library && result.top_match.jellyfin_id ? (
) : result.top_match.spotify_id ? (
) : null}
{result.matches.length > 1 && (
Other possible matches:
{result.matches.slice(1, 4).map((match) => (
{match.title}
{match.artist}
{match.confidence_score.toFixed(1)}%
{match.jellyfin_id ? (
) : match.spotify_id ? (
) : null}
))}
)}
>
) : (
No matches found
Try recording again with less background noise
)}
);
const IdentificationHistory: React.FC<{
history: HistoryEntry[];
onPlayTrack: (id: string) => void;
onDownloadTrack: (spotify_id: string, title: string, artist: string) => void;
}> = ({ history, onPlayTrack, onDownloadTrack }) => (
{history.length === 0 ? (
No identification history yet
Start identifying songs to see your history here
) : (
{history.map((entry) => (
{entry.timestamp.toLocaleString()}
{entry.result.identified && entry.result.top_match ? (
{entry.result.top_match.title}
{entry.result.top_match.artist}
{entry.user_action && (
{entry.user_action}
)}
{entry.result.top_match.jellyfin_id ? (
) : entry.result.top_match.spotify_id ? (
) : null}
) : (
No match found ({entry.audio_duration.toFixed(1)}s recording)
)}
))}
)}
);
export const AudioIdentifier: React.FC = ({
mode = 'icon',
onIdentified
}) => {
const [isRecording, setIsRecording] = useState(false);
const [isIdentifying, setIsIdentifying] = useState(false);
const [showModal, setShowModal] = useState(false);
const [result, setResult] = useState(null);
const [showHistory, setShowHistory] = useState(false);
const [history, setHistory] = useState([]);
const mediaRecorderRef = useRef(null);
const audioChunksRef = useRef([]);
const streamRef = useRef(null);
// Load identification history on mount
useEffect(() => {
loadIdentificationHistory();
}, []);
const loadIdentificationHistory = async () => {
try {
const response = await fetch('/api/audio/identification-history');
if (response.ok) {
const historyData = await response.json();
setHistory(historyData.map((entry: any) => ({
...entry,
timestamp: new Date(entry.timestamp)
})));
}
} catch (error) {
console.error('Error loading identification history:', error);
}
};
const saveToHistory = async (result: IdentificationResult, audioDuration: number) => {
try {
const historyEntry = {
timestamp: new Date(),
result,
audio_duration: audioDuration
};
const response = await fetch('/api/audio/save-identification', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(historyEntry)
});
if (response.ok) {
const saved = await response.json();
setHistory(prev => [{ ...historyEntry, id: saved.id }, ...prev]);
}
} catch (error) {
console.error('Error saving to history:', error);
}
};
const startRecording = useCallback(async () => {
try {
// Request high-quality audio with optimal settings for music identification
const stream = await navigator.mediaDevices.getUserMedia({
audio: {
sampleRate: { ideal: 44100, min: 22050 }, // Higher sample rate for better quality
channelCount: 1,
echoCancellation: false, // Keep original audio for fingerprinting
noiseSuppression: false, // Keep original audio for fingerprinting
autoGainControl: false // Prevent automatic gain changes
}
});
streamRef.current = stream;
const mediaRecorder = new MediaRecorder(stream);
mediaRecorderRef.current = mediaRecorder;
audioChunksRef.current = [];
const startTime = Date.now();
mediaRecorder.ondataavailable = (event) => {
if (event.data.size > 0) {
audioChunksRef.current.push(event.data);
}
};
mediaRecorder.onstop = async () => {
const audioBlob = new Blob(audioChunksRef.current, { type: 'audio/wav' });
const duration = (Date.now() - startTime) / 1000;
await identifyAudio(audioBlob, duration);
};
mediaRecorder.start();
setIsRecording(true);
// Open modal if in icon mode
if (mode === 'icon') {
setShowModal(true);
}
// Auto-stop after 20 seconds (longer for better identification)
setTimeout(() => {
if (mediaRecorderRef.current && isRecording) {
stopRecording();
}
}, 20000);
} catch (error) {
console.error('Error starting recording:', error);
toast.error('Failed to access microphone');
}
}, [mode]);
const stopRecording = useCallback(() => {
if (mediaRecorderRef.current && isRecording) {
mediaRecorderRef.current.stop();
setIsRecording(false);
setIsIdentifying(true);
}
if (streamRef.current) {
streamRef.current.getTracks().forEach((track: MediaStreamTrack) => track.stop());
streamRef.current = null;
}
}, [isRecording]);
const identifyAudio = async (audioBlob: Blob, duration: number) => {
try {
const formData = new FormData();
formData.append('audio', audioBlob, 'recording.wav');
const response = await fetch('/api/audio/identify', {
method: 'POST',
body: formData,
});
const data: IdentificationResult = await response.json();
setResult(data);
setIsIdentifying(false);
// Save to history
await saveToHistory(data, duration);
// Callback for parent component
if (onIdentified) {
onIdentified(data);
}
if (data.identified) {
toast.success(`Identified: ${data.top_match?.title} by ${data.top_match?.artist}`);
} else {
toast.error('No matches found - try recording again with less background noise');
}
} catch (error) {
console.error('Error identifying audio:', error);
toast.error('Audio identification failed');
setIsIdentifying(false);
}
};
const handleFileUpload = async (event: React.ChangeEvent) => {
const file = event.target.files?.[0];
if (!file) return;
setIsIdentifying(true);
if (mode === 'icon') {
setShowModal(true);
}
// Estimate duration (rough)
const duration = 10; // Default for uploaded files
await identifyAudio(file, duration);
event.target.value = ''; // Reset file input
};
const playTrack = async (jellyfin_id: string) => {
try {
const playbackResponse = await fetch(`/api/jellyfin/play/${jellyfin_id}`, {
method: 'POST',
});
if (playbackResponse.ok) {
toast.success('Playing track');
await updateHistoryAction(result!, 'played');
setShowModal(false);
} else {
toast.error('Failed to play track');
}
} catch (error) {
console.error('Error playing track:', error);
toast.error('Failed to play track');
}
};
const downloadTrack = async (spotify_id: string, title: string, artist: string) => {
try {
const spotify_url = `https://open.spotify.com/track/${spotify_id}`;
const downloadResponse = await fetch('/api/download', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
url: spotify_url,
source: 'audio_identification', // Track that this came from audio ID
title,
artist
})
});
if (downloadResponse.ok) {
toast.success(`Added "${title}" to download queue`);
await updateHistoryAction(result!, 'downloaded');
setShowModal(false);
} else {
toast.error('Failed to add to download queue');
}
} catch (error) {
console.error('Error downloading track:', error);
toast.error('Download failed');
}
};
const updateHistoryAction = async (result: IdentificationResult, action: 'played' | 'downloaded' | 'ignored') => {
try {
const response = await fetch('/api/audio/update-history-action', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ result_id: result.top_match?.song_id, action })
});
if (response.ok) {
// Update local history
setHistory(prev => prev.map(entry =>
entry.result.top_match?.song_id === result.top_match?.song_id
? { ...entry, user_action: action }
: entry
));
}
} catch (error) {
console.error('Error updating history action:', error);
}
};
const closeModal = () => {
setShowModal(false);
setResult(null);
setShowHistory(false);
};
// Icon mode - just the button for header/sidebar
if (mode === 'icon') {
return (
<>
{/* Modal for icon mode */}
{showModal && (
{isRecording ? 'Listening...' :
isIdentifying ? 'Identifying...' :
result ? 'Identification Results' :
showHistory ? 'Identification History' : 'Audio Identification'}
{!isRecording && !isIdentifying && (
)}
{showHistory ? (
) : isRecording ? (
) : isIdentifying ? (
) : result ? (
updateHistoryAction(result, 'ignored')}
/>
) : (
)}
)}
>
);
}
// Full mode - for search page integration
return (
Can't find it? Try Audio Identification
Record a snippet of the song you're looking for
{/* Inline results for full mode */}
{result && (
updateHistoryAction(result, 'ignored')}
inline
/>
)}
{/* History access for full mode */}
{/* History modal for full mode */}
{showHistory && (
Identification History
)}
);
};