//! Conservative recording identity checks and in-place library audio upgrades. use std::path::Path; use std::time::Duration; use anyhow::{Context, Result, ensure}; use serde::Deserialize; use sqlx::{FromRow, PgPool}; use super::dto::NormalizedFields; #[derive(Debug)] pub struct DuplicateReview(pub String); impl std::fmt::Display for DuplicateReview { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "Duplicate check needs review: {}", self.0) } } impl std::error::Error for DuplicateReview {} fn review(message: impl Into) -> anyhow::Error { DuplicateReview(message.into()).into() } #[derive(FromRow)] struct Candidate { track_id: i64, media_id: i64, file_path: String, sha256_hash: String, album: String, year: Option, artist_names: Vec, } #[derive(Debug)] struct AudioInfo { codec: String, duration: f64, channels: u32, sample_rate: u32, bit_depth: u32, bitrate: u64, } impl AudioInfo { fn lossless(&self) -> bool { matches!(self.codec.as_str(), "flac" | "alac" | "ape" | "tta") || self.codec.starts_with("pcm_") } fn lossy(&self) -> bool { matches!( self.codec.as_str(), "mp3" | "mp2" | "aac" | "opus" | "vorbis" | "wmav1" | "wmav2" | "ac3" | "eac3" | "amr_nb" | "amr_wb" ) } } #[derive(Debug, PartialEq)] enum Quality { Better, Keep, Incomparable, } fn compare_quality(new: &AudioInfo, old: &AudioInfo) -> Quality { if new.channels != old.channels || new.channels == 0 || new.sample_rate == 0 || old.sample_rate == 0 || (!new.lossless() && !new.lossy()) || (!old.lossless() && !old.lossy()) { return Quality::Incomparable; } if old.lossless() && !new.lossless() { return Quality::Keep; } if new.lossless() && !old.lossless() { return if new.sample_rate >= old.sample_rate && new.bit_depth >= 16 { Quality::Better } else { Quality::Incomparable }; } if new.lossless() && old.lossless() { if new.bit_depth == 0 || old.bit_depth == 0 { return Quality::Incomparable; } if new.sample_rate >= old.sample_rate && new.bit_depth >= old.bit_depth { return if new.sample_rate > old.sample_rate || new.bit_depth > old.bit_depth { Quality::Better } else { Quality::Keep }; } if new.sample_rate <= old.sample_rate && new.bit_depth <= old.bit_depth { return Quality::Keep; } return Quality::Incomparable; } // Bitrate alone is comparable only within the same codec and sampling rate. if new.codec != old.codec || new.sample_rate != old.sample_rate || new.bitrate == 0 || old.bitrate == 0 { return Quality::Incomparable; } if new.bitrate > old.bitrate + old.bitrate / 10 { Quality::Better } else { Quality::Keep } } fn durations_match(a: f64, b: f64) -> bool { a.is_finite() && b.is_finite() && a > 0.0 && b > 0.0 && (a - b).abs() <= 2.0_f64.min(a.min(b) * 0.01) } /// Compare the complete fingerprints, tolerating a small encoder-delay shift. /// Silence and very short/constant fingerprints cannot establish identity. fn fingerprints_match(a: &[u32], b: &[u32]) -> bool { if a.len().min(b.len()) < 64 { return false; } for fp in [a, b] { if fp .iter() .copied() .collect::>() .len() < 8 { return false; } } for offset in -8_i32..=8 { let left = &a[offset.max(0) as usize..]; let right = &b[(-offset).max(0) as usize..]; let length = left.len().min(right.len()); if length * 100 < a.len().max(b.len()) * 95 { continue; } // Every ~8-second section must agree, not just an identical intro. let mut total_errors = 0_u64; let mut all_sections_match = true; for start in (0..length).step_by(64) { let end = (start + 64).min(length); let errors: u64 = left[start..end] .iter() .zip(&right[start..end]) .map(|(x, y)| u64::from((x ^ y).count_ones())) .sum(); if errors * 100 > (end - start) as u64 * 32 * 10 { all_sections_match = false; break; } total_errors += errors; } if all_sections_match && total_errors * 100 <= length as u64 * 32 * 5 { return true; } } false } async fn command_output(program: &str, args: &[&str], path: &Path) -> Result> { let mut command = tokio::process::Command::new(program); command.args(args).arg(path).kill_on_drop(true); let output = tokio::time::timeout(Duration::from_secs(120), command.output()) .await .context("audio inspection timed out")? .with_context(|| format!("cannot run {program}"))?; // Debian's fpcalc 1.5.1 can report EOF after producing complete results. let legacy_eof = program == "fpcalc" && output.status.code() == Some(3) && String::from_utf8_lossy(&output.stderr).trim() == "ERROR: Error decoding audio frame (End of file)"; ensure!( output.status.success() || legacy_eof, "{program} failed: {}", String::from_utf8_lossy(&output.stderr) .chars() .take(300) .collect::() ); Ok(output.stdout) } async fn probe(path: &Path) -> Result { let output = command_output("ffprobe", &[ "-v", "error", "-select_streams", "a:0", "-show_entries", "stream=codec_name,sample_rate,bits_per_sample,bits_per_raw_sample,bit_rate,channels,duration:format=duration,bit_rate", "-of", "json", ], path).await?; let info: serde_json::Value = serde_json::from_slice(&output)?; let stream = info["streams"] .as_array() .and_then(|items| items.first()) .context("file has no audio stream")?; let number = |value: &serde_json::Value| { value .as_f64() .or_else(|| value.as_str().and_then(|s| s.parse().ok())) .unwrap_or(0.0) }; let duration = number(&stream["duration"]); Ok(AudioInfo { codec: stream["codec_name"] .as_str() .context("unknown audio codec")? .to_owned(), duration: if duration > 0.0 { duration } else { number(&info["format"]["duration"]) }, channels: number(&stream["channels"]) as u32, sample_rate: number(&stream["sample_rate"]) as u32, bit_depth: number(&stream["bits_per_raw_sample"]).max(number(&stream["bits_per_sample"])) as u32, // Stream bitrate excludes embedded artwork; container bitrate is not // evidence of a better lossy encoding. bitrate: number(&stream["bit_rate"]) as u64, }) } #[derive(Deserialize)] struct AudioFingerprint { duration: f64, fingerprint: Vec, } async fn fingerprint(path: &Path, expected_duration: f64) -> Result { // One chunk covering the whole recording reports actual decoded duration, // rather than copying a possibly stale duration from the container header. let output = command_output( "fpcalc", &["-json", "-raw", "-length", "0", "-chunk", "86400"], path, ) .await?; let result: AudioFingerprint = serde_json::from_slice(&output)?; ensure!( result.duration.is_finite() && result.duration > 0.0 && (result.duration - expected_duration).abs() <= 0.5, "decoded audio duration disagrees with its header; possible truncated source" ); Ok(result) } /// Returns true when the upload was merged into an existing track. /// Ambiguous matches remain in the inbox for review; no duplicate is created. pub async fn merge_existing( pool: &PgPool, storage_dir: &str, source: &Path, fields: &NormalizedFields, context: &serde_json::Value, ) -> Result { let Some(artist) = fields.artist.as_deref().filter(|s| !s.trim().is_empty()) else { return Ok(false); }; let Some(title) = fields.title.as_deref().filter(|s| !s.trim().is_empty()) else { return Ok(false); }; let candidates = sqlx::query_as::<_, Candidate>( r#"SELECT t.id AS track_id, m.id AS media_id, m.file_path, m.sha256_hash::text, r.title_sort::text AS album, r.year, ARRAY(SELECT DISTINCT a.name_sort::text FROM furumusic__track_artist ta JOIN furumusic__artist a ON a.id = ta.artist_id WHERE ta.track_id = t.id) AS artist_names FROM furumusic__track t JOIN furumusic__release r ON r.id = t.release_id JOIN furumusic__media_file m ON m.id = t.audio_file_id WHERE t.title_sort = $1 AND m.file_type = 'audio' AND EXISTS (SELECT 1 FROM furumusic__track_artist ta JOIN furumusic__artist a ON a.id = ta.artist_id WHERE ta.track_id = t.id AND a.name_sort = $2) ORDER BY t.id"#, ) .bind(title.trim().to_lowercase()) .bind(artist.trim().to_lowercase()) .fetch_all(pool) .await?; let mut artists: Vec<_> = fields .featured_artists .iter() .map(|s| s.trim().to_lowercase()) .collect(); artists.push(artist.trim().to_lowercase()); artists.sort(); artists.dedup(); let candidates: Vec<_> = candidates .into_iter() .map(|mut candidate| { candidate.artist_names.sort(); candidate.artist_names.dedup(); candidate }) .filter(|c| { fields .album .as_deref() .is_none_or(|album| c.album == album.trim().to_lowercase()) }) .collect(); if candidates.is_empty() { return Ok(false); } let inspected_sha = super::mover::sha256_file(source) .await .map_err(|e| review(format!("cannot read incoming audio: {e:#}")))?; let new_info = probe(source) .await .map_err(|e| review(format!("cannot inspect incoming audio: {e:#}")))?; let mut new_fingerprint = None; let mut confirmed = Vec::new(); let mut ambiguous = Vec::new(); for candidate in candidates { let existing = crate::media_paths::resolve_path_from_root(storage_dir, &candidate.file_path); let old_info = probe(&existing).await.map_err(|e| { review(format!( "cannot inspect existing track {}: {e:#}", candidate.track_id )) })?; if !durations_match(new_info.duration, old_info.duration) { ambiguous.push(format!( "track {}: duration differs ({:.3}s vs {:.3}s)", candidate.track_id, new_info.duration, old_info.duration )); continue; } if candidate.artist_names != artists { ambiguous.push(format!( "track {}: artist credits differ", candidate.track_id )); continue; } if fields.album.is_none() || fields.year.zip(candidate.year).is_some_and(|(a, b)| a != b) { ambiguous.push(format!( "track {}: album or edition year needs confirmation", candidate.track_id )); continue; } if new_fingerprint.is_none() { new_fingerprint = Some( fingerprint(source, new_info.duration) .await .map_err(|e| review(format!("cannot fingerprint incoming audio: {e:#}")))?, ); } let old_fp = fingerprint(&existing, old_info.duration) .await .map_err(|e| review(format!("cannot fingerprint existing track: {e:#}")))?; let new_fp = new_fingerprint.as_ref().unwrap(); if (new_fp.duration - old_fp.duration).abs() <= 0.25 && fingerprints_match(&new_fp.fingerprint, &old_fp.fingerprint) { confirmed.push((candidate, existing, old_info)); } else { // Same metadata/duration with different sound is an edition or a // metadata conflict, not permission to discard either recording. ambiguous.push(format!( "track {}: decoded duration or audio fingerprint differs", candidate.track_id )); } } if confirmed.len() > 1 { return Err(review( "multiple existing tracks match this recording; catalog deduplication needs review", )); } if !ambiguous.is_empty() { return Err(review(ambiguous.join("; "))); } let Some((candidate, old_path, old_info)) = confirmed.pop() else { return Ok(false); }; match compare_quality(&new_info, &old_info) { Quality::Keep => { if super::mover::sha256_file(source).await? != inspected_sha { return Err(review("incoming audio changed during duplicate inspection")); } tokio::fs::remove_file(source).await?; tracing::info!( track_id = candidate.track_id, "duplicate upload merged; existing audio retained" ); } Quality::Incomparable => { return Err(review(format!( "track {} matches; cannot automatically rank incoming {} {}kbps/{}Hz/{}bit/{}ch against existing {} {}kbps/{}Hz/{}bit/{}ch", candidate.track_id, new_info.codec, new_info.bitrate / 1000, new_info.sample_rate, new_info.bit_depth, new_info.channels, old_info.codec, old_info.bitrate / 1000, old_info.sample_rate, old_info.bit_depth, old_info.channels ))); } Quality::Better => { let mut checked_context = context.clone(); checked_context["inspected_sha256"] = serde_json::json!(inspected_sha); upgrade( pool, storage_dir, source, (&candidate, &old_path), &new_info, &checked_context, ) .await? } } // A repeated YouTube import must still resolve to this existing media ID. if let Some(sha) = context["sha256"].as_str() { if let Err(error) = crate::import_playlists::link_media(pool, sha, candidate.media_id).await { tracing::warn!(%error, "could not link deduplicated import to destination playlists"); } if let Err(error) = sqlx::query( r#"INSERT INTO furumusic__youtube_import_media (item_id, media_file_id) SELECT DISTINCT item.id, $1 FROM furumusic__youtube_download_item item JOIN furumusic__pending_review review ON review.input_path = item.inbox_path OR left(review.input_path, length(item.inbox_path) + 1) = item.inbox_path || '/' WHERE substring(review.context_json from '"sha256"[[:space:]]*:[[:space:]]*"([0-9a-fA-F]{64})"') = $2 ON CONFLICT (item_id, media_file_id) DO NOTHING"#, ).bind(candidate.media_id).bind(sha).execute(pool).await { tracing::warn!(%error, "could not link merged upload to YouTube history"); } if let Err(error) = crate::youtube::sync_target_playlists_for_imported_media(pool, candidate.media_id).await { tracing::warn!(%error, "could not sync merged YouTube import to playlists"); } } Ok(true) } async fn upgrade( pool: &PgPool, storage_dir: &str, source: &Path, candidate_and_path: (&Candidate, &Path), info: &AudioInfo, context: &serde_json::Value, ) -> Result<()> { let (candidate, old_path) = candidate_and_path; let ext = source .extension() .and_then(|s| s.to_str()) .context("incoming audio has no extension")?; let base_stem = old_path .file_stem() .and_then(|s| s.to_str()) .unwrap_or("audio") .split(" [upgrade-") .next() .unwrap_or("audio"); let mut stem_end = base_stem.len().min(160); while !base_stem.is_char_boundary(stem_end) { stem_end -= 1 } let destination = old_path.with_file_name(format!( "{} [upgrade-{}].{}", &base_stem[..stem_end], uuid::Uuid::new_v4(), ext )); // Keep both original files until the database transaction commits. super::mover::publish_copy(source, &destination).await?; let mut provenance = context.clone(); provenance["original_filename"] = serde_json::json!(source.file_name().and_then(|s| s.to_str())); let result = update_media( pool, storage_dir, &destination, candidate, info, &provenance, ) .await; if let Err(error) = result { let _ = tokio::fs::remove_file(&destination).await; return Err(error); } // Other catalog media records may still refer to the old physical path. let mut old_aliases = vec![ candidate.file_path.clone(), old_path.to_string_lossy().into_owned(), ]; if let Some(relative) = crate::media_paths::media_file_path_for_storage(storage_dir, old_path) { old_aliases.push(relative); } let referenced: bool = sqlx::query_scalar( "SELECT EXISTS(SELECT 1 FROM furumusic__media_file WHERE file_path = ANY($1))", ) .bind(old_aliases) .fetch_one(pool) .await .unwrap_or(true); if !referenced && let Err(error) = tokio::fs::remove_file(old_path).await { tracing::warn!(?old_path, %error, "upgraded audio; could not remove superseded file"); } tokio::fs::remove_file(source).await?; tracing::info!( track_id = candidate.track_id, media_id = candidate.media_id, codec = info.codec, bitrate = info.bitrate, "existing track audio upgraded without creating a duplicate" ); Ok(()) } async fn update_media( pool: &PgPool, storage_dir: &str, destination: &Path, candidate: &Candidate, info: &AudioInfo, context: &serde_json::Value, ) -> Result<()> { let path = crate::media_paths::media_file_path_for_storage(storage_dir, destination) .context("upgraded file is outside storage root")?; let sha = super::mover::sha256_file(destination).await?; if context["inspected_sha256"].as_str() != Some(sha.as_str()) { return Err(review("incoming audio changed while staging the upgrade")); } let size = tokio::fs::metadata(destination).await?.len() as i64; let ext = destination .extension() .and_then(|s| s.to_str()) .unwrap_or("") .to_lowercase(); let mime = match ext.as_str() { "mp3" => "audio/mpeg", "flac" => "audio/flac", "ogg" | "opus" => "audio/ogg", "aac" | "m4a" => "audio/mp4", "wav" => "audio/wav", "aiff" | "aif" => "audio/aiff", _ => "application/octet-stream", }; let mut tx = pool.begin().await?; let updated = sqlx::query( r#"UPDATE furumusic__media_file SET file_path=$1, sha256_hash=$2, file_size_bytes=$3, mime_type=$4, audio_format=$5, audio_bitrate=$6, audio_sample_rate=$7, audio_bit_depth=$8, created_at=$9, original_filename=COALESCE($12, original_filename), uploaded_by_user_id=COALESCE($13, uploaded_by_user_id), uploader_name=COALESCE($14, uploader_name) WHERE id=$10 AND sha256_hash=$11"#, ) .bind(path) .bind(&sha) .bind(size) .bind(mime) .bind(ext) .bind((info.bitrate > 0).then_some((info.bitrate / 1000) as i32)) .bind(info.sample_rate as i32) .bind((info.bit_depth > 0).then_some(info.bit_depth as i32)) .bind(chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true)) .bind(candidate.media_id) .bind(&candidate.sha256_hash) .bind(context["original_filename"].as_str()) .bind(context["uploaded_by_user_id"].as_i64()) .bind(context["uploader_name"].as_str()) .execute(&mut *tx) .await?; ensure!( updated.rows_affected() == 1, "existing audio changed during the upgrade; retry required" ); sqlx::query( "UPDATE furumusic__track SET duration_seconds=$1, updated_at=$2 WHERE audio_file_id=$3", ) .bind(info.duration) .bind(chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true)) .bind(candidate.media_id) .execute(&mut *tx) .await?; sqlx::query("DELETE FROM furumusic__federation_content_id_cache WHERE media_file_id=$1") .bind(candidate.media_id) .execute(&mut *tx) .await?; sqlx::query("DELETE FROM furumusic__track_embedding WHERE track_id IN (SELECT id FROM furumusic__track WHERE audio_file_id=$1)") .bind(candidate.media_id).execute(&mut *tx).await?; tx.commit().await?; Ok(()) } #[cfg(test)] mod tests { use super::*; fn audio(codec: &str, bitrate: u64, rate: u32, depth: u32) -> AudioInfo { AudioInfo { codec: codec.into(), duration: 180.0, channels: 2, sample_rate: rate, bit_depth: depth, bitrate, } } #[test] fn quality_comparison_preserves_lossless_and_compares_codec_like_for_like() { let mp3 = audio("mp3", 128000, 44100, 0); let better_mp3 = audio("mp3", 320000, 44100, 0); let flac = audio("flac", 700000, 44100, 16); assert_eq!(compare_quality(&better_mp3, &mp3), Quality::Better); assert_eq!(compare_quality(&mp3, &better_mp3), Quality::Keep); assert_eq!(compare_quality(&flac, &better_mp3), Quality::Better); assert_eq!(compare_quality(&better_mp3, &flac), Quality::Keep); assert_eq!( compare_quality(&audio("alac", 1400000, 44100, 16), &flac), Quality::Keep ); assert_eq!( compare_quality(&audio("opus", 192000, 48000, 0), &mp3), Quality::Incomparable ); assert_eq!( compare_quality(&audio("flac", 1500000, 96000, 24), &flac), Quality::Better ); let mut mono = better_mp3; mono.channels = 1; assert_eq!(compare_quality(&mono, &mp3), Quality::Incomparable); assert_eq!( compare_quality(&audio("wavpack", 2000000, 44100, 24), &mp3), Quality::Incomparable ); } #[test] fn durations_reject_unknown_values_and_different_versions() { assert!(durations_match(180.0, 180.2)); assert!(!durations_match(180.0, 184.0)); assert!(!durations_match(10.0, 11.0)); assert!(!durations_match(0.0, 0.0)); assert!(!durations_match(f64::NAN, 180.0)); } #[test] fn fingerprints_allow_encoder_shift_but_reject_silence_and_altered_endings() { let fp: Vec = (0..512_u32) .map(|n| n.wrapping_mul(2654435761).rotate_left(7)) .collect(); let mut shifted = vec![0xdeadbeef; 2]; shifted.extend(&fp); assert!(fingerprints_match(&fp, &shifted)); let slight_noise: Vec<_> = fp.iter().map(|n| n ^ 1).collect(); assert!(fingerprints_match(&fp, &slight_noise)); let other: Vec<_> = fp.iter().map(|n| !n).collect(); assert!(!fingerprints_match(&fp, &other)); assert!(!fingerprints_match(&[0; 512], &[0; 512])); assert!(!fingerprints_match(&fp[..32], &fp[..32])); let mut altered = fp.clone(); for n in &mut altered[448..] { *n = !*n } assert!(!fingerprints_match(&fp, &altered)); } async fn encode_test_audio(destination: &Path, input: Option<&Path>, bitrate: &str) { let mut command = tokio::process::Command::new("ffmpeg"); command.args(["-hide_banner", "-loglevel", "error", "-y"]); if let Some(input) = input { command.arg("-i").arg(input); } else { command.args([ "-f", "lavfi", "-i", "aevalsrc=0.3*sin(2*PI*(220+10*t)*t)+0.2*sin(2*PI*(440+17*t)*t):s=44100:d=30", ]); } command.args(["-ac", "2", "-b:a", bitrate]).arg(destination); let output = command.output().await.unwrap(); assert!( output.status.success(), "{}", String::from_utf8_lossy(&output.stderr) ); } #[tokio::test] async fn real_transcodes_match_and_keep_distinct_bitrates() { let root = std::env::temp_dir().join(format!("furumi-fingerprint-{}", uuid::Uuid::new_v4())); tokio::fs::create_dir_all(&root).await.unwrap(); let source = root.join("source.wav"); let low = root.join("low.mp3"); let high = root.join("high.mp3"); encode_test_audio(&source, None, "128k").await; encode_test_audio(&low, Some(&source), "128k").await; encode_test_audio(&high, Some(&source), "320k").await; let low_info = probe(&low).await.unwrap(); let high_info = probe(&high).await.unwrap(); assert!(durations_match(low_info.duration, high_info.duration)); assert_eq!(compare_quality(&high_info, &low_info), Quality::Better); let low_fp = fingerprint(&low, low_info.duration).await.unwrap(); let high_fp = fingerprint(&high, high_info.duration).await.unwrap(); assert!( fingerprints_match(&low_fp.fingerprint, &high_fp.fingerprint), "transcodes did not match" ); let broken = root.join("truncated.mp3"); let bytes = tokio::fs::read(&high).await.unwrap(); tokio::fs::write(&broken, &bytes[..bytes.len() / 2]) .await .unwrap(); let broken_info = probe(&broken).await.unwrap(); if let Ok(broken_fp) = fingerprint(&broken, broken_info.duration).await { assert!((broken_fp.duration - high_fp.duration).abs() > 0.25); assert!(!fingerprints_match( &broken_fp.fingerprint, &high_fp.fingerprint )); } tokio::fs::remove_dir_all(root).await.unwrap(); } #[tokio::test] #[ignore = "requires an isolated PostgreSQL socket in FURUMI_DEDUP_TEST_SOCKET"] async fn upgrades_preserve_ids_and_database_errors_preserve_both_sources() { let socket = std::env::var("FURUMI_DEDUP_TEST_SOCKET").expect("isolated test PostgreSQL socket"); let schema = format!("dedup_test_{}", uuid::Uuid::new_v4().simple()); let search_path = format!("SET search_path TO {schema}"); let pool = sqlx::postgres::PgPoolOptions::new() .max_connections(5) .after_connect(move |conn, _| { let statement = search_path.clone(); Box::pin(async move { sqlx::query(&statement).execute(conn).await?; Ok(()) }) }) .connect_with( sqlx::postgres::PgConnectOptions::new() .host(&socket) .username("furumi_dedup_test") .database("postgres"), ) .await .unwrap(); sqlx::query(&format!("CREATE SCHEMA {schema}")) .execute(&pool) .await .unwrap(); sqlx::raw_sql(r#" CREATE TABLE furumusic__artist (id bigint PRIMARY KEY, name text, name_sort text); CREATE TABLE furumusic__release (id bigint PRIMARY KEY, title_sort text, year int); CREATE TABLE furumusic__media_file (id bigint PRIMARY KEY, file_path text, file_type text, sha256_hash text, file_size_bytes bigint, mime_type text, audio_format text, audio_bitrate int, audio_sample_rate int, audio_bit_depth int, created_at text, original_filename text, uploaded_by_user_id bigint, uploader_name text); CREATE TABLE furumusic__track (id bigint PRIMARY KEY, release_id bigint, audio_file_id bigint, title_sort text, duration_seconds double precision, updated_at text); CREATE TABLE furumusic__track_artist (track_id bigint, artist_id bigint, role text); CREATE TABLE furumusic__federation_content_id_cache (media_file_id bigint); CREATE TABLE furumusic__track_embedding (track_id bigint); CREATE TABLE test_likes (track_id bigint REFERENCES furumusic__track(id)); CREATE TABLE test_playlist_tracks (track_id bigint REFERENCES furumusic__track(id)); CREATE TABLE furumusic__local_upload (target_playlist_id bigint, id text PRIMARY KEY, user_id bigint, filename text, size_bytes bigint, status text, inbox_path text, error text, created_at text, updated_at text, completed_at text); INSERT INTO furumusic__artist VALUES (1, 'Artist', 'artist'); INSERT INTO furumusic__release VALUES (2, 'album', 2002); INSERT INTO furumusic__track VALUES (42, 2, 7, 'song', 30, 'before'); INSERT INTO furumusic__track_artist VALUES (42, 1, 'main'); INSERT INTO test_likes VALUES (42); INSERT INTO test_playlist_tracks VALUES (42); INSERT INTO furumusic__federation_content_id_cache VALUES (7); INSERT INTO furumusic__track_embedding VALUES (42); "#).execute(&pool).await.unwrap(); let root = std::env::temp_dir().join(format!("furumi-dedup-db-{}", uuid::Uuid::new_v4())); let inbox = root.join("incoming"); let storage = root.join("storage"); tokio::fs::create_dir_all(&inbox).await.unwrap(); tokio::fs::create_dir_all(&storage).await.unwrap(); let master = root.join("master.wav"); let low = storage.join("song.mp3"); let high = inbox.join("song.mp3"); encode_test_audio(&master, None, "128k").await; encode_test_audio(&low, Some(&master), "128k").await; encode_test_audio(&high, Some(&master), "320k").await; let low_hash = super::super::mover::sha256_file(&low).await.unwrap(); let high_hash = super::super::mover::sha256_file(&high).await.unwrap(); sqlx::query("INSERT INTO furumusic__media_file (id,file_path,file_type,sha256_hash) VALUES (7,'song.mp3','audio',$1)") .bind(&low_hash).execute(&pool).await.unwrap(); let fields = NormalizedFields { artist: Some("Artist".into()), album: Some("Album".into()), title: Some("Song".into()), year: Some(2002), ..Default::default() }; let context = serde_json::json!({}); assert!( merge_existing(&pool, storage.to_str().unwrap(), &high, &fields, &context) .await .unwrap() ); let (track_id, media_id, path, sha): (i64, i64, String, String) = sqlx::query_as( "SELECT t.id, m.id, m.file_path, m.sha256_hash FROM furumusic__track t JOIN furumusic__media_file m ON m.id=t.audio_file_id" ).fetch_one(&pool).await.unwrap(); assert_eq!((track_id, media_id), (42, 7)); assert_eq!(sha, high_hash); assert!(!high.exists()); assert!(!low.exists()); assert!(storage.join(&path).exists()); assert_eq!( sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM test_likes WHERE track_id=42") .fetch_one(&pool) .await .unwrap(), 1 ); assert_eq!( sqlx::query_scalar::<_, i64>( "SELECT COUNT(*) FROM test_playlist_tracks WHERE track_id=42" ) .fetch_one(&pool) .await .unwrap(), 1 ); assert_eq!( sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM furumusic__track_embedding") .fetch_one(&pool) .await .unwrap(), 0 ); // Lower-quality reuploads disappear without changing the catalog file. encode_test_audio(&high, Some(&master), "128k").await; assert!( merge_existing(&pool, storage.to_str().unwrap(), &high, &fields, &context) .await .unwrap() ); assert!(!high.exists()); let stored_hash: String = sqlx::query_scalar("SELECT sha256_hash FROM furumusic__media_file WHERE id=7") .fetch_one(&pool) .await .unwrap(); assert_eq!(stored_hash, high_hash); // A codec upgrade updates MIME/path while preserving the same IDs. let lossless = inbox.join("song.flac"); encode_test_audio(&lossless, Some(&master), "128k").await; assert!( merge_existing( &pool, storage.to_str().unwrap(), &lossless, &fields, &context ) .await .unwrap() ); let mime: String = sqlx::query_scalar("SELECT mime_type FROM furumusic__media_file WHERE id=7") .fetch_one(&pool) .await .unwrap(); assert_eq!(mime, "audio/flac"); assert!(!lossless.exists()); assert!(!storage.join(&path).exists()); assert_eq!( sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM furumusic__track") .fetch_one(&pool) .await .unwrap(), 1 ); // Guest credit changes must not silently create another track. A // primary/guest ordering change with identical credits is a duplicate. encode_test_audio(&high, Some(&master), "128k").await; let with_guest = NormalizedFields { artist: Some("Artist".into()), featured_artists: vec!["Guest".into()], album: Some("Album".into()), title: Some("Song".into()), year: Some(2002), ..Default::default() }; let error = merge_existing( &pool, storage.to_str().unwrap(), &high, &with_guest, &context, ) .await .unwrap_err(); assert!(error.downcast_ref::().is_some()); assert!(high.exists()); sqlx::raw_sql("INSERT INTO furumusic__artist VALUES (9,'Guest','guest'); INSERT INTO furumusic__track_artist VALUES (42,9,'featuring');") .execute(&pool).await.unwrap(); let flipped = NormalizedFields { artist: Some("Guest".into()), featured_artists: vec!["Artist".into()], album: Some("Album".into()), title: Some("Song".into()), year: Some(2002), ..Default::default() }; assert!( merge_existing(&pool, storage.to_str().unwrap(), &high, &flipped, &context) .await .unwrap() ); assert!(!high.exists()); sqlx::query("DELETE FROM furumusic__track_artist WHERE artist_id=9") .execute(&pool) .await .unwrap(); // Reset to the low-quality file, then force a late SQL failure after the // media UPDATE. Neither the old file nor the inbox may be deleted. encode_test_audio(&low, Some(&master), "128k").await; encode_test_audio(&high, Some(&master), "320k").await; sqlx::query( "UPDATE furumusic__media_file SET file_path='song.mp3', sha256_hash=$1 WHERE id=7", ) .bind(&low_hash) .execute(&pool) .await .unwrap(); let receipt_id = "87654321-e89b-12d3-a456-426614174000"; crate::local_uploads::create( &pool, receipt_id, 6, "song.mp3", 3, "user_uploads/6/batch/song.mp3", None, ) .await .unwrap(); crate::local_uploads::create( &pool, receipt_id, 6, "song.mp3", 3, "user_uploads/6/batch/song.mp3", None, ) .await .unwrap(); crate::local_uploads::mark_queued(&pool, receipt_id, 6) .await .unwrap(); assert!( crate::local_uploads::successful_receipt( &pool, 6, receipt_id, "user_uploads/6/batch/song.mp3" ) .await .unwrap() .is_some() ); assert!( crate::local_uploads::successful_receipt( &pool, 7, receipt_id, "user_uploads/6/batch/song.mp3" ) .await .unwrap() .is_none() ); assert!( crate::local_uploads::create( &pool, receipt_id, 7, "song.mp3", 3, "user_uploads/7/song.mp3", None, ) .await .is_err() ); sqlx::query("DROP TABLE furumusic__track_embedding") .execute(&pool) .await .unwrap(); assert!( merge_existing(&pool, storage.to_str().unwrap(), &high, &fields, &context) .await .is_err() ); let hash: String = sqlx::query_scalar("SELECT sha256_hash FROM furumusic__media_file WHERE id=7") .fetch_one(&pool) .await .unwrap(); assert_eq!(hash, low_hash); assert!(low.exists()); assert!(high.exists()); sqlx::query(&format!("DROP SCHEMA {schema} CASCADE")) .execute(&pool) .await .unwrap(); pool.close().await; tokio::fs::remove_dir_all(root).await.unwrap(); } }