From 1e1453e465e05582200df4e9c372accb8c4fac20 Mon Sep 17 00:00:00 2001 From: Ultradesu Date: Wed, 3 Jun 2026 03:39:16 +0300 Subject: [PATCH] Update metadata jobs and player library --- Cargo.lock | 2 +- src/admin/mod.rs | 26 ++ src/admin/v2.rs | 62 ++++ src/admin/views.rs | 1 + src/i18n/phrases.rs | 2 + src/jobs/artwork_backfill.rs | 653 ++++++++++++++++++++++++++++++---- src/jobs/metadata_backfill.rs | 359 +++++++++++++++++-- src/jobs/mod.rs | 1 + src/jobs/musicbrainz.rs | 587 ++++++++++++++++++++++++++++++ src/music/mod.rs | 46 +++ src/player/dto.rs | 1 + src/player/mod.rs | 111 +++--- templates/admin/v2.html | 48 ++- templates/player/scripts.html | 23 +- templates/player/shell.html | 20 +- templates/player/styles.html | 20 ++ 16 files changed, 1803 insertions(+), 159 deletions(-) create mode 100644 src/jobs/musicbrainz.rs diff --git a/Cargo.lock b/Cargo.lock index d4fb93e..dbd6848 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1418,7 +1418,7 @@ checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" [[package]] name = "furumusic" -version = "0.2.14" +version = "0.2.15" dependencies = [ "anyhow", "async-trait", diff --git a/src/admin/mod.rs b/src/admin/mod.rs index 3d7b785..73d9b8f 100644 --- a/src/admin/mod.rs +++ b/src/admin/mod.rs @@ -357,6 +357,32 @@ impl App for AdminApp { }), "admin_v2_metadata_backfill_run_options", ), + Route::with_handler_and_name( + "/v2/api/jobs/artwork_backfill/run-options", + cot::router::method::post({ + let pool = Arc::clone(&pool); + let pool_config = Arc::clone(&pool_config); + move |session: Session, + db: Database, + json: Json| { + let pool = Arc::clone(&pool); + let pool_config = Arc::clone(&pool_config); + async move { + let pg_pool = pool + .get_or_init(|| async { + sqlx::postgres::PgPoolOptions::new() + .max_connections(5) + .connect(&pool_config.database_url) + .await + .expect("admin pool") + }) + .await; + v2::run_artwork_backfill(session, db, pg_pool, json).await + } + } + }), + "admin_v2_artwork_backfill_run_options", + ), Route::with_handler_and_name( "/v2/api/jobs/{name}/run", cot::router::method::post({ diff --git a/src/admin/v2.rs b/src/admin/v2.rs index 838111a..ea50da4 100644 --- a/src/admin/v2.rs +++ b/src/admin/v2.rs @@ -83,10 +83,18 @@ pub struct MetadataBackfillRunRequest { local_genres: bool, #[serde(default = "default_true")] lastfm_tags: bool, + #[serde(default = "default_true")] + musicbrainz_tags: bool, #[serde(default)] overwrite: bool, } +#[derive(Debug, Deserialize)] +pub struct ArtworkBackfillRunRequest { + #[serde(default)] + overwrite_existing: bool, +} + #[derive(Debug, Deserialize)] pub(super) struct UpdateLibraryItemRequest { kind: String, @@ -296,6 +304,7 @@ struct MetadataTagDto { struct ReviewPageDto { items: Vec, total: i64, + total_all: i64, limit: i64, offset: i64, status: Option, @@ -1076,6 +1085,7 @@ pub async fn run_metadata_backfill( duration_seconds: body.duration_seconds, local_genres: body.local_genres, lastfm_tags: body.lastfm_tags, + musicbrainz_tags: body.musicbrainz_tags, overwrite: body.overwrite, }; if !options.any_field() { @@ -1123,6 +1133,56 @@ pub async fn run_metadata_backfill( Json(JobRunStartedDto { ok: true, run_id }).into_response() } +pub async fn run_artwork_backfill( + session: Session, + db: Database, + pool: &PgPool, + Json(body): Json, +) -> cot::Result { + if let Err(response) = require_admin_json(&session, &db).await { + return Ok(response); + } + + let options = crate::jobs::artwork_backfill::ArtworkBackfillOptions { + overwrite_existing: body.overwrite_existing, + }; + let mut run = JobRun::create_running(&db, "artwork_backfill", "manual") + .await + .map_err(|e| cot::Error::internal(format!("failed to create job run: {e}")))?; + let run_id = run.id_val(); + let (live_config, _) = AppConfig::load_with_db(&db).await; + let db_for_task = db.clone(); + let pool_for_task = pool.clone(); + + tokio::spawn(async move { + let start = std::time::Instant::now(); + let ctx = scheduler::JobContext { + config: std::sync::Arc::new(live_config), + db: db_for_task.clone(), + pool: pool_for_task.clone(), + run_id, + registry: std::sync::Arc::new(JobRegistry::new()), + }; + let mut log = scheduler::JobLog::with_live_flush(pool_for_task.clone(), run_id); + let result = crate::jobs::artwork_backfill::run_with_options(&ctx, &mut log, options).await; + let duration_ms = start.elapsed().as_millis() as i64; + match result { + Ok(()) => { + let _ = run + .set_completed(&db_for_task, duration_ms, &log.output()) + .await; + } + Err(err) => { + let _ = run + .set_failed(&db_for_task, duration_ms, &log.output(), &err.to_string()) + .await; + } + } + }); + + Json(JobRunStartedDto { ok: true, run_id }).into_response() +} + pub async fn toggle_job( session: Session, db: Database, @@ -2109,6 +2169,7 @@ async fn load_review_page(pool: &PgPool, query: ReviewsQuery) -> anyhow::Result< let search_pattern = search.as_ref().map(|s| format!("%{s}%")); let total = count_reviews(pool, status.clone(), search_pattern.clone()).await?; + let total_all = count_reviews(pool, None, search_pattern.clone()).await?; let status_counts = load_review_status_counts(pool, None, search_pattern.clone()).await?; let mut qb = QueryBuilder::::new( @@ -2134,6 +2195,7 @@ async fn load_review_page(pool: &PgPool, query: ReviewsQuery) -> anyhow::Result< Ok(ReviewPageDto { items, total, + total_all, limit, offset, status, diff --git a/src/admin/views.rs b/src/admin/views.rs index 82c1fc0..d03f548 100644 --- a/src/admin/views.rs +++ b/src/admin/views.rs @@ -1472,6 +1472,7 @@ pub async fn metadata_backfill_run( duration_seconds: data.duration_seconds.is_some(), local_genres: data.local_genres.is_some(), lastfm_tags: data.lastfm_tags.is_some(), + musicbrainz_tags: true, overwrite: data.mode.as_deref() == Some("overwrite"), }; diff --git a/src/i18n/phrases.rs b/src/i18n/phrases.rs index c942a98..105d8d1 100644 --- a/src/i18n/phrases.rs +++ b/src/i18n/phrases.rs @@ -272,6 +272,7 @@ translations! { // Player UI player_library: "Library" , "Библиотека"; player_artists: "Artists" , "Артисты"; + player_global_library: "Global" , "Global"; player_release: "Release" , "Релиз"; player_releases: "Releases" , "Релизы"; player_tracks: "Tracks" , "Треки"; @@ -479,6 +480,7 @@ translations! { player_seen: "seen" , "видели"; player_eta: "eta" , "осталось"; player_loading_history: "Loading history..." , "Загрузка истории..."; + player_loading_more: "Loading more..." , "Загружаю ещё..."; player_failed_load_history: "Failed to load history" , "Не удалось загрузить историю"; player_total_plays: "total plays" , "прослушиваний всего"; player_play_history: "Play history" , "История прослушиваний"; diff --git a/src/jobs/artwork_backfill.rs b/src/jobs/artwork_backfill.rs index 18b152d..459763d 100644 --- a/src/jobs/artwork_backfill.rs +++ b/src/jobs/artwork_backfill.rs @@ -15,12 +15,19 @@ pub struct ArtworkBackfillJob; const LASTFM_REQUEST_DELAY: std::time::Duration = std::time::Duration::from_millis(1200); const MAX_LASTFM_RELEASE_LOOKUPS: i64 = 200; const MAX_LASTFM_ARTIST_LOOKUPS: i64 = 200; +const MAX_CAA_RELEASE_LOOKUPS: i64 = 200; + +#[derive(Debug, Clone, Copy)] +pub struct ArtworkBackfillOptions { + pub overwrite_existing: bool, +} #[derive(Debug, sqlx::FromRow)] struct ReleaseCandidate { id: i64, title: String, artist_name: Option, + track_title: Option, } #[derive(Debug, sqlx::FromRow)] @@ -36,6 +43,13 @@ struct ArtworkRefCandidate { file_path: Option, } +#[derive(Debug, sqlx::FromRow)] +struct ArtistImageRepairCandidate { + id: i64, + name: String, + media_file_id: i64, +} + #[derive(Debug, Deserialize)] struct LastfmAlbumResponse { album: Option, @@ -85,7 +99,10 @@ struct ArtworkStats { broken_release_refs_cleared: u64, broken_track_refs_cleared: u64, broken_artist_refs_cleared: u64, + remote_artist_album_fallback_refs_cleared: u64, release_local_assigned: u64, + release_caa_assigned: u64, + release_caa_not_found: u64, release_lastfm_assigned: u64, release_lastfm_not_found: u64, release_skipped_no_audio: u64, @@ -114,66 +131,107 @@ impl Job for ArtworkBackfillJob { } async fn run(&self, ctx: &JobContext, log: &mut JobLog) -> anyhow::Result<()> { - let storage_dir = ctx.config.agent_storage_dir.trim(); - if storage_dir.is_empty() { - log.warn("agent_storage_dir is not configured, skipping artwork backfill"); - return Ok(()); - } - - let client = Client::builder() - .user_agent(format!( - "furumusic-artwork-backfill/{}", - env!("CARGO_PKG_VERSION") - )) - .timeout(std::time::Duration::from_secs(20)) - .build()?; - let mut stats = ArtworkStats::default(); - - let normalized_paths = - crate::media_paths::normalize_media_file_paths(&ctx.pool, storage_dir).await?; - if normalized_paths > 0 { - log.info(&format!( - "Media path normalization pass: rewrote {normalized_paths} media file path(s) to relative storage paths" - )); - } else { - log.info("Media path normalization pass: all media file paths are already relative"); - } - - repair_missing_artwork_refs(ctx, log, storage_dir, &mut stats).await?; - backfill_release_local(ctx, log, storage_dir, &mut stats).await?; - - let api_key = ctx.config.lastfm_api_key.trim(); - if api_key.is_empty() { - log.warn("lastfm_api_key is not configured; skipping Last.fm artwork fallback"); - } else { - backfill_release_lastfm(ctx, log, storage_dir, api_key, &client, &mut stats).await?; - backfill_artist_lastfm(ctx, log, storage_dir, api_key, &client, &mut stats).await?; - } - - backfill_artist_album_fallbacks(ctx, log, &mut stats).await?; - repair_cover_variants(ctx, log, storage_dir, &mut stats).await?; - - log.info(&format!( - "Artwork backfill complete: broken_release_refs_cleared={}, broken_track_refs_cleared={}, broken_artist_refs_cleared={}, release_local_assigned={}, release_lastfm_assigned={}, release_lastfm_not_found={}, release_skipped_no_audio={}, artist_lastfm_assigned={}, artist_lastfm_not_found={}, artist_album_fallback_assigned={}, variants_created={}, variants_unchanged={}, variants_missing_original={}, failed={}", - stats.broken_release_refs_cleared, - stats.broken_track_refs_cleared, - stats.broken_artist_refs_cleared, - stats.release_local_assigned, - stats.release_lastfm_assigned, - stats.release_lastfm_not_found, - stats.release_skipped_no_audio, - stats.artist_lastfm_assigned, - stats.artist_lastfm_not_found, - stats.artist_album_fallback_assigned, - stats.variants_created, - stats.variants_unchanged, - stats.variants_missing_original, - stats.failed - )); - Ok(()) + run_with_options( + ctx, + log, + ArtworkBackfillOptions { + overwrite_existing: false, + }, + ) + .await } } +pub async fn run_with_options( + ctx: &JobContext, + log: &mut JobLog, + options: ArtworkBackfillOptions, +) -> anyhow::Result<()> { + let storage_dir = ctx.config.agent_storage_dir.trim(); + if storage_dir.is_empty() { + log.warn("agent_storage_dir is not configured, skipping artwork backfill"); + return Ok(()); + } + + log.info(&format!( + "Artwork backfill options: mode={}", + if options.overwrite_existing { + "overwrite_existing" + } else { + "missing_only" + } + )); + + let client = Client::builder() + .user_agent(format!( + "furumusic-artwork-backfill/{}", + env!("CARGO_PKG_VERSION") + )) + .timeout(std::time::Duration::from_secs(20)) + .build()?; + let musicbrainz_client = + crate::jobs::musicbrainz::MusicBrainzClient::new("furumusic-artwork-backfill")?; + let mut stats = ArtworkStats::default(); + + let normalized_paths = + crate::media_paths::normalize_media_file_paths(&ctx.pool, storage_dir).await?; + if normalized_paths > 0 { + log.info(&format!( + "Media path normalization pass: rewrote {normalized_paths} media file path(s) to relative storage paths" + )); + } else { + log.info("Media path normalization pass: all media file paths are already relative"); + } + + repair_missing_artwork_refs(ctx, log, storage_dir, &mut stats).await?; + repair_remote_artist_album_fallback_refs(ctx, log, &mut stats).await?; + backfill_release_local(ctx, log, storage_dir, options, &mut stats).await?; + backfill_release_coverartarchive( + ctx, + log, + storage_dir, + &musicbrainz_client, + options, + &mut stats, + ) + .await?; + + let api_key = ctx.config.lastfm_api_key.trim(); + if api_key.is_empty() { + log.warn("lastfm_api_key is not configured; skipping Last.fm artwork fallback"); + } else { + backfill_release_lastfm(ctx, log, storage_dir, api_key, &client, options, &mut stats) + .await?; + backfill_artist_lastfm(ctx, log, storage_dir, api_key, &client, options, &mut stats) + .await?; + } + + backfill_artist_album_fallbacks(ctx, log, options, &mut stats).await?; + repair_cover_variants(ctx, log, storage_dir, &mut stats).await?; + + log.info(&format!( + "Artwork backfill complete: broken_release_refs_cleared={}, broken_track_refs_cleared={}, broken_artist_refs_cleared={}, remote_artist_album_fallback_refs_cleared={}, release_local_assigned={}, release_caa_assigned={}, release_caa_not_found={}, release_lastfm_assigned={}, release_lastfm_not_found={}, release_skipped_no_audio={}, artist_lastfm_assigned={}, artist_lastfm_not_found={}, artist_album_fallback_assigned={}, variants_created={}, variants_unchanged={}, variants_missing_original={}, failed={}", + stats.broken_release_refs_cleared, + stats.broken_track_refs_cleared, + stats.broken_artist_refs_cleared, + stats.remote_artist_album_fallback_refs_cleared, + stats.release_local_assigned, + stats.release_caa_assigned, + stats.release_caa_not_found, + stats.release_lastfm_assigned, + stats.release_lastfm_not_found, + stats.release_skipped_no_audio, + stats.artist_lastfm_assigned, + stats.artist_lastfm_not_found, + stats.artist_album_fallback_assigned, + stats.variants_created, + stats.variants_unchanged, + stats.variants_missing_original, + stats.failed + )); + Ok(()) +} + async fn repair_missing_artwork_refs( ctx: &JobContext, log: &mut JobLog, @@ -186,6 +244,71 @@ async fn repair_missing_artwork_refs( Ok(()) } +async fn repair_remote_artist_album_fallback_refs( + ctx: &JobContext, + log: &mut JobLog, + stats: &mut ArtworkStats, +) -> anyhow::Result<()> { + let rows = sqlx::query_as::<_, ArtistImageRepairCandidate>( + r#"SELECT DISTINCT a.id, + a.name::text AS name, + a.image_file_id AS media_file_id + FROM furumusic__artist a + JOIN furumusic__media_file mf ON mf.id = a.image_file_id + WHERE a.image_file_id IS NOT NULL + AND a.is_hidden = false + AND mf.file_path NOT LIKE '%/__artist_image__/%' + AND EXISTS ( + SELECT 1 + FROM furumusic__release r + JOIN furumusic__artwork_lookup_state s + ON s.entity_kind = 'release' + AND s.entity_id = r.id + AND s.source IN ('lastfm', 'coverartarchive') + AND s.status = 'found' + WHERE r.cover_file_id = a.image_file_id + AND r.is_hidden = false + ) + ORDER BY a.id"#, + ) + .fetch_all(&ctx.pool) + .await?; + + if rows.is_empty() { + return Ok(()); + } + + log.info(&format!( + "Artist fallback repair pass: clearing {} remote release cover fallback(s)", + rows.len() + )); + + for row in rows { + let result = sqlx::query( + r#"UPDATE furumusic__artist + SET image_file_id = NULL, + updated_at = $3 + WHERE id = $1 + AND image_file_id = $2"#, + ) + .bind(row.id) + .bind(row.media_file_id) + .bind(now_iso()) + .execute(&ctx.pool) + .await?; + + if result.rows_affected() > 0 { + stats.remote_artist_album_fallback_refs_cleared += 1; + log.warn(&format!( + "Artist {} \"{}\": cleared remote release cover fallback (media_file_id={})", + row.id, row.name, row.media_file_id + )); + } + } + + Ok(()) +} + async fn repair_missing_release_cover_refs( ctx: &JobContext, log: &mut JobLog, @@ -360,6 +483,7 @@ async fn backfill_release_local( ctx: &JobContext, log: &mut JobLog, storage_dir: &str, + options: ArtworkBackfillOptions, stats: &mut ArtworkStats, ) -> anyhow::Result<()> { let releases = sqlx::query_as::<_, ReleaseCandidate>( @@ -372,17 +496,26 @@ async fn backfill_release_local( WHERE ra.release_id = r.id ORDER BY ra.position LIMIT 1 - ) AS artist_name + ) AS artist_name, + ( + SELECT t.title::text + FROM furumusic__track t + WHERE t.release_id = r.id + AND t.is_hidden = false + ORDER BY t.disc_number NULLS LAST, t.track_number NULLS LAST, t.id + LIMIT 1 + ) AS track_title FROM furumusic__release r - WHERE r.cover_file_id IS NULL + WHERE ($1 OR r.cover_file_id IS NULL) AND r.is_hidden = false ORDER BY r.id"#, ) + .bind(options.overwrite_existing) .fetch_all(&ctx.pool) .await?; if releases.is_empty() { - log.info("Release local artwork pass: all visible releases already have covers"); + log.info("Release local artwork pass: no eligible releases need local cover lookup"); return Ok(()); } log.info(&format!( @@ -451,7 +584,7 @@ async fn backfill_release_local( .await { Ok(cover_file_id) => { - cover_art::assign_cover_to_release(&ctx.pool, release.id, cover_file_id).await?; + assign_cover_to_release(&ctx.pool, release.id, cover_file_id, options).await?; stats.release_local_assigned += 1; log.info(&format!( "Release {} \"{}\": assigned local cover from {source_desc}", @@ -471,12 +604,12 @@ async fn backfill_release_local( Ok(()) } -async fn backfill_release_lastfm( +async fn backfill_release_coverartarchive( ctx: &JobContext, log: &mut JobLog, storage_dir: &str, - api_key: &str, - client: &Client, + client: &crate::jobs::musicbrainz::MusicBrainzClient, + options: ArtworkBackfillOptions, stats: &mut ArtworkStats, ) -> anyhow::Result<()> { let failed_cutoff = cutoff_iso(1); @@ -502,25 +635,285 @@ async fn backfill_release_lastfm( ORDER BY t.disc_number NULLS LAST, t.track_number NULLS LAST, ta.position LIMIT 1 ) - ) AS artist_name + ) AS artist_name, + ( + SELECT t.title::text + FROM furumusic__track t + WHERE t.release_id = r.id + AND t.is_hidden = false + ORDER BY t.disc_number NULLS LAST, t.track_number NULLS LAST, t.id + LIMIT 1 + ) AS track_title FROM furumusic__release r LEFT JOIN furumusic__artwork_lookup_state s ON s.entity_kind = 'release' AND s.entity_id = r.id - AND s.source = 'lastfm' - WHERE r.cover_file_id IS NULL + AND s.source = 'coverartarchive' + WHERE ($3 OR r.cover_file_id IS NULL) AND r.is_hidden = false AND ( - s.entity_id IS NULL + $3 + OR s.entity_id IS NULL OR s.status = 'failed' AND s.last_attempt_at < $1 OR s.status = 'not_found' AND (s.attempt_count < 3 OR s.last_attempt_at < $2) OR s.status = 'found' AND s.last_attempt_at < $1 ) ORDER BY s.last_attempt_at NULLS FIRST, r.id - LIMIT $3"#, + LIMIT $4"#, ) .bind(&failed_cutoff) .bind(¬_found_cutoff) + .bind(options.overwrite_existing) + .bind(MAX_CAA_RELEASE_LOOKUPS) + .fetch_all(&ctx.pool) + .await?; + + if releases.is_empty() { + log.info("Release Cover Art Archive pass: no eligible releases need lookup"); + return Ok(()); + } + log.info(&format!( + "Release Cover Art Archive pass: looking up {} release(s)", + releases.len() + )); + + for (index, release) in releases.iter().enumerate() { + let Some(artist_name) = release + .artist_name + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()) + else { + stats.release_caa_not_found += 1; + record_coverartarchive_lookup_state( + &ctx.pool, + "release", + release.id, + "not_found", + Some("release has no primary artist for MusicBrainz lookup"), + None, + ) + .await?; + continue; + }; + + log.info(&format!( + "Release Cover Art Archive {}/{}: release {} \"{}\" by \"{}\"", + index + 1, + releases.len(), + release.id, + release.title, + artist_name + )); + + let release_mbid = crate::jobs::musicbrainz::load_or_search_release_mbid( + &ctx.pool, + client, + release.id, + artist_name, + &release.title, + release.track_title.as_deref(), + ) + .await; + let (release_mbid, release_group_mbid) = match release_mbid { + Ok(value) => value, + Err(err) => { + stats.failed += 1; + record_coverartarchive_lookup_state( + &ctx.pool, + "release", + release.id, + "failed", + Some(&err.to_string()), + None, + ) + .await?; + log.warn(&format!( + "Release {} \"{}\": MusicBrainz lookup for cover art failed: {err}", + release.id, release.title + )); + continue; + } + }; + + if release_mbid.is_none() && release_group_mbid.is_none() { + stats.release_caa_not_found += 1; + record_coverartarchive_lookup_state( + &ctx.pool, + "release", + release.id, + "not_found", + Some("MusicBrainz release match not found"), + None, + ) + .await?; + continue; + } + + match client + .fetch_cover_art_front_url(release_mbid.as_deref(), release_group_mbid.as_deref()) + .await + { + Ok(Some(image_url)) => { + match download_remote_cover(client.http_client(), &image_url).await { + Ok(cover) => match cover_art::save_cover_to_storage( + &ctx.db, + &ctx.pool, + storage_dir, + artist_name, + &release.title, + &cover, + ) + .await + { + Ok(cover_file_id) => { + assign_cover_to_release(&ctx.pool, release.id, cover_file_id, options) + .await?; + record_coverartarchive_lookup_state( + &ctx.pool, + "release", + release.id, + "found", + None, + Some(&image_url), + ) + .await?; + stats.release_caa_assigned += 1; + log.info(&format!( + "Release {} \"{}\": assigned Cover Art Archive cover", + release.id, release.title + )); + } + Err(err) => { + stats.failed += 1; + record_coverartarchive_lookup_state( + &ctx.pool, + "release", + release.id, + "failed", + Some(&err.to_string()), + Some(&image_url), + ) + .await?; + log.warn(&format!( + "Release {} \"{}\": failed to save Cover Art Archive cover: {err}", + release.id, release.title + )); + } + }, + Err(err) => { + stats.failed += 1; + record_coverartarchive_lookup_state( + &ctx.pool, + "release", + release.id, + "failed", + Some(&err.to_string()), + Some(&image_url), + ) + .await?; + log.warn(&format!( + "Release {} \"{}\": failed to download Cover Art Archive cover: {err}", + release.id, release.title + )); + } + } + } + Ok(None) => { + stats.release_caa_not_found += 1; + record_coverartarchive_lookup_state( + &ctx.pool, + "release", + release.id, + "not_found", + None, + None, + ) + .await?; + } + Err(err) => { + stats.failed += 1; + record_coverartarchive_lookup_state( + &ctx.pool, + "release", + release.id, + "failed", + Some(&err.to_string()), + None, + ) + .await?; + log.warn(&format!( + "Release {} \"{}\": Cover Art Archive lookup failed: {err}", + release.id, release.title + )); + } + } + } + + Ok(()) +} + +async fn backfill_release_lastfm( + ctx: &JobContext, + log: &mut JobLog, + storage_dir: &str, + api_key: &str, + client: &Client, + options: ArtworkBackfillOptions, + stats: &mut ArtworkStats, +) -> anyhow::Result<()> { + let failed_cutoff = cutoff_iso(1); + let not_found_cutoff = cutoff_iso(30); + let releases = sqlx::query_as::<_, ReleaseCandidate>( + r#"SELECT r.id, + r.title::text AS title, + COALESCE( + ( + SELECT a.name::text + FROM furumusic__release_artist ra + JOIN furumusic__artist a ON a.id = ra.artist_id + WHERE ra.release_id = r.id + ORDER BY ra.position + LIMIT 1 + ), + ( + SELECT a.name::text + FROM furumusic__track t + JOIN furumusic__track_artist ta ON ta.track_id = t.id + JOIN furumusic__artist a ON a.id = ta.artist_id + WHERE t.release_id = r.id AND ta.role <> 'featuring' + ORDER BY t.disc_number NULLS LAST, t.track_number NULLS LAST, ta.position + LIMIT 1 + ) + ) AS artist_name, + ( + SELECT t.title::text + FROM furumusic__track t + WHERE t.release_id = r.id + AND t.is_hidden = false + ORDER BY t.disc_number NULLS LAST, t.track_number NULLS LAST, t.id + LIMIT 1 + ) AS track_title + FROM furumusic__release r + LEFT JOIN furumusic__artwork_lookup_state s + ON s.entity_kind = 'release' + AND s.entity_id = r.id + AND s.source = 'lastfm' + WHERE ($3 OR r.cover_file_id IS NULL) + AND r.is_hidden = false + AND ( + $3 + OR s.entity_id IS NULL + OR s.status = 'failed' AND s.last_attempt_at < $1 + OR s.status = 'not_found' AND (s.attempt_count < 3 OR s.last_attempt_at < $2) + OR s.status = 'found' AND s.last_attempt_at < $1 + ) + ORDER BY s.last_attempt_at NULLS FIRST, r.id + LIMIT $4"#, + ) + .bind(&failed_cutoff) + .bind(¬_found_cutoff) + .bind(options.overwrite_existing) .bind(MAX_LASTFM_RELEASE_LOOKUPS) .fetch_all(&ctx.pool) .await?; @@ -580,7 +973,7 @@ async fn backfill_release_lastfm( .await { Ok(cover_file_id) => { - cover_art::assign_cover_to_release(&ctx.pool, release.id, cover_file_id) + assign_cover_to_release(&ctx.pool, release.id, cover_file_id, options) .await?; record_lookup_state( &ctx.pool, @@ -680,12 +1073,37 @@ async fn backfill_release_lastfm( Ok(()) } +async fn assign_cover_to_release( + pool: &sqlx::PgPool, + release_id: i64, + cover_file_id: i64, + options: ArtworkBackfillOptions, +) -> anyhow::Result<()> { + if options.overwrite_existing { + sqlx::query( + r#"UPDATE furumusic__release + SET cover_file_id = $1, + updated_at = $3 + WHERE id = $2"#, + ) + .bind(cover_file_id) + .bind(release_id) + .bind(now_iso()) + .execute(pool) + .await?; + } else { + cover_art::assign_cover_to_release(pool, release_id, cover_file_id).await?; + } + Ok(()) +} + async fn backfill_artist_lastfm( ctx: &JobContext, log: &mut JobLog, storage_dir: &str, api_key: &str, client: &Client, + options: ArtworkBackfillOptions, stats: &mut ArtworkStats, ) -> anyhow::Result<()> { let failed_cutoff = cutoff_iso(1); @@ -699,21 +1117,25 @@ async fn backfill_artist_lastfm( AND s.entity_id = a.id AND s.source = 'lastfm' WHERE ( + $3 + OR a.image_file_id IS NULL OR mf.file_path NOT LIKE '%/__artist_image__/%' ) AND a.is_hidden = false AND ( - s.entity_id IS NULL + $3 + OR s.entity_id IS NULL OR s.status = 'failed' AND s.last_attempt_at < $1 OR s.status = 'not_found' AND (s.attempt_count < 3 OR s.last_attempt_at < $2) OR s.status = 'found' AND s.last_attempt_at < $1 ) ORDER BY s.last_attempt_at NULLS FIRST, a.id - LIMIT $3"#, + LIMIT $4"#, ) .bind(&failed_cutoff) .bind(¬_found_cutoff) + .bind(options.overwrite_existing) .bind(MAX_LASTFM_ARTIST_LOOKUPS) .fetch_all(&ctx.pool) .await?; @@ -755,6 +1177,8 @@ async fn backfill_artist_lastfm( updated_at = $3 WHERE id = $2 AND ( + $4 + OR image_file_id IS NULL OR EXISTS ( SELECT 1 @@ -767,6 +1191,7 @@ async fn backfill_artist_lastfm( .bind(image_file_id) .bind(artist.id) .bind(now_iso()) + .bind(options.overwrite_existing) .execute(&ctx.pool) .await?; record_lookup_state( @@ -826,7 +1251,7 @@ async fn backfill_artist_lastfm( artist.id, artist.name )); stats.artist_lastfm_not_found += 1; - match assign_artist_album_fallback(ctx, artist.id).await { + match assign_artist_album_fallback(ctx, artist.id, options).await { Ok(Some(media_file_id)) => { stats.artist_album_fallback_assigned += 1; log.info(&format!( @@ -892,31 +1317,53 @@ async fn backfill_artist_lastfm( async fn backfill_artist_album_fallbacks( ctx: &JobContext, log: &mut JobLog, + options: ArtworkBackfillOptions, stats: &mut ArtworkStats, ) -> anyhow::Result<()> { let artists = sqlx::query_as::<_, ArtistCandidate>( r#"SELECT a.id, a.name::text AS name FROM furumusic__artist a - WHERE a.image_file_id IS NULL + WHERE ($1 OR a.image_file_id IS NULL) AND a.is_hidden = false AND EXISTS ( SELECT 1 FROM furumusic__release_artist ra JOIN furumusic__release r ON r.id = ra.release_id + JOIN furumusic__media_file mf ON mf.id = r.cover_file_id WHERE ra.artist_id = a.id AND r.cover_file_id IS NOT NULL + AND mf.file_type = 'cover_art' AND r.is_hidden = false + AND NOT EXISTS ( + SELECT 1 + FROM furumusic__artwork_lookup_state s + WHERE s.entity_kind = 'release' + AND s.entity_id = r.id + AND s.source IN ('lastfm', 'coverartarchive') + AND s.status = 'found' + ) UNION SELECT 1 FROM furumusic__track_artist ta JOIN furumusic__track t ON t.id = ta.track_id JOIN furumusic__release r ON r.id = t.release_id + JOIN furumusic__media_file mf ON mf.id = r.cover_file_id WHERE ta.artist_id = a.id AND r.cover_file_id IS NOT NULL + AND mf.file_type = 'cover_art' AND r.is_hidden = false + AND NOT EXISTS ( + SELECT 1 + FROM furumusic__artwork_lookup_state s + WHERE s.entity_kind = 'release' + AND s.entity_id = r.id + AND s.source IN ('lastfm', 'coverartarchive') + AND s.status = 'found' + ) ) ORDER BY a.id"#, ) + .bind(options.overwrite_existing) .fetch_all(&ctx.pool) .await?; @@ -931,7 +1378,7 @@ async fn backfill_artist_album_fallbacks( )); for artist in artists { - match assign_artist_album_fallback(ctx, artist.id).await { + match assign_artist_album_fallback(ctx, artist.id, options).await { Ok(Some(media_file_id)) => { stats.artist_album_fallback_assigned += 1; log.info(&format!( @@ -1016,6 +1463,7 @@ async fn repair_cover_variants( async fn assign_artist_album_fallback( ctx: &JobContext, artist_id: i64, + options: ArtworkBackfillOptions, ) -> anyhow::Result> { let media_file_id: Option = sqlx::query_scalar( r#"SELECT media_file_id @@ -1023,17 +1471,37 @@ async fn assign_artist_album_fallback( SELECT DISTINCT r.cover_file_id AS media_file_id FROM furumusic__release r JOIN furumusic__release_artist ra ON ra.release_id = r.id + JOIN furumusic__media_file mf ON mf.id = r.cover_file_id WHERE ra.artist_id = $1 AND r.cover_file_id IS NOT NULL + AND mf.file_type = 'cover_art' AND r.is_hidden = false + AND NOT EXISTS ( + SELECT 1 + FROM furumusic__artwork_lookup_state s + WHERE s.entity_kind = 'release' + AND s.entity_id = r.id + AND s.source IN ('lastfm', 'coverartarchive') + AND s.status = 'found' + ) UNION SELECT DISTINCT r.cover_file_id AS media_file_id FROM furumusic__release r JOIN furumusic__track t ON t.release_id = r.id JOIN furumusic__track_artist ta ON ta.track_id = t.id + JOIN furumusic__media_file mf ON mf.id = r.cover_file_id WHERE ta.artist_id = $1 AND r.cover_file_id IS NOT NULL + AND mf.file_type = 'cover_art' AND r.is_hidden = false + AND NOT EXISTS ( + SELECT 1 + FROM furumusic__artwork_lookup_state s + WHERE s.entity_kind = 'release' + AND s.entity_id = r.id + AND s.source IN ('lastfm', 'coverartarchive') + AND s.status = 'found' + ) ) covers ORDER BY random() LIMIT 1"#, @@ -1051,11 +1519,12 @@ async fn assign_artist_album_fallback( SET image_file_id = $1, updated_at = $3 WHERE id = $2 - AND image_file_id IS NULL"#, + AND ($4 OR image_file_id IS NULL)"#, ) .bind(media_file_id) .bind(artist_id) .bind(now_iso()) + .bind(options.overwrite_existing) .execute(&ctx.pool) .await?; @@ -1324,6 +1793,36 @@ async fn record_lookup_state( Ok(()) } +async fn record_coverartarchive_lookup_state( + pool: &sqlx::PgPool, + entity_kind: &str, + entity_id: i64, + status: &str, + error: Option<&str>, + source_url: Option<&str>, +) -> anyhow::Result<()> { + sqlx::query( + r#"INSERT INTO furumusic__artwork_lookup_state + (entity_kind, entity_id, source, status, attempt_count, last_attempt_at, last_error, source_url) + VALUES ($1, $2, 'coverartarchive', $3, 1, $4, $5, $6) + ON CONFLICT (entity_kind, entity_id, source) DO UPDATE SET + status = EXCLUDED.status, + attempt_count = furumusic__artwork_lookup_state.attempt_count + 1, + last_attempt_at = EXCLUDED.last_attempt_at, + last_error = EXCLUDED.last_error, + source_url = EXCLUDED.source_url"#, + ) + .bind(entity_kind) + .bind(entity_id) + .bind(status) + .bind(now_iso()) + .bind(error) + .bind(source_url) + .execute(pool) + .await?; + Ok(()) +} + async fn reset_lookup_state( pool: &sqlx::PgPool, entity_kind: &str, @@ -1333,7 +1832,7 @@ async fn reset_lookup_state( r#"DELETE FROM furumusic__artwork_lookup_state WHERE entity_kind = $1 AND entity_id = $2 - AND source = 'lastfm'"#, + AND source IN ('lastfm', 'coverartarchive')"#, ) .bind(entity_kind) .bind(entity_id) diff --git a/src/jobs/metadata_backfill.rs b/src/jobs/metadata_backfill.rs index 43717b8..c3c4557 100644 --- a/src/jobs/metadata_backfill.rs +++ b/src/jobs/metadata_backfill.rs @@ -15,6 +15,7 @@ pub struct MetadataBackfillOptions { pub duration_seconds: bool, pub local_genres: bool, pub lastfm_tags: bool, + pub musicbrainz_tags: bool, pub overwrite: bool, } @@ -26,6 +27,7 @@ impl MetadataBackfillOptions { || self.duration_seconds || self.local_genres || self.lastfm_tags + || self.musicbrainz_tags } fn needs_file_scan(self) -> bool { @@ -59,6 +61,7 @@ struct LastfmReleaseTagRow { id: i64, title: String, artist_name: Option, + track_title: Option, } #[derive(sqlx::FromRow)] @@ -84,6 +87,16 @@ struct LastfmTagStats { failed: u64, } +#[derive(Debug, Default)] +struct MusicBrainzTagStats { + considered: u64, + updated_entities: u64, + tags_saved: u64, + skipped_existing: u64, + not_found: u64, + failed: u64, +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum LastfmTagPassResult { Completed, @@ -117,6 +130,7 @@ impl Job for MetadataBackfillJob { duration_seconds: true, local_genres: true, lastfm_tags: true, + musicbrainz_tags: true, overwrite: false, }, ) @@ -143,10 +157,11 @@ pub async fn run_with_options( let mut failed = 0u64; log.info(&format!( - "Metadata backfill options: file_scan={}, local_genres={}, lastfm_tags={}, mode={}", + "Metadata backfill options: file_scan={}, local_genres={}, lastfm_tags={}, musicbrainz_tags={}, mode={}", options.needs_file_scan(), options.local_genres, options.lastfm_tags, + options.musicbrainz_tags, if options.overwrite { "overwrite" } else { @@ -251,14 +266,9 @@ pub async fn run_with_options( let mut changed_tags = false; if options.local_genres { if let (Some(track_id), Some(genre)) = (row.track_id, raw_meta.genre.as_deref()) { - let saved = save_track_tag_text( - &ctx.pool, - track_id, - genre, - "file", - options.overwrite, - ) - .await?; + let saved = + save_track_tag_text(&ctx.pool, track_id, genre, "file", options.overwrite) + .await?; if saved > 0 { local_tags_updated += saved; changed_tags = true; @@ -312,14 +322,28 @@ pub async fn run_with_options( LastfmTagStats::default() }; + let musicbrainz_stats = if options.musicbrainz_tags { + log.info("Starting MusicBrainz tag backfill"); + backfill_musicbrainz_tags(ctx, log, options.overwrite).await? + } else { + log.info("MusicBrainz tag backfill disabled for this run"); + MusicBrainzTagStats::default() + }; + log.info(&format!( - "Metadata backfill complete: {scanned} scanned, {media_updated} media updated, {track_updated} tracks updated, {local_tags_updated} local tags saved, {unchanged} unchanged, {missing} missing, {failed} failed; Last.fm tags: considered={}, updated_entities={}, tags_saved={}, skipped_existing={}, not_found={}, failed={}", + "Metadata backfill complete: {scanned} scanned, {media_updated} media updated, {track_updated} tracks updated, {local_tags_updated} local tags saved, {unchanged} unchanged, {missing} missing, {failed} failed; Last.fm tags: considered={}, updated_entities={}, tags_saved={}, skipped_existing={}, not_found={}, failed={}; MusicBrainz tags: considered={}, updated_entities={}, tags_saved={}, skipped_existing={}, not_found={}, failed={}", lastfm_stats.considered, lastfm_stats.updated_entities, lastfm_stats.tags_saved, lastfm_stats.skipped_existing, lastfm_stats.not_found, lastfm_stats.failed, + musicbrainz_stats.considered, + musicbrainz_stats.updated_entities, + musicbrainz_stats.tags_saved, + musicbrainz_stats.skipped_existing, + musicbrainz_stats.not_found, + musicbrainz_stats.failed, )); Ok(()) } @@ -369,6 +393,259 @@ async fn backfill_lastfm_tags( Ok(stats) } +async fn backfill_musicbrainz_tags( + ctx: &JobContext, + log: &mut JobLog, + overwrite: bool, +) -> anyhow::Result { + log.info("MusicBrainz tag backfill started"); + let client = crate::jobs::musicbrainz::MusicBrainzClient::new("furumusic-metadata-backfill")?; + let mut stats = MusicBrainzTagStats::default(); + backfill_musicbrainz_artist_tags(ctx, log, &client, overwrite, &mut stats).await?; + backfill_musicbrainz_release_tags(ctx, log, &client, overwrite, &mut stats).await?; + Ok(stats) +} + +async fn backfill_musicbrainz_artist_tags( + ctx: &JobContext, + log: &mut JobLog, + client: &crate::jobs::musicbrainz::MusicBrainzClient, + overwrite: bool, + stats: &mut MusicBrainzTagStats, +) -> anyhow::Result<()> { + let rows = sqlx::query_as::<_, LastfmArtistTagRow>( + r#"SELECT DISTINCT a.id, a.name::text AS name + FROM furumusic__artist a + JOIN furumusic__track_artist ta ON ta.artist_id = a.id + JOIN furumusic__track t ON t.id = ta.track_id + WHERE a.is_hidden = false AND t.is_hidden = false + ORDER BY a.id"#, + ) + .fetch_all(&ctx.pool) + .await?; + + log.info(&format!( + "MusicBrainz artist tag pass: checking {} artist(s)", + rows.len() + )); + let total = rows.len(); + for (index, row) in rows.into_iter().enumerate() { + if should_log_lastfm_item(index + 1, total, 25) { + log.info(&format!( + "MusicBrainz artist tags {}/{}: artist {} \"{}\"", + index + 1, + total, + row.id, + row.name + )); + } + if should_skip_source_entity(&ctx.pool, "artist", row.id, "musicbrainz", overwrite).await? { + stats.skipped_existing += 1; + continue; + } + stats.considered += 1; + + let mbid = + match crate::jobs::musicbrainz::load_external_id(&ctx.pool, "artist", row.id, "artist") + .await? + { + Some(mbid) => Some(mbid), + None => match client.search_artist(&row.name).await { + Ok(Some(found)) => { + crate::jobs::musicbrainz::save_external_id( + &ctx.pool, + "artist", + row.id, + "artist", + &found.mbid, + found.score as f64 / 100.0, + ) + .await?; + Some(found.mbid) + } + Ok(None) => None, + Err(err) => { + stats.failed += 1; + log.warn(&format!( + "MusicBrainz artist search failed for artist {} \"{}\": {err}", + row.id, row.name + )); + None + } + }, + }; + + let Some(mbid) = mbid else { + stats.not_found += 1; + continue; + }; + match client.lookup_artist_tags(&mbid).await { + Ok(tags) if !tags.is_empty() => { + let tags = musicbrainz_tags_to_candidates(&tags); + match replace_entity_tags(&ctx.pool, "artist", row.id, &tags, "musicbrainz", false) + .await + { + Ok(saved) => { + stats.tags_saved += saved; + stats.updated_entities += 1; + } + Err(err) => { + stats.failed += 1; + log.warn(&format!( + "MusicBrainz artist tags save failed for artist {} \"{}\": {err}", + row.id, row.name + )); + } + } + } + Ok(_) => { + stats.not_found += 1; + } + Err(err) => { + stats.failed += 1; + log.warn(&format!( + "MusicBrainz artist tags failed for artist {} \"{}\" mbid={}: {err}", + row.id, row.name, mbid + )); + } + } + } + Ok(()) +} + +async fn backfill_musicbrainz_release_tags( + ctx: &JobContext, + log: &mut JobLog, + client: &crate::jobs::musicbrainz::MusicBrainzClient, + overwrite: bool, + stats: &mut MusicBrainzTagStats, +) -> anyhow::Result<()> { + let rows = sqlx::query_as::<_, LastfmReleaseTagRow>( + r#"SELECT r.id, + r.title::text AS title, + ( + SELECT a.name::text + FROM furumusic__release_artist ra + JOIN furumusic__artist a ON a.id = ra.artist_id + WHERE ra.release_id = r.id + ORDER BY ra.position + LIMIT 1 + ) AS artist_name, + ( + SELECT t.title::text + FROM furumusic__track t + WHERE t.release_id = r.id + AND t.is_hidden = false + ORDER BY t.disc_number NULLS LAST, t.track_number NULLS LAST, t.id + LIMIT 1 + ) AS track_title + FROM furumusic__release r + WHERE r.is_hidden = false + ORDER BY r.id"#, + ) + .fetch_all(&ctx.pool) + .await?; + + log.info(&format!( + "MusicBrainz release tag pass: checking {} release(s)", + rows.len() + )); + let total = rows.len(); + for (index, row) in rows.into_iter().enumerate() { + if should_log_lastfm_item(index + 1, total, 25) { + log.info(&format!( + "MusicBrainz release tags {}/{}: release {} \"{}\"", + index + 1, + total, + row.id, + row.title + )); + } + if should_skip_source_entity(&ctx.pool, "release", row.id, "musicbrainz", overwrite).await? + { + stats.skipped_existing += 1; + continue; + } + let Some(artist) = row + .artist_name + .as_deref() + .filter(|value| !value.trim().is_empty()) + else { + stats.not_found += 1; + continue; + }; + stats.considered += 1; + + let mbid = match crate::jobs::musicbrainz::load_or_search_release_mbid( + &ctx.pool, + client, + row.id, + artist, + &row.title, + row.track_title.as_deref(), + ) + .await + { + Ok((release_mbid, _release_group_mbid)) => release_mbid, + Err(err) => { + stats.failed += 1; + log.warn(&format!( + "MusicBrainz release search failed for release {} \"{}\" / \"{}\": {err}", + row.id, artist, row.title + )); + None + } + }; + + let Some(mbid) = mbid else { + stats.not_found += 1; + continue; + }; + match client.lookup_release_tags(&mbid).await { + Ok(result) if !result.tags.is_empty() => { + if let Some(group_mbid) = result.release_group_mbid.as_deref() { + crate::jobs::musicbrainz::save_external_id( + &ctx.pool, + "release", + row.id, + "release_group", + group_mbid, + 1.0, + ) + .await?; + } + let tags = musicbrainz_tags_to_candidates(&result.tags); + match replace_entity_tags(&ctx.pool, "release", row.id, &tags, "musicbrainz", false) + .await + { + Ok(saved) => { + stats.tags_saved += saved; + stats.updated_entities += 1; + } + Err(err) => { + stats.failed += 1; + log.warn(&format!( + "MusicBrainz release tags save failed for release {} \"{}\" / \"{}\": {err}", + row.id, artist, row.title + )); + } + } + } + Ok(_) => { + stats.not_found += 1; + } + Err(err) => { + stats.failed += 1; + log.warn(&format!( + "MusicBrainz release tags failed for release {} \"{}\" mbid={}: {err}", + row.id, row.title, mbid + )); + } + } + } + Ok(()) +} + async fn backfill_lastfm_artist_tags( ctx: &JobContext, log: &mut JobLog, @@ -428,8 +705,7 @@ async fn backfill_lastfm_artist_tags( stats.considered += 1; match fetch_lastfm_artist_tags(client, api_key, &row.name).await { Ok(tags) if !tags.is_empty() => { - match replace_entity_tags(&ctx.pool, "artist", row.id, &tags, "lastfm", false) - .await + match replace_entity_tags(&ctx.pool, "artist", row.id, &tags, "lastfm", false).await { Ok(saved) => { stats.tags_saved += saved; @@ -493,7 +769,15 @@ async fn backfill_lastfm_release_tags( WHERE ra.release_id = r.id ORDER BY ra.position LIMIT 1 - ) AS artist_name + ) AS artist_name, + ( + SELECT t.title::text + FROM furumusic__track t + WHERE t.release_id = r.id + AND t.is_hidden = false + ORDER BY t.disc_number NULLS LAST, t.track_number NULLS LAST, t.id + LIMIT 1 + ) AS track_title FROM furumusic__release r WHERE r.is_hidden = false ORDER BY r.id"#, @@ -538,7 +822,10 @@ async fn backfill_lastfm_release_tags( continue; } } - let Some(artist) = row.artist_name.as_deref().filter(|value| !value.trim().is_empty()) + let Some(artist) = row + .artist_name + .as_deref() + .filter(|value| !value.trim().is_empty()) else { stats.not_found += 1; if should_log_lastfm_progress(index + 1, total, 25) { @@ -663,7 +950,10 @@ async fn backfill_lastfm_track_tags( continue; } } - let Some(artist) = row.artist_name.as_deref().filter(|value| !value.trim().is_empty()) + let Some(artist) = row + .artist_name + .as_deref() + .filter(|value| !value.trim().is_empty()) else { stats.not_found += 1; if should_log_lastfm_progress(index + 1, total, 50) { @@ -678,8 +968,7 @@ async fn backfill_lastfm_track_tags( stats.considered += 1; match fetch_lastfm_track_tags(client, api_key, artist, &row.title).await { Ok(tags) if !tags.is_empty() => { - match replace_entity_tags(&ctx.pool, "track", row.id, &tags, "lastfm", true).await - { + match replace_entity_tags(&ctx.pool, "track", row.id, &tags, "lastfm", true).await { Ok(saved) => { stats.tags_saved += saved; stats.updated_entities += 1; @@ -741,6 +1030,16 @@ async fn should_skip_lastfm_entity( entity_kind: &str, entity_id: i64, overwrite: bool, +) -> anyhow::Result { + should_skip_source_entity(pool, entity_kind, entity_id, "lastfm", overwrite).await +} + +async fn should_skip_source_entity( + pool: &sqlx::PgPool, + entity_kind: &str, + entity_id: i64, + source: &str, + overwrite: bool, ) -> anyhow::Result { if overwrite { return Ok(false); @@ -748,16 +1047,28 @@ async fn should_skip_lastfm_entity( let exists: Option = sqlx::query_scalar( r#"SELECT 1 FROM furumusic__entity_genre_tag - WHERE entity_kind = $1 AND entity_id = $2 AND source = 'lastfm' + WHERE entity_kind = $1 AND entity_id = $2 AND source = $3 LIMIT 1"#, ) .bind(entity_kind) .bind(entity_id) + .bind(source) .fetch_optional(pool) .await?; Ok(exists.is_some()) } +fn musicbrainz_tags_to_candidates( + tags: &[crate::jobs::musicbrainz::MusicBrainzTag], +) -> Vec { + tags.iter() + .map(|tag| TagCandidate { + name: tag.name.clone(), + weight: tag.weight, + }) + .collect() +} + async fn fetch_lastfm_artist_tags( client: &reqwest::Client, api_key: &str, @@ -854,7 +1165,11 @@ async fn fetch_lastfm_top_tags( Value::Object(_) => tag_from_value(tag_value).into_iter().collect::>(), _ => Vec::new(), }; - tags.sort_by(|a, b| b.weight.total_cmp(&a.weight).then_with(|| a.name.cmp(&b.name))); + tags.sort_by(|a, b| { + b.weight + .total_cmp(&a.weight) + .then_with(|| a.name.cmp(&b.name)) + }); tags.truncate(LASTFM_TAG_LIMIT); Ok(tags) } @@ -1027,9 +1342,9 @@ fn tags_from_text(value: &str) -> Vec { for raw in normalized_separators.split([';', ',']) { if let Some(name) = clean_tag_name(raw) { if !is_ignored_tag(&normalize_tag_name(&name)) - && !tags - .iter() - .any(|tag: &TagCandidate| normalize_tag_name(&tag.name) == normalize_tag_name(&name)) + && !tags.iter().any(|tag: &TagCandidate| { + normalize_tag_name(&tag.name) == normalize_tag_name(&name) + }) { tags.push(TagCandidate { name, weight: 1.0 }); } diff --git a/src/jobs/mod.rs b/src/jobs/mod.rs index df3b5e6..903675a 100644 --- a/src/jobs/mod.rs +++ b/src/jobs/mod.rs @@ -4,6 +4,7 @@ pub mod inbox_process; pub mod lastfm_popularity; pub mod lastfm_scrobble; pub mod metadata_backfill; +pub mod musicbrainz; use std::path::{Component, Path, PathBuf}; diff --git a/src/jobs/musicbrainz.rs b/src/jobs/musicbrainz.rs new file mode 100644 index 0000000..e24d3b8 --- /dev/null +++ b/src/jobs/musicbrainz.rs @@ -0,0 +1,587 @@ +use std::time::{Duration, Instant}; + +use reqwest::{Client, StatusCode}; +use serde::Deserialize; +use tokio::sync::Mutex; + +const MUSICBRAINZ_BASE_URL: &str = "https://musicbrainz.org/ws/2"; +const COVER_ART_ARCHIVE_BASE_URL: &str = "https://coverartarchive.org"; +const MUSICBRAINZ_REQUEST_DELAY: Duration = Duration::from_millis(1100); +const MUSICBRAINZ_TAG_LIMIT: usize = 12; + +#[derive(Debug, Clone)] +pub struct MusicBrainzTag { + pub name: String, + pub weight: f64, +} + +#[derive(Debug, Clone)] +pub struct MusicBrainzArtistMatch { + pub mbid: String, + pub score: i32, +} + +#[derive(Debug, Clone)] +pub struct MusicBrainzReleaseMatch { + pub mbid: String, + pub release_group_mbid: Option, + pub score: i32, +} + +#[derive(Debug, Clone)] +pub struct MusicBrainzReleaseTags { + pub release_group_mbid: Option, + pub tags: Vec, +} + +pub struct MusicBrainzClient { + client: Client, + last_musicbrainz_request: Mutex>, +} + +pub async fn load_external_id( + pool: &sqlx::PgPool, + entity_kind: &str, + entity_id: i64, + id_kind: &str, +) -> anyhow::Result> { + let value = sqlx::query_scalar::<_, String>( + r#"SELECT external_id::text + FROM furumusic__external_metadata_id + WHERE entity_kind = $1 + AND entity_id = $2 + AND source = 'musicbrainz' + AND id_kind = $3 + LIMIT 1"#, + ) + .bind(entity_kind) + .bind(entity_id) + .bind(id_kind) + .fetch_optional(pool) + .await?; + Ok(value) +} + +pub async fn save_external_id( + pool: &sqlx::PgPool, + entity_kind: &str, + entity_id: i64, + id_kind: &str, + external_id: &str, + confidence: f64, +) -> anyhow::Result<()> { + sqlx::query( + r#"INSERT INTO furumusic__external_metadata_id + (entity_kind, entity_id, source, id_kind, external_id, confidence, updated_at) + VALUES ($1, $2, 'musicbrainz', $3, $4, $5, $6) + ON CONFLICT (entity_kind, entity_id, source, id_kind) DO UPDATE SET + external_id = EXCLUDED.external_id, + confidence = EXCLUDED.confidence, + updated_at = EXCLUDED.updated_at"#, + ) + .bind(entity_kind) + .bind(entity_id) + .bind(id_kind) + .bind(external_id) + .bind(confidence) + .bind(now_iso()) + .execute(pool) + .await?; + Ok(()) +} + +pub async fn load_or_search_release_mbid( + pool: &sqlx::PgPool, + client: &MusicBrainzClient, + release_id: i64, + artist_name: &str, + release_title: &str, + representative_track_title: Option<&str>, +) -> anyhow::Result<(Option, Option)> { + let release_mbid = load_external_id(pool, "release", release_id, "release").await?; + let release_group_mbid = load_external_id(pool, "release", release_id, "release_group").await?; + if release_mbid.is_some() || release_group_mbid.is_some() { + return Ok((release_mbid, release_group_mbid)); + } + + let found = match client.search_release(artist_name, release_title).await? { + Some(found) => Some(found), + None => { + if let Some(track_title) = + representative_track_title.filter(|value| !value.trim().is_empty()) + { + client + .search_release_by_recording(artist_name, track_title) + .await? + } else { + None + } + } + }; + + let Some(found) = found else { + return Ok((None, None)); + }; + save_external_id( + pool, + "release", + release_id, + "release", + &found.mbid, + found.score as f64 / 100.0, + ) + .await?; + if let Some(group_mbid) = found.release_group_mbid.as_deref() { + save_external_id( + pool, + "release", + release_id, + "release_group", + group_mbid, + found.score as f64 / 100.0, + ) + .await?; + } + Ok((Some(found.mbid), found.release_group_mbid)) +} + +impl MusicBrainzClient { + pub fn new(user_agent_prefix: &str) -> anyhow::Result { + let client = Client::builder() + .user_agent(format!( + "{}/{} (musicbrainz.org/doc/MusicBrainz_API)", + user_agent_prefix, + env!("CARGO_PKG_VERSION") + )) + .timeout(Duration::from_secs(20)) + .build()?; + Ok(Self { + client, + last_musicbrainz_request: Mutex::new(None), + }) + } + + pub fn http_client(&self) -> &Client { + &self.client + } + + pub async fn search_artist( + &self, + name: &str, + ) -> anyhow::Result> { + let query = format!("artist:\"{}\"", escape_search_value(name)); + let response: Option = self + .get_musicbrainz_json( + "artist", + &[("query", query.as_str()), ("fmt", "json"), ("limit", "5")], + ) + .await?; + let Some(response) = response else { + return Ok(None); + }; + Ok(response + .artists + .into_iter() + .filter(|artist| artist.id.trim().len() == 36) + .max_by_key(|artist| artist.score.unwrap_or(0)) + .and_then(|artist| { + let score = artist.score.unwrap_or(0); + (score >= 70).then_some(MusicBrainzArtistMatch { + mbid: artist.id, + score, + }) + })) + } + + pub async fn search_release( + &self, + artist: &str, + title: &str, + ) -> anyhow::Result> { + let query = format!( + "release:\"{}\" AND artist:\"{}\"", + escape_search_value(title), + escape_search_value(artist) + ); + let response: Option = self + .get_musicbrainz_json( + "release", + &[("query", query.as_str()), ("fmt", "json"), ("limit", "5")], + ) + .await?; + let Some(response) = response else { + return Ok(None); + }; + Ok(response + .releases + .into_iter() + .filter(|release| release.id.trim().len() == 36) + .max_by_key(|release| release.score.unwrap_or(0)) + .and_then(|release| { + let score = release.score.unwrap_or(0); + (score >= 70).then_some(MusicBrainzReleaseMatch { + mbid: release.id, + release_group_mbid: release.release_group.map(|group| group.id), + score, + }) + })) + } + + pub async fn search_release_by_recording( + &self, + artist: &str, + track_title: &str, + ) -> anyhow::Result> { + let query = format!( + "recording:\"{}\" AND artist:\"{}\"", + escape_search_value(track_title), + escape_search_value(artist) + ); + let response: Option = self + .get_musicbrainz_json( + "recording", + &[("query", query.as_str()), ("fmt", "json"), ("limit", "5")], + ) + .await?; + let Some(response) = response else { + return Ok(None); + }; + + Ok(response + .recordings + .into_iter() + .flat_map(|recording| { + let recording_score = recording.score.unwrap_or(0); + recording + .releases + .into_iter() + .filter(move |release| release.id.trim().len() == 36) + .map(move |release| (recording_score, release)) + }) + .max_by_key(|(score, _)| *score) + .and_then(|(score, release)| { + (score >= 70).then_some(MusicBrainzReleaseMatch { + mbid: release.id, + release_group_mbid: release.release_group.map(|group| group.id), + score, + }) + })) + } + + pub async fn lookup_artist_tags(&self, mbid: &str) -> anyhow::Result> { + let response: Option = self + .get_musicbrainz_json( + &format!("artist/{mbid}"), + &[("inc", "tags+genres"), ("fmt", "json")], + ) + .await?; + Ok(response.map(tags_from_entity).unwrap_or_default()) + } + + pub async fn lookup_release_tags(&self, mbid: &str) -> anyhow::Result { + let response: Option = self + .get_musicbrainz_json( + &format!("release/{mbid}"), + &[("inc", "tags+genres+release-groups"), ("fmt", "json")], + ) + .await?; + let Some(response) = response else { + return Ok(MusicBrainzReleaseTags { + release_group_mbid: None, + tags: Vec::new(), + }); + }; + + let mut tags = tags_from_parts(response.tags, response.genres); + let release_group_mbid = response + .release_group + .as_ref() + .map(|group| group.id.clone()); + if let Some(group_mbid) = release_group_mbid.as_deref() { + let group_response: Option = self + .get_musicbrainz_json( + &format!("release-group/{group_mbid}"), + &[("inc", "tags+genres"), ("fmt", "json")], + ) + .await?; + merge_tags( + &mut tags, + group_response.map(tags_from_entity).unwrap_or_default(), + ); + } + tags.sort_by(|a, b| { + b.weight + .total_cmp(&a.weight) + .then_with(|| a.name.cmp(&b.name)) + }); + tags.truncate(MUSICBRAINZ_TAG_LIMIT); + + Ok(MusicBrainzReleaseTags { + release_group_mbid, + tags, + }) + } + + pub async fn fetch_cover_art_front_url( + &self, + release_mbid: Option<&str>, + release_group_mbid: Option<&str>, + ) -> anyhow::Result> { + if let Some(mbid) = release_mbid { + if let Some(url) = self.cover_art_front_url("release", mbid).await? { + return Ok(Some(url)); + } + } + if let Some(mbid) = release_group_mbid { + if let Some(url) = self.cover_art_front_url("release-group", mbid).await? { + return Ok(Some(url)); + } + } + Ok(None) + } + + async fn get_musicbrainz_json( + &self, + path: &str, + query: &[(&str, &str)], + ) -> anyhow::Result> + where + T: for<'de> Deserialize<'de>, + { + self.wait_for_musicbrainz_slot().await; + let url = format!("{MUSICBRAINZ_BASE_URL}/{}", path.trim_start_matches('/')); + let response = self.client.get(url).query(query).send().await?; + if response.status() == StatusCode::NOT_FOUND { + return Ok(None); + } + if response.status() == StatusCode::TOO_MANY_REQUESTS + || response.status() == StatusCode::SERVICE_UNAVAILABLE + { + anyhow::bail!( + "MusicBrainz rate limit or service unavailable: {}", + response.status() + ); + } + let response = response.error_for_status()?; + Ok(Some(response.json::().await?)) + } + + async fn cover_art_front_url(&self, kind: &str, mbid: &str) -> anyhow::Result> { + let url = format!("{COVER_ART_ARCHIVE_BASE_URL}/{kind}/{mbid}"); + let response = self.client.get(url).send().await?; + if response.status() == StatusCode::NOT_FOUND { + return Ok(None); + } + if response.status() == StatusCode::TOO_MANY_REQUESTS + || response.status() == StatusCode::SERVICE_UNAVAILABLE + { + anyhow::bail!("Cover Art Archive unavailable: {}", response.status()); + } + let response = response.error_for_status()?; + let body = response.json::().await?; + Ok(best_cover_art_url(body.images)) + } + + async fn wait_for_musicbrainz_slot(&self) { + let mut last = self.last_musicbrainz_request.lock().await; + if let Some(previous) = *last { + let elapsed = previous.elapsed(); + if elapsed < MUSICBRAINZ_REQUEST_DELAY { + tokio::time::sleep(MUSICBRAINZ_REQUEST_DELAY - elapsed).await; + } + } + *last = Some(Instant::now()); + } +} + +fn now_iso() -> String { + chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ").to_string() +} + +#[derive(Debug, Deserialize)] +struct ArtistSearchResponse { + #[serde(default)] + artists: Vec, +} + +#[derive(Debug, Deserialize)] +struct ArtistSearchItem { + id: String, + score: Option, +} + +#[derive(Debug, Deserialize)] +struct ReleaseSearchResponse { + #[serde(default)] + releases: Vec, +} + +#[derive(Debug, Deserialize)] +struct ReleaseSearchItem { + id: String, + score: Option, + #[serde(rename = "release-group")] + release_group: Option, +} + +#[derive(Debug, Deserialize)] +struct RecordingSearchResponse { + #[serde(default)] + recordings: Vec, +} + +#[derive(Debug, Deserialize)] +struct RecordingSearchItem { + score: Option, + #[serde(default)] + releases: Vec, +} + +#[derive(Debug, Deserialize)] +struct RecordingReleaseItem { + id: String, + #[serde(rename = "release-group")] + release_group: Option, +} + +#[derive(Debug, Deserialize)] +struct ReleaseLookupResponse { + #[serde(default)] + tags: Vec, + #[serde(default)] + genres: Vec, + #[serde(rename = "release-group")] + release_group: Option, +} + +#[derive(Debug, Deserialize)] +struct TaggedEntityResponse { + #[serde(default)] + tags: Vec, + #[serde(default)] + genres: Vec, +} + +#[derive(Debug, Deserialize)] +struct MusicBrainzIdRef { + id: String, +} + +#[derive(Debug, Deserialize)] +struct MusicBrainzTagItem { + name: String, + count: Option, +} + +#[derive(Debug, Deserialize)] +struct CoverArtArchiveResponse { + #[serde(default)] + images: Vec, +} + +#[derive(Debug, Deserialize)] +struct CoverArtArchiveImage { + image: Option, + front: Option, + approved: Option, + #[serde(default)] + types: Vec, + thumbnails: Option, +} + +#[derive(Debug, Deserialize)] +struct CoverArtArchiveThumbnails { + #[serde(rename = "1200")] + size_1200: Option, + #[serde(rename = "500")] + size_500: Option, + large: Option, +} + +fn escape_search_value(value: &str) -> String { + value.replace('\\', "\\\\").replace('"', "\\\"") +} + +fn tags_from_entity(entity: TaggedEntityResponse) -> Vec { + tags_from_parts(entity.tags, entity.genres) +} + +fn tags_from_parts( + tags: Vec, + genres: Vec, +) -> Vec { + let mut result = Vec::new(); + merge_items(&mut result, genres, 2.0); + merge_items(&mut result, tags, 1.0); + result.sort_by(|a, b| { + b.weight + .total_cmp(&a.weight) + .then_with(|| a.name.cmp(&b.name)) + }); + result.truncate(MUSICBRAINZ_TAG_LIMIT); + result +} + +fn merge_items(result: &mut Vec, items: Vec, multiplier: f64) { + for item in items { + let name = item.name.trim(); + if name.is_empty() { + continue; + } + let weight = item.count.unwrap_or(1).max(1) as f64 * multiplier; + if let Some(existing) = result + .iter_mut() + .find(|tag| tag.name.eq_ignore_ascii_case(name)) + { + existing.weight = existing.weight.max(weight); + } else { + result.push(MusicBrainzTag { + name: name.to_string(), + weight, + }); + } + } +} + +fn merge_tags(result: &mut Vec, extra: Vec) { + for tag in extra { + if let Some(existing) = result + .iter_mut() + .find(|candidate| candidate.name.eq_ignore_ascii_case(&tag.name)) + { + existing.weight = existing.weight.max(tag.weight); + } else { + result.push(tag); + } + } +} + +fn best_cover_art_url(mut images: Vec) -> Option { + images.sort_by_key(|image| { + let front = image.front.unwrap_or(false) + || image + .types + .iter() + .any(|value| value.eq_ignore_ascii_case("front")); + let approved = image.approved.unwrap_or(false); + (u8::from(front), u8::from(approved)) + }); + images + .into_iter() + .rev() + .find_map(|image| { + image + .thumbnails + .as_ref() + .and_then(|thumbs| { + thumbs + .size_1200 + .as_deref() + .or(thumbs.size_500.as_deref()) + .or(thumbs.large.as_deref()) + }) + .map(str::to_string) + .or(image.image) + }) + .filter(|url| !url.trim().is_empty()) +} diff --git a/src/music/mod.rs b/src/music/mod.rs index 3a87e3f..36a8ad4 100644 --- a/src/music/mod.rs +++ b/src/music/mod.rs @@ -1865,6 +1865,51 @@ pub mod db_migrations { &[Operation::custom(create_entity_genre_tags).build()]; } + // -- M0036: External metadata identifiers -------------------------------- + + #[cot::db::migrations::migration_op] + async fn create_external_metadata_ids( + ctx: migrations::MigrationContext<'_>, + ) -> cot::db::Result<()> { + ctx.db + .raw( + "CREATE TABLE IF NOT EXISTS furumusic__external_metadata_id ( + id BIGSERIAL PRIMARY KEY, + entity_kind VARCHAR(32) NOT NULL, + entity_id BIGINT NOT NULL, + source VARCHAR(32) NOT NULL, + id_kind VARCHAR(64) NOT NULL, + external_id VARCHAR(255) NOT NULL, + confidence DOUBLE PRECISION NOT NULL DEFAULT 1, + updated_at VARCHAR(32) NOT NULL, + UNIQUE(entity_kind, entity_id, source, id_kind) + )", + ) + .await?; + ctx.db + .raw( + "CREATE INDEX IF NOT EXISTS idx_external_metadata_id_lookup + ON furumusic__external_metadata_id (source, id_kind, external_id)", + ) + .await?; + Ok(()) + } + + #[derive(Debug, Copy, Clone)] + pub struct M0036CreateExternalMetadataIds; + + impl migrations::Migration for M0036CreateExternalMetadataIds { + const APP_NAME: &'static str = "furumusic"; + const MIGRATION_NAME: &'static str = "m_0036_create_external_metadata_ids"; + const DEPENDENCIES: &'static [migrations::MigrationDependency] = + &[migrations::MigrationDependency::migration( + "furumusic", + "m_0035_create_entity_genre_tags", + )]; + const OPERATIONS: &'static [Operation] = + &[Operation::custom(create_external_metadata_ids).build()]; + } + pub const MIGRATIONS: &[&SyncDynMigration] = &[ &M0006CreateMediaFile, &M0007CreateArtist, @@ -1891,5 +1936,6 @@ pub mod db_migrations { &M0033CreateLastfmScrobbling, &M0034CreateArtworkLookupState, &M0035CreateEntityGenreTags, + &M0036CreateExternalMetadataIds, ]; } diff --git a/src/player/dto.rs b/src/player/dto.rs index 167c588..f0614b2 100644 --- a/src/player/dto.rs +++ b/src/player/dto.rs @@ -16,6 +16,7 @@ pub(super) struct Paginated { pub(super) total: i64, pub(super) page: i32, pub(super) per_page: i32, + pub(super) has_more: bool, } #[derive(Debug, Serialize, JsonSchema)] diff --git a/src/player/mod.rs b/src/player/mod.rs index 5c93de8..c38ac2f 100644 --- a/src/player/mod.rs +++ b/src/player/mod.rs @@ -2787,7 +2787,7 @@ async fn artists_handler( let total_row = sqlx::query_as::<_, CountRow>( r#"SELECT COUNT(DISTINCT a.id) AS count FROM furumusic__artist a - JOIN furumusic__track_artist ta ON ta.artist_id = a.id AND ta.role <> 'featuring' + JOIN furumusic__track_artist ta ON ta.artist_id = a.id JOIN furumusic__track t ON t.id = ta.track_id AND t.is_hidden = false JOIN furumusic__release r ON r.id = t.release_id AND r.is_hidden = false JOIN furumusic__media_file mf ON mf.id = t.audio_file_id @@ -2801,17 +2801,24 @@ async fn artists_handler( let rows = sqlx::query_as::<_, ArtistRow>( r#"SELECT a.id, a.name::text as name, a.image_file_id, - COUNT(DISTINCT r.id) AS release_count, + COUNT(DISTINCT r.id) FILTER (WHERE primary_release.artist_id IS NOT NULL) AS release_count, COUNT(DISTINCT t.id) AS track_count FROM furumusic__artist a - JOIN furumusic__track_artist ta ON ta.artist_id = a.id AND ta.role <> 'featuring' + JOIN furumusic__track_artist ta ON ta.artist_id = a.id JOIN furumusic__track t ON t.id = ta.track_id AND t.is_hidden = false JOIN furumusic__release r ON r.id = t.release_id AND r.is_hidden = false JOIN furumusic__media_file mf ON mf.id = t.audio_file_id + LEFT JOIN furumusic__release_artist primary_release + ON primary_release.release_id = r.id + AND primary_release.artist_id = a.id + AND primary_release.position = 0 WHERE a.is_hidden = false AND mf.uploaded_by_user_id = $1 GROUP BY a.id, a.name, a.name_sort, a.image_file_id - ORDER BY release_count DESC, track_count DESC, a.name_sort + ORDER BY (COUNT(DISTINCT r.id) FILTER (WHERE primary_release.artist_id IS NOT NULL) > 0) DESC, + release_count DESC, + track_count DESC, + a.name_sort LIMIT $2 OFFSET $3"#, ) .bind(user.id) @@ -2831,22 +2838,22 @@ async fn artists_handler( track_count: r.track_count, }) .collect(); + let has_more = offset + (items.len() as i64) < total_row.count; return Json(Paginated { items, total: total_row.count, page, per_page, + has_more, }) .into_response(); } let total_row = sqlx::query_as::<_, CountRow>( - r#"SELECT COUNT(DISTINCT a.id) AS count + r#"SELECT COUNT(*) AS count FROM furumusic__artist a - JOIN furumusic__release_artist ra ON ra.artist_id = a.id - JOIN furumusic__release r ON r.id = ra.release_id - WHERE a.is_hidden = false AND r.is_hidden = false AND ra.position = 0"#, + WHERE a.is_hidden = false"#, ) .fetch_one(pool) .await @@ -2854,21 +2861,33 @@ async fn artists_handler( let rows = sqlx::query_as::<_, ArtistRow>( r#"SELECT a.id, a.name::text as name, a.image_file_id, - s.release_count, - s.track_count + COALESCE(s.release_count, 0) AS release_count, + COALESCE(s.track_count, 0) AS track_count FROM furumusic__artist a - JOIN ( - SELECT ra.artist_id, - COUNT(DISTINCT r.id) AS release_count, - COUNT(t.id) AS track_count - FROM furumusic__release_artist ra - JOIN furumusic__release r ON r.id = ra.release_id AND r.is_hidden = false - LEFT JOIN furumusic__track t ON t.release_id = r.id AND t.is_hidden = false - WHERE ra.position = 0 - GROUP BY ra.artist_id + LEFT JOIN ( + SELECT appearance.artist_id, + COUNT(DISTINCT appearance.release_id) FILTER (WHERE appearance.is_primary_release_artist) AS release_count, + COUNT(DISTINCT appearance.track_id) AS track_count + FROM ( + SELECT ta.artist_id, + t.id AS track_id, + r.id AS release_id, + primary_release.artist_id IS NOT NULL AS is_primary_release_artist + FROM furumusic__track_artist ta + JOIN furumusic__track t ON t.id = ta.track_id AND t.is_hidden = false + JOIN furumusic__release r ON r.id = t.release_id AND r.is_hidden = false + LEFT JOIN furumusic__release_artist primary_release + ON primary_release.release_id = r.id + AND primary_release.artist_id = ta.artist_id + AND primary_release.position = 0 + ) appearance + GROUP BY appearance.artist_id ) s ON s.artist_id = a.id WHERE a.is_hidden = false - ORDER BY s.release_count DESC, s.track_count DESC, a.name_sort + ORDER BY (COALESCE(s.release_count, 0) > 0) DESC, + COALESCE(s.release_count, 0) DESC, + COALESCE(s.track_count, 0) DESC, + a.name_sort LIMIT $1 OFFSET $2"#, ) .bind(per_page as i64) @@ -2887,12 +2906,14 @@ async fn artists_handler( track_count: r.track_count, }) .collect(); + let has_more = offset + (items.len() as i64) < total_row.count; Json(Paginated { items, total: total_row.count, page, per_page, + has_more, }) .into_response() } @@ -3480,10 +3501,7 @@ async fn build_track_items( .collect()) } -async fn load_track_items_by_ids( - pool: &sqlx::PgPool, - ids: &[i64], -) -> cot::Result> { +async fn load_track_items_by_ids(pool: &sqlx::PgPool, ids: &[i64]) -> cot::Result> { if ids.is_empty() { return Ok(Vec::new()); } @@ -4073,7 +4091,10 @@ async fn devices_command_handler( } fn stamp_jam_queue_tracks(payload: &mut serde_json::Value, user_id: i64, user_name: &str) { - let Some(tracks) = payload.get_mut("tracks").and_then(serde_json::Value::as_array_mut) else { + let Some(tracks) = payload + .get_mut("tracks") + .and_then(serde_json::Value::as_array_mut) + else { return; }; for track in tracks { @@ -5792,7 +5813,8 @@ async fn build_track_radio_ids( let artist_ids = track_primary_artist_ids(pool, track_id).await?; let remaining = PLAYER_RADIO_TRACK_LIMIT.saturating_sub(ids.len()) as i64; - let fallback_ids = fallback_radio_track_ids(pool, user_id, &artist_ids, &ids, remaining).await?; + let fallback_ids = + fallback_radio_track_ids(pool, user_id, &artist_ids, &ids, remaining).await?; append_unique_track_ids(&mut ids, fallback_ids, PLAYER_RADIO_TRACK_LIMIT); Ok(Some(ids)) @@ -5848,7 +5870,8 @@ async fn build_release_radio_ids( let artist_ids = release_primary_artist_ids(pool, release_id).await?; let remaining = PLAYER_RADIO_TRACK_LIMIT.saturating_sub(ids.len()) as i64; - let fallback_ids = fallback_radio_track_ids(pool, user_id, &artist_ids, &ids, remaining).await?; + let fallback_ids = + fallback_radio_track_ids(pool, user_id, &artist_ids, &ids, remaining).await?; append_unique_track_ids(&mut ids, fallback_ids, PLAYER_RADIO_TRACK_LIMIT); Ok(Some(ids)) @@ -6751,22 +6774,24 @@ impl App for PlayerApp { { let pool = Arc::clone(&pool); let pool_config = Arc::clone(&pool_config); - get(move |session: Session, db: Database, path: Path| { - let pool = Arc::clone(&pool); - let pool_config = Arc::clone(&pool_config); - async move { - let pg_pool = pool - .get_or_init(|| async { - sqlx::postgres::PgPoolOptions::new() - .max_connections(5) - .connect(&pool_config.database_url) - .await - .expect("player pool") - }) - .await; - radio_handler(session, db, pg_pool, path).await - } - }) + get( + move |session: Session, db: Database, path: Path| { + let pool = Arc::clone(&pool); + let pool_config = Arc::clone(&pool_config); + async move { + let pg_pool = pool + .get_or_init(|| async { + sqlx::postgres::PgPoolOptions::new() + .max_connections(5) + .connect(&pool_config.database_url) + .await + .expect("player pool") + }) + .await; + radio_handler(session, db, pg_pool, path).await + } + }, + ) }, "player_radio", ), diff --git a/templates/admin/v2.html b/templates/admin/v2.html index 598964a..2c12405 100644 --- a/templates/admin/v2.html +++ b/templates/admin/v2.html @@ -1434,7 +1434,7 @@ tbody tr:hover {