ab/furumusic
1
0
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases Activity

Update metadata jobs and player library

Browse Source
This commit is contained in:
Ultradesu
2026-06-03 03:39:16 +03:00
parent d2a8f301b8
commit 1e1453e465
16 changed files with 1803 additions and 159 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+1 -1
View File
@@ -1418,7 +1418,7 @@ checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c"
[[package]]
name = "furumusic"
version = "0.2.14"
version = "0.2.15"
dependencies = [
"anyhow",
"async-trait",
src/admin/mod.rs
+26
View File
@@ -357,6 +357,32 @@ impl App for AdminApp {
}),
"admin_v2_metadata_backfill_run_options",
),
Route::with_handler_and_name(
"/v2/api/jobs/artwork_backfill/run-options",
cot::router::method::post({
let pool = Arc::clone(&pool);
let pool_config = Arc::clone(&pool_config);
move |session: Session,
db: Database,
json: Json<v2::ArtworkBackfillRunRequest>| {
let pool = Arc::clone(&pool);
let pool_config = Arc::clone(&pool_config);
async move {
let pg_pool = pool
.get_or_init(|| async {
sqlx::postgres::PgPoolOptions::new()
.max_connections(5)
.connect(&pool_config.database_url)
.await
.expect("admin pool")
})
.await;
v2::run_artwork_backfill(session, db, pg_pool, json).await
}
}
}),
"admin_v2_artwork_backfill_run_options",
),
Route::with_handler_and_name(
"/v2/api/jobs/{name}/run",
cot::router::method::post({
src/admin/v2.rs
+62
View File
@@ -83,10 +83,18 @@ pub struct MetadataBackfillRunRequest {
local_genres: bool,
#[serde(default = "default_true")]
lastfm_tags: bool,
#[serde(default = "default_true")]
musicbrainz_tags: bool,
#[serde(default)]
overwrite: bool,
}
#[derive(Debug, Deserialize)]
pub struct ArtworkBackfillRunRequest {
#[serde(default)]
overwrite_existing: bool,
}
#[derive(Debug, Deserialize)]
pub(super) struct UpdateLibraryItemRequest {
kind: String,
@@ -296,6 +304,7 @@ struct MetadataTagDto {
struct ReviewPageDto {
items: Vec<ReviewDto>,
total: i64,
total_all: i64,
limit: i64,
offset: i64,
status: Option<String>,
@@ -1076,6 +1085,7 @@ pub async fn run_metadata_backfill(
duration_seconds: body.duration_seconds,
local_genres: body.local_genres,
lastfm_tags: body.lastfm_tags,
musicbrainz_tags: body.musicbrainz_tags,
overwrite: body.overwrite,
};
if !options.any_field() {
@@ -1123,6 +1133,56 @@ pub async fn run_metadata_backfill(
Json(JobRunStartedDto { ok: true, run_id }).into_response()
}
pub async fn run_artwork_backfill(
session: Session,
db: Database,
pool: &PgPool,
Json(body): Json<ArtworkBackfillRunRequest>,
) -> cot::Result<cot::response::Response> {
if let Err(response) = require_admin_json(&session, &db).await {
return Ok(response);
}
let options = crate::jobs::artwork_backfill::ArtworkBackfillOptions {
overwrite_existing: body.overwrite_existing,
};
let mut run = JobRun::create_running(&db, "artwork_backfill", "manual")
.await
.map_err(|e| cot::Error::internal(format!("failed to create job run: {e}")))?;
let run_id = run.id_val();
let (live_config, _) = AppConfig::load_with_db(&db).await;
let db_for_task = db.clone();
let pool_for_task = pool.clone();
tokio::spawn(async move {
let start = std::time::Instant::now();
let ctx = scheduler::JobContext {
config: std::sync::Arc::new(live_config),
db: db_for_task.clone(),
pool: pool_for_task.clone(),
run_id,
registry: std::sync::Arc::new(JobRegistry::new()),
};
let mut log = scheduler::JobLog::with_live_flush(pool_for_task.clone(), run_id);
let result = crate::jobs::artwork_backfill::run_with_options(&ctx, &mut log, options).await;
let duration_ms = start.elapsed().as_millis() as i64;
match result {
Ok(()) => {
let _ = run
.set_completed(&db_for_task, duration_ms, &log.output())
.await;
}
Err(err) => {
let _ = run
.set_failed(&db_for_task, duration_ms, &log.output(), &err.to_string())
.await;
}
}
});
Json(JobRunStartedDto { ok: true, run_id }).into_response()
}
pub async fn toggle_job(
session: Session,
db: Database,
@@ -2109,6 +2169,7 @@ async fn load_review_page(pool: &PgPool, query: ReviewsQuery) -> anyhow::Result<
let search_pattern = search.as_ref().map(|s| format!("%{s}%"));
let total = count_reviews(pool, status.clone(), search_pattern.clone()).await?;
let total_all = count_reviews(pool, None, search_pattern.clone()).await?;
let status_counts = load_review_status_counts(pool, None, search_pattern.clone()).await?;
let mut qb = QueryBuilder::<Postgres>::new(
@@ -2134,6 +2195,7 @@ async fn load_review_page(pool: &PgPool, query: ReviewsQuery) -> anyhow::Result<
Ok(ReviewPageDto {
items,
total,
total_all,
limit,
offset,
status,
src/admin/views.rs
+1
View File
@@ -1472,6 +1472,7 @@ pub async fn metadata_backfill_run(
duration_seconds: data.duration_seconds.is_some(),
local_genres: data.local_genres.is_some(),
lastfm_tags: data.lastfm_tags.is_some(),
musicbrainz_tags: true,
overwrite: data.mode.as_deref() == Some("overwrite"),
};
src/i18n/phrases.rs
+2
View File
@@ -272,6 +272,7 @@ translations! {
// Player UI
player_library: "Library" , "Библиотека";
player_artists: "Artists" , "Артисты";
player_global_library: "Global" , "Global";
player_release: "Release" , "Релиз";
player_releases: "Releases" , "Релизы";
player_tracks: "Tracks" , "Треки";
@@ -479,6 +480,7 @@ translations! {
player_seen: "seen" , "видели";
player_eta: "eta" , "осталось";
player_loading_history: "Loading history..." , "Загрузка истории...";
player_loading_more: "Loading more..." , "Загружаю ещё...";
player_failed_load_history: "Failed to load history" , "Не удалось загрузить историю";
player_total_plays: "total plays" , "прослушиваний всего";
player_play_history: "Play history" , "История прослушиваний";
src/jobs/artwork_backfill.rs
+576 -77
View File
@@ -15,12 +15,19 @@ pub struct ArtworkBackfillJob;
const LASTFM_REQUEST_DELAY: std::time::Duration = std::time::Duration::from_millis(1200);
const MAX_LASTFM_RELEASE_LOOKUPS: i64 = 200;
const MAX_LASTFM_ARTIST_LOOKUPS: i64 = 200;
const MAX_CAA_RELEASE_LOOKUPS: i64 = 200;
#[derive(Debug, Clone, Copy)]
pub struct ArtworkBackfillOptions {
pub overwrite_existing: bool,
}
#[derive(Debug, sqlx::FromRow)]
struct ReleaseCandidate {
id: i64,
title: String,
artist_name: Option<String>,
track_title: Option<String>,
}
#[derive(Debug, sqlx::FromRow)]
@@ -36,6 +43,13 @@ struct ArtworkRefCandidate {
file_path: Option<String>,
}
#[derive(Debug, sqlx::FromRow)]
struct ArtistImageRepairCandidate {
id: i64,
name: String,
media_file_id: i64,
}
#[derive(Debug, Deserialize)]
struct LastfmAlbumResponse {
album: Option<LastfmImageContainer>,
@@ -85,7 +99,10 @@ struct ArtworkStats {
broken_release_refs_cleared: u64,
broken_track_refs_cleared: u64,
broken_artist_refs_cleared: u64,
remote_artist_album_fallback_refs_cleared: u64,
release_local_assigned: u64,
release_caa_assigned: u64,
release_caa_not_found: u64,
release_lastfm_assigned: u64,
release_lastfm_not_found: u64,
release_skipped_no_audio: u64,
@@ -114,66 +131,107 @@ impl Job for ArtworkBackfillJob {
}
async fn run(&self, ctx: &JobContext, log: &mut JobLog) -> anyhow::Result<()> {
let storage_dir = ctx.config.agent_storage_dir.trim();
if storage_dir.is_empty() {
log.warn("agent_storage_dir is not configured, skipping artwork backfill");
return Ok(());
}
let client = Client::builder()
.user_agent(format!(
"furumusic-artwork-backfill/{}",
env!("CARGO_PKG_VERSION")
))
.timeout(std::time::Duration::from_secs(20))
.build()?;
let mut stats = ArtworkStats::default();
let normalized_paths =
crate::media_paths::normalize_media_file_paths(&ctx.pool, storage_dir).await?;
if normalized_paths > 0 {
log.info(&format!(
"Media path normalization pass: rewrote {normalized_paths} media file path(s) to relative storage paths"
));
} else {
log.info("Media path normalization pass: all media file paths are already relative");
}
repair_missing_artwork_refs(ctx, log, storage_dir, &mut stats).await?;
backfill_release_local(ctx, log, storage_dir, &mut stats).await?;
let api_key = ctx.config.lastfm_api_key.trim();
if api_key.is_empty() {
log.warn("lastfm_api_key is not configured; skipping Last.fm artwork fallback");
} else {
backfill_release_lastfm(ctx, log, storage_dir, api_key, &client, &mut stats).await?;
backfill_artist_lastfm(ctx, log, storage_dir, api_key, &client, &mut stats).await?;
}
backfill_artist_album_fallbacks(ctx, log, &mut stats).await?;
repair_cover_variants(ctx, log, storage_dir, &mut stats).await?;
log.info(&format!(
"Artwork backfill complete: broken_release_refs_cleared={}, broken_track_refs_cleared={}, broken_artist_refs_cleared={}, release_local_assigned={}, release_lastfm_assigned={}, release_lastfm_not_found={}, release_skipped_no_audio={}, artist_lastfm_assigned={}, artist_lastfm_not_found={}, artist_album_fallback_assigned={}, variants_created={}, variants_unchanged={}, variants_missing_original={}, failed={}",
stats.broken_release_refs_cleared,
stats.broken_track_refs_cleared,
stats.broken_artist_refs_cleared,
stats.release_local_assigned,
stats.release_lastfm_assigned,
stats.release_lastfm_not_found,
stats.release_skipped_no_audio,
stats.artist_lastfm_assigned,
stats.artist_lastfm_not_found,
stats.artist_album_fallback_assigned,
stats.variants_created,
stats.variants_unchanged,
stats.variants_missing_original,
stats.failed
));
Ok(())
run_with_options(
ctx,
log,
ArtworkBackfillOptions {
overwrite_existing: false,
},
)
.await
}
}
pub async fn run_with_options(
ctx: &JobContext,
log: &mut JobLog,
options: ArtworkBackfillOptions,
) -> anyhow::Result<()> {
let storage_dir = ctx.config.agent_storage_dir.trim();
if storage_dir.is_empty() {
log.warn("agent_storage_dir is not configured, skipping artwork backfill");
return Ok(());
}
log.info(&format!(
"Artwork backfill options: mode={}",
if options.overwrite_existing {
"overwrite_existing"
} else {
"missing_only"
}
));
let client = Client::builder()
.user_agent(format!(
"furumusic-artwork-backfill/{}",
env!("CARGO_PKG_VERSION")
))
.timeout(std::time::Duration::from_secs(20))
.build()?;
let musicbrainz_client =
crate::jobs::musicbrainz::MusicBrainzClient::new("furumusic-artwork-backfill")?;
let mut stats = ArtworkStats::default();
let normalized_paths =
crate::media_paths::normalize_media_file_paths(&ctx.pool, storage_dir).await?;
if normalized_paths > 0 {
log.info(&format!(
"Media path normalization pass: rewrote {normalized_paths} media file path(s) to relative storage paths"
));
} else {
log.info("Media path normalization pass: all media file paths are already relative");
}
repair_missing_artwork_refs(ctx, log, storage_dir, &mut stats).await?;
repair_remote_artist_album_fallback_refs(ctx, log, &mut stats).await?;
backfill_release_local(ctx, log, storage_dir, options, &mut stats).await?;
backfill_release_coverartarchive(
ctx,
log,
storage_dir,
&musicbrainz_client,
options,
&mut stats,
)
.await?;
let api_key = ctx.config.lastfm_api_key.trim();
if api_key.is_empty() {
log.warn("lastfm_api_key is not configured; skipping Last.fm artwork fallback");
} else {
backfill_release_lastfm(ctx, log, storage_dir, api_key, &client, options, &mut stats)
.await?;
backfill_artist_lastfm(ctx, log, storage_dir, api_key, &client, options, &mut stats)
.await?;
}
backfill_artist_album_fallbacks(ctx, log, options, &mut stats).await?;
repair_cover_variants(ctx, log, storage_dir, &mut stats).await?;
log.info(&format!(
"Artwork backfill complete: broken_release_refs_cleared={}, broken_track_refs_cleared={}, broken_artist_refs_cleared={}, remote_artist_album_fallback_refs_cleared={}, release_local_assigned={}, release_caa_assigned={}, release_caa_not_found={}, release_lastfm_assigned={}, release_lastfm_not_found={}, release_skipped_no_audio={}, artist_lastfm_assigned={}, artist_lastfm_not_found={}, artist_album_fallback_assigned={}, variants_created={}, variants_unchanged={}, variants_missing_original={}, failed={}",
stats.broken_release_refs_cleared,
stats.broken_track_refs_cleared,
stats.broken_artist_refs_cleared,
stats.remote_artist_album_fallback_refs_cleared,
stats.release_local_assigned,
stats.release_caa_assigned,
stats.release_caa_not_found,
stats.release_lastfm_assigned,
stats.release_lastfm_not_found,
stats.release_skipped_no_audio,
stats.artist_lastfm_assigned,
stats.artist_lastfm_not_found,
stats.artist_album_fallback_assigned,
stats.variants_created,
stats.variants_unchanged,
stats.variants_missing_original,
stats.failed
));
Ok(())
}
async fn repair_missing_artwork_refs(
ctx: &JobContext,
log: &mut JobLog,
@@ -186,6 +244,71 @@ async fn repair_missing_artwork_refs(
Ok(())
}
async fn repair_remote_artist_album_fallback_refs(
ctx: &JobContext,
log: &mut JobLog,
stats: &mut ArtworkStats,
) -> anyhow::Result<()> {
let rows = sqlx::query_as::<_, ArtistImageRepairCandidate>(
r#"SELECT DISTINCT a.id,
a.name::text AS name,
a.image_file_id AS media_file_id
FROM furumusic__artist a
JOIN furumusic__media_file mf ON mf.id = a.image_file_id
WHERE a.image_file_id IS NOT NULL
AND a.is_hidden = false
AND mf.file_path NOT LIKE '%/__artist_image__/%'
AND EXISTS (
SELECT 1
FROM furumusic__release r
JOIN furumusic__artwork_lookup_state s
ON s.entity_kind = 'release'
AND s.entity_id = r.id
AND s.source IN ('lastfm', 'coverartarchive')
AND s.status = 'found'
WHERE r.cover_file_id = a.image_file_id
AND r.is_hidden = false
)
ORDER BY a.id"#,
)
.fetch_all(&ctx.pool)
.await?;
if rows.is_empty() {
return Ok(());
}
log.info(&format!(
"Artist fallback repair pass: clearing {} remote release cover fallback(s)",
rows.len()
));
for row in rows {
let result = sqlx::query(
r#"UPDATE furumusic__artist
SET image_file_id = NULL,
updated_at = $3
WHERE id = $1
AND image_file_id = $2"#,
)
.bind(row.id)
.bind(row.media_file_id)
.bind(now_iso())
.execute(&ctx.pool)
.await?;
if result.rows_affected() > 0 {
stats.remote_artist_album_fallback_refs_cleared += 1;
log.warn(&format!(
"Artist {} \"{}\": cleared remote release cover fallback (media_file_id={})",
row.id, row.name, row.media_file_id
));
}
}
Ok(())
}
async fn repair_missing_release_cover_refs(
ctx: &JobContext,
log: &mut JobLog,
@@ -360,6 +483,7 @@ async fn backfill_release_local(
ctx: &JobContext,
log: &mut JobLog,
storage_dir: &str,
options: ArtworkBackfillOptions,
stats: &mut ArtworkStats,
) -> anyhow::Result<()> {
let releases = sqlx::query_as::<_, ReleaseCandidate>(
@@ -372,17 +496,26 @@ async fn backfill_release_local(
WHERE ra.release_id = r.id
ORDER BY ra.position
LIMIT 1
) AS artist_name
) AS artist_name,
(
SELECT t.title::text
FROM furumusic__track t
WHERE t.release_id = r.id
AND t.is_hidden = false
ORDER BY t.disc_number NULLS LAST, t.track_number NULLS LAST, t.id
LIMIT 1
) AS track_title
FROM furumusic__release r
WHERE r.cover_file_id IS NULL
WHERE ($1 OR r.cover_file_id IS NULL)
AND r.is_hidden = false
ORDER BY r.id"#,
)
.bind(options.overwrite_existing)
.fetch_all(&ctx.pool)
.await?;
if releases.is_empty() {
log.info("Release local artwork pass: all visible releases already have covers");
log.info("Release local artwork pass: no eligible releases need local cover lookup");
return Ok(());
}
log.info(&format!(
@@ -451,7 +584,7 @@ async fn backfill_release_local(
.await
{
Ok(cover_file_id) => {
cover_art::assign_cover_to_release(&ctx.pool, release.id, cover_file_id).await?;
assign_cover_to_release(&ctx.pool, release.id, cover_file_id, options).await?;
stats.release_local_assigned += 1;
log.info(&format!(
"Release {} \"{}\": assigned local cover from {source_desc}",
@@ -471,12 +604,12 @@ async fn backfill_release_local(
Ok(())
}
async fn backfill_release_lastfm(
async fn backfill_release_coverartarchive(
ctx: &JobContext,
log: &mut JobLog,
storage_dir: &str,
api_key: &str,
client: &Client,
client: &crate::jobs::musicbrainz::MusicBrainzClient,
options: ArtworkBackfillOptions,
stats: &mut ArtworkStats,
) -> anyhow::Result<()> {
let failed_cutoff = cutoff_iso(1);
@@ -502,25 +635,285 @@ async fn backfill_release_lastfm(
ORDER BY t.disc_number NULLS LAST, t.track_number NULLS LAST, ta.position
LIMIT 1
)
) AS artist_name
) AS artist_name,
(
SELECT t.title::text
FROM furumusic__track t
WHERE t.release_id = r.id
AND t.is_hidden = false
ORDER BY t.disc_number NULLS LAST, t.track_number NULLS LAST, t.id
LIMIT 1
) AS track_title
FROM furumusic__release r
LEFT JOIN furumusic__artwork_lookup_state s
ON s.entity_kind = 'release'
AND s.entity_id = r.id
AND s.source = 'lastfm'
WHERE r.cover_file_id IS NULL
AND s.source = 'coverartarchive'
WHERE ($3 OR r.cover_file_id IS NULL)
AND r.is_hidden = false
AND (
s.entity_id IS NULL
$3
OR s.entity_id IS NULL
OR s.status = 'failed' AND s.last_attempt_at < $1
OR s.status = 'not_found' AND (s.attempt_count < 3 OR s.last_attempt_at < $2)
OR s.status = 'found' AND s.last_attempt_at < $1
)
ORDER BY s.last_attempt_at NULLS FIRST, r.id
LIMIT $3"#,
LIMIT $4"#,
)
.bind(&failed_cutoff)
.bind(&not_found_cutoff)
.bind(options.overwrite_existing)
.bind(MAX_CAA_RELEASE_LOOKUPS)
.fetch_all(&ctx.pool)
.await?;
if releases.is_empty() {
log.info("Release Cover Art Archive pass: no eligible releases need lookup");
return Ok(());
}
log.info(&format!(
"Release Cover Art Archive pass: looking up {} release(s)",
releases.len()
));
for (index, release) in releases.iter().enumerate() {
let Some(artist_name) = release
.artist_name
.as_deref()
.map(str::trim)
.filter(|value| !value.is_empty())
else {
stats.release_caa_not_found += 1;
record_coverartarchive_lookup_state(
&ctx.pool,
"release",
release.id,
"not_found",
Some("release has no primary artist for MusicBrainz lookup"),
None,
)
.await?;
continue;
};
log.info(&format!(
"Release Cover Art Archive {}/{}: release {} \"{}\" by \"{}\"",
index + 1,
releases.len(),
release.id,
release.title,
artist_name
));
let release_mbid = crate::jobs::musicbrainz::load_or_search_release_mbid(
&ctx.pool,
client,
release.id,
artist_name,
&release.title,
release.track_title.as_deref(),
)
.await;
let (release_mbid, release_group_mbid) = match release_mbid {
Ok(value) => value,
Err(err) => {
stats.failed += 1;
record_coverartarchive_lookup_state(
&ctx.pool,
"release",
release.id,
"failed",
Some(&err.to_string()),
None,
)
.await?;
log.warn(&format!(
"Release {} \"{}\": MusicBrainz lookup for cover art failed: {err}",
release.id, release.title
));
continue;
}
};
if release_mbid.is_none() && release_group_mbid.is_none() {
stats.release_caa_not_found += 1;
record_coverartarchive_lookup_state(
&ctx.pool,
"release",
release.id,
"not_found",
Some("MusicBrainz release match not found"),
None,
)
.await?;
continue;
}
match client
.fetch_cover_art_front_url(release_mbid.as_deref(), release_group_mbid.as_deref())
.await
{
Ok(Some(image_url)) => {
match download_remote_cover(client.http_client(), &image_url).await {
Ok(cover) => match cover_art::save_cover_to_storage(
&ctx.db,
&ctx.pool,
storage_dir,
artist_name,
&release.title,
&cover,
)
.await
{
Ok(cover_file_id) => {
assign_cover_to_release(&ctx.pool, release.id, cover_file_id, options)
.await?;
record_coverartarchive_lookup_state(
&ctx.pool,
"release",
release.id,
"found",
None,
Some(&image_url),
)
.await?;
stats.release_caa_assigned += 1;
log.info(&format!(
"Release {} \"{}\": assigned Cover Art Archive cover",
release.id, release.title
));
}
Err(err) => {
stats.failed += 1;
record_coverartarchive_lookup_state(
&ctx.pool,
"release",
release.id,
"failed",
Some(&err.to_string()),
Some(&image_url),
)
.await?;
log.warn(&format!(
"Release {} \"{}\": failed to save Cover Art Archive cover: {err}",
release.id, release.title
));
}
},
Err(err) => {
stats.failed += 1;
record_coverartarchive_lookup_state(
&ctx.pool,
"release",
release.id,
"failed",
Some(&err.to_string()),
Some(&image_url),
)
.await?;
log.warn(&format!(
"Release {} \"{}\": failed to download Cover Art Archive cover: {err}",
release.id, release.title
));
}
}
}
Ok(None) => {
stats.release_caa_not_found += 1;
record_coverartarchive_lookup_state(
&ctx.pool,
"release",
release.id,
"not_found",
None,
None,
)
.await?;
}
Err(err) => {
stats.failed += 1;
record_coverartarchive_lookup_state(
&ctx.pool,
"release",
release.id,
"failed",
Some(&err.to_string()),
None,
)
.await?;
log.warn(&format!(
"Release {} \"{}\": Cover Art Archive lookup failed: {err}",
release.id, release.title
));
}
}
}
Ok(())
}
async fn backfill_release_lastfm(
ctx: &JobContext,
log: &mut JobLog,
storage_dir: &str,
api_key: &str,
client: &Client,
options: ArtworkBackfillOptions,
stats: &mut ArtworkStats,
) -> anyhow::Result<()> {
let failed_cutoff = cutoff_iso(1);
let not_found_cutoff = cutoff_iso(30);
let releases = sqlx::query_as::<_, ReleaseCandidate>(
r#"SELECT r.id,
r.title::text AS title,
COALESCE(
(
SELECT a.name::text
FROM furumusic__release_artist ra
JOIN furumusic__artist a ON a.id = ra.artist_id
WHERE ra.release_id = r.id
ORDER BY ra.position
LIMIT 1
),
(
SELECT a.name::text
FROM furumusic__track t
JOIN furumusic__track_artist ta ON ta.track_id = t.id
JOIN furumusic__artist a ON a.id = ta.artist_id
WHERE t.release_id = r.id AND ta.role <> 'featuring'
ORDER BY t.disc_number NULLS LAST, t.track_number NULLS LAST, ta.position
LIMIT 1
)
) AS artist_name,
(
SELECT t.title::text
FROM furumusic__track t
WHERE t.release_id = r.id
AND t.is_hidden = false
ORDER BY t.disc_number NULLS LAST, t.track_number NULLS LAST, t.id
LIMIT 1
) AS track_title
FROM furumusic__release r
LEFT JOIN furumusic__artwork_lookup_state s
ON s.entity_kind = 'release'
AND s.entity_id = r.id
AND s.source = 'lastfm'
WHERE ($3 OR r.cover_file_id IS NULL)
AND r.is_hidden = false
AND (
$3
OR s.entity_id IS NULL
OR s.status = 'failed' AND s.last_attempt_at < $1
OR s.status = 'not_found' AND (s.attempt_count < 3 OR s.last_attempt_at < $2)
OR s.status = 'found' AND s.last_attempt_at < $1
)
ORDER BY s.last_attempt_at NULLS FIRST, r.id
LIMIT $4"#,
)
.bind(&failed_cutoff)
.bind(&not_found_cutoff)
.bind(options.overwrite_existing)
.bind(MAX_LASTFM_RELEASE_LOOKUPS)
.fetch_all(&ctx.pool)
.await?;
@@ -580,7 +973,7 @@ async fn backfill_release_lastfm(
.await
{
Ok(cover_file_id) => {
cover_art::assign_cover_to_release(&ctx.pool, release.id, cover_file_id)
assign_cover_to_release(&ctx.pool, release.id, cover_file_id, options)
.await?;
record_lookup_state(
&ctx.pool,
@@ -680,12 +1073,37 @@ async fn backfill_release_lastfm(
Ok(())
}
async fn assign_cover_to_release(
pool: &sqlx::PgPool,
release_id: i64,
cover_file_id: i64,
options: ArtworkBackfillOptions,
) -> anyhow::Result<()> {
if options.overwrite_existing {
sqlx::query(
r#"UPDATE furumusic__release
SET cover_file_id = $1,
updated_at = $3
WHERE id = $2"#,
)
.bind(cover_file_id)
.bind(release_id)
.bind(now_iso())
.execute(pool)
.await?;
} else {
cover_art::assign_cover_to_release(pool, release_id, cover_file_id).await?;
}
Ok(())
}
async fn backfill_artist_lastfm(
ctx: &JobContext,
log: &mut JobLog,
storage_dir: &str,
api_key: &str,
client: &Client,
options: ArtworkBackfillOptions,
stats: &mut ArtworkStats,
) -> anyhow::Result<()> {
let failed_cutoff = cutoff_iso(1);
@@ -699,21 +1117,25 @@ async fn backfill_artist_lastfm(
AND s.entity_id = a.id
AND s.source = 'lastfm'
WHERE (
$3
OR
a.image_file_id IS NULL
OR mf.file_path NOT LIKE '%/__artist_image__/%'
)
AND a.is_hidden = false
AND (
s.entity_id IS NULL
$3
OR s.entity_id IS NULL
OR s.status = 'failed' AND s.last_attempt_at < $1
OR s.status = 'not_found' AND (s.attempt_count < 3 OR s.last_attempt_at < $2)
OR s.status = 'found' AND s.last_attempt_at < $1
)
ORDER BY s.last_attempt_at NULLS FIRST, a.id
LIMIT $3"#,
LIMIT $4"#,
)
.bind(&failed_cutoff)
.bind(&not_found_cutoff)
.bind(options.overwrite_existing)
.bind(MAX_LASTFM_ARTIST_LOOKUPS)
.fetch_all(&ctx.pool)
.await?;
@@ -755,6 +1177,8 @@ async fn backfill_artist_lastfm(
updated_at = $3
WHERE id = $2
AND (
$4
OR
image_file_id IS NULL
OR EXISTS (
SELECT 1
@@ -767,6 +1191,7 @@ async fn backfill_artist_lastfm(
.bind(image_file_id)
.bind(artist.id)
.bind(now_iso())
.bind(options.overwrite_existing)
.execute(&ctx.pool)
.await?;
record_lookup_state(
@@ -826,7 +1251,7 @@ async fn backfill_artist_lastfm(
artist.id, artist.name
));
stats.artist_lastfm_not_found += 1;
match assign_artist_album_fallback(ctx, artist.id).await {
match assign_artist_album_fallback(ctx, artist.id, options).await {
Ok(Some(media_file_id)) => {
stats.artist_album_fallback_assigned += 1;
log.info(&format!(
@@ -892,31 +1317,53 @@ async fn backfill_artist_lastfm(
async fn backfill_artist_album_fallbacks(
ctx: &JobContext,
log: &mut JobLog,
options: ArtworkBackfillOptions,
stats: &mut ArtworkStats,
) -> anyhow::Result<()> {
let artists = sqlx::query_as::<_, ArtistCandidate>(
r#"SELECT a.id, a.name::text AS name
FROM furumusic__artist a
WHERE a.image_file_id IS NULL
WHERE ($1 OR a.image_file_id IS NULL)
AND a.is_hidden = false
AND EXISTS (
SELECT 1
FROM furumusic__release_artist ra
JOIN furumusic__release r ON r.id = ra.release_id
JOIN furumusic__media_file mf ON mf.id = r.cover_file_id
WHERE ra.artist_id = a.id
AND r.cover_file_id IS NOT NULL
AND mf.file_type = 'cover_art'
AND r.is_hidden = false
AND NOT EXISTS (
SELECT 1
FROM furumusic__artwork_lookup_state s
WHERE s.entity_kind = 'release'
AND s.entity_id = r.id
AND s.source IN ('lastfm', 'coverartarchive')
AND s.status = 'found'
)
UNION
SELECT 1
FROM furumusic__track_artist ta
JOIN furumusic__track t ON t.id = ta.track_id
JOIN furumusic__release r ON r.id = t.release_id
JOIN furumusic__media_file mf ON mf.id = r.cover_file_id
WHERE ta.artist_id = a.id
AND r.cover_file_id IS NOT NULL
AND mf.file_type = 'cover_art'
AND r.is_hidden = false
AND NOT EXISTS (
SELECT 1
FROM furumusic__artwork_lookup_state s
WHERE s.entity_kind = 'release'
AND s.entity_id = r.id
AND s.source IN ('lastfm', 'coverartarchive')
AND s.status = 'found'
)
)
ORDER BY a.id"#,
)
.bind(options.overwrite_existing)
.fetch_all(&ctx.pool)
.await?;
@@ -931,7 +1378,7 @@ async fn backfill_artist_album_fallbacks(
));
for artist in artists {
match assign_artist_album_fallback(ctx, artist.id).await {
match assign_artist_album_fallback(ctx, artist.id, options).await {
Ok(Some(media_file_id)) => {
stats.artist_album_fallback_assigned += 1;
log.info(&format!(
@@ -1016,6 +1463,7 @@ async fn repair_cover_variants(
async fn assign_artist_album_fallback(
ctx: &JobContext,
artist_id: i64,
options: ArtworkBackfillOptions,
) -> anyhow::Result<Option<i64>> {
let media_file_id: Option<i64> = sqlx::query_scalar(
r#"SELECT media_file_id
@@ -1023,17 +1471,37 @@ async fn assign_artist_album_fallback(
SELECT DISTINCT r.cover_file_id AS media_file_id
FROM furumusic__release r
JOIN furumusic__release_artist ra ON ra.release_id = r.id
JOIN furumusic__media_file mf ON mf.id = r.cover_file_id
WHERE ra.artist_id = $1
AND r.cover_file_id IS NOT NULL
AND mf.file_type = 'cover_art'
AND r.is_hidden = false
AND NOT EXISTS (
SELECT 1
FROM furumusic__artwork_lookup_state s
WHERE s.entity_kind = 'release'
AND s.entity_id = r.id
AND s.source IN ('lastfm', 'coverartarchive')
AND s.status = 'found'
)
UNION
SELECT DISTINCT r.cover_file_id AS media_file_id
FROM furumusic__release r
JOIN furumusic__track t ON t.release_id = r.id
JOIN furumusic__track_artist ta ON ta.track_id = t.id
JOIN furumusic__media_file mf ON mf.id = r.cover_file_id
WHERE ta.artist_id = $1
AND r.cover_file_id IS NOT NULL
AND mf.file_type = 'cover_art'
AND r.is_hidden = false
AND NOT EXISTS (
SELECT 1
FROM furumusic__artwork_lookup_state s
WHERE s.entity_kind = 'release'
AND s.entity_id = r.id
AND s.source IN ('lastfm', 'coverartarchive')
AND s.status = 'found'
)
) covers
ORDER BY random()
LIMIT 1"#,
@@ -1051,11 +1519,12 @@ async fn assign_artist_album_fallback(
SET image_file_id = $1,
updated_at = $3
WHERE id = $2
AND image_file_id IS NULL"#,
AND ($4 OR image_file_id IS NULL)"#,
)
.bind(media_file_id)
.bind(artist_id)
.bind(now_iso())
.bind(options.overwrite_existing)
.execute(&ctx.pool)
.await?;
@@ -1324,6 +1793,36 @@ async fn record_lookup_state(
Ok(())
}
async fn record_coverartarchive_lookup_state(
pool: &sqlx::PgPool,
entity_kind: &str,
entity_id: i64,
status: &str,
error: Option<&str>,
source_url: Option<&str>,
) -> anyhow::Result<()> {
sqlx::query(
r#"INSERT INTO furumusic__artwork_lookup_state
(entity_kind, entity_id, source, status, attempt_count, last_attempt_at, last_error, source_url)
VALUES ($1, $2, 'coverartarchive', $3, 1, $4, $5, $6)
ON CONFLICT (entity_kind, entity_id, source) DO UPDATE SET
status = EXCLUDED.status,
attempt_count = furumusic__artwork_lookup_state.attempt_count + 1,
last_attempt_at = EXCLUDED.last_attempt_at,
last_error = EXCLUDED.last_error,
source_url = EXCLUDED.source_url"#,
)
.bind(entity_kind)
.bind(entity_id)
.bind(status)
.bind(now_iso())
.bind(error)
.bind(source_url)
.execute(pool)
.await?;
Ok(())
}
async fn reset_lookup_state(
pool: &sqlx::PgPool,
entity_kind: &str,
@@ -1333,7 +1832,7 @@ async fn reset_lookup_state(
r#"DELETE FROM furumusic__artwork_lookup_state
WHERE entity_kind = $1
AND entity_id = $2
AND source = 'lastfm'"#,
AND source IN ('lastfm', 'coverartarchive')"#,
)
.bind(entity_kind)
.bind(entity_id)
src/jobs/metadata_backfill.rs
+337 -22
View File
@@ -15,6 +15,7 @@ pub struct MetadataBackfillOptions {
pub duration_seconds: bool,
pub local_genres: bool,
pub lastfm_tags: bool,
pub musicbrainz_tags: bool,
pub overwrite: bool,
}
@@ -26,6 +27,7 @@ impl MetadataBackfillOptions {
|| self.duration_seconds
|| self.local_genres
|| self.lastfm_tags
|| self.musicbrainz_tags
}
fn needs_file_scan(self) -> bool {
@@ -59,6 +61,7 @@ struct LastfmReleaseTagRow {
id: i64,
title: String,
artist_name: Option<String>,
track_title: Option<String>,
}
#[derive(sqlx::FromRow)]
@@ -84,6 +87,16 @@ struct LastfmTagStats {
failed: u64,
}
#[derive(Debug, Default)]
struct MusicBrainzTagStats {
considered: u64,
updated_entities: u64,
tags_saved: u64,
skipped_existing: u64,
not_found: u64,
failed: u64,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum LastfmTagPassResult {
Completed,
@@ -117,6 +130,7 @@ impl Job for MetadataBackfillJob {
duration_seconds: true,
local_genres: true,
lastfm_tags: true,
musicbrainz_tags: true,
overwrite: false,
},
)
@@ -143,10 +157,11 @@ pub async fn run_with_options(
let mut failed = 0u64;
log.info(&format!(
"Metadata backfill options: file_scan={}, local_genres={}, lastfm_tags={}, mode={}",
"Metadata backfill options: file_scan={}, local_genres={}, lastfm_tags={}, musicbrainz_tags={}, mode={}",
options.needs_file_scan(),
options.local_genres,
options.lastfm_tags,
options.musicbrainz_tags,
if options.overwrite {
"overwrite"
} else {
@@ -251,14 +266,9 @@ pub async fn run_with_options(
let mut changed_tags = false;
if options.local_genres {
if let (Some(track_id), Some(genre)) = (row.track_id, raw_meta.genre.as_deref()) {
let saved = save_track_tag_text(
&ctx.pool,
track_id,
genre,
"file",
options.overwrite,
)
.await?;
let saved =
save_track_tag_text(&ctx.pool, track_id, genre, "file", options.overwrite)
.await?;
if saved > 0 {
local_tags_updated += saved;
changed_tags = true;
@@ -312,14 +322,28 @@ pub async fn run_with_options(
LastfmTagStats::default()
};
let musicbrainz_stats = if options.musicbrainz_tags {
log.info("Starting MusicBrainz tag backfill");
backfill_musicbrainz_tags(ctx, log, options.overwrite).await?
} else {
log.info("MusicBrainz tag backfill disabled for this run");
MusicBrainzTagStats::default()
};
log.info(&format!(
"Metadata backfill complete: {scanned} scanned, {media_updated} media updated, {track_updated} tracks updated, {local_tags_updated} local tags saved, {unchanged} unchanged, {missing} missing, {failed} failed; Last.fm tags: considered={}, updated_entities={}, tags_saved={}, skipped_existing={}, not_found={}, failed={}",
"Metadata backfill complete: {scanned} scanned, {media_updated} media updated, {track_updated} tracks updated, {local_tags_updated} local tags saved, {unchanged} unchanged, {missing} missing, {failed} failed; Last.fm tags: considered={}, updated_entities={}, tags_saved={}, skipped_existing={}, not_found={}, failed={}; MusicBrainz tags: considered={}, updated_entities={}, tags_saved={}, skipped_existing={}, not_found={}, failed={}",
lastfm_stats.considered,
lastfm_stats.updated_entities,
lastfm_stats.tags_saved,
lastfm_stats.skipped_existing,
lastfm_stats.not_found,
lastfm_stats.failed,
musicbrainz_stats.considered,
musicbrainz_stats.updated_entities,
musicbrainz_stats.tags_saved,
musicbrainz_stats.skipped_existing,
musicbrainz_stats.not_found,
musicbrainz_stats.failed,
));
Ok(())
}
@@ -369,6 +393,259 @@ async fn backfill_lastfm_tags(
Ok(stats)
}
async fn backfill_musicbrainz_tags(
ctx: &JobContext,
log: &mut JobLog,
overwrite: bool,
) -> anyhow::Result<MusicBrainzTagStats> {
log.info("MusicBrainz tag backfill started");
let client = crate::jobs::musicbrainz::MusicBrainzClient::new("furumusic-metadata-backfill")?;
let mut stats = MusicBrainzTagStats::default();
backfill_musicbrainz_artist_tags(ctx, log, &client, overwrite, &mut stats).await?;
backfill_musicbrainz_release_tags(ctx, log, &client, overwrite, &mut stats).await?;
Ok(stats)
}
async fn backfill_musicbrainz_artist_tags(
ctx: &JobContext,
log: &mut JobLog,
client: &crate::jobs::musicbrainz::MusicBrainzClient,
overwrite: bool,
stats: &mut MusicBrainzTagStats,
) -> anyhow::Result<()> {
let rows = sqlx::query_as::<_, LastfmArtistTagRow>(
r#"SELECT DISTINCT a.id, a.name::text AS name
FROM furumusic__artist a
JOIN furumusic__track_artist ta ON ta.artist_id = a.id
JOIN furumusic__track t ON t.id = ta.track_id
WHERE a.is_hidden = false AND t.is_hidden = false
ORDER BY a.id"#,
)
.fetch_all(&ctx.pool)
.await?;
log.info(&format!(
"MusicBrainz artist tag pass: checking {} artist(s)",
rows.len()
));
let total = rows.len();
for (index, row) in rows.into_iter().enumerate() {
if should_log_lastfm_item(index + 1, total, 25) {
log.info(&format!(
"MusicBrainz artist tags {}/{}: artist {} \"{}\"",
index + 1,
total,
row.id,
row.name
));
}
if should_skip_source_entity(&ctx.pool, "artist", row.id, "musicbrainz", overwrite).await? {
stats.skipped_existing += 1;
continue;
}
stats.considered += 1;
let mbid =
match crate::jobs::musicbrainz::load_external_id(&ctx.pool, "artist", row.id, "artist")
.await?
{
Some(mbid) => Some(mbid),
None => match client.search_artist(&row.name).await {
Ok(Some(found)) => {
crate::jobs::musicbrainz::save_external_id(
&ctx.pool,
"artist",
row.id,
"artist",
&found.mbid,
found.score as f64 / 100.0,
)
.await?;
Some(found.mbid)
}
Ok(None) => None,
Err(err) => {
stats.failed += 1;
log.warn(&format!(
"MusicBrainz artist search failed for artist {} \"{}\": {err}",
row.id, row.name
));
None
}
},
};
let Some(mbid) = mbid else {
stats.not_found += 1;
continue;
};
match client.lookup_artist_tags(&mbid).await {
Ok(tags) if !tags.is_empty() => {
let tags = musicbrainz_tags_to_candidates(&tags);
match replace_entity_tags(&ctx.pool, "artist", row.id, &tags, "musicbrainz", false)
.await
{
Ok(saved) => {
stats.tags_saved += saved;
stats.updated_entities += 1;
}
Err(err) => {
stats.failed += 1;
log.warn(&format!(
"MusicBrainz artist tags save failed for artist {} \"{}\": {err}",
row.id, row.name
));
}
}
}
Ok(_) => {
stats.not_found += 1;
}
Err(err) => {
stats.failed += 1;
log.warn(&format!(
"MusicBrainz artist tags failed for artist {} \"{}\" mbid={}: {err}",
row.id, row.name, mbid
));
}
}
}
Ok(())
}
async fn backfill_musicbrainz_release_tags(
ctx: &JobContext,
log: &mut JobLog,
client: &crate::jobs::musicbrainz::MusicBrainzClient,
overwrite: bool,
stats: &mut MusicBrainzTagStats,
) -> anyhow::Result<()> {
let rows = sqlx::query_as::<_, LastfmReleaseTagRow>(
r#"SELECT r.id,
r.title::text AS title,
(
SELECT a.name::text
FROM furumusic__release_artist ra
JOIN furumusic__artist a ON a.id = ra.artist_id
WHERE ra.release_id = r.id
ORDER BY ra.position
LIMIT 1
) AS artist_name,
(
SELECT t.title::text
FROM furumusic__track t
WHERE t.release_id = r.id
AND t.is_hidden = false
ORDER BY t.disc_number NULLS LAST, t.track_number NULLS LAST, t.id
LIMIT 1
) AS track_title
FROM furumusic__release r
WHERE r.is_hidden = false
ORDER BY r.id"#,
)
.fetch_all(&ctx.pool)
.await?;
log.info(&format!(
"MusicBrainz release tag pass: checking {} release(s)",
rows.len()
));
let total = rows.len();
for (index, row) in rows.into_iter().enumerate() {
if should_log_lastfm_item(index + 1, total, 25) {
log.info(&format!(
"MusicBrainz release tags {}/{}: release {} \"{}\"",
index + 1,
total,
row.id,
row.title
));
}
if should_skip_source_entity(&ctx.pool, "release", row.id, "musicbrainz", overwrite).await?
{
stats.skipped_existing += 1;
continue;
}
let Some(artist) = row
.artist_name
.as_deref()
.filter(|value| !value.trim().is_empty())
else {
stats.not_found += 1;
continue;
};
stats.considered += 1;
let mbid = match crate::jobs::musicbrainz::load_or_search_release_mbid(
&ctx.pool,
client,
row.id,
artist,
&row.title,
row.track_title.as_deref(),
)
.await
{
Ok((release_mbid, _release_group_mbid)) => release_mbid,
Err(err) => {
stats.failed += 1;
log.warn(&format!(
"MusicBrainz release search failed for release {} \"{}\" / \"{}\": {err}",
row.id, artist, row.title
));
None
}
};
let Some(mbid) = mbid else {
stats.not_found += 1;
continue;
};
match client.lookup_release_tags(&mbid).await {
Ok(result) if !result.tags.is_empty() => {
if let Some(group_mbid) = result.release_group_mbid.as_deref() {
crate::jobs::musicbrainz::save_external_id(
&ctx.pool,
"release",
row.id,
"release_group",
group_mbid,
1.0,
)
.await?;
}
let tags = musicbrainz_tags_to_candidates(&result.tags);
match replace_entity_tags(&ctx.pool, "release", row.id, &tags, "musicbrainz", false)
.await
{
Ok(saved) => {
stats.tags_saved += saved;
stats.updated_entities += 1;
}
Err(err) => {
stats.failed += 1;
log.warn(&format!(
"MusicBrainz release tags save failed for release {} \"{}\" / \"{}\": {err}",
row.id, artist, row.title
));
}
}
}
Ok(_) => {
stats.not_found += 1;
}
Err(err) => {
stats.failed += 1;
log.warn(&format!(
"MusicBrainz release tags failed for release {} \"{}\" mbid={}: {err}",
row.id, row.title, mbid
));
}
}
}
Ok(())
}
async fn backfill_lastfm_artist_tags(
ctx: &JobContext,
log: &mut JobLog,
@@ -428,8 +705,7 @@ async fn backfill_lastfm_artist_tags(
stats.considered += 1;
match fetch_lastfm_artist_tags(client, api_key, &row.name).await {
Ok(tags) if !tags.is_empty() => {
match replace_entity_tags(&ctx.pool, "artist", row.id, &tags, "lastfm", false)
.await
match replace_entity_tags(&ctx.pool, "artist", row.id, &tags, "lastfm", false).await
{
Ok(saved) => {
stats.tags_saved += saved;
@@ -493,7 +769,15 @@ async fn backfill_lastfm_release_tags(
WHERE ra.release_id = r.id
ORDER BY ra.position
LIMIT 1
) AS artist_name
) AS artist_name,
(
SELECT t.title::text
FROM furumusic__track t
WHERE t.release_id = r.id
AND t.is_hidden = false
ORDER BY t.disc_number NULLS LAST, t.track_number NULLS LAST, t.id
LIMIT 1
) AS track_title
FROM furumusic__release r
WHERE r.is_hidden = false
ORDER BY r.id"#,
@@ -538,7 +822,10 @@ async fn backfill_lastfm_release_tags(
continue;
}
}
let Some(artist) = row.artist_name.as_deref().filter(|value| !value.trim().is_empty())
let Some(artist) = row
.artist_name
.as_deref()
.filter(|value| !value.trim().is_empty())
else {
stats.not_found += 1;
if should_log_lastfm_progress(index + 1, total, 25) {
@@ -663,7 +950,10 @@ async fn backfill_lastfm_track_tags(
continue;
}
}
let Some(artist) = row.artist_name.as_deref().filter(|value| !value.trim().is_empty())
let Some(artist) = row
.artist_name
.as_deref()
.filter(|value| !value.trim().is_empty())
else {
stats.not_found += 1;
if should_log_lastfm_progress(index + 1, total, 50) {
@@ -678,8 +968,7 @@ async fn backfill_lastfm_track_tags(
stats.considered += 1;
match fetch_lastfm_track_tags(client, api_key, artist, &row.title).await {
Ok(tags) if !tags.is_empty() => {
match replace_entity_tags(&ctx.pool, "track", row.id, &tags, "lastfm", true).await
{
match replace_entity_tags(&ctx.pool, "track", row.id, &tags, "lastfm", true).await {
Ok(saved) => {
stats.tags_saved += saved;
stats.updated_entities += 1;
@@ -741,6 +1030,16 @@ async fn should_skip_lastfm_entity(
entity_kind: &str,
entity_id: i64,
overwrite: bool,
) -> anyhow::Result<bool> {
should_skip_source_entity(pool, entity_kind, entity_id, "lastfm", overwrite).await
}
async fn should_skip_source_entity(
pool: &sqlx::PgPool,
entity_kind: &str,
entity_id: i64,
source: &str,
overwrite: bool,
) -> anyhow::Result<bool> {
if overwrite {
return Ok(false);
@@ -748,16 +1047,28 @@ async fn should_skip_lastfm_entity(
let exists: Option<i64> = sqlx::query_scalar(
r#"SELECT 1
FROM furumusic__entity_genre_tag
WHERE entity_kind = $1 AND entity_id = $2 AND source = 'lastfm'
WHERE entity_kind = $1 AND entity_id = $2 AND source = $3
LIMIT 1"#,
)
.bind(entity_kind)
.bind(entity_id)
.bind(source)
.fetch_optional(pool)
.await?;
Ok(exists.is_some())
}
fn musicbrainz_tags_to_candidates(
tags: &[crate::jobs::musicbrainz::MusicBrainzTag],
) -> Vec<TagCandidate> {
tags.iter()
.map(|tag| TagCandidate {
name: tag.name.clone(),
weight: tag.weight,
})
.collect()
}
async fn fetch_lastfm_artist_tags(
client: &reqwest::Client,
api_key: &str,
@@ -854,7 +1165,11 @@ async fn fetch_lastfm_top_tags(
Value::Object(_) => tag_from_value(tag_value).into_iter().collect::<Vec<_>>(),
_ => Vec::new(),
};
tags.sort_by(|a, b| b.weight.total_cmp(&a.weight).then_with(|| a.name.cmp(&b.name)));
tags.sort_by(|a, b| {
b.weight
.total_cmp(&a.weight)
.then_with(|| a.name.cmp(&b.name))
});
tags.truncate(LASTFM_TAG_LIMIT);
Ok(tags)
}
@@ -1027,9 +1342,9 @@ fn tags_from_text(value: &str) -> Vec<TagCandidate> {
for raw in normalized_separators.split([';', ',']) {
if let Some(name) = clean_tag_name(raw) {
if !is_ignored_tag(&normalize_tag_name(&name))
&& !tags
.iter()
.any(|tag: &TagCandidate| normalize_tag_name(&tag.name) == normalize_tag_name(&name))
&& !tags.iter().any(|tag: &TagCandidate| {
normalize_tag_name(&tag.name) == normalize_tag_name(&name)
})
{
tags.push(TagCandidate { name, weight: 1.0 });
}
src/jobs/mod.rs
+1
View File
@@ -4,6 +4,7 @@ pub mod inbox_process;
pub mod lastfm_popularity;
pub mod lastfm_scrobble;
pub mod metadata_backfill;
pub mod musicbrainz;
use std::path::{Component, Path, PathBuf};
src/jobs/musicbrainz.rs
+587
View File
@@ -0,0 +1,587 @@
use std::time::{Duration, Instant};
use reqwest::{Client, StatusCode};
use serde::Deserialize;
use tokio::sync::Mutex;
const MUSICBRAINZ_BASE_URL: &str = "https://musicbrainz.org/ws/2";
const COVER_ART_ARCHIVE_BASE_URL: &str = "https://coverartarchive.org";
const MUSICBRAINZ_REQUEST_DELAY: Duration = Duration::from_millis(1100);
const MUSICBRAINZ_TAG_LIMIT: usize = 12;
#[derive(Debug, Clone)]
pub struct MusicBrainzTag {
pub name: String,
pub weight: f64,
}
#[derive(Debug, Clone)]
pub struct MusicBrainzArtistMatch {
pub mbid: String,
pub score: i32,
}
#[derive(Debug, Clone)]
pub struct MusicBrainzReleaseMatch {
pub mbid: String,
pub release_group_mbid: Option<String>,
pub score: i32,
}
#[derive(Debug, Clone)]
pub struct MusicBrainzReleaseTags {
pub release_group_mbid: Option<String>,
pub tags: Vec<MusicBrainzTag>,
}
pub struct MusicBrainzClient {
client: Client,
last_musicbrainz_request: Mutex<Option<Instant>>,
}
pub async fn load_external_id(
pool: &sqlx::PgPool,
entity_kind: &str,
entity_id: i64,
id_kind: &str,
) -> anyhow::Result<Option<String>> {
let value = sqlx::query_scalar::<_, String>(
r#"SELECT external_id::text
FROM furumusic__external_metadata_id
WHERE entity_kind = $1
AND entity_id = $2
AND source = 'musicbrainz'
AND id_kind = $3
LIMIT 1"#,
)
.bind(entity_kind)
.bind(entity_id)
.bind(id_kind)
.fetch_optional(pool)
.await?;
Ok(value)
}
pub async fn save_external_id(
pool: &sqlx::PgPool,
entity_kind: &str,
entity_id: i64,
id_kind: &str,
external_id: &str,
confidence: f64,
) -> anyhow::Result<()> {
sqlx::query(
r#"INSERT INTO furumusic__external_metadata_id
(entity_kind, entity_id, source, id_kind, external_id, confidence, updated_at)
VALUES ($1, $2, 'musicbrainz', $3, $4, $5, $6)
ON CONFLICT (entity_kind, entity_id, source, id_kind) DO UPDATE SET
external_id = EXCLUDED.external_id,
confidence = EXCLUDED.confidence,
updated_at = EXCLUDED.updated_at"#,
)
.bind(entity_kind)
.bind(entity_id)
.bind(id_kind)
.bind(external_id)
.bind(confidence)
.bind(now_iso())
.execute(pool)
.await?;
Ok(())
}
pub async fn load_or_search_release_mbid(
pool: &sqlx::PgPool,
client: &MusicBrainzClient,
release_id: i64,
artist_name: &str,
release_title: &str,
representative_track_title: Option<&str>,
) -> anyhow::Result<(Option<String>, Option<String>)> {
let release_mbid = load_external_id(pool, "release", release_id, "release").await?;
let release_group_mbid = load_external_id(pool, "release", release_id, "release_group").await?;
if release_mbid.is_some() || release_group_mbid.is_some() {
return Ok((release_mbid, release_group_mbid));
}
let found = match client.search_release(artist_name, release_title).await? {
Some(found) => Some(found),
None => {
if let Some(track_title) =
representative_track_title.filter(|value| !value.trim().is_empty())
{
client
.search_release_by_recording(artist_name, track_title)
.await?
} else {
None
}
}
};
let Some(found) = found else {
return Ok((None, None));
};
save_external_id(
pool,
"release",
release_id,
"release",
&found.mbid,
found.score as f64 / 100.0,
)
.await?;
if let Some(group_mbid) = found.release_group_mbid.as_deref() {
save_external_id(
pool,
"release",
release_id,
"release_group",
group_mbid,
found.score as f64 / 100.0,
)
.await?;
}
Ok((Some(found.mbid), found.release_group_mbid))
}
impl MusicBrainzClient {
pub fn new(user_agent_prefix: &str) -> anyhow::Result<Self> {
let client = Client::builder()
.user_agent(format!(
"{}/{} (musicbrainz.org/doc/MusicBrainz_API)",
user_agent_prefix,
env!("CARGO_PKG_VERSION")
))
.timeout(Duration::from_secs(20))
.build()?;
Ok(Self {
client,
last_musicbrainz_request: Mutex::new(None),
})
}
pub fn http_client(&self) -> &Client {
&self.client
}
pub async fn search_artist(
&self,
name: &str,
) -> anyhow::Result<Option<MusicBrainzArtistMatch>> {
let query = format!("artist:\"{}\"", escape_search_value(name));
let response: Option<ArtistSearchResponse> = self
.get_musicbrainz_json(
"artist",
&[("query", query.as_str()), ("fmt", "json"), ("limit", "5")],
)
.await?;
let Some(response) = response else {
return Ok(None);
};
Ok(response
.artists
.into_iter()
.filter(|artist| artist.id.trim().len() == 36)
.max_by_key(|artist| artist.score.unwrap_or(0))
.and_then(|artist| {
let score = artist.score.unwrap_or(0);
(score >= 70).then_some(MusicBrainzArtistMatch {
mbid: artist.id,
score,
})
}))
}
pub async fn search_release(
&self,
artist: &str,
title: &str,
) -> anyhow::Result<Option<MusicBrainzReleaseMatch>> {
let query = format!(
"release:\"{}\" AND artist:\"{}\"",
escape_search_value(title),
escape_search_value(artist)
);
let response: Option<ReleaseSearchResponse> = self
.get_musicbrainz_json(
"release",
&[("query", query.as_str()), ("fmt", "json"), ("limit", "5")],
)
.await?;
let Some(response) = response else {
return Ok(None);
};
Ok(response
.releases
.into_iter()
.filter(|release| release.id.trim().len() == 36)
.max_by_key(|release| release.score.unwrap_or(0))
.and_then(|release| {
let score = release.score.unwrap_or(0);
(score >= 70).then_some(MusicBrainzReleaseMatch {
mbid: release.id,
release_group_mbid: release.release_group.map(|group| group.id),
score,
})
}))
}
pub async fn search_release_by_recording(
&self,
artist: &str,
track_title: &str,
) -> anyhow::Result<Option<MusicBrainzReleaseMatch>> {
let query = format!(
"recording:\"{}\" AND artist:\"{}\"",
escape_search_value(track_title),
escape_search_value(artist)
);
let response: Option<RecordingSearchResponse> = self
.get_musicbrainz_json(
"recording",
&[("query", query.as_str()), ("fmt", "json"), ("limit", "5")],
)
.await?;
let Some(response) = response else {
return Ok(None);
};
Ok(response
.recordings
.into_iter()
.flat_map(|recording| {
let recording_score = recording.score.unwrap_or(0);
recording
.releases
.into_iter()
.filter(move |release| release.id.trim().len() == 36)
.map(move |release| (recording_score, release))
})
.max_by_key(|(score, _)| *score)
.and_then(|(score, release)| {
(score >= 70).then_some(MusicBrainzReleaseMatch {
mbid: release.id,
release_group_mbid: release.release_group.map(|group| group.id),
score,
})
}))
}
pub async fn lookup_artist_tags(&self, mbid: &str) -> anyhow::Result<Vec<MusicBrainzTag>> {
let response: Option<TaggedEntityResponse> = self
.get_musicbrainz_json(
&format!("artist/{mbid}"),
&[("inc", "tags+genres"), ("fmt", "json")],
)
.await?;
Ok(response.map(tags_from_entity).unwrap_or_default())
}
pub async fn lookup_release_tags(&self, mbid: &str) -> anyhow::Result<MusicBrainzReleaseTags> {
let response: Option<ReleaseLookupResponse> = self
.get_musicbrainz_json(
&format!("release/{mbid}"),
&[("inc", "tags+genres+release-groups"), ("fmt", "json")],
)
.await?;
let Some(response) = response else {
return Ok(MusicBrainzReleaseTags {
release_group_mbid: None,
tags: Vec::new(),
});
};
let mut tags = tags_from_parts(response.tags, response.genres);
let release_group_mbid = response
.release_group
.as_ref()
.map(|group| group.id.clone());
if let Some(group_mbid) = release_group_mbid.as_deref() {
let group_response: Option<TaggedEntityResponse> = self
.get_musicbrainz_json(
&format!("release-group/{group_mbid}"),
&[("inc", "tags+genres"), ("fmt", "json")],
)
.await?;
merge_tags(
&mut tags,
group_response.map(tags_from_entity).unwrap_or_default(),
);
}
tags.sort_by(|a, b| {
b.weight
.total_cmp(&a.weight)
.then_with(|| a.name.cmp(&b.name))
});
tags.truncate(MUSICBRAINZ_TAG_LIMIT);
Ok(MusicBrainzReleaseTags {
release_group_mbid,
tags,
})
}
pub async fn fetch_cover_art_front_url(
&self,
release_mbid: Option<&str>,
release_group_mbid: Option<&str>,
) -> anyhow::Result<Option<String>> {
if let Some(mbid) = release_mbid {
if let Some(url) = self.cover_art_front_url("release", mbid).await? {
return Ok(Some(url));
}
}
if let Some(mbid) = release_group_mbid {
if let Some(url) = self.cover_art_front_url("release-group", mbid).await? {
return Ok(Some(url));
}
}
Ok(None)
}
async fn get_musicbrainz_json<T>(
&self,
path: &str,
query: &[(&str, &str)],
) -> anyhow::Result<Option<T>>
where
T: for<'de> Deserialize<'de>,
{
self.wait_for_musicbrainz_slot().await;
let url = format!("{MUSICBRAINZ_BASE_URL}/{}", path.trim_start_matches('/'));
let response = self.client.get(url).query(query).send().await?;
if response.status() == StatusCode::NOT_FOUND {
return Ok(None);
}
if response.status() == StatusCode::TOO_MANY_REQUESTS
|| response.status() == StatusCode::SERVICE_UNAVAILABLE
{
anyhow::bail!(
"MusicBrainz rate limit or service unavailable: {}",
response.status()
);
}
let response = response.error_for_status()?;
Ok(Some(response.json::<T>().await?))
}
async fn cover_art_front_url(&self, kind: &str, mbid: &str) -> anyhow::Result<Option<String>> {
let url = format!("{COVER_ART_ARCHIVE_BASE_URL}/{kind}/{mbid}");
let response = self.client.get(url).send().await?;
if response.status() == StatusCode::NOT_FOUND {
return Ok(None);
}
if response.status() == StatusCode::TOO_MANY_REQUESTS
|| response.status() == StatusCode::SERVICE_UNAVAILABLE
{
anyhow::bail!("Cover Art Archive unavailable: {}", response.status());
}
let response = response.error_for_status()?;
let body = response.json::<CoverArtArchiveResponse>().await?;
Ok(best_cover_art_url(body.images))
}
async fn wait_for_musicbrainz_slot(&self) {
let mut last = self.last_musicbrainz_request.lock().await;
if let Some(previous) = *last {
let elapsed = previous.elapsed();
if elapsed < MUSICBRAINZ_REQUEST_DELAY {
tokio::time::sleep(MUSICBRAINZ_REQUEST_DELAY - elapsed).await;
}
}
*last = Some(Instant::now());
}
}
fn now_iso() -> String {
chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ").to_string()
}
#[derive(Debug, Deserialize)]
struct ArtistSearchResponse {
#[serde(default)]
artists: Vec<ArtistSearchItem>,
}
#[derive(Debug, Deserialize)]
struct ArtistSearchItem {
id: String,
score: Option<i32>,
}
#[derive(Debug, Deserialize)]
struct ReleaseSearchResponse {
#[serde(default)]
releases: Vec<ReleaseSearchItem>,
}
#[derive(Debug, Deserialize)]
struct ReleaseSearchItem {
id: String,
score: Option<i32>,
#[serde(rename = "release-group")]
release_group: Option<MusicBrainzIdRef>,
}
#[derive(Debug, Deserialize)]
struct RecordingSearchResponse {
#[serde(default)]
recordings: Vec<RecordingSearchItem>,
}
#[derive(Debug, Deserialize)]
struct RecordingSearchItem {
score: Option<i32>,
#[serde(default)]
releases: Vec<RecordingReleaseItem>,
}
#[derive(Debug, Deserialize)]
struct RecordingReleaseItem {
id: String,
#[serde(rename = "release-group")]
release_group: Option<MusicBrainzIdRef>,
}
#[derive(Debug, Deserialize)]
struct ReleaseLookupResponse {
#[serde(default)]
tags: Vec<MusicBrainzTagItem>,
#[serde(default)]
genres: Vec<MusicBrainzTagItem>,
#[serde(rename = "release-group")]
release_group: Option<MusicBrainzIdRef>,
}
#[derive(Debug, Deserialize)]
struct TaggedEntityResponse {
#[serde(default)]
tags: Vec<MusicBrainzTagItem>,
#[serde(default)]
genres: Vec<MusicBrainzTagItem>,
}
#[derive(Debug, Deserialize)]
struct MusicBrainzIdRef {
id: String,
}
#[derive(Debug, Deserialize)]
struct MusicBrainzTagItem {
name: String,
count: Option<i64>,
}
#[derive(Debug, Deserialize)]
struct CoverArtArchiveResponse {
#[serde(default)]
images: Vec<CoverArtArchiveImage>,
}
#[derive(Debug, Deserialize)]
struct CoverArtArchiveImage {
image: Option<String>,
front: Option<bool>,
approved: Option<bool>,
#[serde(default)]
types: Vec<String>,
thumbnails: Option<CoverArtArchiveThumbnails>,
}
#[derive(Debug, Deserialize)]
struct CoverArtArchiveThumbnails {
#[serde(rename = "1200")]
size_1200: Option<String>,
#[serde(rename = "500")]
size_500: Option<String>,
large: Option<String>,
}
fn escape_search_value(value: &str) -> String {
value.replace('\\', "\\\\").replace('"', "\\\"")
}
fn tags_from_entity(entity: TaggedEntityResponse) -> Vec<MusicBrainzTag> {
tags_from_parts(entity.tags, entity.genres)
}
fn tags_from_parts(
tags: Vec<MusicBrainzTagItem>,
genres: Vec<MusicBrainzTagItem>,
) -> Vec<MusicBrainzTag> {
let mut result = Vec::new();
merge_items(&mut result, genres, 2.0);
merge_items(&mut result, tags, 1.0);
result.sort_by(|a, b| {
b.weight
.total_cmp(&a.weight)
.then_with(|| a.name.cmp(&b.name))
});
result.truncate(MUSICBRAINZ_TAG_LIMIT);
result
}
fn merge_items(result: &mut Vec<MusicBrainzTag>, items: Vec<MusicBrainzTagItem>, multiplier: f64) {
for item in items {
let name = item.name.trim();
if name.is_empty() {
continue;
}
let weight = item.count.unwrap_or(1).max(1) as f64 * multiplier;
if let Some(existing) = result
.iter_mut()
.find(|tag| tag.name.eq_ignore_ascii_case(name))
{
existing.weight = existing.weight.max(weight);
} else {
result.push(MusicBrainzTag {
name: name.to_string(),
weight,
});
}
}
}
fn merge_tags(result: &mut Vec<MusicBrainzTag>, extra: Vec<MusicBrainzTag>) {
for tag in extra {
if let Some(existing) = result
.iter_mut()
.find(|candidate| candidate.name.eq_ignore_ascii_case(&tag.name))
{
existing.weight = existing.weight.max(tag.weight);
} else {
result.push(tag);
}
}
}
fn best_cover_art_url(mut images: Vec<CoverArtArchiveImage>) -> Option<String> {
images.sort_by_key(|image| {
let front = image.front.unwrap_or(false)
|| image
.types
.iter()
.any(|value| value.eq_ignore_ascii_case("front"));
let approved = image.approved.unwrap_or(false);
(u8::from(front), u8::from(approved))
});
images
.into_iter()
.rev()
.find_map(|image| {
image
.thumbnails
.as_ref()
.and_then(|thumbs| {
thumbs
.size_1200
.as_deref()
.or(thumbs.size_500.as_deref())
.or(thumbs.large.as_deref())
})
.map(str::to_string)
.or(image.image)
})
.filter(|url| !url.trim().is_empty())
}
src/music/mod.rs
+46
View File
@@ -1865,6 +1865,51 @@ pub mod db_migrations {
&[Operation::custom(create_entity_genre_tags).build()];
}
// -- M0036: External metadata identifiers --------------------------------
#[cot::db::migrations::migration_op]
async fn create_external_metadata_ids(
ctx: migrations::MigrationContext<'_>,
) -> cot::db::Result<()> {
ctx.db
.raw(
"CREATE TABLE IF NOT EXISTS furumusic__external_metadata_id (
id BIGSERIAL PRIMARY KEY,
entity_kind VARCHAR(32) NOT NULL,
entity_id BIGINT NOT NULL,
source VARCHAR(32) NOT NULL,
id_kind VARCHAR(64) NOT NULL,
external_id VARCHAR(255) NOT NULL,
confidence DOUBLE PRECISION NOT NULL DEFAULT 1,
updated_at VARCHAR(32) NOT NULL,
UNIQUE(entity_kind, entity_id, source, id_kind)
)",
)
.await?;
ctx.db
.raw(
"CREATE INDEX IF NOT EXISTS idx_external_metadata_id_lookup
ON furumusic__external_metadata_id (source, id_kind, external_id)",
)
.await?;
Ok(())
}
#[derive(Debug, Copy, Clone)]
pub struct M0036CreateExternalMetadataIds;
impl migrations::Migration for M0036CreateExternalMetadataIds {
const APP_NAME: &'static str = "furumusic";
const MIGRATION_NAME: &'static str = "m_0036_create_external_metadata_ids";
const DEPENDENCIES: &'static [migrations::MigrationDependency] =
&[migrations::MigrationDependency::migration(
"furumusic",
"m_0035_create_entity_genre_tags",
)];
const OPERATIONS: &'static [Operation] =
&[Operation::custom(create_external_metadata_ids).build()];
}
pub const MIGRATIONS: &[&SyncDynMigration] = &[
&M0006CreateMediaFile,
&M0007CreateArtist,
@@ -1891,5 +1936,6 @@ pub mod db_migrations {
&M0033CreateLastfmScrobbling,
&M0034CreateArtworkLookupState,
&M0035CreateEntityGenreTags,
&M0036CreateExternalMetadataIds,
];
}
src/player/dto.rs
+1
View File
@@ -16,6 +16,7 @@ pub(super) struct Paginated<T: Serialize> {
pub(super) total: i64,
pub(super) page: i32,
pub(super) per_page: i32,
pub(super) has_more: bool,
}
#[derive(Debug, Serialize, JsonSchema)]
src/player/mod.rs
+68 -43
View File
@@ -2787,7 +2787,7 @@ async fn artists_handler(
let total_row = sqlx::query_as::<_, CountRow>(
r#"SELECT COUNT(DISTINCT a.id) AS count
FROM furumusic__artist a
JOIN furumusic__track_artist ta ON ta.artist_id = a.id AND ta.role <> 'featuring'
JOIN furumusic__track_artist ta ON ta.artist_id = a.id
JOIN furumusic__track t ON t.id = ta.track_id AND t.is_hidden = false
JOIN furumusic__release r ON r.id = t.release_id AND r.is_hidden = false
JOIN furumusic__media_file mf ON mf.id = t.audio_file_id
@@ -2801,17 +2801,24 @@ async fn artists_handler(
let rows = sqlx::query_as::<_, ArtistRow>(
r#"SELECT a.id, a.name::text as name, a.image_file_id,
COUNT(DISTINCT r.id) AS release_count,
COUNT(DISTINCT r.id) FILTER (WHERE primary_release.artist_id IS NOT NULL) AS release_count,
COUNT(DISTINCT t.id) AS track_count
FROM furumusic__artist a
JOIN furumusic__track_artist ta ON ta.artist_id = a.id AND ta.role <> 'featuring'
JOIN furumusic__track_artist ta ON ta.artist_id = a.id
JOIN furumusic__track t ON t.id = ta.track_id AND t.is_hidden = false
JOIN furumusic__release r ON r.id = t.release_id AND r.is_hidden = false
JOIN furumusic__media_file mf ON mf.id = t.audio_file_id
LEFT JOIN furumusic__release_artist primary_release
ON primary_release.release_id = r.id
AND primary_release.artist_id = a.id
AND primary_release.position = 0
WHERE a.is_hidden = false
AND mf.uploaded_by_user_id = $1
GROUP BY a.id, a.name, a.name_sort, a.image_file_id
ORDER BY release_count DESC, track_count DESC, a.name_sort
ORDER BY (COUNT(DISTINCT r.id) FILTER (WHERE primary_release.artist_id IS NOT NULL) > 0) DESC,
release_count DESC,
track_count DESC,
a.name_sort
LIMIT $2 OFFSET $3"#,
)
.bind(user.id)
@@ -2831,22 +2838,22 @@ async fn artists_handler(
track_count: r.track_count,
})
.collect();
let has_more = offset + (items.len() as i64) < total_row.count;
return Json(Paginated {
items,
total: total_row.count,
page,
per_page,
has_more,
})
.into_response();
}
let total_row = sqlx::query_as::<_, CountRow>(
r#"SELECT COUNT(DISTINCT a.id) AS count
r#"SELECT COUNT(*) AS count
FROM furumusic__artist a
JOIN furumusic__release_artist ra ON ra.artist_id = a.id
JOIN furumusic__release r ON r.id = ra.release_id
WHERE a.is_hidden = false AND r.is_hidden = false AND ra.position = 0"#,
WHERE a.is_hidden = false"#,
)
.fetch_one(pool)
.await
@@ -2854,21 +2861,33 @@ async fn artists_handler(
let rows = sqlx::query_as::<_, ArtistRow>(
r#"SELECT a.id, a.name::text as name, a.image_file_id,
s.release_count,
s.track_count
COALESCE(s.release_count, 0) AS release_count,
COALESCE(s.track_count, 0) AS track_count
FROM furumusic__artist a
JOIN (
SELECT ra.artist_id,
COUNT(DISTINCT r.id) AS release_count,
COUNT(t.id) AS track_count
FROM furumusic__release_artist ra
JOIN furumusic__release r ON r.id = ra.release_id AND r.is_hidden = false
LEFT JOIN furumusic__track t ON t.release_id = r.id AND t.is_hidden = false
WHERE ra.position = 0
GROUP BY ra.artist_id
LEFT JOIN (
SELECT appearance.artist_id,
COUNT(DISTINCT appearance.release_id) FILTER (WHERE appearance.is_primary_release_artist) AS release_count,
COUNT(DISTINCT appearance.track_id) AS track_count
FROM (
SELECT ta.artist_id,
t.id AS track_id,
r.id AS release_id,
primary_release.artist_id IS NOT NULL AS is_primary_release_artist
FROM furumusic__track_artist ta
JOIN furumusic__track t ON t.id = ta.track_id AND t.is_hidden = false
JOIN furumusic__release r ON r.id = t.release_id AND r.is_hidden = false
LEFT JOIN furumusic__release_artist primary_release
ON primary_release.release_id = r.id
AND primary_release.artist_id = ta.artist_id
AND primary_release.position = 0
) appearance
GROUP BY appearance.artist_id
) s ON s.artist_id = a.id
WHERE a.is_hidden = false
ORDER BY s.release_count DESC, s.track_count DESC, a.name_sort
ORDER BY (COALESCE(s.release_count, 0) > 0) DESC,
COALESCE(s.release_count, 0) DESC,
COALESCE(s.track_count, 0) DESC,
a.name_sort
LIMIT $1 OFFSET $2"#,
)
.bind(per_page as i64)
@@ -2887,12 +2906,14 @@ async fn artists_handler(
track_count: r.track_count,
})
.collect();
let has_more = offset + (items.len() as i64) < total_row.count;
Json(Paginated {
items,
total: total_row.count,
page,
per_page,
has_more,
})
.into_response()
}
@@ -3480,10 +3501,7 @@ async fn build_track_items(
.collect())
}
async fn load_track_items_by_ids(
pool: &sqlx::PgPool,
ids: &[i64],
) -> cot::Result<Vec<TrackItem>> {
async fn load_track_items_by_ids(pool: &sqlx::PgPool, ids: &[i64]) -> cot::Result<Vec<TrackItem>> {
if ids.is_empty() {
return Ok(Vec::new());
}
@@ -4073,7 +4091,10 @@ async fn devices_command_handler(
}
fn stamp_jam_queue_tracks(payload: &mut serde_json::Value, user_id: i64, user_name: &str) {
let Some(tracks) = payload.get_mut("tracks").and_then(serde_json::Value::as_array_mut) else {
let Some(tracks) = payload
.get_mut("tracks")
.and_then(serde_json::Value::as_array_mut)
else {
return;
};
for track in tracks {
@@ -5792,7 +5813,8 @@ async fn build_track_radio_ids(
let artist_ids = track_primary_artist_ids(pool, track_id).await?;
let remaining = PLAYER_RADIO_TRACK_LIMIT.saturating_sub(ids.len()) as i64;
let fallback_ids = fallback_radio_track_ids(pool, user_id, &artist_ids, &ids, remaining).await?;
let fallback_ids =
fallback_radio_track_ids(pool, user_id, &artist_ids, &ids, remaining).await?;
append_unique_track_ids(&mut ids, fallback_ids, PLAYER_RADIO_TRACK_LIMIT);
Ok(Some(ids))
@@ -5848,7 +5870,8 @@ async fn build_release_radio_ids(
let artist_ids = release_primary_artist_ids(pool, release_id).await?;
let remaining = PLAYER_RADIO_TRACK_LIMIT.saturating_sub(ids.len()) as i64;
let fallback_ids = fallback_radio_track_ids(pool, user_id, &artist_ids, &ids, remaining).await?;
let fallback_ids =
fallback_radio_track_ids(pool, user_id, &artist_ids, &ids, remaining).await?;
append_unique_track_ids(&mut ids, fallback_ids, PLAYER_RADIO_TRACK_LIMIT);
Ok(Some(ids))
@@ -6751,22 +6774,24 @@ impl App for PlayerApp {
{
let pool = Arc::clone(&pool);
let pool_config = Arc::clone(&pool_config);
get(move |session: Session, db: Database, path: Path<PathRadioSeed>| {
let pool = Arc::clone(&pool);
let pool_config = Arc::clone(&pool_config);
async move {
let pg_pool = pool
.get_or_init(|| async {
sqlx::postgres::PgPoolOptions::new()
.max_connections(5)
.connect(&pool_config.database_url)
.await
.expect("player pool")
})
.await;
radio_handler(session, db, pg_pool, path).await
}
})
get(
move |session: Session, db: Database, path: Path<PathRadioSeed>| {
let pool = Arc::clone(&pool);
let pool_config = Arc::clone(&pool_config);
async move {
let pg_pool = pool
.get_or_init(|| async {
sqlx::postgres::PgPoolOptions::new()
.max_connections(5)
.connect(&pool_config.database_url)
.await
.expect("player pool")
})
.await;
radio_handler(session, db, pg_pool, path).await
}
},
)
},
"player_radio",
),
templates/admin/v2.html
+43 -5
View File
@@ -1434,7 +1434,7 @@ tbody tr:hover {
<button class="nav-btn" :class="{active: activeView === 'reviews'}" @click="openReviews()">
<i data-lucide="inbox"></i>
<span>Review Queue</span>
<span class="nav-count" x-text="reviews.total || 0"></span>
<span class="nav-count" x-text="reviewTotalAll()"></span>
</button>
<button class="nav-btn" :class="{active: activeView === 'jobs'}" @click="openJobs()">
<i data-lucide="calendar-clock"></i>
@@ -1526,7 +1526,7 @@ tbody tr:hover {
<template x-for="status in reviewStatuses" :key="status.value">
<button class="seg-btn" :class="{active: reviewFilter.status === status.value}" @click="setReviewStatus(status.value)">
<span x-text="status.label"></span>
<span class="muted" x-text="status.value ? statusCount(status.value) : reviews.total"></span>
<span class="muted" x-text="status.value ? statusCount(status.value) : reviewTotalAll()"></span>
</button>
</template>
</div>
@@ -1715,12 +1715,19 @@ tbody tr:hover {
<label><input type="checkbox" x-model="metadataBackfillOptions.duration_seconds" /> duration_seconds</label>
<label><input type="checkbox" x-model="metadataBackfillOptions.local_genres" /> local genres from files</label>
<label><input type="checkbox" x-model="metadataBackfillOptions.lastfm_tags" /> Last.fm tags</label>
<label><input type="checkbox" x-model="metadataBackfillOptions.musicbrainz_tags" /> MusicBrainz tags</label>
</div>
<div class="mode-row">
<label><input type="radio" value="fill_missing" x-model="metadataBackfillOptions.mode" /> Fill missing only</label>
<label><input type="radio" value="overwrite" x-model="metadataBackfillOptions.mode" /> Overwrite existing values</label>
</div>
</div>
<div class="metadata-backfill-options" x-show="isArtworkBackfillJob(activeJob)">
<div class="mode-row">
<label><input type="radio" value="missing" x-model="artworkBackfillOptions.mode" /> Missing images only</label>
<label><input type="radio" value="overwrite" x-model="artworkBackfillOptions.mode" /> Search all and replace existing</label>
</div>
</div>
<div class="job-param-note" x-show="activeJob && !jobHasParameterForm(activeJob)">
This task has no manual parameters.
</div>
@@ -2609,7 +2616,7 @@ function adminV2() {
stats: {},
runtime: { agent: {}, storage: [], node: {} },
libraryOverview: {},
reviews: { items: [], total: 0, limit: 80, offset: 0, status_counts: [] },
reviews: { items: [], total: 0, total_all: 0, limit: 80, offset: 0, status_counts: [] },
users: { items: [], total: 0, limit: 40, offset: 0, online_count: 0 },
usersLoading: false,
userSearch: '',
@@ -2656,8 +2663,12 @@ function adminV2() {
duration_seconds: true,
local_genres: true,
lastfm_tags: true,
musicbrainz_tags: true,
mode: 'fill_missing'
},
artworkBackfillOptions: {
mode: 'missing'
},
libraryKind: 'artists',
librarySearch: '',
library: { items: [], total: 0, limit: 40, offset: 0 },
@@ -3180,6 +3191,11 @@ function adminV2() {
method: 'POST',
body: JSON.stringify(this.metadataBackfillPayload())
})
: this.isArtworkBackfillJob(job)
? await this.request(`${this.apiBase}/jobs/artwork_backfill/run-options`, {
method: 'POST',
body: JSON.stringify(this.artworkBackfillPayload())
})
: await this.request(`${this.apiBase}/jobs/${encodeURIComponent(job.name)}/run`, { method: 'POST' });
this.showToast(`Run #${result.run_id} started`);
this.activeJobName = job.name;
@@ -3199,8 +3215,12 @@ function adminV2() {
return job && job.name === 'metadata_backfill';
},
isArtworkBackfillJob(job) {
return job && job.name === 'artwork_backfill';
},
jobHasParameterForm(job) {
return this.isMetadataBackfillJob(job);
return this.isMetadataBackfillJob(job) || this.isArtworkBackfillJob(job);
},
metadataBackfillPayload() {
@@ -3212,10 +3232,18 @@ function adminV2() {
duration_seconds: Boolean(options.duration_seconds),
local_genres: Boolean(options.local_genres),
lastfm_tags: Boolean(options.lastfm_tags),
musicbrainz_tags: Boolean(options.musicbrainz_tags),
overwrite: options.mode === 'overwrite'
};
},
artworkBackfillPayload() {
const options = this.artworkBackfillOptions || {};
return {
overwrite_existing: options.mode === 'overwrite'
};
},
async toggleJob(job) {
try {
const result = await this.request(`${this.apiBase}/jobs/${encodeURIComponent(job.name)}/toggle`, { method: 'POST' });
@@ -3844,7 +3872,11 @@ function adminV2() {
},
reviewPanelSubtitle() {
return `${this.fmt(this.reviews.total || 0)} rows · ${this.reviewFilter.status || 'all statuses'}`;
const total = this.reviews.total || 0;
const totalAll = this.reviewTotalAll();
const status = this.reviewFilter.status || 'all statuses';
if (this.reviewFilter.status) return `${this.fmt(total)} ${status} · ${this.fmt(totalAll)} total`;
return `${this.fmt(totalAll)} rows · ${status}`;
},
jobPanelSubtitle() {
@@ -3955,6 +3987,12 @@ function adminV2() {
return row ? row.count : 0;
},
reviewTotalAll() {
const explicit = Number(this.reviews.total_all || 0);
if (explicit > 0 || Object.prototype.hasOwnProperty.call(this.reviews || {}, 'total_all')) return explicit;
return (this.reviews.status_counts || []).reduce((sum, row) => sum + Number(row.count || 0), 0);
},
formatConfidence(value) {
return typeof value === 'number' ? `${Math.round(value * 100)}%` : '-';
},
templates/player/scripts.html
+18 -5
View File
@@ -2025,23 +2025,26 @@ document.addEventListener('alpine:init', () => {
this._artistsLoadToken = token;
try {
const mine = filter === 'uploads' ? '&mine=true' : '';
const res = await fetch(`/api/player/artists?page=${page}&limit=60${mine}`);
const res = await fetch(`/api/player/artists?page=${page}&limit=80${mine}`);
if (!res.ok) throw new Error('failed');
const data = await res.json();
if (token !== this._artistsLoadToken || filter !== this.artistFilter) return;
const incoming = Array.isArray(data.items) ? data.items : [];
if (page === 1) {
this.artists = data.items;
this.artists = incoming;
} else {
this.artists = [...this.artists, ...data.items];
const existing = new Set(this.artists.map(artist => artist.id));
this.artists = [...this.artists, ...incoming.filter(artist => !existing.has(artist.id))];
}
this.artistsPage = data.page;
this.artistsTotal = data.total;
if (this.artists.length >= data.total) {
if (data.has_more === false || this.artists.length >= data.total) {
this._allLoaded = true;
}
} catch {}
if (token === this._artistsLoadToken) {
this.loading = false;
this.$nextTick(() => { this._fillArtistViewport(); });
}
},
@@ -2400,11 +2403,21 @@ document.addEventListener('alpine:init', () => {
if (entries[0].isIntersecting && !this.loading && !this._allLoaded) {
this.loadArtists(this.artistsPage + 1);
}
}, { root: document.getElementById('center-scroll'), threshold: 0.1 });
}, { root: document.getElementById('center-scroll'), rootMargin: '900px 0px', threshold: 0.01 });
this._observer.observe(sentinel);
this._fillArtistViewport();
});
},
_fillArtistViewport() {
if (!(this.view === 'artists' || this.view === 'my_uploads') || this.loading || this._allLoaded) return;
const el = this._scrollElement();
if (!el) return;
if (el.scrollHeight <= el.clientHeight + 900) {
this.loadArtists(this.artistsPage + 1);
}
},
$nextTick(fn) {
setTimeout(fn, 50);
},
templates/player/shell.html
+14 -6
View File
@@ -60,7 +60,7 @@
:class="{ active: $store.library.view === 'artists' }"
@click="$store.library.goArtists()">
<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"><circle cx="12" cy="8" r="4"/><path d="M20 21a8 8 0 10-16 0"/></svg>
{{ t.player_artists }}
{{ t.player_global_library }}
</div>
<div class="sidebar-nav-item"
:class="{ active: $store.library.view === 'my_uploads' }"
@@ -174,7 +174,7 @@
:class="{ active: $store.library.view === 'artists' }"
@click="$store.library.goArtists(); $store.mobile.closeLibrary()">
<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"><circle cx="12" cy="8" r="4"/><path d="M20 21a8 8 0 10-16 0"/></svg>
{{ t.player_artists }}
{{ t.player_global_library }}
</div>
<div class="sidebar-nav-item"
:class="{ active: $store.library.view === 'my_uploads' }"
@@ -521,7 +521,7 @@
<!-- Artists Grid -->
<template x-if="$store.library.view === 'artists' || $store.library.view === 'my_uploads'">
<div>
<h1 class="section-title" x-text="$store.library.view === 'my_uploads' ? '{{ t.player_my_uploads }}' : '{{ t.player_artists }}'"></h1>
<h1 class="section-title" x-text="$store.library.view === 'my_uploads' ? '{{ t.player_my_uploads }}' : '{{ t.player_global_library }}'"></h1>
<div class="card-grid">
<template x-for="artist in $store.library.artists" :key="artist.id">
<div class="card" @click="$store.library.openArtist(artist.id)">
@@ -554,9 +554,17 @@
<p x-text="$store.library.view === 'my_uploads' ? '{{ t.player_no_uploaded_tracks }}' : '{{ t.artists_empty }}'"></p>
</div>
</template>
<template x-if="$store.library.loading">
<template x-if="$store.library.loading && $store.library.artists.length === 0">
<div class="loading-spinner"><div class="spinner"></div></div>
</template>
<div class="scroll-load-indicator"
x-show="$store.library.loading && $store.library.artists.length > 0"
x-cloak
role="status"
aria-live="polite">
<div class="scroll-load-spinner"></div>
<span>{{ t.player_loading_more }}</span>
</div>
<div id="artist-sentinel" style="height:1px"></div>
</div>
</template>
@@ -566,7 +574,7 @@
<div>
<div class="breadcrumb">
<a @click="$store.library.artistFilter === 'uploads' ? $store.library.goMyUploads() : $store.library.goArtists()"
x-text="$store.library.artistFilter === 'uploads' ? '{{ t.player_my_uploads }}' : '{{ t.player_artists }}'"></a>
x-text="$store.library.artistFilter === 'uploads' ? '{{ t.player_my_uploads }}' : '{{ t.player_global_library }}'"></a>
<span>/</span>
<span x-text="$store.library.currentArtist.name"></span>
</div>
@@ -727,7 +735,7 @@
<div>
<div class="breadcrumb">
<a @click="$store.library.artistFilter === 'uploads' ? $store.library.goMyUploads() : $store.library.goArtists()"
x-text="$store.library.artistFilter === 'uploads' ? '{{ t.player_my_uploads }}' : '{{ t.player_artists }}'"></a>
x-text="$store.library.artistFilter === 'uploads' ? '{{ t.player_my_uploads }}' : '{{ t.player_global_library }}'"></a>
<span>/</span>
<template x-if="$store.library.currentRelease.artists.length > 0">
<a @click="$store.library.openArtist($store.library.currentRelease.artists[0].id)" x-text="$store.library.currentRelease.artists[0].name"></a>
templates/player/styles.html
+20
View File
@@ -1814,6 +1814,26 @@ button.user-stat:hover {
animation: spin 0.8s linear infinite;
}
.scroll-load-indicator {
min-height: 56px;
margin: 18px 0 8px;
display: flex;
align-items: center;
justify-content: center;
gap: 10px;
color: var(--text-secondary);
font-size: 13px;
}
.scroll-load-spinner {
width: 18px;
height: 18px;
border: 2px solid var(--bg-active);
border-top-color: var(--accent);
border-radius: 50%;
animation: spin 0.8s linear infinite;
}
@keyframes spin { to { transform: rotate(360deg); } }
/* Empty state */