Compare commits
3 Commits
v0.4.0
...
8fa06038fe
| Author | SHA1 | Date | |
|---|---|---|---|
| 8fa06038fe | |||
| 6b69cc0fc0 | |||
| 624cadab64 |
Generated
+1
-1
@@ -1418,7 +1418,7 @@ checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "furumusic"
|
name = "furumusic"
|
||||||
version = "0.3.1"
|
version = "0.4.1"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"async-trait",
|
"async-trait",
|
||||||
|
|||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "furumusic"
|
name = "furumusic"
|
||||||
version = "0.4.0"
|
version = "0.4.2"
|
||||||
edition = "2024"
|
edition = "2024"
|
||||||
description = "Reusable web-app boilerplate: auth, OIDC/SSO, admin panel, user management, i18n, PostgreSQL"
|
description = "Reusable web-app boilerplate: auth, OIDC/SSO, admin panel, user management, i18n, PostgreSQL"
|
||||||
|
|
||||||
|
|||||||
@@ -96,17 +96,11 @@ fn generate_missing_variants_sync(
|
|||||||
image::ExtendedColorType::Rgb8,
|
image::ExtendedColorType::Rgb8,
|
||||||
);
|
);
|
||||||
match result {
|
match result {
|
||||||
Ok(()) => crate::metrics::record_agent_cover_variant(
|
Ok(()) => {
|
||||||
variant.name,
|
crate::metrics::record_agent_cover_variant(variant.name, "ok", start.elapsed())
|
||||||
"ok",
|
}
|
||||||
start.elapsed(),
|
|
||||||
),
|
|
||||||
Err(err) => {
|
Err(err) => {
|
||||||
crate::metrics::record_agent_cover_variant(
|
crate::metrics::record_agent_cover_variant(variant.name, "error", start.elapsed());
|
||||||
variant.name,
|
|
||||||
"error",
|
|
||||||
start.elapsed(),
|
|
||||||
);
|
|
||||||
return Err(err.into());
|
return Err(err.into());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,431 @@
|
|||||||
|
use std::collections::BTreeSet;
|
||||||
|
use std::io::ErrorKind;
|
||||||
|
|
||||||
|
use sqlx::PgPool;
|
||||||
|
|
||||||
|
use crate::scheduler::{Job, JobContext, JobLog};
|
||||||
|
|
||||||
|
const SAMPLE_LOG_LIMIT: usize = 50;
|
||||||
|
|
||||||
|
pub struct ArchiveCleanupJob;
|
||||||
|
|
||||||
|
#[derive(Debug, sqlx::FromRow)]
|
||||||
|
struct TrackFileRow {
|
||||||
|
track_id: i64,
|
||||||
|
track_title: String,
|
||||||
|
release_id: i64,
|
||||||
|
release_title: Option<String>,
|
||||||
|
media_file_id: Option<i64>,
|
||||||
|
file_type: Option<String>,
|
||||||
|
file_path: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
struct MissingTrack {
|
||||||
|
track_id: i64,
|
||||||
|
track_title: String,
|
||||||
|
release_id: i64,
|
||||||
|
release_title: Option<String>,
|
||||||
|
media_file_id: Option<i64>,
|
||||||
|
file_path: Option<String>,
|
||||||
|
reason: MissingReason,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
enum MissingReason {
|
||||||
|
MissingMediaRow,
|
||||||
|
InvalidMediaType(String),
|
||||||
|
EmptyPath,
|
||||||
|
MissingFile,
|
||||||
|
NotRegularFile,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Default)]
|
||||||
|
struct DeleteStats {
|
||||||
|
playback_states_cleared: u64,
|
||||||
|
playlist_entries_deleted: u64,
|
||||||
|
likes_deleted: u64,
|
||||||
|
play_history_deleted: u64,
|
||||||
|
popularity_history_deleted: u64,
|
||||||
|
scrobble_outbox_deleted: u64,
|
||||||
|
track_genres_deleted: u64,
|
||||||
|
entity_tags_deleted: u64,
|
||||||
|
external_ids_deleted: u64,
|
||||||
|
track_artists_deleted: u64,
|
||||||
|
tracks_deleted: u64,
|
||||||
|
media_files_deleted: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait::async_trait]
|
||||||
|
impl Job for ArchiveCleanupJob {
|
||||||
|
fn name(&self) -> &'static str {
|
||||||
|
"archive_cleanup"
|
||||||
|
}
|
||||||
|
|
||||||
|
fn description(&self) -> &'static str {
|
||||||
|
"Clean stale archive records, starting with tracks whose audio files are missing"
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_cron(&self) -> &'static str {
|
||||||
|
// Daily at 04:45.
|
||||||
|
"0 45 4 * * *"
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn run(&self, ctx: &JobContext, log: &mut JobLog) -> anyhow::Result<()> {
|
||||||
|
run_missing_audio_cleanup(ctx, log).await
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn run_missing_audio_cleanup(ctx: &JobContext, log: &mut JobLog) -> anyhow::Result<()> {
|
||||||
|
let storage_dir = ctx.config.agent_storage_dir.trim();
|
||||||
|
if storage_dir.is_empty() {
|
||||||
|
log.warn("Archive cleanup: agent_storage_dir is not configured, skipping file checks");
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
|
||||||
|
let rows = sqlx::query_as::<_, TrackFileRow>(
|
||||||
|
r#"SELECT t.id AS track_id,
|
||||||
|
t.title::text AS track_title,
|
||||||
|
t.release_id,
|
||||||
|
r.title::text AS release_title,
|
||||||
|
mf.id AS media_file_id,
|
||||||
|
mf.file_type::text AS file_type,
|
||||||
|
mf.file_path::text AS file_path
|
||||||
|
FROM furumusic__track t
|
||||||
|
LEFT JOIN furumusic__release r ON r.id = t.release_id
|
||||||
|
LEFT JOIN furumusic__media_file mf ON mf.id = t.audio_file_id
|
||||||
|
ORDER BY t.id"#,
|
||||||
|
)
|
||||||
|
.fetch_all(&ctx.pool)
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
if rows.is_empty() {
|
||||||
|
log.info("Archive cleanup: no tracks found");
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
|
||||||
|
log.info(&format!(
|
||||||
|
"Archive cleanup: checking {} track audio reference(s)",
|
||||||
|
rows.len()
|
||||||
|
));
|
||||||
|
|
||||||
|
let mut missing_tracks = Vec::new();
|
||||||
|
let mut skipped_io_errors = 0u64;
|
||||||
|
|
||||||
|
for row in rows {
|
||||||
|
let Some(media_file_id) = row.media_file_id else {
|
||||||
|
missing_tracks.push(MissingTrack::from_row(row, MissingReason::MissingMediaRow));
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
|
||||||
|
let file_type = row.file_type.clone();
|
||||||
|
match file_type.as_deref() {
|
||||||
|
Some("audio") => {}
|
||||||
|
Some(file_type) => {
|
||||||
|
missing_tracks.push(MissingTrack::from_row(
|
||||||
|
row,
|
||||||
|
MissingReason::InvalidMediaType(file_type.to_owned()),
|
||||||
|
));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
None => {
|
||||||
|
missing_tracks.push(MissingTrack::from_row(row, MissingReason::MissingMediaRow));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let Some(file_path) = row
|
||||||
|
.file_path
|
||||||
|
.as_deref()
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|path| !path.is_empty())
|
||||||
|
else {
|
||||||
|
missing_tracks.push(MissingTrack::from_row(row, MissingReason::EmptyPath));
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
|
||||||
|
let absolute_path = crate::media_paths::resolve_media_file_path(storage_dir, file_path);
|
||||||
|
match tokio::fs::metadata(&absolute_path).await {
|
||||||
|
Ok(meta) if meta.is_file() => {}
|
||||||
|
Ok(_) => {
|
||||||
|
missing_tracks.push(MissingTrack::from_row(row, MissingReason::NotRegularFile));
|
||||||
|
}
|
||||||
|
Err(err) if err.kind() == ErrorKind::NotFound => {
|
||||||
|
missing_tracks.push(MissingTrack::from_row(row, MissingReason::MissingFile));
|
||||||
|
}
|
||||||
|
Err(err) => {
|
||||||
|
skipped_io_errors += 1;
|
||||||
|
log.warn(&format!(
|
||||||
|
"Archive cleanup: skipping track {} media_file_id={media_file_id}; cannot inspect {}: {err}",
|
||||||
|
row.track_id,
|
||||||
|
absolute_path.display()
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if missing_tracks.is_empty() {
|
||||||
|
log.info(&format!(
|
||||||
|
"Archive cleanup: all checked tracks have readable audio files; skipped_io_errors={skipped_io_errors}"
|
||||||
|
));
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
|
||||||
|
for (index, track) in missing_tracks.iter().take(SAMPLE_LOG_LIMIT).enumerate() {
|
||||||
|
log.warn(&format!(
|
||||||
|
"Archive cleanup: deleting stale track {} \"{}\"{}{} ({})",
|
||||||
|
track.track_id,
|
||||||
|
track.track_title,
|
||||||
|
track
|
||||||
|
.release_title
|
||||||
|
.as_deref()
|
||||||
|
.map(|title| format!(" from \"{title}\""))
|
||||||
|
.unwrap_or_default(),
|
||||||
|
track
|
||||||
|
.file_path
|
||||||
|
.as_deref()
|
||||||
|
.map(|path| format!(", path={path}"))
|
||||||
|
.unwrap_or_default(),
|
||||||
|
track.reason
|
||||||
|
));
|
||||||
|
if index + 1 == SAMPLE_LOG_LIMIT && missing_tracks.len() > SAMPLE_LOG_LIMIT {
|
||||||
|
log.warn(&format!(
|
||||||
|
"Archive cleanup: suppressing per-track logs for remaining {} stale track(s)",
|
||||||
|
missing_tracks.len() - SAMPLE_LOG_LIMIT
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let track_ids = unique_sorted(
|
||||||
|
missing_tracks
|
||||||
|
.iter()
|
||||||
|
.map(|track| track.track_id)
|
||||||
|
.collect::<Vec<_>>(),
|
||||||
|
);
|
||||||
|
let media_file_ids = unique_sorted(
|
||||||
|
missing_tracks
|
||||||
|
.iter()
|
||||||
|
.filter_map(|track| track.media_file_id)
|
||||||
|
.collect::<Vec<_>>(),
|
||||||
|
);
|
||||||
|
let release_ids = unique_sorted(
|
||||||
|
missing_tracks
|
||||||
|
.iter()
|
||||||
|
.map(|track| track.release_id)
|
||||||
|
.collect::<Vec<_>>(),
|
||||||
|
);
|
||||||
|
|
||||||
|
let stats =
|
||||||
|
delete_tracks_and_unreferenced_audio_media(&ctx.pool, &track_ids, &media_file_ids).await?;
|
||||||
|
let empty_release_count = count_empty_releases(&ctx.pool, &release_ids).await?;
|
||||||
|
|
||||||
|
log.info(&format!(
|
||||||
|
"Archive cleanup: deleted {} track(s), {} unreferenced audio media_file row(s); cleared playback_states={}, playlist_entries={}, likes={}, play_history={}, popularity_history={}, scrobble_outbox={}, track_genres={}, entity_tags={}, external_ids={}, track_artists={}; skipped_io_errors={skipped_io_errors}; empty_releases_left={empty_release_count}",
|
||||||
|
stats.tracks_deleted,
|
||||||
|
stats.media_files_deleted,
|
||||||
|
stats.playback_states_cleared,
|
||||||
|
stats.playlist_entries_deleted,
|
||||||
|
stats.likes_deleted,
|
||||||
|
stats.play_history_deleted,
|
||||||
|
stats.popularity_history_deleted,
|
||||||
|
stats.scrobble_outbox_deleted,
|
||||||
|
stats.track_genres_deleted,
|
||||||
|
stats.entity_tags_deleted,
|
||||||
|
stats.external_ids_deleted,
|
||||||
|
stats.track_artists_deleted,
|
||||||
|
));
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
impl MissingTrack {
|
||||||
|
fn from_row(row: TrackFileRow, reason: MissingReason) -> Self {
|
||||||
|
Self {
|
||||||
|
track_id: row.track_id,
|
||||||
|
track_title: row.track_title,
|
||||||
|
release_id: row.release_id,
|
||||||
|
release_title: row.release_title,
|
||||||
|
media_file_id: row.media_file_id,
|
||||||
|
file_path: row.file_path,
|
||||||
|
reason,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::fmt::Display for MissingReason {
|
||||||
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
match self {
|
||||||
|
Self::MissingMediaRow => f.write_str("missing media_file row"),
|
||||||
|
Self::InvalidMediaType(file_type) => write!(f, "invalid media_file type {file_type:?}"),
|
||||||
|
Self::EmptyPath => f.write_str("empty media file path"),
|
||||||
|
Self::MissingFile => f.write_str("audio file not found on disk"),
|
||||||
|
Self::NotRegularFile => f.write_str("audio path is not a regular file"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn unique_sorted(values: Vec<i64>) -> Vec<i64> {
|
||||||
|
values
|
||||||
|
.into_iter()
|
||||||
|
.collect::<BTreeSet<_>>()
|
||||||
|
.into_iter()
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn delete_tracks_and_unreferenced_audio_media(
|
||||||
|
pool: &PgPool,
|
||||||
|
track_ids: &[i64],
|
||||||
|
media_file_ids: &[i64],
|
||||||
|
) -> anyhow::Result<DeleteStats> {
|
||||||
|
if track_ids.is_empty() {
|
||||||
|
return Ok(DeleteStats::default());
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut tx = pool.begin().await?;
|
||||||
|
let mut stats = DeleteStats::default();
|
||||||
|
|
||||||
|
stats.playback_states_cleared = sqlx::query(
|
||||||
|
r#"UPDATE furumusic__playback_state
|
||||||
|
SET current_track_id = NULL
|
||||||
|
WHERE current_track_id = ANY($1)"#,
|
||||||
|
)
|
||||||
|
.bind(track_ids)
|
||||||
|
.execute(&mut *tx)
|
||||||
|
.await?
|
||||||
|
.rows_affected();
|
||||||
|
|
||||||
|
stats.playlist_entries_deleted =
|
||||||
|
delete_track_rows(&mut tx, "furumusic__playlist_track", track_ids).await?;
|
||||||
|
stats.likes_deleted =
|
||||||
|
delete_track_rows(&mut tx, "furumusic__user_liked_track", track_ids).await?;
|
||||||
|
stats.play_history_deleted =
|
||||||
|
delete_track_rows(&mut tx, "furumusic__play_history", track_ids).await?;
|
||||||
|
stats.popularity_history_deleted =
|
||||||
|
delete_track_rows(&mut tx, "furumusic__track_popularity_history", track_ids).await?;
|
||||||
|
stats.scrobble_outbox_deleted =
|
||||||
|
delete_track_rows(&mut tx, "furumusic__lastfm_scrobble_outbox", track_ids).await?;
|
||||||
|
stats.track_genres_deleted =
|
||||||
|
delete_track_rows(&mut tx, "furumusic__track_genre", track_ids).await?;
|
||||||
|
|
||||||
|
stats.entity_tags_deleted = sqlx::query(
|
||||||
|
r#"DELETE FROM furumusic__entity_genre_tag
|
||||||
|
WHERE entity_kind = 'track'
|
||||||
|
AND entity_id = ANY($1)"#,
|
||||||
|
)
|
||||||
|
.bind(track_ids)
|
||||||
|
.execute(&mut *tx)
|
||||||
|
.await?
|
||||||
|
.rows_affected();
|
||||||
|
|
||||||
|
stats.external_ids_deleted = sqlx::query(
|
||||||
|
r#"DELETE FROM furumusic__external_metadata_id
|
||||||
|
WHERE entity_kind = 'track'
|
||||||
|
AND entity_id = ANY($1)"#,
|
||||||
|
)
|
||||||
|
.bind(track_ids)
|
||||||
|
.execute(&mut *tx)
|
||||||
|
.await?
|
||||||
|
.rows_affected();
|
||||||
|
|
||||||
|
stats.track_artists_deleted =
|
||||||
|
delete_track_rows(&mut tx, "furumusic__track_artist", track_ids).await?;
|
||||||
|
|
||||||
|
stats.tracks_deleted = sqlx::query("DELETE FROM furumusic__track WHERE id = ANY($1)")
|
||||||
|
.bind(track_ids)
|
||||||
|
.execute(&mut *tx)
|
||||||
|
.await?
|
||||||
|
.rows_affected();
|
||||||
|
|
||||||
|
if !media_file_ids.is_empty() {
|
||||||
|
stats.media_files_deleted = sqlx::query(
|
||||||
|
r#"DELETE FROM furumusic__media_file mf
|
||||||
|
WHERE mf.id = ANY($1)
|
||||||
|
AND mf.file_type = 'audio'
|
||||||
|
AND NOT EXISTS (
|
||||||
|
SELECT 1
|
||||||
|
FROM furumusic__track t
|
||||||
|
WHERE t.audio_file_id = mf.id
|
||||||
|
OR t.cover_file_id = mf.id
|
||||||
|
)
|
||||||
|
AND NOT EXISTS (
|
||||||
|
SELECT 1
|
||||||
|
FROM furumusic__release r
|
||||||
|
WHERE r.cover_file_id = mf.id
|
||||||
|
)
|
||||||
|
AND NOT EXISTS (
|
||||||
|
SELECT 1
|
||||||
|
FROM furumusic__artist a
|
||||||
|
WHERE a.image_file_id = mf.id
|
||||||
|
)
|
||||||
|
AND NOT EXISTS (
|
||||||
|
SELECT 1
|
||||||
|
FROM furumusic__playlist p
|
||||||
|
WHERE p.cover_file_id = mf.id
|
||||||
|
)"#,
|
||||||
|
)
|
||||||
|
.bind(media_file_ids)
|
||||||
|
.execute(&mut *tx)
|
||||||
|
.await?
|
||||||
|
.rows_affected();
|
||||||
|
}
|
||||||
|
|
||||||
|
tx.commit().await?;
|
||||||
|
Ok(stats)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn delete_track_rows(
|
||||||
|
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
|
||||||
|
table: &str,
|
||||||
|
track_ids: &[i64],
|
||||||
|
) -> anyhow::Result<u64> {
|
||||||
|
let sql = format!("DELETE FROM {table} WHERE track_id = ANY($1)");
|
||||||
|
Ok(sqlx::query(&sql)
|
||||||
|
.bind(track_ids)
|
||||||
|
.execute(&mut **tx)
|
||||||
|
.await?
|
||||||
|
.rows_affected())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn count_empty_releases(pool: &PgPool, release_ids: &[i64]) -> anyhow::Result<i64> {
|
||||||
|
if release_ids.is_empty() {
|
||||||
|
return Ok(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
let count = sqlx::query_scalar::<_, i64>(
|
||||||
|
r#"SELECT COUNT(*)
|
||||||
|
FROM furumusic__release r
|
||||||
|
WHERE r.id = ANY($1)
|
||||||
|
AND NOT EXISTS (
|
||||||
|
SELECT 1
|
||||||
|
FROM furumusic__track t
|
||||||
|
WHERE t.release_id = r.id
|
||||||
|
)"#,
|
||||||
|
)
|
||||||
|
.bind(release_ids)
|
||||||
|
.fetch_one(pool)
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
Ok(count)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn unique_sorted_deduplicates_ids() {
|
||||||
|
assert_eq!(unique_sorted(vec![3, 1, 3, 2, 1]), vec![1, 2, 3]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn missing_reason_display_is_stable() {
|
||||||
|
assert_eq!(
|
||||||
|
MissingReason::InvalidMediaType("cover_art".to_owned()).to_string(),
|
||||||
|
"invalid media_file type \"cover_art\""
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
MissingReason::MissingFile.to_string(),
|
||||||
|
"audio file not found on disk"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -128,7 +128,11 @@ impl Job for InboxDiscoverJob {
|
|||||||
v
|
v
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
crate::metrics::record_agent_file_hash(hash_start.elapsed(), 0, "error");
|
crate::metrics::record_agent_file_hash(
|
||||||
|
hash_start.elapsed(),
|
||||||
|
0,
|
||||||
|
"error",
|
||||||
|
);
|
||||||
log.warn(&format!("Failed to hash {}: {e}", file_path.display()));
|
log.warn(&format!("Failed to hash {}: {e}", file_path.display()));
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -494,7 +494,12 @@ async fn process_folder_batch(
|
|||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
Ok(Ok(results)) => {
|
Ok(Ok(results)) => {
|
||||||
crate::metrics::record_agent_rag("artist", "ok", rag_start.elapsed(), results.len());
|
crate::metrics::record_agent_rag(
|
||||||
|
"artist",
|
||||||
|
"ok",
|
||||||
|
rag_start.elapsed(),
|
||||||
|
results.len(),
|
||||||
|
);
|
||||||
for a in results {
|
for a in results {
|
||||||
if !all_similar_artists
|
if !all_similar_artists
|
||||||
.iter()
|
.iter()
|
||||||
@@ -525,7 +530,12 @@ async fn process_folder_batch(
|
|||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
Ok(Ok(results)) => {
|
Ok(Ok(results)) => {
|
||||||
crate::metrics::record_agent_rag("release", "ok", rag_start.elapsed(), results.len());
|
crate::metrics::record_agent_rag(
|
||||||
|
"release",
|
||||||
|
"ok",
|
||||||
|
rag_start.elapsed(),
|
||||||
|
results.len(),
|
||||||
|
);
|
||||||
for r in results {
|
for r in results {
|
||||||
if !all_similar_releases
|
if !all_similar_releases
|
||||||
.iter()
|
.iter()
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
pub mod archive_cleanup;
|
||||||
pub mod artwork_backfill;
|
pub mod artwork_backfill;
|
||||||
pub mod inbox_discover;
|
pub mod inbox_discover;
|
||||||
pub mod inbox_process;
|
pub mod inbox_process;
|
||||||
|
|||||||
@@ -52,6 +52,7 @@ fn build_registry() -> Arc<JobRegistry> {
|
|||||||
registry.register(jobs::inbox_discover::InboxDiscoverJob);
|
registry.register(jobs::inbox_discover::InboxDiscoverJob);
|
||||||
registry.register(jobs::inbox_process::InboxProcessJob);
|
registry.register(jobs::inbox_process::InboxProcessJob);
|
||||||
registry.register(jobs::inbox_process::FileProcessJob);
|
registry.register(jobs::inbox_process::FileProcessJob);
|
||||||
|
registry.register(jobs::archive_cleanup::ArchiveCleanupJob);
|
||||||
registry.register(jobs::artwork_backfill::ArtworkBackfillJob);
|
registry.register(jobs::artwork_backfill::ArtworkBackfillJob);
|
||||||
registry.register(jobs::metadata_backfill::MetadataBackfillJob);
|
registry.register(jobs::metadata_backfill::MetadataBackfillJob);
|
||||||
registry.register(jobs::lastfm_popularity::LastfmPopularityJob);
|
registry.register(jobs::lastfm_popularity::LastfmPopularityJob);
|
||||||
|
|||||||
+195
-2
@@ -981,11 +981,151 @@ fn safe_mobile_redirect_uri(raw: Option<&str>) -> Option<String> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn mobile_redirect_success(app_redirect_uri: &str, code: &str) -> cot::response::Response {
|
fn mobile_redirect_success(app_redirect_uri: &str, code: &str) -> cot::response::Response {
|
||||||
auth::redirect(&append_query_param(app_redirect_uri, "code", code))
|
let deep_link = append_query_param(app_redirect_uri, "code", code);
|
||||||
|
mobile_deep_link_page(
|
||||||
|
"success",
|
||||||
|
"Sign-in complete",
|
||||||
|
"Furumi should open automatically. You can close this window after the app opens.",
|
||||||
|
None,
|
||||||
|
&deep_link,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn mobile_redirect_error(app_redirect_uri: &str, error: &str) -> cot::response::Response {
|
fn mobile_redirect_error(app_redirect_uri: &str, error: &str) -> cot::response::Response {
|
||||||
auth::redirect(&append_query_param(app_redirect_uri, "error", error))
|
let deep_link = append_query_param(app_redirect_uri, "error", error);
|
||||||
|
mobile_deep_link_page(
|
||||||
|
"error",
|
||||||
|
"Sign-in failed",
|
||||||
|
"Furumi should open automatically and show the sign-in error. You can close this window after the app opens.",
|
||||||
|
Some(error),
|
||||||
|
&deep_link,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn mobile_deep_link_page(
|
||||||
|
state: &str,
|
||||||
|
title: &str,
|
||||||
|
message: &str,
|
||||||
|
detail: Option<&str>,
|
||||||
|
deep_link: &str,
|
||||||
|
) -> cot::response::Response {
|
||||||
|
let state_class = html_escape(state);
|
||||||
|
let title_html = html_escape(title);
|
||||||
|
let message_html = html_escape(message);
|
||||||
|
let detail_html = detail
|
||||||
|
.map(|value| format!(r#"<p class="detail">Reason: {}</p>"#, html_escape(value)))
|
||||||
|
.unwrap_or_default();
|
||||||
|
let deep_link_html = html_escape(deep_link);
|
||||||
|
let deep_link_js =
|
||||||
|
serde_json::to_string(deep_link).expect("serializing URL string cannot fail");
|
||||||
|
|
||||||
|
let html = format!(
|
||||||
|
r#"<!doctype html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8">
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>{title_html}</title>
|
||||||
|
<style>
|
||||||
|
:root {{
|
||||||
|
color-scheme: light dark;
|
||||||
|
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
|
||||||
|
background: #101114;
|
||||||
|
color: #f5f2ea;
|
||||||
|
}}
|
||||||
|
body {{
|
||||||
|
min-height: 100vh;
|
||||||
|
margin: 0;
|
||||||
|
display: grid;
|
||||||
|
place-items: center;
|
||||||
|
padding: 24px;
|
||||||
|
box-sizing: border-box;
|
||||||
|
}}
|
||||||
|
main {{
|
||||||
|
width: min(420px, 100%);
|
||||||
|
text-align: center;
|
||||||
|
}}
|
||||||
|
.mark {{
|
||||||
|
width: 54px;
|
||||||
|
height: 54px;
|
||||||
|
margin: 0 auto 18px;
|
||||||
|
border-radius: 999px;
|
||||||
|
display: grid;
|
||||||
|
place-items: center;
|
||||||
|
font-size: 18px;
|
||||||
|
font-weight: 700;
|
||||||
|
background: #2f7d52;
|
||||||
|
color: white;
|
||||||
|
}}
|
||||||
|
.mark.error {{
|
||||||
|
background: #9d3d42;
|
||||||
|
}}
|
||||||
|
h1 {{
|
||||||
|
margin: 0 0 10px;
|
||||||
|
font-size: 26px;
|
||||||
|
line-height: 1.15;
|
||||||
|
letter-spacing: 0;
|
||||||
|
}}
|
||||||
|
p {{
|
||||||
|
margin: 0;
|
||||||
|
color: #c9c2b7;
|
||||||
|
font-size: 15px;
|
||||||
|
line-height: 1.55;
|
||||||
|
}}
|
||||||
|
.detail {{
|
||||||
|
margin-top: 12px;
|
||||||
|
color: #f1b3b7;
|
||||||
|
overflow-wrap: anywhere;
|
||||||
|
}}
|
||||||
|
a {{
|
||||||
|
display: inline-flex;
|
||||||
|
align-items: center;
|
||||||
|
justify-content: center;
|
||||||
|
min-height: 44px;
|
||||||
|
margin-top: 24px;
|
||||||
|
padding: 0 18px;
|
||||||
|
border-radius: 8px;
|
||||||
|
background: #e8d8a8;
|
||||||
|
color: #17150f;
|
||||||
|
font-weight: 700;
|
||||||
|
text-decoration: none;
|
||||||
|
}}
|
||||||
|
.hint {{
|
||||||
|
margin-top: 14px;
|
||||||
|
font-size: 13px;
|
||||||
|
color: #89847c;
|
||||||
|
}}
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<main>
|
||||||
|
<div class="mark {state_class}" aria-hidden="true">{mark}</div>
|
||||||
|
<h1>{title_html}</h1>
|
||||||
|
<p>{message_html}</p>
|
||||||
|
{detail_html}
|
||||||
|
<a href="{deep_link_html}">Open Furumi</a>
|
||||||
|
<p class="hint">If nothing happens, use the button above.</p>
|
||||||
|
</main>
|
||||||
|
<script>
|
||||||
|
const deepLink = {deep_link_js};
|
||||||
|
window.setTimeout(() => {{
|
||||||
|
window.location.href = deepLink;
|
||||||
|
}}, 100);
|
||||||
|
window.setTimeout(() => {{
|
||||||
|
window.close();
|
||||||
|
}}, 1800);
|
||||||
|
</script>
|
||||||
|
</body>
|
||||||
|
</html>"#,
|
||||||
|
mark = if state == "error" { "!" } else { "OK" }
|
||||||
|
);
|
||||||
|
|
||||||
|
cot::http::Response::builder()
|
||||||
|
.status(cot::http::StatusCode::OK)
|
||||||
|
.header(cot::http::header::CONTENT_TYPE, "text/html; charset=utf-8")
|
||||||
|
.header(cot::http::header::CACHE_CONTROL, "no-store")
|
||||||
|
.body(cot::Body::fixed(html))
|
||||||
|
.expect("valid response")
|
||||||
}
|
}
|
||||||
|
|
||||||
fn append_query_param(uri: &str, key: &str, value: &str) -> String {
|
fn append_query_param(uri: &str, key: &str, value: &str) -> String {
|
||||||
@@ -999,6 +1139,21 @@ fn append_query_param(uri: &str, key: &str, value: &str) -> String {
|
|||||||
out
|
out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn html_escape(value: &str) -> String {
|
||||||
|
let mut out = String::with_capacity(value.len());
|
||||||
|
for ch in value.chars() {
|
||||||
|
match ch {
|
||||||
|
'&' => out.push_str("&"),
|
||||||
|
'<' => out.push_str("<"),
|
||||||
|
'>' => out.push_str(">"),
|
||||||
|
'"' => out.push_str("""),
|
||||||
|
'\'' => out.push_str("'"),
|
||||||
|
_ => out.push(ch),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
fn extract_groups_from_jwt(token: &str) -> Vec<String> {
|
fn extract_groups_from_jwt(token: &str) -> Vec<String> {
|
||||||
use base64::Engine;
|
use base64::Engine;
|
||||||
|
|
||||||
@@ -1038,3 +1193,41 @@ fn urlencoded(s: &str) -> String {
|
|||||||
}
|
}
|
||||||
out
|
out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn mobile_oidc_append_query_param_preserves_fragment() {
|
||||||
|
assert_eq!(
|
||||||
|
append_query_param("furumi://auth/callback#done", "code", "a b"),
|
||||||
|
"furumi://auth/callback?code=a%20b#done"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
append_query_param("furumi://auth/callback?desktop=1", "error", "oidc_error"),
|
||||||
|
"furumi://auth/callback?desktop=1&error=oidc_error"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn mobile_oidc_html_escape_escapes_page_values() {
|
||||||
|
assert_eq!(
|
||||||
|
html_escape(r#"<tag attr="x&y">'text'</tag>"#),
|
||||||
|
"<tag attr="x&y">'text'</tag>"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn mobile_oidc_redirect_uri_allows_only_furumi_schemes() {
|
||||||
|
assert_eq!(
|
||||||
|
safe_mobile_redirect_uri(Some("furumi://auth/callback")).as_deref(),
|
||||||
|
Some("furumi://auth/callback")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
safe_mobile_redirect_uri(Some("furumusic://auth/callback")).as_deref(),
|
||||||
|
Some("furumusic://auth/callback")
|
||||||
|
);
|
||||||
|
assert!(safe_mobile_redirect_uri(Some("https://example.com/callback")).is_none());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -804,6 +804,10 @@ fn normalize_device_id(raw: &str) -> Option<String> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn device_name_from_user_agent(user_agent: Option<&str>) -> String {
|
fn device_name_from_user_agent(user_agent: Option<&str>) -> String {
|
||||||
|
if let Some(name) = native_device_name_from_user_agent(user_agent) {
|
||||||
|
return name;
|
||||||
|
}
|
||||||
|
|
||||||
let ua = user_agent.unwrap_or_default().to_ascii_lowercase();
|
let ua = user_agent.unwrap_or_default().to_ascii_lowercase();
|
||||||
let browser = if ua.contains("edg/") || ua.contains("edgios/") || ua.contains("edga/") {
|
let browser = if ua.contains("edg/") || ua.contains("edgios/") || ua.contains("edga/") {
|
||||||
"Edge"
|
"Edge"
|
||||||
@@ -838,8 +842,54 @@ fn device_name_from_user_agent(user_agent: Option<&str>) -> String {
|
|||||||
format!("{browser} on {os}")
|
format!("{browser} on {os}")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn native_device_name_from_user_agent(user_agent: Option<&str>) -> Option<String> {
|
||||||
|
let raw = user_agent?.trim();
|
||||||
|
for token in raw.split_ascii_whitespace() {
|
||||||
|
let Some((product, version)) = token.split_once('/') else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let version = sanitize_user_agent_version(version);
|
||||||
|
if product.eq_ignore_ascii_case("FurumiAndroid") {
|
||||||
|
return Some(match version.as_deref() {
|
||||||
|
Some(v) => format!("Furumi Android {v}"),
|
||||||
|
None => "Furumi Android".to_string(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if product.eq_ignore_ascii_case("FurumiMacOS") {
|
||||||
|
return Some(match version.as_deref() {
|
||||||
|
Some(v) => format!("Furumi MacOS {v}"),
|
||||||
|
None => "Furumi MacOS".to_string(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
|
fn sanitize_user_agent_version(version: &str) -> Option<String> {
|
||||||
|
let version = version
|
||||||
|
.chars()
|
||||||
|
.take(32)
|
||||||
|
.filter(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '.' | '-' | '_'))
|
||||||
|
.collect::<String>();
|
||||||
|
if version.is_empty() {
|
||||||
|
None
|
||||||
|
} else {
|
||||||
|
Some(version)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn device_kind_from_user_agent(user_agent: Option<&str>) -> &'static str {
|
fn device_kind_from_user_agent(user_agent: Option<&str>) -> &'static str {
|
||||||
let ua = user_agent.unwrap_or_default().to_ascii_lowercase();
|
let ua = user_agent.unwrap_or_default().to_ascii_lowercase();
|
||||||
|
if ua.contains("furumiandroid/") {
|
||||||
|
return if ua.contains("tablet") || (ua.contains("android") && !ua.contains("mobile")) {
|
||||||
|
"tablet"
|
||||||
|
} else {
|
||||||
|
"phone"
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if ua.contains("furumimac") {
|
||||||
|
return "computer";
|
||||||
|
}
|
||||||
if ua.contains("iphone") || (ua.contains("android") && ua.contains("mobile")) {
|
if ua.contains("iphone") || (ua.contains("android") && ua.contains("mobile")) {
|
||||||
"phone"
|
"phone"
|
||||||
} else if ua.contains("ipad") || ua.contains("tablet") || ua.contains("android") {
|
} else if ua.contains("ipad") || ua.contains("tablet") || ua.contains("android") {
|
||||||
@@ -849,6 +899,33 @@ fn device_kind_from_user_agent(user_agent: Option<&str>) -> &'static str {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod device_tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn detects_furumi_android_native_client() {
|
||||||
|
let user_agent = Some("FurumiAndroid/1.0 Android Mobile");
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
device_name_from_user_agent(user_agent),
|
||||||
|
"Furumi Android 1.0"
|
||||||
|
);
|
||||||
|
assert_eq!(device_kind_from_user_agent(user_agent), "phone");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn keeps_browser_fallback_for_generic_android_user_agents() {
|
||||||
|
let user_agent = Some("Mozilla/5.0 Android Mobile");
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
device_name_from_user_agent(user_agent),
|
||||||
|
"Browser on Android"
|
||||||
|
);
|
||||||
|
assert_eq!(device_kind_from_user_agent(user_agent), "phone");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, sqlx::FromRow)]
|
#[derive(Debug, sqlx::FromRow)]
|
||||||
struct LastfmAccountApiRow {
|
struct LastfmAccountApiRow {
|
||||||
session_key: String,
|
session_key: String,
|
||||||
|
|||||||
@@ -1372,6 +1372,7 @@ async fn run_scheduled_job(
|
|||||||
if !live_config.agent_enabled
|
if !live_config.agent_enabled
|
||||||
&& job_name != "lastfm_popularity"
|
&& job_name != "lastfm_popularity"
|
||||||
&& job_name != "lastfm_scrobble"
|
&& job_name != "lastfm_scrobble"
|
||||||
|
&& job_name != "archive_cleanup"
|
||||||
&& job_name != "artwork_backfill"
|
&& job_name != "artwork_backfill"
|
||||||
{
|
{
|
||||||
tracing::warn!(job = job_name, "Skipping: agent_enabled=false");
|
tracing::warn!(job = job_name, "Skipping: agent_enabled=false");
|
||||||
|
|||||||
Reference in New Issue
Block a user