Improved AI metadata processing
Build and Publish / Build and Publish Docker Image (push) Successful in 5m33s

This commit is contained in:
Aleksandr Bogomiakov committed 2026-10-15 00:29:37 +01:00
1 parent 8c0ea444f0
commit 2b3b50f7af
20 files changed
+1916 -68

No files matched your search

+1046
View File
File diff suppressed because it is too large. Load diff
+3
View File
@@ -1,5 +1,6 @@
pub mod cover_art;
pub mod cover_variants;
pub mod dedup;
pub mod dto;
pub mod metadata;
pub mod mover;
@@ -54,6 +55,8 @@ pub async fn probe_llm(llm_url: &str, llm_model: &str, llm_auth: &str) -> AgentP
],
"stream": false,
"temperature": 0.3,
"reasoning_effort": "none",
"chat_template_kwargs": {"enable_thinking": false},
"max_tokens": 256
});
+86 -15
View File
@@ -11,9 +11,9 @@ pub enum MoveOutcome {
///
/// Creates the directory structure: `storage_dir/artist/album/filename`
///
/// If `rename` fails (cross-device), falls back to copy + remove.
/// If the destination already exists the inbox copy is removed and
/// `MoveOutcome::Merged` is returned.
/// Copies into the destination filesystem before removing the inbox source.
/// Filename collisions are merged only when both physical files have the
/// same SHA-256. Distinct recordings receive a separate storage filename.
pub async fn move_to_storage(
storage_dir: &Path,
artist: &str,
@@ -27,31 +27,72 @@ pub async fn move_to_storage(
let dest_dir = storage_dir.join(&artist_dir).join(&album_dir);
tokio::fs::create_dir_all(&dest_dir).await?;
let dest = dest_dir.join(filename);
let mut dest = dest_dir.join(filename);
// File already at destination — remove the inbox duplicate
if dest.exists() {
if source.exists() {
if sha256_file(source).await? == sha256_file(&dest).await? {
tokio::fs::remove_file(source).await?;
tracing::info!(from = ?source, to = ?dest, "merged duplicate into existing storage file");
return Ok(MoveOutcome::Merged(dest));
}
return Ok(MoveOutcome::Merged(dest));
let stem = Path::new(filename)
.file_stem()
.and_then(|s| s.to_str())
.unwrap_or("audio");
let ext = Path::new(filename)
.extension()
.and_then(|s| s.to_str())
.unwrap_or("audio");
dest = dest_dir.join(format!("{stem} [{}].{ext}", uuid::Uuid::new_v4()));
}
// Try atomic rename first (same filesystem)
match tokio::fs::rename(source, &dest).await {
Ok(()) => {}
Err(_) => {
// Cross-device: copy then remove
tokio::fs::copy(source, &dest).await?;
tokio::fs::remove_file(source).await?;
}
}
publish_copy(source, &dest).await?;
tokio::fs::remove_file(source).await?;
tracing::info!(from = ?source, to = ?dest, "moved file to storage");
Ok(MoveOutcome::Moved(dest))
}
/// Copy into the destination filesystem, then expose the complete file without
/// replacing an existing destination. The source remains available on errors.
pub async fn publish_copy(source: &Path, destination: &Path) -> anyhow::Result<()> {
let parent = destination
.parent()
.ok_or_else(|| anyhow::anyhow!("destination has no parent"))?;
tokio::fs::create_dir_all(parent).await?;
let temporary = parent.join(format!(".{}.moving", uuid::Uuid::new_v4()));
let result = async {
tokio::fs::copy(source, &temporary).await?;
tokio::fs::hard_link(&temporary, destination).await
}
.await;
let _ = tokio::fs::remove_file(temporary).await;
result?;
Ok(())
}
pub async fn sha256_file(path: &Path) -> anyhow::Result<String> {
let path = path.to_owned();
tokio::task::spawn_blocking(move || {
use sha2::{Digest, Sha256};
use std::io::Read;
let mut file = std::fs::File::open(path)?;
let mut hash = Sha256::new();
let mut buffer = [0; 65536];
loop {
let count = file.read(&mut buffer)?;
if count == 0 {
break;
}
hash.update(&buffer[..count]);
}
Ok::<_, std::io::Error>(format!("{:x}", hash.finalize()))
})
.await?
.map_err(Into::into)
}
/// Remove characters that are unsafe for directory names.
fn sanitize_dir_name(name: &str) -> String {
name.chars()
@@ -64,3 +105,33 @@ fn sanitize_dir_name(name: &str) -> String {
.trim_matches('.')
.to_owned()
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn filename_collision_does_not_discard_a_different_recording() {
let root = std::env::temp_dir().join(format!("furumi-move-{}", uuid::Uuid::new_v4()));
let storage = root.join("storage");
tokio::fs::create_dir_all(storage.join("Artist/Album"))
.await
.unwrap();
let old = storage.join("Artist/Album/song.mp3");
let incoming = root.join("incoming.mp3");
tokio::fs::write(&old, b"original recording").await.unwrap();
tokio::fs::write(&incoming, b"different recording")
.await
.unwrap();
let outcome = move_to_storage(&storage, "Artist", "Album", "song.mp3", &incoming)
.await
.unwrap();
let MoveOutcome::Moved(new) = outcome else {
panic!("different recording was merged")
};
assert_ne!(new, old);
assert_eq!(tokio::fs::read(&old).await.unwrap(), b"original recording");
assert_eq!(tokio::fs::read(new).await.unwrap(), b"different recording");
tokio::fs::remove_dir_all(root).await.unwrap();
}
}
+140 -3
View File
@@ -1,3 +1,4 @@
use anyhow::Context;
use serde::{Deserialize, Serialize};
use super::dto::{
@@ -8,6 +9,9 @@ use super::dto::{
// Types
// ---------------------------------------------------------------------------
// Large album batches can take longer than ten minutes on the local model.
const LLM_REQUEST_TIMEOUT_SECS: u64 = 3600;
/// A single message in the chat history.
#[derive(Clone, Serialize)]
pub struct ChatMessage {
@@ -22,6 +26,13 @@ struct ChatRequest {
response_format: ChatResponseFormat,
stream: bool,
temperature: f64,
reasoning_effort: &'static str,
chat_template_kwargs: ChatTemplateKwargs,
}
#[derive(Serialize)]
struct ChatTemplateKwargs {
enable_thinking: bool,
}
#[derive(Serialize)]
@@ -60,7 +71,8 @@ async fn call_llm_chat(
auth: Option<&str>,
) -> anyhow::Result<(String, String, ChatUsage)> {
let client = reqwest::Client::builder()
.timeout(std::time::Duration::from_secs(600))
.connect_timeout(std::time::Duration::from_secs(15))
.timeout(std::time::Duration::from_secs(LLM_REQUEST_TIMEOUT_SECS))
.build()?;
let request = ChatRequest {
@@ -71,6 +83,11 @@ async fn call_llm_chat(
},
stream: false,
temperature: 0.1,
// Disable reasoning through both the API and the model's chat template.
reasoning_effort: "none",
chat_template_kwargs: ChatTemplateKwargs {
enable_thinking: false,
},
};
let url = format!("{}/v1/chat/completions", base_url.trim_end_matches('/'));
@@ -78,6 +95,8 @@ async fn call_llm_chat(
%url,
model,
message_count = messages.len(),
timeout_secs = LLM_REQUEST_TIMEOUT_SECS,
thinking = false,
"Calling LLM API (chat mode)..."
);
@@ -86,7 +105,17 @@ async fn call_llm_chat(
if let Some(auth_header) = auth {
req = req.header("Authorization", auth_header);
}
let resp = req.send().await?;
let resp = req.send().await.map_err(|error| {
let elapsed_secs = start.elapsed().as_secs();
let message = if error.is_timeout() {
format!(
"LLM request timed out after {elapsed_secs}s (request timeout: {LLM_REQUEST_TIMEOUT_SECS}s, connect timeout: 15s)"
)
} else {
format!("LLM request failed after {elapsed_secs}s")
};
anyhow::Error::new(error).context(message)
})?;
let elapsed = start.elapsed();
if !resp.status().is_success() {
@@ -97,7 +126,10 @@ async fn call_llm_chat(
anyhow::bail!("LLM returned {}: {}", status, body);
}
let chat_resp: ChatResponse = resp.json().await?;
let chat_resp: ChatResponse = resp
.json()
.await
.context("Failed to decode LLM chat response")?;
let resp_model = chat_resp.model.unwrap_or_else(|| model.to_owned());
let usage = chat_resp.usage.unwrap_or_default();
let content = chat_resp
@@ -605,3 +637,108 @@ fn parse_batch_response(
Ok(results)
}
#[cfg(test)]
mod tests {
use super::*;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
#[test]
fn mixed_folder_results_keep_metadata_per_file() {
let files = vec![
BatchFileInput {
filename: "01 - One.mp3".into(),
raw: RawMetadata::default(),
hints: PathHints::default(),
},
BatchFileInput {
filename: "02 - Two.mp3".into(),
raw: RawMetadata::default(),
hints: PathHints::default(),
},
];
let response = r#"{"results":[
{"filename":"02 - Two.mp3","artist":"Artist B","album":"Album B","year":2020},
{"filename":"01 - One.mp3","artist":"Artist A","album":"Album A","year":2002}
]}"#;
let results: std::collections::HashMap<_, _> = parse_batch_response(response, &files)
.unwrap()
.into_iter()
.collect();
assert_eq!(results["01 - One.mp3"].artist.as_deref(), Some("Artist A"));
assert_eq!(results["01 - One.mp3"].album.as_deref(), Some("Album A"));
assert_eq!(results["01 - One.mp3"].year, Some(2002));
assert_eq!(results["02 - Two.mp3"].artist.as_deref(), Some("Artist B"));
assert_eq!(results["02 - Two.mp3"].album.as_deref(), Some("Album B"));
assert_eq!(results["02 - Two.mp3"].year, Some(2020));
}
#[tokio::test]
async fn chat_request_disables_thinking_on_the_wire() {
tokio::time::timeout(std::time::Duration::from_secs(5), async {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
.await
.unwrap();
let url = format!("http://{}", listener.local_addr().unwrap());
let server = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.unwrap();
let mut request = Vec::new();
let mut buffer = [0; 4096];
let payload = loop {
let count = socket.read(&mut buffer).await.unwrap();
assert!(count > 0, "client closed before sending the request");
request.extend_from_slice(&buffer[..count]);
let Some(header_end) = request.windows(4).position(|w| w == b"\r\n\r\n")
else {
continue;
};
let headers = std::str::from_utf8(&request[..header_end]).unwrap();
assert!(headers.starts_with("POST /v1/chat/completions HTTP/1.1"));
let length = headers
.lines()
.find_map(|line| {
let (key, value) = line.split_once(':')?;
key.eq_ignore_ascii_case("content-length")
.then(|| value.trim().parse::<usize>().unwrap())
})
.unwrap();
if request.len() >= header_end + 4 + length {
break serde_json::from_slice::<serde_json::Value>(
&request[header_end + 4..header_end + 4 + length],
)
.unwrap();
}
};
let response = serde_json::json!({
"model": "local-model",
"choices": [{"message": {"content": "{\"results\":[]}"}}],
"usage": {"prompt_tokens": 10, "completion_tokens": 5}
})
.to_string();
socket
.write_all(
format!(
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{response}",
response.len()
)
.as_bytes(),
)
.await
.unwrap();
payload
});
let (content, model, usage) = call_llm_chat(&url, "local-model", &[], None)
.await
.unwrap();
let payload = server.await.unwrap();
assert_eq!(payload["reasoning_effort"], "none");
assert_eq!(payload["chat_template_kwargs"]["enable_thinking"], false);
assert_eq!(payload["response_format"]["type"], "json_object");
assert_eq!(content, "{\"results\":[]}");
assert_eq!(model, "local-model");
assert_eq!(usage.completion_tokens, Some(5));
})
.await
.expect("mock LLM request should finish within five seconds");
}
}
+89 -6
View File
@@ -28,13 +28,23 @@ pub fn parse(relative_path: &Path) -> PathHints {
parse_filename(stem, &mut hints);
match components.len() {
// Artist/Album/file.ext
// Artist/.../Album/file.ext: use the nearest release folder rather
// than an organizational directory such as "Альбомы и синглы".
3.. => {
hints.artist = Some(components[0].to_owned());
let album_raw = components[1];
let mut album_index = components.len() - 2;
if album_index > 0 && is_disc_folder(components[album_index]) {
album_index -= 1;
}
hints.artist = components[..album_index]
.iter()
.rev()
.find(|name| !is_organizational_folder(name))
.map(|name| (*name).to_owned());
let album_raw = components[album_index];
let (album, year) = parse_album_with_year(album_raw);
hints.album = Some(album);
if !is_organizational_folder(&album) {
hints.album = Some(album);
}
if year.is_some() {
hints.year = year;
}
@@ -43,7 +53,9 @@ pub fn parse(relative_path: &Path) -> PathHints {
2 => {
let dir = components[0];
let (name, year) = parse_album_with_year(dir);
hints.album = Some(name);
if !is_organizational_folder(&name) {
hints.album = Some(name);
}
if year.is_some() {
hints.year = year;
}
@@ -55,6 +67,34 @@ pub fn parse(relative_path: &Path) -> PathHints {
hints
}
fn is_organizational_folder(name: &str) -> bool {
matches!(
name.trim().to_lowercase().as_str(),
"music"
| "downloads"
| "albums"
| "singles"
| "albums and singles"
| "mixed"
| "mix"
| "музыка"
| "загрузки"
| "альбомы"
| "синглы"
| "альбомы и синглы"
| "разное"
)
}
fn is_disc_folder(name: &str) -> bool {
let name = name.trim().to_lowercase();
["cd", "disc", "disk", "диск"].iter().any(|prefix| {
name.strip_prefix(prefix)
.and_then(|number| number.trim().parse::<u32>().ok())
.is_some_and(|number| number > 0)
})
}
/// Try to extract track number and title from a filename stem.
///
/// Patterns: "01 - Title", "01. Title", "1 Title", "Title"
@@ -106,6 +146,14 @@ fn try_strip_track_prefix(s: &str) -> Option<(&str, &str)> {
///
/// Patterns: "Album (2001)", "(2001) Album", "Album [2001]", "Album"
fn parse_album_with_year(dir: &str) -> (String, Option<i32>) {
// Common discography folders: "2002 - Album".
if let Some((prefix, album)) = dir.split_once(" - ")
&& let Ok(year) = prefix.trim().parse::<i32>()
&& (1900..=2100).contains(&year)
&& !album.trim().is_empty()
{
return (album.trim().to_owned(), Some(year));
}
// Try "Album (YYYY)" or "Album [YYYY]"
for (open, close) in [('(', ')'), ('[', ']')] {
if let Some(start) = dir.rfind(open) {
@@ -191,4 +239,39 @@ mod tests {
assert_eq!(h.album.as_deref(), Some("Album"));
assert_eq!(h.year, Some(1999));
}
#[test]
fn nested_discography_uses_the_release_folder() {
let h = parse(Path::new(
"Diskoteka_Avariya-Diskografia/Альбомы и синглы/2002 - Х.Х.Х.И.Р.Н.Р/08 - Где же ваши руки.mp3",
));
assert_eq!(h.artist.as_deref(), Some("Diskoteka_Avariya-Diskografia"));
assert_eq!(h.album.as_deref(), Some("Х.Х.Х.И.Р.Н.Р"));
assert_eq!(h.year, Some(2002));
assert_eq!(h.track_number, Some(8));
}
#[test]
fn mixed_folder_does_not_supply_an_album() {
let h = parse(Path::new("Mix/Track.mp3"));
assert_eq!(h.artist, None);
assert_eq!(h.album, None);
assert_eq!(h.title.as_deref(), Some("Track"));
}
#[test]
fn collection_root_does_not_replace_the_artist() {
let h = parse(Path::new("Music/Artist/2020 - Album/01 - Song.flac"));
assert_eq!(h.artist.as_deref(), Some("Artist"));
assert_eq!(h.album.as_deref(), Some("Album"));
assert_eq!(h.year, Some(2020));
}
#[test]
fn disc_subfolder_keeps_the_album_and_artist() {
let h = parse(Path::new("Artist/2002 - Album/CD 1/01 - Song.flac"));
assert_eq!(h.artist.as_deref(), Some("Artist"));
assert_eq!(h.album.as_deref(), Some("Album"));
assert_eq!(h.year, Some(2002));
}
}
+32 -7
View File
@@ -83,15 +83,40 @@ pub async fn find_similar_releases(
/// are ignored so that re-discovery is possible after the user deletes
/// artists/releases/tracks.
pub async fn file_hash_exists(pool: &PgPool, sha256: &str) -> anyhow::Result<bool> {
let row: (bool,) = sqlx::query_as(
"SELECT EXISTS(\
SELECT 1 FROM furumusic__media_file mf \
JOIN furumusic__track t ON t.audio_file_id = mf.id \
WHERE mf.sha256_hash = $1\
)",
sqlx::query_scalar(
"SELECT EXISTS(SELECT 1 FROM furumusic__media_file mf \
JOIN furumusic__track t ON t.audio_file_id = mf.id \
WHERE mf.sha256_hash = $1)",
)
.bind(sha256)
.fetch_one(pool)
.await
.map_err(Into::into)
}
/// Verify the bytes on disk as well as the database hash before dropping an
/// inbox upload. Stale catalog hashes must not discard a replacement file.
pub async fn file_hash_exists_verified(
pool: &PgPool,
sha256: &str,
storage_dir: &str,
) -> anyhow::Result<bool> {
let paths: Vec<String> = sqlx::query_scalar(
"SELECT DISTINCT mf.file_path FROM furumusic__media_file mf \
JOIN furumusic__track t ON t.audio_file_id = mf.id \
WHERE mf.sha256_hash = $1",
)
.bind(sha256)
.fetch_all(pool)
.await?;
Ok(row.0)
for path in paths {
let path = crate::media_paths::resolve_media_file_path(storage_dir, &path);
if super::mover::sha256_file(&path)
.await
.is_ok_and(|actual| actual == sha256)
{
return Ok(true);
}
}
Ok(false)
}
+3
View File
@@ -525,6 +525,9 @@ translations! {
player_drop_audio_title: "Drop audio files here" , "Перетащите аудиофайлы сюда";
player_drop_audio_hint: "or click to choose files" , "или нажмите, чтобы выбрать файлы";
player_drop_audio_formats: "MP3, FLAC, WAV, M4A, OGG, Opus and AAC" , "MP3, FLAC, WAV, M4A, OGG, Opus и AAC";
player_choose_audio_files: "Choose files" , "Выбрать файлы";
player_choose_audio_folder: "Choose folder" , "Выбрать папку";
player_audio_folder_hint: "Audio files from subfolders are included; folder structure is preserved." , "Аудиофайлы из вложенных папок тоже загрузятся; структура папок сохранится.";
player_upload_selected_files: "Upload selected files" , "Загрузить выбранные файлы";
player_upload_history: "Upload history" , "История загрузок";
player_no_file_uploads: "No file uploads yet" , "Загрузок файлов пока нет";
+13 -3
View File
@@ -180,10 +180,17 @@ impl Job for InboxDiscoverJob {
};
// Skip if hash already in media_files
if crate::agent::rag::file_hash_exists(&ctx.pool, &hash)
.await
.unwrap_or(false)
if crate::agent::rag::file_hash_exists_verified(
&ctx.pool,
&hash,
&config.agent_storage_dir,
)
.await
.unwrap_or(false)
{
// Close the upload by removing the exact inbox duplicate;
// otherwise its history remains queued forever.
tokio::fs::remove_file(file_path).await?;
skipped_hash += 1;
continue;
}
@@ -320,6 +327,9 @@ pub fn group_by_folder(files: &[PathBuf]) -> Vec<(PathBuf, Vec<PathBuf>)> {
}
pub async fn collect_audio_files(dir: &Path, audio: &mut Vec<PathBuf>) -> anyhow::Result<()> {
if crate::local_uploads::incomplete_batch(dir).await? {
return Ok(());
}
let mut entries = tokio::fs::read_dir(dir).await?;
while let Some(entry) = entries.next_entry().await? {
let name = entry.file_name().to_string_lossy().into_owned();
+99 -9
View File
@@ -7,6 +7,9 @@ use cot::db::{Database, Model};
/// Guard to prevent multiple inbox_process orchestrators from running simultaneously.
static ORCHESTRATOR_RUNNING: AtomicBool = AtomicBool::new(false);
// Bound approvals waiting on database import locks so active imports retain
// spare pool connections for candidate queries and upgrade transactions.
static IMPORT_PERMITS: tokio::sync::Semaphore = tokio::sync::Semaphore::const_new(2);
/// Well-known advisory lock ID for the inbox_process orchestrator.
/// PostgreSQL advisory locks use a 64-bit key; this is an arbitrary unique value.
@@ -387,7 +390,7 @@ async fn process_folder_batch(
.to_owned();
let file_exists = file_path.exists();
if !sha.is_empty()
&& crate::agent::rag::file_hash_exists(pool, &sha)
&& crate::agent::rag::file_hash_exists_verified(pool, &sha, &config.agent_storage_dir)
.await
.unwrap_or(false)
{
@@ -642,7 +645,9 @@ async fn process_folder_batch(
folder_files.sort();
let track_count = folder_files.len();
FolderContext {
folder_path: folder_rel.to_owned(),
folder_path: crate::jobs::strip_user_upload_prefix(Path::new(folder_rel))
.to_string_lossy()
.into_owned(),
folder_files,
track_count,
}
@@ -696,7 +701,7 @@ async fn process_folder_batch(
Err(e) => {
crate::metrics::record_agent_failed("llm");
crate::metrics::record_agent_folder_batch("failed", file_count, batch_start.elapsed());
let err_msg = format!("Batch LLM call failed: {e}");
let err_msg = format!("Batch LLM call failed: {e:#}");
log.error(&err_msg);
// Mark all files as failed
let prepared_count = prepared.len() as u64;
@@ -810,6 +815,30 @@ async fn process_folder_batch(
crate::metrics::record_agent_file_processed("ok", "auto_approved");
ok_count += 1;
}
Err(e)
if e.downcast_ref::<crate::agent::dedup::DuplicateReview>()
.is_some() =>
{
let message = e.to_string();
p.review.status = cot::db::LimitedString::new("pending").unwrap();
p.review.error_message = Some(message.clone());
p.review.updated_at = cot::db::LimitedString::new(
chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ").to_string(),
)
.unwrap();
if let Some(result) = p.review.result_json.as_deref()
&& let Ok(mut result) = serde_json::from_str::<serde_json::Value>(result)
{
let previous = result["notes"].as_str().unwrap_or_default();
result["notes"] =
serde_json::json!(format!("{previous}\n{message}").trim());
p.review.result_json = Some(result.to_string());
}
let _ = p.review.save(db).await;
crate::metrics::record_agent_file_processed("ok", "pending_review");
log.warn(&format!("{filename}: {message}"));
ok_count += 1;
}
Err(e) => {
crate::metrics::record_agent_failed("finalize");
crate::metrics::record_agent_file_processed("failed", "failed");
@@ -869,8 +898,54 @@ pub async fn finalize_approved(
let year = normalized.year;
let track_number = normalized.track_number;
// Serialize imports of a recording across orchestrators and manual approvals.
// This transaction owns only the advisory lock; the existing Cot writes use
// their normal connections and remain protected until finalization returns.
let _import_permit = IMPORT_PERMITS.acquire().await?;
let mut import_lock = pool.begin().await?;
let mut import_artists: Vec<_> = normalized
.featured_artists
.iter()
.map(|name| name.trim().to_lowercase())
.collect();
import_artists.push(artist_name.trim().to_lowercase());
import_artists.sort();
import_artists.dedup();
for name in import_artists {
let lock_key = serde_json::to_string(&(
"furumi-recording-import",
name,
track_title.trim().to_lowercase(),
))?;
sqlx::query("SELECT pg_advisory_xact_lock(hashtextextended($1, 0))")
.bind(lock_key)
.execute(&mut *import_lock)
.await?;
}
if crate::agent::dedup::merge_existing(
pool,
storage_dir_str,
Path::new(input_path_str),
normalized,
context,
)
.await?
{
import_lock.commit().await?;
return Ok(());
}
let artist = find_or_create_artist(db, artist_name, model_name).await?;
let release = find_or_create_release(db, release_title, release_type, year, model_name).await?;
let release = find_or_create_release(
db,
pool,
artist.id_val(),
release_title,
release_type,
year,
model_name,
)
.await?;
// Link ReleaseArtist
let existing_links = ReleaseArtist::find_by_release(db, release.id_val())
@@ -1215,6 +1290,7 @@ pub async fn finalize_approved(
"Track finalized"
);
import_lock.commit().await?;
Ok(())
}
@@ -1241,17 +1317,31 @@ async fn find_or_create_artist(
async fn find_or_create_release(
db: &cot::db::Database,
pool: &sqlx::PgPool,
artist_id: i64,
title: &str,
release_type: &str,
year: Option<i32>,
model_name: Option<&str>,
) -> anyhow::Result<Release> {
let title_sort = title.trim().to_lowercase();
let all = Release::list_all(db).await.unwrap_or_default();
for r in &all {
if r.title_sort.as_str() == title_sort && r.release_type.as_str() == release_type {
return Ok(r.clone());
}
// Release type describes an album; it is not a separate identity key.
let existing_id: Option<i64> = sqlx::query_scalar(
r#"SELECT r.id FROM furumusic__release r
JOIN furumusic__release_artist ra ON ra.release_id=r.id
WHERE ra.artist_id=$1 AND r.title_sort=$2
AND ($3::integer IS NULL OR r.year IS NULL OR r.year=$3)
ORDER BY r.id LIMIT 1"#,
)
.bind(artist_id)
.bind(title_sort)
.bind(year)
.fetch_optional(pool)
.await?;
if let Some(id) = existing_id
&& let Some(release) = Release::get_by_id(db, id).await?
{
return Ok(release);
}
Release::create(db, title, release_type, year, model_name)
.await
+39 -1
View File
@@ -29,7 +29,15 @@ pub fn strip_user_upload_prefix(relative_path: &Path) -> PathBuf {
if components.len() >= 3
&& matches!(components[0], Component::Normal(value) if value == "user_uploads")
{
components[2..].iter().collect()
let start = if matches!(components[2], Component::Normal(value)
if value.to_str().and_then(|v| v.strip_prefix("local-"))
.and_then(|v| uuid::Uuid::parse_str(v).ok()).is_some())
{
3
} else {
2
};
components[start..].iter().collect()
} else {
relative_path.to_path_buf()
}
@@ -76,3 +84,33 @@ pub async fn uploader_from_relative_path(
_ => UploaderAttribution::unknown(),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn local_upload_batch_id_is_not_an_artist_hint() {
let path = Path::new(
"user_uploads/6/local-123e4567-e89b-12d3-a456-426614174000/Artist/Album/01.mp3",
);
assert_eq!(
strip_user_upload_prefix(path),
PathBuf::from("Artist/Album/01.mp3")
);
assert_eq!(
strip_user_upload_prefix(Path::new(
"user_uploads/6/local-123e4567-e89b-12d3-a456-426614174000/01.mp3"
)),
PathBuf::from("01.mp3")
);
}
#[test]
fn ordinary_upload_folder_names_are_preserved() {
assert_eq!(
strip_user_upload_prefix(Path::new("user_uploads/6/local-band/Album/01.mp3")),
PathBuf::from("local-band/Album/01.mp3")
);
}
}
+207 -2
View File
@@ -1,4 +1,5 @@
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use anyhow::{Context, bail};
use serde::Serialize;
@@ -6,6 +7,92 @@ use sqlx::{FromRow, PgPool};
const LOCAL_UPLOAD_LIST_LIMIT: i64 = 100;
pub async fn begin_batch(root: &Path) -> std::io::Result<()> {
tokio::fs::create_dir_all(root).await?;
tokio::fs::write(root.join(".uploading"), b"uploading").await
}
pub async fn finish_batch(root: &Path) -> std::io::Result<()> {
match tokio::fs::remove_file(root.join(".uploading")).await {
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()),
result => result,
}
}
pub async fn incomplete_batch(root: &Path) -> std::io::Result<bool> {
let is_batch = root
.file_name()
.and_then(|s| s.to_str())
.and_then(|s| s.strip_prefix("local-"))
.is_some_and(|s| uuid::Uuid::parse_str(s).is_ok());
if !is_batch {
return Ok(false);
}
let marker = match tokio::fs::metadata(root.join(".uploading")).await {
Ok(metadata) => metadata,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(false),
Err(error) => return Err(error),
};
// A crashed browser must not strand accepted files indefinitely.
if marker.modified()?.elapsed().unwrap_or_default().as_secs() < 3600 {
return Ok(true);
}
finish_batch(root).await?;
Ok(false)
}
/// Validate browser-provided folder paths before appending them to a user's inbox.
pub fn relative_upload_path(value: &str) -> anyhow::Result<PathBuf> {
if value.is_empty() || value.contains('\\') || value.chars().any(char::is_control) {
bail!("invalid relative upload path");
}
let mut path = PathBuf::new();
for part in value.split('/') {
if part.is_empty() || matches!(part, "." | "..") || part.ends_with(':') {
bail!("upload path must be relative and contain no traversal components");
}
path.push(part);
}
if !path.is_relative() {
bail!("upload path must be relative");
}
Ok(path)
}
/// Publish a complete file atomically without overwriting another upload.
pub async fn save_file(destination: &Path, upload_id: &str, bytes: &[u8]) -> std::io::Result<()> {
let parent = destination.parent().ok_or_else(|| {
std::io::Error::new(
std::io::ErrorKind::InvalidInput,
"upload has no parent folder",
)
})?;
tokio::fs::create_dir_all(parent).await?;
let temporary = parent.join(format!(".{upload_id}.{}.uploading", uuid::Uuid::new_v4()));
let result = async {
tokio::fs::write(&temporary, bytes).await?;
// Unlike rename, hard_link fails if the destination already exists.
match tokio::fs::hard_link(&temporary, destination).await {
Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {
use sha2::{Digest, Sha256};
let incoming_hash = format!("{:x}", Sha256::digest(bytes));
let existing_hash = crate::agent::mover::sha256_file(destination)
.await
.map_err(std::io::Error::other)?;
if existing_hash == incoming_hash {
Ok(())
} else {
Err(error)
}
}
result => result,
}
}
.await;
let _ = tokio::fs::remove_file(&temporary).await;
result
}
#[derive(Debug, Clone, Serialize)]
pub struct LocalUploadDto {
pub id: String,
@@ -56,11 +143,16 @@ pub async fn create(
inbox_path: &str,
) -> anyhow::Result<()> {
let now = now_string();
sqlx::query(
let created = sqlx::query(
r#"INSERT INTO furumusic__local_upload
(id, user_id, filename, size_bytes, status, inbox_path, error,
created_at, updated_at, completed_at)
VALUES ($1, $2, $3, $4, 'uploading', $5, NULL, $6, $6, NULL)"#,
VALUES ($1, $2, $3, $4, 'uploading', $5, NULL, $6, $6, NULL)
ON CONFLICT (id) DO UPDATE SET status='uploading', error=NULL, updated_at=$6
WHERE furumusic__local_upload.user_id=$2
AND furumusic__local_upload.inbox_path=$5
AND furumusic__local_upload.size_bytes=$4
AND furumusic__local_upload.status IN ('uploading', 'failed')"#,
)
.bind(id)
.bind(user_id)
@@ -70,9 +162,34 @@ pub async fn create(
.bind(now)
.execute(pool)
.await?;
if created.rows_affected() == 0 {
bail!("upload ID is already in use");
}
Ok(())
}
/// Return a completed upload receipt when a client retries after losing the
/// HTTP response. A receipt is scoped to this user and exact destination.
pub async fn successful_receipt(
pool: &PgPool,
user_id: i64,
id: &str,
inbox_path: &str,
) -> anyhow::Result<Option<LocalUploadDto>> {
let row = sqlx::query_as::<_, LocalUploadRow>(
r#"SELECT id, user_id, filename, size_bytes, status, inbox_path, error,
created_at, updated_at, completed_at
FROM furumusic__local_upload WHERE id=$1 AND user_id=$2 AND inbox_path=$3
AND status NOT IN ('uploading','failed')"#,
)
.bind(id)
.bind(user_id)
.bind(inbox_path)
.fetch_optional(pool)
.await?;
Ok(row.map(|row| row.dto()))
}
pub async fn mark_queued(pool: &PgPool, id: &str, user_id: i64) -> anyhow::Result<LocalUploadDto> {
update_status(pool, id, user_id, "queued", None).await?;
load(pool, user_id, id).await.map(|row| row.dto())
@@ -257,3 +374,91 @@ fn trim_error(value: &str) -> String {
fn now_string() -> String {
chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ").to_string()
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn folder_discovery_waits_for_completion_and_recovers_abandoned_uploads() {
let parent =
std::env::temp_dir().join(format!("furumi-batch-test-{}", uuid::Uuid::new_v4()));
let root = parent.join(format!("local-{}", uuid::Uuid::new_v4()));
begin_batch(&root).await.unwrap();
save_file(&root.join("Artist/Album/01.mp3"), "first", b"audio")
.await
.unwrap();
let mut found = Vec::new();
crate::jobs::inbox_discover::collect_audio_files(&parent, &mut found)
.await
.unwrap();
assert!(found.is_empty(), "partial folder was queued for AI");
finish_batch(&root).await.unwrap();
crate::jobs::inbox_discover::collect_audio_files(&parent, &mut found)
.await
.unwrap();
assert_eq!(found.len(), 1);
begin_batch(&root).await.unwrap();
std::fs::File::options()
.write(true)
.open(root.join(".uploading"))
.unwrap()
.set_times(
std::fs::FileTimes::new().set_modified(
std::time::SystemTime::now() - std::time::Duration::from_secs(7200),
),
)
.unwrap();
assert!(!incomplete_batch(&root).await.unwrap());
assert!(!root.join(".uploading").exists());
tokio::fs::remove_dir_all(parent).await.unwrap();
}
#[test]
fn folder_paths_preserve_unicode_and_nested_names() {
let path = "Музыка/Исполнитель/2002 - Альбом/08 - Где же ваши руки.mp3";
assert_eq!(relative_upload_path(path).unwrap(), PathBuf::from(path));
}
#[test]
fn folder_paths_reject_absolute_paths_and_traversal() {
for path in [
"",
"/etc/file.mp3",
"../file.mp3",
"Album/../../file.mp3",
"Album/./file.mp3",
"Album//file.mp3",
"Album/",
"C:/file.mp3",
"Album\\..\\file.mp3",
"Album/\0file.mp3",
] {
assert!(relative_upload_path(path).is_err(), "accepted {path:?}");
}
}
#[tokio::test]
async fn folder_uploads_keep_siblings_and_never_overwrite_existing_files() {
let root =
std::env::temp_dir().join(format!("furumi-upload-test-{}", uuid::Uuid::new_v4()));
let first = root.join("Artist A/Album/01.mp3");
let second = root.join("Artist B/Album/01.mp3");
save_file(&first, "first", b"first audio").await.unwrap();
save_file(&second, "second", b"second audio").await.unwrap();
save_file(&first, "first-retry", b"first audio")
.await
.unwrap();
let error = save_file(&first, "duplicate", b"replacement")
.await
.unwrap_err();
assert_eq!(error.kind(), std::io::ErrorKind::AlreadyExists);
assert_eq!(tokio::fs::read(&first).await.unwrap(), b"first audio");
assert_eq!(tokio::fs::read(&second).await.unwrap(), b"second audio");
let mut entries = tokio::fs::read_dir(first.parent().unwrap()).await.unwrap();
while let Some(entry) = entries.next_entry().await.unwrap() {
assert!(!entry.file_name().to_string_lossy().ends_with(".uploading"));
}
tokio::fs::remove_dir_all(root).await.unwrap();
}
}
+81 -13
View File
@@ -5157,6 +5157,47 @@ async fn local_upload_handler(
));
}
let batch_header_id = request
.headers()
.get("x-furumusic-upload-batch-id")
.and_then(|value| value.to_str().ok())
.and_then(|value| uuid::Uuid::parse_str(value.trim()).ok());
let batch_root = batch_header_id.map(|id| {
inbox_root
.join("user_uploads")
.join(user.id.to_string())
.join(format!("local-{id}"))
});
if request
.headers()
.get("x-furumusic-upload-complete")
.is_some_and(|value| value == "true")
{
let Some(root) = batch_root else {
return Ok(json_error(
StatusCode::BAD_REQUEST,
"batch completion requires a valid batch ID",
));
};
crate::local_uploads::finish_batch(&root)
.await
.map_err(|err| cot::Error::internal(err.to_string()))?;
if let Some(handle) = scheduler_handle.get() {
let handle = Arc::clone(handle);
tokio::spawn(async move {
if let Err(err) = handle.trigger_job_now("inbox_discover").await {
tracing::warn!("failed to trigger discovery after folder upload: {err}");
}
});
}
return Json(serde_json::json!({"ok": true})).into_response();
}
if let Some(root) = &batch_root {
crate::local_uploads::begin_batch(root)
.await
.map_err(|err| cot::Error::internal(err.to_string()))?;
}
let filename_header = HeaderName::from_static("x-furumusic-filename");
let original_name = request
.headers()
@@ -5176,6 +5217,32 @@ async fn local_upload_handler(
.unwrap_or_else(uuid::Uuid::new_v4)
.to_string();
// One batch directory preserves sibling and nested folders from the picker.
// Older clients keep their existing per-file upload directories.
let batch_id = request
.headers()
.get("x-furumusic-upload-batch-id")
.and_then(|value| value.to_str().ok())
.and_then(|value| uuid::Uuid::parse_str(value.trim()).ok())
.map(|value| value.to_string())
.unwrap_or_else(|| upload_id.clone());
let relative_path = match request.headers().get("x-furumusic-relative-path") {
Some(value) => match value.to_str().ok().map(percent_decode_header) {
Some(value) => match crate::local_uploads::relative_upload_path(&value) {
Ok(path) => path,
Err(err) => return Ok(json_error(StatusCode::BAD_REQUEST, &err.to_string())),
},
None => {
return Ok(json_error(
StatusCode::BAD_REQUEST,
"invalid upload path header",
));
}
},
None => std::path::PathBuf::from(&filename),
};
let display_name = relative_path.to_string_lossy().to_string();
let bytes = request
.into_body()
.into_bytes()
@@ -5191,8 +5258,8 @@ async fn local_upload_handler(
let upload_dir = inbox_root
.join("user_uploads")
.join(user.id.to_string())
.join(format!("local-{upload_id}"));
let destination = upload_dir.join(&filename);
.join(format!("local-{batch_id}"));
let destination = upload_dir.join(&relative_path);
let Some(inbox_path) =
crate::media_paths::path_for_root(&inbox_root.to_string_lossy(), &destination)
else {
@@ -5201,11 +5268,18 @@ async fn local_upload_handler(
"upload destination escaped agent_inbox_dir",
));
};
if let Some(upload) =
crate::local_uploads::successful_receipt(pool, user.id, &upload_id, &inbox_path)
.await
.map_err(|err| cot::Error::internal(err.to_string()))?
{
return Json(LocalUploadResponse { ok: true, upload }).into_response();
}
if let Err(err) = crate::local_uploads::create(
pool,
&upload_id,
user.id,
&filename,
&display_name,
bytes.len() as u64,
&inbox_path,
)
@@ -5214,17 +5288,9 @@ async fn local_upload_handler(
return Ok(json_error(StatusCode::BAD_REQUEST, &err.to_string()));
}
let temporary = upload_dir.join(format!(".{upload_id}.uploading"));
let write_result = async {
tokio::fs::create_dir_all(&upload_dir).await?;
tokio::fs::write(&temporary, &bytes).await?;
tokio::fs::rename(&temporary, &destination).await?;
Ok::<(), std::io::Error>(())
}
.await;
let write_result = crate::local_uploads::save_file(&destination, &upload_id, &bytes).await;
if let Err(err) = write_result {
let message = format!("could not save uploaded file: {err}");
let _ = tokio::fs::remove_dir_all(&upload_dir).await;
let _ = crate::local_uploads::mark_failed(pool, &upload_id, user.id, &message).await;
return Ok(json_error(StatusCode::INTERNAL_SERVER_ERROR, &message));
}
@@ -5239,7 +5305,9 @@ async fn local_upload_handler(
}
};
if let Some(handle) = scheduler_handle.get() {
if batch_root.is_none()
&& let Some(handle) = scheduler_handle.get()
{
let handle = Arc::clone(handle);
tokio::spawn(async move {
if let Err(err) = handle.trigger_job_now("inbox_discover").await {