Improved AI metadata processing
Build and Publish / Build and Publish Docker Image (push) Successful in 5m33s
Build and Publish / Build and Publish Docker Image (push) Successful in 5m33s
This commit is contained in:
1 parent
8c0ea444f0
commit
2b3b50f7af
20 files changed
+1916
-68
No files matched your search
+1046
File diff suppressed because it is too large.
Load diff
@@ -1,5 +1,6 @@
|
||||
pub mod cover_art;
|
||||
pub mod cover_variants;
|
||||
pub mod dedup;
|
||||
pub mod dto;
|
||||
pub mod metadata;
|
||||
pub mod mover;
|
||||
@@ -54,6 +55,8 @@ pub async fn probe_llm(llm_url: &str, llm_model: &str, llm_auth: &str) -> AgentP
|
||||
],
|
||||
"stream": false,
|
||||
"temperature": 0.3,
|
||||
"reasoning_effort": "none",
|
||||
"chat_template_kwargs": {"enable_thinking": false},
|
||||
"max_tokens": 256
|
||||
});
|
||||
|
||||
|
||||
+86
-15
@@ -11,9 +11,9 @@ pub enum MoveOutcome {
|
||||
///
|
||||
/// Creates the directory structure: `storage_dir/artist/album/filename`
|
||||
///
|
||||
/// If `rename` fails (cross-device), falls back to copy + remove.
|
||||
/// If the destination already exists the inbox copy is removed and
|
||||
/// `MoveOutcome::Merged` is returned.
|
||||
/// Copies into the destination filesystem before removing the inbox source.
|
||||
/// Filename collisions are merged only when both physical files have the
|
||||
/// same SHA-256. Distinct recordings receive a separate storage filename.
|
||||
pub async fn move_to_storage(
|
||||
storage_dir: &Path,
|
||||
artist: &str,
|
||||
@@ -27,31 +27,72 @@ pub async fn move_to_storage(
|
||||
let dest_dir = storage_dir.join(&artist_dir).join(&album_dir);
|
||||
tokio::fs::create_dir_all(&dest_dir).await?;
|
||||
|
||||
let dest = dest_dir.join(filename);
|
||||
let mut dest = dest_dir.join(filename);
|
||||
|
||||
// File already at destination — remove the inbox duplicate
|
||||
if dest.exists() {
|
||||
if source.exists() {
|
||||
if sha256_file(source).await? == sha256_file(&dest).await? {
|
||||
tokio::fs::remove_file(source).await?;
|
||||
tracing::info!(from = ?source, to = ?dest, "merged duplicate into existing storage file");
|
||||
return Ok(MoveOutcome::Merged(dest));
|
||||
}
|
||||
return Ok(MoveOutcome::Merged(dest));
|
||||
let stem = Path::new(filename)
|
||||
.file_stem()
|
||||
.and_then(|s| s.to_str())
|
||||
.unwrap_or("audio");
|
||||
let ext = Path::new(filename)
|
||||
.extension()
|
||||
.and_then(|s| s.to_str())
|
||||
.unwrap_or("audio");
|
||||
dest = dest_dir.join(format!("{stem} [{}].{ext}", uuid::Uuid::new_v4()));
|
||||
}
|
||||
|
||||
// Try atomic rename first (same filesystem)
|
||||
match tokio::fs::rename(source, &dest).await {
|
||||
Ok(()) => {}
|
||||
Err(_) => {
|
||||
// Cross-device: copy then remove
|
||||
tokio::fs::copy(source, &dest).await?;
|
||||
tokio::fs::remove_file(source).await?;
|
||||
}
|
||||
}
|
||||
publish_copy(source, &dest).await?;
|
||||
tokio::fs::remove_file(source).await?;
|
||||
|
||||
tracing::info!(from = ?source, to = ?dest, "moved file to storage");
|
||||
Ok(MoveOutcome::Moved(dest))
|
||||
}
|
||||
|
||||
/// Copy into the destination filesystem, then expose the complete file without
|
||||
/// replacing an existing destination. The source remains available on errors.
|
||||
pub async fn publish_copy(source: &Path, destination: &Path) -> anyhow::Result<()> {
|
||||
let parent = destination
|
||||
.parent()
|
||||
.ok_or_else(|| anyhow::anyhow!("destination has no parent"))?;
|
||||
tokio::fs::create_dir_all(parent).await?;
|
||||
let temporary = parent.join(format!(".{}.moving", uuid::Uuid::new_v4()));
|
||||
let result = async {
|
||||
tokio::fs::copy(source, &temporary).await?;
|
||||
tokio::fs::hard_link(&temporary, destination).await
|
||||
}
|
||||
.await;
|
||||
let _ = tokio::fs::remove_file(temporary).await;
|
||||
result?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn sha256_file(path: &Path) -> anyhow::Result<String> {
|
||||
let path = path.to_owned();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::io::Read;
|
||||
let mut file = std::fs::File::open(path)?;
|
||||
let mut hash = Sha256::new();
|
||||
let mut buffer = [0; 65536];
|
||||
loop {
|
||||
let count = file.read(&mut buffer)?;
|
||||
if count == 0 {
|
||||
break;
|
||||
}
|
||||
hash.update(&buffer[..count]);
|
||||
}
|
||||
Ok::<_, std::io::Error>(format!("{:x}", hash.finalize()))
|
||||
})
|
||||
.await?
|
||||
.map_err(Into::into)
|
||||
}
|
||||
|
||||
/// Remove characters that are unsafe for directory names.
|
||||
fn sanitize_dir_name(name: &str) -> String {
|
||||
name.chars()
|
||||
@@ -64,3 +105,33 @@ fn sanitize_dir_name(name: &str) -> String {
|
||||
.trim_matches('.')
|
||||
.to_owned()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn filename_collision_does_not_discard_a_different_recording() {
|
||||
let root = std::env::temp_dir().join(format!("furumi-move-{}", uuid::Uuid::new_v4()));
|
||||
let storage = root.join("storage");
|
||||
tokio::fs::create_dir_all(storage.join("Artist/Album"))
|
||||
.await
|
||||
.unwrap();
|
||||
let old = storage.join("Artist/Album/song.mp3");
|
||||
let incoming = root.join("incoming.mp3");
|
||||
tokio::fs::write(&old, b"original recording").await.unwrap();
|
||||
tokio::fs::write(&incoming, b"different recording")
|
||||
.await
|
||||
.unwrap();
|
||||
let outcome = move_to_storage(&storage, "Artist", "Album", "song.mp3", &incoming)
|
||||
.await
|
||||
.unwrap();
|
||||
let MoveOutcome::Moved(new) = outcome else {
|
||||
panic!("different recording was merged")
|
||||
};
|
||||
assert_ne!(new, old);
|
||||
assert_eq!(tokio::fs::read(&old).await.unwrap(), b"original recording");
|
||||
assert_eq!(tokio::fs::read(new).await.unwrap(), b"different recording");
|
||||
tokio::fs::remove_dir_all(root).await.unwrap();
|
||||
}
|
||||
}
|
||||
+140
-3
@@ -1,3 +1,4 @@
|
||||
use anyhow::Context;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::dto::{
|
||||
@@ -8,6 +9,9 @@ use super::dto::{
|
||||
// Types
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// Large album batches can take longer than ten minutes on the local model.
|
||||
const LLM_REQUEST_TIMEOUT_SECS: u64 = 3600;
|
||||
|
||||
/// A single message in the chat history.
|
||||
#[derive(Clone, Serialize)]
|
||||
pub struct ChatMessage {
|
||||
@@ -22,6 +26,13 @@ struct ChatRequest {
|
||||
response_format: ChatResponseFormat,
|
||||
stream: bool,
|
||||
temperature: f64,
|
||||
reasoning_effort: &'static str,
|
||||
chat_template_kwargs: ChatTemplateKwargs,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct ChatTemplateKwargs {
|
||||
enable_thinking: bool,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
@@ -60,7 +71,8 @@ async fn call_llm_chat(
|
||||
auth: Option<&str>,
|
||||
) -> anyhow::Result<(String, String, ChatUsage)> {
|
||||
let client = reqwest::Client::builder()
|
||||
.timeout(std::time::Duration::from_secs(600))
|
||||
.connect_timeout(std::time::Duration::from_secs(15))
|
||||
.timeout(std::time::Duration::from_secs(LLM_REQUEST_TIMEOUT_SECS))
|
||||
.build()?;
|
||||
|
||||
let request = ChatRequest {
|
||||
@@ -71,6 +83,11 @@ async fn call_llm_chat(
|
||||
},
|
||||
stream: false,
|
||||
temperature: 0.1,
|
||||
// Disable reasoning through both the API and the model's chat template.
|
||||
reasoning_effort: "none",
|
||||
chat_template_kwargs: ChatTemplateKwargs {
|
||||
enable_thinking: false,
|
||||
},
|
||||
};
|
||||
|
||||
let url = format!("{}/v1/chat/completions", base_url.trim_end_matches('/'));
|
||||
@@ -78,6 +95,8 @@ async fn call_llm_chat(
|
||||
%url,
|
||||
model,
|
||||
message_count = messages.len(),
|
||||
timeout_secs = LLM_REQUEST_TIMEOUT_SECS,
|
||||
thinking = false,
|
||||
"Calling LLM API (chat mode)..."
|
||||
);
|
||||
|
||||
@@ -86,7 +105,17 @@ async fn call_llm_chat(
|
||||
if let Some(auth_header) = auth {
|
||||
req = req.header("Authorization", auth_header);
|
||||
}
|
||||
let resp = req.send().await?;
|
||||
let resp = req.send().await.map_err(|error| {
|
||||
let elapsed_secs = start.elapsed().as_secs();
|
||||
let message = if error.is_timeout() {
|
||||
format!(
|
||||
"LLM request timed out after {elapsed_secs}s (request timeout: {LLM_REQUEST_TIMEOUT_SECS}s, connect timeout: 15s)"
|
||||
)
|
||||
} else {
|
||||
format!("LLM request failed after {elapsed_secs}s")
|
||||
};
|
||||
anyhow::Error::new(error).context(message)
|
||||
})?;
|
||||
let elapsed = start.elapsed();
|
||||
|
||||
if !resp.status().is_success() {
|
||||
@@ -97,7 +126,10 @@ async fn call_llm_chat(
|
||||
anyhow::bail!("LLM returned {}: {}", status, body);
|
||||
}
|
||||
|
||||
let chat_resp: ChatResponse = resp.json().await?;
|
||||
let chat_resp: ChatResponse = resp
|
||||
.json()
|
||||
.await
|
||||
.context("Failed to decode LLM chat response")?;
|
||||
let resp_model = chat_resp.model.unwrap_or_else(|| model.to_owned());
|
||||
let usage = chat_resp.usage.unwrap_or_default();
|
||||
let content = chat_resp
|
||||
@@ -605,3 +637,108 @@ fn parse_batch_response(
|
||||
|
||||
Ok(results)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
|
||||
#[test]
|
||||
fn mixed_folder_results_keep_metadata_per_file() {
|
||||
let files = vec![
|
||||
BatchFileInput {
|
||||
filename: "01 - One.mp3".into(),
|
||||
raw: RawMetadata::default(),
|
||||
hints: PathHints::default(),
|
||||
},
|
||||
BatchFileInput {
|
||||
filename: "02 - Two.mp3".into(),
|
||||
raw: RawMetadata::default(),
|
||||
hints: PathHints::default(),
|
||||
},
|
||||
];
|
||||
let response = r#"{"results":[
|
||||
{"filename":"02 - Two.mp3","artist":"Artist B","album":"Album B","year":2020},
|
||||
{"filename":"01 - One.mp3","artist":"Artist A","album":"Album A","year":2002}
|
||||
]}"#;
|
||||
let results: std::collections::HashMap<_, _> = parse_batch_response(response, &files)
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.collect();
|
||||
assert_eq!(results["01 - One.mp3"].artist.as_deref(), Some("Artist A"));
|
||||
assert_eq!(results["01 - One.mp3"].album.as_deref(), Some("Album A"));
|
||||
assert_eq!(results["01 - One.mp3"].year, Some(2002));
|
||||
assert_eq!(results["02 - Two.mp3"].artist.as_deref(), Some("Artist B"));
|
||||
assert_eq!(results["02 - Two.mp3"].album.as_deref(), Some("Album B"));
|
||||
assert_eq!(results["02 - Two.mp3"].year, Some(2020));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn chat_request_disables_thinking_on_the_wire() {
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), async {
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
||||
.await
|
||||
.unwrap();
|
||||
let url = format!("http://{}", listener.local_addr().unwrap());
|
||||
let server = tokio::spawn(async move {
|
||||
let (mut socket, _) = listener.accept().await.unwrap();
|
||||
let mut request = Vec::new();
|
||||
let mut buffer = [0; 4096];
|
||||
let payload = loop {
|
||||
let count = socket.read(&mut buffer).await.unwrap();
|
||||
assert!(count > 0, "client closed before sending the request");
|
||||
request.extend_from_slice(&buffer[..count]);
|
||||
let Some(header_end) = request.windows(4).position(|w| w == b"\r\n\r\n")
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
let headers = std::str::from_utf8(&request[..header_end]).unwrap();
|
||||
assert!(headers.starts_with("POST /v1/chat/completions HTTP/1.1"));
|
||||
let length = headers
|
||||
.lines()
|
||||
.find_map(|line| {
|
||||
let (key, value) = line.split_once(':')?;
|
||||
key.eq_ignore_ascii_case("content-length")
|
||||
.then(|| value.trim().parse::<usize>().unwrap())
|
||||
})
|
||||
.unwrap();
|
||||
if request.len() >= header_end + 4 + length {
|
||||
break serde_json::from_slice::<serde_json::Value>(
|
||||
&request[header_end + 4..header_end + 4 + length],
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
};
|
||||
let response = serde_json::json!({
|
||||
"model": "local-model",
|
||||
"choices": [{"message": {"content": "{\"results\":[]}"}}],
|
||||
"usage": {"prompt_tokens": 10, "completion_tokens": 5}
|
||||
})
|
||||
.to_string();
|
||||
socket
|
||||
.write_all(
|
||||
format!(
|
||||
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{response}",
|
||||
response.len()
|
||||
)
|
||||
.as_bytes(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
payload
|
||||
});
|
||||
let (content, model, usage) = call_llm_chat(&url, "local-model", &[], None)
|
||||
.await
|
||||
.unwrap();
|
||||
let payload = server.await.unwrap();
|
||||
assert_eq!(payload["reasoning_effort"], "none");
|
||||
assert_eq!(payload["chat_template_kwargs"]["enable_thinking"], false);
|
||||
assert_eq!(payload["response_format"]["type"], "json_object");
|
||||
assert_eq!(content, "{\"results\":[]}");
|
||||
assert_eq!(model, "local-model");
|
||||
assert_eq!(usage.completion_tokens, Some(5));
|
||||
})
|
||||
.await
|
||||
.expect("mock LLM request should finish within five seconds");
|
||||
}
|
||||
}
|
||||
+89
-6
@@ -28,13 +28,23 @@ pub fn parse(relative_path: &Path) -> PathHints {
|
||||
parse_filename(stem, &mut hints);
|
||||
|
||||
match components.len() {
|
||||
// Artist/Album/file.ext
|
||||
// Artist/.../Album/file.ext: use the nearest release folder rather
|
||||
// than an organizational directory such as "Альбомы и синглы".
|
||||
3.. => {
|
||||
hints.artist = Some(components[0].to_owned());
|
||||
|
||||
let album_raw = components[1];
|
||||
let mut album_index = components.len() - 2;
|
||||
if album_index > 0 && is_disc_folder(components[album_index]) {
|
||||
album_index -= 1;
|
||||
}
|
||||
hints.artist = components[..album_index]
|
||||
.iter()
|
||||
.rev()
|
||||
.find(|name| !is_organizational_folder(name))
|
||||
.map(|name| (*name).to_owned());
|
||||
let album_raw = components[album_index];
|
||||
let (album, year) = parse_album_with_year(album_raw);
|
||||
hints.album = Some(album);
|
||||
if !is_organizational_folder(&album) {
|
||||
hints.album = Some(album);
|
||||
}
|
||||
if year.is_some() {
|
||||
hints.year = year;
|
||||
}
|
||||
@@ -43,7 +53,9 @@ pub fn parse(relative_path: &Path) -> PathHints {
|
||||
2 => {
|
||||
let dir = components[0];
|
||||
let (name, year) = parse_album_with_year(dir);
|
||||
hints.album = Some(name);
|
||||
if !is_organizational_folder(&name) {
|
||||
hints.album = Some(name);
|
||||
}
|
||||
if year.is_some() {
|
||||
hints.year = year;
|
||||
}
|
||||
@@ -55,6 +67,34 @@ pub fn parse(relative_path: &Path) -> PathHints {
|
||||
hints
|
||||
}
|
||||
|
||||
fn is_organizational_folder(name: &str) -> bool {
|
||||
matches!(
|
||||
name.trim().to_lowercase().as_str(),
|
||||
"music"
|
||||
| "downloads"
|
||||
| "albums"
|
||||
| "singles"
|
||||
| "albums and singles"
|
||||
| "mixed"
|
||||
| "mix"
|
||||
| "музыка"
|
||||
| "загрузки"
|
||||
| "альбомы"
|
||||
| "синглы"
|
||||
| "альбомы и синглы"
|
||||
| "разное"
|
||||
)
|
||||
}
|
||||
|
||||
fn is_disc_folder(name: &str) -> bool {
|
||||
let name = name.trim().to_lowercase();
|
||||
["cd", "disc", "disk", "диск"].iter().any(|prefix| {
|
||||
name.strip_prefix(prefix)
|
||||
.and_then(|number| number.trim().parse::<u32>().ok())
|
||||
.is_some_and(|number| number > 0)
|
||||
})
|
||||
}
|
||||
|
||||
/// Try to extract track number and title from a filename stem.
|
||||
///
|
||||
/// Patterns: "01 - Title", "01. Title", "1 Title", "Title"
|
||||
@@ -106,6 +146,14 @@ fn try_strip_track_prefix(s: &str) -> Option<(&str, &str)> {
|
||||
///
|
||||
/// Patterns: "Album (2001)", "(2001) Album", "Album [2001]", "Album"
|
||||
fn parse_album_with_year(dir: &str) -> (String, Option<i32>) {
|
||||
// Common discography folders: "2002 - Album".
|
||||
if let Some((prefix, album)) = dir.split_once(" - ")
|
||||
&& let Ok(year) = prefix.trim().parse::<i32>()
|
||||
&& (1900..=2100).contains(&year)
|
||||
&& !album.trim().is_empty()
|
||||
{
|
||||
return (album.trim().to_owned(), Some(year));
|
||||
}
|
||||
// Try "Album (YYYY)" or "Album [YYYY]"
|
||||
for (open, close) in [('(', ')'), ('[', ']')] {
|
||||
if let Some(start) = dir.rfind(open) {
|
||||
@@ -191,4 +239,39 @@ mod tests {
|
||||
assert_eq!(h.album.as_deref(), Some("Album"));
|
||||
assert_eq!(h.year, Some(1999));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nested_discography_uses_the_release_folder() {
|
||||
let h = parse(Path::new(
|
||||
"Diskoteka_Avariya-Diskografia/Альбомы и синглы/2002 - Х.Х.Х.И.Р.Н.Р/08 - Где же ваши руки.mp3",
|
||||
));
|
||||
assert_eq!(h.artist.as_deref(), Some("Diskoteka_Avariya-Diskografia"));
|
||||
assert_eq!(h.album.as_deref(), Some("Х.Х.Х.И.Р.Н.Р"));
|
||||
assert_eq!(h.year, Some(2002));
|
||||
assert_eq!(h.track_number, Some(8));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mixed_folder_does_not_supply_an_album() {
|
||||
let h = parse(Path::new("Mix/Track.mp3"));
|
||||
assert_eq!(h.artist, None);
|
||||
assert_eq!(h.album, None);
|
||||
assert_eq!(h.title.as_deref(), Some("Track"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn collection_root_does_not_replace_the_artist() {
|
||||
let h = parse(Path::new("Music/Artist/2020 - Album/01 - Song.flac"));
|
||||
assert_eq!(h.artist.as_deref(), Some("Artist"));
|
||||
assert_eq!(h.album.as_deref(), Some("Album"));
|
||||
assert_eq!(h.year, Some(2020));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn disc_subfolder_keeps_the_album_and_artist() {
|
||||
let h = parse(Path::new("Artist/2002 - Album/CD 1/01 - Song.flac"));
|
||||
assert_eq!(h.artist.as_deref(), Some("Artist"));
|
||||
assert_eq!(h.album.as_deref(), Some("Album"));
|
||||
assert_eq!(h.year, Some(2002));
|
||||
}
|
||||
}
|
||||
+32
-7
@@ -83,15 +83,40 @@ pub async fn find_similar_releases(
|
||||
/// are ignored so that re-discovery is possible after the user deletes
|
||||
/// artists/releases/tracks.
|
||||
pub async fn file_hash_exists(pool: &PgPool, sha256: &str) -> anyhow::Result<bool> {
|
||||
let row: (bool,) = sqlx::query_as(
|
||||
"SELECT EXISTS(\
|
||||
SELECT 1 FROM furumusic__media_file mf \
|
||||
JOIN furumusic__track t ON t.audio_file_id = mf.id \
|
||||
WHERE mf.sha256_hash = $1\
|
||||
)",
|
||||
sqlx::query_scalar(
|
||||
"SELECT EXISTS(SELECT 1 FROM furumusic__media_file mf \
|
||||
JOIN furumusic__track t ON t.audio_file_id = mf.id \
|
||||
WHERE mf.sha256_hash = $1)",
|
||||
)
|
||||
.bind(sha256)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.map_err(Into::into)
|
||||
}
|
||||
|
||||
/// Verify the bytes on disk as well as the database hash before dropping an
|
||||
/// inbox upload. Stale catalog hashes must not discard a replacement file.
|
||||
pub async fn file_hash_exists_verified(
|
||||
pool: &PgPool,
|
||||
sha256: &str,
|
||||
storage_dir: &str,
|
||||
) -> anyhow::Result<bool> {
|
||||
let paths: Vec<String> = sqlx::query_scalar(
|
||||
"SELECT DISTINCT mf.file_path FROM furumusic__media_file mf \
|
||||
JOIN furumusic__track t ON t.audio_file_id = mf.id \
|
||||
WHERE mf.sha256_hash = $1",
|
||||
)
|
||||
.bind(sha256)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
Ok(row.0)
|
||||
for path in paths {
|
||||
let path = crate::media_paths::resolve_media_file_path(storage_dir, &path);
|
||||
if super::mover::sha256_file(&path)
|
||||
.await
|
||||
.is_ok_and(|actual| actual == sha256)
|
||||
{
|
||||
return Ok(true);
|
||||
}
|
||||
}
|
||||
Ok(false)
|
||||
}
|
||||
@@ -525,6 +525,9 @@ translations! {
|
||||
player_drop_audio_title: "Drop audio files here" , "Перетащите аудиофайлы сюда";
|
||||
player_drop_audio_hint: "or click to choose files" , "или нажмите, чтобы выбрать файлы";
|
||||
player_drop_audio_formats: "MP3, FLAC, WAV, M4A, OGG, Opus and AAC" , "MP3, FLAC, WAV, M4A, OGG, Opus и AAC";
|
||||
player_choose_audio_files: "Choose files" , "Выбрать файлы";
|
||||
player_choose_audio_folder: "Choose folder" , "Выбрать папку";
|
||||
player_audio_folder_hint: "Audio files from subfolders are included; folder structure is preserved." , "Аудиофайлы из вложенных папок тоже загрузятся; структура папок сохранится.";
|
||||
player_upload_selected_files: "Upload selected files" , "Загрузить выбранные файлы";
|
||||
player_upload_history: "Upload history" , "История загрузок";
|
||||
player_no_file_uploads: "No file uploads yet" , "Загрузок файлов пока нет";
|
||||
|
||||
@@ -180,10 +180,17 @@ impl Job for InboxDiscoverJob {
|
||||
};
|
||||
|
||||
// Skip if hash already in media_files
|
||||
if crate::agent::rag::file_hash_exists(&ctx.pool, &hash)
|
||||
.await
|
||||
.unwrap_or(false)
|
||||
if crate::agent::rag::file_hash_exists_verified(
|
||||
&ctx.pool,
|
||||
&hash,
|
||||
&config.agent_storage_dir,
|
||||
)
|
||||
.await
|
||||
.unwrap_or(false)
|
||||
{
|
||||
// Close the upload by removing the exact inbox duplicate;
|
||||
// otherwise its history remains queued forever.
|
||||
tokio::fs::remove_file(file_path).await?;
|
||||
skipped_hash += 1;
|
||||
continue;
|
||||
}
|
||||
@@ -320,6 +327,9 @@ pub fn group_by_folder(files: &[PathBuf]) -> Vec<(PathBuf, Vec<PathBuf>)> {
|
||||
}
|
||||
|
||||
pub async fn collect_audio_files(dir: &Path, audio: &mut Vec<PathBuf>) -> anyhow::Result<()> {
|
||||
if crate::local_uploads::incomplete_batch(dir).await? {
|
||||
return Ok(());
|
||||
}
|
||||
let mut entries = tokio::fs::read_dir(dir).await?;
|
||||
while let Some(entry) = entries.next_entry().await? {
|
||||
let name = entry.file_name().to_string_lossy().into_owned();
|
||||
|
||||
@@ -7,6 +7,9 @@ use cot::db::{Database, Model};
|
||||
|
||||
/// Guard to prevent multiple inbox_process orchestrators from running simultaneously.
|
||||
static ORCHESTRATOR_RUNNING: AtomicBool = AtomicBool::new(false);
|
||||
// Bound approvals waiting on database import locks so active imports retain
|
||||
// spare pool connections for candidate queries and upgrade transactions.
|
||||
static IMPORT_PERMITS: tokio::sync::Semaphore = tokio::sync::Semaphore::const_new(2);
|
||||
|
||||
/// Well-known advisory lock ID for the inbox_process orchestrator.
|
||||
/// PostgreSQL advisory locks use a 64-bit key; this is an arbitrary unique value.
|
||||
@@ -387,7 +390,7 @@ async fn process_folder_batch(
|
||||
.to_owned();
|
||||
let file_exists = file_path.exists();
|
||||
if !sha.is_empty()
|
||||
&& crate::agent::rag::file_hash_exists(pool, &sha)
|
||||
&& crate::agent::rag::file_hash_exists_verified(pool, &sha, &config.agent_storage_dir)
|
||||
.await
|
||||
.unwrap_or(false)
|
||||
{
|
||||
@@ -642,7 +645,9 @@ async fn process_folder_batch(
|
||||
folder_files.sort();
|
||||
let track_count = folder_files.len();
|
||||
FolderContext {
|
||||
folder_path: folder_rel.to_owned(),
|
||||
folder_path: crate::jobs::strip_user_upload_prefix(Path::new(folder_rel))
|
||||
.to_string_lossy()
|
||||
.into_owned(),
|
||||
folder_files,
|
||||
track_count,
|
||||
}
|
||||
@@ -696,7 +701,7 @@ async fn process_folder_batch(
|
||||
Err(e) => {
|
||||
crate::metrics::record_agent_failed("llm");
|
||||
crate::metrics::record_agent_folder_batch("failed", file_count, batch_start.elapsed());
|
||||
let err_msg = format!("Batch LLM call failed: {e}");
|
||||
let err_msg = format!("Batch LLM call failed: {e:#}");
|
||||
log.error(&err_msg);
|
||||
// Mark all files as failed
|
||||
let prepared_count = prepared.len() as u64;
|
||||
@@ -810,6 +815,30 @@ async fn process_folder_batch(
|
||||
crate::metrics::record_agent_file_processed("ok", "auto_approved");
|
||||
ok_count += 1;
|
||||
}
|
||||
Err(e)
|
||||
if e.downcast_ref::<crate::agent::dedup::DuplicateReview>()
|
||||
.is_some() =>
|
||||
{
|
||||
let message = e.to_string();
|
||||
p.review.status = cot::db::LimitedString::new("pending").unwrap();
|
||||
p.review.error_message = Some(message.clone());
|
||||
p.review.updated_at = cot::db::LimitedString::new(
|
||||
chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ").to_string(),
|
||||
)
|
||||
.unwrap();
|
||||
if let Some(result) = p.review.result_json.as_deref()
|
||||
&& let Ok(mut result) = serde_json::from_str::<serde_json::Value>(result)
|
||||
{
|
||||
let previous = result["notes"].as_str().unwrap_or_default();
|
||||
result["notes"] =
|
||||
serde_json::json!(format!("{previous}\n{message}").trim());
|
||||
p.review.result_json = Some(result.to_string());
|
||||
}
|
||||
let _ = p.review.save(db).await;
|
||||
crate::metrics::record_agent_file_processed("ok", "pending_review");
|
||||
log.warn(&format!("{filename}: {message}"));
|
||||
ok_count += 1;
|
||||
}
|
||||
Err(e) => {
|
||||
crate::metrics::record_agent_failed("finalize");
|
||||
crate::metrics::record_agent_file_processed("failed", "failed");
|
||||
@@ -869,8 +898,54 @@ pub async fn finalize_approved(
|
||||
let year = normalized.year;
|
||||
let track_number = normalized.track_number;
|
||||
|
||||
// Serialize imports of a recording across orchestrators and manual approvals.
|
||||
// This transaction owns only the advisory lock; the existing Cot writes use
|
||||
// their normal connections and remain protected until finalization returns.
|
||||
let _import_permit = IMPORT_PERMITS.acquire().await?;
|
||||
let mut import_lock = pool.begin().await?;
|
||||
let mut import_artists: Vec<_> = normalized
|
||||
.featured_artists
|
||||
.iter()
|
||||
.map(|name| name.trim().to_lowercase())
|
||||
.collect();
|
||||
import_artists.push(artist_name.trim().to_lowercase());
|
||||
import_artists.sort();
|
||||
import_artists.dedup();
|
||||
for name in import_artists {
|
||||
let lock_key = serde_json::to_string(&(
|
||||
"furumi-recording-import",
|
||||
name,
|
||||
track_title.trim().to_lowercase(),
|
||||
))?;
|
||||
sqlx::query("SELECT pg_advisory_xact_lock(hashtextextended($1, 0))")
|
||||
.bind(lock_key)
|
||||
.execute(&mut *import_lock)
|
||||
.await?;
|
||||
}
|
||||
if crate::agent::dedup::merge_existing(
|
||||
pool,
|
||||
storage_dir_str,
|
||||
Path::new(input_path_str),
|
||||
normalized,
|
||||
context,
|
||||
)
|
||||
.await?
|
||||
{
|
||||
import_lock.commit().await?;
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let artist = find_or_create_artist(db, artist_name, model_name).await?;
|
||||
let release = find_or_create_release(db, release_title, release_type, year, model_name).await?;
|
||||
let release = find_or_create_release(
|
||||
db,
|
||||
pool,
|
||||
artist.id_val(),
|
||||
release_title,
|
||||
release_type,
|
||||
year,
|
||||
model_name,
|
||||
)
|
||||
.await?;
|
||||
|
||||
// Link ReleaseArtist
|
||||
let existing_links = ReleaseArtist::find_by_release(db, release.id_val())
|
||||
@@ -1215,6 +1290,7 @@ pub async fn finalize_approved(
|
||||
"Track finalized"
|
||||
);
|
||||
|
||||
import_lock.commit().await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -1241,17 +1317,31 @@ async fn find_or_create_artist(
|
||||
|
||||
async fn find_or_create_release(
|
||||
db: &cot::db::Database,
|
||||
pool: &sqlx::PgPool,
|
||||
artist_id: i64,
|
||||
title: &str,
|
||||
release_type: &str,
|
||||
year: Option<i32>,
|
||||
model_name: Option<&str>,
|
||||
) -> anyhow::Result<Release> {
|
||||
let title_sort = title.trim().to_lowercase();
|
||||
let all = Release::list_all(db).await.unwrap_or_default();
|
||||
for r in &all {
|
||||
if r.title_sort.as_str() == title_sort && r.release_type.as_str() == release_type {
|
||||
return Ok(r.clone());
|
||||
}
|
||||
// Release type describes an album; it is not a separate identity key.
|
||||
let existing_id: Option<i64> = sqlx::query_scalar(
|
||||
r#"SELECT r.id FROM furumusic__release r
|
||||
JOIN furumusic__release_artist ra ON ra.release_id=r.id
|
||||
WHERE ra.artist_id=$1 AND r.title_sort=$2
|
||||
AND ($3::integer IS NULL OR r.year IS NULL OR r.year=$3)
|
||||
ORDER BY r.id LIMIT 1"#,
|
||||
)
|
||||
.bind(artist_id)
|
||||
.bind(title_sort)
|
||||
.bind(year)
|
||||
.fetch_optional(pool)
|
||||
.await?;
|
||||
if let Some(id) = existing_id
|
||||
&& let Some(release) = Release::get_by_id(db, id).await?
|
||||
{
|
||||
return Ok(release);
|
||||
}
|
||||
Release::create(db, title, release_type, year, model_name)
|
||||
.await
|
||||
|
||||
+39
-1
@@ -29,7 +29,15 @@ pub fn strip_user_upload_prefix(relative_path: &Path) -> PathBuf {
|
||||
if components.len() >= 3
|
||||
&& matches!(components[0], Component::Normal(value) if value == "user_uploads")
|
||||
{
|
||||
components[2..].iter().collect()
|
||||
let start = if matches!(components[2], Component::Normal(value)
|
||||
if value.to_str().and_then(|v| v.strip_prefix("local-"))
|
||||
.and_then(|v| uuid::Uuid::parse_str(v).ok()).is_some())
|
||||
{
|
||||
3
|
||||
} else {
|
||||
2
|
||||
};
|
||||
components[start..].iter().collect()
|
||||
} else {
|
||||
relative_path.to_path_buf()
|
||||
}
|
||||
@@ -76,3 +84,33 @@ pub async fn uploader_from_relative_path(
|
||||
_ => UploaderAttribution::unknown(),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn local_upload_batch_id_is_not_an_artist_hint() {
|
||||
let path = Path::new(
|
||||
"user_uploads/6/local-123e4567-e89b-12d3-a456-426614174000/Artist/Album/01.mp3",
|
||||
);
|
||||
assert_eq!(
|
||||
strip_user_upload_prefix(path),
|
||||
PathBuf::from("Artist/Album/01.mp3")
|
||||
);
|
||||
assert_eq!(
|
||||
strip_user_upload_prefix(Path::new(
|
||||
"user_uploads/6/local-123e4567-e89b-12d3-a456-426614174000/01.mp3"
|
||||
)),
|
||||
PathBuf::from("01.mp3")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ordinary_upload_folder_names_are_preserved() {
|
||||
assert_eq!(
|
||||
strip_user_upload_prefix(Path::new("user_uploads/6/local-band/Album/01.mp3")),
|
||||
PathBuf::from("local-band/Album/01.mp3")
|
||||
);
|
||||
}
|
||||
}
|
||||
+207
-2
@@ -1,4 +1,5 @@
|
||||
use std::collections::HashMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use anyhow::{Context, bail};
|
||||
use serde::Serialize;
|
||||
@@ -6,6 +7,92 @@ use sqlx::{FromRow, PgPool};
|
||||
|
||||
const LOCAL_UPLOAD_LIST_LIMIT: i64 = 100;
|
||||
|
||||
pub async fn begin_batch(root: &Path) -> std::io::Result<()> {
|
||||
tokio::fs::create_dir_all(root).await?;
|
||||
tokio::fs::write(root.join(".uploading"), b"uploading").await
|
||||
}
|
||||
|
||||
pub async fn finish_batch(root: &Path) -> std::io::Result<()> {
|
||||
match tokio::fs::remove_file(root.join(".uploading")).await {
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()),
|
||||
result => result,
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn incomplete_batch(root: &Path) -> std::io::Result<bool> {
|
||||
let is_batch = root
|
||||
.file_name()
|
||||
.and_then(|s| s.to_str())
|
||||
.and_then(|s| s.strip_prefix("local-"))
|
||||
.is_some_and(|s| uuid::Uuid::parse_str(s).is_ok());
|
||||
if !is_batch {
|
||||
return Ok(false);
|
||||
}
|
||||
let marker = match tokio::fs::metadata(root.join(".uploading")).await {
|
||||
Ok(metadata) => metadata,
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(false),
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
// A crashed browser must not strand accepted files indefinitely.
|
||||
if marker.modified()?.elapsed().unwrap_or_default().as_secs() < 3600 {
|
||||
return Ok(true);
|
||||
}
|
||||
finish_batch(root).await?;
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
/// Validate browser-provided folder paths before appending them to a user's inbox.
|
||||
pub fn relative_upload_path(value: &str) -> anyhow::Result<PathBuf> {
|
||||
if value.is_empty() || value.contains('\\') || value.chars().any(char::is_control) {
|
||||
bail!("invalid relative upload path");
|
||||
}
|
||||
let mut path = PathBuf::new();
|
||||
for part in value.split('/') {
|
||||
if part.is_empty() || matches!(part, "." | "..") || part.ends_with(':') {
|
||||
bail!("upload path must be relative and contain no traversal components");
|
||||
}
|
||||
path.push(part);
|
||||
}
|
||||
if !path.is_relative() {
|
||||
bail!("upload path must be relative");
|
||||
}
|
||||
Ok(path)
|
||||
}
|
||||
|
||||
/// Publish a complete file atomically without overwriting another upload.
|
||||
pub async fn save_file(destination: &Path, upload_id: &str, bytes: &[u8]) -> std::io::Result<()> {
|
||||
let parent = destination.parent().ok_or_else(|| {
|
||||
std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidInput,
|
||||
"upload has no parent folder",
|
||||
)
|
||||
})?;
|
||||
tokio::fs::create_dir_all(parent).await?;
|
||||
let temporary = parent.join(format!(".{upload_id}.{}.uploading", uuid::Uuid::new_v4()));
|
||||
let result = async {
|
||||
tokio::fs::write(&temporary, bytes).await?;
|
||||
// Unlike rename, hard_link fails if the destination already exists.
|
||||
match tokio::fs::hard_link(&temporary, destination).await {
|
||||
Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {
|
||||
use sha2::{Digest, Sha256};
|
||||
let incoming_hash = format!("{:x}", Sha256::digest(bytes));
|
||||
let existing_hash = crate::agent::mover::sha256_file(destination)
|
||||
.await
|
||||
.map_err(std::io::Error::other)?;
|
||||
if existing_hash == incoming_hash {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(error)
|
||||
}
|
||||
}
|
||||
result => result,
|
||||
}
|
||||
}
|
||||
.await;
|
||||
let _ = tokio::fs::remove_file(&temporary).await;
|
||||
result
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct LocalUploadDto {
|
||||
pub id: String,
|
||||
@@ -56,11 +143,16 @@ pub async fn create(
|
||||
inbox_path: &str,
|
||||
) -> anyhow::Result<()> {
|
||||
let now = now_string();
|
||||
sqlx::query(
|
||||
let created = sqlx::query(
|
||||
r#"INSERT INTO furumusic__local_upload
|
||||
(id, user_id, filename, size_bytes, status, inbox_path, error,
|
||||
created_at, updated_at, completed_at)
|
||||
VALUES ($1, $2, $3, $4, 'uploading', $5, NULL, $6, $6, NULL)"#,
|
||||
VALUES ($1, $2, $3, $4, 'uploading', $5, NULL, $6, $6, NULL)
|
||||
ON CONFLICT (id) DO UPDATE SET status='uploading', error=NULL, updated_at=$6
|
||||
WHERE furumusic__local_upload.user_id=$2
|
||||
AND furumusic__local_upload.inbox_path=$5
|
||||
AND furumusic__local_upload.size_bytes=$4
|
||||
AND furumusic__local_upload.status IN ('uploading', 'failed')"#,
|
||||
)
|
||||
.bind(id)
|
||||
.bind(user_id)
|
||||
@@ -70,9 +162,34 @@ pub async fn create(
|
||||
.bind(now)
|
||||
.execute(pool)
|
||||
.await?;
|
||||
if created.rows_affected() == 0 {
|
||||
bail!("upload ID is already in use");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Return a completed upload receipt when a client retries after losing the
|
||||
/// HTTP response. A receipt is scoped to this user and exact destination.
|
||||
pub async fn successful_receipt(
|
||||
pool: &PgPool,
|
||||
user_id: i64,
|
||||
id: &str,
|
||||
inbox_path: &str,
|
||||
) -> anyhow::Result<Option<LocalUploadDto>> {
|
||||
let row = sqlx::query_as::<_, LocalUploadRow>(
|
||||
r#"SELECT id, user_id, filename, size_bytes, status, inbox_path, error,
|
||||
created_at, updated_at, completed_at
|
||||
FROM furumusic__local_upload WHERE id=$1 AND user_id=$2 AND inbox_path=$3
|
||||
AND status NOT IN ('uploading','failed')"#,
|
||||
)
|
||||
.bind(id)
|
||||
.bind(user_id)
|
||||
.bind(inbox_path)
|
||||
.fetch_optional(pool)
|
||||
.await?;
|
||||
Ok(row.map(|row| row.dto()))
|
||||
}
|
||||
|
||||
pub async fn mark_queued(pool: &PgPool, id: &str, user_id: i64) -> anyhow::Result<LocalUploadDto> {
|
||||
update_status(pool, id, user_id, "queued", None).await?;
|
||||
load(pool, user_id, id).await.map(|row| row.dto())
|
||||
@@ -257,3 +374,91 @@ fn trim_error(value: &str) -> String {
|
||||
fn now_string() -> String {
|
||||
chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ").to_string()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn folder_discovery_waits_for_completion_and_recovers_abandoned_uploads() {
|
||||
let parent =
|
||||
std::env::temp_dir().join(format!("furumi-batch-test-{}", uuid::Uuid::new_v4()));
|
||||
let root = parent.join(format!("local-{}", uuid::Uuid::new_v4()));
|
||||
begin_batch(&root).await.unwrap();
|
||||
save_file(&root.join("Artist/Album/01.mp3"), "first", b"audio")
|
||||
.await
|
||||
.unwrap();
|
||||
let mut found = Vec::new();
|
||||
crate::jobs::inbox_discover::collect_audio_files(&parent, &mut found)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(found.is_empty(), "partial folder was queued for AI");
|
||||
finish_batch(&root).await.unwrap();
|
||||
crate::jobs::inbox_discover::collect_audio_files(&parent, &mut found)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(found.len(), 1);
|
||||
begin_batch(&root).await.unwrap();
|
||||
std::fs::File::options()
|
||||
.write(true)
|
||||
.open(root.join(".uploading"))
|
||||
.unwrap()
|
||||
.set_times(
|
||||
std::fs::FileTimes::new().set_modified(
|
||||
std::time::SystemTime::now() - std::time::Duration::from_secs(7200),
|
||||
),
|
||||
)
|
||||
.unwrap();
|
||||
assert!(!incomplete_batch(&root).await.unwrap());
|
||||
assert!(!root.join(".uploading").exists());
|
||||
tokio::fs::remove_dir_all(parent).await.unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn folder_paths_preserve_unicode_and_nested_names() {
|
||||
let path = "Музыка/Исполнитель/2002 - Альбом/08 - Где же ваши руки.mp3";
|
||||
assert_eq!(relative_upload_path(path).unwrap(), PathBuf::from(path));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn folder_paths_reject_absolute_paths_and_traversal() {
|
||||
for path in [
|
||||
"",
|
||||
"/etc/file.mp3",
|
||||
"../file.mp3",
|
||||
"Album/../../file.mp3",
|
||||
"Album/./file.mp3",
|
||||
"Album//file.mp3",
|
||||
"Album/",
|
||||
"C:/file.mp3",
|
||||
"Album\\..\\file.mp3",
|
||||
"Album/\0file.mp3",
|
||||
] {
|
||||
assert!(relative_upload_path(path).is_err(), "accepted {path:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn folder_uploads_keep_siblings_and_never_overwrite_existing_files() {
|
||||
let root =
|
||||
std::env::temp_dir().join(format!("furumi-upload-test-{}", uuid::Uuid::new_v4()));
|
||||
let first = root.join("Artist A/Album/01.mp3");
|
||||
let second = root.join("Artist B/Album/01.mp3");
|
||||
save_file(&first, "first", b"first audio").await.unwrap();
|
||||
save_file(&second, "second", b"second audio").await.unwrap();
|
||||
save_file(&first, "first-retry", b"first audio")
|
||||
.await
|
||||
.unwrap();
|
||||
let error = save_file(&first, "duplicate", b"replacement")
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(error.kind(), std::io::ErrorKind::AlreadyExists);
|
||||
assert_eq!(tokio::fs::read(&first).await.unwrap(), b"first audio");
|
||||
assert_eq!(tokio::fs::read(&second).await.unwrap(), b"second audio");
|
||||
let mut entries = tokio::fs::read_dir(first.parent().unwrap()).await.unwrap();
|
||||
while let Some(entry) = entries.next_entry().await.unwrap() {
|
||||
assert!(!entry.file_name().to_string_lossy().ends_with(".uploading"));
|
||||
}
|
||||
tokio::fs::remove_dir_all(root).await.unwrap();
|
||||
}
|
||||
}
|
||||
+81
-13
@@ -5157,6 +5157,47 @@ async fn local_upload_handler(
|
||||
));
|
||||
}
|
||||
|
||||
let batch_header_id = request
|
||||
.headers()
|
||||
.get("x-furumusic-upload-batch-id")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.and_then(|value| uuid::Uuid::parse_str(value.trim()).ok());
|
||||
let batch_root = batch_header_id.map(|id| {
|
||||
inbox_root
|
||||
.join("user_uploads")
|
||||
.join(user.id.to_string())
|
||||
.join(format!("local-{id}"))
|
||||
});
|
||||
if request
|
||||
.headers()
|
||||
.get("x-furumusic-upload-complete")
|
||||
.is_some_and(|value| value == "true")
|
||||
{
|
||||
let Some(root) = batch_root else {
|
||||
return Ok(json_error(
|
||||
StatusCode::BAD_REQUEST,
|
||||
"batch completion requires a valid batch ID",
|
||||
));
|
||||
};
|
||||
crate::local_uploads::finish_batch(&root)
|
||||
.await
|
||||
.map_err(|err| cot::Error::internal(err.to_string()))?;
|
||||
if let Some(handle) = scheduler_handle.get() {
|
||||
let handle = Arc::clone(handle);
|
||||
tokio::spawn(async move {
|
||||
if let Err(err) = handle.trigger_job_now("inbox_discover").await {
|
||||
tracing::warn!("failed to trigger discovery after folder upload: {err}");
|
||||
}
|
||||
});
|
||||
}
|
||||
return Json(serde_json::json!({"ok": true})).into_response();
|
||||
}
|
||||
if let Some(root) = &batch_root {
|
||||
crate::local_uploads::begin_batch(root)
|
||||
.await
|
||||
.map_err(|err| cot::Error::internal(err.to_string()))?;
|
||||
}
|
||||
|
||||
let filename_header = HeaderName::from_static("x-furumusic-filename");
|
||||
let original_name = request
|
||||
.headers()
|
||||
@@ -5176,6 +5217,32 @@ async fn local_upload_handler(
|
||||
.unwrap_or_else(uuid::Uuid::new_v4)
|
||||
.to_string();
|
||||
|
||||
// One batch directory preserves sibling and nested folders from the picker.
|
||||
// Older clients keep their existing per-file upload directories.
|
||||
let batch_id = request
|
||||
.headers()
|
||||
.get("x-furumusic-upload-batch-id")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.and_then(|value| uuid::Uuid::parse_str(value.trim()).ok())
|
||||
.map(|value| value.to_string())
|
||||
.unwrap_or_else(|| upload_id.clone());
|
||||
let relative_path = match request.headers().get("x-furumusic-relative-path") {
|
||||
Some(value) => match value.to_str().ok().map(percent_decode_header) {
|
||||
Some(value) => match crate::local_uploads::relative_upload_path(&value) {
|
||||
Ok(path) => path,
|
||||
Err(err) => return Ok(json_error(StatusCode::BAD_REQUEST, &err.to_string())),
|
||||
},
|
||||
None => {
|
||||
return Ok(json_error(
|
||||
StatusCode::BAD_REQUEST,
|
||||
"invalid upload path header",
|
||||
));
|
||||
}
|
||||
},
|
||||
None => std::path::PathBuf::from(&filename),
|
||||
};
|
||||
let display_name = relative_path.to_string_lossy().to_string();
|
||||
|
||||
let bytes = request
|
||||
.into_body()
|
||||
.into_bytes()
|
||||
@@ -5191,8 +5258,8 @@ async fn local_upload_handler(
|
||||
let upload_dir = inbox_root
|
||||
.join("user_uploads")
|
||||
.join(user.id.to_string())
|
||||
.join(format!("local-{upload_id}"));
|
||||
let destination = upload_dir.join(&filename);
|
||||
.join(format!("local-{batch_id}"));
|
||||
let destination = upload_dir.join(&relative_path);
|
||||
let Some(inbox_path) =
|
||||
crate::media_paths::path_for_root(&inbox_root.to_string_lossy(), &destination)
|
||||
else {
|
||||
@@ -5201,11 +5268,18 @@ async fn local_upload_handler(
|
||||
"upload destination escaped agent_inbox_dir",
|
||||
));
|
||||
};
|
||||
if let Some(upload) =
|
||||
crate::local_uploads::successful_receipt(pool, user.id, &upload_id, &inbox_path)
|
||||
.await
|
||||
.map_err(|err| cot::Error::internal(err.to_string()))?
|
||||
{
|
||||
return Json(LocalUploadResponse { ok: true, upload }).into_response();
|
||||
}
|
||||
if let Err(err) = crate::local_uploads::create(
|
||||
pool,
|
||||
&upload_id,
|
||||
user.id,
|
||||
&filename,
|
||||
&display_name,
|
||||
bytes.len() as u64,
|
||||
&inbox_path,
|
||||
)
|
||||
@@ -5214,17 +5288,9 @@ async fn local_upload_handler(
|
||||
return Ok(json_error(StatusCode::BAD_REQUEST, &err.to_string()));
|
||||
}
|
||||
|
||||
let temporary = upload_dir.join(format!(".{upload_id}.uploading"));
|
||||
let write_result = async {
|
||||
tokio::fs::create_dir_all(&upload_dir).await?;
|
||||
tokio::fs::write(&temporary, &bytes).await?;
|
||||
tokio::fs::rename(&temporary, &destination).await?;
|
||||
Ok::<(), std::io::Error>(())
|
||||
}
|
||||
.await;
|
||||
let write_result = crate::local_uploads::save_file(&destination, &upload_id, &bytes).await;
|
||||
if let Err(err) = write_result {
|
||||
let message = format!("could not save uploaded file: {err}");
|
||||
let _ = tokio::fs::remove_dir_all(&upload_dir).await;
|
||||
let _ = crate::local_uploads::mark_failed(pool, &upload_id, user.id, &message).await;
|
||||
return Ok(json_error(StatusCode::INTERNAL_SERVER_ERROR, &message));
|
||||
}
|
||||
@@ -5239,7 +5305,9 @@ async fn local_upload_handler(
|
||||
}
|
||||
};
|
||||
|
||||
if let Some(handle) = scheduler_handle.get() {
|
||||
if batch_root.is_none()
|
||||
&& let Some(handle) = scheduler_handle.get()
|
||||
{
|
||||
let handle = Arc::clone(handle);
|
||||
tokio::spawn(async move {
|
||||
if let Err(err) = handle.trigger_job_now("inbox_discover").await {
|
||||
|
||||
Reference in new issue
Block a user