Furumi init
This commit is contained in:
@@ -0,0 +1,408 @@
|
||||
//! Cover art extraction and management.
|
||||
//!
|
||||
//! Sources (in priority order):
|
||||
//! 1. Standalone image files in the album folder (cover.jpg, folder.jpg, etc.)
|
||||
//! 2. Embedded cover art in audio file metadata (ID3 APIC, Vorbis METADATA_BLOCK_PICTURE, etc.)
|
||||
//!
|
||||
//! The first usable image found is saved as a MediaFile with file_type="cover_art"
|
||||
//! and linked to the Release via cover_file_id.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
/// Image data extracted from an audio file or found on disk.
|
||||
#[derive(Debug)]
|
||||
pub struct CoverImage {
|
||||
pub data: Vec<u8>,
|
||||
pub mime_type: String,
|
||||
/// Where this image came from (for logging).
|
||||
pub source: CoverSource,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum CoverSource {
|
||||
/// A standalone image file in the folder.
|
||||
FolderFile(PathBuf),
|
||||
/// Embedded in an audio file's metadata.
|
||||
Embedded(PathBuf),
|
||||
}
|
||||
|
||||
/// Well-known cover art filenames, in priority order.
|
||||
/// Case-insensitive matching is used.
|
||||
const COVER_FILENAMES: &[&str] = &[
|
||||
"cover",
|
||||
"folder",
|
||||
"front",
|
||||
"album",
|
||||
"albumart",
|
||||
"albumartsmall",
|
||||
"thumb",
|
||||
"artwork",
|
||||
];
|
||||
|
||||
const IMAGE_EXTENSIONS: &[&str] = &["jpg", "jpeg", "png", "webp", "bmp", "gif"];
|
||||
|
||||
fn is_image_file(name: &str) -> bool {
|
||||
let ext = name.rsplit('.').next().unwrap_or("").to_lowercase();
|
||||
IMAGE_EXTENSIONS.contains(&ext.as_str())
|
||||
}
|
||||
|
||||
fn mime_for_image(path: &Path) -> String {
|
||||
let ext = path
|
||||
.extension()
|
||||
.and_then(|e| e.to_str())
|
||||
.unwrap_or("")
|
||||
.to_lowercase();
|
||||
match ext.as_str() {
|
||||
"jpg" | "jpeg" => "image/jpeg".to_string(),
|
||||
"png" => "image/png".to_string(),
|
||||
"webp" => "image/webp".to_string(),
|
||||
"gif" => "image/gif".to_string(),
|
||||
"bmp" => "image/bmp".to_string(),
|
||||
_ => "application/octet-stream".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Scan a folder for image files that look like cover art.
|
||||
///
|
||||
/// Returns image file paths sorted by priority:
|
||||
/// - Files with well-known names (cover.jpg, front.png, etc.) first
|
||||
/// - Then any other image files
|
||||
pub fn find_folder_images(folder: &Path) -> Vec<PathBuf> {
|
||||
let entries = match std::fs::read_dir(folder) {
|
||||
Ok(rd) => rd,
|
||||
Err(_) => return Vec::new(),
|
||||
};
|
||||
|
||||
let mut images: Vec<PathBuf> = entries
|
||||
.filter_map(|e| e.ok())
|
||||
.filter(|e| {
|
||||
let name = e.file_name().to_string_lossy().into_owned();
|
||||
!name.starts_with('.') && is_image_file(&name)
|
||||
})
|
||||
.map(|e| e.path())
|
||||
.collect();
|
||||
|
||||
// Sort: well-known names first (by priority index), then alphabetically
|
||||
images.sort_by(|a, b| {
|
||||
let pri_a = cover_name_priority(a);
|
||||
let pri_b = cover_name_priority(b);
|
||||
pri_a.cmp(&pri_b).then_with(|| a.cmp(b))
|
||||
});
|
||||
|
||||
images
|
||||
}
|
||||
|
||||
/// Return a priority index for a filename (lower = higher priority).
|
||||
/// Well-known cover filenames get indices 0..N, unknown ones get usize::MAX.
|
||||
fn cover_name_priority(path: &Path) -> usize {
|
||||
let stem = path
|
||||
.file_stem()
|
||||
.and_then(|s| s.to_str())
|
||||
.unwrap_or("")
|
||||
.to_lowercase();
|
||||
|
||||
for (i, &known) in COVER_FILENAMES.iter().enumerate() {
|
||||
if stem == known {
|
||||
return i;
|
||||
}
|
||||
}
|
||||
usize::MAX
|
||||
}
|
||||
|
||||
/// Try to find the best cover image for a folder of audio files.
|
||||
///
|
||||
/// Strategy:
|
||||
/// 1. Look for standalone image files in the folder (prioritized by filename).
|
||||
/// 2. Try to extract embedded cover art from each audio file.
|
||||
///
|
||||
/// Returns the first usable image found, or None.
|
||||
pub async fn find_best_cover(
|
||||
folder: &Path,
|
||||
audio_files: &[PathBuf],
|
||||
) -> Option<CoverImage> {
|
||||
// Strategy 1: folder images
|
||||
let folder_images = find_folder_images(folder);
|
||||
for img_path in &folder_images {
|
||||
match tokio::fs::read(img_path).await {
|
||||
Ok(data) if !data.is_empty() => {
|
||||
let mime = mime_for_image(img_path);
|
||||
return Some(CoverImage {
|
||||
data,
|
||||
mime_type: mime,
|
||||
source: CoverSource::FolderFile(img_path.clone()),
|
||||
});
|
||||
}
|
||||
_ => continue,
|
||||
}
|
||||
}
|
||||
|
||||
// Strategy 2: embedded cover art from audio files
|
||||
for audio_path in audio_files {
|
||||
let path = audio_path.to_path_buf();
|
||||
let result = tokio::task::spawn_blocking(move || extract_embedded_cover(&path)).await;
|
||||
if let Ok(Some(cover)) = result {
|
||||
return Some(cover);
|
||||
}
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
/// Extract embedded cover art from an audio file.
|
||||
///
|
||||
/// Tries Symphonia first (works for FLAC, OGG, etc.), then falls back to
|
||||
/// id3 crate for MP3 files.
|
||||
///
|
||||
/// Must be called from a blocking context.
|
||||
fn extract_embedded_cover(path: &Path) -> Option<CoverImage> {
|
||||
// Try Symphonia visuals first
|
||||
if let Some(cover) = extract_cover_symphonia(path) {
|
||||
return Some(cover);
|
||||
}
|
||||
|
||||
// Fallback: id3 for MP3
|
||||
let is_mp3 = path
|
||||
.extension()
|
||||
.and_then(|e| e.to_str())
|
||||
.map(|e| e.eq_ignore_ascii_case("mp3"))
|
||||
.unwrap_or(false);
|
||||
|
||||
if is_mp3 {
|
||||
return extract_cover_id3(path);
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
fn extract_cover_symphonia(path: &Path) -> Option<CoverImage> {
|
||||
use symphonia::core::formats::FormatOptions;
|
||||
use symphonia::core::io::MediaSourceStream;
|
||||
use symphonia::core::meta::MetadataOptions;
|
||||
use symphonia::core::probe::Hint;
|
||||
|
||||
let file = std::fs::File::open(path).ok()?;
|
||||
let mss = MediaSourceStream::new(Box::new(file), Default::default());
|
||||
|
||||
let mut hint = Hint::new();
|
||||
if let Some(ext) = path.extension().and_then(|e| e.to_str()) {
|
||||
hint.with_extension(ext);
|
||||
}
|
||||
|
||||
let mut probed = symphonia::default::get_probe()
|
||||
.format(
|
||||
&hint,
|
||||
mss,
|
||||
&FormatOptions {
|
||||
enable_gapless: false,
|
||||
..Default::default()
|
||||
},
|
||||
&MetadataOptions::default(),
|
||||
)
|
||||
.ok()?;
|
||||
|
||||
// Check side-data metadata (ID3 before format)
|
||||
if let Some(rev) = probed.metadata.get().as_ref().and_then(|m| m.current()) {
|
||||
for visual in rev.visuals() {
|
||||
if !visual.data.is_empty() {
|
||||
let mime = if visual.media_type.is_empty() {
|
||||
guess_image_mime(&visual.data)
|
||||
} else {
|
||||
visual.media_type.to_string()
|
||||
};
|
||||
return Some(CoverImage {
|
||||
data: visual.data.to_vec(),
|
||||
mime_type: mime,
|
||||
source: CoverSource::Embedded(path.to_path_buf()),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check format-level metadata
|
||||
if let Some(rev) = probed.format.metadata().current() {
|
||||
for visual in rev.visuals() {
|
||||
if !visual.data.is_empty() {
|
||||
let mime = if visual.media_type.is_empty() {
|
||||
guess_image_mime(&visual.data)
|
||||
} else {
|
||||
visual.media_type.to_string()
|
||||
};
|
||||
return Some(CoverImage {
|
||||
data: visual.data.to_vec(),
|
||||
mime_type: mime,
|
||||
source: CoverSource::Embedded(path.to_path_buf()),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
fn extract_cover_id3(path: &Path) -> Option<CoverImage> {
|
||||
let tag = id3::Tag::read_from_path(path).ok()?;
|
||||
|
||||
// Prefer front cover (picture type 3), then any picture
|
||||
let mut best: Option<&id3::frame::Picture> = None;
|
||||
for pic in tag.pictures() {
|
||||
if pic.picture_type == id3::frame::PictureType::CoverFront {
|
||||
best = Some(pic);
|
||||
break;
|
||||
}
|
||||
if best.is_none() {
|
||||
best = Some(pic);
|
||||
}
|
||||
}
|
||||
|
||||
let pic = best?;
|
||||
if pic.data.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mime = if pic.mime_type.is_empty() || pic.mime_type == "image/" {
|
||||
guess_image_mime(&pic.data)
|
||||
} else {
|
||||
pic.mime_type.clone()
|
||||
};
|
||||
|
||||
Some(CoverImage {
|
||||
data: pic.data.clone(),
|
||||
mime_type: mime,
|
||||
source: CoverSource::Embedded(path.to_path_buf()),
|
||||
})
|
||||
}
|
||||
|
||||
/// Guess MIME type from image magic bytes.
|
||||
fn guess_image_mime(data: &[u8]) -> String {
|
||||
if data.starts_with(&[0xFF, 0xD8, 0xFF]) {
|
||||
"image/jpeg".to_string()
|
||||
} else if data.starts_with(&[0x89, 0x50, 0x4E, 0x47]) {
|
||||
"image/png".to_string()
|
||||
} else if data.starts_with(b"RIFF") && data.len() > 12 && &data[8..12] == b"WEBP" {
|
||||
"image/webp".to_string()
|
||||
} else if data.starts_with(b"GIF8") {
|
||||
"image/gif".to_string()
|
||||
} else if data.starts_with(&[0x42, 0x4D]) {
|
||||
"image/bmp".to_string()
|
||||
} else {
|
||||
"image/jpeg".to_string() // default assumption
|
||||
}
|
||||
}
|
||||
|
||||
/// Compute SHA-256 hash of image data.
|
||||
pub fn hash_image(data: &[u8]) -> String {
|
||||
let digest = Sha256::digest(data);
|
||||
format!("{:x}", digest)
|
||||
}
|
||||
|
||||
/// Extension for a MIME type.
|
||||
pub fn extension_for_mime(mime: &str) -> &str {
|
||||
match mime {
|
||||
"image/jpeg" => "jpg",
|
||||
"image/png" => "png",
|
||||
"image/webp" => "webp",
|
||||
"image/gif" => "gif",
|
||||
"image/bmp" => "bmp",
|
||||
_ => "jpg",
|
||||
}
|
||||
}
|
||||
|
||||
/// Save cover image data to the storage directory and create a MediaFile record.
|
||||
///
|
||||
/// Returns the MediaFile ID on success.
|
||||
pub async fn save_cover_to_storage(
|
||||
db: &cot::db::Database,
|
||||
pool: &sqlx::PgPool,
|
||||
storage_dir: &str,
|
||||
artist_name: &str,
|
||||
release_title: &str,
|
||||
cover: &CoverImage,
|
||||
) -> anyhow::Result<i64> {
|
||||
let hash = hash_image(&cover.data);
|
||||
|
||||
// Check if we already have this exact image in the DB
|
||||
let existing: Option<(i64,)> = sqlx::query_as(
|
||||
"SELECT id FROM furumusic__media_file WHERE sha256_hash = $1 AND file_type = 'cover_art' LIMIT 1",
|
||||
)
|
||||
.bind(&hash)
|
||||
.fetch_optional(pool)
|
||||
.await?;
|
||||
|
||||
if let Some((id,)) = existing {
|
||||
return Ok(id);
|
||||
}
|
||||
|
||||
let ext = extension_for_mime(&cover.mime_type);
|
||||
let filename = format!("cover.{ext}");
|
||||
|
||||
let artist_dir = sanitize_dir_name(artist_name);
|
||||
let album_dir = sanitize_dir_name(release_title);
|
||||
|
||||
let dest_dir = Path::new(storage_dir).join(&artist_dir).join(&album_dir);
|
||||
tokio::fs::create_dir_all(&dest_dir).await?;
|
||||
|
||||
let dest_path = dest_dir.join(&filename);
|
||||
|
||||
// Write image data
|
||||
tokio::fs::write(&dest_path, &cover.data).await?;
|
||||
|
||||
let relative_path = dest_path.to_string_lossy().to_string();
|
||||
let file_size = cover.data.len() as i64;
|
||||
|
||||
let media_file = crate::music::MediaFile::create(
|
||||
db,
|
||||
"cover_art",
|
||||
&relative_path,
|
||||
&filename,
|
||||
&cover.mime_type,
|
||||
file_size,
|
||||
&hash,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("failed to create cover MediaFile: {e}"))?;
|
||||
|
||||
tracing::info!(
|
||||
media_file_id = media_file.id_val(),
|
||||
hash = %hash,
|
||||
mime = %cover.mime_type,
|
||||
size = file_size,
|
||||
"Saved cover art"
|
||||
);
|
||||
|
||||
Ok(media_file.id_val())
|
||||
}
|
||||
|
||||
/// Set the cover_file_id on a release (if not already set).
|
||||
pub async fn assign_cover_to_release(
|
||||
pool: &sqlx::PgPool,
|
||||
release_id: i64,
|
||||
cover_file_id: i64,
|
||||
) -> anyhow::Result<()> {
|
||||
sqlx::query(
|
||||
"UPDATE furumusic__release SET cover_file_id = $1, updated_at = $3 WHERE id = $2 AND cover_file_id IS NULL",
|
||||
)
|
||||
.bind(cover_file_id)
|
||||
.bind(release_id)
|
||||
.bind(chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ").to_string())
|
||||
.execute(pool)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn sanitize_dir_name(name: &str) -> String {
|
||||
name.chars()
|
||||
.map(|c| match c {
|
||||
'/' | '\\' | ':' | '*' | '?' | '"' | '<' | '>' | '|' | '\0' => '_',
|
||||
_ => c,
|
||||
})
|
||||
.collect::<String>()
|
||||
.trim()
|
||||
.trim_matches('.')
|
||||
.to_owned()
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Raw metadata extracted from audio file tags.
|
||||
#[derive(Debug, Default)]
|
||||
pub struct RawMetadata {
|
||||
pub title: Option<String>,
|
||||
pub artist: Option<String>,
|
||||
pub album: Option<String>,
|
||||
pub track_number: Option<u32>,
|
||||
pub year: Option<u32>,
|
||||
pub genre: Option<String>,
|
||||
pub duration_secs: Option<f64>,
|
||||
}
|
||||
|
||||
/// Hints parsed from the file path (directory structure + filename).
|
||||
#[derive(Debug, Default)]
|
||||
pub struct PathHints {
|
||||
pub title: Option<String>,
|
||||
pub artist: Option<String>,
|
||||
pub album: Option<String>,
|
||||
pub year: Option<i32>,
|
||||
pub track_number: Option<i32>,
|
||||
}
|
||||
|
||||
/// Normalized metadata returned by the LLM.
|
||||
#[derive(Debug, Default, Serialize, Deserialize)]
|
||||
pub struct NormalizedFields {
|
||||
pub title: Option<String>,
|
||||
pub artist: Option<String>,
|
||||
pub album: Option<String>,
|
||||
pub year: Option<i32>,
|
||||
pub track_number: Option<i32>,
|
||||
pub genre: Option<String>,
|
||||
#[serde(default)]
|
||||
pub featured_artists: Vec<String>,
|
||||
pub release_type: Option<String>,
|
||||
pub confidence: Option<f64>,
|
||||
pub notes: Option<String>,
|
||||
}
|
||||
|
||||
/// A similar artist found via pg_trgm fuzzy search.
|
||||
#[derive(Debug, Clone)]
|
||||
#[allow(dead_code)]
|
||||
pub struct SimilarArtist {
|
||||
pub id: i64,
|
||||
pub name: String,
|
||||
pub similarity: f32,
|
||||
}
|
||||
|
||||
/// A similar release found via pg_trgm fuzzy search.
|
||||
#[derive(Debug, Clone)]
|
||||
#[allow(dead_code)]
|
||||
pub struct SimilarRelease {
|
||||
pub id: i64,
|
||||
pub title: String,
|
||||
pub year: Option<i32>,
|
||||
pub similarity: f32,
|
||||
}
|
||||
|
||||
/// Context about other files in the same folder (for the LLM).
|
||||
pub struct FolderContext {
|
||||
pub folder_path: String,
|
||||
pub folder_files: Vec<String>,
|
||||
pub track_count: usize,
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
use std::path::Path;
|
||||
|
||||
use symphonia::core::{
|
||||
codecs::CODEC_TYPE_NULL,
|
||||
formats::FormatOptions,
|
||||
io::MediaSourceStream,
|
||||
meta::{MetadataOptions, StandardTagKey},
|
||||
probe::Hint,
|
||||
};
|
||||
|
||||
use super::dto::RawMetadata;
|
||||
|
||||
/// Extract metadata from an audio file.
|
||||
///
|
||||
/// For MP3, falls back to the `id3` crate when Symphonia cannot probe the file
|
||||
/// (e.g. ID3 tag with large embedded cover art exceeds Symphonia's probe limit).
|
||||
///
|
||||
/// Must be called from a blocking context (`spawn_blocking`).
|
||||
pub fn extract(path: &Path) -> anyhow::Result<RawMetadata> {
|
||||
match extract_via_symphonia(path) {
|
||||
Ok(meta) => Ok(meta),
|
||||
Err(e) => {
|
||||
let is_mp3 = path
|
||||
.extension()
|
||||
.and_then(|e| e.to_str())
|
||||
.map(|e| e.eq_ignore_ascii_case("mp3"))
|
||||
.unwrap_or(false);
|
||||
if is_mp3 {
|
||||
tracing::debug!(error = %e, "Symphonia failed on MP3, falling back to id3 crate");
|
||||
extract_mp3_via_id3(path)
|
||||
} else {
|
||||
Err(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn extract_via_symphonia(path: &Path) -> anyhow::Result<RawMetadata> {
|
||||
let file = std::fs::File::open(path)?;
|
||||
let mss = MediaSourceStream::new(Box::new(file), Default::default());
|
||||
|
||||
let mut hint = Hint::new();
|
||||
if let Some(ext) = path.extension().and_then(|e| e.to_str()) {
|
||||
hint.with_extension(ext);
|
||||
}
|
||||
|
||||
let mut probed = symphonia::default::get_probe().format(
|
||||
&hint,
|
||||
mss,
|
||||
&FormatOptions {
|
||||
enable_gapless: false,
|
||||
..Default::default()
|
||||
},
|
||||
&MetadataOptions::default(),
|
||||
)?;
|
||||
|
||||
let mut meta = RawMetadata::default();
|
||||
|
||||
// Check metadata side-data (e.g. ID3 tags probed before format)
|
||||
if let Some(rev) = probed.metadata.get().as_ref().and_then(|m| m.current()) {
|
||||
extract_tags(rev.tags(), &mut meta);
|
||||
}
|
||||
|
||||
// Also check format-embedded metadata
|
||||
if let Some(rev) = probed.format.metadata().current() {
|
||||
if meta.title.is_none() {
|
||||
extract_tags(rev.tags(), &mut meta);
|
||||
}
|
||||
}
|
||||
|
||||
// Duration
|
||||
meta.duration_secs = probed
|
||||
.format
|
||||
.tracks()
|
||||
.iter()
|
||||
.find(|t| t.codec_params.codec != CODEC_TYPE_NULL)
|
||||
.and_then(|t| {
|
||||
let n_frames = t.codec_params.n_frames?;
|
||||
let tb = t.codec_params.time_base?;
|
||||
Some(n_frames as f64 * tb.numer as f64 / tb.denom as f64)
|
||||
});
|
||||
|
||||
Ok(meta)
|
||||
}
|
||||
|
||||
/// Read MP3 tags via the `id3` crate. Duration is not available this way.
|
||||
fn extract_mp3_via_id3(path: &Path) -> anyhow::Result<RawMetadata> {
|
||||
use id3::TagLike;
|
||||
|
||||
let tag =
|
||||
id3::Tag::read_from_path(path).map_err(|e| anyhow::anyhow!("id3 read failed: {}", e))?;
|
||||
|
||||
let mut meta = RawMetadata::default();
|
||||
meta.title = tag.title().map(|s| fix_encoding(s.to_owned()));
|
||||
meta.artist = tag.artist().map(|s| fix_encoding(s.to_owned()));
|
||||
meta.album = tag.album().map(|s| fix_encoding(s.to_owned()));
|
||||
meta.year = tag.year().and_then(|y| u32::try_from(y).ok());
|
||||
meta.track_number = tag.track();
|
||||
meta.genre = tag.genre().map(|s: &str| fix_encoding(s.to_owned()));
|
||||
|
||||
Ok(meta)
|
||||
}
|
||||
|
||||
fn extract_tags(tags: &[symphonia::core::meta::Tag], meta: &mut RawMetadata) {
|
||||
for tag in tags {
|
||||
let value = fix_encoding(tag.value.to_string());
|
||||
if let Some(key) = tag.std_key {
|
||||
match key {
|
||||
StandardTagKey::TrackTitle => {
|
||||
if meta.title.is_none() {
|
||||
meta.title = Some(value);
|
||||
}
|
||||
}
|
||||
StandardTagKey::Artist | StandardTagKey::Performer => {
|
||||
if meta.artist.is_none() {
|
||||
meta.artist = Some(value);
|
||||
}
|
||||
}
|
||||
StandardTagKey::Album => {
|
||||
if meta.album.is_none() {
|
||||
meta.album = Some(value);
|
||||
}
|
||||
}
|
||||
StandardTagKey::TrackNumber => {
|
||||
if meta.track_number.is_none() {
|
||||
meta.track_number = value.parse().ok();
|
||||
}
|
||||
}
|
||||
StandardTagKey::Date | StandardTagKey::OriginalDate => {
|
||||
if meta.year.is_none() {
|
||||
meta.year = value[..4.min(value.len())].parse().ok();
|
||||
}
|
||||
}
|
||||
StandardTagKey::Genre => {
|
||||
if meta.genre.is_none() {
|
||||
meta.genre = Some(value);
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Heuristic to fix mojibake (CP1251 bytes interpreted as Latin-1/Windows-1252).
|
||||
fn fix_encoding(s: String) -> String {
|
||||
let bytes: Vec<u8> = s
|
||||
.chars()
|
||||
.map(|c| c as u32)
|
||||
.filter(|&c| c <= 255)
|
||||
.map(|c| c as u8)
|
||||
.collect();
|
||||
|
||||
if bytes.len() != s.chars().count() {
|
||||
return s;
|
||||
}
|
||||
|
||||
let has_mojibake = bytes.iter().any(|&b| b >= 0xC0);
|
||||
if !has_mojibake {
|
||||
return s;
|
||||
}
|
||||
|
||||
let (decoded, _, errors) = encoding_rs::WINDOWS_1251.decode(&bytes);
|
||||
if errors {
|
||||
return s;
|
||||
}
|
||||
|
||||
decoded.into_owned()
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
pub mod cover_art;
|
||||
pub mod dto;
|
||||
pub mod metadata;
|
||||
pub mod mover;
|
||||
pub mod normalize;
|
||||
pub mod path_hints;
|
||||
pub mod rag;
|
||||
|
||||
use serde::Deserialize;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// LLM health probe — called from the admin settings page
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Result of probing the LLM API.
|
||||
#[derive(Debug, Default)]
|
||||
pub struct AgentProbeResult {
|
||||
pub ok: bool,
|
||||
pub model_intro: String,
|
||||
pub model_name: String,
|
||||
pub prompt_tokens: Option<u32>,
|
||||
pub completion_tokens: Option<u32>,
|
||||
pub tokens_per_sec: Option<f64>,
|
||||
pub latency_ms: u64,
|
||||
pub error: String,
|
||||
}
|
||||
|
||||
/// Send a lightweight "introduce yourself" prompt to the LLM and return the
|
||||
/// response together with timing / usage statistics when available.
|
||||
pub async fn probe_llm(
|
||||
llm_url: &str,
|
||||
llm_model: &str,
|
||||
llm_auth: &str,
|
||||
) -> AgentProbeResult {
|
||||
let start = std::time::Instant::now();
|
||||
|
||||
let client = match reqwest::Client::builder()
|
||||
.timeout(std::time::Duration::from_secs(30))
|
||||
.build()
|
||||
{
|
||||
Ok(c) => c,
|
||||
Err(e) => {
|
||||
return AgentProbeResult {
|
||||
error: format!("failed to create HTTP client: {e}"),
|
||||
..Default::default()
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
let body = serde_json::json!({
|
||||
"model": llm_model,
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Introduce yourself briefly: what model are you, who made you? Reply in 1–2 sentences."
|
||||
}
|
||||
],
|
||||
"stream": false,
|
||||
"temperature": 0.3,
|
||||
"max_tokens": 256
|
||||
});
|
||||
|
||||
let url = format!("{}/v1/chat/completions", llm_url.trim_end_matches('/'));
|
||||
let mut req = client.post(&url).json(&body);
|
||||
if !llm_auth.is_empty() {
|
||||
req = req.header("Authorization", llm_auth);
|
||||
}
|
||||
|
||||
let resp = match req.send().await {
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
return AgentProbeResult {
|
||||
latency_ms: start.elapsed().as_millis() as u64,
|
||||
error: format!("connection failed: {e}"),
|
||||
..Default::default()
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
let elapsed = start.elapsed();
|
||||
let latency_ms = elapsed.as_millis() as u64;
|
||||
|
||||
if !resp.status().is_success() {
|
||||
let status = resp.status();
|
||||
let body_text = resp.text().await.unwrap_or_default();
|
||||
return AgentProbeResult {
|
||||
latency_ms,
|
||||
error: format!("HTTP {status}: {}", &body_text[..body_text.len().min(300)]),
|
||||
..Default::default()
|
||||
};
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct ProbeResponse {
|
||||
choices: Option<Vec<ProbeChoice>>,
|
||||
model: Option<String>,
|
||||
usage: Option<ProbeUsage>,
|
||||
}
|
||||
#[derive(Deserialize)]
|
||||
struct ProbeChoice {
|
||||
message: Option<ProbeMessage>,
|
||||
}
|
||||
#[derive(Deserialize)]
|
||||
struct ProbeMessage {
|
||||
content: Option<String>,
|
||||
}
|
||||
#[derive(Deserialize)]
|
||||
struct ProbeUsage {
|
||||
prompt_tokens: Option<u32>,
|
||||
completion_tokens: Option<u32>,
|
||||
}
|
||||
|
||||
let raw: ProbeResponse = match resp.json().await {
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
return AgentProbeResult {
|
||||
latency_ms,
|
||||
error: format!("failed to parse response: {e}"),
|
||||
..Default::default()
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
let model_intro = raw
|
||||
.choices
|
||||
.as_ref()
|
||||
.and_then(|c| c.first())
|
||||
.and_then(|c| c.message.as_ref())
|
||||
.and_then(|m| m.content.clone())
|
||||
.unwrap_or_default();
|
||||
|
||||
let model_name = raw.model.unwrap_or_default();
|
||||
|
||||
let prompt_tokens = raw.usage.as_ref().and_then(|u| u.prompt_tokens);
|
||||
let completion_tokens = raw.usage.as_ref().and_then(|u| u.completion_tokens);
|
||||
|
||||
// Compute tokens/sec from completion tokens and wall time
|
||||
let tokens_per_sec = completion_tokens.map(|ct| {
|
||||
if elapsed.as_secs_f64() > 0.0 {
|
||||
ct as f64 / elapsed.as_secs_f64()
|
||||
} else {
|
||||
0.0
|
||||
}
|
||||
});
|
||||
|
||||
AgentProbeResult {
|
||||
ok: true,
|
||||
model_intro,
|
||||
model_name,
|
||||
prompt_tokens,
|
||||
completion_tokens,
|
||||
tokens_per_sec,
|
||||
latency_ms,
|
||||
error: String::new(),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
pub enum MoveOutcome {
|
||||
/// File was moved/renamed to destination.
|
||||
Moved(PathBuf),
|
||||
/// Destination already existed; inbox duplicate was removed.
|
||||
Merged(PathBuf),
|
||||
}
|
||||
|
||||
/// Move a file from inbox to the permanent storage directory.
|
||||
///
|
||||
/// Creates the directory structure: `storage_dir/artist/album/filename`
|
||||
///
|
||||
/// If `rename` fails (cross-device), falls back to copy + remove.
|
||||
/// If the destination already exists the inbox copy is removed and
|
||||
/// `MoveOutcome::Merged` is returned.
|
||||
pub async fn move_to_storage(
|
||||
storage_dir: &Path,
|
||||
artist: &str,
|
||||
album: &str,
|
||||
filename: &str,
|
||||
source: &Path,
|
||||
) -> anyhow::Result<MoveOutcome> {
|
||||
let artist_dir = sanitize_dir_name(artist);
|
||||
let album_dir = sanitize_dir_name(album);
|
||||
|
||||
let dest_dir = storage_dir.join(&artist_dir).join(&album_dir);
|
||||
tokio::fs::create_dir_all(&dest_dir).await?;
|
||||
|
||||
let dest = dest_dir.join(filename);
|
||||
|
||||
// File already at destination — remove the inbox duplicate
|
||||
if dest.exists() {
|
||||
if source.exists() {
|
||||
tokio::fs::remove_file(source).await?;
|
||||
tracing::info!(from = ?source, to = ?dest, "merged duplicate into existing storage file");
|
||||
}
|
||||
return Ok(MoveOutcome::Merged(dest));
|
||||
}
|
||||
|
||||
// Try atomic rename first (same filesystem)
|
||||
match tokio::fs::rename(source, &dest).await {
|
||||
Ok(()) => {}
|
||||
Err(_) => {
|
||||
// Cross-device: copy then remove
|
||||
tokio::fs::copy(source, &dest).await?;
|
||||
tokio::fs::remove_file(source).await?;
|
||||
}
|
||||
}
|
||||
|
||||
tracing::info!(from = ?source, to = ?dest, "moved file to storage");
|
||||
Ok(MoveOutcome::Moved(dest))
|
||||
}
|
||||
|
||||
/// Remove characters that are unsafe for directory names.
|
||||
fn sanitize_dir_name(name: &str) -> String {
|
||||
name.chars()
|
||||
.map(|c| match c {
|
||||
'/' | '\\' | ':' | '*' | '?' | '"' | '<' | '>' | '|' | '\0' => '_',
|
||||
_ => c,
|
||||
})
|
||||
.collect::<String>()
|
||||
.trim()
|
||||
.trim_matches('.')
|
||||
.to_owned()
|
||||
}
|
||||
@@ -0,0 +1,483 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::dto::{FolderContext, NormalizedFields, PathHints, RawMetadata, SimilarArtist, SimilarRelease};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Types
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// A single message in the chat history.
|
||||
#[derive(Clone, Serialize)]
|
||||
pub struct ChatMessage {
|
||||
pub role: String,
|
||||
pub content: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct ChatRequest {
|
||||
model: String,
|
||||
messages: Vec<ChatMessage>,
|
||||
response_format: ChatResponseFormat,
|
||||
stream: bool,
|
||||
temperature: f64,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct ChatResponseFormat {
|
||||
#[serde(rename = "type")]
|
||||
kind: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct ChatResponse {
|
||||
model: Option<String>,
|
||||
choices: Vec<ChatChoice>,
|
||||
usage: Option<ChatUsage>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct ChatChoice {
|
||||
message: ChatResponseMessage,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct ChatResponseMessage {
|
||||
content: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Default)]
|
||||
struct ChatUsage {
|
||||
prompt_tokens: Option<u32>,
|
||||
completion_tokens: Option<u32>,
|
||||
}
|
||||
|
||||
async fn call_llm_chat(
|
||||
base_url: &str,
|
||||
model: &str,
|
||||
messages: &[ChatMessage],
|
||||
auth: Option<&str>,
|
||||
) -> anyhow::Result<(String, String, ChatUsage)> {
|
||||
let client = reqwest::Client::builder()
|
||||
.timeout(std::time::Duration::from_secs(600))
|
||||
.build()?;
|
||||
|
||||
let request = ChatRequest {
|
||||
model: model.to_owned(),
|
||||
messages: messages.to_vec(),
|
||||
response_format: ChatResponseFormat {
|
||||
kind: "json_object".to_owned(),
|
||||
},
|
||||
stream: false,
|
||||
temperature: 0.1,
|
||||
};
|
||||
|
||||
let url = format!("{}/v1/chat/completions", base_url.trim_end_matches('/'));
|
||||
tracing::info!(
|
||||
%url,
|
||||
model,
|
||||
message_count = messages.len(),
|
||||
"Calling LLM API (chat mode)..."
|
||||
);
|
||||
|
||||
let start = std::time::Instant::now();
|
||||
let mut req = client.post(&url).json(&request);
|
||||
if let Some(auth_header) = auth {
|
||||
req = req.header("Authorization", auth_header);
|
||||
}
|
||||
let resp = req.send().await?;
|
||||
let elapsed = start.elapsed();
|
||||
|
||||
if !resp.status().is_success() {
|
||||
let status = resp.status();
|
||||
let body = resp.text().await.unwrap_or_default();
|
||||
tracing::error!(%status, body = &body[..body.len().min(500)], "LLM API error");
|
||||
anyhow::bail!("LLM returned {}: {}", status, body);
|
||||
}
|
||||
|
||||
let chat_resp: ChatResponse = resp.json().await?;
|
||||
let resp_model = chat_resp.model.unwrap_or_else(|| model.to_owned());
|
||||
let usage = chat_resp.usage.unwrap_or_default();
|
||||
let content = chat_resp
|
||||
.choices
|
||||
.into_iter()
|
||||
.next()
|
||||
.ok_or_else(|| anyhow::anyhow!("LLM returned empty choices"))?
|
||||
.message
|
||||
.content;
|
||||
|
||||
tracing::info!(
|
||||
elapsed_ms = elapsed.as_millis() as u64,
|
||||
response_len = content.len(),
|
||||
prompt_tokens = usage.prompt_tokens.unwrap_or(0),
|
||||
completion_tokens = usage.completion_tokens.unwrap_or(0),
|
||||
model = %resp_model,
|
||||
"LLM response received"
|
||||
);
|
||||
tracing::debug!(raw_response = %content, "LLM raw output");
|
||||
|
||||
Ok((content, resp_model, usage))
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Batch normalize — process multiple files in one LLM call
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Input for one file in a batch normalize call.
|
||||
pub struct BatchFileInput {
|
||||
pub filename: String,
|
||||
pub raw: RawMetadata,
|
||||
pub hints: PathHints,
|
||||
}
|
||||
|
||||
/// Result of a batch normalize call.
|
||||
pub struct BatchNormalizeResult {
|
||||
/// (filename, normalized_fields) pairs.
|
||||
pub results: Vec<(String, NormalizedFields)>,
|
||||
pub model: String,
|
||||
pub prompt_tokens: u64,
|
||||
pub completion_tokens: u64,
|
||||
pub duration_ms: u64,
|
||||
}
|
||||
|
||||
/// Estimate the token count for a batch of files.
|
||||
/// Uses the rough heuristic of 1 token per 4 characters.
|
||||
fn estimate_batch_tokens(
|
||||
system_prompt: &str,
|
||||
files: &[BatchFileInput],
|
||||
similar_artists: &[SimilarArtist],
|
||||
similar_releases: &[SimilarRelease],
|
||||
folder_ctx: Option<&FolderContext>,
|
||||
) -> u64 {
|
||||
let system_tokens = system_prompt.len() as u64 / 4;
|
||||
|
||||
// Shared context (RAG + folder) — sent once
|
||||
let mut shared_chars: u64 = 0;
|
||||
for a in similar_artists {
|
||||
shared_chars += 40 + a.name.len() as u64;
|
||||
}
|
||||
for r in similar_releases {
|
||||
shared_chars += 50 + r.title.len() as u64;
|
||||
}
|
||||
if let Some(ctx) = folder_ctx {
|
||||
shared_chars += 60 + ctx.folder_path.len() as u64;
|
||||
for f in &ctx.folder_files {
|
||||
shared_chars += 4 + f.len() as u64;
|
||||
}
|
||||
}
|
||||
let shared_tokens = shared_chars / 4;
|
||||
|
||||
// Per-file: metadata input + expected response
|
||||
let mut per_file_tokens: u64 = 0;
|
||||
for f in files {
|
||||
let mut chars: u64 = 40 + f.filename.len() as u64; // header
|
||||
if let Some(v) = &f.raw.title { chars += 10 + v.len() as u64; }
|
||||
if let Some(v) = &f.raw.artist { chars += 12 + v.len() as u64; }
|
||||
if let Some(v) = &f.raw.album { chars += 12 + v.len() as u64; }
|
||||
if f.raw.year.is_some() { chars += 12; }
|
||||
if f.raw.track_number.is_some() { chars += 18; }
|
||||
if let Some(v) = &f.raw.genre { chars += 10 + v.len() as u64; }
|
||||
// hints
|
||||
if let Some(v) = &f.hints.artist { chars += 16 + v.len() as u64; }
|
||||
if let Some(v) = &f.hints.album { chars += 16 + v.len() as u64; }
|
||||
if let Some(v) = &f.hints.title { chars += 15 + v.len() as u64; }
|
||||
if f.hints.year.is_some() { chars += 14; }
|
||||
if f.hints.track_number.is_some() { chars += 20; }
|
||||
per_file_tokens += chars / 4;
|
||||
// Expected response per file (~150 tokens)
|
||||
per_file_tokens += 150;
|
||||
}
|
||||
|
||||
system_tokens + shared_tokens + per_file_tokens
|
||||
}
|
||||
|
||||
/// Build the user message for a batch of files.
|
||||
fn build_batch_user_message(
|
||||
files: &[BatchFileInput],
|
||||
similar_artists: &[SimilarArtist],
|
||||
similar_releases: &[SimilarRelease],
|
||||
folder_ctx: Option<&FolderContext>,
|
||||
) -> String {
|
||||
let mut msg = String::with_capacity(4096);
|
||||
|
||||
// Shared context first
|
||||
if let Some(ctx) = folder_ctx {
|
||||
msg.push_str("## Folder context\n");
|
||||
msg.push_str(&format!("Folder path: \"{}\"\n", ctx.folder_path));
|
||||
msg.push_str(&format!("Total files in folder: {}\n\n", ctx.track_count));
|
||||
}
|
||||
|
||||
if !similar_artists.is_empty() {
|
||||
msg.push_str("## Existing artists in database\n");
|
||||
for a in similar_artists {
|
||||
msg.push_str(&format!("- \"{}\" (similarity: {:.2})\n", a.name, a.similarity));
|
||||
}
|
||||
msg.push('\n');
|
||||
}
|
||||
|
||||
if !similar_releases.is_empty() {
|
||||
msg.push_str("## Existing releases in database\n");
|
||||
for r in similar_releases {
|
||||
let year_str = r.year.map(|y| format!(", year: {y}")).unwrap_or_default();
|
||||
msg.push_str(&format!("- \"{}\" (similarity: {:.2}{})\n", r.title, r.similarity, year_str));
|
||||
}
|
||||
msg.push('\n');
|
||||
}
|
||||
|
||||
// Per-file metadata
|
||||
msg.push_str(&format!("## Files to process ({})\n\n", files.len()));
|
||||
|
||||
for f in files {
|
||||
msg.push_str(&format!("### {}\n", f.filename));
|
||||
|
||||
if let Some(v) = &f.raw.title { msg.push_str(&format!("Title: \"{v}\"\n")); }
|
||||
if let Some(v) = &f.raw.artist { msg.push_str(&format!("Artist: \"{v}\"\n")); }
|
||||
if let Some(v) = &f.raw.album { msg.push_str(&format!("Release: \"{v}\"\n")); }
|
||||
if let Some(v) = f.raw.year { msg.push_str(&format!("Year: {v}\n")); }
|
||||
if let Some(v) = f.raw.track_number { msg.push_str(&format!("Track: {v}\n")); }
|
||||
if let Some(v) = &f.raw.genre { msg.push_str(&format!("Genre: \"{v}\"\n")); }
|
||||
|
||||
// Path hints (only if different from tag metadata)
|
||||
let has_hints = f.hints.artist.is_some()
|
||||
|| f.hints.album.is_some()
|
||||
|| f.hints.title.is_some()
|
||||
|| f.hints.year.is_some()
|
||||
|| f.hints.track_number.is_some();
|
||||
if has_hints {
|
||||
if let Some(v) = &f.hints.artist { msg.push_str(&format!("Path artist: \"{v}\"\n")); }
|
||||
if let Some(v) = &f.hints.album { msg.push_str(&format!("Path release: \"{v}\"\n")); }
|
||||
if let Some(v) = &f.hints.title { msg.push_str(&format!("Path title: \"{v}\"\n")); }
|
||||
if let Some(v) = f.hints.year { msg.push_str(&format!("Path year: {v}\n")); }
|
||||
if let Some(v) = f.hints.track_number { msg.push_str(&format!("Path track: {v}\n")); }
|
||||
}
|
||||
msg.push('\n');
|
||||
}
|
||||
|
||||
msg
|
||||
}
|
||||
|
||||
/// Normalize a batch of files in one LLM call.
|
||||
/// If the batch is too large for the context window, it is automatically
|
||||
/// split in half and each half is processed recursively.
|
||||
pub async fn normalize_batch(
|
||||
llm_url: &str,
|
||||
llm_model: &str,
|
||||
llm_auth: &str,
|
||||
system_prompt: &str,
|
||||
context_limit: u64,
|
||||
files: Vec<BatchFileInput>,
|
||||
similar_artists: &[SimilarArtist],
|
||||
similar_releases: &[SimilarRelease],
|
||||
folder_ctx: Option<&FolderContext>,
|
||||
) -> anyhow::Result<BatchNormalizeResult> {
|
||||
// Estimate tokens
|
||||
let estimated = estimate_batch_tokens(
|
||||
system_prompt, &files, similar_artists, similar_releases, folder_ctx,
|
||||
);
|
||||
|
||||
// If over 80% of context limit and more than 1 file, split
|
||||
let limit_80 = context_limit * 80 / 100;
|
||||
if estimated > limit_80 && files.len() > 1 {
|
||||
tracing::info!(
|
||||
estimated_tokens = estimated,
|
||||
context_limit,
|
||||
file_count = files.len(),
|
||||
"Batch too large, splitting in half"
|
||||
);
|
||||
let mid = files.len() / 2;
|
||||
let mut files_vec = files;
|
||||
let right = files_vec.split_off(mid);
|
||||
let left = files_vec;
|
||||
|
||||
let left_result = Box::pin(normalize_batch(
|
||||
llm_url, llm_model, llm_auth, system_prompt, context_limit,
|
||||
left, similar_artists, similar_releases, folder_ctx,
|
||||
)).await?;
|
||||
|
||||
let right_result = Box::pin(normalize_batch(
|
||||
llm_url, llm_model, llm_auth, system_prompt, context_limit,
|
||||
right, similar_artists, similar_releases, folder_ctx,
|
||||
)).await?;
|
||||
|
||||
// Merge results
|
||||
let mut results = left_result.results;
|
||||
results.extend(right_result.results);
|
||||
return Ok(BatchNormalizeResult {
|
||||
results,
|
||||
model: left_result.model,
|
||||
prompt_tokens: left_result.prompt_tokens + right_result.prompt_tokens,
|
||||
completion_tokens: left_result.completion_tokens + right_result.completion_tokens,
|
||||
duration_ms: left_result.duration_ms + right_result.duration_ms,
|
||||
});
|
||||
}
|
||||
|
||||
// Build and send
|
||||
let user_message = build_batch_user_message(
|
||||
&files, similar_artists, similar_releases, folder_ctx,
|
||||
);
|
||||
|
||||
let messages = vec![
|
||||
ChatMessage { role: "system".into(), content: system_prompt.to_owned() },
|
||||
ChatMessage { role: "user".into(), content: user_message },
|
||||
];
|
||||
|
||||
let start = std::time::Instant::now();
|
||||
let call_result = call_llm_chat(
|
||||
llm_url, llm_model, &messages,
|
||||
if llm_auth.is_empty() { None } else { Some(llm_auth) },
|
||||
).await;
|
||||
let duration_ms = start.elapsed().as_millis() as u64;
|
||||
|
||||
// If LLM error and batch > 1, try splitting (handles context overflow errors)
|
||||
let (response_text, resp_model, usage) = match call_result {
|
||||
Ok(r) => r,
|
||||
Err(e) if files.len() > 1 => {
|
||||
let err_str = e.to_string().to_lowercase();
|
||||
let is_context_error = err_str.contains("context")
|
||||
|| err_str.contains("too long")
|
||||
|| err_str.contains("maximum")
|
||||
|| err_str.contains("length")
|
||||
|| err_str.contains("token");
|
||||
if is_context_error {
|
||||
tracing::warn!(
|
||||
file_count = files.len(),
|
||||
"LLM error suggests context overflow, splitting batch: {e}"
|
||||
);
|
||||
let mid = files.len() / 2;
|
||||
let mut files_vec = files;
|
||||
let right = files_vec.split_off(mid);
|
||||
let left = files_vec;
|
||||
|
||||
let left_result = Box::pin(normalize_batch(
|
||||
llm_url, llm_model, llm_auth, system_prompt, context_limit,
|
||||
left, similar_artists, similar_releases, folder_ctx,
|
||||
)).await?;
|
||||
let right_result = Box::pin(normalize_batch(
|
||||
llm_url, llm_model, llm_auth, system_prompt, context_limit,
|
||||
right, similar_artists, similar_releases, folder_ctx,
|
||||
)).await?;
|
||||
|
||||
let mut results = left_result.results;
|
||||
results.extend(right_result.results);
|
||||
return Ok(BatchNormalizeResult {
|
||||
results,
|
||||
model: left_result.model,
|
||||
prompt_tokens: left_result.prompt_tokens + right_result.prompt_tokens,
|
||||
completion_tokens: left_result.completion_tokens + right_result.completion_tokens,
|
||||
duration_ms: left_result.duration_ms + right_result.duration_ms,
|
||||
});
|
||||
}
|
||||
return Err(e);
|
||||
}
|
||||
Err(e) => return Err(e),
|
||||
};
|
||||
|
||||
let prompt_tokens = usage.prompt_tokens.unwrap_or(0) as u64;
|
||||
let completion_tokens = usage.completion_tokens.unwrap_or(0) as u64;
|
||||
|
||||
// Parse batch response
|
||||
let results = parse_batch_response(&response_text, &files)?;
|
||||
|
||||
Ok(BatchNormalizeResult {
|
||||
results,
|
||||
model: resp_model,
|
||||
prompt_tokens,
|
||||
completion_tokens,
|
||||
duration_ms,
|
||||
})
|
||||
}
|
||||
|
||||
/// Parse a batch JSON array response from the LLM.
|
||||
/// Returns (filename, NormalizedFields) pairs.
|
||||
/// Handles: clean JSON array, markdown-fenced JSON, and wrapped `{"results": [...]}`.
|
||||
fn parse_batch_response(
|
||||
response: &str,
|
||||
files: &[BatchFileInput],
|
||||
) -> anyhow::Result<Vec<(String, NormalizedFields)>> {
|
||||
let cleaned = response.trim();
|
||||
|
||||
// Strip markdown code fences if present
|
||||
let json_str = if cleaned.starts_with("```") {
|
||||
let start = cleaned.find('[')
|
||||
.or_else(|| cleaned.find('{'))
|
||||
.unwrap_or(0);
|
||||
let end_bracket = cleaned.rfind(']').map(|i| i + 1);
|
||||
let end_brace = cleaned.rfind('}').map(|i| i + 1);
|
||||
let end = end_bracket.or(end_brace).unwrap_or(cleaned.len());
|
||||
&cleaned[start..end]
|
||||
} else {
|
||||
cleaned
|
||||
};
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct BatchLlmOutput {
|
||||
filename: Option<String>,
|
||||
artist: Option<String>,
|
||||
album: Option<String>,
|
||||
title: Option<String>,
|
||||
year: Option<i32>,
|
||||
track_number: Option<i32>,
|
||||
genre: Option<String>,
|
||||
#[serde(default)]
|
||||
featured_artists: Vec<String>,
|
||||
release_type: Option<String>,
|
||||
confidence: Option<f64>,
|
||||
notes: Option<String>,
|
||||
}
|
||||
|
||||
// Try parsing as array first, then as {"results": [...]} wrapper
|
||||
let items: Vec<BatchLlmOutput> = if json_str.starts_with('[') {
|
||||
serde_json::from_str(json_str)
|
||||
} else {
|
||||
// Try as wrapper object with a "results" or "files" key
|
||||
#[derive(Deserialize)]
|
||||
struct Wrapper {
|
||||
#[serde(alias = "files")]
|
||||
results: Vec<BatchLlmOutput>,
|
||||
}
|
||||
serde_json::from_str::<Wrapper>(json_str).map(|w| w.results)
|
||||
}
|
||||
.map_err(|e| {
|
||||
anyhow::anyhow!(
|
||||
"Failed to parse batch LLM response: {} — raw: {}",
|
||||
e,
|
||||
&response[..response.len().min(500)]
|
||||
)
|
||||
})?;
|
||||
|
||||
// Build a map of filename → NormalizedFields
|
||||
let mut results = Vec::with_capacity(files.len());
|
||||
let mut matched = std::collections::HashSet::new();
|
||||
|
||||
for item in &items {
|
||||
let filename = match &item.filename {
|
||||
Some(f) => f.clone(),
|
||||
None => continue,
|
||||
};
|
||||
let fields = NormalizedFields {
|
||||
title: item.title.clone(),
|
||||
artist: item.artist.clone(),
|
||||
album: item.album.clone(),
|
||||
year: item.year,
|
||||
track_number: item.track_number,
|
||||
genre: item.genre.clone(),
|
||||
featured_artists: item.featured_artists.clone(),
|
||||
release_type: item.release_type.clone(),
|
||||
confidence: item.confidence,
|
||||
notes: item.notes.clone(),
|
||||
};
|
||||
matched.insert(filename.clone());
|
||||
results.push((filename, fields));
|
||||
}
|
||||
|
||||
// Warn about files the LLM missed
|
||||
for f in files {
|
||||
if !matched.contains(&f.filename) {
|
||||
tracing::warn!(
|
||||
filename = %f.filename,
|
||||
"LLM batch response missing result for file"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(results)
|
||||
}
|
||||
@@ -0,0 +1,197 @@
|
||||
use std::path::Path;
|
||||
|
||||
use super::dto::PathHints;
|
||||
|
||||
/// Parse metadata hints from the file path relative to the inbox directory.
|
||||
///
|
||||
/// Recognized patterns:
|
||||
/// Artist/Album/01 - Title.ext
|
||||
/// Artist/Album (Year)/01 - Title.ext
|
||||
/// Artist/(Year) Album/01 - Title.ext
|
||||
/// Artist/Album [Year]/01 - Title.ext
|
||||
/// 01 - Title.ext (flat, no artist/album)
|
||||
pub fn parse(relative_path: &Path) -> PathHints {
|
||||
let components: Vec<&str> = relative_path
|
||||
.components()
|
||||
.filter_map(|c| c.as_os_str().to_str())
|
||||
.collect();
|
||||
|
||||
let mut hints = PathHints::default();
|
||||
|
||||
let filename = components.last().copied().unwrap_or("");
|
||||
let stem = Path::new(filename)
|
||||
.file_stem()
|
||||
.and_then(|s| s.to_str())
|
||||
.unwrap_or("");
|
||||
|
||||
// Parse track number and title from filename
|
||||
parse_filename(stem, &mut hints);
|
||||
|
||||
match components.len() {
|
||||
// Artist/Album/file.ext
|
||||
3.. => {
|
||||
hints.artist = Some(components[0].to_owned());
|
||||
|
||||
let album_raw = components[1];
|
||||
let (album, year) = parse_album_with_year(album_raw);
|
||||
hints.album = Some(album);
|
||||
if year.is_some() {
|
||||
hints.year = year;
|
||||
}
|
||||
}
|
||||
// Album/file.ext (or Artist/file.ext — ambiguous, treat as album)
|
||||
2 => {
|
||||
let dir = components[0];
|
||||
let (name, year) = parse_album_with_year(dir);
|
||||
hints.album = Some(name);
|
||||
if year.is_some() {
|
||||
hints.year = year;
|
||||
}
|
||||
}
|
||||
// Just file.ext
|
||||
_ => {}
|
||||
}
|
||||
|
||||
hints
|
||||
}
|
||||
|
||||
/// Try to extract track number and title from a filename stem.
|
||||
///
|
||||
/// Patterns: "01 - Title", "01. Title", "1 Title", "Title"
|
||||
fn parse_filename(stem: &str, hints: &mut PathHints) {
|
||||
let trimmed = stem.trim();
|
||||
|
||||
// Try "NN - Title" or "NN. Title"
|
||||
if let Some(rest) = try_strip_track_prefix(trimmed) {
|
||||
let (num_str, title) = rest;
|
||||
if let Ok(num) = num_str.parse::<i32>() {
|
||||
hints.track_number = Some(num);
|
||||
if !title.is_empty() {
|
||||
hints.title = Some(title.to_owned());
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// No track number found, use full stem as title
|
||||
if !trimmed.is_empty() {
|
||||
hints.title = Some(trimmed.to_owned());
|
||||
}
|
||||
}
|
||||
|
||||
/// Try to parse "NN - Rest" or "NN. Rest" from a string.
|
||||
/// Returns (number_str, rest) if successful.
|
||||
fn try_strip_track_prefix(s: &str) -> Option<(&str, &str)> {
|
||||
let digit_end = s.find(|c: char| !c.is_ascii_digit())?;
|
||||
if digit_end == 0 {
|
||||
return None;
|
||||
}
|
||||
let num_str = &s[..digit_end];
|
||||
let rest = s[digit_end..].trim_start();
|
||||
|
||||
let title = if let Some(stripped) = rest.strip_prefix("- ") {
|
||||
stripped.trim()
|
||||
} else if let Some(stripped) = rest.strip_prefix(". ") {
|
||||
stripped.trim()
|
||||
} else if let Some(stripped) = rest.strip_prefix('.') {
|
||||
stripped.trim()
|
||||
} else {
|
||||
rest
|
||||
};
|
||||
|
||||
Some((num_str, title))
|
||||
}
|
||||
|
||||
/// Extract album name and optional year from directory name.
|
||||
///
|
||||
/// Patterns: "Album (2001)", "(2001) Album", "Album [2001]", "Album"
|
||||
fn parse_album_with_year(dir: &str) -> (String, Option<i32>) {
|
||||
// Try "Album (YYYY)" or "Album [YYYY]"
|
||||
for (open, close) in [('(', ')'), ('[', ']')] {
|
||||
if let Some(start) = dir.rfind(open) {
|
||||
if let Some(end) = dir[start..].find(close) {
|
||||
let inside = &dir[start + 1..start + end];
|
||||
if let Ok(year) = inside.trim().parse::<i32>() {
|
||||
if (1900..=2100).contains(&year) {
|
||||
let album = format!(
|
||||
"{}{}",
|
||||
&dir[..start].trim(),
|
||||
&dir[start + end + 1..].trim()
|
||||
);
|
||||
let album = album.trim().to_owned();
|
||||
return (album, Some(year));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Try "(YYYY) Album"
|
||||
if dir.starts_with('(') {
|
||||
if let Some(end) = dir.find(')') {
|
||||
let inside = &dir[1..end];
|
||||
if let Ok(year) = inside.trim().parse::<i32>() {
|
||||
if (1900..=2100).contains(&year) {
|
||||
let album = dir[end + 1..].trim().to_owned();
|
||||
return (album, Some(year));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
(dir.to_owned(), None)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::path::PathBuf;
|
||||
|
||||
#[test]
|
||||
fn test_artist_album_track() {
|
||||
let p = PathBuf::from("Pink Floyd/Wish You Were Here (1975)/03 - Have a Cigar.flac");
|
||||
let h = parse(&p);
|
||||
assert_eq!(h.artist.as_deref(), Some("Pink Floyd"));
|
||||
assert_eq!(h.album.as_deref(), Some("Wish You Were Here"));
|
||||
assert_eq!(h.year, Some(1975));
|
||||
assert_eq!(h.track_number, Some(3));
|
||||
assert_eq!(h.title.as_deref(), Some("Have a Cigar"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_year_prefix() {
|
||||
let p = PathBuf::from("Artist/(2020) Album Name/01. Song.flac");
|
||||
let h = parse(&p);
|
||||
assert_eq!(h.artist.as_deref(), Some("Artist"));
|
||||
assert_eq!(h.album.as_deref(), Some("Album Name"));
|
||||
assert_eq!(h.year, Some(2020));
|
||||
assert_eq!(h.track_number, Some(1));
|
||||
assert_eq!(h.title.as_deref(), Some("Song"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_flat_file() {
|
||||
let p = PathBuf::from("05 - Something.mp3");
|
||||
let h = parse(&p);
|
||||
assert_eq!(h.artist, None);
|
||||
assert_eq!(h.album, None);
|
||||
assert_eq!(h.track_number, Some(5));
|
||||
assert_eq!(h.title.as_deref(), Some("Something"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_no_track_number() {
|
||||
let p = PathBuf::from("Artist/Album/Song Name.flac");
|
||||
let h = parse(&p);
|
||||
assert_eq!(h.track_number, None);
|
||||
assert_eq!(h.title.as_deref(), Some("Song Name"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_square_bracket_year() {
|
||||
let p = PathBuf::from("Band/Album [1999]/track.flac");
|
||||
let h = parse(&p);
|
||||
assert_eq!(h.album.as_deref(), Some("Album"));
|
||||
assert_eq!(h.year, Some(1999));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
use sqlx::PgPool;
|
||||
|
||||
use super::dto::{SimilarArtist, SimilarRelease};
|
||||
|
||||
/// Find artists with similar names using pg_trgm.
|
||||
/// Short names (<3 chars) fall back to ILIKE prefix match.
|
||||
pub async fn find_similar_artists(
|
||||
pool: &PgPool,
|
||||
name: &str,
|
||||
limit: i32,
|
||||
) -> anyhow::Result<Vec<SimilarArtist>> {
|
||||
if name.chars().count() < 3 {
|
||||
let rows: Vec<(i64, String, f32)> = sqlx::query_as(
|
||||
"SELECT id, name, 1.0::real AS similarity FROM furumusic__artist \
|
||||
WHERE name_sort ILIKE $1 || '%' ORDER BY name LIMIT $2",
|
||||
)
|
||||
.bind(name.to_lowercase())
|
||||
.bind(limit)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
Ok(rows
|
||||
.into_iter()
|
||||
.map(|(id, name, similarity)| SimilarArtist {
|
||||
id,
|
||||
name,
|
||||
similarity,
|
||||
})
|
||||
.collect())
|
||||
} else {
|
||||
let rows: Vec<(i64, String, f32)> = sqlx::query_as(
|
||||
r#"SELECT id, name, MAX(sim) AS similarity FROM (
|
||||
SELECT id, name, similarity(name_sort, $1) AS sim
|
||||
FROM furumusic__artist WHERE name_sort % $1
|
||||
UNION ALL
|
||||
SELECT id, name, 0.01::real AS sim
|
||||
FROM furumusic__artist WHERE name_sort ILIKE '%' || $1 || '%'
|
||||
) sub GROUP BY id, name ORDER BY similarity DESC LIMIT $2"#,
|
||||
)
|
||||
.bind(name.to_lowercase())
|
||||
.bind(limit)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
Ok(rows
|
||||
.into_iter()
|
||||
.map(|(id, name, similarity)| SimilarArtist {
|
||||
id,
|
||||
name,
|
||||
similarity,
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
}
|
||||
|
||||
/// Find releases with similar titles using pg_trgm.
|
||||
pub async fn find_similar_releases(
|
||||
pool: &PgPool,
|
||||
title: &str,
|
||||
limit: i32,
|
||||
) -> anyhow::Result<Vec<SimilarRelease>> {
|
||||
let rows: Vec<(i64, String, Option<i32>, f32)> = sqlx::query_as(
|
||||
"SELECT id, title, year, similarity(title_sort, $1) AS similarity \
|
||||
FROM furumusic__release WHERE title_sort % $1 \
|
||||
ORDER BY similarity DESC LIMIT $2",
|
||||
)
|
||||
.bind(title.to_lowercase())
|
||||
.bind(limit)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
Ok(rows
|
||||
.into_iter()
|
||||
.map(|(id, title, year, similarity)| SimilarRelease {
|
||||
id,
|
||||
title,
|
||||
year,
|
||||
similarity,
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// Check if a file with the given SHA-256 hash is actively used in the library.
|
||||
/// Returns true only if a media_file with this hash exists AND at least one
|
||||
/// track references it via audio_file_id. Orphaned media_files (no track)
|
||||
/// are ignored so that re-discovery is possible after the user deletes
|
||||
/// artists/releases/tracks.
|
||||
pub async fn file_hash_exists(pool: &PgPool, sha256: &str) -> anyhow::Result<bool> {
|
||||
let row: (bool,) = sqlx::query_as(
|
||||
"SELECT EXISTS(\
|
||||
SELECT 1 FROM furumusic__media_file mf \
|
||||
JOIN furumusic__track t ON t.audio_file_id = mf.id \
|
||||
WHERE mf.sha256_hash = $1\
|
||||
)",
|
||||
)
|
||||
.bind(sha256)
|
||||
.fetch_one(pool)
|
||||
.await?;
|
||||
Ok(row.0)
|
||||
}
|
||||
Reference in New Issue
Block a user