From 03f74cc91e8cd7eb2a39fbbeeac524834ebb3b0f Mon Sep 17 00:00:00 2001 From: Aleksandr Bogomiakov Date: Mon, 10 Aug 2026 00:22:20 +0100 Subject: [PATCH] Add federated music similarity search --- ARCHITECTURE.md | 28 + CHANGELOG.md | 26 + Cargo.lock | 715 +++++++++++++++++++++- Cargo.toml | 6 +- README.md | 8 + src/app/cmdline.rs | 61 ++ src/app/event.rs | 10 + src/app/mod.rs | 60 ++ src/app/popup.rs | 70 +++ src/app/state.rs | 40 ++ src/app/update.rs | 63 +- src/config/settings.rs | 58 ++ src/federation/capabilities.rs | 1 + src/federation/mod.rs | 32 + src/federation/similarity.rs | 277 +++++++++ src/library/mod.rs | 258 ++++++++ src/library/tests.rs | 75 +++ src/main.rs | 1 + src/similarity.rs | 1047 ++++++++++++++++++++++++++++++++ src/ui/federation.rs | 104 +++- src/ui/global.rs | 7 +- src/ui/popup.rs | 73 ++- 22 files changed, 2998 insertions(+), 22 deletions(-) create mode 100644 src/federation/similarity.rs create mode 100644 src/similarity.rs diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index d685566..c57e25c 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -282,6 +282,31 @@ intended role: transforming current audio features into drawing commands. It is not a plugin mechanism for accessing the library, network, or player controls. +## Music-similarity indexing + +Similarity search is an optional local capability and is disabled by default. +When enabled, a background pipeline downloads a selected ONNX model, verifies +its pinned SHA-256 digest, decodes durable local tracks, and stores normalized +embeddings in the library SQLite database. Embeddings are keyed by an exact +fingerprint of the model artifact and preprocessing profile. Old profile rows +remain available while a new profile is calculated, and the in-memory exact +cosine index switches only after the replacement profile is usable. + +The SQLite rows are the canonical derived store. The in-memory index can be +discarded and rebuilt, and neither is required for import, browsing, or +playback. Remote/cache-only tracks are never scheduled for local embedding. + +Federated similarity uses a separate versioned direct-stream protocol. With +explicit privacy consent, the requester sends only a normalized embedding and +its profile fingerprint to a bounded set of known peers. It does not publish +queries to the DHT. Each peer searches its own active local index and returns a +bounded metadata result with a compact embedding SimHash. The requester uses +that signature to suppress near-duplicate recordings across peers without +receiving every result vector. Fan-out, concurrency, message sizes, and +timeouts are bounded; incompatible profiles are rejected. This direct +peer-selection layer can later be replaced by DHT routing without changing +local storage or ranking. + ## Persistence boundaries Furumi stores different kinds of state according to their lifetime: @@ -290,6 +315,7 @@ Furumi stores different kinds of state according to their lifetime: | --- | --- | --- | | Library, playlists, likes, history | SQLite | Durable local source of truth | | Device operation log and replicas | SQLite | Offline synchronization | +| Versioned track embeddings | SQLite | Durable, locally rebuildable similarity data | | Federation catalog cache | SQLite/cache | Faster network browsing | | Audio and artwork cache | Filesystem cache | Reusable fetched data | | Settings, keymap, identity | Platform config/data dirs | Node configuration | @@ -321,6 +347,8 @@ The source tree follows the architectural responsibilities: - `library/` owns the local catalog and import pipeline; - `player/` owns audio playback and analysis; +- `similarity.rs` owns model acquisition, preprocessing, background indexing, + and the replaceable exact in-memory index; - `federation/` owns DHT-facing search, peer catalogs, and audio exchange; - `devices.rs` owns trusted-device replication and playback coordination; - `app/` owns state transitions and runtime orchestration; diff --git a/CHANGELOG.md b/CHANGELOG.md index feed566..23c0c6f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,32 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Optional offline music-similarity search for local tracks, backed by + versioned SQLite embeddings and a replaceable exact in-memory cosine index. +- Automatic SHA-256-verified download and local inference for the first ONNX + embedding model, with retained model/profile generations and background + backfilling of existing library tracks. +- Similarity settings for enablement, model/profile information, numeric worker + count, derived-data cleanup, processing progress, and federation privacy + consent. +- Track-seeded similarity search from the track-information popup, including + bounded federated queries to compatible known peers. +- The `furumi-fd/similarity/1` protocol in the visible protocol-version status. + +### Changed + +- Similarity wire types, bounds, validation, and stream framing now come from + the shared `music-dht 0.3.1` API so native, web, and future clients can + interoperate without sharing an embedding implementation. +- A similarity result page keeps the source track first as query context while + excluding it from the actual nearest-neighbor ranking, labels the mode as + `Search similar to`, and suppresses near-identical embeddings across releases + and federated peer responses. +- Preprocessing profiles now open a read-only details window describing their + audio selection, resampling, spectrogram, patching, and aggregation contract. + ## [0.2.5] - 2026-08-02 ### Added diff --git a/Cargo.lock b/Cargo.lock index 8c90a68..fa1d9ef 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -109,6 +109,12 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" +[[package]] +name = "anymap3" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5dfbc6d8d2675589ccbe4d0fd61df2419075625f8c1a62325e718e2b0049f9" + [[package]] name = "approx" version = "0.5.1" @@ -396,7 +402,16 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" dependencies = [ - "bit-vec", + "bit-vec 0.6.3", +] + +[[package]] +name = "bit-set" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2f926cc3060f09db9ebc5b52823d85268d24bb917e472c0c4bea35780a7d" +dependencies = [ + "bit-vec 0.9.1", ] [[package]] @@ -405,6 +420,15 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" +[[package]] +name = "bit-vec" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b71798fca2c1fe1086445a7258a4bc81e6e49dcd24c8d0dd9a1e57395b603f51" +dependencies = [ + "serde", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -893,6 +917,16 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + [[package]] name = "crossbeam-epoch" version = "0.9.20" @@ -1119,6 +1153,17 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "derive-new" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cdc8d50f426189eef89dac62fabfa0abb27d5cc008f25bf4156a0203325becc" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "derive_more" version = "2.1.1" @@ -1238,12 +1283,30 @@ dependencies = [ "litrs", ] +[[package]] +name = "downcast-rs" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "117240f60069e65410b3ae1bb213295bd828f707b5bec6596a1afc8793ce0cbc" + [[package]] name = "dyn-clone" version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" +[[package]] +name = "dyn-eq" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c2d035d21af5cde1a6f5c7b444a5bf963520a9f142e5d06931178433d7d5388" + +[[package]] +name = "dyn-hash" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fdab65db9274e0168143841eb8f864a0a21f8b1b8d2ba6812bbe6024346e99e" + [[package]] name = "ed25519" version = "3.0.0" @@ -1336,6 +1399,17 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +[[package]] +name = "erased-serde" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2add8a07dd6a8d93ff627029c51de145e12686fbc36ecb298ac22e74cf02dec" +dependencies = [ + "serde", + "serde_core", + "typeid", +] + [[package]] name = "errno" version = "0.3.14" @@ -1416,7 +1490,7 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b95f7c0680e4142284cf8b22c14a476e87d61b004a3a0861872b32ef7ead40a2" dependencies = [ - "bit-set", + "bit-set 0.5.3", "regex", ] @@ -1453,8 +1527,6 @@ dependencies = [ [[package]] name = "federation-net" version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15a8707baeccb46b5935138f9cb3df3c988c0730b807a2634d901f26b39250d6" dependencies = [ "blake3", "data-encoding", @@ -1488,6 +1560,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "filetime" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" +dependencies = [ + "cfg-if", + "libc", +] + [[package]] name = "find-msvc-tools" version = "0.1.9" @@ -1516,6 +1598,12 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "float-ord" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ce81f49ae8a0482e4c55ea62ebbd7e5a686af544c00b9d090bba3ff9be97b3d" + [[package]] name = "flume" version = "0.12.0" @@ -1579,17 +1667,21 @@ dependencies = [ "lofty", "music-dht", "ratatui", + "reqwest 0.12.28", "rhai", "rodio", "rusqlite", + "rustfft", "serde", "serde_json", + "sha2 0.10.9", "souvlaki", "thiserror 2.0.19", "tokio", "toml", "tracing", "tracing-subscriber", + "tract-onnx", "unicode-width", "windows-sys 0.61.2", ] @@ -1838,6 +1930,18 @@ dependencies = [ "tracing", ] +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "num-traits", + "zerocopy", +] + [[package]] name = "hash32" version = "0.2.1" @@ -1865,6 +1969,8 @@ dependencies = [ "allocator-api2", "equivalent", "foldhash", + "serde", + "serde_core", ] [[package]] @@ -2091,6 +2197,7 @@ dependencies = [ "tokio", "tokio-rustls", "tower-service", + "webpki-roots", ] [[package]] @@ -2353,6 +2460,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "inventory" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4f0c30c76f2f4ccee3fe55a2435f691ca00c0e4bd87abe4f4a851b1d4dac39b" +dependencies = [ + "rustversion", +] + [[package]] name = "io-lifetimes" version = "1.0.11" @@ -2421,7 +2537,7 @@ dependencies = [ "portable-atomic", "portmapper", "rand 0.10.2", - "reqwest", + "reqwest 0.13.4", "rustc-hash", "rustls", "rustls-pki-types", @@ -2536,7 +2652,7 @@ dependencies = [ "pin-project", "postcard", "rand 0.10.2", - "reqwest", + "reqwest 0.13.4", "rustls", "rustls-pki-types", "serde", @@ -2891,6 +3007,12 @@ dependencies = [ "libc", ] +[[package]] +name = "maplit" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" + [[package]] name = "matchers" version = "0.2.0" @@ -2900,18 +3022,43 @@ dependencies = [ "regex-automata", ] +[[package]] +name = "matrixmultiply" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f607c237553f086e7043417a51df26b2eb899d3caff94e6a67592ff992fedc7" +dependencies = [ + "autocfg", + "rawpointer", +] + [[package]] name = "memchr" version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" +[[package]] +name = "memmap2" +version = "0.9.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" +dependencies = [ + "libc", +] + [[package]] name = "memmem" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a64a92489e2744ce060c349162be1c5f33c6969234104dbd99ddb5feb08b8c15" +[[package]] +name = "memo-map" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38d1115007560874e373613744c6fba374c17688327a71c1476d1a5954cc857b" + [[package]] name = "memoffset" version = "0.7.1" @@ -2930,6 +3077,16 @@ dependencies = [ "autocfg", ] +[[package]] +name = "minijinja" +version = "2.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42d74234349a775546a83af0f0c0c0e3a73227dee4950a542cda81a47240b3e6" +dependencies = [ + "memo-map", + "serde", +] + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -2987,9 +3144,7 @@ dependencies = [ [[package]] name = "music-dht" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f32daa9edf769fb5686e92ae6884e9fda6ea082452e4208309fc14301e26aef" +version = "0.3.1" dependencies = [ "async-trait", "blake3", @@ -3060,6 +3215,21 @@ dependencies = [ "n0-future", ] +[[package]] +name = "ndarray" +version = "0.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520080814a7a6b4a6e9070823bb24b4531daac8c4627e08ba5de8c5ef2f2752d" +dependencies = [ + "matrixmultiply", + "num-complex", + "num-integer", + "num-traits", + "portable-atomic", + "portable-atomic-util", + "rawpointer", +] + [[package]] name = "ndk" version = "0.9.0" @@ -3246,6 +3416,24 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + +[[package]] +name = "nom-language" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2de2bc5b451bfedaef92c90b8939a8fff5770bdcc1fafd6239d086aab8fa6b29" +dependencies = [ + "nom 8.0.0", +] + [[package]] name = "noq" version = "1.1.0" @@ -3327,6 +3515,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + [[package]] name = "num-conv" version = "0.2.2" @@ -3371,6 +3568,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", + "libm", ] [[package]] @@ -3676,6 +3874,12 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "pastey" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4" + [[package]] name = "pem-rfc7468" version = "1.0.0" @@ -3931,6 +4135,15 @@ dependencies = [ "serde", ] +[[package]] +name = "portable-atomic-util" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" +dependencies = [ + "portable-atomic", +] + [[package]] name = "portmapper" version = "0.19.1" @@ -4020,6 +4233,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "primal-check" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0d895b311e3af9902528fbb8f928688abbd95872819320517cc24ca6b2bd08" +dependencies = [ + "num-integer", +] + [[package]] name = "proc-macro-crate" version = "1.3.1" @@ -4048,6 +4270,29 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "prost" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-derive" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "pxfm" version = "0.1.30" @@ -4069,6 +4314,62 @@ dependencies = [ "memchr", ] +[[package]] +name = "quinn" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2 0.6.5", + "thiserror 2.0.19", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" +dependencies = [ + "bytes", + "getrandom 0.4.3", + "lru-slab", + "rand 0.10.2", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.19", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2 0.6.5", + "tracing", + "windows-sys 0.61.2", +] + [[package]] name = "quote" version = "1.0.47" @@ -4166,6 +4467,16 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" +[[package]] +name = "rand_distr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d431c2703ccf129de4d45253c03f49ebb22b97d6ad79ee3ecfc7e3f4862c1d8" +dependencies = [ + "num-traits", + "rand 0.10.2", +] + [[package]] name = "rand_pcg" version = "0.10.2" @@ -4276,6 +4587,32 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "rawpointer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -4325,6 +4662,47 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams 0.4.2", + "web-sys", + "webpki-roots", +] + [[package]] name = "reqwest" version = "0.13.4" @@ -4358,7 +4736,7 @@ dependencies = [ "url", "wasm-bindgen", "wasm-bindgen-futures", - "wasm-streams", + "wasm-streams 0.5.0", "web-sys", ] @@ -4453,6 +4831,20 @@ dependencies = [ "semver", ] +[[package]] +name = "rustfft" +version = "6.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21db5f9893e91f41798c88680037dba611ca6674703c1a18601b01a72c8adb89" +dependencies = [ + "num-complex", + "num-integer", + "num-traits", + "primal-check", + "strength_reduce", + "transpose", +] + [[package]] name = "rustix" version = "0.37.28" @@ -4580,6 +4972,19 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "safetensors" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79b079b829cb27a1c3c374341345ed2e8b2c0c839034522cee576c140bd7f846" +dependencies = [ + "hashbrown 0.16.1", + "libc", + "serde", + "serde_json", + "tempfile", +] + [[package]] name = "same-file" version = "1.0.6" @@ -4589,6 +4994,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "scan_fmt" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b53b0a5db882a8e2fdaae0a43f7b39e7e9082389e978398bdf223a55b581248" +dependencies = [ + "regex", +] + [[package]] name = "schannel" version = "0.1.29" @@ -4738,6 +5152,18 @@ dependencies = [ "serde_core", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + [[package]] name = "serdect" version = "0.4.3" @@ -5001,12 +5427,28 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "strength_reduce" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe895eb47f22e2ddd4dabc02bce419d2e643c8e3b585c78158b349195bc24d82" + [[package]] name = "strict" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f42444fea5b87a39db4218d9422087e66a85d0e7a0963a439b07bcdf91804006" +[[package]] +name = "string-interner" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad3df9b59e2eded8d825c7c4363ad339a20fb6bc0b9a4778560f518f59910b15" +dependencies = [ + "hashbrown 0.16.1", + "serde", +] + [[package]] name = "strsim" version = "0.11.1" @@ -5278,6 +5720,17 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" +[[package]] +name = "tar" +version = "0.4.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "tempfile" version = "3.27.0" @@ -5311,7 +5764,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4ea810f0692f9f51b382fff5893887bb4580f5fa246fde546e0b13e7fcee662" dependencies = [ "fnv", - "nom", + "nom 7.1.3", "phf", "phf_codegen", ] @@ -5751,12 +6204,229 @@ dependencies = [ "tracing-log", ] +[[package]] +name = "tract-core" +version = "0.23.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "608e176a669d5da02cccc92bbfe5ee4e57686ed8841022608a9eba014d3b7886" +dependencies = [ + "anyhow", + "anymap3", + "bit-set 0.10.0", + "derive-new", + "downcast-rs", + "dyn-clone", + "dyn-eq", + "erased-serde", + "inventory", + "lazy_static", + "log", + "maplit", + "ndarray", + "num-complex", + "num-integer", + "num-traits", + "pastey", + "rustfft", + "serde", + "smallvec", + "tract-data", + "tract-linalg", +] + +[[package]] +name = "tract-data" +version = "0.23.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "870236dd45aaeb1381023cb709a67ff14ece608ee0b37f99aa166d166db9b0d0" +dependencies = [ + "anyhow", + "downcast-rs", + "dyn-clone", + "dyn-eq", + "dyn-hash", + "half", + "inventory", + "itertools", + "lazy_static", + "libm", + "maplit", + "ndarray", + "nom 8.0.0", + "nom-language", + "num-integer", + "num-traits", + "parking_lot", + "scan_fmt", + "smallvec", + "string-interner", +] + +[[package]] +name = "tract-extra" +version = "0.23.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "effcae1ebfce133e8bf6c79ad31298cbee0a9eceb3e4642fe1d14cb54cca78e6" +dependencies = [ + "tract-nnef", + "tract-pulse", +] + +[[package]] +name = "tract-hir" +version = "0.23.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28783b2bb583177685f65016a866b01eb3d383bce4fa7ae1b9692b325b64449d" +dependencies = [ + "derive-new", + "log", + "tract-core", +] + +[[package]] +name = "tract-linalg" +version = "0.23.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3e01491f7360806ef061af4c016a2d0a801d586768896394a0b8a7d6872c2b0" +dependencies = [ + "byteorder", + "cc", + "derive-new", + "downcast-rs", + "dyn-clone", + "dyn-eq", + "dyn-hash", + "half", + "lazy_static", + "log", + "minijinja", + "num-traits", + "pastey", + "scan_fmt", + "tract-data", + "walkdir", +] + +[[package]] +name = "tract-nnef" +version = "0.23.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00417fabf01aeea7bc56107367862e5bb8da18c9ad850c9061d3823700479ecc" +dependencies = [ + "byteorder", + "erased-serde", + "flate2", + "log", + "minijinja", + "nom 8.0.0", + "nom-language", + "safetensors", + "serde", + "serde_json", + "simd-adler32", + "tar", + "tract-core", + "walkdir", +] + +[[package]] +name = "tract-onnx" +version = "0.23.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3215dd27bddd2a041a20fee750013b400135186d3485501c9274c755b19ceb0" +dependencies = [ + "bytes", + "derive-new", + "dyn-eq", + "log", + "memmap2", + "num-integer", + "prost", + "smallvec", + "tract-extra", + "tract-hir", + "tract-nnef", + "tract-onnx-opl", + "tract-transformers", +] + +[[package]] +name = "tract-onnx-opl" +version = "0.23.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44f549ad3f245c1c00ce710c66cc0e086be0b637f35934d37b655ff175e1ab3e" +dependencies = [ + "dyn-eq", + "getrandom 0.4.3", + "log", + "rand 0.10.2", + "rand_distr", + "rustfft", + "tract-extra", + "tract-nnef", +] + +[[package]] +name = "tract-pulse" +version = "0.23.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a164e22e96ab9b5c90458fa270700570d87963e3dec9ccaac1ab18d9fa763ce2" +dependencies = [ + "downcast-rs", + "dyn-eq", + "erased-serde", + "lazy_static", + "log", + "serde", + "tract-pulse-opl", + "tract-transformers", +] + +[[package]] +name = "tract-pulse-opl" +version = "0.23.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65418f9e93e0af0d567f4f2f1bf2309b53930fa2543c635a69fda10c87a04987" +dependencies = [ + "downcast-rs", + "dyn-eq", + "lazy_static", + "tract-nnef", +] + +[[package]] +name = "tract-transformers" +version = "0.23.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8471ebf7f52d226552283d9130539595540c28ed41b8e032df3830507d3ccdd5" +dependencies = [ + "float-ord", + "rayon", + "tract-nnef", +] + +[[package]] +name = "transpose" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad61aed86bc3faea4300c7aee358b4c6d0c8d6ccc36524c96e4c92ccf26e77e" +dependencies = [ + "num-integer", + "strength_reduce", +] + [[package]] name = "try-lock" version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "typeid" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" + [[package]] name = "typenum" version = "1.20.1" @@ -5999,6 +6669,19 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "wasm-streams" version = "0.5.0" @@ -6582,6 +7265,16 @@ dependencies = [ "web-sys", ] +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix 1.1.4", +] + [[package]] name = "xdg-home" version = "1.3.0" diff --git a/Cargo.toml b/Cargo.toml index 5fdb817..e50545a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,19 +21,23 @@ image = { version = "0.25.10", default-features = false, features = ["jpeg", "pn lofty = "0.22" # P2P federation: library index in a shared DHT + audio streaming between # peers (same protocol as furumi-fd). -music-dht = "0.3" +music-dht = { version = "0.3.1", path = "../frid/crates/music-dht" } ratatui = "0.30.1" +reqwest = { version = "0.12.28", default-features = false, features = ["rustls-tls", "stream"] } rhai = { version = "1", features = ["sync"] } rodio = { version = "0.22.2", default-features = false, features = ["playback", "mp3", "flac", "vorbis", "wav", "symphonia-aac", "symphonia-isomp4", "symphonia-alac"] } +rustfft = "6.4.1" rusqlite = { version = "0.32", features = ["bundled", "functions"] } serde = { version = "1.0.228", features = ["derive"] } serde_json = "1.0.150" +sha2 = "0.10.9" souvlaki = { version = "0.8.3", default-features = false, features = ["use_zbus"] } thiserror = "2.0.18" tokio = { version = "1.52.3", features = ["rt-multi-thread", "macros", "sync", "time", "fs", "io-util"] } toml = "1.1.2" tracing = "0.1.44" tracing-subscriber = { version = "0.3.23", features = ["env-filter"] } +tract-onnx = "0.23.4" unicode-width = "0.2.2" [patch.crates-io] diff --git a/README.md b/README.md index 399c0ea..acd7b9e 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,13 @@ library management are included. Visualizations are runtime-loadable Rhai scripts executed in a resource-limited sandbox, so they can be added or edited without rebuilding the player. +Optional similarity search calculates versioned embeddings for local tracks +in the background and keeps them in SQLite. It works offline; after a separate +privacy consent it can also ask a bounded set of federation peers for matches. +The first selectable model is downloaded on demand and is licensed separately +by MTG under CC BY-NC-SA 4.0 (a proprietary license is also available from +MTG); Furumi itself remains WTFPL. + ## Install ### macOS @@ -144,6 +151,7 @@ Furumi is a Rust application built with: - `ratatui` and `crossterm` for the cross-platform TUI; - `rodio` for local audio playback; - SQLite for the personal library and synchronization state; +- tract ONNX inference for optional local music embeddings; - a dedicated DHT for decentralized discovery; - iroh-based P2P streams for client-to-client communication; - an offline-tolerant operation log for trusted-device synchronization; diff --git a/src/app/cmdline.rs b/src/app/cmdline.rs index 9286038..15286a4 100644 --- a/src/app/cmdline.rs +++ b/src/app/cmdline.rs @@ -8,6 +8,7 @@ use crate::app::Runtime; use crate::app::command::{self, Command, Parsed}; use crate::app::event::AppEvent; use crate::app::state::{AppState, GlobalView, SearchState, Tab}; +use crate::library::models::SearchResults; const SEARCH_DEBOUNCE: Duration = Duration::from_millis(180); const SEARCH_LIMIT: i64 = 12; @@ -97,6 +98,7 @@ fn set_view_cursor_zero(state: &mut AppState) { /// spawned task only queries if it is still the latest after the debounce, /// and the receiver drops responses that arrive out of date. pub(super) fn schedule_search(state: &mut AppState, runtime: &Runtime) { + state.search.similarity_source = None; let seq = runtime.search_seq.fetch_add(1, Ordering::SeqCst) + 1; let query = state.search.query.clone(); if query.is_empty() { @@ -145,6 +147,62 @@ pub(super) fn schedule_search(state: &mut AppState, runtime: &Runtime) { } } +pub(super) fn schedule_similarity_search( + state: &mut AppState, + runtime: &Runtime, + track: &crate::library::models::TrackItem, +) { + let seq = runtime.search_seq.fetch_add(1, Ordering::SeqCst) + 1; + let artist = track.artist_line(); + state.search.query = if artist.is_empty() { + track.title.clone() + } else { + format!("{} — {artist}", track.title) + }; + state.search.similarity_source = Some(track.id); + state.search.loading = true; + state.search.results = None; + state.search.fed_tracks.clear(); + state.search.fed_artists.clear(); + state.search.fed_loading = false; + state.active_tab = crate::app::state::Tab::Global; + if let Some(crate::app::state::GlobalView::Search { cursor }) = state.global.stack.last_mut() { + *cursor = 0; + } else { + state + .global + .stack + .push(crate::app::state::GlobalView::Search { cursor: 0 }); + } + + let similarity = Arc::clone(&runtime.similarity); + let tx = runtime.event_tx.clone(); + let track_id = track.id; + let source_track = track.clone(); + tokio::task::spawn_blocking(move || { + let result = similarity + .search_track(track_id, 49) + .map(|(matches, query)| { + let mut tracks = Vec::with_capacity(1 + matches.len()); + tracks.push(source_track); + tracks.extend(matches.into_iter().map(|found| found.track)); + ( + SearchResults { + artists: Vec::new(), + releases: Vec::new(), + tracks, + }, + query, + ) + }); + let (result, query) = match result { + Ok((results, query)) => (Ok(results), Some(query)), + Err(err) => (Err(format!("{err:#}")), None), + }; + let _ = tx.send(AppEvent::SimilaritySearchLoaded { seq, result, query }); + }); +} + /// Refresh only the local-library half of an already open search. /// /// Library/device sync notifications can arrive while federated search @@ -152,6 +210,9 @@ pub(super) fn schedule_search(state: &mut AppState, runtime: &Runtime) { /// federation rows and bump the shared sequence, causing valid network /// responses to be dropped or flicker away. pub(super) fn refresh_local_search(state: &mut AppState, runtime: &Runtime) { + if state.search.similarity_source.is_some() { + return; + } let query = state.search.query.clone(); if query.is_empty() { return; diff --git a/src/app/event.rs b/src/app/event.rs index 504abf4..3e1cbf1 100644 --- a/src/app/event.rs +++ b/src/app/event.rs @@ -32,6 +32,16 @@ pub enum AppEvent { seq: u64, result: Result, }, + /// Local similar-track search completed. It uses the same sequence as + /// text search so stale pages cannot overwrite a newer request. + SimilaritySearchLoaded { + seq: u64, + result: Result, + query: Option, + }, + SimilarityStatus(crate::similarity::SimilarityStatus), + /// `None` is emitted after clearing every stored embedding. + SimilarityProfileActivated(Option), /// Artwork loaded and decoded for the shared art cache. ArtLoaded { key: String, diff --git a/src/app/mod.rs b/src/app/mod.rs index 0fbc944..7276b59 100644 --- a/src/app/mod.rs +++ b/src/app/mod.rs @@ -41,6 +41,7 @@ pub struct Runtime { pub devices: Arc, pub jam: Arc, pub federation: Arc, + pub similarity: Arc, /// When the last Federation-tab status snapshot was requested. pub fed_status_at: Option, pub library_network_refresh_at: Option, @@ -262,6 +263,7 @@ pub async fn run( state.player.volume = settings.volume; state.global.filters = settings.library; state.music_dir = settings.music_dir.clone(); + state.similarity.settings = settings.similarity.clone(); if let Err(err) = state.visualizer.load_library() { state.status_message = Some(format!("visualizations disabled: {err:#}")); } @@ -269,10 +271,17 @@ pub async fn run( let devices = crate::devices::DeviceSync::new(Arc::clone(&library))?; devices.set_event_tx(event_tx.clone()); let jam = crate::jam::JamManager::new(event_tx.clone()); + let similarity = crate::similarity::Manager::new( + Arc::clone(&library), + event_tx.clone(), + settings.similarity.clone(), + ); + state.similarity.status = similarity.status(); let federation = crate::federation::Federation::new( Arc::clone(&library), Arc::clone(&devices), Arc::clone(&jam), + Arc::clone(&similarity), settings.music_dir.clone(), ); state.music_dir = federation.media_dir(); @@ -292,6 +301,7 @@ pub async fn run( devices, jam, federation, + similarity, fed_status_at: None, library_network_refresh_at: None, library_network_refreshing: Arc::new(std::sync::atomic::AtomicBool::new(false)), @@ -319,6 +329,9 @@ pub async fn run( status_publisher: crate::status::Publisher::spawn(), }; spawn_content_id_backfill(&runtime); + if state.similarity.settings.enabled { + runtime.similarity.start(); + } { let fed = Arc::clone(&runtime.federation); @@ -1581,6 +1594,15 @@ fn perform_effect(state: &mut AppState, runtime: &mut Runtime, effect: Effect) { let _ = tx.send(event); }); } + Effect::SimilarityApplySettings => { + save_app_settings(state); + runtime.similarity.apply(state.similarity.settings.clone()); + state.similarity.status = runtime.similarity.status(); + } + Effect::SimilarityClear => { + state.status_message = Some("clearing stored embeddings…".to_string()); + runtime.similarity.clear(); + } Effect::FedApplySettings => fed_apply_settings(state, runtime), Effect::FedSyncNow => { state.federation.publishing = true; @@ -2785,6 +2807,7 @@ fn save_app_settings(state: &AppState) { volume: state.player.volume, library: state.global.filters, music_dir: state.music_dir.clone(), + similarity: state.similarity.settings.clone(), }; if let Err(err) = crate::config::settings::save(&settings) { tracing::warn!(%err, "saving app settings failed"); @@ -3632,6 +3655,40 @@ fn handle_app_event(state: &mut AppState, runtime: &mut Runtime, event: AppEvent Err(message) => state.status_message = Some(message), } } + AppEvent::SimilaritySearchLoaded { seq, result, query } => { + if seq != runtime.search_seq.load(std::sync::atomic::Ordering::SeqCst) { + return; + } + state.search.loading = false; + match result { + Ok(results) => state.search.results = Some(results), + Err(message) => { + state.status_message = Some(format!("similarity search failed: {message}")); + return; + } + } + if let Some(query) = query + && state.federation.settings.enabled + && runtime.similarity.network_allowed() + { + state.search.fed_loading = true; + let federation = Arc::clone(&runtime.federation); + let tx = runtime.event_tx.clone(); + tokio::spawn(async move { + let result = federation + .search_similar(query, 50) + .await + .map_err(|err| format!("{err:#}")); + let _ = tx.send(AppEvent::FedSearchLoaded { seq, result }); + }); + } + } + AppEvent::SimilarityStatus(status) => state.similarity.status = status, + AppEvent::SimilarityProfileActivated(profile_id) => { + state.similarity.settings.active_profile = profile_id; + state.similarity.status = runtime.similarity.status(); + save_app_settings(state); + } AppEvent::ArtLoaded { key, art } => { let entry = match art { Some(image) => state::ArtState::Ready(image), @@ -3857,6 +3914,9 @@ fn handle_app_event(state: &mut AppState, runtime: &mut Runtime, event: AppEvent } AppEvent::LibraryChanged { message } => { on_library_changed(state, runtime); + if state.similarity.settings.enabled { + runtime.similarity.start(); + } if let Some(message) = message { state.status_message = Some(message); } diff --git a/src/app/popup.rs b/src/app/popup.rs index 0310138..79509f9 100644 --- a/src/app/popup.rs +++ b/src/app/popup.rs @@ -115,6 +115,39 @@ pub fn handle_key(state: &mut AppState, runtime: &mut Runtime, key: KeyEvent) { Popup::ConfirmDelete { target, label } => { handle_confirm_delete(state, runtime, target, label, key); } + Popup::SimilarityPrivacyConsent { enable_federation } => match key.code { + KeyCode::Enter | KeyCode::Char('y') => { + state.similarity.settings.federation_consent = true; + state.similarity.settings.enabled = true; + super::perform_effect( + state, + runtime, + crate::app::update::Effect::SimilarityApplySettings, + ); + if enable_federation { + super::perform_effect( + state, + runtime, + crate::app::update::Effect::FedApplySettings, + ); + } + } + KeyCode::Esc | KeyCode::Char('n') | KeyCode::Char('q') => { + if enable_federation { + state.federation.settings.enabled = false; + } + } + _ => { + state.popup = Some(Popup::SimilarityPrivacyConsent { enable_federation }); + } + }, + Popup::ConfirmClearEmbeddings => match key.code { + KeyCode::Enter | KeyCode::Char('y') => { + super::perform_effect(state, runtime, crate::app::update::Effect::SimilarityClear) + } + KeyCode::Esc | KeyCode::Char('n') | KeyCode::Char('q') => {} + _ => state.popup = Some(Popup::ConfirmClearEmbeddings), + }, Popup::LibraryFilters { cursor } => handle_library_filters(state, runtime, cursor, key), Popup::TrackInfo { tracks, @@ -544,6 +577,21 @@ fn handle_fed_input( super::validate_music_directory(state, runtime, value.into()); } } + FedInputField::SimilarityWorkers => match value.parse::() { + Ok(workers @ 1..=16) => { + state.similarity.settings.workers = workers; + super::perform_effect( + state, + runtime, + crate::app::update::Effect::SimilarityApplySettings, + ); + } + _ => { + state.status_message = + Some("similarity workers must be a number from 1 to 16".into()); + state.popup = Some(Popup::FedInput { field, input }); + } + }, FedInputField::ConnectTicket => { if value.is_empty() { state.status_message = Some("ticket is empty".into()); @@ -1019,6 +1067,28 @@ fn handle_track_info( scroll, }); } + KeyCode::Char('s') => { + let Some(track) = tracks.get(cursor.min(len.saturating_sub(1))) else { + return; + }; + if !state.similarity.settings.enabled { + state.status_message = Some("enable Similarity search in Settings first".into()); + state.popup = Some(Popup::TrackInfo { + tracks, + cursor, + scroll, + }); + } else if track.id < 0 || track.file_path.is_empty() { + state.status_message = Some("similarity search starts from a local track".into()); + state.popup = Some(Popup::TrackInfo { + tracks, + cursor, + scroll, + }); + } else { + super::cmdline::schedule_similarity_search(state, runtime, track); + } + } _ => { state.popup = Some(Popup::TrackInfo { tracks, diff --git a/src/app/state.rs b/src/app/state.rs index 2c9164a..dd6c9eb 100644 --- a/src/app/state.rs +++ b/src/app/state.rs @@ -687,6 +687,10 @@ pub enum Popup { }, /// Delete confirmation; Enter/y deletes, Esc/n cancels. ConfirmDelete { target: DeleteTarget, label: String }, + /// Enabling network similarity reveals an embedding query to peers. + SimilarityPrivacyConsent { enable_federation: bool }, + /// Derived data is safe to recreate but potentially expensive. + ConfirmClearEmbeddings, /// Library-home filters. Cursor is kept for the next filters added here. LibraryFilters { cursor: usize }, /// Track metadata viewer; left/right switch between selected tracks. @@ -812,6 +816,7 @@ impl StatusDetailFocus { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum FedInputField { MusicDirectory, + SimilarityWorkers, NetworkId, ConnectTicket, DeviceName, @@ -823,6 +828,7 @@ impl FedInputField { pub fn title(self) -> &'static str { match self { FedInputField::MusicDirectory => "Music save directory", + FedInputField::SimilarityWorkers => "Similarity background workers", FedInputField::NetworkId => "Network ID", FedInputField::ConnectTicket => "Connect to peer (paste ticket)", FedInputField::DeviceName => "Device name", @@ -836,6 +842,9 @@ impl FedInputField { FedInputField::MusicDirectory => { "Federated tracks saved to your library use this directory. The directory is checked for write access before anything changes." } + FedInputField::SimilarityWorkers => { + "Enter the maximum number of tracks processed in parallel, from 1 to 16. The change takes effect immediately." + } FedInputField::NetworkId => { "A unique network id. It must match exactly on every client that should see and connect to the same peers." } @@ -866,6 +875,25 @@ pub enum FedRow { Connect, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SimilarityRow { + Toggle, + Model, + Profile, + Workers, + Clear, +} + +impl SimilarityRow { + pub const ALL: [SimilarityRow; 5] = [ + SimilarityRow::Toggle, + SimilarityRow::Model, + SimilarityRow::Profile, + SimilarityRow::Workers, + SimilarityRow::Clear, + ]; +} + impl FedRow { pub const ALL: [FedRow; 6] = [ FedRow::Toggle, @@ -883,6 +911,7 @@ impl FedRow { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum SettingsRow { MusicDirectory, + Similarity(SimilarityRow), Federation(FedRow), StatusDetails, DeviceName, @@ -1021,6 +1050,7 @@ pub fn device_status_order(state: &AppState) -> Vec { pub fn settings_rows(state: &AppState) -> Vec { let mut rows = vec![SettingsRow::MusicDirectory]; + rows.extend(SimilarityRow::ALL.into_iter().map(SettingsRow::Similarity)); rows.extend(FedRow::ALL.into_iter().map(SettingsRow::Federation)); rows.push(SettingsRow::DeviceName); rows.push(SettingsRow::DeviceInvite); @@ -1059,6 +1089,12 @@ pub struct FederationTab { pub device_syncing: bool, } +#[derive(Debug, Default)] +pub struct SimilarityTab { + pub settings: crate::config::settings::SimilaritySettings, + pub status: crate::similarity::SimilarityStatus, +} + /// Playlists eligible as add-targets (the virtual Likes playlist is managed /// through likes, not direct adds). pub fn addable_playlists(state: &AppState) -> Vec<(i64, String)> { @@ -1187,6 +1223,9 @@ pub struct SearchState { /// from the artist names of matching tracks. pub fed_artists: Vec, pub fed_loading: bool, + /// Present only for a track-seeded search; text-search refreshes must not + /// replace this page with a title query. + pub similarity_source: Option, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] @@ -1423,6 +1462,7 @@ pub struct AppState { pub logs: LogsTab, pub queue_tab: QueueTab, pub federation: FederationTab, + pub similarity: SimilarityTab, /// The one federated artist card being viewed (name + loading state); /// opening another card replaces it. pub fed_artist_view: Option<(String, Loadable)>, diff --git a/src/app/update.rs b/src/app/update.rs index 4e824d6..431022d 100644 --- a/src/app/update.rs +++ b/src/app/update.rs @@ -63,6 +63,9 @@ pub enum Effect { }, /// Persist the federation settings and start/stop the node. FedApplySettings, + /// Persist/apply embedding model, profile, worker or enable changes. + SimilarityApplySettings, + SimilarityClear, /// Force an immediate library publish into the DHT. FedSyncNow, /// Fetch this peer's ticket and show it in a popup. @@ -2720,7 +2723,7 @@ fn fed_card_featured_artist_names(track: &crate::federation::FedCardTrack) -> Ve /// Enter on Settings: toggle switches, open text inputs, run /// one-shot operations. The heavy lifting happens in perform_effect(). fn federation_select(state: &mut AppState) -> Option { - use super::state::{FedInputField, FedRow, Popup, SettingsRow}; + use super::state::{FedInputField, FedRow, Popup, SettingsRow, SimilarityRow}; match settings_rows(state).get(state.settings_cursor).copied()? { SettingsRow::MusicDirectory => { if state.music_dir_changing { @@ -2735,6 +2738,52 @@ fn federation_select(state: &mut AppState) -> Option { }); None } + SettingsRow::Similarity(SimilarityRow::Toggle) => { + if !state.similarity.settings.enabled + && state.federation.settings.enabled + && !state.similarity.settings.federation_consent + { + state.popup = Some(Popup::SimilarityPrivacyConsent { + enable_federation: false, + }); + return None; + } + state.similarity.settings.enabled = !state.similarity.settings.enabled; + Some(Effect::SimilarityApplySettings) + } + SettingsRow::Similarity(SimilarityRow::Model) => { + let models = crate::similarity::MODELS; + let current = models + .iter() + .position(|model| model.id == state.similarity.settings.model) + .unwrap_or(0); + state.similarity.settings.model = models[(current + 1) % models.len()].id.to_string(); + Some(Effect::SimilarityApplySettings) + } + SettingsRow::Similarity(SimilarityRow::Profile) => { + let profile = state.similarity.settings.profile.clone(); + let text = + crate::similarity::profile_details(&profile, &state.similarity.settings.model) + .unwrap_or_else(|| format!("Unknown preprocessing profile: {profile}")); + state.popup = Some(Popup::FedText { + title: format!("Preprocessing profile: {profile}"), + text, + }); + None + } + SettingsRow::Similarity(SimilarityRow::Workers) => { + state.popup = Some(Popup::FedInput { + field: FedInputField::SimilarityWorkers, + input: crate::app::input::LineEdit::new( + state.similarity.settings.workers.to_string(), + ), + }); + None + } + SettingsRow::Similarity(SimilarityRow::Clear) => { + state.popup = Some(Popup::ConfirmClearEmbeddings); + None + } SettingsRow::Federation(FedRow::Toggle) => { let settings = &mut state.federation.settings; if !settings.enabled && settings.network_id.trim().is_empty() { @@ -2744,7 +2793,17 @@ fn federation_select(state: &mut AppState) -> Option { }); return None; } - settings.enabled = !settings.enabled; + let enabling = !settings.enabled; + settings.enabled = enabling; + if enabling + && state.similarity.settings.enabled + && !state.similarity.settings.federation_consent + { + state.popup = Some(Popup::SimilarityPrivacyConsent { + enable_federation: true, + }); + return None; + } Some(Effect::FedApplySettings) } SettingsRow::Federation(FedRow::NetworkId) => { diff --git a/src/config/settings.rs b/src/config/settings.rs index 8670ab7..21d485d 100644 --- a/src/config/settings.rs +++ b/src/config/settings.rs @@ -45,6 +45,40 @@ pub struct LibraryFilters { pub source_mode: LibrarySourceMode, } +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct SimilaritySettings { + /// Local embedding/search master switch. Network participation follows + /// federation and additionally requires the explicit privacy consent. + #[serde(default)] + pub enabled: bool, + #[serde(default = "default_similarity_model")] + pub model: String, + #[serde(default = "default_similarity_profile")] + pub profile: String, + #[serde(default = "default_similarity_workers")] + pub workers: usize, + #[serde(default)] + pub federation_consent: bool, + /// Exact fingerprint of the last fully usable profile. Keeping this + /// separate from the selected target lets an old index serve searches + /// while a newly selected model/profile is being calculated. + #[serde(default)] + pub active_profile: Option, +} + +impl Default for SimilaritySettings { + fn default() -> Self { + Self { + enabled: false, + model: default_similarity_model(), + profile: default_similarity_profile(), + workers: default_similarity_workers(), + federation_consent: false, + active_profile: None, + } + } +} + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct AppSettings { #[serde(default = "default_volume")] @@ -54,6 +88,8 @@ pub struct AppSettings { /// Root used for music materialized from federation peers. #[serde(default = "default_music_dir")] pub music_dir: PathBuf, + #[serde(default)] + pub similarity: SimilaritySettings, } impl Default for AppSettings { @@ -62,6 +98,7 @@ impl Default for AppSettings { volume: default_volume(), library: LibraryFilters::default(), music_dir: default_music_dir(), + similarity: SimilaritySettings::default(), } } } @@ -72,6 +109,13 @@ impl AppSettings { if self.music_dir.as_os_str().is_empty() { self.music_dir = default_music_dir(); } + if self.similarity.model.trim().is_empty() { + self.similarity.model = default_similarity_model(); + } + if self.similarity.profile.trim().is_empty() { + self.similarity.profile = default_similarity_profile(); + } + self.similarity.workers = self.similarity.workers.clamp(1, 16); self } } @@ -80,6 +124,20 @@ fn default_volume() -> u8 { 80 } +fn default_similarity_model() -> String { + "discogs-effnet-bsdynamic-1".to_string() +} + +fn default_similarity_profile() -> String { + "furumi-full-track-v1".to_string() +} + +fn default_similarity_workers() -> usize { + std::thread::available_parallelism() + .map(|count| (count.get() / 2).clamp(1, 4)) + .unwrap_or(1) +} + /// The historical permanent-download location, kept as the default for /// backward compatibility with existing installations. pub fn default_music_dir() -> PathBuf { diff --git a/src/federation/capabilities.rs b/src/federation/capabilities.rs index b3aa935..604afe9 100644 --- a/src/federation/capabilities.rs +++ b/src/federation/capabilities.rs @@ -170,6 +170,7 @@ mod tests { "music_dht", "catalog", "audio", + "similarity", "device_sync", "jam", ] { diff --git a/src/federation/mod.rs b/src/federation/mod.rs index 17c613a..ce87617 100644 --- a/src/federation/mod.rs +++ b/src/federation/mod.rs @@ -15,6 +15,7 @@ mod audio; mod capabilities; pub mod catalog; +mod similarity; use std::collections::{HashMap, VecDeque}; use std::path::{Path, PathBuf}; @@ -39,6 +40,7 @@ use crate::library::models::{ArtistRef, TrackItem}; pub use audio::{AUDIO_ALPN, DownloadProgress, StreamingStart, TrackMetadata}; pub use capabilities::ProtocolVersions; pub use catalog::{CATALOG_ALPN, FedAppearsOn, FedArtistCard, FedCardTrack, FedRelease}; +pub use similarity::SIMILARITY_ALPN; /// How often the published library is re-synchronized with the local index. const SYNC_INTERVAL: Duration = Duration::from_secs(60); @@ -420,6 +422,7 @@ pub struct Federation { library: Arc, devices: Arc, jam: Arc, + similarity: Arc, data_dir: PathBuf, cache_dir: PathBuf, media_dir: std::sync::Mutex, @@ -526,6 +529,7 @@ impl Federation { library: Arc, devices: Arc, jam: Arc, + similarity: Arc, media_dir: PathBuf, ) -> Arc { let dirs = crate::config::project_dirs(); @@ -549,6 +553,7 @@ impl Federation { library, devices, jam, + similarity, data_dir, cache_dir, media_dir: std::sync::Mutex::new(media_dir), @@ -692,6 +697,8 @@ impl Federation { .stream_protocol(AUDIO_ALPN) // ...and browse each other's per-artist catalogs over this one. .stream_protocol(CATALOG_ALPN) + // Anonymous, bounded direct embedding queries. + .stream_protocol(SIMILARITY_ALPN) // Personal-device sync (likes, playlists, trusted devices). .stream_protocol(crate::devices::SYNC_ALPN) // Capability-scoped shared playback control. @@ -746,6 +753,15 @@ impl Federation { service.endpoint_id(), Arc::clone(&self.transport_stats), )); + let similarity_acceptor = service + .stream_acceptor(SIMILARITY_ALPN) + .map_err(|err| anyhow::anyhow!("failed to take the similarity acceptor: {err}"))?; + let similarity_task = tokio::spawn(similarity::serve_peers( + similarity_acceptor, + Arc::clone(&self.similarity), + service.endpoint_id(), + Arc::clone(&self.transport_stats), + )); let sync_acceptor = service .stream_acceptor(crate::devices::SYNC_ALPN) .map_err(|err| anyhow::anyhow!("failed to take the device-sync acceptor: {err}"))?; @@ -788,6 +804,7 @@ impl Federation { sync_task, audio_task, catalog_task, + similarity_task, device_sync_task, device_tick_task, jam_serve_task, @@ -1147,6 +1164,21 @@ impl Federation { Ok(FedSearchResults { artists, tracks }) } + /// Bounded fan-out to known peers using the exact model/profile + /// fingerprint carried with the query. No DHT records are written. + pub async fn search_similar( + &self, + query: crate::similarity::QueryVector, + limit: usize, + ) -> Result { + anyhow::ensure!( + self.similarity.network_allowed(), + "similarity federation has no consent" + ); + let service = self.service().await?; + similarity::search(service, query, limit, Arc::clone(&self.transport_stats)).await + } + /// Resolves a share-link content id to one playable federated track. /// /// Resolution order: the in-session metadata cache, the DHT content key, diff --git a/src/federation/similarity.rs b/src/federation/similarity.rs new file mode 100644 index 0000000..263a167 --- /dev/null +++ b/src/federation/similarity.rs @@ -0,0 +1,277 @@ +//! Furumi policy and local-index adapter for the shared similarity protocol. +//! +//! `music_dht::similarity` owns the versioned wire contract and framing. This +//! module owns application policy: consent, peer fan-out, local index access, +//! result conversion, deduplication, and ranking limits. + +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; +use std::time::Duration; + +use anyhow::{Context as _, Result}; +use futures_util::stream::{self, StreamExt as _}; +use music_dht::similarity::{self as wire, SimilarityHit, SimilarityRequest, SimilarityResponse}; +use music_dht::{ByteStream, EndpointId, ItemId, ItemKind, MusicDhtService, StreamAcceptor}; + +use crate::federation::{FedSearchResults, FedTrack, TransportStats}; +use crate::similarity::{Manager, QueryVector}; + +pub use music_dht::similarity::SIMILARITY_ALPN; + +const MAX_QUERY_PEERS: usize = 16; +const QUERY_CONCURRENCY: usize = 6; +const QUERY_TIMEOUT: Duration = Duration::from_secs(5); +const MAX_PER_ARTIST: usize = 3; +const MAX_NEAR_DUPLICATE_SIGNATURE_DISTANCE: u32 = 8; + +pub async fn serve_peers( + mut acceptor: StreamAcceptor, + similarity: Arc, + own: EndpointId, + transport: Arc, +) { + while let Some(stream) = acceptor.accept().await { + let similarity = Arc::clone(&similarity); + let transport = Arc::clone(&transport); + tokio::spawn(async move { + let peer = stream.peer_id; + if let Err(err) = serve_one(stream, similarity, own, transport).await { + tracing::warn!(peer = %peer, "similarity request failed: {err:#}"); + } + }); + } +} + +async fn serve_one( + mut stream: ByteStream, + similarity: Arc, + own: EndpointId, + transport: Arc, +) -> Result<()> { + super::record_stream_transport(&transport, "similarity", "inbound", "open", &stream); + let request = wire::read_request(&mut stream).await?; + let response = if !similarity.network_allowed() { + SimilarityResponse::refused("similarity federation is disabled or has no privacy consent")? + } else { + let profile = request.profile_id; + let vector = request.vector; + let limit = request.limit; + let matches = tokio::task::spawn_blocking(move || { + similarity.search_vector(&profile, &vector, None, None, limit) + }) + .await + .context("local similarity task failed") + .and_then(|result| result); + match matches { + Ok(matches) => { + let hits = matches + .into_iter() + .filter_map(|found| { + let track = found.track; + let hit = SimilarityHit { + score: found.score, + item_id: super::audio::hex_encode( + ItemId::derive( + &own, + ItemKind::Track, + &format!("track:{}", track.id), + ) + .as_bytes(), + ), + title: track.title, + artist_names: track + .artists + .into_iter() + .map(|artist| artist.name) + .collect(), + featured_artist_names: track + .featured_artists + .into_iter() + .map(|artist| artist.name) + .collect(), + year: track.release_year, + duration_seconds: Some(track.duration_seconds.round() as i64), + content_id: track.content_id, + release_title: Some(track.release_title), + track_number: track.track_number, + disc_number: track.disc_number, + embedding_signature: Some(found.embedding_signature), + }; + match hit.validate() { + Ok(()) => Some(hit), + Err(err) => { + tracing::debug!(%err, "invalid local similarity metadata skipped"); + None + } + } + }) + .collect(); + SimilarityResponse::success(hits)? + } + Err(err) => { + SimilarityResponse::refused(format!("similarity query is unavailable: {err:#}"))? + } + } + }; + wire::write_response(&mut stream, &response).await?; + stream.send.finish()?; + let _ = stream.send.stopped().await; + super::record_stream_transport(&transport, "similarity", "inbound", "done", &stream); + Ok(()) +} + +pub async fn search( + service: Arc, + query: QueryVector, + limit: usize, + transport: Arc, +) -> Result { + let own = service.endpoint_id(); + let mut peers = Vec::new(); + let mut seen = HashSet::new(); + for peer in service + .connected_peers() + .into_iter() + .chain(service.known_peers().into_iter().map(|peer| peer.peer_id)) + { + if peer != own && seen.insert(peer) { + peers.push(peer); + } + if peers.len() >= MAX_QUERY_PEERS { + break; + } + } + let query_signature = wire::embedding_signature(&query.vector)?; + let request = Arc::new(SimilarityRequest::new( + query.profile_id, + query.vector, + limit.clamp(1, wire::MAX_SIMILARITY_RESULTS), + )?); + + let responses = stream::iter(peers.into_iter().map(|peer| { + let service = Arc::clone(&service); + let request = Arc::clone(&request); + let transport = Arc::clone(&transport); + async move { + tokio::time::timeout( + QUERY_TIMEOUT, + query_peer(service, peer, &request, transport), + ) + .await + .map_err(|_| anyhow::anyhow!("similarity peer timed out"))? + } + })) + .buffer_unordered(QUERY_CONCURRENCY) + .collect::>() + .await; + + let mut hits = Vec::new(); + for response in responses { + match response { + Ok(peer_hits) => hits.extend(peer_hits), + Err(err) => tracing::debug!(%err, "similarity peer query skipped"), + } + } + hits.sort_by(|left, right| right.1.total_cmp(&left.1)); + let mut dedup = HashSet::new(); + let mut embedding_signatures = vec![query_signature]; + let mut artist_counts: HashMap = HashMap::new(); + let mut tracks = Vec::new(); + for (track, _, embedding_signature) in hits { + if query + .source_content_id + .as_deref() + .is_some_and(|source| track.content_id.as_deref() == Some(source)) + { + continue; + } + let key = track + .content_id + .clone() + .unwrap_or_else(|| format!("{}:{}", track.owner, track.item_id)); + if !dedup.insert(key) { + continue; + } + if embedding_signature.is_some_and(|candidate| { + embedding_signatures.iter().any(|existing| { + wire::signature_distance(&candidate, existing) + <= MAX_NEAR_DUPLICATE_SIGNATURE_DISTANCE + }) + }) { + continue; + } + let artist = track + .artist_names + .first() + .map(|name| music_dht::normalize_name(name)) + .unwrap_or_default(); + let count = artist_counts.entry(artist.clone()).or_default(); + if !artist.is_empty() && *count >= MAX_PER_ARTIST { + continue; + } + *count += 1; + if let Some(signature) = embedding_signature { + embedding_signatures.push(signature); + } + tracks.push(track); + if tracks.len() >= limit.min(wire::MAX_SIMILARITY_RESULTS) { + break; + } + } + Ok(FedSearchResults { + artists: Vec::new(), + tracks, + }) +} + +async fn query_peer( + service: Arc, + owner: EndpointId, + request: &SimilarityRequest, + transport: Arc, +) -> Result< + Vec<( + FedTrack, + f32, + Option<[u8; wire::SIMILARITY_SIGNATURE_BYTES]>, + )>, +> { + let mut stream = service + .open_stream(owner, SIMILARITY_ALPN) + .await + .map_err(|err| anyhow::anyhow!("cannot reach similarity peer: {err}"))?; + super::record_stream_transport(&transport, "similarity", "outbound", "open", &stream); + let response = wire::exchange(&mut stream, request).await?; + super::record_stream_transport(&transport, "similarity", "outbound", "done", &stream); + anyhow::ensure!( + response.ok, + "peer refused similarity query: {}", + response.error.unwrap_or_default() + ); + Ok(response + .hits + .into_iter() + .map(|hit| { + let score = hit.score; + let embedding_signature = hit.embedding_signature; + ( + FedTrack { + item_id: hit.item_id, + owner: owner.to_string(), + own: false, + title: hit.title, + artist_names: hit.artist_names, + featured_artist_names: hit.featured_artist_names, + year: hit.year, + duration_seconds: hit.duration_seconds, + content_id: hit.content_id, + release_title: hit.release_title, + track_number: hit.track_number, + disc_number: hit.disc_number, + }, + score, + embedding_signature, + ) + }) + .collect()) +} diff --git a/src/library/mod.rs b/src/library/mod.rs index 3664a09..8d45d82 100644 --- a/src/library/mod.rs +++ b/src/library/mod.rs @@ -193,6 +193,26 @@ CREATE INDEX IF NOT EXISTS idx_network_artist_cache_kind ON network_artist_cache(source_kind, seen_at_ms); CREATE INDEX IF NOT EXISTS idx_network_artist_cache_artist ON network_artist_cache(artist_key); +CREATE TABLE IF NOT EXISTS similarity_profiles ( + profile_id TEXT PRIMARY KEY, + model_id TEXT NOT NULL, + model_version TEXT NOT NULL, + model_sha256 TEXT NOT NULL, + preprocessing TEXT NOT NULL, + dimensions INTEGER NOT NULL, + created_at_ms INTEGER NOT NULL +); +CREATE TABLE IF NOT EXISTS track_embeddings ( + track_id INTEGER NOT NULL REFERENCES tracks(id) ON DELETE CASCADE, + profile_id TEXT NOT NULL REFERENCES similarity_profiles(profile_id) ON DELETE CASCADE, + dimensions INTEGER NOT NULL, + vector BLOB NOT NULL, + source_content_id TEXT, + computed_at_ms INTEGER NOT NULL, + PRIMARY KEY (track_id, profile_id) +); +CREATE INDEX IF NOT EXISTS idx_track_embeddings_profile + ON track_embeddings(profile_id, track_id); "; /// The SELECT column list every TrackItem row is built from; artist lists @@ -271,6 +291,32 @@ pub struct NetworkArtistImageRequest { pub name: String, } +/// Minimal durable-track row used by the background embedding pipeline. +#[derive(Debug, Clone)] +pub struct SimilarityTrack { + pub id: i64, + pub title: String, + pub file_path: String, + pub content_id: Option, + pub duration_seconds: f64, +} + +/// One validated vector loaded from SQLite for the in-memory exact index. +#[derive(Debug, Clone)] +pub struct StoredEmbedding { + pub track_id: i64, + pub vector: Vec, + pub artist_key: String, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct SimilarityStorageStats { + pub total_tracks: usize, + pub embedded_tracks: usize, + pub stored_vectors: usize, + pub stored_bytes: u64, +} + pub struct Library { conn: Mutex, db_path: PathBuf, @@ -1433,6 +1479,192 @@ impl Library { Ok(tracks.pop()) } + // ----------------------------------------------------------------- + // Similarity embeddings. SQLite is the canonical store; callers build + // replaceable in-memory indexes from these rows. + // ----------------------------------------------------------------- + + pub fn ensure_similarity_profile( + &self, + profile_id: &str, + model_id: &str, + model_version: &str, + model_sha256: &str, + preprocessing: &str, + dimensions: usize, + ) -> Result<()> { + let conn = self.lock(); + conn.execute( + "INSERT INTO similarity_profiles ( + profile_id, model_id, model_version, model_sha256, + preprocessing, dimensions, created_at_ms + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7) + ON CONFLICT(profile_id) DO NOTHING", + params![ + profile_id, + model_id, + model_version, + model_sha256, + preprocessing, + dimensions as i64, + now_ms_i64(), + ], + )?; + Ok(()) + } + + pub fn pending_similarity_tracks(&self, profile_id: &str) -> Result> { + let conn = self.lock(); + let mut statement = conn.prepare( + "SELECT t.id, t.title, t.file_path, t.content_id, t.duration_seconds + FROM tracks t + WHERE NOT EXISTS ( + SELECT 1 FROM track_embeddings e + WHERE e.track_id = t.id + AND e.profile_id = ?1 + AND e.source_content_id IS t.content_id + ) + ORDER BY t.id", + )?; + Ok(statement + .query_map([profile_id], |row| { + Ok(SimilarityTrack { + id: row.get(0)?, + title: row.get(1)?, + file_path: row.get(2)?, + content_id: row.get(3)?, + duration_seconds: row.get(4)?, + }) + })? + .collect::>>()?) + } + + pub fn store_similarity_embedding( + &self, + track: &SimilarityTrack, + profile_id: &str, + vector: &[f32], + ) -> Result<()> { + anyhow::ensure!(!vector.is_empty(), "embedding vector is empty"); + anyhow::ensure!( + vector.iter().all(|value| value.is_finite()), + "embedding contains a non-finite value" + ); + let bytes = embedding_to_bytes(vector); + let conn = self.lock(); + conn.execute( + "INSERT INTO track_embeddings ( + track_id, profile_id, dimensions, vector, + source_content_id, computed_at_ms + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6) + ON CONFLICT(track_id, profile_id) DO UPDATE SET + dimensions = excluded.dimensions, + vector = excluded.vector, + source_content_id = excluded.source_content_id, + computed_at_ms = excluded.computed_at_ms", + params![ + track.id, + profile_id, + vector.len() as i64, + bytes, + track.content_id.as_deref(), + now_ms_i64(), + ], + )?; + Ok(()) + } + + pub fn similarity_embedding( + &self, + track_id: i64, + profile_id: &str, + ) -> Result>> { + let conn = self.lock(); + let row = conn + .query_row( + "SELECT e.dimensions, e.vector + FROM track_embeddings e + JOIN tracks t ON t.id = e.track_id + WHERE e.track_id = ?1 + AND e.profile_id = ?2 + AND e.source_content_id IS t.content_id", + params![track_id, profile_id], + |row| Ok((row.get::<_, i64>(0)?, row.get::<_, Vec>(1)?)), + ) + .optional()?; + row.map(|(dimensions, bytes)| embedding_from_bytes(dimensions, &bytes)) + .transpose() + } + + pub fn load_similarity_index(&self, profile_id: &str) -> Result> { + let conn = self.lock(); + let mut statement = conn.prepare( + "SELECT e.track_id, e.dimensions, e.vector, + COALESCE(( + SELECT norm(a.name) + FROM track_artists ta + JOIN artists a ON a.id = ta.artist_id + WHERE ta.track_id = e.track_id AND ta.role = 'main' + ORDER BY ta.position LIMIT 1 + ), '') + FROM track_embeddings e + JOIN tracks t ON t.id = e.track_id + WHERE e.profile_id = ?1 + AND e.source_content_id IS t.content_id + ORDER BY e.track_id", + )?; + let rows = statement.query_map([profile_id], |row| { + Ok(( + row.get::<_, i64>(0)?, + row.get::<_, i64>(1)?, + row.get::<_, Vec>(2)?, + row.get::<_, String>(3)?, + )) + })?; + let mut embeddings = Vec::new(); + for row in rows { + let (track_id, dimensions, bytes, artist_key) = row?; + embeddings.push(StoredEmbedding { + track_id, + vector: embedding_from_bytes(dimensions, &bytes)?, + artist_key, + }); + } + Ok(embeddings) + } + + pub fn similarity_storage_stats(&self, profile_id: &str) -> Result { + let conn = self.lock(); + let total_tracks = conn.query_row("SELECT COUNT(*) FROM tracks", [], |row| { + row.get::<_, i64>(0) + })?; + let embedded_tracks = conn.query_row( + "SELECT COUNT(*) + FROM track_embeddings e JOIN tracks t ON t.id = e.track_id + WHERE e.profile_id = ?1 AND e.source_content_id IS t.content_id", + [profile_id], + |row| row.get::<_, i64>(0), + )?; + let (stored_vectors, stored_bytes) = conn.query_row( + "SELECT COUNT(*), COALESCE(SUM(length(vector)), 0) FROM track_embeddings", + [], + |row| Ok((row.get::<_, i64>(0)?, row.get::<_, i64>(1)?)), + )?; + Ok(SimilarityStorageStats { + total_tracks: total_tracks.max(0) as usize, + embedded_tracks: embedded_tracks.max(0) as usize, + stored_vectors: stored_vectors.max(0) as usize, + stored_bytes: stored_bytes.max(0) as u64, + }) + } + + pub fn clear_similarity_embeddings(&self) -> Result<()> { + let conn = self.lock(); + conn.execute("DELETE FROM track_embeddings", [])?; + conn.execute("DELETE FROM similarity_profiles", [])?; + Ok(()) + } + // ----------------------------------------------------------------- // Playlists & likes // ----------------------------------------------------------------- @@ -3131,6 +3363,32 @@ fn now_ms_i64() -> i64 { .unwrap_or(0) } +fn embedding_to_bytes(vector: &[f32]) -> Vec { + let mut bytes = Vec::with_capacity(std::mem::size_of_val(vector)); + for value in vector { + bytes.extend_from_slice(&value.to_le_bytes()); + } + bytes +} + +fn embedding_from_bytes(dimensions: i64, bytes: &[u8]) -> Result> { + anyhow::ensure!(dimensions > 0, "stored embedding has invalid dimensions"); + let dimensions = dimensions as usize; + anyhow::ensure!( + bytes.len() == dimensions * std::mem::size_of::(), + "stored embedding byte length does not match its dimensions" + ); + let vector: Vec = bytes + .chunks_exact(4) + .map(|chunk| f32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]])) + .collect(); + anyhow::ensure!( + vector.iter().all(|value| value.is_finite()), + "stored embedding contains a non-finite value" + ); + Ok(vector) +} + fn remote_artist_id(artist_key: &str) -> i64 { let hash = blake3::hash(artist_key.as_bytes()); let mut bytes = [0u8; 8]; diff --git a/src/library/tests.rs b/src/library/tests.rs index 2381d40..4977e67 100644 --- a/src/library/tests.rs +++ b/src/library/tests.rs @@ -686,3 +686,78 @@ fn listen_history_hides_unqualified_events_and_keeps_remote_metadata() { assert!(lib.apply_listen_event(&event, "remote-device").unwrap()); assert!(lib.listen_history(20).unwrap().is_empty()); } + +#[test] +fn similarity_embeddings_round_trip_and_keep_profiles_separate() { + let lib = test_library(); + let track_id = add_track(&lib, "Song", "Artist", "Album"); + for profile in ["profile-a", "profile-b"] { + lib.ensure_similarity_profile(profile, "model", "1", "sha", "prep", 3) + .unwrap(); + } + let track = lib + .pending_similarity_tracks("profile-a") + .unwrap() + .into_iter() + .find(|track| track.id == track_id) + .unwrap(); + lib.store_similarity_embedding(&track, "profile-a", &[0.1, 0.2, 0.3]) + .unwrap(); + lib.store_similarity_embedding(&track, "profile-b", &[0.3, 0.2, 0.1]) + .unwrap(); + + assert_eq!( + lib.similarity_embedding(track_id, "profile-a").unwrap(), + Some(vec![0.1, 0.2, 0.3]) + ); + assert_eq!( + lib.similarity_embedding(track_id, "profile-b").unwrap(), + Some(vec![0.3, 0.2, 0.1]) + ); + let stats = lib.similarity_storage_stats("profile-a").unwrap(); + assert_eq!(stats.total_tracks, 1); + assert_eq!(stats.embedded_tracks, 1); + assert_eq!(stats.stored_vectors, 2); + assert_eq!(stats.stored_bytes, 24); +} + +#[test] +fn changed_content_id_invalidates_only_the_stale_embedding() { + let lib = test_library(); + let track_id = add_track(&lib, "Song", "Artist", "Album"); + lib.ensure_similarity_profile("profile", "model", "1", "sha", "prep", 2) + .unwrap(); + let track = lib.pending_similarity_tracks("profile").unwrap().remove(0); + lib.store_similarity_embedding(&track, "profile", &[0.6, 0.8]) + .unwrap(); + lib.lock() + .execute( + "UPDATE tracks SET content_id = ?2 WHERE id = ?1", + params![track_id, format!("b3:{}", "f".repeat(64))], + ) + .unwrap(); + + assert_eq!(lib.pending_similarity_tracks("profile").unwrap().len(), 1); + assert_eq!(lib.similarity_embedding(track_id, "profile").unwrap(), None); + assert!(lib.load_similarity_index("profile").unwrap().is_empty()); +} + +#[test] +fn clearing_embeddings_preserves_the_library() { + let lib = test_library(); + let track_id = add_track(&lib, "Song", "Artist", "Album"); + lib.ensure_similarity_profile("profile", "model", "1", "sha", "prep", 2) + .unwrap(); + let track = lib.pending_similarity_tracks("profile").unwrap().remove(0); + lib.store_similarity_embedding(&track, "profile", &[0.6, 0.8]) + .unwrap(); + + lib.clear_similarity_embeddings().unwrap(); + + assert_eq!(lib.tracks_by_ids(&[track_id]).unwrap().len(), 1); + assert!( + lib.similarity_embedding(track_id, "profile") + .unwrap() + .is_none() + ); +} diff --git a/src/main.rs b/src/main.rs index f186197..f40ea1e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -8,6 +8,7 @@ mod library; mod media; mod player; mod share; +mod similarity; mod status; mod streaming; mod ui; diff --git a/src/similarity.rs b/src/similarity.rs new file mode 100644 index 0000000..4a90370 --- /dev/null +++ b/src/similarity.rs @@ -0,0 +1,1047 @@ +//! Local, offline-first music embeddings and exact cosine search. +//! +//! SQLite owns the durable vectors. The in-memory index is deliberately +//! replaceable: it is rebuilt for the active profile and never becomes a +//! second source of truth. + +use std::collections::{HashMap, VecDeque}; +use std::fs::File; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex, RwLock}; +use std::time::Instant; + +use anyhow::{Context as _, Result}; +use futures_util::StreamExt as _; +use rodio::{Decoder, Source as _}; +use rustfft::FftPlanner; +use rustfft::num_complex::Complex; +use sha2::{Digest as _, Sha256}; +use tokio::io::AsyncWriteExt as _; +use tract_onnx::prelude::*; +use tract_onnx::tract_core::dims; + +use crate::app::event::AppEvent; +use crate::config::settings::SimilaritySettings; +use crate::library::models::TrackItem; +use crate::library::{Library, StoredEmbedding}; + +const SAMPLE_RATE: usize = 16_000; +const FRAME_SIZE: usize = 512; +const HOP_SIZE: usize = 256; +const MEL_BANDS: usize = 96; +const PATCH_FRAMES: usize = 128; +const PATCH_HOP: usize = 62; +const EMBEDDING_DIMENSIONS: usize = 1280; +const MODEL_BATCH: usize = 8; +const MAX_MODEL_BYTES: usize = 64 * 1024 * 1024; +const RESULT_LIMIT: usize = 50; +const MAX_PER_ARTIST: usize = 3; +const NEAR_DUPLICATE_COSINE: f32 = 0.995; +const FULL_TRACK_MAX_SECONDS: u32 = 5 * 60; +const LONG_TRACK_WINDOW_SECONDS: u32 = 60; +const LONG_TRACK_WINDOWS: usize = 3; + +pub const DEFAULT_MODEL_ID: &str = "discogs-effnet-bsdynamic-1"; +pub const DEFAULT_PROFILE_ID: &str = "furumi-full-track-v1"; + +#[derive(Debug, Clone, Copy)] +pub struct ProfileSpec { + pub id: &'static str, + pub title: &'static str, +} + +pub const PROFILES: &[ProfileSpec] = &[ProfileSpec { + id: DEFAULT_PROFILE_ID, + title: "Full track / balanced long track", +}]; + +#[derive(Debug, Clone, Copy)] +pub struct ModelSpec { + pub id: &'static str, + pub version: &'static str, + pub filename: &'static str, + pub url: &'static str, + pub sha256: &'static str, + pub dimensions: usize, + pub license: &'static str, +} + +pub const MODELS: &[ModelSpec] = &[ModelSpec { + id: DEFAULT_MODEL_ID, + version: "1", + filename: "discogs-effnet-bsdynamic-1.onnx", + url: "https://essentia.upf.edu/models/feature-extractors/discogs-effnet/discogs-effnet-bsdynamic-1.onnx", + sha256: "a280825b334797cf677939db8cd5762c0392aedd0ca6415dbc1cd083f045e43c", + dimensions: EMBEDDING_DIMENSIONS, + license: "CC BY-NC-SA 4.0 (or proprietary from MTG)", +}]; + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum Phase { + #[default] + Disabled, + Downloading, + Loading, + Processing, + Ready, + Error, +} + +impl Phase { + pub fn label(self) -> &'static str { + match self { + Self::Disabled => "disabled", + Self::Downloading => "downloading model", + Self::Loading => "loading model", + Self::Processing => "processing", + Self::Ready => "ready", + Self::Error => "error", + } + } +} + +#[derive(Debug, Clone, Default)] +pub struct SimilarityStatus { + pub phase: Phase, + pub active_profile: Option, + pub target_profile: Option, + pub model: String, + pub total_tracks: usize, + pub completed_tracks: usize, + pub failed_tracks: usize, + pub stored_vectors: usize, + pub stored_bytes: u64, + pub current_track: Option, + pub last_error: Option, +} + +#[derive(Debug, Clone)] +pub struct QueryVector { + pub profile_id: String, + pub vector: Vec, + pub source_content_id: Option, +} + +#[derive(Debug, Clone)] +pub struct SimilarTrack { + pub track: TrackItem, + pub score: f32, + pub embedding_signature: [u8; music_dht::similarity::SIMILARITY_SIGNATURE_BYTES], +} + +#[derive(Default)] +struct Index { + profile_id: Option, + entries: Vec, +} + +type RunnableModel = Arc; + +pub struct Manager { + library: Arc, + event_tx: tokio::sync::mpsc::UnboundedSender, + settings: Mutex, + workers: AtomicUsize, + generation: AtomicU64, + status: Mutex, + index: RwLock, + model: Mutex>, + model_dir: PathBuf, +} + +impl Manager { + pub fn new( + library: Arc, + event_tx: tokio::sync::mpsc::UnboundedSender, + settings: SimilaritySettings, + ) -> Arc { + let model_dir = crate::config::project_dirs() + .map(|dirs| dirs.cache_dir().join("similarity-models")) + .unwrap_or_else(|| PathBuf::from("similarity-models")); + let mut index = Index::default(); + if let Some(profile_id) = settings.active_profile.as_deref() { + match library.load_similarity_index(profile_id) { + Ok(entries) => { + index.profile_id = Some(profile_id.to_string()); + index.entries = entries; + } + Err(err) => tracing::warn!(%err, "similarity index restore failed"), + } + } + let status = SimilarityStatus { + phase: if settings.enabled { + Phase::Loading + } else { + Phase::Disabled + }, + active_profile: index.profile_id.clone(), + model: settings.model.clone(), + ..SimilarityStatus::default() + }; + Arc::new(Self { + library, + event_tx, + workers: AtomicUsize::new(settings.workers.clamp(1, 16)), + generation: AtomicU64::new(0), + settings: Mutex::new(settings), + status: Mutex::new(status), + index: RwLock::new(index), + model: Mutex::new(None), + model_dir, + }) + } + + pub fn settings(&self) -> SimilaritySettings { + lock(&self.settings).clone() + } + + pub fn status(&self) -> SimilarityStatus { + lock(&self.status).clone() + } + + pub fn network_allowed(&self) -> bool { + let settings = lock(&self.settings); + settings.enabled && settings.federation_consent + } + + pub fn apply(self: &Arc, settings: SimilaritySettings) { + self.workers + .store(settings.workers.clamp(1, 16), Ordering::Release); + let previous = std::mem::replace(&mut *lock(&self.settings), settings.clone()); + if !settings.enabled { + self.generation.fetch_add(1, Ordering::AcqRel); + self.update_status(|status| { + status.phase = Phase::Disabled; + status.current_track = None; + status.target_profile = None; + status.last_error = None; + }); + return; + } + if !previous.enabled + || previous.model != settings.model + || previous.profile != settings.profile + { + self.start(); + } + } + + pub fn start(self: &Arc) { + let generation = self.generation.fetch_add(1, Ordering::AcqRel) + 1; + let this = Arc::clone(self); + tokio::spawn(async move { + if let Err(err) = this.run_pipeline(generation).await + && this.generation.load(Ordering::Acquire) == generation + { + tracing::error!(%err, "similarity pipeline failed"); + this.update_status(|status| { + status.phase = Phase::Error; + status.current_track = None; + status.last_error = Some(format!("{err:#}")); + }); + } + }); + } + + pub fn clear(self: &Arc) { + self.generation.fetch_add(1, Ordering::AcqRel); + let this = Arc::clone(self); + tokio::spawn(async move { + let library = Arc::clone(&this.library); + let result = tokio::task::spawn_blocking(move || library.clear_similarity_embeddings()) + .await + .context("embedding clear task failed") + .and_then(|result| result); + match result { + Ok(()) => { + *write(&this.index) = Index::default(); + lock(&this.settings).active_profile = None; + this.update_status(|status| { + *status = SimilarityStatus { + phase: if this.settings().enabled { + Phase::Loading + } else { + Phase::Disabled + }, + model: this.settings().model, + ..SimilarityStatus::default() + }; + }); + let _ = this + .event_tx + .send(AppEvent::SimilarityProfileActivated(None)); + if this.settings().enabled { + this.start(); + } + } + Err(err) => this.update_status(|status| { + status.phase = Phase::Error; + status.last_error = Some(format!("clear failed: {err:#}")); + }), + } + }); + } + + pub fn query_for_track(&self, track_id: i64) -> Result { + let profile_id = read(&self.index) + .profile_id + .clone() + .context("no similarity profile is ready yet")?; + let vector = self + .library + .similarity_embedding(track_id, &profile_id)? + .context("this track has not been processed yet")?; + let source_track = self.library.tracks_by_ids(&[track_id])?.into_iter().next(); + let source_content_id = source_track.as_ref().and_then(|track| { + track + .content_id + .clone() + .or_else(|| crate::library::audio_content_id(&track.file_path)) + }); + Ok(QueryVector { + profile_id, + vector, + source_content_id, + }) + } + + pub fn search_track( + &self, + track_id: i64, + limit: usize, + ) -> Result<(Vec, QueryVector)> { + let query = self.query_for_track(track_id)?; + let matches = self.search_vector( + &query.profile_id, + &query.vector, + Some(track_id), + query.source_content_id.as_deref(), + limit, + )?; + Ok((matches, query)) + } + + pub fn search_vector( + &self, + profile_id: &str, + vector: &[f32], + exclude_track_id: Option, + exclude_content_id: Option<&str>, + limit: usize, + ) -> Result> { + anyhow::ensure!( + !vector.is_empty() && vector.len() <= 4096, + "wrong embedding dimensions" + ); + anyhow::ensure!( + vector.iter().all(|value| value.is_finite()), + "invalid embedding" + ); + let index = read(&self.index); + anyhow::ensure!( + index.profile_id.as_deref() == Some(profile_id), + "the requested similarity profile is not active" + ); + let mut scores: Vec<(i64, f32, &str, &[f32])> = index + .entries + .iter() + .filter(|entry| { + Some(entry.track_id) != exclude_track_id && entry.vector.len() == vector.len() + }) + .map(|entry| { + ( + entry.track_id, + dot(vector, &entry.vector), + entry.artist_key.as_str(), + entry.vector.as_slice(), + ) + }) + .filter(|(_, score, _, _)| score.is_finite()) + .collect(); + scores.sort_by(|left, right| right.1.total_cmp(&left.1)); + + // Pull a wider candidate set, then cap each primary artist so a large + // discography cannot fill the whole result page. + let mut artist_counts: HashMap = HashMap::new(); + let mut kept_vectors = vec![vector]; + let mut selected = Vec::new(); + for (track_id, score, artist, candidate_vector) in scores { + if is_near_duplicate(candidate_vector, &kept_vectors) { + continue; + } + let count = artist_counts.entry(artist.to_string()).or_default(); + if !artist.is_empty() && *count >= MAX_PER_ARTIST { + continue; + } + *count += 1; + let embedding_signature = music_dht::similarity::embedding_signature(candidate_vector)?; + kept_vectors.push(candidate_vector); + selected.push((track_id, score, embedding_signature)); + if selected.len() >= limit.min(RESULT_LIMIT) { + break; + } + } + drop(index); + + let ids: Vec = selected.iter().map(|(id, _, _)| *id).collect(); + let tracks = self.library.tracks_by_ids(&ids)?; + let by_id: HashMap = + tracks.into_iter().map(|track| (track.id, track)).collect(); + Ok(selected + .into_iter() + .filter_map(|(id, score, signature)| { + by_id + .get(&id) + .cloned() + .map(|track| (track, score, signature)) + }) + .filter(|(track, _, _)| { + !exclude_content_id + .is_some_and(|source| track.content_id.as_deref() == Some(source)) + }) + .map(|(track, score, embedding_signature)| SimilarTrack { + track, + score, + embedding_signature, + }) + .collect()) + } + + async fn run_pipeline(self: &Arc, generation: u64) -> Result<()> { + let settings = self.settings(); + if !settings.enabled { + return Ok(()); + } + let spec = model_by_id(&settings.model) + .with_context(|| format!("unknown similarity model '{}'", settings.model))?; + anyhow::ensure!( + profile_by_id(&settings.profile).is_some(), + "unknown preprocessing profile '{}'", + settings.profile + ); + let profile_id = profile_fingerprint(spec, &settings.profile); + self.library.ensure_similarity_profile( + &profile_id, + spec.id, + spec.version, + spec.sha256, + &settings.profile, + spec.dimensions, + )?; + let stats = self.library.similarity_storage_stats(&profile_id)?; + self.update_status(|status| { + status.phase = Phase::Downloading; + status.target_profile = Some(profile_id.clone()); + status.model = spec.id.to_string(); + status.total_tracks = stats.total_tracks; + status.completed_tracks = stats.embedded_tracks; + status.stored_vectors = stats.stored_vectors; + status.stored_bytes = stats.stored_bytes; + status.failed_tracks = 0; + status.current_track = None; + status.last_error = None; + }); + let model_path = self.ensure_model(spec, generation).await?; + self.ensure_generation(generation)?; + self.update_status(|status| status.phase = Phase::Loading); + let model = self.load_model(&profile_id, &model_path).await?; + self.ensure_generation(generation)?; + + let mut pending: VecDeque<_> = self.library.pending_similarity_tracks(&profile_id)?.into(); + let pending_total = pending.len(); + self.update_status(|status| { + status.phase = if pending_total == 0 { + Phase::Loading + } else { + Phase::Processing + }; + }); + + let mut jobs = tokio::task::JoinSet::new(); + while !pending.is_empty() || !jobs.is_empty() { + self.ensure_generation(generation)?; + let workers = self.workers.load(Ordering::Acquire).clamp(1, 16); + while jobs.len() < workers { + let Some(track) = pending.pop_front() else { + break; + }; + self.update_status(|status| status.current_track = Some(track.title.clone())); + let library = Arc::clone(&self.library); + let model = Arc::clone(&model); + let profile_id = profile_id.clone(); + jobs.spawn_blocking(move || { + let started = Instant::now(); + let result = + embed_track(&model, Path::new(&track.file_path), track.duration_seconds) + .and_then(|vector| { + library.store_similarity_embedding(&track, &profile_id, &vector) + }); + (track, result, started.elapsed()) + }); + } + let Some(result) = jobs.join_next().await else { + continue; + }; + let (track, result, elapsed) = result.context("embedding worker panicked")?; + self.ensure_generation(generation)?; + match result { + Ok(()) => { + tracing::info!( + track_id = track.id, + title = %track.title, + elapsed_ms = elapsed.as_millis(), + profile = %profile_id, + "track embedding calculated" + ); + self.update_status(|status| status.completed_tracks += 1); + } + Err(err) => { + tracing::warn!(track_id = track.id, title = %track.title, %err, "track embedding failed"); + self.update_status(|status| { + status.failed_tracks += 1; + status.last_error = Some(format!("{}: {err:#}", track.title)); + }); + } + } + } + self.ensure_generation(generation)?; + let entries = self.library.load_similarity_index(&profile_id)?; + let total_tracks = self + .library + .similarity_storage_stats(&profile_id)? + .total_tracks; + anyhow::ensure!( + total_tracks == 0 || !entries.is_empty(), + "no tracks could be processed" + ); + *write(&self.index) = Index { + profile_id: Some(profile_id.clone()), + entries, + }; + lock(&self.settings).active_profile = Some(profile_id.clone()); + let stats = self.library.similarity_storage_stats(&profile_id)?; + self.update_status(|status| { + status.phase = Phase::Ready; + status.active_profile = Some(profile_id.clone()); + status.target_profile = Some(profile_id.clone()); + status.total_tracks = stats.total_tracks; + status.completed_tracks = stats.embedded_tracks; + status.stored_vectors = stats.stored_vectors; + status.stored_bytes = stats.stored_bytes; + status.current_track = None; + }); + let _ = self + .event_tx + .send(AppEvent::SimilarityProfileActivated(Some(profile_id))); + Ok(()) + } + + fn ensure_generation(&self, generation: u64) -> Result<()> { + anyhow::ensure!( + self.generation.load(Ordering::Acquire) == generation, + "similarity processing superseded by newer settings" + ); + Ok(()) + } + + async fn ensure_model(&self, spec: &ModelSpec, generation: u64) -> Result { + tokio::fs::create_dir_all(&self.model_dir).await?; + let path = self.model_dir.join(spec.filename); + if path.exists() { + let verify_path = path.clone(); + let expected = spec.sha256.to_string(); + let valid = tokio::task::spawn_blocking(move || sha256_file(&verify_path)) + .await + .context("model hash task failed")?? + == expected; + if valid { + return Ok(path); + } + tokio::fs::remove_file(&path).await?; + } + + let response = reqwest::get(spec.url).await?.error_for_status()?; + let tmp = path.with_extension(format!("part-{}-{generation}", std::process::id())); + let mut file = tokio::fs::File::create(&tmp).await?; + let mut hasher = Sha256::new(); + let mut received = 0usize; + let mut stream = response.bytes_stream(); + while let Some(chunk) = stream.next().await { + let chunk = chunk?; + received = received.saturating_add(chunk.len()); + anyhow::ensure!( + received <= MAX_MODEL_BYTES, + "model download exceeds size limit" + ); + hasher.update(&chunk); + file.write_all(&chunk).await?; + } + file.flush().await?; + drop(file); + let actual = format!("{:x}", hasher.finalize()); + if actual != spec.sha256 { + let _ = tokio::fs::remove_file(&tmp).await; + anyhow::bail!("downloaded model hash mismatch"); + } + if let Err(err) = tokio::fs::rename(&tmp, &path).await { + // A superseding pipeline may have installed the same verified + // artifact first. This is expected on platforms where rename + // does not replace an existing destination. + if path.exists() { + let _ = tokio::fs::remove_file(&tmp).await; + } else { + return Err(err.into()); + } + } + Ok(path) + } + + async fn load_model(&self, profile_id: &str, path: &Path) -> Result { + if let Some((cached_profile, model)) = lock(&self.model).as_ref() + && cached_profile == profile_id + { + return Ok(Arc::clone(model)); + } + let path = path.to_path_buf(); + let model = tokio::task::spawn_blocking(move || load_onnx(&path)) + .await + .context("model loading task failed")??; + *lock(&self.model) = Some((profile_id.to_string(), Arc::clone(&model))); + Ok(model) + } + + fn update_status(&self, update: impl FnOnce(&mut SimilarityStatus)) { + let snapshot = { + let mut status = lock(&self.status); + update(&mut status); + status.clone() + }; + let _ = self.event_tx.send(AppEvent::SimilarityStatus(snapshot)); + } +} + +pub fn model_by_id(id: &str) -> Option<&'static ModelSpec> { + MODELS.iter().find(|model| model.id == id) +} + +pub fn profile_by_id(id: &str) -> Option<&'static ProfileSpec> { + PROFILES.iter().find(|profile| profile.id == id) +} + +pub fn profile_details(profile_id: &str, model_id: &str) -> Option { + let profile = profile_by_id(profile_id)?; + let dimensions = model_by_id(model_id) + .map(|model| model.dimensions.to_string()) + .unwrap_or_else(|| "model-defined".to_string()); + Some(format!( + "{}\n\nTrack selection:\n• Up to {} seconds: entire track.\n• Longer: {} × {}-second windows (start, middle, end).\n\nAudio: mono, {} Hz; 16-tap windowed-sinc resampling.\nSpectrogram: Hann window; FFT {}, hop {} samples (16 ms).\nMel: {} Slaney bands, 0–8 kHz, unit-triangle normalization.\nCompression: log10(1 + 10000 × energy).\nPatches: {} frames (~2.05 s), hop {} frames (~0.99 s).\nAggregation: mean of patch embeddings, then L2 normalization.\nOutput dimensions for selected model: {}.\n\nCompatibility includes the exact model version and SHA-256; peers compare only matching profiles.\n\nEnter / Esc: close", + profile.title, + FULL_TRACK_MAX_SECONDS, + LONG_TRACK_WINDOWS, + LONG_TRACK_WINDOW_SECONDS, + SAMPLE_RATE, + FRAME_SIZE, + HOP_SIZE, + MEL_BANDS, + PATCH_FRAMES, + PATCH_HOP, + dimensions, + )) +} + +pub fn profile_fingerprint(model: &ModelSpec, profile: &str) -> String { + let contract = format!( + "furumi-similarity-v1\nmodel={}\nversion={}\nsha256={}\nprofile={}\ninput={}-mono-windowed-sinc16\nselection=full-to-{}s-else-first-middle-last-{}s\nframe={}\nhop={}\nmel=slaney-{}-unit-tri\npatch={}\npatch-hop={}\naggregate=mean-l2\ndimensions={}", + model.id, + model.version, + model.sha256, + profile, + SAMPLE_RATE, + FULL_TRACK_MAX_SECONDS, + LONG_TRACK_WINDOW_SECONDS, + FRAME_SIZE, + HOP_SIZE, + MEL_BANDS, + PATCH_FRAMES, + PATCH_HOP, + model.dimensions + ); + format!("sim1:{}", blake3::hash(contract.as_bytes()).to_hex()) +} + +fn load_onnx(path: &Path) -> Result { + let model = tract_onnx::onnx().model_for_path(path)?; + let batch = model.sym("batch_size"); + let model = model + .with_input_fact(0, f32::fact(dims!(batch, PATCH_FRAMES, MEL_BANDS)).into())? + .into_optimized()? + .into_runnable()?; + Ok(model) +} + +fn embed_track(model: &RunnableModel, path: &Path, duration_seconds: f64) -> Result> { + let signal = decode_mono_16k(path, duration_seconds)?; + let mel = mel_spectrogram(&signal)?; + anyhow::ensure!( + mel.len() >= PATCH_FRAMES, + "track is too short for the model" + ); + let starts: Vec = (0..=mel.len() - PATCH_FRAMES).step_by(PATCH_HOP).collect(); + let mut sum = vec![0.0f32; EMBEDDING_DIMENSIONS]; + let mut count = 0usize; + for batch in starts.chunks(MODEL_BATCH) { + let mut input = vec![0.0f32; MODEL_BATCH * PATCH_FRAMES * MEL_BANDS]; + for (batch_index, &start) in batch.iter().enumerate() { + let offset = batch_index * PATCH_FRAMES * MEL_BANDS; + for frame in 0..PATCH_FRAMES { + let dst = offset + frame * MEL_BANDS; + input[dst..dst + MEL_BANDS].copy_from_slice(&mel[start + frame]); + } + } + let tensor = Tensor::from_shape(&[MODEL_BATCH, PATCH_FRAMES, MEL_BANDS], &input)?; + let outputs = model.run(tvec!(tensor.into_tvalue()))?; + let embedding = outputs + .iter() + .find(|output| output.len() == MODEL_BATCH * EMBEDDING_DIMENSIONS) + .context("model did not return its 1280-dimensional embedding output")? + .to_plain_array_view::()?; + let values = embedding + .as_slice() + .context("model embedding output is not contiguous")?; + for batch_index in 0..batch.len() { + let row = &values + [batch_index * EMBEDDING_DIMENSIONS..(batch_index + 1) * EMBEDDING_DIMENSIONS]; + for (total, value) in sum.iter_mut().zip(row) { + *total += *value; + } + count += 1; + } + } + anyhow::ensure!(count > 0, "model produced no patches"); + for value in &mut sum { + *value /= count as f32; + } + normalize(&mut sum)?; + Ok(sum) +} + +fn decode_mono_16k(path: &Path, duration_seconds: f64) -> Result> { + if duration_seconds.is_finite() && duration_seconds > f64::from(FULL_TRACK_MAX_SECONDS) { + let window = f64::from(LONG_TRACK_WINDOW_SECONDS); + let starts = [ + 0.0, + (duration_seconds / 2.0 - window / 2.0).max(0.0), + (duration_seconds - window).max(0.0), + ]; + let mut selected = Vec::new(); + for start in starts { + selected.extend(decode_mono_window(path, start, Some(window))?); + } + anyhow::ensure!(!selected.is_empty(), "decoded track is empty"); + return Ok(selected); + } + decode_mono_window(path, 0.0, None) +} + +fn decode_mono_window( + path: &Path, + start_seconds: f64, + length_seconds: Option, +) -> Result> { + let file = File::open(path).with_context(|| format!("opening {}", path.display()))?; + let mut decoder = + Decoder::try_from(file).with_context(|| format!("decoding {}", path.display()))?; + let channels = decoder.channels().get() as usize; + let source_rate = decoder.sample_rate().get() as usize; + if start_seconds > 0.0 { + decoder + .try_seek(std::time::Duration::from_secs_f64(start_seconds)) + .with_context(|| format!("seeking {}", path.display()))?; + } + let max_samples = + length_seconds.map(|seconds| (seconds * source_rate as f64).ceil() as usize * channels); + let mut mono = Vec::new(); + let mut channel_sum = 0.0f32; + let mut channel_index = 0usize; + for (sample_index, sample) in decoder.enumerate() { + if max_samples.is_some_and(|limit| sample_index >= limit) { + break; + } + channel_sum += sample; + channel_index += 1; + if channel_index == channels { + mono.push(channel_sum / channels as f32); + channel_sum = 0.0; + channel_index = 0; + } + } + anyhow::ensure!(!mono.is_empty(), "decoded track is empty"); + if source_rate == SAMPLE_RATE { + return Ok(mono); + } + Ok(resample_sinc(&mono, source_rate, SAMPLE_RATE)) +} + +fn resample_sinc(input: &[f32], source_rate: usize, target_rate: usize) -> Vec { + if input.len() < 2 || source_rate == 0 { + return input.to_vec(); + } + let output_len = input + .len() + .saturating_mul(target_rate) + .checked_div(source_rate) + .unwrap_or(0) + .max(1); + let ratio = source_rate as f64 / target_rate as f64; + let cutoff = (target_rate as f64 / source_rate as f64).min(1.0) * 0.95; + const HALF_TAPS: isize = 8; + (0..output_len) + .map(|index| { + let position = index as f64 * ratio; + let center = position.floor() as isize; + let mut value = 0.0f64; + let mut weight_sum = 0.0f64; + for sample_index in center - HALF_TAPS + 1..=center + HALF_TAPS { + if sample_index < 0 || sample_index >= input.len() as isize { + continue; + } + let distance = position - sample_index as f64; + let phase = std::f64::consts::PI * distance * cutoff; + let sinc = if phase.abs() < 1e-12 { + 1.0 + } else { + phase.sin() / phase + }; + let window_position = distance / HALF_TAPS as f64; + let window = if window_position.abs() <= 1.0 { + 0.5 + 0.5 * (std::f64::consts::PI * window_position).cos() + } else { + 0.0 + }; + let weight = cutoff * sinc * window; + value += input[sample_index as usize] as f64 * weight; + weight_sum += weight; + } + if weight_sum.abs() < 1e-12 { + input[center.clamp(0, input.len() as isize - 1) as usize] + } else { + (value / weight_sum) as f32 + } + }) + .collect() +} + +fn mel_spectrogram(signal: &[f32]) -> Result> { + let frame_count = 1 + signal + .len() + .saturating_sub(FRAME_SIZE / 2) + .div_ceil(HOP_SIZE); + let filters = mel_filters(); + let mut planner = FftPlanner::::new(); + let fft = planner.plan_fft_forward(FRAME_SIZE); + let mut mel = Vec::with_capacity(frame_count); + let mut spectrum = vec![Complex::new(0.0f32, 0.0); FRAME_SIZE]; + for frame_index in 0..frame_count { + let start = frame_index as isize * HOP_SIZE as isize - (FRAME_SIZE / 2) as isize; + for (index, value) in spectrum.iter_mut().enumerate() { + let source = start + index as isize; + let sample = if source >= 0 { + signal.get(source as usize).copied().unwrap_or(0.0) + } else { + 0.0 + }; + let window = 0.5 + - 0.5 * (2.0 * std::f32::consts::PI * index as f32 / (FRAME_SIZE - 1) as f32).cos(); + *value = Complex::new(sample * window, 0.0); + } + fft.process(&mut spectrum); + let powers: Vec = spectrum[..=FRAME_SIZE / 2] + .iter() + .map(|value| value.norm_sqr()) + .collect(); + let mut bands = [0.0f32; MEL_BANDS]; + for (band, weights) in filters.iter().enumerate() { + let energy: f32 = powers + .iter() + .zip(weights) + .map(|(power, weight)| power * weight) + .sum(); + bands[band] = (1.0 + 10_000.0 * energy.max(0.0)).log10(); + } + mel.push(bands); + } + Ok(mel) +} + +fn mel_filters() -> Vec> { + let low = hz_to_mel_slaney(0.0); + let high = hz_to_mel_slaney((SAMPLE_RATE / 2) as f32); + let points: Vec = (0..MEL_BANDS + 2) + .map(|index| mel_to_hz_slaney(low + (high - low) * index as f32 / (MEL_BANDS + 1) as f32)) + .collect(); + let frequency_scale = (SAMPLE_RATE as f32 / 2.0) / (FRAME_SIZE / 2) as f32; + (0..MEL_BANDS) + .map(|band| { + let left = points[band]; + let center = points[band + 1]; + let right = points[band + 2]; + let area = ((center - left) + (right - center)) / 2.0; + (0..=FRAME_SIZE / 2) + .map(|bin| { + let frequency = bin as f32 * frequency_scale; + let triangle = if frequency < left || frequency > right { + 0.0 + } else if frequency < center { + (frequency - left) / (center - left) + } else { + (right - frequency) / (right - center) + }; + triangle.max(0.0) / area + }) + .collect() + }) + .collect() +} + +fn hz_to_mel_slaney(hz: f32) -> f32 { + if hz < 1000.0 { + hz / (200.0 / 3.0) + } else { + 15.0 + 27.0 * (hz / 1000.0).ln() / 6.4f32.ln() + } +} + +fn mel_to_hz_slaney(mel: f32) -> f32 { + if mel < 15.0 { + mel * (200.0 / 3.0) + } else { + 1000.0 * (6.4f32.ln() * (mel - 15.0) / 27.0).exp() + } +} + +fn normalize(vector: &mut [f32]) -> Result<()> { + let norm = vector.iter().map(|value| value * value).sum::().sqrt(); + anyhow::ensure!( + norm.is_finite() && norm > f32::EPSILON, + "zero or invalid embedding" + ); + for value in vector { + *value /= norm; + } + Ok(()) +} + +fn dot(left: &[f32], right: &[f32]) -> f32 { + left.iter().zip(right).map(|(a, b)| a * b).sum() +} + +fn is_near_duplicate(candidate: &[f32], kept: &[&[f32]]) -> bool { + kept.iter() + .any(|existing| dot(candidate, existing) >= NEAR_DUPLICATE_COSINE) +} + +fn sha256_file(path: &Path) -> Result { + use std::io::Read as _; + + let mut file = File::open(path)?; + let mut hasher = Sha256::new(); + let mut buffer = [0u8; 64 * 1024]; + loop { + let read = file.read(&mut buffer)?; + if read == 0 { + break; + } + hasher.update(&buffer[..read]); + } + Ok(format!("{:x}", hasher.finalize())) +} + +fn lock(mutex: &Mutex) -> std::sync::MutexGuard<'_, T> { + mutex + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) +} + +fn read(lock: &RwLock) -> std::sync::RwLockReadGuard<'_, T> { + lock.read() + .unwrap_or_else(std::sync::PoisonError::into_inner) +} + +fn write(lock: &RwLock) -> std::sync::RwLockWriteGuard<'_, T> { + lock.write() + .unwrap_or_else(std::sync::PoisonError::into_inner) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn profile_fingerprint_changes_with_contract() { + let model = &MODELS[0]; + let first = profile_fingerprint(model, DEFAULT_PROFILE_ID); + let second = profile_fingerprint(model, "another-profile"); + assert_ne!(first, second); + assert_eq!(first, profile_fingerprint(model, DEFAULT_PROFILE_ID)); + } + + #[test] + fn profile_details_describe_the_processing_contract() { + let details = profile_details(DEFAULT_PROFILE_ID, DEFAULT_MODEL_ID).unwrap(); + assert!(details.contains("Up to 300 seconds")); + assert!(details.contains("16000 Hz")); + assert!(details.contains("1280")); + } + + #[test] + fn vectors_are_normalized() { + let mut vector = vec![3.0, 4.0]; + normalize(&mut vector).unwrap(); + assert!((dot(&vector, &vector) - 1.0).abs() < 1e-6); + } + + #[test] + fn near_duplicate_embeddings_are_filtered_but_distinct_tracks_remain() { + let query = [1.0, 0.0, 0.0]; + let near_duplicate = [0.99995, 0.01, 0.0]; + let distinct = [0.0, 1.0, 0.0]; + assert!(is_near_duplicate(&near_duplicate, &[&query])); + assert!(!is_near_duplicate(&distinct, &[&query])); + } + + #[test] + fn resampling_keeps_a_constant_signal() { + let output = resample_sinc(&vec![0.25; 441], 44_100, 16_000); + assert_eq!(output.len(), 160); + assert!(output.iter().all(|value| (*value - 0.25).abs() < 1e-6)); + } + + /// Manual compatibility check for the separately downloaded model: + /// `FURUMI_TEST_MODEL=/path/model.onnx cargo test onnx_model_smoke -- --ignored`. + #[test] + #[ignore] + fn onnx_model_smoke() { + let path = std::env::var_os("FURUMI_TEST_MODEL") + .map(PathBuf::from) + .expect("set FURUMI_TEST_MODEL"); + let model = load_onnx(&path).unwrap(); + let mut input = vec![0.0f32; MODEL_BATCH * PATCH_FRAMES * MEL_BANDS]; + input[0] = 1.0; + let tensor = Tensor::from_shape(&[MODEL_BATCH, PATCH_FRAMES, MEL_BANDS], &input).unwrap(); + let outputs = model.run(tvec!(tensor.clone().into_tvalue())).unwrap(); + assert!( + outputs + .iter() + .any(|output| output.len() == MODEL_BATCH * EMBEDDING_DIMENSIONS) + ); + let started = Instant::now(); + for _ in 0..5 { + model.run(tvec!(tensor.clone().into_tvalue())).unwrap(); + } + eprintln!( + "five warm batch-{MODEL_BATCH} runs: {:.3}s", + started.elapsed().as_secs_f64() + ); + } +} diff --git a/src/ui/federation.rs b/src/ui/federation.rs index 231b55e..7cb3475 100644 --- a/src/ui/federation.rs +++ b/src/ui/federation.rs @@ -7,7 +7,7 @@ use ratatui::text::{Line, Span}; use ratatui::widgets::{Block, Paragraph}; use super::theme; -use crate::app::state::{AppState, DevicePresenceSection, FedRow, settings_rows}; +use crate::app::state::{AppState, DevicePresenceSection, FedRow, SimilarityRow, settings_rows}; pub fn draw(frame: &mut Frame, area: Rect, state: &AppState) { let block = Block::bordered() @@ -28,12 +28,12 @@ pub fn draw(frame: &mut Frame, area: Rect, state: &AppState) { .areas(inner); draw_settings_rows(frame, rows_area, state); - draw_status(frame, status_area, state); + draw_status_column(frame, status_area, state); return; } let rows_height = - (settings_rows(state).len() + 8 + device_presence_sections(state).len()) as u16; + (settings_rows(state).len() + 10 + device_presence_sections(state).len()) as u16; let [rows_area, _, status_area] = Layout::vertical([ Constraint::Length(rows_height.min(inner.height)), Constraint::Length(1), @@ -42,7 +42,7 @@ pub fn draw(frame: &mut Frame, area: Rect, state: &AppState) { .areas(inner); draw_settings_rows(frame, rows_area, state); - draw_status(frame, status_area, state); + draw_status_column(frame, status_area, state); } fn device_presence_sections(state: &AppState) -> Vec { @@ -88,6 +88,39 @@ fn draw_settings_rows(frame: &mut Frame, area: Rect, state: &AppState) { y = y.saturating_add(1); + draw_section(frame, area, state, &mut y, "Similarity Search"); + let similarity = &state.similarity.settings; + for row in SimilarityRow::ALL { + let (label, value) = match row { + SimilarityRow::Toggle => ("Similarity search", on_off(similarity.enabled).to_string()), + SimilarityRow::Model => ( + "Embedding model", + crate::similarity::model_by_id(&similarity.model) + .map(|model| format!("{} · {}", model.id, model.license)) + .unwrap_or_else(|| similarity.model.clone()), + ), + SimilarityRow::Profile => ( + "Preprocessing profile", + format!("{} (enter for details)", similarity.profile), + ), + SimilarityRow::Workers => ("Background workers", similarity.workers.to_string()), + SimilarityRow::Clear => ("Clear all stored embeddings", "↵".to_string()), + }; + draw_row( + frame, + area, + state, + &mut y, + cursor, + state.settings_cursor, + label, + value, + ); + cursor += 1; + } + + y = y.saturating_add(1); + draw_section(frame, area, state, &mut y, "Federation"); for row in FedRow::ALL { let (label, value) = match row { @@ -366,6 +399,7 @@ fn protocol_label(id: &str) -> &str { "music_dht" => "Music DHT", "catalog" => "Catalog", "audio" => "Audio transfer", + "similarity" => "Similarity search", "device_sync" => "Device sync", "jam" => "Jam", other => other, @@ -505,6 +539,68 @@ fn short_id(id: &str) -> String { id.chars().take(12).collect::() + "…" } +fn draw_status_column(frame: &mut Frame, area: Rect, state: &AppState) { + if area.height < 12 { + draw_status(frame, area, state); + return; + } + let [similarity_area, _, federation_area] = Layout::vertical([ + Constraint::Length(8), + Constraint::Length(1), + Constraint::Min(0), + ]) + .areas(area); + draw_similarity_status(frame, similarity_area, state); + draw_status(frame, federation_area, state); +} + +fn draw_similarity_status(frame: &mut Frame, area: Rect, state: &AppState) { + let status = &state.similarity.status; + let progress = if status.total_tracks == 0 { + "0 / 0".to_string() + } else { + format!("{} / {}", status.completed_tracks, status.total_tracks) + }; + let active = status + .active_profile + .as_deref() + .map(short_id) + .unwrap_or_else(|| "not ready".to_string()); + let target = status + .target_profile + .as_deref() + .map(short_id) + .unwrap_or_else(|| "—".to_string()); + draw_summary_card( + frame, + area, + state, + " Similarity Processing ", + vec![ + status_line("State", status.phase.label().to_string()), + status_line("Progress", progress), + status_line("Active", active), + status_line("Processing", target), + status_line( + "Stored", + format!( + "{} vectors / {}", + status.stored_vectors, + short_bytes_label(status.stored_bytes) + ), + ), + status_line( + "Current / errors", + status + .current_track + .clone() + .or_else(|| status.last_error.clone()) + .unwrap_or_else(|| format!("{} errors", status.failed_tracks)), + ), + ], + ); +} + fn draw_status(frame: &mut Frame, area: Rect, state: &AppState) { if area.width == 0 || area.height == 0 { return; diff --git a/src/ui/global.rs b/src/ui/global.rs index ff3d5ca..ba3a004 100644 --- a/src/ui/global.rs +++ b/src/ui/global.rs @@ -890,7 +890,12 @@ fn draw_release(frame: &mut Frame, area: Rect, state: &AppState, id: i64, cursor fn draw_search(frame: &mut Frame, area: Rect, state: &AppState, cursor: usize) { let search = &state.search; - let mut title = format!(" Search: {} ", search.query); + let prefix = if search.similarity_source.is_some() { + "Search similar to" + } else { + "Search" + }; + let mut title = format!(" {prefix}: {} ", search.query); if search.loading { title.push_str("· searching… "); } diff --git a/src/ui/popup.rs b/src/ui/popup.rs index e1dbc53..d6ba588 100644 --- a/src/ui/popup.rs +++ b/src/ui/popup.rs @@ -25,6 +25,10 @@ pub fn draw(frame: &mut Frame, state: &AppState) { .. }) => draw_edit(frame, state, title, fields, *focus, error.as_deref()), Some(Popup::ConfirmDelete { label, .. }) => draw_confirm_delete(frame, state, label), + Some(Popup::SimilarityPrivacyConsent { .. }) => { + draw_similarity_privacy_consent(frame, state) + } + Some(Popup::ConfirmClearEmbeddings) => draw_confirm_clear_embeddings(frame, state), Some(Popup::LibraryFilters { cursor }) => draw_library_filters(frame, state, *cursor), Some(Popup::TrackInfo { tracks, @@ -100,6 +104,51 @@ pub fn draw(frame: &mut Frame, state: &AppState) { } } +fn draw_similarity_privacy_consent(frame: &mut Frame, state: &AppState) { + let area = centered(frame.area(), 78, 11); + let block = Block::bordered() + .title(" Similarity search and federation ") + .title_style(theme::header_for(state)) + .border_style(theme::strong_border_for(state)); + let inner = block.inner(area); + frame.render_widget(Clear, area); + frame.render_widget(block, area); + frame.render_widget( + Paragraph::new(vec![ + Line::raw("To find similar music on other peers, Furumi sends them an"), + Line::raw("anonymous numeric embedding of the selected track."), + Line::raw(""), + Line::raw("It contains no account identity, but a peer may technically"), + Line::raw("infer what kind of music you are searching from it."), + Line::raw(""), + Line::styled("enter/y: agree and enable · n/esc: cancel", theme::dim()), + ]) + .wrap(Wrap { trim: true }), + inner, + ); +} + +fn draw_confirm_clear_embeddings(frame: &mut Frame, state: &AppState) { + let area = centered(frame.area(), 70, 8); + let block = Block::bordered() + .title(" Clear embeddings? ") + .title_style(theme::header_for(state)) + .border_style(theme::strong_border_for(state)); + let inner = block.inner(area); + frame.render_widget(Clear, area); + frame.render_widget(block, area); + frame.render_widget( + Paragraph::new(vec![ + Line::raw("All model/profile embeddings will be removed from SQLite."), + Line::raw("Audio and library metadata stay untouched."), + Line::raw("If enabled, processing starts again automatically."), + Line::raw(""), + Line::styled("enter/y: clear · n/esc: cancel", theme::dim()), + ]), + inner, + ); +} + fn draw_music_directory_confirmation(frame: &mut Frame, state: &AppState, path: &std::path::Path) { let area = centered(frame.area(), 76, 9); let block = Block::bordered() @@ -873,11 +922,19 @@ fn draw_fed_input( ); } -/// Read-only wrapped text (this peer's federation ticket). +/// Read-only wrapped text. fn draw_fed_text(frame: &mut Frame, state: &AppState, title: &str, text: &str) { let width = frame.area().width.saturating_sub(8).clamp(24, 90); let text_width = usize::from(width.saturating_sub(2)); - let lines_needed = (text.chars().count() / text_width.max(1) + 3) as u16; + let lines_needed = text + .lines() + .map(|line| { + UnicodeWidthStr::width(line) + .max(1) + .div_ceil(text_width.max(1)) + }) + .sum::() + .saturating_add(2) as u16; let area = centered( frame.area(), width, @@ -1281,12 +1338,22 @@ fn draw_track_info( ); let can_share = crate::share::track_can_share(track); - let hint = if tracks.len() > 1 && can_share { + let can_similar = + state.similarity.settings.enabled && track.id >= 0 && !track.file_path.is_empty(); + let hint = if tracks.len() > 1 && can_share && can_similar { + "j/k scroll · h/left previous · l/right next · a artist · s similar · c copy link · esc" + } else if tracks.len() > 1 && can_share { "j/k scroll · h/left previous · l/right next · a artist · c copy frid link · esc close" + } else if tracks.len() > 1 && can_similar { + "j/k scroll · h/left previous · l/right next · a artist · s similar · esc close" } else if tracks.len() > 1 { "j/k scroll · h/left previous · l/right next · a artist · esc close" + } else if can_share && can_similar { + "j/k scroll · a artist · s similar · c copy link · esc close" } else if can_share { "j/k scroll · a artist · c copy frid link · esc close" + } else if can_similar { + "j/k scroll · a artist · s similar · esc close" } else { "j/k scroll · a artist · esc close" };