Run the podcast discovery overlay in the federation node

The node serves FRID's podcast_dht overlay next to the similarity overlay
(cache federation/podcast-dht.sqlite3), runs its maintenance, hands peer
capability manifests to note_peer_capabilities and advertises podcast_dht
only while the overlay runs. A cache that cannot be opened is logged and
only turns discovery off. The running overlay is exposed through a watch
channel for the publication reconciler.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
sanapachandClaude Opus 5.5 committed 2026-11-20 15:13:07 +02:00
1 parent ea7cdc4b7f
commit c453b0a52f
3 files changed
+182 -31

No files matched your search

+59 -13
View File
@@ -10,6 +10,7 @@ use music_dht::capabilities::{
CAPABILITIES_PROTOCOL_VERSION, CapabilityManifest, CapabilityMessage, read_message,
write_message,
};
use music_dht::podcast_dht::PodcastDht;
use music_dht::{ByteStream, EndpointId, MusicDhtService, StreamAcceptor};
const PROBE_INTERVAL: Duration = Duration::from_secs(30);
@@ -30,8 +31,10 @@ pub struct NewerProtocol {
}
impl ProtocolVersions {
pub fn snapshot(observed: &ObservedVersions) -> Self {
let local = local_manifest().protocols;
/// `podcast_dht`: whether this node runs the podcast-discovery overlay
/// (it is advertised only then).
pub fn snapshot(observed: &ObservedVersions, podcast_dht: bool) -> Self {
let local = local_manifest(podcast_dht).protocols;
let observed_versions = lock(&observed.versions).clone();
let newer = observed_versions
.iter()
@@ -59,27 +62,35 @@ pub struct ObservedVersions {
peers: Mutex<BTreeMap<String, String>>,
}
fn local_manifest() -> CapabilityManifest {
CapabilityManifest::frid("furumi", env!("CARGO_PKG_VERSION"))
.with_protocol("audio", super::audio::AUDIO_PROTOCOL_VERSION)
/// This node's manifest. The podcast-discovery overlay is opt-in in FRID:
/// it is advertised only when this node serves it (`podcast_dht`), so
/// peers never dial an overlay that failed to open here.
fn local_manifest(podcast_dht: bool) -> CapabilityManifest {
let manifest = CapabilityManifest::frid("furumi", env!("CARGO_PKG_VERSION"))
.with_protocol("audio", super::audio::AUDIO_PROTOCOL_VERSION);
if podcast_dht {
manifest.with_podcast_dht()
} else {
manifest
}
}
pub async fn serve(mut acceptor: StreamAcceptor) {
pub async fn serve(mut acceptor: StreamAcceptor, podcast_dht: bool) {
while let Some(stream) = acceptor.accept().await {
tokio::spawn(async move {
if let Err(error) = serve_one(stream).await {
if let Err(error) = serve_one(stream, podcast_dht).await {
tracing::debug!("capability stream failed: {error:#}");
}
});
}
}
async fn serve_one(mut stream: ByteStream) -> Result<()> {
async fn serve_one(mut stream: ByteStream, podcast_dht: bool) -> Result<()> {
let response = match read_message(&mut stream).await? {
CapabilityMessage::Get {
version: CAPABILITIES_PROTOCOL_VERSION,
} => CapabilityMessage::Manifest {
manifest: local_manifest(),
manifest: local_manifest(podcast_dht),
},
CapabilityMessage::Get { version } => CapabilityMessage::Error {
message: format!("unsupported capability protocol {version}"),
@@ -94,7 +105,14 @@ async fn serve_one(mut stream: ByteStream) -> Result<()> {
Ok(())
}
pub async fn probe_loop(service: Arc<MusicDhtService>, observed: Arc<ObservedVersions>) {
/// Polls the manifests of known peers. With the podcast-discovery overlay
/// running, every manifest is handed to `PodcastDht::note_peer_capabilities`,
/// which stops dialing peers that do not serve the overlay.
pub async fn probe_loop(
service: Arc<MusicDhtService>,
observed: Arc<ObservedVersions>,
podcast_dht: Option<Arc<PodcastDht>>,
) {
let mut interval = tokio::time::interval(PROBE_INTERVAL);
loop {
interval.tick().await;
@@ -109,7 +127,8 @@ pub async fn probe_loop(service: Arc<MusicDhtService>, observed: Arc<ObservedVer
)
.collect::<std::collections::BTreeSet<_>>();
for peer in peers {
if let Err(error) = probe_peer(&service, peer, &observed).await {
if let Err(error) = probe_peer(&service, peer, &observed, podcast_dht.as_deref()).await
{
tracing::trace!(%peer, "peer capability probe unavailable: {error:#}");
}
}
@@ -120,6 +139,7 @@ async fn probe_peer(
service: &MusicDhtService,
peer: EndpointId,
observed: &ObservedVersions,
podcast_dht: Option<&PodcastDht>,
) -> Result<()> {
let mut stream = service.open_stream(peer, CAPABILITIES_ALPN).await?;
write_message(
@@ -137,6 +157,11 @@ async fn probe_peer(
anyhow::bail!("peer did not return a capability manifest");
};
manifest.validate()?;
if let Some(podcast_dht) = podcast_dht
&& let Err(error) = podcast_dht.note_peer_capabilities(peer, &manifest).await
{
tracing::debug!(%peer, "podcast DHT capability note failed: {error:#}");
}
{
let mut versions = lock(&observed.versions);
for (id, version) in manifest.protocols {
@@ -162,7 +187,7 @@ mod tests {
#[test]
fn local_manifest_lists_every_player_protocol() {
let manifest = local_manifest();
let manifest = local_manifest(true);
for id in [
"federation_net",
"ticket",
@@ -174,10 +199,31 @@ mod tests {
"similarity_dht",
"device_sync",
"jam",
"podcast_dht",
] {
assert!(manifest.protocols.contains_key(id), "missing {id}");
}
manifest.validate().unwrap();
assert!(manifest.supports_podcast_dht());
}
#[test]
fn podcast_discovery_is_advertised_only_when_it_runs() {
let manifest = local_manifest(false);
assert!(!manifest.supports_podcast_dht());
assert!(!manifest.protocols.contains_key("podcast_dht"));
manifest.validate().unwrap();
let observed = ObservedVersions::default();
assert!(
!ProtocolVersions::snapshot(&observed, false)
.local
.contains_key("podcast_dht")
);
assert!(
ProtocolVersions::snapshot(&observed, true)
.local
.contains_key("podcast_dht")
);
}
#[test]
@@ -185,7 +231,7 @@ mod tests {
let observed = ObservedVersions::default();
lock(&observed.versions).insert("music_dht".to_string(), 99);
lock(&observed.versions).insert("jam".to_string(), crate::jam::PROTOCOL_VERSION);
let snapshot = ProtocolVersions::snapshot(&observed);
let snapshot = ProtocolVersions::snapshot(&observed, true);
assert_eq!(snapshot.newer.len(), 1);
assert_eq!(snapshot.newer[0].id, "music_dht");
}
+91 -18
View File
@@ -25,6 +25,7 @@ use std::sync::atomic::{AtomicBool, AtomicI64, AtomicU64, Ordering};
use std::time::{Duration, Instant};
use anyhow::{Context, Result};
use music_dht::podcast_dht::{PODCAST_DHT_ALPN, PodcastDht};
use music_dht::similarity_dht::SimilarityDht;
use music_dht::similarity_lsh::SIMILARITY_DHT_ALPN;
use music_dht::{
@@ -44,6 +45,10 @@ pub use capabilities::ProtocolVersions;
pub use catalog::{CATALOG_ALPN, FedAppearsOn, FedArtistCard, FedCardTrack, FedRelease};
pub use similarity::SIMILARITY_ALPN;
/// Cache of the public podcast-discovery overlay (`music_dht::podcast_dht`),
/// next to the similarity overlay's cache in the federation data directory.
pub const PODCAST_DHT_CACHE_FILE: &str = "podcast-dht.sqlite3";
/// How often the published library is re-synchronized with the local index.
const SYNC_INTERVAL: Duration = Duration::from_secs(60);
/// How often the application checks whether the shared transport needs a
@@ -412,6 +417,9 @@ pub struct NetworkLibrarySource {
struct Running {
service: Arc<MusicDhtService>,
similarity_dht: Arc<SimilarityDht>,
/// Public podcast discovery; `None` when its cache could not be opened
/// (discovery is then off and the overlay is not advertised).
podcast_dht: Option<Arc<PodcastDht>>,
network_name: String,
network_id: NetworkId,
tasks: Vec<tokio::task::JoinHandle<()>>,
@@ -438,6 +446,9 @@ pub struct Federation {
last_error: std::sync::Mutex<Option<String>>,
transport_stats: Arc<TransportStats>,
observed_protocols: Arc<capabilities::ObservedVersions>,
/// The running podcast-discovery overlay, for the podcast publication
/// reconciler (`podcasts::publication`); `None` while federation is off.
podcast_dht: tokio::sync::watch::Sender<Option<Arc<PodcastDht>>>,
}
#[derive(Debug, Clone)]
@@ -572,9 +583,17 @@ impl Federation {
last_error: std::sync::Mutex::new(initial_error),
transport_stats: Arc::new(TransportStats::default()),
observed_protocols: Arc::new(capabilities::ObservedVersions::default()),
podcast_dht: tokio::sync::watch::Sender::new(None),
})
}
/// Follows the podcast-discovery overlay of the running node: `Some`
/// while federation runs with the overlay, `None` otherwise. Every
/// (re)start publishes a new value.
pub fn podcast_dht(&self) -> tokio::sync::watch::Receiver<Option<Arc<PodcastDht>>> {
self.podcast_dht.subscribe()
}
pub fn settings(&self) -> FedSettings {
lock(&self.settings).clone()
}
@@ -730,6 +749,7 @@ impl Federation {
} else if recovery_only {
return Ok(false);
}
self.podcast_dht.send_replace(None);
stop_running(guard.take()).await;
} else if recovery_only {
tracing::warn!(network = %network_name, "retrying stopped federation service");
@@ -756,6 +776,8 @@ impl Federation {
.stream_protocol(crate::jam::JAM_ALPN)
// Signed LSH summaries form their own upgrade-safe DHT overlay.
.schema_independent_stream_protocol(SIMILARITY_DHT_ALPN)
// Public cards of podcasts the user chose to share.
.schema_independent_stream_protocol(PODCAST_DHT_ALPN)
// Informational application/protocol versions.
.schema_independent_stream_protocol(capabilities::CAPABILITIES_ALPN)
.build()
@@ -788,6 +810,18 @@ impl Federation {
Arc::clone(&self.similarity),
Arc::clone(&self.library),
));
// Public podcast discovery: serving, bootstrap, republishing and
// expiry run here; what to publish is decided by the podcast
// publication reconciler from the user's share decisions.
let mut podcast_dht_tasks = Vec::new();
let podcast_dht =
open_podcast_dht(&service, &self.data_dir)
.await
.map(|(dht, acceptor)| {
podcast_dht_tasks.push(tokio::spawn(Arc::clone(&dht).serve(acceptor)));
podcast_dht_tasks.push(tokio::spawn(Arc::clone(&dht).maintenance()));
dht
});
// Drain DHT events into the log; the channel is bounded.
let event_task = tokio::spawn(async move {
@@ -861,33 +895,41 @@ impl Federation {
let capabilities_acceptor = service
.stream_acceptor(capabilities::CAPABILITIES_ALPN)
.map_err(|err| anyhow::anyhow!("failed to take capabilities acceptor: {err}"))?;
let capabilities_serve_task = tokio::spawn(capabilities::serve(capabilities_acceptor));
let capabilities_serve_task = tokio::spawn(capabilities::serve(
capabilities_acceptor,
podcast_dht.is_some(),
));
let capabilities_probe_task = tokio::spawn(capabilities::probe_loop(
Arc::clone(&service),
Arc::clone(&self.observed_protocols),
podcast_dht.clone(),
));
let mut tasks = podcast_dht_tasks;
tasks.extend([
event_task,
sync_task,
audio_task,
catalog_task,
similarity_task,
device_sync_task,
device_tick_task,
jam_serve_task,
jam_poll_task,
capabilities_serve_task,
capabilities_probe_task,
similarity_dht_serve_task,
similarity_dht_maintenance_task,
similarity_dht_sync_task,
]);
self.podcast_dht.send_replace(podcast_dht.clone());
*guard = Some(Running {
service,
similarity_dht,
podcast_dht,
network_name,
network_id,
tasks: vec![
event_task,
sync_task,
audio_task,
catalog_task,
similarity_task,
device_sync_task,
device_tick_task,
jam_serve_task,
jam_poll_task,
capabilities_serve_task,
capabilities_probe_task,
similarity_dht_serve_task,
similarity_dht_maintenance_task,
similarity_dht_sync_task,
],
tasks,
});
self.set_error(None);
Ok(true)
@@ -895,6 +937,7 @@ impl Federation {
async fn stop(&self) {
let mut guard = self.running.lock().await;
self.podcast_dht.send_replace(None);
stop_running(guard.take()).await;
}
@@ -1088,7 +1131,12 @@ impl Federation {
recovery_count: self.recovery_count.load(Ordering::Relaxed),
last_sync: lock(&self.last_sync).clone(),
last_error: lock(&self.last_error).clone(),
protocols: ProtocolVersions::snapshot(&self.observed_protocols),
protocols: ProtocolVersions::snapshot(
&self.observed_protocols,
guard
.as_ref()
.is_some_and(|running| running.podcast_dht.is_some()),
),
..FedStatus::default()
};
if let Some(running) = guard.as_ref() {
@@ -2804,6 +2852,31 @@ async fn similarity_route_sync_loop(
}
}
/// Opens the podcast-discovery overlay and takes its stream acceptor. A
/// failure (unreadable or newer cache, …) is logged and only disables public
/// podcast discovery: the acceptor is dropped so the overlay's ALPN refuses
/// streams, and the overlay is not advertised.
async fn open_podcast_dht(
service: &Arc<MusicDhtService>,
data_dir: &Path,
) -> Option<(Arc<PodcastDht>, music_dht::StreamAcceptor)> {
let acceptor = match service.stream_acceptor(PODCAST_DHT_ALPN) {
Ok(acceptor) => acceptor,
Err(err) => {
tracing::warn!("podcast discovery is off: cannot take its acceptor: {err}");
return None;
}
};
match PodcastDht::open(Arc::clone(service), data_dir.join(PODCAST_DHT_CACHE_FILE)).await {
Ok(dht) => Some((dht, acceptor)),
Err(err) => {
tracing::warn!("podcast discovery is off: cannot open its cache: {err}");
drop(acceptor);
None
}
}
}
async fn stop_running(running: Option<Running>) {
let Some(running) = running else { return };
for task in &running.tasks {
+32
View File
@@ -135,3 +135,35 @@ fn artist_catalog_discovery_prioritizes_matching_owners_and_bounds_fanout() {
);
}
}
/// The podcast-discovery overlay opens next to the other overlay caches; a
/// cache that cannot be opened only turns discovery off.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn podcast_discovery_opens_or_turns_itself_off() {
async fn service(dir: &Path) -> Arc<MusicDhtService> {
let config = MusicDhtConfig::builder()
.data_dir(dir)
.network_id(NetworkId::from_name("furumi-tui-podcast-dht-open-test"))
.schema_independent_stream_protocol(PODCAST_DHT_ALPN)
.build()
.unwrap();
let (service, mut events) = MusicDhtService::start(config).await.unwrap();
tokio::spawn(async move { while events.recv().await.is_some() {} });
Arc::new(service)
}
let ok_dir = tempfile::tempdir().unwrap();
let ok = service(ok_dir.path()).await;
let opened = open_podcast_dht(&ok, ok_dir.path()).await;
assert!(opened.is_some());
assert!(ok_dir.path().join(PODCAST_DHT_CACHE_FILE).is_file());
drop(opened);
ok.shutdown().await.unwrap();
// A directory where the cache file belongs cannot be opened as SQLite.
let broken_dir = tempfile::tempdir().unwrap();
std::fs::create_dir(broken_dir.path().join(PODCAST_DHT_CACHE_FILE)).unwrap();
let broken = service(broken_dir.path()).await;
assert!(open_podcast_dht(&broken, broken_dir.path()).await.is_none());
broken.shutdown().await.unwrap();
}