Run the podcast discovery overlay in the federation node
The node serves FRID's podcast_dht overlay next to the similarity overlay (cache federation/podcast-dht.sqlite3), runs its maintenance, hands peer capability manifests to note_peer_capabilities and advertises podcast_dht only while the overlay runs. A cache that cannot be opened is logged and only turns discovery off. The running overlay is exposed through a watch channel for the publication reconciler. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
ea7cdc4b7f
commit
c453b0a52f
3 files changed
+182
-31
No files matched your search
@@ -10,6 +10,7 @@ use music_dht::capabilities::{
|
||||
CAPABILITIES_PROTOCOL_VERSION, CapabilityManifest, CapabilityMessage, read_message,
|
||||
write_message,
|
||||
};
|
||||
use music_dht::podcast_dht::PodcastDht;
|
||||
use music_dht::{ByteStream, EndpointId, MusicDhtService, StreamAcceptor};
|
||||
|
||||
const PROBE_INTERVAL: Duration = Duration::from_secs(30);
|
||||
@@ -30,8 +31,10 @@ pub struct NewerProtocol {
|
||||
}
|
||||
|
||||
impl ProtocolVersions {
|
||||
pub fn snapshot(observed: &ObservedVersions) -> Self {
|
||||
let local = local_manifest().protocols;
|
||||
/// `podcast_dht`: whether this node runs the podcast-discovery overlay
|
||||
/// (it is advertised only then).
|
||||
pub fn snapshot(observed: &ObservedVersions, podcast_dht: bool) -> Self {
|
||||
let local = local_manifest(podcast_dht).protocols;
|
||||
let observed_versions = lock(&observed.versions).clone();
|
||||
let newer = observed_versions
|
||||
.iter()
|
||||
@@ -59,27 +62,35 @@ pub struct ObservedVersions {
|
||||
peers: Mutex<BTreeMap<String, String>>,
|
||||
}
|
||||
|
||||
fn local_manifest() -> CapabilityManifest {
|
||||
CapabilityManifest::frid("furumi", env!("CARGO_PKG_VERSION"))
|
||||
.with_protocol("audio", super::audio::AUDIO_PROTOCOL_VERSION)
|
||||
/// This node's manifest. The podcast-discovery overlay is opt-in in FRID:
|
||||
/// it is advertised only when this node serves it (`podcast_dht`), so
|
||||
/// peers never dial an overlay that failed to open here.
|
||||
fn local_manifest(podcast_dht: bool) -> CapabilityManifest {
|
||||
let manifest = CapabilityManifest::frid("furumi", env!("CARGO_PKG_VERSION"))
|
||||
.with_protocol("audio", super::audio::AUDIO_PROTOCOL_VERSION);
|
||||
if podcast_dht {
|
||||
manifest.with_podcast_dht()
|
||||
} else {
|
||||
manifest
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn serve(mut acceptor: StreamAcceptor) {
|
||||
pub async fn serve(mut acceptor: StreamAcceptor, podcast_dht: bool) {
|
||||
while let Some(stream) = acceptor.accept().await {
|
||||
tokio::spawn(async move {
|
||||
if let Err(error) = serve_one(stream).await {
|
||||
if let Err(error) = serve_one(stream, podcast_dht).await {
|
||||
tracing::debug!("capability stream failed: {error:#}");
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async fn serve_one(mut stream: ByteStream) -> Result<()> {
|
||||
async fn serve_one(mut stream: ByteStream, podcast_dht: bool) -> Result<()> {
|
||||
let response = match read_message(&mut stream).await? {
|
||||
CapabilityMessage::Get {
|
||||
version: CAPABILITIES_PROTOCOL_VERSION,
|
||||
} => CapabilityMessage::Manifest {
|
||||
manifest: local_manifest(),
|
||||
manifest: local_manifest(podcast_dht),
|
||||
},
|
||||
CapabilityMessage::Get { version } => CapabilityMessage::Error {
|
||||
message: format!("unsupported capability protocol {version}"),
|
||||
@@ -94,7 +105,14 @@ async fn serve_one(mut stream: ByteStream) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn probe_loop(service: Arc<MusicDhtService>, observed: Arc<ObservedVersions>) {
|
||||
/// Polls the manifests of known peers. With the podcast-discovery overlay
|
||||
/// running, every manifest is handed to `PodcastDht::note_peer_capabilities`,
|
||||
/// which stops dialing peers that do not serve the overlay.
|
||||
pub async fn probe_loop(
|
||||
service: Arc<MusicDhtService>,
|
||||
observed: Arc<ObservedVersions>,
|
||||
podcast_dht: Option<Arc<PodcastDht>>,
|
||||
) {
|
||||
let mut interval = tokio::time::interval(PROBE_INTERVAL);
|
||||
loop {
|
||||
interval.tick().await;
|
||||
@@ -109,7 +127,8 @@ pub async fn probe_loop(service: Arc<MusicDhtService>, observed: Arc<ObservedVer
|
||||
)
|
||||
.collect::<std::collections::BTreeSet<_>>();
|
||||
for peer in peers {
|
||||
if let Err(error) = probe_peer(&service, peer, &observed).await {
|
||||
if let Err(error) = probe_peer(&service, peer, &observed, podcast_dht.as_deref()).await
|
||||
{
|
||||
tracing::trace!(%peer, "peer capability probe unavailable: {error:#}");
|
||||
}
|
||||
}
|
||||
@@ -120,6 +139,7 @@ async fn probe_peer(
|
||||
service: &MusicDhtService,
|
||||
peer: EndpointId,
|
||||
observed: &ObservedVersions,
|
||||
podcast_dht: Option<&PodcastDht>,
|
||||
) -> Result<()> {
|
||||
let mut stream = service.open_stream(peer, CAPABILITIES_ALPN).await?;
|
||||
write_message(
|
||||
@@ -137,6 +157,11 @@ async fn probe_peer(
|
||||
anyhow::bail!("peer did not return a capability manifest");
|
||||
};
|
||||
manifest.validate()?;
|
||||
if let Some(podcast_dht) = podcast_dht
|
||||
&& let Err(error) = podcast_dht.note_peer_capabilities(peer, &manifest).await
|
||||
{
|
||||
tracing::debug!(%peer, "podcast DHT capability note failed: {error:#}");
|
||||
}
|
||||
{
|
||||
let mut versions = lock(&observed.versions);
|
||||
for (id, version) in manifest.protocols {
|
||||
@@ -162,7 +187,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn local_manifest_lists_every_player_protocol() {
|
||||
let manifest = local_manifest();
|
||||
let manifest = local_manifest(true);
|
||||
for id in [
|
||||
"federation_net",
|
||||
"ticket",
|
||||
@@ -174,10 +199,31 @@ mod tests {
|
||||
"similarity_dht",
|
||||
"device_sync",
|
||||
"jam",
|
||||
"podcast_dht",
|
||||
] {
|
||||
assert!(manifest.protocols.contains_key(id), "missing {id}");
|
||||
}
|
||||
manifest.validate().unwrap();
|
||||
assert!(manifest.supports_podcast_dht());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn podcast_discovery_is_advertised_only_when_it_runs() {
|
||||
let manifest = local_manifest(false);
|
||||
assert!(!manifest.supports_podcast_dht());
|
||||
assert!(!manifest.protocols.contains_key("podcast_dht"));
|
||||
manifest.validate().unwrap();
|
||||
let observed = ObservedVersions::default();
|
||||
assert!(
|
||||
!ProtocolVersions::snapshot(&observed, false)
|
||||
.local
|
||||
.contains_key("podcast_dht")
|
||||
);
|
||||
assert!(
|
||||
ProtocolVersions::snapshot(&observed, true)
|
||||
.local
|
||||
.contains_key("podcast_dht")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -185,7 +231,7 @@ mod tests {
|
||||
let observed = ObservedVersions::default();
|
||||
lock(&observed.versions).insert("music_dht".to_string(), 99);
|
||||
lock(&observed.versions).insert("jam".to_string(), crate::jam::PROTOCOL_VERSION);
|
||||
let snapshot = ProtocolVersions::snapshot(&observed);
|
||||
let snapshot = ProtocolVersions::snapshot(&observed, true);
|
||||
assert_eq!(snapshot.newer.len(), 1);
|
||||
assert_eq!(snapshot.newer[0].id, "music_dht");
|
||||
}
|
||||
|
||||
+91
-18
@@ -25,6 +25,7 @@ use std::sync::atomic::{AtomicBool, AtomicI64, AtomicU64, Ordering};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use music_dht::podcast_dht::{PODCAST_DHT_ALPN, PodcastDht};
|
||||
use music_dht::similarity_dht::SimilarityDht;
|
||||
use music_dht::similarity_lsh::SIMILARITY_DHT_ALPN;
|
||||
use music_dht::{
|
||||
@@ -44,6 +45,10 @@ pub use capabilities::ProtocolVersions;
|
||||
pub use catalog::{CATALOG_ALPN, FedAppearsOn, FedArtistCard, FedCardTrack, FedRelease};
|
||||
pub use similarity::SIMILARITY_ALPN;
|
||||
|
||||
/// Cache of the public podcast-discovery overlay (`music_dht::podcast_dht`),
|
||||
/// next to the similarity overlay's cache in the federation data directory.
|
||||
pub const PODCAST_DHT_CACHE_FILE: &str = "podcast-dht.sqlite3";
|
||||
|
||||
/// How often the published library is re-synchronized with the local index.
|
||||
const SYNC_INTERVAL: Duration = Duration::from_secs(60);
|
||||
/// How often the application checks whether the shared transport needs a
|
||||
@@ -412,6 +417,9 @@ pub struct NetworkLibrarySource {
|
||||
struct Running {
|
||||
service: Arc<MusicDhtService>,
|
||||
similarity_dht: Arc<SimilarityDht>,
|
||||
/// Public podcast discovery; `None` when its cache could not be opened
|
||||
/// (discovery is then off and the overlay is not advertised).
|
||||
podcast_dht: Option<Arc<PodcastDht>>,
|
||||
network_name: String,
|
||||
network_id: NetworkId,
|
||||
tasks: Vec<tokio::task::JoinHandle<()>>,
|
||||
@@ -438,6 +446,9 @@ pub struct Federation {
|
||||
last_error: std::sync::Mutex<Option<String>>,
|
||||
transport_stats: Arc<TransportStats>,
|
||||
observed_protocols: Arc<capabilities::ObservedVersions>,
|
||||
/// The running podcast-discovery overlay, for the podcast publication
|
||||
/// reconciler (`podcasts::publication`); `None` while federation is off.
|
||||
podcast_dht: tokio::sync::watch::Sender<Option<Arc<PodcastDht>>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
@@ -572,9 +583,17 @@ impl Federation {
|
||||
last_error: std::sync::Mutex::new(initial_error),
|
||||
transport_stats: Arc::new(TransportStats::default()),
|
||||
observed_protocols: Arc::new(capabilities::ObservedVersions::default()),
|
||||
podcast_dht: tokio::sync::watch::Sender::new(None),
|
||||
})
|
||||
}
|
||||
|
||||
/// Follows the podcast-discovery overlay of the running node: `Some`
|
||||
/// while federation runs with the overlay, `None` otherwise. Every
|
||||
/// (re)start publishes a new value.
|
||||
pub fn podcast_dht(&self) -> tokio::sync::watch::Receiver<Option<Arc<PodcastDht>>> {
|
||||
self.podcast_dht.subscribe()
|
||||
}
|
||||
|
||||
pub fn settings(&self) -> FedSettings {
|
||||
lock(&self.settings).clone()
|
||||
}
|
||||
@@ -730,6 +749,7 @@ impl Federation {
|
||||
} else if recovery_only {
|
||||
return Ok(false);
|
||||
}
|
||||
self.podcast_dht.send_replace(None);
|
||||
stop_running(guard.take()).await;
|
||||
} else if recovery_only {
|
||||
tracing::warn!(network = %network_name, "retrying stopped federation service");
|
||||
@@ -756,6 +776,8 @@ impl Federation {
|
||||
.stream_protocol(crate::jam::JAM_ALPN)
|
||||
// Signed LSH summaries form their own upgrade-safe DHT overlay.
|
||||
.schema_independent_stream_protocol(SIMILARITY_DHT_ALPN)
|
||||
// Public cards of podcasts the user chose to share.
|
||||
.schema_independent_stream_protocol(PODCAST_DHT_ALPN)
|
||||
// Informational application/protocol versions.
|
||||
.schema_independent_stream_protocol(capabilities::CAPABILITIES_ALPN)
|
||||
.build()
|
||||
@@ -788,6 +810,18 @@ impl Federation {
|
||||
Arc::clone(&self.similarity),
|
||||
Arc::clone(&self.library),
|
||||
));
|
||||
// Public podcast discovery: serving, bootstrap, republishing and
|
||||
// expiry run here; what to publish is decided by the podcast
|
||||
// publication reconciler from the user's share decisions.
|
||||
let mut podcast_dht_tasks = Vec::new();
|
||||
let podcast_dht =
|
||||
open_podcast_dht(&service, &self.data_dir)
|
||||
.await
|
||||
.map(|(dht, acceptor)| {
|
||||
podcast_dht_tasks.push(tokio::spawn(Arc::clone(&dht).serve(acceptor)));
|
||||
podcast_dht_tasks.push(tokio::spawn(Arc::clone(&dht).maintenance()));
|
||||
dht
|
||||
});
|
||||
|
||||
// Drain DHT events into the log; the channel is bounded.
|
||||
let event_task = tokio::spawn(async move {
|
||||
@@ -861,33 +895,41 @@ impl Federation {
|
||||
let capabilities_acceptor = service
|
||||
.stream_acceptor(capabilities::CAPABILITIES_ALPN)
|
||||
.map_err(|err| anyhow::anyhow!("failed to take capabilities acceptor: {err}"))?;
|
||||
let capabilities_serve_task = tokio::spawn(capabilities::serve(capabilities_acceptor));
|
||||
let capabilities_serve_task = tokio::spawn(capabilities::serve(
|
||||
capabilities_acceptor,
|
||||
podcast_dht.is_some(),
|
||||
));
|
||||
let capabilities_probe_task = tokio::spawn(capabilities::probe_loop(
|
||||
Arc::clone(&service),
|
||||
Arc::clone(&self.observed_protocols),
|
||||
podcast_dht.clone(),
|
||||
));
|
||||
|
||||
let mut tasks = podcast_dht_tasks;
|
||||
tasks.extend([
|
||||
event_task,
|
||||
sync_task,
|
||||
audio_task,
|
||||
catalog_task,
|
||||
similarity_task,
|
||||
device_sync_task,
|
||||
device_tick_task,
|
||||
jam_serve_task,
|
||||
jam_poll_task,
|
||||
capabilities_serve_task,
|
||||
capabilities_probe_task,
|
||||
similarity_dht_serve_task,
|
||||
similarity_dht_maintenance_task,
|
||||
similarity_dht_sync_task,
|
||||
]);
|
||||
self.podcast_dht.send_replace(podcast_dht.clone());
|
||||
*guard = Some(Running {
|
||||
service,
|
||||
similarity_dht,
|
||||
podcast_dht,
|
||||
network_name,
|
||||
network_id,
|
||||
tasks: vec![
|
||||
event_task,
|
||||
sync_task,
|
||||
audio_task,
|
||||
catalog_task,
|
||||
similarity_task,
|
||||
device_sync_task,
|
||||
device_tick_task,
|
||||
jam_serve_task,
|
||||
jam_poll_task,
|
||||
capabilities_serve_task,
|
||||
capabilities_probe_task,
|
||||
similarity_dht_serve_task,
|
||||
similarity_dht_maintenance_task,
|
||||
similarity_dht_sync_task,
|
||||
],
|
||||
tasks,
|
||||
});
|
||||
self.set_error(None);
|
||||
Ok(true)
|
||||
@@ -895,6 +937,7 @@ impl Federation {
|
||||
|
||||
async fn stop(&self) {
|
||||
let mut guard = self.running.lock().await;
|
||||
self.podcast_dht.send_replace(None);
|
||||
stop_running(guard.take()).await;
|
||||
}
|
||||
|
||||
@@ -1088,7 +1131,12 @@ impl Federation {
|
||||
recovery_count: self.recovery_count.load(Ordering::Relaxed),
|
||||
last_sync: lock(&self.last_sync).clone(),
|
||||
last_error: lock(&self.last_error).clone(),
|
||||
protocols: ProtocolVersions::snapshot(&self.observed_protocols),
|
||||
protocols: ProtocolVersions::snapshot(
|
||||
&self.observed_protocols,
|
||||
guard
|
||||
.as_ref()
|
||||
.is_some_and(|running| running.podcast_dht.is_some()),
|
||||
),
|
||||
..FedStatus::default()
|
||||
};
|
||||
if let Some(running) = guard.as_ref() {
|
||||
@@ -2804,6 +2852,31 @@ async fn similarity_route_sync_loop(
|
||||
}
|
||||
}
|
||||
|
||||
/// Opens the podcast-discovery overlay and takes its stream acceptor. A
|
||||
/// failure (unreadable or newer cache, …) is logged and only disables public
|
||||
/// podcast discovery: the acceptor is dropped so the overlay's ALPN refuses
|
||||
/// streams, and the overlay is not advertised.
|
||||
async fn open_podcast_dht(
|
||||
service: &Arc<MusicDhtService>,
|
||||
data_dir: &Path,
|
||||
) -> Option<(Arc<PodcastDht>, music_dht::StreamAcceptor)> {
|
||||
let acceptor = match service.stream_acceptor(PODCAST_DHT_ALPN) {
|
||||
Ok(acceptor) => acceptor,
|
||||
Err(err) => {
|
||||
tracing::warn!("podcast discovery is off: cannot take its acceptor: {err}");
|
||||
return None;
|
||||
}
|
||||
};
|
||||
match PodcastDht::open(Arc::clone(service), data_dir.join(PODCAST_DHT_CACHE_FILE)).await {
|
||||
Ok(dht) => Some((dht, acceptor)),
|
||||
Err(err) => {
|
||||
tracing::warn!("podcast discovery is off: cannot open its cache: {err}");
|
||||
drop(acceptor);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn stop_running(running: Option<Running>) {
|
||||
let Some(running) = running else { return };
|
||||
for task in &running.tasks {
|
||||
|
||||
@@ -135,3 +135,35 @@ fn artist_catalog_discovery_prioritizes_matching_owners_and_bounds_fanout() {
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// The podcast-discovery overlay opens next to the other overlay caches; a
|
||||
/// cache that cannot be opened only turns discovery off.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn podcast_discovery_opens_or_turns_itself_off() {
|
||||
async fn service(dir: &Path) -> Arc<MusicDhtService> {
|
||||
let config = MusicDhtConfig::builder()
|
||||
.data_dir(dir)
|
||||
.network_id(NetworkId::from_name("furumi-tui-podcast-dht-open-test"))
|
||||
.schema_independent_stream_protocol(PODCAST_DHT_ALPN)
|
||||
.build()
|
||||
.unwrap();
|
||||
let (service, mut events) = MusicDhtService::start(config).await.unwrap();
|
||||
tokio::spawn(async move { while events.recv().await.is_some() {} });
|
||||
Arc::new(service)
|
||||
}
|
||||
|
||||
let ok_dir = tempfile::tempdir().unwrap();
|
||||
let ok = service(ok_dir.path()).await;
|
||||
let opened = open_podcast_dht(&ok, ok_dir.path()).await;
|
||||
assert!(opened.is_some());
|
||||
assert!(ok_dir.path().join(PODCAST_DHT_CACHE_FILE).is_file());
|
||||
drop(opened);
|
||||
ok.shutdown().await.unwrap();
|
||||
|
||||
// A directory where the cache file belongs cannot be opened as SQLite.
|
||||
let broken_dir = tempfile::tempdir().unwrap();
|
||||
std::fs::create_dir(broken_dir.path().join(PODCAST_DHT_CACHE_FILE)).unwrap();
|
||||
let broken = service(broken_dir.path()).await;
|
||||
assert!(open_podcast_dht(&broken, broken_dir.path()).await.is_none());
|
||||
broken.shutdown().await.unwrap();
|
||||
}
|
||||
Reference in new issue
Block a user