//! Conditional, size- and time-bounded feed downloads through [`SafeHttp`]. use std::fmt; use std::time::Duration; use reqwest::header::{ ACCEPT, ACCEPT_ENCODING, ETAG, HeaderMap, HeaderValue, IF_MODIFIED_SINCE, IF_NONE_MATCH, LAST_MODIFIED, RETRY_AFTER, }; use reqwest::{StatusCode, Url}; use music_dht::podcast::FeedUrl; use music_dht::podcast::feed::{ParsedFeed, parse_feed_with_base}; use super::net::{GuardError, REFUSED_MESSAGE, SafeHttp, SafeHttpConfig}; /// Hard cap on a feed document (large catalogs are a few MiB). pub(crate) const MAX_FEED_BYTES: usize = 32 * 1024 * 1024; /// Whole fetch, from connect to the last body byte. pub(crate) const FEED_FETCH_TIMEOUT: Duration = Duration::from_secs(60); /// Redirect hops followed for a feed. pub(crate) const MAX_FEED_REDIRECTS: usize = 5; /// `Retry-After` values above this are clamped. const MAX_RETRY_AFTER: Duration = Duration::from_secs(24 * 60 * 60); const FEED_ACCEPT: &str = "application/rss+xml, application/atom+xml, application/xml;q=0.9, \ text/xml;q=0.9, */*;q=0.5"; /// HTTP client settings of the feed fetcher. pub(crate) const FEED_HTTP: SafeHttpConfig = SafeHttpConfig { connect_timeout: Duration::from_secs(10), read_timeout: Duration::from_secs(30), total_timeout: Some(FEED_FETCH_TIMEOUT), max_redirects: MAX_FEED_REDIRECTS, }; /// Validators of the previous successful fetch. #[derive(Debug, Clone, Default, PartialEq, Eq, Hash)] pub(crate) struct Validators { pub(crate) etag: Option, pub(crate) last_modified: Option, } /// A successfully fetched and parsed document. #[derive(Debug, Clone)] pub(crate) struct FetchedFeed { pub(crate) feed: ParsedFeed, /// Where the feed now lives: the final URL after a chain of permanent /// redirects, otherwise the requested URL. pub(crate) fetched_from: FeedUrl, /// Whether the feed moved: at least one redirect, all permanent (see /// [`super::net::Followed::moved_permanently`]), to a valid feed URL. pub(crate) moved_permanently: bool, /// New validators for the next conditional request. pub(crate) validators: Validators, } /// Outcome of [`FeedFetcher::fetch`]. #[derive(Debug, Clone)] pub(crate) enum FeedFetch { /// `304 Not Modified`: the stored catalog is current. NotModified, /// A new document. Fetched(Box), } /// Kind of a failed fetch. #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) enum FetchErrorKind { /// Refused by the SSRF guard (blocked address or port, scheme, /// credentials). Refused, /// Non-success HTTP status. Status(u16), /// Body over [`MAX_FEED_BYTES`]. TooLarge, /// The document is not a feed. Parse, /// Timed out. Timeout, /// DNS, connect, TLS or a dropped connection. Transport, } /// A failed fetch. The stored catalog is never touched on failure. #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct FetchError { pub(crate) kind: FetchErrorKind, /// Details for the server logs (no URLs; see [`Self::public_message`] /// for what users see). pub(crate) message: String, /// Server-requested delay (`Retry-After` on 429/503), clamped to a day. pub(crate) retry_after: Option, } impl FetchError { fn new(kind: FetchErrorKind, message: impl Into) -> Self { Self { kind, message: message.into(), retry_after: None, } } /// What the signed-in user is told (API answers, the stored last /// error): the kind of failure, never resolved addresses or transport /// details, which would let users probe the server's network. pub(crate) fn public_message(&self) -> String { match &self.kind { FetchErrorKind::Refused => REFUSED_MESSAGE.to_string(), FetchErrorKind::Status(status) => format!("the server answered HTTP {status}"), FetchErrorKind::TooLarge => format!( "the feed is larger than {} MiB", MAX_FEED_BYTES / (1024 * 1024) ), FetchErrorKind::Parse => "the document is not a podcast feed".to_string(), FetchErrorKind::Timeout => "the request timed out".to_string(), FetchErrorKind::Transport => "could not connect to the server".to_string(), } } } impl fmt::Display for FetchError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str(&self.message) } } impl std::error::Error for FetchError {} impl From for FetchError { fn from(error: GuardError) -> Self { let kind = if error.is_refusal() { FetchErrorKind::Refused } else if error.is_timeout() { FetchErrorKind::Timeout } else { FetchErrorKind::Transport }; Self::new(kind, error.to_string()) } } /// Parses `Retry-After` (delta seconds or an HTTP date). pub(crate) fn parse_retry_after( value: &str, now: chrono::DateTime, ) -> Option { let value = value.trim(); let delay = if let Ok(seconds) = value.parse::() { Duration::from_secs(seconds) } else { let at = chrono::DateTime::parse_from_rfc2822(value).ok()?; (at.with_timezone(&chrono::Utc) - now) .to_std() .unwrap_or_default() }; Some(delay.min(MAX_RETRY_AFTER)) } /// Downloads feeds with the SSRF guard, conditional requests, redirects /// and hard size/time limits; parses them with the shared parser. #[derive(Debug, Clone)] pub(crate) struct FeedFetcher { http: SafeHttp, max_bytes: usize, } impl FeedFetcher { /// The production fetcher (public addresses only). pub(crate) fn new() -> anyhow::Result { Ok(Self::with_http(SafeHttp::public(FEED_HTTP)?)) } pub(crate) fn with_http(http: SafeHttp) -> Self { Self { http, max_bytes: MAX_FEED_BYTES, } } #[cfg(test)] pub(crate) fn with_max_bytes(mut self, max_bytes: usize) -> Self { self.max_bytes = max_bytes; self } /// Fetches `url`, conditionally when `validators` are known. pub(crate) async fn fetch( &self, url: &FeedUrl, validators: &Validators, ) -> Result { match tokio::time::timeout(FEED_FETCH_TIMEOUT, self.fetch_inner(url, validators)).await { Ok(result) => result, Err(_) => Err(FetchError::new( FetchErrorKind::Timeout, format!( "feed fetch timed out after {} s", FEED_FETCH_TIMEOUT.as_secs() ), )), } } async fn fetch_inner( &self, feed_url: &FeedUrl, validators: &Validators, ) -> Result { let url = Url::parse(feed_url.as_str()) .map_err(|error| FetchError::new(FetchErrorKind::Refused, error.to_string()))?; let mut headers = HeaderMap::new(); headers.insert(ACCEPT, HeaderValue::from_static(FEED_ACCEPT)); // Compressed bodies are not negotiated (no decoder is enabled), so // the size cap applies to what is parsed. headers.insert(ACCEPT_ENCODING, HeaderValue::from_static("identity")); if let Some(etag) = validators .etag .as_deref() .and_then(|value| HeaderValue::from_str(value).ok()) { headers.insert(IF_NONE_MATCH, etag); } if let Some(date) = validators .last_modified .as_deref() .and_then(|value| HeaderValue::from_str(value).ok()) { headers.insert(IF_MODIFIED_SINCE, date); } let followed = self.http.get(url, &headers).await?; let mut moved_permanently = followed.moved_permanently(); let final_url = followed.final_url; let response = followed.response; let status = response.status(); if status == StatusCode::NOT_MODIFIED { return Ok(FeedFetch::NotModified); } if !status.is_success() { let mut error = FetchError::new( FetchErrorKind::Status(status.as_u16()), format!("feed request failed: HTTP {status}"), ); if matches!(status.as_u16(), 429 | 503) { error.retry_after = response .headers() .get(RETRY_AFTER) .and_then(|value| value.to_str().ok()) .and_then(|value| parse_retry_after(value, chrono::Utc::now())); } return Err(error); } let header = |name| { response .headers() .get(name) .and_then(|value: &HeaderValue| value.to_str().ok()) .map(str::to_string) }; let validators = Validators { etag: header(ETAG), last_modified: header(LAST_MODIFIED), }; let body = read_capped(response, self.max_bytes).await?; // Relative enclosure URLs resolve against where the feed really is. let base = final_url.to_string(); let feed = tokio::task::spawn_blocking(move || parse_feed_with_base(&body, Some(&base))) .await .map_err(|error| { FetchError::new( FetchErrorKind::Parse, format!("feed parser task failed: {error}"), ) })? .map_err(|error| { FetchError::new( FetchErrorKind::Parse, format!("not a podcast feed: {error}"), ) })?; // Only a permanent move changes the feed URL; a target that is not a // valid feed URL (over-long, ...) is not learned. let moved_to = moved_permanently .then(|| FeedUrl::parse(final_url.as_str()).ok()) .flatten(); moved_permanently = moved_to.is_some(); let fetched_from = moved_to.unwrap_or_else(|| feed_url.clone()); Ok(FeedFetch::Fetched(Box::new(FetchedFeed { feed, fetched_from, moved_permanently, validators, }))) } } async fn read_capped(mut response: reqwest::Response, cap: usize) -> Result, FetchError> { let too_large = |detail: String| { FetchError::new( FetchErrorKind::TooLarge, format!("feed is too large ({detail}, limit {cap} bytes)"), ) }; if let Some(len) = response.content_length() && len > cap as u64 { return Err(too_large(format!("{len} bytes"))); } let mut body = Vec::new(); while let Some(chunk) = response .chunk() .await .map_err(|error| FetchError::from(GuardError::from(error)))? { if body.len() + chunk.len() > cap { return Err(too_large("body exceeds the limit".to_string())); } body.extend_from_slice(&chunk); } Ok(body) }