Users see the kind of failure only; resolved addresses and transport details go to the logs, without URLs. Feed and audio requests are limited to the web ports. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
323 lines
11 KiB
Rust
323 lines
11 KiB
Rust
//! Conditional, size- and time-bounded feed downloads through [`SafeHttp`].
|
|
|
|
use std::fmt;
|
|
use std::time::Duration;
|
|
|
|
use reqwest::header::{
|
|
ACCEPT, ACCEPT_ENCODING, ETAG, HeaderMap, HeaderValue, IF_MODIFIED_SINCE, IF_NONE_MATCH,
|
|
LAST_MODIFIED, RETRY_AFTER,
|
|
};
|
|
use reqwest::{StatusCode, Url};
|
|
|
|
use music_dht::podcast::FeedUrl;
|
|
use music_dht::podcast::feed::{ParsedFeed, parse_feed_with_base};
|
|
|
|
use super::net::{GuardError, REFUSED_MESSAGE, SafeHttp, SafeHttpConfig};
|
|
|
|
/// Hard cap on a feed document (large catalogs are a few MiB).
|
|
pub(crate) const MAX_FEED_BYTES: usize = 32 * 1024 * 1024;
|
|
/// Whole fetch, from connect to the last body byte.
|
|
pub(crate) const FEED_FETCH_TIMEOUT: Duration = Duration::from_secs(60);
|
|
/// Redirect hops followed for a feed.
|
|
pub(crate) const MAX_FEED_REDIRECTS: usize = 5;
|
|
/// `Retry-After` values above this are clamped.
|
|
const MAX_RETRY_AFTER: Duration = Duration::from_secs(24 * 60 * 60);
|
|
|
|
const FEED_ACCEPT: &str = "application/rss+xml, application/atom+xml, application/xml;q=0.9, \
|
|
text/xml;q=0.9, */*;q=0.5";
|
|
|
|
/// HTTP client settings of the feed fetcher.
|
|
pub(crate) const FEED_HTTP: SafeHttpConfig = SafeHttpConfig {
|
|
connect_timeout: Duration::from_secs(10),
|
|
read_timeout: Duration::from_secs(30),
|
|
total_timeout: Some(FEED_FETCH_TIMEOUT),
|
|
max_redirects: MAX_FEED_REDIRECTS,
|
|
};
|
|
|
|
/// Validators of the previous successful fetch.
|
|
#[derive(Debug, Clone, Default, PartialEq, Eq, Hash)]
|
|
pub(crate) struct Validators {
|
|
pub(crate) etag: Option<String>,
|
|
pub(crate) last_modified: Option<String>,
|
|
}
|
|
|
|
/// A successfully fetched and parsed document.
|
|
#[derive(Debug, Clone)]
|
|
pub(crate) struct FetchedFeed {
|
|
pub(crate) feed: ParsedFeed,
|
|
/// Where the feed now lives: the final URL after a chain of permanent
|
|
/// redirects, otherwise the requested URL.
|
|
pub(crate) fetched_from: FeedUrl,
|
|
/// Whether the feed moved: at least one redirect, all permanent (see
|
|
/// [`super::net::Followed::moved_permanently`]), to a valid feed URL.
|
|
pub(crate) moved_permanently: bool,
|
|
/// New validators for the next conditional request.
|
|
pub(crate) validators: Validators,
|
|
}
|
|
|
|
/// Outcome of [`FeedFetcher::fetch`].
|
|
#[derive(Debug, Clone)]
|
|
pub(crate) enum FeedFetch {
|
|
/// `304 Not Modified`: the stored catalog is current.
|
|
NotModified,
|
|
/// A new document.
|
|
Fetched(Box<FetchedFeed>),
|
|
}
|
|
|
|
/// Kind of a failed fetch.
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
pub(crate) enum FetchErrorKind {
|
|
/// Refused by the SSRF guard (blocked address or port, scheme,
|
|
/// credentials).
|
|
Refused,
|
|
/// Non-success HTTP status.
|
|
Status(u16),
|
|
/// Body over [`MAX_FEED_BYTES`].
|
|
TooLarge,
|
|
/// The document is not a feed.
|
|
Parse,
|
|
/// Timed out.
|
|
Timeout,
|
|
/// DNS, connect, TLS or a dropped connection.
|
|
Transport,
|
|
}
|
|
|
|
/// A failed fetch. The stored catalog is never touched on failure.
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
pub(crate) struct FetchError {
|
|
pub(crate) kind: FetchErrorKind,
|
|
/// Details for the server logs (no URLs; see [`Self::public_message`]
|
|
/// for what users see).
|
|
pub(crate) message: String,
|
|
/// Server-requested delay (`Retry-After` on 429/503), clamped to a day.
|
|
pub(crate) retry_after: Option<Duration>,
|
|
}
|
|
|
|
impl FetchError {
|
|
fn new(kind: FetchErrorKind, message: impl Into<String>) -> Self {
|
|
Self {
|
|
kind,
|
|
message: message.into(),
|
|
retry_after: None,
|
|
}
|
|
}
|
|
|
|
/// What the signed-in user is told (API answers, the stored last
|
|
/// error): the kind of failure, never resolved addresses or transport
|
|
/// details, which would let users probe the server's network.
|
|
pub(crate) fn public_message(&self) -> String {
|
|
match &self.kind {
|
|
FetchErrorKind::Refused => REFUSED_MESSAGE.to_string(),
|
|
FetchErrorKind::Status(status) => format!("the server answered HTTP {status}"),
|
|
FetchErrorKind::TooLarge => format!(
|
|
"the feed is larger than {} MiB",
|
|
MAX_FEED_BYTES / (1024 * 1024)
|
|
),
|
|
FetchErrorKind::Parse => "the document is not a podcast feed".to_string(),
|
|
FetchErrorKind::Timeout => "the request timed out".to_string(),
|
|
FetchErrorKind::Transport => "could not connect to the server".to_string(),
|
|
}
|
|
}
|
|
}
|
|
|
|
impl fmt::Display for FetchError {
|
|
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
|
f.write_str(&self.message)
|
|
}
|
|
}
|
|
|
|
impl std::error::Error for FetchError {}
|
|
|
|
impl From<GuardError> for FetchError {
|
|
fn from(error: GuardError) -> Self {
|
|
let kind = if error.is_refusal() {
|
|
FetchErrorKind::Refused
|
|
} else if error.is_timeout() {
|
|
FetchErrorKind::Timeout
|
|
} else {
|
|
FetchErrorKind::Transport
|
|
};
|
|
Self::new(kind, error.to_string())
|
|
}
|
|
}
|
|
|
|
/// Parses `Retry-After` (delta seconds or an HTTP date).
|
|
pub(crate) fn parse_retry_after(
|
|
value: &str,
|
|
now: chrono::DateTime<chrono::Utc>,
|
|
) -> Option<Duration> {
|
|
let value = value.trim();
|
|
let delay = if let Ok(seconds) = value.parse::<u64>() {
|
|
Duration::from_secs(seconds)
|
|
} else {
|
|
let at = chrono::DateTime::parse_from_rfc2822(value).ok()?;
|
|
(at.with_timezone(&chrono::Utc) - now)
|
|
.to_std()
|
|
.unwrap_or_default()
|
|
};
|
|
Some(delay.min(MAX_RETRY_AFTER))
|
|
}
|
|
|
|
/// Downloads feeds with the SSRF guard, conditional requests, redirects
|
|
/// and hard size/time limits; parses them with the shared parser.
|
|
#[derive(Debug, Clone)]
|
|
pub(crate) struct FeedFetcher {
|
|
http: SafeHttp,
|
|
max_bytes: usize,
|
|
}
|
|
|
|
impl FeedFetcher {
|
|
/// The production fetcher (public addresses only).
|
|
pub(crate) fn new() -> anyhow::Result<Self> {
|
|
Ok(Self::with_http(SafeHttp::public(FEED_HTTP)?))
|
|
}
|
|
|
|
pub(crate) fn with_http(http: SafeHttp) -> Self {
|
|
Self {
|
|
http,
|
|
max_bytes: MAX_FEED_BYTES,
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
pub(crate) fn with_max_bytes(mut self, max_bytes: usize) -> Self {
|
|
self.max_bytes = max_bytes;
|
|
self
|
|
}
|
|
|
|
/// Fetches `url`, conditionally when `validators` are known.
|
|
pub(crate) async fn fetch(
|
|
&self,
|
|
url: &FeedUrl,
|
|
validators: &Validators,
|
|
) -> Result<FeedFetch, FetchError> {
|
|
match tokio::time::timeout(FEED_FETCH_TIMEOUT, self.fetch_inner(url, validators)).await {
|
|
Ok(result) => result,
|
|
Err(_) => Err(FetchError::new(
|
|
FetchErrorKind::Timeout,
|
|
format!(
|
|
"feed fetch timed out after {} s",
|
|
FEED_FETCH_TIMEOUT.as_secs()
|
|
),
|
|
)),
|
|
}
|
|
}
|
|
|
|
async fn fetch_inner(
|
|
&self,
|
|
feed_url: &FeedUrl,
|
|
validators: &Validators,
|
|
) -> Result<FeedFetch, FetchError> {
|
|
let url = Url::parse(feed_url.as_str())
|
|
.map_err(|error| FetchError::new(FetchErrorKind::Refused, error.to_string()))?;
|
|
let mut headers = HeaderMap::new();
|
|
headers.insert(ACCEPT, HeaderValue::from_static(FEED_ACCEPT));
|
|
// Compressed bodies are not negotiated (no decoder is enabled), so
|
|
// the size cap applies to what is parsed.
|
|
headers.insert(ACCEPT_ENCODING, HeaderValue::from_static("identity"));
|
|
if let Some(etag) = validators
|
|
.etag
|
|
.as_deref()
|
|
.and_then(|value| HeaderValue::from_str(value).ok())
|
|
{
|
|
headers.insert(IF_NONE_MATCH, etag);
|
|
}
|
|
if let Some(date) = validators
|
|
.last_modified
|
|
.as_deref()
|
|
.and_then(|value| HeaderValue::from_str(value).ok())
|
|
{
|
|
headers.insert(IF_MODIFIED_SINCE, date);
|
|
}
|
|
let followed = self.http.get(url, &headers).await?;
|
|
let mut moved_permanently = followed.moved_permanently();
|
|
let final_url = followed.final_url;
|
|
let response = followed.response;
|
|
let status = response.status();
|
|
if status == StatusCode::NOT_MODIFIED {
|
|
return Ok(FeedFetch::NotModified);
|
|
}
|
|
if !status.is_success() {
|
|
let mut error = FetchError::new(
|
|
FetchErrorKind::Status(status.as_u16()),
|
|
format!("feed request failed: HTTP {status}"),
|
|
);
|
|
if matches!(status.as_u16(), 429 | 503) {
|
|
error.retry_after = response
|
|
.headers()
|
|
.get(RETRY_AFTER)
|
|
.and_then(|value| value.to_str().ok())
|
|
.and_then(|value| parse_retry_after(value, chrono::Utc::now()));
|
|
}
|
|
return Err(error);
|
|
}
|
|
let header = |name| {
|
|
response
|
|
.headers()
|
|
.get(name)
|
|
.and_then(|value: &HeaderValue| value.to_str().ok())
|
|
.map(str::to_string)
|
|
};
|
|
let validators = Validators {
|
|
etag: header(ETAG),
|
|
last_modified: header(LAST_MODIFIED),
|
|
};
|
|
let body = read_capped(response, self.max_bytes).await?;
|
|
// Relative enclosure URLs resolve against where the feed really is.
|
|
let base = final_url.to_string();
|
|
let feed = tokio::task::spawn_blocking(move || parse_feed_with_base(&body, Some(&base)))
|
|
.await
|
|
.map_err(|error| {
|
|
FetchError::new(
|
|
FetchErrorKind::Parse,
|
|
format!("feed parser task failed: {error}"),
|
|
)
|
|
})?
|
|
.map_err(|error| {
|
|
FetchError::new(
|
|
FetchErrorKind::Parse,
|
|
format!("not a podcast feed: {error}"),
|
|
)
|
|
})?;
|
|
// Only a permanent move changes the feed URL; a target that is not a
|
|
// valid feed URL (over-long, ...) is not learned.
|
|
let moved_to = moved_permanently
|
|
.then(|| FeedUrl::parse(final_url.as_str()).ok())
|
|
.flatten();
|
|
moved_permanently = moved_to.is_some();
|
|
let fetched_from = moved_to.unwrap_or_else(|| feed_url.clone());
|
|
Ok(FeedFetch::Fetched(Box::new(FetchedFeed {
|
|
feed,
|
|
fetched_from,
|
|
moved_permanently,
|
|
validators,
|
|
})))
|
|
}
|
|
}
|
|
|
|
async fn read_capped(mut response: reqwest::Response, cap: usize) -> Result<Vec<u8>, FetchError> {
|
|
let too_large = |detail: String| {
|
|
FetchError::new(
|
|
FetchErrorKind::TooLarge,
|
|
format!("feed is too large ({detail}, limit {cap} bytes)"),
|
|
)
|
|
};
|
|
if let Some(len) = response.content_length()
|
|
&& len > cap as u64
|
|
{
|
|
return Err(too_large(format!("{len} bytes")));
|
|
}
|
|
let mut body = Vec::new();
|
|
while let Some(chunk) = response
|
|
.chunk()
|
|
.await
|
|
.map_err(|error| FetchError::from(GuardError::from(error)))?
|
|
{
|
|
if body.len() + chunk.len() > cap {
|
|
return Err(too_large("body exceeds the limit".to_string()));
|
|
}
|
|
body.extend_from_slice(&chunk);
|
|
}
|
|
Ok(body)
|
|
}
|