Files
furumi-web/src/podcasts/fetch.rs
T
sanapachandClaude Opus 5.5 c51a9af9e5 Keep network details and URLs out of podcast fetch errors
Users see the kind of failure only; resolved addresses and transport
details go to the logs, without URLs. Feed and audio requests are
limited to the web ports.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 15:12:38 +02:00

323 lines
11 KiB
Rust

//! Conditional, size- and time-bounded feed downloads through [`SafeHttp`].
use std::fmt;
use std::time::Duration;
use reqwest::header::{
ACCEPT, ACCEPT_ENCODING, ETAG, HeaderMap, HeaderValue, IF_MODIFIED_SINCE, IF_NONE_MATCH,
LAST_MODIFIED, RETRY_AFTER,
};
use reqwest::{StatusCode, Url};
use music_dht::podcast::FeedUrl;
use music_dht::podcast::feed::{ParsedFeed, parse_feed_with_base};
use super::net::{GuardError, REFUSED_MESSAGE, SafeHttp, SafeHttpConfig};
/// Hard cap on a feed document (large catalogs are a few MiB).
pub(crate) const MAX_FEED_BYTES: usize = 32 * 1024 * 1024;
/// Whole fetch, from connect to the last body byte.
pub(crate) const FEED_FETCH_TIMEOUT: Duration = Duration::from_secs(60);
/// Redirect hops followed for a feed.
pub(crate) const MAX_FEED_REDIRECTS: usize = 5;
/// `Retry-After` values above this are clamped.
const MAX_RETRY_AFTER: Duration = Duration::from_secs(24 * 60 * 60);
const FEED_ACCEPT: &str = "application/rss+xml, application/atom+xml, application/xml;q=0.9, \
text/xml;q=0.9, */*;q=0.5";
/// HTTP client settings of the feed fetcher.
pub(crate) const FEED_HTTP: SafeHttpConfig = SafeHttpConfig {
connect_timeout: Duration::from_secs(10),
read_timeout: Duration::from_secs(30),
total_timeout: Some(FEED_FETCH_TIMEOUT),
max_redirects: MAX_FEED_REDIRECTS,
};
/// Validators of the previous successful fetch.
#[derive(Debug, Clone, Default, PartialEq, Eq, Hash)]
pub(crate) struct Validators {
pub(crate) etag: Option<String>,
pub(crate) last_modified: Option<String>,
}
/// A successfully fetched and parsed document.
#[derive(Debug, Clone)]
pub(crate) struct FetchedFeed {
pub(crate) feed: ParsedFeed,
/// Where the feed now lives: the final URL after a chain of permanent
/// redirects, otherwise the requested URL.
pub(crate) fetched_from: FeedUrl,
/// Whether the feed moved: at least one redirect, all permanent (see
/// [`super::net::Followed::moved_permanently`]), to a valid feed URL.
pub(crate) moved_permanently: bool,
/// New validators for the next conditional request.
pub(crate) validators: Validators,
}
/// Outcome of [`FeedFetcher::fetch`].
#[derive(Debug, Clone)]
pub(crate) enum FeedFetch {
/// `304 Not Modified`: the stored catalog is current.
NotModified,
/// A new document.
Fetched(Box<FetchedFeed>),
}
/// Kind of a failed fetch.
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum FetchErrorKind {
/// Refused by the SSRF guard (blocked address or port, scheme,
/// credentials).
Refused,
/// Non-success HTTP status.
Status(u16),
/// Body over [`MAX_FEED_BYTES`].
TooLarge,
/// The document is not a feed.
Parse,
/// Timed out.
Timeout,
/// DNS, connect, TLS or a dropped connection.
Transport,
}
/// A failed fetch. The stored catalog is never touched on failure.
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct FetchError {
pub(crate) kind: FetchErrorKind,
/// Details for the server logs (no URLs; see [`Self::public_message`]
/// for what users see).
pub(crate) message: String,
/// Server-requested delay (`Retry-After` on 429/503), clamped to a day.
pub(crate) retry_after: Option<Duration>,
}
impl FetchError {
fn new(kind: FetchErrorKind, message: impl Into<String>) -> Self {
Self {
kind,
message: message.into(),
retry_after: None,
}
}
/// What the signed-in user is told (API answers, the stored last
/// error): the kind of failure, never resolved addresses or transport
/// details, which would let users probe the server's network.
pub(crate) fn public_message(&self) -> String {
match &self.kind {
FetchErrorKind::Refused => REFUSED_MESSAGE.to_string(),
FetchErrorKind::Status(status) => format!("the server answered HTTP {status}"),
FetchErrorKind::TooLarge => format!(
"the feed is larger than {} MiB",
MAX_FEED_BYTES / (1024 * 1024)
),
FetchErrorKind::Parse => "the document is not a podcast feed".to_string(),
FetchErrorKind::Timeout => "the request timed out".to_string(),
FetchErrorKind::Transport => "could not connect to the server".to_string(),
}
}
}
impl fmt::Display for FetchError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(&self.message)
}
}
impl std::error::Error for FetchError {}
impl From<GuardError> for FetchError {
fn from(error: GuardError) -> Self {
let kind = if error.is_refusal() {
FetchErrorKind::Refused
} else if error.is_timeout() {
FetchErrorKind::Timeout
} else {
FetchErrorKind::Transport
};
Self::new(kind, error.to_string())
}
}
/// Parses `Retry-After` (delta seconds or an HTTP date).
pub(crate) fn parse_retry_after(
value: &str,
now: chrono::DateTime<chrono::Utc>,
) -> Option<Duration> {
let value = value.trim();
let delay = if let Ok(seconds) = value.parse::<u64>() {
Duration::from_secs(seconds)
} else {
let at = chrono::DateTime::parse_from_rfc2822(value).ok()?;
(at.with_timezone(&chrono::Utc) - now)
.to_std()
.unwrap_or_default()
};
Some(delay.min(MAX_RETRY_AFTER))
}
/// Downloads feeds with the SSRF guard, conditional requests, redirects
/// and hard size/time limits; parses them with the shared parser.
#[derive(Debug, Clone)]
pub(crate) struct FeedFetcher {
http: SafeHttp,
max_bytes: usize,
}
impl FeedFetcher {
/// The production fetcher (public addresses only).
pub(crate) fn new() -> anyhow::Result<Self> {
Ok(Self::with_http(SafeHttp::public(FEED_HTTP)?))
}
pub(crate) fn with_http(http: SafeHttp) -> Self {
Self {
http,
max_bytes: MAX_FEED_BYTES,
}
}
#[cfg(test)]
pub(crate) fn with_max_bytes(mut self, max_bytes: usize) -> Self {
self.max_bytes = max_bytes;
self
}
/// Fetches `url`, conditionally when `validators` are known.
pub(crate) async fn fetch(
&self,
url: &FeedUrl,
validators: &Validators,
) -> Result<FeedFetch, FetchError> {
match tokio::time::timeout(FEED_FETCH_TIMEOUT, self.fetch_inner(url, validators)).await {
Ok(result) => result,
Err(_) => Err(FetchError::new(
FetchErrorKind::Timeout,
format!(
"feed fetch timed out after {} s",
FEED_FETCH_TIMEOUT.as_secs()
),
)),
}
}
async fn fetch_inner(
&self,
feed_url: &FeedUrl,
validators: &Validators,
) -> Result<FeedFetch, FetchError> {
let url = Url::parse(feed_url.as_str())
.map_err(|error| FetchError::new(FetchErrorKind::Refused, error.to_string()))?;
let mut headers = HeaderMap::new();
headers.insert(ACCEPT, HeaderValue::from_static(FEED_ACCEPT));
// Compressed bodies are not negotiated (no decoder is enabled), so
// the size cap applies to what is parsed.
headers.insert(ACCEPT_ENCODING, HeaderValue::from_static("identity"));
if let Some(etag) = validators
.etag
.as_deref()
.and_then(|value| HeaderValue::from_str(value).ok())
{
headers.insert(IF_NONE_MATCH, etag);
}
if let Some(date) = validators
.last_modified
.as_deref()
.and_then(|value| HeaderValue::from_str(value).ok())
{
headers.insert(IF_MODIFIED_SINCE, date);
}
let followed = self.http.get(url, &headers).await?;
let mut moved_permanently = followed.moved_permanently();
let final_url = followed.final_url;
let response = followed.response;
let status = response.status();
if status == StatusCode::NOT_MODIFIED {
return Ok(FeedFetch::NotModified);
}
if !status.is_success() {
let mut error = FetchError::new(
FetchErrorKind::Status(status.as_u16()),
format!("feed request failed: HTTP {status}"),
);
if matches!(status.as_u16(), 429 | 503) {
error.retry_after = response
.headers()
.get(RETRY_AFTER)
.and_then(|value| value.to_str().ok())
.and_then(|value| parse_retry_after(value, chrono::Utc::now()));
}
return Err(error);
}
let header = |name| {
response
.headers()
.get(name)
.and_then(|value: &HeaderValue| value.to_str().ok())
.map(str::to_string)
};
let validators = Validators {
etag: header(ETAG),
last_modified: header(LAST_MODIFIED),
};
let body = read_capped(response, self.max_bytes).await?;
// Relative enclosure URLs resolve against where the feed really is.
let base = final_url.to_string();
let feed = tokio::task::spawn_blocking(move || parse_feed_with_base(&body, Some(&base)))
.await
.map_err(|error| {
FetchError::new(
FetchErrorKind::Parse,
format!("feed parser task failed: {error}"),
)
})?
.map_err(|error| {
FetchError::new(
FetchErrorKind::Parse,
format!("not a podcast feed: {error}"),
)
})?;
// Only a permanent move changes the feed URL; a target that is not a
// valid feed URL (over-long, ...) is not learned.
let moved_to = moved_permanently
.then(|| FeedUrl::parse(final_url.as_str()).ok())
.flatten();
moved_permanently = moved_to.is_some();
let fetched_from = moved_to.unwrap_or_else(|| feed_url.clone());
Ok(FeedFetch::Fetched(Box::new(FetchedFeed {
feed,
fetched_from,
moved_permanently,
validators,
})))
}
}
async fn read_capped(mut response: reqwest::Response, cap: usize) -> Result<Vec<u8>, FetchError> {
let too_large = |detail: String| {
FetchError::new(
FetchErrorKind::TooLarge,
format!("feed is too large ({detail}, limit {cap} bytes)"),
)
};
if let Some(len) = response.content_length()
&& len > cap as u64
{
return Err(too_large(format!("{len} bytes")));
}
let mut body = Vec::new();
while let Some(chunk) = response
.chunk()
.await
.map_err(|error| FetchError::from(GuardError::from(error)))?
{
if body.len() + chunk.len() > cap {
return Err(too_large("body exceeds the limit".to_string()));
}
body.extend_from_slice(&chunk);
}
Ok(body)
}