Keep network details and URLs out of podcast fetch errors

Users see the kind of failure only; resolved addresses and transport
details go to the logs, without URLs. Feed and audio requests are
limited to the web ports.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
sanapachandClaude Opus 5.5 committed 2026-11-20 15:12:38 +02:00
1 parent 99b2935b35
commit c51a9af9e5
10 files changed
+256 -42

No files matched your search

+7 -4
View File
@@ -123,10 +123,13 @@ feed metadata only, never episode audio. A failing feed keeps its known
episodes. Accounts subscribed to the same feed share one request.
The server therefore makes outgoing HTTP(S) requests to podcast publishers.
It only connects to public addresses: feed or audio URLs that point to
loopback, private, link-local, CGNAT or cloud metadata addresses, directly,
through DNS or through a redirect, are refused. System proxy settings are not
used for these requests.
It only connects to public addresses on the web ports 80, 443, 8080 and 8443:
feed or audio URLs that point to loopback, private, link-local, CGNAT or cloud
metadata addresses, directly, through DNS or through a redirect, or to any
other port, are refused. System proxy settings are not used for these
requests. Users only see the kind of failure (address not allowed, could not
connect, HTTP status, not a feed); the details, without the URL, are in the
server log and, for scheduled refreshes, in the job log with the podcast id.
Episode audio is streamed by the browser directly from the publisher. Some
publishers cannot be played that way (plain HTTP while this server uses
+2 -2
View File
@@ -58,8 +58,8 @@ impl Job for PodcastRefreshJob {
.await;
conn.close().await?;
let summary = result?;
for (user_id, podcast_id, error) in summary.failures.iter().take(LOGGED_FAILURES) {
log.warn(&format!("user {user_id}, podcast {podcast_id}: {error}"));
for (podcast_id, error) in summary.failures.iter().take(LOGGED_FAILURES) {
log.warn(&format!("podcast {podcast_id}: {error}"));
}
if summary.failures.len() > LOGGED_FAILURES {
log.warn(&format!(
+26 -2
View File
@@ -81,6 +81,9 @@ impl From<anyhow::Error> for ApiError {
FetchErrorKind::Timeout | FetchErrorKind::Transport => 502,
_ => 422,
};
// The user sees the kind of failure only (no resolved
// addresses or transport details); the log keeps the details.
tracing::info!("podcast feed could not be added: {}", fetch.message);
return Self::new(status, error.to_string());
}
tracing::warn!("podcast API request failed: {error:#}");
@@ -975,12 +978,21 @@ mod tests {
let refused: ApiError =
anyhow::Error::from(SubscribeUrlError::Fetch(super::super::fetch::FetchError {
kind: FetchErrorKind::Refused,
message: "address 127.0.0.1 is not public".into(),
message: "intranet.corp resolves to a private address (10.1.2.3)".into(),
retry_after: None,
}))
.into();
assert_eq!(refused.status, 422);
assert!(refused.message.contains("not public"));
assert!(
refused.message.contains("only public"),
"{}",
refused.message
);
assert!(
!refused.message.contains("10.1.2.3"),
"no resolved addresses"
);
assert!(!refused.message.contains("intranet"));
let timeout: ApiError =
anyhow::Error::from(SubscribeUrlError::Fetch(super::super::fetch::FetchError {
kind: FetchErrorKind::Timeout,
@@ -989,6 +1001,18 @@ mod tests {
}))
.into();
assert_eq!(timeout.status, 502);
let transport: ApiError =
anyhow::Error::from(SubscribeUrlError::Fetch(super::super::fetch::FetchError {
kind: FetchErrorKind::Transport,
message: "error sending request: tcp connect error: Connection refused".into(),
retry_after: None,
}))
.into();
assert_eq!(transport.status, 502);
assert_eq!(
transport.message,
"the feed could not be loaded: could not connect to the server"
);
let other: ApiError = anyhow::anyhow!("connection reset").into();
assert_eq!(other.status, 500);
assert_eq!(other.message, "internal error", "internals are not shown");
+28 -9
View File
@@ -12,7 +12,7 @@ use reqwest::{StatusCode, Url};
use music_dht::podcast::FeedUrl;
use music_dht::podcast::feed::{ParsedFeed, parse_feed_with_base};
use super::net::{GuardError, SafeHttp, SafeHttpConfig};
use super::net::{GuardError, REFUSED_MESSAGE, SafeHttp, SafeHttpConfig};
/// Hard cap on a feed document (large catalogs are a few MiB).
pub(crate) const MAX_FEED_BYTES: usize = 32 * 1024 * 1024;
@@ -67,7 +67,8 @@ pub(crate) enum FeedFetch {
/// Kind of a failed fetch.
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum FetchErrorKind {
/// Refused by the SSRF guard (blocked address, scheme, credentials).
/// Refused by the SSRF guard (blocked address or port, scheme,
/// credentials).
Refused,
/// Non-success HTTP status.
Status(u16),
@@ -85,6 +86,8 @@ pub(crate) enum FetchErrorKind {
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct FetchError {
pub(crate) kind: FetchErrorKind,
/// Details for the server logs (no URLs; see [`Self::public_message`]
/// for what users see).
pub(crate) message: String,
/// Server-requested delay (`Retry-After` on 429/503), clamped to a day.
pub(crate) retry_after: Option<Duration>,
@@ -98,6 +101,23 @@ impl FetchError {
retry_after: None,
}
}
/// What the signed-in user is told (API answers, the stored last
/// error): the kind of failure, never resolved addresses or transport
/// details, which would let users probe the server's network.
pub(crate) fn public_message(&self) -> String {
match &self.kind {
FetchErrorKind::Refused => REFUSED_MESSAGE.to_string(),
FetchErrorKind::Status(status) => format!("the server answered HTTP {status}"),
FetchErrorKind::TooLarge => format!(
"the feed is larger than {} MiB",
MAX_FEED_BYTES / (1024 * 1024)
),
FetchErrorKind::Parse => "the document is not a podcast feed".to_string(),
FetchErrorKind::Timeout => "the request timed out".to_string(),
FetchErrorKind::Transport => "could not connect to the server".to_string(),
}
}
}
impl fmt::Display for FetchError {
@@ -110,13 +130,12 @@ impl std::error::Error for FetchError {}
impl From<GuardError> for FetchError {
fn from(error: GuardError) -> Self {
let kind = match &error {
GuardError::UnsupportedScheme(_)
| GuardError::Credentials
| GuardError::MissingHost
| GuardError::Blocked(_) => FetchErrorKind::Refused,
_ if error.is_timeout() => FetchErrorKind::Timeout,
_ => FetchErrorKind::Transport,
let kind = if error.is_refusal() {
FetchErrorKind::Refused
} else if error.is_timeout() {
FetchErrorKind::Timeout
} else {
FetchErrorKind::Transport
};
Self::new(kind, error.to_string())
}
+3 -2
View File
@@ -369,10 +369,11 @@ async fn audio_handler(
let audio = match proxy.fetch(&audio_url, mime.as_deref(), range).await {
Ok(audio) => audio,
Err(error) => {
tracing::info!(user_id = user.id, episode = %episode_id, "podcast proxy: {error}");
// Neither the URL (access tokens) nor the account is logged.
tracing::info!(episode = %episode_id, "podcast proxy: {error}");
let status =
StatusCode::from_u16(error.http_status()).unwrap_or(StatusCode::BAD_GATEWAY);
return Ok(json_error(status, &error.to_string()));
return Ok(json_error(status, &error.public_message()));
}
};
let mut builder = cot::http::Response::builder()
+50 -5
View File
@@ -14,6 +14,9 @@
//! that a second DNS answer could bypass (DNS rebinding);
//! * checks IP-literal hosts the same way (the connector does not resolve
//! them);
//! * connects only to the web ports [`ALLOWED_PORTS`] (80, 443, 8080, 8443):
//! a public address can still be the server's own, and its other services
//! (databases, admin ports) are often reachable from the host itself;
//! * follows redirects itself, at most `max_redirects` hops, checking every
//! hop like the first URL;
//! * never uses a system proxy (it would resolve the target instead of the
@@ -200,26 +203,39 @@ pub(crate) fn classify(ip: IpAddr) -> Option<AddressClass> {
}
}
/// Ports feeds and enclosures may be fetched from.
pub(crate) const ALLOWED_PORTS: [u16; 4] = [80, 443, 8080, 8443];
/// Which addresses [`SafeHttp`] may connect to.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) struct AddressPolicy {
allow_loopback: bool,
any_port: bool,
}
impl AddressPolicy {
/// Public addresses only. The only policy available outside tests.
/// Public addresses on [`ALLOWED_PORTS`] only. The only policy
/// available outside tests.
pub(crate) const PUBLIC_ONLY: Self = Self {
allow_loopback: false,
any_port: false,
};
/// Public addresses plus loopback, for the local test servers.
/// Public addresses plus loopback on any port, for the local test
/// servers.
#[cfg(test)]
pub(crate) const fn test_allow_loopback() -> Self {
Self {
allow_loopback: true,
any_port: true,
}
}
/// Whether a connection to `port` is allowed.
pub(crate) fn allows_port(&self, port: u16) -> bool {
self.any_port || ALLOWED_PORTS.contains(&port)
}
/// Checks one address.
pub(crate) fn check(&self, ip: IpAddr) -> Result<(), AddressClass> {
match classify(ip) {
@@ -350,6 +366,8 @@ pub(crate) enum GuardError {
MissingHost,
/// The host is (or resolves to) a blocked address.
Blocked(BlockedAddress),
/// The port is not one of [`ALLOWED_PORTS`].
Port(u16),
/// More redirects than allowed.
TooManyRedirects(usize),
/// A redirect without a usable `Location`.
@@ -362,8 +380,27 @@ impl GuardError {
pub(crate) fn is_timeout(&self) -> bool {
matches!(self, Self::Http(error) if error.is_timeout())
}
/// Whether the guard refused the URL before or instead of connecting.
pub(crate) fn is_refusal(&self) -> bool {
matches!(
self,
Self::UnsupportedScheme(_)
| Self::Credentials
| Self::MissingHost
| Self::Blocked(_)
| Self::Port(_)
)
}
}
/// What a signed-in user is told about a refused or failed request. The
/// details ([`GuardError`]'s `Display`: resolved addresses, transport error
/// chains) stay in the server logs; showing them would turn feed and audio
/// requests into a scanner of the server's network.
pub(crate) const REFUSED_MESSAGE: &str =
"only public http and https addresses on ports 80, 443, 8080 and 8443 are allowed";
impl fmt::Display for GuardError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
@@ -373,6 +410,7 @@ impl fmt::Display for GuardError {
Self::Credentials => f.write_str("URLs with credentials are not allowed"),
Self::MissingHost => f.write_str("URL has no host"),
Self::Blocked(blocked) => blocked.fmt(f),
Self::Port(port) => write!(f, "port {port} is not allowed"),
Self::TooManyRedirects(limit) => write!(f, "more than {limit} redirects"),
Self::InvalidRedirect(reason) => write!(f, "invalid redirect: {reason}"),
Self::Http(error) => {
@@ -392,7 +430,10 @@ impl std::error::Error for GuardError {}
impl From<reqwest::Error> for GuardError {
/// Surfaces a resolver rejection buried in the transport error chain.
/// The URL is dropped: feed and enclosure URLs often carry access
/// tokens, and these errors end up in logs.
fn from(error: reqwest::Error) -> Self {
let error = error.without_url();
let mut source = std::error::Error::source(&error);
while let Some(inner) = source {
if let Some(blocked) = inner.downcast_ref::<BlockedAddress>() {
@@ -530,9 +571,9 @@ impl SafeHttp {
.ok_or_else(|| {
GuardError::InvalidRedirect(format!("HTTP {status} without Location"))
})?;
let next = url
.join(location)
.map_err(|error| GuardError::InvalidRedirect(format!("{location:?}: {error}")))?;
let next = url.join(location).map_err(|error| {
GuardError::InvalidRedirect(format!("unusable Location: {error}"))
})?;
redirects.push(status);
url = next;
}
@@ -566,5 +607,9 @@ pub(crate) fn check_url(policy: AddressPolicy, url: &Url) -> Result<(), GuardErr
})
})?;
}
let port = url.port_or_known_default().ok_or(GuardError::MissingHost)?;
if !policy.allows_port(port) {
return Err(GuardError::Port(port));
}
Ok(())
}
+87 -7
View File
@@ -182,6 +182,45 @@ fn urls_are_checked_before_connecting() {
assert!(check_url(policy, &url("http://localhost/rss")).is_ok());
}
#[test]
fn only_web_ports_are_allowed() {
// A public address can be the server's own; its databases and admin
// ports must not be reachable through feed or audio requests.
let policy = AddressPolicy::PUBLIC_ONLY;
for text in [
"https://feeds.example.com/rss",
"http://feeds.example.com/rss",
"https://feeds.example.com:443/rss",
"http://feeds.example.com:8080/rss",
"https://feeds.example.com:8443/rss",
"https://8.8.8.8/rss",
] {
assert!(check_url(policy, &url(text)).is_ok(), "{text}");
}
for (text, port) in [
("http://feeds.example.com:5432/", 5432),
("https://feeds.example.com:22/", 22),
("http://8.8.8.8:6379/feed", 6379),
("https://feeds.example.com:3000/", 3000),
] {
match check_url(policy, &url(text)) {
Err(error @ GuardError::Port(refused)) => {
assert_eq!(refused, port, "{text}");
assert!(error.is_refusal());
}
other => panic!("{text}: {other:?}"),
}
}
// The local test servers listen on random ports.
assert!(
check_url(
AddressPolicy::test_allow_loopback(),
&url("http://127.0.0.1:5432/")
)
.is_ok()
);
}
// ---------------------------------------------------------------------------
// Resolver
// ---------------------------------------------------------------------------
@@ -226,14 +265,21 @@ async fn production_policy_never_reaches_a_local_server() {
server.set("/feed", Route::new("ok", "text/plain"));
let lookup = ScriptedLookup::with(&[("loop.test", &[&["127.0.0.1"]])]);
let http = public_http(Arc::clone(&lookup));
for target in [server.url("/feed"), server.url_on("loop.test", "/feed")] {
match http.get(url(&target), &HeaderMap::new()).await {
Err(GuardError::Blocked(blocked)) => {
assert_eq!(blocked.class, AddressClass::Loopback, "{target}")
}
other => panic!("{target}: {other:?}"),
}
match http.get(url(&server.url("/feed")), &HeaderMap::new()).await {
Err(GuardError::Blocked(blocked)) => assert_eq!(blocked.class, AddressClass::Loopback),
other => panic!("{other:?}"),
}
// The test server's random port is refused before any lookup.
let named = url(&server.url_on("loop.test", "/feed"));
match http.get(named.clone(), &HeaderMap::new()).await {
Err(GuardError::Port(port)) => assert_eq!(Some(port), named.port()),
other => panic!("{other:?}"),
}
assert_eq!(lookup.calls.load(Ordering::SeqCst), 0);
// On an allowed port the resolver refuses the loopback answer.
let resolver = GuardedResolver::new(AddressPolicy::PUBLIC_ONLY, lookup);
let error = resolver.resolve_checked("loop.test").await.unwrap_err();
assert_eq!(blocked_class(error.as_ref()), Some(AddressClass::Loopback));
assert_eq!(server.request_count(), 0);
}
@@ -327,6 +373,40 @@ async fn every_redirect_hop_is_checked() {
assert_eq!(followed.final_url.path(), "/hop4");
}
#[tokio::test]
async fn transport_errors_never_carry_the_url() {
// Feed and enclosure URLs often carry access tokens; errors are logged.
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let port = listener.local_addr().unwrap().port();
drop(listener);
let http = test_http(ScriptedLookup::with(&[]));
let error = http
.get(
url(&format!(
"http://127.0.0.1:{port}/private/feed.xml?token=secret"
)),
&HeaderMap::new(),
)
.await
.unwrap_err();
assert!(matches!(error, GuardError::Http(_)), "{error:?}");
let text = error.to_string();
assert!(
!text.contains("secret") && !text.contains("feed.xml"),
"{text}"
);
assert!(!format!("{error:?}").contains("secret"));
// A redirect to an unusable location does not echo it either.
let server = TestServer::start();
server.set("/bad", Route::redirect(302, "http://[::1/?token=secret"));
let error = http
.get(url(&server.url("/bad")), &HeaderMap::new())
.await
.unwrap_err();
assert!(matches!(error, GuardError::InvalidRedirect(_)), "{error:?}");
assert!(!error.to_string().contains("secret"), "{error}");
}
#[tokio::test]
async fn no_cookies_or_credentials_are_sent() {
let server = TestServer::start();
+5
View File
@@ -1903,6 +1903,11 @@ async fn postgres_podcast_api_subscribes_plays_and_keeps_progress() {
.await
.unwrap_err();
assert_eq!(refused.status, 422, "{}", refused.message);
assert_eq!(
refused.message,
"the feed could not be loaded: only public http and https addresses on ports 80, 443, \
8080 and 8443 are allowed"
);
assert!(logged_kinds(&pool, user).await.is_empty());
assert_eq!(server.requests("/show.xml").len(), 0);
+19 -1
View File
@@ -20,7 +20,7 @@ use reqwest::header::{
};
use reqwest::{StatusCode, Url};
use super::net::{GuardError, SafeHttp, SafeHttpConfig};
use super::net::{GuardError, REFUSED_MESSAGE, SafeHttp, SafeHttpConfig};
/// Largest response body passed through (a long episode at a high bitrate
/// stays far below).
@@ -69,6 +69,24 @@ impl ProxyError {
_ => 502,
}
}
/// What the browser is told; the details (`Display`) stay in the
/// server log.
pub(crate) fn public_message(&self) -> String {
match self {
Self::InvalidUrl(_) => "the episode's audio URL is invalid".to_string(),
Self::Upstream(error) if error.is_refusal() => REFUSED_MESSAGE.to_string(),
Self::Upstream(error) if error.is_timeout() => {
"the publisher did not answer in time".to_string()
}
Self::Upstream(_) => "could not connect to the publisher".to_string(),
Self::Status(status) => format!("the publisher answered HTTP {status}"),
Self::ContentType(_) => "the publisher's file is not audio".to_string(),
Self::TooLarge(_) => "the publisher's file is too large".to_string(),
Self::Timeout => "the publisher did not answer in time".to_string(),
Self::Body(_) => "the publisher stream failed".to_string(),
}
}
}
impl fmt::Display for ProxyError {
+29 -10
View File
@@ -68,7 +68,10 @@ pub(crate) enum TargetResult {
NotModified,
/// The fetch or the ingestion failed; the catalog is unchanged.
Failed {
/// What the user sees (stored as the last error).
error: String,
/// Details for the server log (no URLs).
detail: String,
consecutive_failures: u32,
next_check_at_ms: i64,
},
@@ -89,8 +92,9 @@ pub(crate) struct RunSummary {
pub(crate) skipped: usize,
pub(crate) new_episodes: usize,
pub(crate) updated_episodes: usize,
/// `(user_id, podcast_id, error)` of failed targets, for the job log.
pub(crate) failures: Vec<(i64, PodcastId, String)>,
/// `(podcast_id, details)` of failed targets, for the job log (no
/// account ids or URLs: feed URLs often carry access tokens).
pub(crate) failures: Vec<(PodcastId, String)>,
}
impl RunSummary {
@@ -102,10 +106,10 @@ impl RunSummary {
self.updated_episodes += outcome.updated_episodes;
}
TargetResult::NotModified => self.not_modified += 1,
TargetResult::Failed { error, .. } => {
TargetResult::Failed { detail, .. } => {
self.failed += 1;
self.failures
.push((target.user_id, target.podcast_id.clone(), error.clone()));
.push((target.podcast_id.clone(), detail.clone()));
}
TargetResult::Skipped(_) => self.skipped += 1,
}
@@ -232,12 +236,12 @@ async fn record_result(
now_ms: i64,
) -> Result<TargetResult> {
let (user_id, podcast_id) = (target.user_id, &target.podcast_id);
let failure = |error: String, retry_after: Option<Duration>| {
let failure = |error: String, detail: String, retry_after: Option<Duration>| {
let consecutive_failures = previous_failures.saturating_add(1);
let next_check_at_ms = now_ms + failure_backoff_ms(consecutive_failures, retry_after);
(error, consecutive_failures, next_check_at_ms)
(error, detail, consecutive_failures, next_check_at_ms)
};
let (error, consecutive_failures, next_check_at_ms) = match fetched {
let (error, detail, consecutive_failures, next_check_at_ms) = match fetched {
Ok(FeedFetch::NotModified) => {
let next = next_regular_check_ms(now_ms, user_id, podcast_id);
return match store
@@ -254,10 +258,18 @@ async fn record_result(
Err(error) if is_gone(&error) => return skipped_or(error),
// A store failure is recorded like a fetch failure, so the
// feed backs off instead of being retried every run.
Err(error) => failure(format!("storing the feed failed: {error:#}"), None),
Err(error) => failure(
"the feed could not be stored".to_string(),
format!("storing the feed failed: {error:#}"),
None,
),
}
}
Err(error) => failure(error.message.clone(), error.retry_after),
Err(error) => failure(
error.public_message(),
error.message.clone(),
error.retry_after,
),
};
match store
.record_fetch_failure(user_id, podcast_id, now_ms, &error, Some(next_check_at_ms))
@@ -265,6 +277,7 @@ async fn record_result(
{
Ok(_) => Ok(TargetResult::Failed {
error,
detail,
consecutive_failures,
next_check_at_ms,
}),
@@ -408,7 +421,13 @@ pub(crate) enum SubscribeUrlError {
impl fmt::Display for SubscribeUrlError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Fetch(error) => write!(f, "the feed could not be loaded: {}", error.message),
Self::Fetch(error) => {
write!(
f,
"the feed could not be loaded: {}",
error.public_message()
)
}
}
}
}