Keep network details and URLs out of podcast fetch errors

Users see the kind of failure only; resolved addresses and transport
details go to the logs, without URLs. Feed and audio requests are
limited to the web ports.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
sanapachandClaude Opus 5.5 committed 2026-11-20 15:12:38 +02:00
1 parent 99b2935b35
commit c51a9af9e5
10 files changed
+256 -42

No files matched your search

+2 -2
View File
@@ -58,8 +58,8 @@ impl Job for PodcastRefreshJob {
.await;
conn.close().await?;
let summary = result?;
for (user_id, podcast_id, error) in summary.failures.iter().take(LOGGED_FAILURES) {
log.warn(&format!("user {user_id}, podcast {podcast_id}: {error}"));
for (podcast_id, error) in summary.failures.iter().take(LOGGED_FAILURES) {
log.warn(&format!("podcast {podcast_id}: {error}"));
}
if summary.failures.len() > LOGGED_FAILURES {
log.warn(&format!(
+26 -2
View File
@@ -81,6 +81,9 @@ impl From<anyhow::Error> for ApiError {
FetchErrorKind::Timeout | FetchErrorKind::Transport => 502,
_ => 422,
};
// The user sees the kind of failure only (no resolved
// addresses or transport details); the log keeps the details.
tracing::info!("podcast feed could not be added: {}", fetch.message);
return Self::new(status, error.to_string());
}
tracing::warn!("podcast API request failed: {error:#}");
@@ -975,12 +978,21 @@ mod tests {
let refused: ApiError =
anyhow::Error::from(SubscribeUrlError::Fetch(super::super::fetch::FetchError {
kind: FetchErrorKind::Refused,
message: "address 127.0.0.1 is not public".into(),
message: "intranet.corp resolves to a private address (10.1.2.3)".into(),
retry_after: None,
}))
.into();
assert_eq!(refused.status, 422);
assert!(refused.message.contains("not public"));
assert!(
refused.message.contains("only public"),
"{}",
refused.message
);
assert!(
!refused.message.contains("10.1.2.3"),
"no resolved addresses"
);
assert!(!refused.message.contains("intranet"));
let timeout: ApiError =
anyhow::Error::from(SubscribeUrlError::Fetch(super::super::fetch::FetchError {
kind: FetchErrorKind::Timeout,
@@ -989,6 +1001,18 @@ mod tests {
}))
.into();
assert_eq!(timeout.status, 502);
let transport: ApiError =
anyhow::Error::from(SubscribeUrlError::Fetch(super::super::fetch::FetchError {
kind: FetchErrorKind::Transport,
message: "error sending request: tcp connect error: Connection refused".into(),
retry_after: None,
}))
.into();
assert_eq!(transport.status, 502);
assert_eq!(
transport.message,
"the feed could not be loaded: could not connect to the server"
);
let other: ApiError = anyhow::anyhow!("connection reset").into();
assert_eq!(other.status, 500);
assert_eq!(other.message, "internal error", "internals are not shown");
+28 -9
View File
@@ -12,7 +12,7 @@ use reqwest::{StatusCode, Url};
use music_dht::podcast::FeedUrl;
use music_dht::podcast::feed::{ParsedFeed, parse_feed_with_base};
use super::net::{GuardError, SafeHttp, SafeHttpConfig};
use super::net::{GuardError, REFUSED_MESSAGE, SafeHttp, SafeHttpConfig};
/// Hard cap on a feed document (large catalogs are a few MiB).
pub(crate) const MAX_FEED_BYTES: usize = 32 * 1024 * 1024;
@@ -67,7 +67,8 @@ pub(crate) enum FeedFetch {
/// Kind of a failed fetch.
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum FetchErrorKind {
/// Refused by the SSRF guard (blocked address, scheme, credentials).
/// Refused by the SSRF guard (blocked address or port, scheme,
/// credentials).
Refused,
/// Non-success HTTP status.
Status(u16),
@@ -85,6 +86,8 @@ pub(crate) enum FetchErrorKind {
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct FetchError {
pub(crate) kind: FetchErrorKind,
/// Details for the server logs (no URLs; see [`Self::public_message`]
/// for what users see).
pub(crate) message: String,
/// Server-requested delay (`Retry-After` on 429/503), clamped to a day.
pub(crate) retry_after: Option<Duration>,
@@ -98,6 +101,23 @@ impl FetchError {
retry_after: None,
}
}
/// What the signed-in user is told (API answers, the stored last
/// error): the kind of failure, never resolved addresses or transport
/// details, which would let users probe the server's network.
pub(crate) fn public_message(&self) -> String {
match &self.kind {
FetchErrorKind::Refused => REFUSED_MESSAGE.to_string(),
FetchErrorKind::Status(status) => format!("the server answered HTTP {status}"),
FetchErrorKind::TooLarge => format!(
"the feed is larger than {} MiB",
MAX_FEED_BYTES / (1024 * 1024)
),
FetchErrorKind::Parse => "the document is not a podcast feed".to_string(),
FetchErrorKind::Timeout => "the request timed out".to_string(),
FetchErrorKind::Transport => "could not connect to the server".to_string(),
}
}
}
impl fmt::Display for FetchError {
@@ -110,13 +130,12 @@ impl std::error::Error for FetchError {}
impl From<GuardError> for FetchError {
fn from(error: GuardError) -> Self {
let kind = match &error {
GuardError::UnsupportedScheme(_)
| GuardError::Credentials
| GuardError::MissingHost
| GuardError::Blocked(_) => FetchErrorKind::Refused,
_ if error.is_timeout() => FetchErrorKind::Timeout,
_ => FetchErrorKind::Transport,
let kind = if error.is_refusal() {
FetchErrorKind::Refused
} else if error.is_timeout() {
FetchErrorKind::Timeout
} else {
FetchErrorKind::Transport
};
Self::new(kind, error.to_string())
}
+3 -2
View File
@@ -369,10 +369,11 @@ async fn audio_handler(
let audio = match proxy.fetch(&audio_url, mime.as_deref(), range).await {
Ok(audio) => audio,
Err(error) => {
tracing::info!(user_id = user.id, episode = %episode_id, "podcast proxy: {error}");
// Neither the URL (access tokens) nor the account is logged.
tracing::info!(episode = %episode_id, "podcast proxy: {error}");
let status =
StatusCode::from_u16(error.http_status()).unwrap_or(StatusCode::BAD_GATEWAY);
return Ok(json_error(status, &error.to_string()));
return Ok(json_error(status, &error.public_message()));
}
};
let mut builder = cot::http::Response::builder()
+50 -5
View File
@@ -14,6 +14,9 @@
//! that a second DNS answer could bypass (DNS rebinding);
//! * checks IP-literal hosts the same way (the connector does not resolve
//! them);
//! * connects only to the web ports [`ALLOWED_PORTS`] (80, 443, 8080, 8443):
//! a public address can still be the server's own, and its other services
//! (databases, admin ports) are often reachable from the host itself;
//! * follows redirects itself, at most `max_redirects` hops, checking every
//! hop like the first URL;
//! * never uses a system proxy (it would resolve the target instead of the
@@ -200,26 +203,39 @@ pub(crate) fn classify(ip: IpAddr) -> Option<AddressClass> {
}
}
/// Ports feeds and enclosures may be fetched from.
pub(crate) const ALLOWED_PORTS: [u16; 4] = [80, 443, 8080, 8443];
/// Which addresses [`SafeHttp`] may connect to.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) struct AddressPolicy {
allow_loopback: bool,
any_port: bool,
}
impl AddressPolicy {
/// Public addresses only. The only policy available outside tests.
/// Public addresses on [`ALLOWED_PORTS`] only. The only policy
/// available outside tests.
pub(crate) const PUBLIC_ONLY: Self = Self {
allow_loopback: false,
any_port: false,
};
/// Public addresses plus loopback, for the local test servers.
/// Public addresses plus loopback on any port, for the local test
/// servers.
#[cfg(test)]
pub(crate) const fn test_allow_loopback() -> Self {
Self {
allow_loopback: true,
any_port: true,
}
}
/// Whether a connection to `port` is allowed.
pub(crate) fn allows_port(&self, port: u16) -> bool {
self.any_port || ALLOWED_PORTS.contains(&port)
}
/// Checks one address.
pub(crate) fn check(&self, ip: IpAddr) -> Result<(), AddressClass> {
match classify(ip) {
@@ -350,6 +366,8 @@ pub(crate) enum GuardError {
MissingHost,
/// The host is (or resolves to) a blocked address.
Blocked(BlockedAddress),
/// The port is not one of [`ALLOWED_PORTS`].
Port(u16),
/// More redirects than allowed.
TooManyRedirects(usize),
/// A redirect without a usable `Location`.
@@ -362,8 +380,27 @@ impl GuardError {
pub(crate) fn is_timeout(&self) -> bool {
matches!(self, Self::Http(error) if error.is_timeout())
}
/// Whether the guard refused the URL before or instead of connecting.
pub(crate) fn is_refusal(&self) -> bool {
matches!(
self,
Self::UnsupportedScheme(_)
| Self::Credentials
| Self::MissingHost
| Self::Blocked(_)
| Self::Port(_)
)
}
}
/// What a signed-in user is told about a refused or failed request. The
/// details ([`GuardError`]'s `Display`: resolved addresses, transport error
/// chains) stay in the server logs; showing them would turn feed and audio
/// requests into a scanner of the server's network.
pub(crate) const REFUSED_MESSAGE: &str =
"only public http and https addresses on ports 80, 443, 8080 and 8443 are allowed";
impl fmt::Display for GuardError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
@@ -373,6 +410,7 @@ impl fmt::Display for GuardError {
Self::Credentials => f.write_str("URLs with credentials are not allowed"),
Self::MissingHost => f.write_str("URL has no host"),
Self::Blocked(blocked) => blocked.fmt(f),
Self::Port(port) => write!(f, "port {port} is not allowed"),
Self::TooManyRedirects(limit) => write!(f, "more than {limit} redirects"),
Self::InvalidRedirect(reason) => write!(f, "invalid redirect: {reason}"),
Self::Http(error) => {
@@ -392,7 +430,10 @@ impl std::error::Error for GuardError {}
impl From<reqwest::Error> for GuardError {
/// Surfaces a resolver rejection buried in the transport error chain.
/// The URL is dropped: feed and enclosure URLs often carry access
/// tokens, and these errors end up in logs.
fn from(error: reqwest::Error) -> Self {
let error = error.without_url();
let mut source = std::error::Error::source(&error);
while let Some(inner) = source {
if let Some(blocked) = inner.downcast_ref::<BlockedAddress>() {
@@ -530,9 +571,9 @@ impl SafeHttp {
.ok_or_else(|| {
GuardError::InvalidRedirect(format!("HTTP {status} without Location"))
})?;
let next = url
.join(location)
.map_err(|error| GuardError::InvalidRedirect(format!("{location:?}: {error}")))?;
let next = url.join(location).map_err(|error| {
GuardError::InvalidRedirect(format!("unusable Location: {error}"))
})?;
redirects.push(status);
url = next;
}
@@ -566,5 +607,9 @@ pub(crate) fn check_url(policy: AddressPolicy, url: &Url) -> Result<(), GuardErr
})
})?;
}
let port = url.port_or_known_default().ok_or(GuardError::MissingHost)?;
if !policy.allows_port(port) {
return Err(GuardError::Port(port));
}
Ok(())
}
+87 -7
View File
@@ -182,6 +182,45 @@ fn urls_are_checked_before_connecting() {
assert!(check_url(policy, &url("http://localhost/rss")).is_ok());
}
#[test]
fn only_web_ports_are_allowed() {
// A public address can be the server's own; its databases and admin
// ports must not be reachable through feed or audio requests.
let policy = AddressPolicy::PUBLIC_ONLY;
for text in [
"https://feeds.example.com/rss",
"http://feeds.example.com/rss",
"https://feeds.example.com:443/rss",
"http://feeds.example.com:8080/rss",
"https://feeds.example.com:8443/rss",
"https://8.8.8.8/rss",
] {
assert!(check_url(policy, &url(text)).is_ok(), "{text}");
}
for (text, port) in [
("http://feeds.example.com:5432/", 5432),
("https://feeds.example.com:22/", 22),
("http://8.8.8.8:6379/feed", 6379),
("https://feeds.example.com:3000/", 3000),
] {
match check_url(policy, &url(text)) {
Err(error @ GuardError::Port(refused)) => {
assert_eq!(refused, port, "{text}");
assert!(error.is_refusal());
}
other => panic!("{text}: {other:?}"),
}
}
// The local test servers listen on random ports.
assert!(
check_url(
AddressPolicy::test_allow_loopback(),
&url("http://127.0.0.1:5432/")
)
.is_ok()
);
}
// ---------------------------------------------------------------------------
// Resolver
// ---------------------------------------------------------------------------
@@ -226,14 +265,21 @@ async fn production_policy_never_reaches_a_local_server() {
server.set("/feed", Route::new("ok", "text/plain"));
let lookup = ScriptedLookup::with(&[("loop.test", &[&["127.0.0.1"]])]);
let http = public_http(Arc::clone(&lookup));
for target in [server.url("/feed"), server.url_on("loop.test", "/feed")] {
match http.get(url(&target), &HeaderMap::new()).await {
Err(GuardError::Blocked(blocked)) => {
assert_eq!(blocked.class, AddressClass::Loopback, "{target}")
}
other => panic!("{target}: {other:?}"),
}
match http.get(url(&server.url("/feed")), &HeaderMap::new()).await {
Err(GuardError::Blocked(blocked)) => assert_eq!(blocked.class, AddressClass::Loopback),
other => panic!("{other:?}"),
}
// The test server's random port is refused before any lookup.
let named = url(&server.url_on("loop.test", "/feed"));
match http.get(named.clone(), &HeaderMap::new()).await {
Err(GuardError::Port(port)) => assert_eq!(Some(port), named.port()),
other => panic!("{other:?}"),
}
assert_eq!(lookup.calls.load(Ordering::SeqCst), 0);
// On an allowed port the resolver refuses the loopback answer.
let resolver = GuardedResolver::new(AddressPolicy::PUBLIC_ONLY, lookup);
let error = resolver.resolve_checked("loop.test").await.unwrap_err();
assert_eq!(blocked_class(error.as_ref()), Some(AddressClass::Loopback));
assert_eq!(server.request_count(), 0);
}
@@ -327,6 +373,40 @@ async fn every_redirect_hop_is_checked() {
assert_eq!(followed.final_url.path(), "/hop4");
}
#[tokio::test]
async fn transport_errors_never_carry_the_url() {
// Feed and enclosure URLs often carry access tokens; errors are logged.
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let port = listener.local_addr().unwrap().port();
drop(listener);
let http = test_http(ScriptedLookup::with(&[]));
let error = http
.get(
url(&format!(
"http://127.0.0.1:{port}/private/feed.xml?token=secret"
)),
&HeaderMap::new(),
)
.await
.unwrap_err();
assert!(matches!(error, GuardError::Http(_)), "{error:?}");
let text = error.to_string();
assert!(
!text.contains("secret") && !text.contains("feed.xml"),
"{text}"
);
assert!(!format!("{error:?}").contains("secret"));
// A redirect to an unusable location does not echo it either.
let server = TestServer::start();
server.set("/bad", Route::redirect(302, "http://[::1/?token=secret"));
let error = http
.get(url(&server.url("/bad")), &HeaderMap::new())
.await
.unwrap_err();
assert!(matches!(error, GuardError::InvalidRedirect(_)), "{error:?}");
assert!(!error.to_string().contains("secret"), "{error}");
}
#[tokio::test]
async fn no_cookies_or_credentials_are_sent() {
let server = TestServer::start();
+5
View File
@@ -1903,6 +1903,11 @@ async fn postgres_podcast_api_subscribes_plays_and_keeps_progress() {
.await
.unwrap_err();
assert_eq!(refused.status, 422, "{}", refused.message);
assert_eq!(
refused.message,
"the feed could not be loaded: only public http and https addresses on ports 80, 443, \
8080 and 8443 are allowed"
);
assert!(logged_kinds(&pool, user).await.is_empty());
assert_eq!(server.requests("/show.xml").len(), 0);
+19 -1
View File
@@ -20,7 +20,7 @@ use reqwest::header::{
};
use reqwest::{StatusCode, Url};
use super::net::{GuardError, SafeHttp, SafeHttpConfig};
use super::net::{GuardError, REFUSED_MESSAGE, SafeHttp, SafeHttpConfig};
/// Largest response body passed through (a long episode at a high bitrate
/// stays far below).
@@ -69,6 +69,24 @@ impl ProxyError {
_ => 502,
}
}
/// What the browser is told; the details (`Display`) stay in the
/// server log.
pub(crate) fn public_message(&self) -> String {
match self {
Self::InvalidUrl(_) => "the episode's audio URL is invalid".to_string(),
Self::Upstream(error) if error.is_refusal() => REFUSED_MESSAGE.to_string(),
Self::Upstream(error) if error.is_timeout() => {
"the publisher did not answer in time".to_string()
}
Self::Upstream(_) => "could not connect to the publisher".to_string(),
Self::Status(status) => format!("the publisher answered HTTP {status}"),
Self::ContentType(_) => "the publisher's file is not audio".to_string(),
Self::TooLarge(_) => "the publisher's file is too large".to_string(),
Self::Timeout => "the publisher did not answer in time".to_string(),
Self::Body(_) => "the publisher stream failed".to_string(),
}
}
}
impl fmt::Display for ProxyError {
+29 -10
View File
@@ -68,7 +68,10 @@ pub(crate) enum TargetResult {
NotModified,
/// The fetch or the ingestion failed; the catalog is unchanged.
Failed {
/// What the user sees (stored as the last error).
error: String,
/// Details for the server log (no URLs).
detail: String,
consecutive_failures: u32,
next_check_at_ms: i64,
},
@@ -89,8 +92,9 @@ pub(crate) struct RunSummary {
pub(crate) skipped: usize,
pub(crate) new_episodes: usize,
pub(crate) updated_episodes: usize,
/// `(user_id, podcast_id, error)` of failed targets, for the job log.
pub(crate) failures: Vec<(i64, PodcastId, String)>,
/// `(podcast_id, details)` of failed targets, for the job log (no
/// account ids or URLs: feed URLs often carry access tokens).
pub(crate) failures: Vec<(PodcastId, String)>,
}
impl RunSummary {
@@ -102,10 +106,10 @@ impl RunSummary {
self.updated_episodes += outcome.updated_episodes;
}
TargetResult::NotModified => self.not_modified += 1,
TargetResult::Failed { error, .. } => {
TargetResult::Failed { detail, .. } => {
self.failed += 1;
self.failures
.push((target.user_id, target.podcast_id.clone(), error.clone()));
.push((target.podcast_id.clone(), detail.clone()));
}
TargetResult::Skipped(_) => self.skipped += 1,
}
@@ -232,12 +236,12 @@ async fn record_result(
now_ms: i64,
) -> Result<TargetResult> {
let (user_id, podcast_id) = (target.user_id, &target.podcast_id);
let failure = |error: String, retry_after: Option<Duration>| {
let failure = |error: String, detail: String, retry_after: Option<Duration>| {
let consecutive_failures = previous_failures.saturating_add(1);
let next_check_at_ms = now_ms + failure_backoff_ms(consecutive_failures, retry_after);
(error, consecutive_failures, next_check_at_ms)
(error, detail, consecutive_failures, next_check_at_ms)
};
let (error, consecutive_failures, next_check_at_ms) = match fetched {
let (error, detail, consecutive_failures, next_check_at_ms) = match fetched {
Ok(FeedFetch::NotModified) => {
let next = next_regular_check_ms(now_ms, user_id, podcast_id);
return match store
@@ -254,10 +258,18 @@ async fn record_result(
Err(error) if is_gone(&error) => return skipped_or(error),
// A store failure is recorded like a fetch failure, so the
// feed backs off instead of being retried every run.
Err(error) => failure(format!("storing the feed failed: {error:#}"), None),
Err(error) => failure(
"the feed could not be stored".to_string(),
format!("storing the feed failed: {error:#}"),
None,
),
}
}
Err(error) => failure(error.message.clone(), error.retry_after),
Err(error) => failure(
error.public_message(),
error.message.clone(),
error.retry_after,
),
};
match store
.record_fetch_failure(user_id, podcast_id, now_ms, &error, Some(next_check_at_ms))
@@ -265,6 +277,7 @@ async fn record_result(
{
Ok(_) => Ok(TargetResult::Failed {
error,
detail,
consecutive_failures,
next_check_at_ms,
}),
@@ -408,7 +421,13 @@ pub(crate) enum SubscribeUrlError {
impl fmt::Display for SubscribeUrlError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Fetch(error) => write!(f, "the feed could not be loaded: {}", error.message),
Self::Fetch(error) => {
write!(
f,
"the feed could not be loaded: {}",
error.public_message()
)
}
}
}
}