From 04ae7d5bd1b5609e15f6f9b3f0c8020fd7926318 Mon Sep 17 00:00:00 2001 From: Sanapach Date: Thu, 19 Nov 2026 20:24:03 +0200 Subject: [PATCH] Guard podcast HTTP requests against SSRF Feed and enclosure URLs come from users and third-party feeds. SafeHttp accepts only http(s) URLs without credentials, resolves names with a resolver that refuses the whole answer unless every address is public (loopback, private, CGNAT, link-local, cloud metadata, unique local, multicast, reserved; IPv4-mapped, NAT64 and 6to4 judged by their IPv4 part), and connects to exactly the checked addresses, so DNS rebinding cannot swap them. IP literals and every redirect hop are checked; system proxies, cookies and Referer are off. Loopback is reachable only through a #[cfg(test)] policy used by the local test server. Co-Authored-By: Claude Opus 5.5 --- src/podcasts/mod.rs | 5 + src/podcasts/net.rs | 570 ++++++++++++++++++++++++++++++++++++ src/podcasts/net_tests.rs | 358 ++++++++++++++++++++++ src/podcasts/test_server.rs | 379 ++++++++++++++++++++++++ 4 files changed, 1312 insertions(+) create mode 100644 src/podcasts/net.rs create mode 100644 src/podcasts/net_tests.rs create mode 100644 src/podcasts/test_server.rs diff --git a/src/podcasts/mod.rs b/src/podcasts/mod.rs index df00fa9..cd4b469 100644 --- a/src/podcasts/mod.rs +++ b/src/podcasts/mod.rs @@ -50,13 +50,18 @@ mod actions; mod apply; +mod net; mod queries; mod refresh; pub(crate) mod schema; +#[cfg(test)] +mod net_tests; #[cfg(test)] pub(crate) mod postgres_tests; #[cfg(test)] +pub(crate) mod test_server; +#[cfg(test)] mod tests; use std::fmt; diff --git a/src/podcasts/net.rs b/src/podcasts/net.rs new file mode 100644 index 0000000..576e596 --- /dev/null +++ b/src/podcasts/net.rs @@ -0,0 +1,570 @@ +//! Outbound HTTP for podcast feeds and the audio proxy, guarded against SSRF. +//! +//! Feed URLs and enclosure URLs come from users and from third-party feed +//! documents, so the server must never be tricked into requesting its own +//! network. Every request made through [`SafeHttp`]: +//! +//! * accepts only `http` and `https` URLs with a host and without userinfo; +//! * resolves host names with [`GuardedResolver`], which rejects the whole +//! answer when **any** address is not public (loopback, private, shared +//! CGNAT, link-local, cloud metadata, multicast, unspecified, documentation, +//! reserved; IPv4 and IPv6, including IPv4-mapped, NAT64 and 6to4 forms). +//! The connection is made to exactly the addresses that were checked: the +//! resolver *is* the connector's resolver, there is no separate pre-check +//! that a second DNS answer could bypass (DNS rebinding); +//! * checks IP-literal hosts the same way (the connector does not resolve +//! them); +//! * follows redirects itself, at most `max_redirects` hops, checking every +//! hop like the first URL; +//! * never uses a system proxy (it would resolve the target instead of the +//! guarded resolver), never stores or sends cookies and never forwards +//! credentials: callers pass the few headers they need explicitly. +//! +//! Loopback can be allowed only in tests (`AddressPolicy::test_allow_loopback` +//! is `#[cfg(test)]`), so the local test servers work and nothing else does. + +use std::fmt; +use std::future::Future; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}; +use std::pin::Pin; +use std::sync::Arc; +use std::time::Duration; + +use reqwest::header::{HeaderMap, LOCATION}; +use reqwest::{StatusCode, Url}; + +/// Why an address is not reachable through [`SafeHttp`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum AddressClass { + /// `0.0.0.0/8`, `::`. + Unspecified, + /// `127.0.0.0/8`, `::1`. + Loopback, + /// RFC 1918 and the deprecated IPv6 site-local range. + Private, + /// `100.64.0.0/10` (carrier-grade NAT). + SharedCgnat, + /// `169.254.0.0/16`, `fe80::/10`. + LinkLocal, + /// Cloud instance metadata endpoints. + CloudMetadata, + /// `fc00::/7`. + UniqueLocal, + /// `224.0.0.0/4`, `ff00::/8`. + Multicast, + /// `255.255.255.255`. + Broadcast, + /// Documentation ranges (`192.0.2.0/24`, `2001:db8::/32`, ...). + Documentation, + /// `198.18.0.0/15`, `2001:2::/48`. + Benchmarking, + /// Everything else outside global unicast (IETF protocol assignments, + /// `240.0.0.0/4`, IPv4-compatible IPv6, Teredo, local NAT64, ...). + Reserved, +} + +impl fmt::Display for AddressClass { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Self::Unspecified => "unspecified", + Self::Loopback => "loopback", + Self::Private => "private", + Self::SharedCgnat => "shared (CGNAT)", + Self::LinkLocal => "link-local", + Self::CloudMetadata => "cloud metadata", + Self::UniqueLocal => "unique local", + Self::Multicast => "multicast", + Self::Broadcast => "broadcast", + Self::Documentation => "documentation", + Self::Benchmarking => "benchmarking", + Self::Reserved => "reserved", + }) + } +} + +/// Well-known cloud metadata endpoints. They also fall into blocked ranges +/// (link-local, CGNAT, unique local); naming them makes errors clearer. +const METADATA_V4: [Ipv4Addr; 3] = [ + Ipv4Addr::new(169, 254, 169, 254), + Ipv4Addr::new(169, 254, 170, 2), + Ipv4Addr::new(100, 100, 100, 200), +]; +const METADATA_V6: [Ipv6Addr; 1] = [Ipv6Addr::new(0xfd00, 0x0ec2, 0, 0, 0, 0, 0, 0x0254)]; + +fn in_v4(ip: Ipv4Addr, net: [u8; 4], prefix: u32) -> bool { + let mask = if prefix == 0 { + 0 + } else { + u32::MAX << (32 - prefix) + }; + u32::from(ip) & mask == u32::from(Ipv4Addr::from(net)) & mask +} + +fn in_v6(ip: Ipv6Addr, net: [u16; 8], prefix: u32) -> bool { + let mask = if prefix == 0 { + 0 + } else { + u128::MAX << (128 - prefix) + }; + let [a, b, c, d, e, f, g, h] = net; + u128::from(ip) & mask == u128::from(Ipv6Addr::new(a, b, c, d, e, f, g, h)) & mask +} + +/// Class of a non-public IPv4 address; `None` for a public one. +fn classify_v4(ip: Ipv4Addr) -> Option { + use AddressClass::*; + if METADATA_V4.contains(&ip) { + return Some(CloudMetadata); + } + const RULES: &[([u8; 4], u32, AddressClass)] = &[ + ([0, 0, 0, 0], 8, Unspecified), + ([10, 0, 0, 0], 8, Private), + ([100, 64, 0, 0], 10, SharedCgnat), + ([127, 0, 0, 0], 8, Loopback), + ([169, 254, 0, 0], 16, LinkLocal), + ([172, 16, 0, 0], 12, Private), + ([192, 0, 0, 0], 24, Reserved), + ([192, 0, 2, 0], 24, Documentation), + ([192, 88, 99, 0], 24, Reserved), + ([192, 168, 0, 0], 16, Private), + ([198, 18, 0, 0], 15, Benchmarking), + ([198, 51, 100, 0], 24, Documentation), + ([203, 0, 113, 0], 24, Documentation), + ([224, 0, 0, 0], 4, Multicast), + ([255, 255, 255, 255], 32, Broadcast), + ([240, 0, 0, 0], 4, Reserved), + ]; + RULES + .iter() + .find(|(net, prefix, _)| in_v4(ip, *net, *prefix)) + .map(|(_, _, class)| *class) +} + +/// Class of a non-public IPv6 address; `None` for a public one. Forms that +/// embed an IPv4 address (mapped, NAT64, 6to4) are judged by that address. +fn classify_v6(ip: Ipv6Addr) -> Option { + use AddressClass::*; + if METADATA_V6.contains(&ip) { + return Some(CloudMetadata); + } + if ip.is_unspecified() { + return Some(Unspecified); + } + if ip.is_loopback() { + return Some(Loopback); + } + let bits = u128::from(ip); + let embedded_v4 = |shift: u32| Ipv4Addr::from((bits >> shift) as u32); + // ::ffff:a.b.c.d + if let Some(v4) = ip.to_ipv4_mapped() { + return classify_v4(v4); + } + // 64:ff9b::/96, the well-known NAT64 prefix. + if in_v6(ip, [0x64, 0xff9b, 0, 0, 0, 0, 0, 0], 96) { + return classify_v4(embedded_v4(0)); + } + // 2002::/16, 6to4: the IPv4 address follows the prefix. + if in_v6(ip, [0x2002, 0, 0, 0, 0, 0, 0, 0], 16) { + return classify_v4(embedded_v4(80)); + } + const RULES: &[([u16; 8], u32, AddressClass)] = &[ + // IPv4-compatible (deprecated) and the rest of ::/96. + ([0, 0, 0, 0, 0, 0, 0, 0], 96, Reserved), + ([0x64, 0xff9b, 1, 0, 0, 0, 0, 0], 48, Reserved), + ([0x100, 0, 0, 0, 0, 0, 0, 0], 64, Reserved), + ([0x2001, 0x2, 0, 0, 0, 0, 0, 0], 48, Benchmarking), + ([0x2001, 0xdb8, 0, 0, 0, 0, 0, 0], 32, Documentation), + // IETF protocol assignments, including Teredo (2001::/32). + ([0x2001, 0, 0, 0, 0, 0, 0, 0], 23, Reserved), + ([0x3fff, 0, 0, 0, 0, 0, 0, 0], 20, Documentation), + ([0xfc00, 0, 0, 0, 0, 0, 0, 0], 7, UniqueLocal), + ([0xfe80, 0, 0, 0, 0, 0, 0, 0], 10, LinkLocal), + ([0xfec0, 0, 0, 0, 0, 0, 0, 0], 10, Private), + ([0xff00, 0, 0, 0, 0, 0, 0, 0], 8, Multicast), + ]; + if let Some((_, _, class)) = RULES + .iter() + .find(|(net, prefix, _)| in_v6(ip, *net, *prefix)) + { + return Some(*class); + } + // Only global unicast (2000::/3) is public. + (!in_v6(ip, [0x2000, 0, 0, 0, 0, 0, 0, 0], 3)).then_some(Reserved) +} + +/// Class of a non-public address; `None` when it is public. +pub(crate) fn classify(ip: IpAddr) -> Option { + match ip { + IpAddr::V4(ip) => classify_v4(ip), + IpAddr::V6(ip) => classify_v6(ip), + } +} + +/// Which addresses [`SafeHttp`] may connect to. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct AddressPolicy { + allow_loopback: bool, +} + +impl AddressPolicy { + /// Public addresses only. The only policy available outside tests. + pub(crate) const PUBLIC_ONLY: Self = Self { + allow_loopback: false, + }; + + /// Public addresses plus loopback, for the local test servers. + #[cfg(test)] + pub(crate) const fn test_allow_loopback() -> Self { + Self { + allow_loopback: true, + } + } + + /// Checks one address. + pub(crate) fn check(&self, ip: IpAddr) -> Result<(), AddressClass> { + match classify(ip) { + None => Ok(()), + Some(AddressClass::Loopback) if self.allow_loopback => Ok(()), + Some(class) => Err(class), + } + } +} + +/// A host name or literal resolved to an address that is not allowed. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct BlockedAddress { + /// Host as written in the URL. + pub(crate) host: String, + /// The offending address. + pub(crate) address: IpAddr, + /// Why it is blocked. + pub(crate) class: AddressClass, +} + +impl fmt::Display for BlockedAddress { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let literal = self.host.trim_start_matches('[').trim_end_matches(']'); + if literal.parse::().is_ok() { + write!(f, "{} is a {} address", self.host, self.class)?; + } else { + write!( + f, + "{} resolves to a {} address ({})", + self.host, self.class, self.address + )?; + } + f.write_str("; only public addresses are allowed") + } +} + +impl std::error::Error for BlockedAddress {} + +/// Name resolution used by [`GuardedResolver`]. +pub(crate) trait HostLookup: Send + Sync + 'static { + /// Every address of `host` (a DNS name, never an IP literal). + fn lookup(&self, host: &str) -> LookupFuture; +} + +pub(crate) type LookupFuture = + Pin>> + Send + 'static>>; + +/// The operating system resolver (`getaddrinfo` via tokio). +#[derive(Debug, Clone, Copy, Default)] +pub(crate) struct SystemLookup; + +impl HostLookup for SystemLookup { + fn lookup(&self, host: &str) -> LookupFuture { + let host = host.to_string(); + Box::pin(async move { + Ok(tokio::net::lookup_host((host.as_str(), 0)) + .await? + .map(|addr| addr.ip()) + .collect()) + }) + } +} + +const LOOKUP_TIMEOUT: Duration = Duration::from_secs(10); + +/// DNS resolver of the guarded client: resolves, then rejects the answer +/// unless every address is allowed. The connector connects to exactly the +/// returned addresses, so a checked answer cannot be swapped afterwards. +pub(crate) struct GuardedResolver { + policy: AddressPolicy, + lookup: Arc, +} + +impl GuardedResolver { + pub(crate) fn new(policy: AddressPolicy, lookup: Arc) -> Self { + Self { policy, lookup } + } + + /// Resolves and checks `host`; the addresses a connection may use. + pub(crate) async fn resolve_checked( + &self, + host: &str, + ) -> Result, Box> { + let ips = tokio::time::timeout(LOOKUP_TIMEOUT, self.lookup.lookup(host)) + .await + .map_err(|_| format!("resolving {host} timed out"))??; + if ips.is_empty() { + return Err(format!("{host} has no addresses").into()); + } + for ip in &ips { + if let Err(class) = self.policy.check(*ip) { + return Err(Box::new(BlockedAddress { + host: host.to_string(), + address: *ip, + class, + })); + } + } + Ok(ips) + } +} + +impl reqwest::dns::Resolve for GuardedResolver { + fn resolve(&self, name: reqwest::dns::Name) -> reqwest::dns::Resolving { + let resolver = Self { + policy: self.policy, + lookup: Arc::clone(&self.lookup), + }; + let host = name.as_str().to_string(); + Box::pin(async move { + let ips = resolver.resolve_checked(&host).await?; + // Port 0: the connector uses the URL's port. + let addrs: Vec = ips.into_iter().map(|ip| SocketAddr::new(ip, 0)).collect(); + Ok(Box::new(addrs.into_iter()) as reqwest::dns::Addrs) + }) + } +} + +/// Errors of [`SafeHttp`]. +#[derive(Debug)] +pub(crate) enum GuardError { + /// Not `http`/`https`. + UnsupportedScheme(String), + /// The URL carries `user:password@`. + Credentials, + /// The URL has no host. + MissingHost, + /// The host is (or resolves to) a blocked address. + Blocked(BlockedAddress), + /// More redirects than allowed. + TooManyRedirects(usize), + /// A redirect without a usable `Location`. + InvalidRedirect(String), + /// Any other transport failure (DNS, connect, TLS, timeout, ...). + Http(reqwest::Error), +} + +impl GuardError { + pub(crate) fn is_timeout(&self) -> bool { + matches!(self, Self::Http(error) if error.is_timeout()) + } +} + +impl fmt::Display for GuardError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::UnsupportedScheme(scheme) => { + write!(f, "unsupported URL scheme {scheme:?} (only http and https)") + } + Self::Credentials => f.write_str("URLs with credentials are not allowed"), + Self::MissingHost => f.write_str("URL has no host"), + Self::Blocked(blocked) => blocked.fmt(f), + Self::TooManyRedirects(limit) => write!(f, "more than {limit} redirects"), + Self::InvalidRedirect(reason) => write!(f, "invalid redirect: {reason}"), + Self::Http(error) => { + write!(f, "{error}")?; + let mut source = std::error::Error::source(error); + while let Some(inner) = source { + write!(f, ": {inner}")?; + source = inner.source(); + } + Ok(()) + } + } + } +} + +impl std::error::Error for GuardError {} + +impl From for GuardError { + /// Surfaces a resolver rejection buried in the transport error chain. + fn from(error: reqwest::Error) -> Self { + let mut source = std::error::Error::source(&error); + while let Some(inner) = source { + if let Some(blocked) = inner.downcast_ref::() { + return Self::Blocked(blocked.clone()); + } + source = inner.source(); + } + Self::Http(error) + } +} + +/// Timeouts and limits of a [`SafeHttp`] client. +#[derive(Debug, Clone, Copy)] +pub(crate) struct SafeHttpConfig { + /// TCP + TLS connect timeout (name resolution has its own 10 s bound). + pub(crate) connect_timeout: Duration, + /// Maximum time between two reads of a response. + pub(crate) read_timeout: Duration, + /// Whole-request timeout including the body; `None` for streams. + pub(crate) total_timeout: Option, + /// Redirect hops followed before giving up. + pub(crate) max_redirects: usize, +} + +/// A response after following redirects. +#[derive(Debug)] +pub(crate) struct Followed { + /// The final response (never a followed redirect). + pub(crate) response: reqwest::Response, + /// URL of the final response. + pub(crate) final_url: Url, + /// Status of every redirect followed, in order. + pub(crate) redirects: Vec, +} + +impl Followed { + /// Whether at least one redirect was followed and all of them were + /// permanent (301/308): only then has the resource moved. + pub(crate) fn moved_permanently(&self) -> bool { + !self.redirects.is_empty() + && self + .redirects + .iter() + .all(|status| matches!(status.as_u16(), 301 | 308)) + } +} + +/// SSRF-guarded HTTP client (see the module documentation). +#[derive(Clone)] +pub(crate) struct SafeHttp { + client: reqwest::Client, + policy: AddressPolicy, + max_redirects: usize, +} + +impl fmt::Debug for SafeHttp { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SafeHttp") + .field("policy", &self.policy) + .field("max_redirects", &self.max_redirects) + .finish_non_exhaustive() + } +} + +impl SafeHttp { + /// A production client: public addresses only, system DNS. + pub(crate) fn public(config: SafeHttpConfig) -> anyhow::Result { + Self::with_lookup(config, AddressPolicy::PUBLIC_ONLY, Arc::new(SystemLookup)) + } + + /// A client with an explicit policy and resolver (tests inject both). + pub(crate) fn with_lookup( + config: SafeHttpConfig, + policy: AddressPolicy, + lookup: Arc, + ) -> anyhow::Result { + let mut builder = reqwest::Client::builder() + .dns_resolver(Arc::new(GuardedResolver::new(policy, lookup))) + // A proxy would resolve the target itself, bypassing the guard. + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .referer(false) + .connect_timeout(config.connect_timeout) + .read_timeout(config.read_timeout) + .user_agent(concat!( + "furumusic/", + env!("CARGO_PKG_VERSION"), + " (podcasts)" + )); + if let Some(timeout) = config.total_timeout { + builder = builder.timeout(timeout); + } + Ok(Self { + client: builder.build()?, + policy, + max_redirects: config.max_redirects, + }) + } + + /// Checks a URL before any connection: scheme, credentials, host, and + /// IP-literal hosts against the policy (names are checked by the + /// resolver at connect time). + pub(crate) fn check_url(&self, url: &Url) -> Result<(), GuardError> { + check_url(self.policy, url) + } + + /// `GET url` with `headers`, following redirects (each hop checked, the + /// same headers re-sent; nothing else is added). + pub(crate) async fn get(&self, url: Url, headers: &HeaderMap) -> Result { + let mut url = url; + let mut redirects = Vec::new(); + loop { + self.check_url(&url)?; + let response = self + .client + .get(url.clone()) + .headers(headers.clone()) + .send() + .await?; + let status = response.status(); + if !matches!(status.as_u16(), 301 | 302 | 303 | 307 | 308) { + return Ok(Followed { + response, + final_url: url, + redirects, + }); + } + if redirects.len() >= self.max_redirects { + return Err(GuardError::TooManyRedirects(self.max_redirects)); + } + let location = response + .headers() + .get(LOCATION) + .and_then(|value| value.to_str().ok()) + .ok_or_else(|| { + GuardError::InvalidRedirect(format!("HTTP {status} without Location")) + })?; + let next = url + .join(location) + .map_err(|error| GuardError::InvalidRedirect(format!("{location:?}: {error}")))?; + redirects.push(status); + url = next; + } + } +} + +/// See [`SafeHttp::check_url`]. +pub(crate) fn check_url(policy: AddressPolicy, url: &Url) -> Result<(), GuardError> { + if !matches!(url.scheme(), "http" | "https") { + return Err(GuardError::UnsupportedScheme(url.scheme().to_string())); + } + if !url.username().is_empty() || url.password().is_some() { + return Err(GuardError::Credentials); + } + let host = url + .host_str() + .filter(|host| !host.is_empty()) + .ok_or(GuardError::MissingHost)?; + // The URL parser normalizes every IPv4 spelling (decimal, octal, hex, + // trailing dot) to dotted quads and keeps IPv6 literals in brackets. + let bare = host + .strip_prefix('[') + .and_then(|host| host.strip_suffix(']')) + .unwrap_or(host); + if let Ok(ip) = bare.parse::() { + policy.check(ip).map_err(|class| { + GuardError::Blocked(BlockedAddress { + host: host.to_string(), + address: ip, + class, + }) + })?; + } + Ok(()) +} diff --git a/src/podcasts/net_tests.rs b/src/podcasts/net_tests.rs new file mode 100644 index 0000000..fe9a8ab --- /dev/null +++ b/src/podcasts/net_tests.rs @@ -0,0 +1,358 @@ +//! SSRF guard: address classes, URL checks, the resolver, DNS rebinding, +//! redirects and headers. Network tests use the local test server; loopback +//! is allowed only through the test-only `AddressPolicy::test_allow_loopback`. + +use std::net::IpAddr; +use std::sync::Arc; +use std::sync::atomic::Ordering; + +use reqwest::Url; +use reqwest::header::HeaderMap; + +use super::net::{ + AddressClass, AddressPolicy, BlockedAddress, GuardError, GuardedResolver, SafeHttp, check_url, + classify, +}; +use super::test_server::{Route, ScriptedLookup, TEST_HTTP, TestServer, test_http}; + +// --------------------------------------------------------------------------- +// Address classes +// --------------------------------------------------------------------------- + +fn ip(text: &str) -> IpAddr { + text.parse().unwrap() +} + +#[test] +fn public_addresses_are_allowed() { + for address in [ + "8.8.8.8", + "1.1.1.1", + "93.184.215.14", + "172.32.0.1", + "100.128.0.1", + "192.0.3.1", + "2606:4700:4700::1111", + "2a00:1450:4001:80b::200e", + "::ffff:8.8.8.8", + "64:ff9b::808:808", + "2002:808:808::1", + ] { + assert_eq!(classify(ip(address)), None, "{address}"); + assert_eq!(AddressPolicy::PUBLIC_ONLY.check(ip(address)), Ok(())); + } +} + +#[test] +fn non_public_addresses_are_classified() { + use AddressClass::*; + let cases = [ + // IPv4 + ("0.0.0.0", Unspecified), + ("0.1.2.3", Unspecified), + ("10.1.2.3", Private), + ("100.64.0.1", SharedCgnat), + ("100.127.255.254", SharedCgnat), + ("127.0.0.1", Loopback), + ("127.255.0.9", Loopback), + ("169.254.1.1", LinkLocal), + ("169.254.169.254", CloudMetadata), + ("169.254.170.2", CloudMetadata), + ("100.100.100.200", CloudMetadata), + ("172.16.0.1", Private), + ("172.31.255.255", Private), + ("192.0.0.8", Reserved), + ("192.0.2.5", Documentation), + ("192.88.99.1", Reserved), + ("192.168.1.1", Private), + ("198.18.0.1", Benchmarking), + ("198.19.255.255", Benchmarking), + ("198.51.100.1", Documentation), + ("203.0.113.1", Documentation), + ("224.0.0.1", Multicast), + ("239.255.255.250", Multicast), + ("240.0.0.1", Reserved), + ("255.255.255.255", Broadcast), + // IPv6 + ("::", Unspecified), + ("::1", Loopback), + ("::ffff:127.0.0.1", Loopback), + ("::ffff:10.0.0.1", Private), + ("::ffff:169.254.169.254", CloudMetadata), + ("::ffff:0.0.0.0", Unspecified), + ("::127.0.0.1", Reserved), + ("::8.8.8.8", Reserved), + ("64:ff9b::a00:1", Private), + ("64:ff9b::7f00:1", Loopback), + ("64:ff9b:1::1", Reserved), + ("2002:7f00:1::1", Loopback), + ("2002:a9fe:a9fe::1", CloudMetadata), + ("2002:c0a8:101::1", Private), + ("100::1", Reserved), + ("2001::1", Reserved), + ("2001:2::1", Benchmarking), + ("2001:db8::1", Documentation), + ("3fff::1", Documentation), + ("fc00::1", UniqueLocal), + ("fd12:3456::1", UniqueLocal), + ("fd00:ec2::254", CloudMetadata), + ("fe80::1", LinkLocal), + ("febf::1", LinkLocal), + ("fec0::1", Private), + ("ff02::1", Multicast), + ("4000::1", Reserved), + ("::2", Reserved), + ]; + for (address, class) in cases { + assert_eq!(classify(ip(address)), Some(class), "{address}"); + assert_eq!( + AddressPolicy::PUBLIC_ONLY.check(ip(address)), + Err(class), + "{address}" + ); + } +} + +#[test] +fn only_the_test_policy_allows_loopback_and_nothing_else() { + let test = AddressPolicy::test_allow_loopback(); + assert_eq!(test.check(ip("127.0.0.1")), Ok(())); + assert_eq!(test.check(ip("::1")), Ok(())); + assert_eq!(test.check(ip("::ffff:127.0.0.1")), Ok(())); + assert_eq!(test.check(ip("10.0.0.1")), Err(AddressClass::Private)); + assert_eq!( + test.check(ip("169.254.169.254")), + Err(AddressClass::CloudMetadata) + ); + assert_eq!( + AddressPolicy::PUBLIC_ONLY.check(ip("127.0.0.1")), + Err(AddressClass::Loopback) + ); +} + +fn url(text: &str) -> Url { + Url::parse(text).unwrap() +} + +#[test] +fn urls_are_checked_before_connecting() { + let policy = AddressPolicy::PUBLIC_ONLY; + for (text, class) in [ + ("http://127.0.0.1/feed", AddressClass::Loopback), + // Every IPv4 spelling is normalized by the URL parser. + ("http://2130706433/feed", AddressClass::Loopback), + ("http://0x7f.1/feed", AddressClass::Loopback), + ("http://127.1/feed", AddressClass::Loopback), + ("http://0177.0.0.1/feed", AddressClass::Loopback), + ("http://127.0.0.1./feed", AddressClass::Loopback), + ("http://0/feed", AddressClass::Unspecified), + ("http://[::1]:8080/feed", AddressClass::Loopback), + ("http://[::ffff:7f00:1]/feed", AddressClass::Loopback), + ("http://[::ffff:a9fe:a9fe]/", AddressClass::CloudMetadata), + ( + "http://169.254.169.254/latest/meta-data/", + AddressClass::CloudMetadata, + ), + ("https://10.0.0.5/audio.mp3", AddressClass::Private), + ("https://[fd00::1]/audio.mp3", AddressClass::UniqueLocal), + ] { + match check_url(policy, &url(text)) { + Err(GuardError::Blocked(blocked)) => assert_eq!(blocked.class, class, "{text}"), + other => panic!("{text}: {other:?}"), + } + } + assert!(matches!( + check_url(policy, &url("ftp://example.com/feed")), + Err(GuardError::UnsupportedScheme(_)) + )); + assert!(matches!( + check_url(policy, &url("file:///etc/passwd")), + Err(GuardError::UnsupportedScheme(_)) + )); + assert!(matches!( + check_url(policy, &url("https://user:secret@example.com/feed")), + Err(GuardError::Credentials) + )); + assert!(matches!( + check_url(policy, &url("https://user@example.com/feed")), + Err(GuardError::Credentials) + )); + // Names are checked by the resolver at connect time. + assert!(check_url(policy, &url("https://feeds.example.com/rss")).is_ok()); + assert!(check_url(policy, &url("http://localhost/rss")).is_ok()); +} + +// --------------------------------------------------------------------------- +// Resolver +// --------------------------------------------------------------------------- + +fn blocked_class(error: &(dyn std::error::Error + Send + Sync + 'static)) -> Option { + error + .downcast_ref::() + .map(|blocked| blocked.class) +} + +#[tokio::test] +async fn resolver_rejects_the_whole_answer_when_any_address_is_not_public() { + let lookup = ScriptedLookup::with(&[ + ("public.test", &[&["8.8.8.8", "2606:4700:4700::1111"]]), + ("mixed.test", &[&["8.8.8.8", "10.0.0.1"]]), + ("mapped.test", &[&["::ffff:192.168.0.10"]]), + ("localhost", &[&["127.0.0.1", "::1"]]), + ("empty.test", &[&[]]), + ]); + let resolver = GuardedResolver::new(AddressPolicy::PUBLIC_ONLY, lookup); + assert_eq!( + resolver.resolve_checked("public.test").await.unwrap(), + vec![ip("8.8.8.8"), ip("2606:4700:4700::1111")] + ); + let error = resolver.resolve_checked("mixed.test").await.unwrap_err(); + assert_eq!(blocked_class(error.as_ref()), Some(AddressClass::Private)); + let error = resolver.resolve_checked("mapped.test").await.unwrap_err(); + assert_eq!(blocked_class(error.as_ref()), Some(AddressClass::Private)); + let error = resolver.resolve_checked("localhost").await.unwrap_err(); + assert_eq!(blocked_class(error.as_ref()), Some(AddressClass::Loopback)); + assert!(resolver.resolve_checked("empty.test").await.is_err()); + assert!(resolver.resolve_checked("nowhere.test").await.is_err()); +} + +fn public_http(lookup: Arc) -> SafeHttp { + SafeHttp::with_lookup(TEST_HTTP, AddressPolicy::PUBLIC_ONLY, lookup).unwrap() +} + +#[tokio::test] +async fn production_policy_never_reaches_a_local_server() { + let server = TestServer::start(); + server.set("/feed", Route::new("ok", "text/plain")); + let lookup = ScriptedLookup::with(&[("loop.test", &[&["127.0.0.1"]])]); + let http = public_http(Arc::clone(&lookup)); + for target in [server.url("/feed"), server.url_on("loop.test", "/feed")] { + match http.get(url(&target), &HeaderMap::new()).await { + Err(GuardError::Blocked(blocked)) => { + assert_eq!(blocked.class, AddressClass::Loopback, "{target}") + } + other => panic!("{target}: {other:?}"), + } + } + assert_eq!(server.request_count(), 0); +} + +#[tokio::test] +async fn every_connection_uses_the_addresses_it_checked() { + // DNS rebinding: the first answer is allowed, the next one points to a + // private address. Each connection resolves and checks once and + // connects to exactly those addresses, so the rebound answer is refused + // instead of being used after an earlier successful check. + let server = TestServer::start(); + server.set("/feed", Route::new("first", "text/plain")); + let lookup = ScriptedLookup::with(&[("rebind.test", &[&["127.0.0.1"], &["10.0.0.7"]])]); + let http = test_http(Arc::clone(&lookup)); + let target = url(&server.url_on("rebind.test", "/feed")); + let first = http.get(target.clone(), &HeaderMap::new()).await.unwrap(); + assert_eq!(first.response.text().await.unwrap(), "first"); + match http.get(target, &HeaderMap::new()).await { + Err(GuardError::Blocked(blocked)) => { + assert_eq!(blocked.class, AddressClass::Private); + assert_eq!(blocked.address, ip("10.0.0.7")); + } + other => panic!("{other:?}"), + } + assert_eq!(lookup.calls.load(Ordering::SeqCst), 2); + assert_eq!(server.requests("/feed").len(), 1); +} + +#[tokio::test] +async fn every_redirect_hop_is_checked() { + let server = TestServer::start(); + let lookup = ScriptedLookup::with(&[("intranet.test", &[&["192.168.1.20"]])]); + let http = test_http(lookup); + let cases = [ + ( + "/to-private", + "http://10.0.0.1/feed".to_string(), + AddressClass::Private, + ), + ( + "/to-metadata", + "http://169.254.169.254/latest/meta-data/".to_string(), + AddressClass::CloudMetadata, + ), + ( + "/to-mapped", + "http://[::ffff:10.1.1.1]/".to_string(), + AddressClass::Private, + ), + ( + "/to-name", + server.url_on("intranet.test", "/feed"), + AddressClass::Private, + ), + ]; + for (path, location, class) in &cases { + server.set(path, Route::redirect(302, location)); + match http.get(url(&server.url(path)), &HeaderMap::new()).await { + Err(GuardError::Blocked(blocked)) => assert_eq!(blocked.class, *class, "{path}"), + other => panic!("{path}: {other:?}"), + } + } + server.set("/to-file", Route::redirect(301, "file:///etc/passwd")); + assert!(matches!( + http.get(url(&server.url("/to-file")), &HeaderMap::new()) + .await, + Err(GuardError::UnsupportedScheme(_)) + )); + server.set("/no-location", Route::new("", "text/plain").status(302)); + assert!(matches!( + http.get(url(&server.url("/no-location")), &HeaderMap::new()) + .await, + Err(GuardError::InvalidRedirect(_)) + )); + // Hop limit (3 in the test client): /hop0 -> /hop1 -> ... -> /hop4. + for hop in 0..4 { + server.set( + &format!("/hop{hop}"), + Route::redirect(307, &format!("/hop{}", hop + 1)), + ); + } + server.set("/hop4", Route::new("end", "text/plain")); + assert!(matches!( + http.get(url(&server.url("/hop0")), &HeaderMap::new()).await, + Err(GuardError::TooManyRedirects(3)) + )); + let followed = http + .get(url(&server.url("/hop1")), &HeaderMap::new()) + .await + .unwrap(); + assert_eq!(followed.redirects.len(), 3); + assert_eq!(followed.final_url.path(), "/hop4"); +} + +#[tokio::test] +async fn no_cookies_or_credentials_are_sent() { + let server = TestServer::start(); + server.set( + "/start", + Route::redirect(302, "/next").header("Set-Cookie", "session=leak; Path=/"), + ); + server.set( + "/next", + Route::new("ok", "text/plain").header("Set-Cookie", "other=leak; Path=/"), + ); + let http = test_http(ScriptedLookup::with(&[])); + for _ in 0..2 { + let followed = http + .get(url(&server.url("/start")), &HeaderMap::new()) + .await + .unwrap(); + assert_eq!(followed.response.text().await.unwrap(), "ok"); + } + for request in server + .requests("/start") + .iter() + .chain(&server.requests("/next")) + { + for header in ["cookie", "authorization", "referer", "proxy-authorization"] { + assert!(!request.contains_key(header), "{header} sent: {request:?}"); + } + } +} diff --git a/src/podcasts/test_server.rs b/src/podcasts/test_server.rs new file mode 100644 index 0000000..4f9c199 --- /dev/null +++ b/src/podcasts/test_server.rs @@ -0,0 +1,379 @@ +//! Test support of the podcast runtime: a scripted resolver, a client that +//! may reach loopback, and a minimal HTTP/1.1 server on 127.0.0.1 (no +//! internet). One request per connection (`Connection: close`), so every +//! request opens a new connection and resolves its host again. +//! +//! Adapted from the furumi-tui prototype's test server: routes can redirect, +//! send extra headers and answer conditional requests, and every request's +//! headers are recorded. + +use std::collections::{HashMap, VecDeque}; +use std::io::{BufRead, BufReader, Write}; +use std::net::{IpAddr, TcpListener, TcpStream}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex, PoisonError}; +use std::time::Duration; + +use super::net::{AddressPolicy, HostLookup, LookupFuture, SafeHttp, SafeHttpConfig}; + +#[derive(Clone)] +pub(crate) struct Route { + pub(crate) status: u16, + pub(crate) body: Arc>, + pub(crate) content_type: Option, + pub(crate) etag: Option, + pub(crate) last_modified: Option, + /// Honour `Range` requests with 206 responses. + pub(crate) ranges: bool, + /// Send `Content-Length` (otherwise the body is close-delimited). + pub(crate) send_length: bool, + /// Close the connection once the body reaches this absolute offset. + pub(crate) cut_after: Option, + /// Extra response headers. + pub(crate) headers: Vec<(String, String)>, + /// Pause between the headers and the body. + pub(crate) body_delay: Option, +} + +impl Route { + pub(crate) fn new(body: impl Into>, content_type: &str) -> Route { + Route { + status: 200, + body: Arc::new(body.into()), + content_type: Some(content_type.to_string()), + etag: None, + last_modified: None, + ranges: true, + send_length: true, + cut_after: None, + headers: Vec::new(), + body_delay: None, + } + } + + /// A redirect to `location` with `status`. + pub(crate) fn redirect(status: u16, location: &str) -> Route { + Route { + status, + body: Arc::new(Vec::new()), + content_type: None, + ranges: false, + ..Route::new(Vec::new(), "text/plain") + } + .header("Location", location) + } + + pub(crate) fn status(mut self, status: u16) -> Route { + self.status = status; + self + } + + pub(crate) fn etag(mut self, etag: &str) -> Route { + self.etag = Some(etag.to_string()); + self + } + + pub(crate) fn last_modified(mut self, date: &str) -> Route { + self.last_modified = Some(date.to_string()); + self + } + + pub(crate) fn no_ranges(mut self) -> Route { + self.ranges = false; + self + } + + pub(crate) fn no_length(mut self) -> Route { + self.send_length = false; + self + } + + pub(crate) fn no_content_type(mut self) -> Route { + self.content_type = None; + self + } + + pub(crate) fn cut(mut self, after: usize) -> Route { + self.cut_after = Some(after); + self + } + + pub(crate) fn delay_body(mut self, delay: Duration) -> Route { + self.body_delay = Some(delay); + self + } + + pub(crate) fn header(mut self, name: &str, value: &str) -> Route { + self.headers.push((name.to_string(), value.to_string())); + self + } +} + +/// Lowercased request headers of one request. +pub(crate) type RequestHeaders = HashMap; + +#[derive(Default)] +struct ServerState { + routes: HashMap, + requests: HashMap>, +} + +pub(crate) struct TestServer { + port: u16, + state: Arc>, +} + +impl TestServer { + pub(crate) fn start() -> TestServer { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind test server"); + let port = listener.local_addr().expect("local addr").port(); + let state = Arc::new(Mutex::new(ServerState::default())); + let accept_state = Arc::clone(&state); + std::thread::spawn(move || { + for stream in listener.incoming() { + let Ok(stream) = stream else { continue }; + let state = Arc::clone(&accept_state); + std::thread::spawn(move || handle(stream, &state)); + } + }); + TestServer { port, state } + } + + pub(crate) fn port(&self) -> u16 { + self.port + } + + /// `http://127.0.0.1:`. + pub(crate) fn url(&self, path: &str) -> String { + format!("http://127.0.0.1:{}{path}", self.port) + } + + /// `http://:` (for hosts mapped by a test resolver). + pub(crate) fn url_on(&self, host: &str, path: &str) -> String { + format!("http://{host}:{}{path}", self.port) + } + + pub(crate) fn set(&self, path: &str, route: Route) { + self.lock().routes.insert(path.to_string(), route); + } + + /// Headers of every request to `path`, in order. + pub(crate) fn requests(&self, path: &str) -> Vec { + self.lock().requests.get(path).cloned().unwrap_or_default() + } + + /// Requests to any path. + pub(crate) fn request_count(&self) -> usize { + self.lock().requests.values().map(Vec::len).sum() + } + + fn lock(&self) -> std::sync::MutexGuard<'_, ServerState> { + self.state.lock().unwrap_or_else(PoisonError::into_inner) + } +} + +fn handle(stream: TcpStream, state: &Mutex) { + let _ = stream.set_read_timeout(Some(Duration::from_secs(10))); + let mut reader = BufReader::new(match stream.try_clone() { + Ok(stream) => stream, + Err(_) => return, + }); + let mut request_line = String::new(); + if reader.read_line(&mut request_line).is_err() { + return; + } + let path = request_line + .split_whitespace() + .nth(1) + .unwrap_or("/") + .to_string(); + let mut headers = HashMap::new(); + loop { + let mut line = String::new(); + match reader.read_line(&mut line) { + Ok(0) | Err(_) => return, + Ok(_) => {} + } + let line = line.trim_end(); + if line.is_empty() { + break; + } + if let Some((name, value)) = line.split_once(':') { + headers.insert(name.trim().to_ascii_lowercase(), value.trim().to_string()); + } + } + let route = { + let mut state = state.lock().unwrap_or_else(PoisonError::into_inner); + state + .requests + .entry(path.clone()) + .or_default() + .push(headers.clone()); + state.routes.get(&path).cloned() + }; + let mut out = stream; + let Some(route) = route else { + let _ = out + .write_all(b"HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"); + return; + }; + let not_modified = match (&route.etag, headers.get("if-none-match")) { + (Some(etag), Some(sent)) => etag == sent, + _ => matches!( + (&route.last_modified, headers.get("if-modified-since")), + (Some(date), Some(sent)) if date == sent + ), + }; + if route.status == 200 && not_modified { + let _ = write!( + out, + "HTTP/1.1 304 Not Modified\r\nConnection: close\r\n\r\n" + ); + return; + } + let total = route.body.len(); + let mut status = route.status; + let mut start = 0usize; + let mut end = total; + let mut extra = String::new(); + if route.status == 200 + && route.ranges + && let Some(range) = headers.get("range").and_then(|r| r.strip_prefix("bytes=")) + && let Some((from, to)) = range.split_once('-') + { + let (from, to) = match (from.trim().parse::(), to.trim().parse::()) { + (Ok(from), Ok(to)) => (from, (to + 1).min(total)), + (Ok(from), Err(_)) => (from, total), + (Err(_), Ok(suffix)) => (total.saturating_sub(suffix), total), + _ => (0, total), + }; + if from >= total { + let _ = write!( + out, + "HTTP/1.1 416 Range Not Satisfiable\r\nContent-Range: bytes */{total}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" + ); + return; + } + status = 206; + start = from; + end = to; + extra.push_str(&format!( + "Content-Range: bytes {start}-{}/{total}\r\n", + end - 1 + )); + } + if route.ranges { + extra.push_str("Accept-Ranges: bytes\r\n"); + } + if let Some(content_type) = &route.content_type { + extra.push_str(&format!("Content-Type: {content_type}\r\n")); + } + if let Some(etag) = &route.etag { + extra.push_str(&format!("ETag: {etag}\r\n")); + } + if let Some(date) = &route.last_modified { + extra.push_str(&format!("Last-Modified: {date}\r\n")); + } + for (name, value) in &route.headers { + extra.push_str(&format!("{name}: {value}\r\n")); + } + if route.send_length { + extra.push_str(&format!("Content-Length: {}\r\n", end - start)); + } + if write!( + out, + "HTTP/1.1 {status} Status\r\n{extra}Connection: close\r\n\r\n" + ) + .is_err() + { + return; + } + let _ = out.flush(); + if let Some(delay) = route.body_delay { + std::thread::sleep(delay); + } + let limit = route.cut_after.map_or(end, |cut| cut.clamp(start, end)); + for chunk in route.body[start..limit].chunks(64 * 1024) { + if out.write_all(chunk).is_err() { + return; + } + } + let _ = out.flush(); + let _ = out.shutdown(std::net::Shutdown::Both); +} + +/// Deterministic pseudo-random bytes. +pub(crate) fn noise(len: usize, seed: u64) -> Vec { + let mut state = seed | 1; + let mut out = Vec::with_capacity(len); + while out.len() < len { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + out.extend_from_slice(&state.to_le_bytes()); + } + out.truncate(len); + out +} + +// --------------------------------------------------------------------------- +// Resolver and client +// --------------------------------------------------------------------------- + +/// Test resolver: each host answers from a script; the last answer repeats. +#[derive(Default)] +pub(crate) struct ScriptedLookup { + answers: std::sync::Mutex>>>, + pub(crate) calls: AtomicUsize, +} + +impl ScriptedLookup { + pub(crate) fn with(entries: &[(&str, &[&[&str]])]) -> Arc { + let lookup = Self::default(); + { + let mut answers = lookup.answers.lock().unwrap(); + for (host, script) in entries { + answers.insert( + host.to_string(), + script + .iter() + .map(|answer| { + answer + .iter() + .map(|a| a.parse::().unwrap()) + .collect() + }) + .collect(), + ); + } + } + Arc::new(lookup) + } +} + +impl HostLookup for ScriptedLookup { + fn lookup(&self, host: &str) -> LookupFuture { + self.calls.fetch_add(1, Ordering::SeqCst); + let mut answers = self.answers.lock().unwrap(); + let answer = match answers.get_mut(host) { + Some(script) if script.len() > 1 => Ok(script.pop_front().unwrap()), + Some(script) => Ok(script.front().cloned().unwrap_or_default()), + None => Err(std::io::Error::new( + std::io::ErrorKind::NotFound, + format!("unknown test host {host}"), + )), + }; + Box::pin(async move { answer }) + } +} + +pub(crate) const TEST_HTTP: SafeHttpConfig = SafeHttpConfig { + connect_timeout: Duration::from_secs(5), + read_timeout: Duration::from_secs(5), + total_timeout: Some(Duration::from_secs(20)), + max_redirects: 3, +}; + +pub(crate) fn test_http(lookup: Arc) -> SafeHttp { + SafeHttp::with_lookup(TEST_HTTP, AddressPolicy::test_allow_loopback(), lookup).unwrap() +}