2026-07-02 04:11:40 +00:00
|
|
|
"""SSRF URL validation for docreader outbound HTTP requests.
|
|
|
|
|
|
|
|
|
|
Mirrors the core policy in internal/utils/security.go so redirect targets
|
|
|
|
|
during Playwright navigation are blocked the same way as Go-side imports.
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
from __future__ import annotations
|
|
|
|
|
|
|
|
|
|
import ipaddress
|
|
|
|
|
import os
|
|
|
|
|
import re
|
|
|
|
|
import socket
|
|
|
|
|
from functools import lru_cache
|
|
|
|
|
from typing import FrozenSet, Optional, Tuple, Union
|
|
|
|
|
from urllib.parse import urlparse
|
|
|
|
|
|
|
|
|
|
RESTRICTED_HOSTNAMES: FrozenSet[str] = frozenset(
|
|
|
|
|
{
|
|
|
|
|
"localhost",
|
|
|
|
|
"127.0.0.1",
|
|
|
|
|
"::1",
|
|
|
|
|
"0.0.0.0",
|
|
|
|
|
"metadata.google.internal",
|
|
|
|
|
"metadata.tencentyun.com",
|
|
|
|
|
"metadata.aws.internal",
|
|
|
|
|
"host.docker.internal",
|
|
|
|
|
"gateway.docker.internal",
|
|
|
|
|
"kubernetes.docker.internal",
|
|
|
|
|
"kubernetes",
|
|
|
|
|
"kubernetes.default",
|
|
|
|
|
"kubernetes.default.svc",
|
|
|
|
|
"kubernetes.default.svc.cluster.local",
|
|
|
|
|
}
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
RESTRICTED_SUFFIXES: Tuple[str, ...] = (
|
|
|
|
|
".local",
|
|
|
|
|
".localhost",
|
|
|
|
|
".internal",
|
|
|
|
|
".corp",
|
|
|
|
|
".lan",
|
|
|
|
|
".home",
|
|
|
|
|
".localdomain",
|
|
|
|
|
".svc.cluster.local",
|
|
|
|
|
".pod.cluster.local",
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
EXTRA_RESTRICTED_CIDRS: Tuple[Union[ipaddress.IPv4Network, ipaddress.IPv6Network], ...] = tuple(
|
|
|
|
|
ipaddress.ip_network(cidr)
|
|
|
|
|
for cidr in (
|
|
|
|
|
"100.64.0.0/10",
|
|
|
|
|
"198.18.0.0/15",
|
|
|
|
|
"198.51.100.0/24",
|
|
|
|
|
"203.0.113.0/24",
|
|
|
|
|
"192.0.0.0/24",
|
|
|
|
|
"192.0.2.0/24",
|
|
|
|
|
"0.0.0.0/8",
|
|
|
|
|
"240.0.0.0/4",
|
|
|
|
|
"255.255.255.255/32",
|
|
|
|
|
"172.17.0.0/16",
|
|
|
|
|
"172.18.0.0/16",
|
|
|
|
|
"172.19.0.0/16",
|
|
|
|
|
"172.20.0.0/16",
|
|
|
|
|
)
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
BLOCKED_PORTS: FrozenSet[str] = frozenset(
|
|
|
|
|
{
|
|
|
|
|
"22",
|
|
|
|
|
"23",
|
|
|
|
|
"25",
|
|
|
|
|
"445",
|
|
|
|
|
"3389",
|
|
|
|
|
"5432",
|
|
|
|
|
"3306",
|
|
|
|
|
"6379",
|
|
|
|
|
"27017",
|
|
|
|
|
"9200",
|
|
|
|
|
"2379",
|
|
|
|
|
"2380",
|
|
|
|
|
"8500",
|
|
|
|
|
"4001",
|
|
|
|
|
}
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
_IP_LIKE_PATTERNS = (
|
|
|
|
|
re.compile(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$"),
|
|
|
|
|
re.compile(r"^\d{8,10}$"),
|
|
|
|
|
re.compile(r"^0[0-7]+\."),
|
|
|
|
|
re.compile(r"(?i)^0x[0-9a-f]+\."),
|
|
|
|
|
re.compile(r"(?i)^0x[0-9a-f]{6,8}$"),
|
|
|
|
|
re.compile(r"(?i)^[0-9a-f:]+::[0-9a-f:]*$"),
|
|
|
|
|
re.compile(r"(?i)^[0-9a-f]{1,4}(:[0-9a-f]{1,4}){7}$"),
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _normalize_url(raw_url: str) -> str:
|
|
|
|
|
if "://" not in raw_url:
|
|
|
|
|
return f"https://{raw_url}"
|
|
|
|
|
return raw_url
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@lru_cache(maxsize=1)
|
|
|
|
|
def _load_whitelist() -> Tuple[FrozenSet[str], Tuple[str, ...], Tuple[Union[ipaddress.IPv4Network, ipaddress.IPv6Network], ...]]:
|
|
|
|
|
entries: list[str] = []
|
|
|
|
|
for env_key in ("SSRF_WHITELIST", "SSRF_WHITELIST_EXTRA"):
|
|
|
|
|
raw = os.environ.get(env_key, "")
|
|
|
|
|
if raw.strip():
|
|
|
|
|
entries.extend(part.strip() for part in raw.split(",") if part.strip())
|
|
|
|
|
|
|
|
|
|
exact_hosts: set[str] = set()
|
|
|
|
|
suffix_hosts: list[str] = []
|
|
|
|
|
cidr_nets: list[Union[ipaddress.IPv4Network, ipaddress.IPv6Network]] = []
|
|
|
|
|
for entry in entries:
|
|
|
|
|
lowered = entry.lower()
|
|
|
|
|
if lowered.startswith("*."):
|
|
|
|
|
suffix_hosts.append(lowered[1:])
|
|
|
|
|
continue
|
|
|
|
|
if "/" in lowered:
|
|
|
|
|
try:
|
|
|
|
|
cidr_nets.append(ipaddress.ip_network(lowered, strict=False))
|
|
|
|
|
except ValueError:
|
|
|
|
|
continue
|
|
|
|
|
continue
|
|
|
|
|
exact_hosts.add(lowered)
|
|
|
|
|
return frozenset(exact_hosts), tuple(suffix_hosts), tuple(cidr_nets)
|
|
|
|
|
|
|
|
|
|
|
feat(security): add a whitelist-only outbound mode that refuses before DNS (#3393)
* feat(security): add a whitelist-only outbound mode that refuses before DNS
A penetration test of a private deployment found that a host outside
SSRF_WHITELIST still produces a DNS query: every outbound guard resolves
the name first and judges the addresses afterwards, which is what makes
it immune to DNS rebinding, but it also means an attacker-chosen name
reaches the resolver and can carry data out of an otherwise offline
network (#3378).
SSRF_DNS_WHITELIST_ONLY makes the whitelist the whole egress policy, and
the check is placed wherever a name is still in hand.
Where a URL is available, CheckSSRFWhitelistOnly refuses a host the
whitelist does not name, IP literals included: URL validation, the
sandbox outbound policy, and the web-fetch target resolver.
Where only a dial address is available, CheckDialWhitelistOnly judges
hostnames and leaves addresses to the checks that already exist, because
an IP arriving at a dialer is either a name something else resolved or a
literal the URL layer already refused. That covers the shared SSRF-safe
dialer, the web-fetch dialer, the Yunzhijia webhook dialer, and every
sandbox transport: the HTTP and websocket data planes, the guarded
transport and the Docker engine now share one GuardedDialContext instead
of each building its own dialer.
Where the client resolves its own target, the name is judged before the
client exists. gRPC's resolver runs before any dialer it was given, so
the env-configured qdrant and milvus clients are built through small
constructors that refuse a non-whitelisted host outright; a whitelisted
one is resolved by gRPC as before.
docreader enforces the same switch in its own validator, and compose
passes the variable to both services.
Matching is by name only in this mode: the whitelist's CIDR entries
still match an IP literal, but a hostname is no longer resolved to see
whether its addresses fall inside one, because that lookup is the query
the mode exists to prevent.
The value is parsed with ParseBool, and a non-empty value that parses as
neither enables the mode: silently leaving an egress lockdown off
because an operator wrote "yes" is the one failure mode a security
control must not have.
Still not covered, and documented as such: the Langfuse OTLP exporter
builds its own client and never reaches these guards; and with
HTTP(S)_PROXY set the proxy host is dialled unguarded when the dial
address equals the proxy URL's host exactly, while a mismatch makes the
proxy host itself subject to the whitelist.
The switch defaults to off, so nothing changes for deployments that do
not set it.
* docs(security): document SSRF_DNS_WHITELIST_ONLY in website-docs
docs/ has been retired in favour of website-docs/ (#3580), so the notes that
lived in docs/QA.md move to the configuration table: the new switch, what to
whitelist before turning it on, and the three outbound paths it still does
not cover.
The existing SSRF_WHITELIST row was also stale: compose gives only the app its
SSRF_WHITELIST_EXTRA default (which now includes minio), and docreader's is
empty, so "shared by app and docreader" held for SSRF_WHITELIST alone.
2026-09-23 04:02:06 +00:00
|
|
|
def _whitelist_only_enabled() -> bool:
|
|
|
|
|
"""Whether SSRF_DNS_WHITELIST_ONLY makes the whitelist the whole egress
|
|
|
|
|
policy: a host outside it is refused before any DNS query (#3378).
|
|
|
|
|
|
|
|
|
|
Parsed like the Go side: 1/t/true enable it, 0/f/false disable it, and a
|
|
|
|
|
non-empty value that is neither enables it — silently leaving an egress
|
|
|
|
|
lockdown off because someone wrote "yes" is the one failure mode this
|
|
|
|
|
control must not have.
|
|
|
|
|
"""
|
|
|
|
|
raw = os.environ.get("SSRF_DNS_WHITELIST_ONLY", "").strip()
|
|
|
|
|
if not raw:
|
|
|
|
|
return False
|
|
|
|
|
return raw.lower() not in {"0", "f", "false"}
|
|
|
|
|
|
|
|
|
|
|
2026-07-02 04:11:40 +00:00
|
|
|
def _is_whitelisted(hostname: str) -> bool:
|
|
|
|
|
lowered = hostname.lower()
|
|
|
|
|
exact_hosts, suffix_hosts, cidr_nets = _load_whitelist()
|
|
|
|
|
if lowered in exact_hosts:
|
|
|
|
|
return True
|
|
|
|
|
for suffix in suffix_hosts:
|
|
|
|
|
if lowered.endswith(suffix) or lowered == suffix.lstrip("."):
|
|
|
|
|
return True
|
|
|
|
|
try:
|
|
|
|
|
ip = ipaddress.ip_address(lowered)
|
|
|
|
|
except ValueError:
|
|
|
|
|
return False
|
|
|
|
|
return any(ip in net for net in cidr_nets)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _is_ip_like_hostname(hostname: str) -> bool:
|
|
|
|
|
return any(pattern.search(hostname) for pattern in _IP_LIKE_PATTERNS)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _is_restricted_ip(ip: Union[ipaddress.IPv4Address, ipaddress.IPv6Address]) -> Optional[str]:
|
|
|
|
|
if ip.is_private:
|
|
|
|
|
return "private IP address"
|
|
|
|
|
if ip.is_loopback:
|
|
|
|
|
return "loopback address"
|
|
|
|
|
if ip.is_link_local:
|
|
|
|
|
return "link-local address"
|
|
|
|
|
if ip.is_multicast:
|
|
|
|
|
return "multicast address"
|
|
|
|
|
if ip.is_unspecified:
|
|
|
|
|
return "unspecified address"
|
|
|
|
|
if isinstance(ip, ipaddress.IPv4Address):
|
|
|
|
|
for net in EXTRA_RESTRICTED_CIDRS:
|
|
|
|
|
if isinstance(net, ipaddress.IPv4Network) and ip in net:
|
|
|
|
|
return f"restricted range {net}"
|
|
|
|
|
if isinstance(ip, ipaddress.IPv6Address):
|
2026-08-10 05:24:14 +00:00
|
|
|
embedded_ipv4 = ip.ipv4_mapped
|
|
|
|
|
if embedded_ipv4 is not None:
|
|
|
|
|
reason = _is_restricted_ip(embedded_ipv4)
|
|
|
|
|
if reason:
|
|
|
|
|
return f"IPv4-mapped {reason}"
|
|
|
|
|
if ip.sixtofour is not None:
|
|
|
|
|
reason = _is_restricted_ip(ip.sixtofour)
|
|
|
|
|
if reason:
|
|
|
|
|
return f"6to4-embedded {reason}"
|
|
|
|
|
if ip.teredo is not None:
|
|
|
|
|
server_ip, client_ip = ip.teredo
|
|
|
|
|
for label, embedded_ip in (("server", server_ip), ("client", client_ip)):
|
|
|
|
|
reason = _is_restricted_ip(embedded_ip)
|
|
|
|
|
if reason:
|
|
|
|
|
return f"Teredo {label} embeds {reason}"
|
2026-09-01 07:27:18 +00:00
|
|
|
# NAT64's well-known prefix (64:ff9b::/96, RFC 6052) and the deprecated
|
|
|
|
|
# IPv4-compatible form (::a.b.c.d) also carry a plain IPv4 address, but
|
|
|
|
|
# ipaddress exposes no accessor for either, so is_private and
|
|
|
|
|
# .sixtofour never see the payload. Without these two branches
|
|
|
|
|
# ::169.254.169.254 reads as an ordinary public IPv6 address.
|
|
|
|
|
# Mirrors internal/ipclass on the Go side.
|
|
|
|
|
packed = ip.packed
|
2026-09-16 08:20:57 +00:00
|
|
|
# RFC 8215 local-use NAT64 has deployment-specific IPv4 layouts.
|
|
|
|
|
# Reject the entire /48, matching internal/ipclass, on every Python version.
|
|
|
|
|
if packed[:6] == b"\x00\x64\xff\x9b\x00\x01":
|
|
|
|
|
return "local-use NAT64 translation address"
|
2026-09-01 07:27:18 +00:00
|
|
|
if packed[0:4] == b"\x00\x64\xff\x9b" and packed[4:12] == bytes(8):
|
|
|
|
|
reason = _is_restricted_ip(ipaddress.IPv4Address(packed[12:16]))
|
|
|
|
|
if reason:
|
|
|
|
|
return f"NAT64-embedded {reason}"
|
|
|
|
|
if packed[0:12] == bytes(12):
|
|
|
|
|
# :: and ::1 already returned above; ::ffff:a.b.c.d is handled by
|
|
|
|
|
# the ipv4_mapped branch.
|
|
|
|
|
reason = _is_restricted_ip(ipaddress.IPv4Address(packed[12:16]))
|
|
|
|
|
if reason:
|
|
|
|
|
return f"IPv4-compatible {reason}"
|
2026-07-02 04:11:40 +00:00
|
|
|
# Site-local (fec0::/10)
|
|
|
|
|
if (ip.packed[0] == 0xFE) and (ip.packed[1] & 0xC0) == 0xC0:
|
|
|
|
|
return "site-local IPv6 address"
|
|
|
|
|
return None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _resolve_host_ips(hostname: str) -> Tuple[Tuple[Union[ipaddress.IPv4Address, ipaddress.IPv6Address], ...], Optional[str]]:
|
|
|
|
|
try:
|
|
|
|
|
infos = socket.getaddrinfo(hostname, None, type=socket.SOCK_STREAM)
|
|
|
|
|
except socket.gaierror as exc:
|
|
|
|
|
return (), f"DNS resolution failed for hostname {hostname}: {exc}"
|
|
|
|
|
ips: list[Union[ipaddress.IPv4Address, ipaddress.IPv6Address]] = []
|
|
|
|
|
seen: set[str] = set()
|
|
|
|
|
for info in infos:
|
|
|
|
|
sockaddr = info[4]
|
|
|
|
|
if not sockaddr:
|
|
|
|
|
continue
|
|
|
|
|
ip_str = sockaddr[0]
|
|
|
|
|
if ip_str in seen:
|
|
|
|
|
continue
|
|
|
|
|
seen.add(ip_str)
|
|
|
|
|
try:
|
|
|
|
|
ips.append(ipaddress.ip_address(ip_str))
|
|
|
|
|
except ValueError:
|
|
|
|
|
continue
|
|
|
|
|
if not ips:
|
|
|
|
|
return (), f"DNS resolution failed for hostname {hostname}: no addresses"
|
|
|
|
|
return tuple(ips), None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def is_ssrf_safe_url(raw_url: str) -> Tuple[bool, str]:
|
|
|
|
|
"""Return (safe, reason). reason is empty when safe is True."""
|
|
|
|
|
if not raw_url or not raw_url.strip():
|
|
|
|
|
return False, "URL is empty"
|
|
|
|
|
|
|
|
|
|
normalized = _normalize_url(raw_url.strip())
|
|
|
|
|
parsed = urlparse(normalized)
|
|
|
|
|
scheme = (parsed.scheme or "").lower()
|
|
|
|
|
if scheme not in {"http", "https"}:
|
|
|
|
|
return False, f"invalid scheme: {scheme or '(none)'} (only http/https allowed)"
|
|
|
|
|
|
|
|
|
|
hostname = (parsed.hostname or "").strip()
|
|
|
|
|
if not hostname:
|
|
|
|
|
return False, "URL has no hostname"
|
|
|
|
|
|
|
|
|
|
hostname_lower = hostname.lower()
|
|
|
|
|
if _is_whitelisted(hostname_lower):
|
|
|
|
|
return True, ""
|
|
|
|
|
|
feat(security): add a whitelist-only outbound mode that refuses before DNS (#3393)
* feat(security): add a whitelist-only outbound mode that refuses before DNS
A penetration test of a private deployment found that a host outside
SSRF_WHITELIST still produces a DNS query: every outbound guard resolves
the name first and judges the addresses afterwards, which is what makes
it immune to DNS rebinding, but it also means an attacker-chosen name
reaches the resolver and can carry data out of an otherwise offline
network (#3378).
SSRF_DNS_WHITELIST_ONLY makes the whitelist the whole egress policy, and
the check is placed wherever a name is still in hand.
Where a URL is available, CheckSSRFWhitelistOnly refuses a host the
whitelist does not name, IP literals included: URL validation, the
sandbox outbound policy, and the web-fetch target resolver.
Where only a dial address is available, CheckDialWhitelistOnly judges
hostnames and leaves addresses to the checks that already exist, because
an IP arriving at a dialer is either a name something else resolved or a
literal the URL layer already refused. That covers the shared SSRF-safe
dialer, the web-fetch dialer, the Yunzhijia webhook dialer, and every
sandbox transport: the HTTP and websocket data planes, the guarded
transport and the Docker engine now share one GuardedDialContext instead
of each building its own dialer.
Where the client resolves its own target, the name is judged before the
client exists. gRPC's resolver runs before any dialer it was given, so
the env-configured qdrant and milvus clients are built through small
constructors that refuse a non-whitelisted host outright; a whitelisted
one is resolved by gRPC as before.
docreader enforces the same switch in its own validator, and compose
passes the variable to both services.
Matching is by name only in this mode: the whitelist's CIDR entries
still match an IP literal, but a hostname is no longer resolved to see
whether its addresses fall inside one, because that lookup is the query
the mode exists to prevent.
The value is parsed with ParseBool, and a non-empty value that parses as
neither enables the mode: silently leaving an egress lockdown off
because an operator wrote "yes" is the one failure mode a security
control must not have.
Still not covered, and documented as such: the Langfuse OTLP exporter
builds its own client and never reaches these guards; and with
HTTP(S)_PROXY set the proxy host is dialled unguarded when the dial
address equals the proxy URL's host exactly, while a mismatch makes the
proxy host itself subject to the whitelist.
The switch defaults to off, so nothing changes for deployments that do
not set it.
* docs(security): document SSRF_DNS_WHITELIST_ONLY in website-docs
docs/ has been retired in favour of website-docs/ (#3580), so the notes that
lived in docs/QA.md move to the configuration table: the new switch, what to
whitelist before turning it on, and the three outbound paths it still does
not cover.
The existing SSRF_WHITELIST row was also stale: compose gives only the app its
SSRF_WHITELIST_EXTRA default (which now includes minio), and docreader's is
empty, so "shared by app and docreader" held for SSRF_WHITELIST alone.
2026-09-23 04:02:06 +00:00
|
|
|
# Whitelist-only mode stops here: everything below resolves the name, and
|
|
|
|
|
# a host outside the whitelist must not reach DNS at all (#3378).
|
|
|
|
|
if _whitelist_only_enabled():
|
|
|
|
|
return (
|
|
|
|
|
False,
|
|
|
|
|
f"host is not in the SSRF whitelist: {hostname_lower} "
|
|
|
|
|
"(SSRF_DNS_WHITELIST_ONLY is on; add it to SSRF_WHITELIST to allow it)",
|
|
|
|
|
)
|
|
|
|
|
|
2026-07-02 04:11:40 +00:00
|
|
|
if hostname_lower in RESTRICTED_HOSTNAMES:
|
|
|
|
|
return False, f"hostname {hostname_lower} is restricted"
|
|
|
|
|
|
|
|
|
|
for suffix in RESTRICTED_SUFFIXES:
|
|
|
|
|
if hostname_lower.endswith(suffix):
|
|
|
|
|
return False, f"hostname suffix {suffix} is restricted"
|
|
|
|
|
|
|
|
|
|
try:
|
|
|
|
|
ipaddress.ip_address(hostname_lower)
|
|
|
|
|
return False, "direct IP address access is not allowed, use domain name or add to SSRF_WHITELIST"
|
|
|
|
|
except ValueError:
|
|
|
|
|
pass
|
|
|
|
|
|
|
|
|
|
if _is_ip_like_hostname(hostname_lower):
|
|
|
|
|
return False, "IP-like hostname format is not allowed"
|
|
|
|
|
|
|
|
|
|
resolved_ips, resolve_err = _resolve_host_ips(hostname_lower)
|
|
|
|
|
if resolve_err:
|
|
|
|
|
return False, resolve_err
|
|
|
|
|
|
|
|
|
|
for resolved_ip in resolved_ips:
|
|
|
|
|
reason = _is_restricted_ip(resolved_ip)
|
|
|
|
|
if reason:
|
|
|
|
|
return (
|
|
|
|
|
False,
|
|
|
|
|
f"hostname {hostname_lower} resolves to restricted IP {resolved_ip}: {reason}",
|
|
|
|
|
)
|
|
|
|
|
|
2026-08-10 05:24:14 +00:00
|
|
|
try:
|
|
|
|
|
port = parsed.port
|
|
|
|
|
except ValueError as exc:
|
|
|
|
|
return False, f"invalid port: {exc}"
|
2026-07-02 04:11:40 +00:00
|
|
|
if port is not None and str(port) in BLOCKED_PORTS:
|
|
|
|
|
return False, f"port {port} is blocked for security reasons"
|
|
|
|
|
|
|
|
|
|
return True, ""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def reset_ssrf_whitelist_cache_for_test() -> None:
|
|
|
|
|
"""Clear cached whitelist entries (for unit tests only)."""
|
|
|
|
|
_load_whitelist.cache_clear()
|