diff --git a/src/Embed.php b/src/Embed.php index 6f2d583a..625e95f4 100644 --- a/src/Embed.php +++ b/src/Embed.php @@ -4,11 +4,15 @@ namespace Embed; use Embed\Http\Crawler; +use InvalidArgumentException; use Psr\Http\Message\RequestInterface; use Psr\Http\Message\ResponseInterface; +use Psr\Http\Message\UriInterface; class Embed { + private const MAX_HTTP_REDIRECTS = 10; + private Crawler $crawler; private ExtractorFactory $extractorFactory; @@ -20,6 +24,9 @@ public function __construct(?Crawler $crawler = null, ?ExtractorFactory $extract public function get(string $url): Extractor { + if (!isValidUrl($url)) { + throw new InvalidArgumentException(sprintf('Access to this URL is blocked for security reasons (%s)', $url)); + } $request = $this->crawler->createRequest('GET', $url); $response = $this->crawler->sendRequest($request); @@ -32,7 +39,15 @@ public function get(string $url): Extractor public function getMulti(string ...$urls): array { $requests = array_map( - fn ($url) => $this->crawler->createRequest('GET', $url), + function ($url): RequestInterface { + if (!isValidUrl($url)) { + throw new InvalidArgumentException(sprintf( + 'Access to this URL is blocked for security reasons (%s)', + $url + )); + } + return $this->crawler->createRequest('GET', $url); + }, $urls ); @@ -68,8 +83,20 @@ public function setSettings(array $settings): void $this->extractorFactory->setSettings($settings); } - private function extract(RequestInterface $request, ResponseInterface $response, bool $redirect = true): Extractor + private function extract(RequestInterface $request, ResponseInterface $response, bool $redirect = true, int $httpRedirects = 0): Extractor { + $httpRedirectUri = $this->getHttpRedirectUri($request, $response); + if ($httpRedirectUri !== null) { + if ($httpRedirects >= self::MAX_HTTP_REDIRECTS) { + throw new InvalidArgumentException('Maximum number of HTTP redirects exceeded'); + } + + $request = $this->createSafeRequest($httpRedirectUri); + $response = $this->crawler->sendRequest($request); + + return $this->extract($request, $response, $redirect, $httpRedirects + 1); + } + $uri = $this->crawler->getResponseUri($response); if ($uri === null) { $uri = $request->getUri(); @@ -87,7 +114,7 @@ private function extract(RequestInterface $request, ResponseInterface $response, return $extractor; } - $request = $this->crawler->createRequest('GET', (string) $redirectUri); + $request = $this->createSafeRequest($redirectUri); $response = $this->crawler->sendRequest($request); return $this->extract($request, $response, false); @@ -103,4 +130,26 @@ private function mustRedirect(Extractor $extractor): bool $redirectUri = $extractor->redirect; return $redirectUri instanceof \Psr\Http\Message\UriInterface; } + + private function getHttpRedirectUri(RequestInterface $request, ResponseInterface $response): ?UriInterface + { + $status = $response->getStatusCode(); + $location = $response->getHeaderLine('Location'); + + if ($status < 300 || $status >= 400 || $location === '') { + return null; + } + + return resolveUri($request->getUri(), $this->crawler->createUri($location)); + } + + private function createSafeRequest(UriInterface $uri): RequestInterface + { + $url = (string) $uri; + if (!isValidUrl($url)) { + throw new InvalidArgumentException(sprintf('Access to this URL is blocked for security reasons (%s)', $url)); + } + + return $this->crawler->createRequest('GET', $url); + } } diff --git a/src/Extractor.php b/src/Extractor.php index 641968a9..4db64e89 100644 --- a/src/Extractor.php +++ b/src/Extractor.php @@ -253,7 +253,9 @@ public function resolveUri($uri): UriInterface if (!isHttp($uri)) { throw new InvalidArgumentException(sprintf('Uri string must use http or https scheme (%s)', $uri)); } - + if (!isValidUrl($uri)) { + throw new InvalidArgumentException(sprintf('Access to this URL is blocked for security reasons (%s)', $uri)); + } $uri = $this->crawler->createUri($uri); } diff --git a/src/Http/CurlDispatcher.php b/src/Http/CurlDispatcher.php index 0658c0fd..06b0a018 100644 --- a/src/Http/CurlDispatcher.php +++ b/src/Http/CurlDispatcher.php @@ -9,6 +9,8 @@ use Psr\Http\Message\ResponseInterface; use Psr\Http\Message\StreamFactoryInterface; use Psr\Http\Message\StreamInterface; +use function Embed\getValidUrlIps; +use InvalidArgumentException; /** * Class to fetch html pages @@ -115,7 +117,23 @@ public static function fetch(array $settings, ResponseFactoryInterface $response private function __construct(array $settings, RequestInterface $request, ?StreamFactoryInterface $streamFactory = null) { $this->request = $request; - $this->curl = curl_init((string) $request->getUri()); + $uri = $request->getUri(); + $url = (string) $uri; + $ips = getValidUrlIps($url); + + if ($ips === []) { + throw new InvalidArgumentException(sprintf('Access to this URL is blocked for security reasons (%s)', $url)); + } + + $host = $uri->getHost(); + $port = $uri->getPort() ?? ($uri->getScheme() === 'https' ? 443 : 80); + $resolveHost = strpos($host, ':') === false ? $host : "[{$host}]"; + $resolveAddresses = array_values(array_map( + static fn (string $ip): string => strpos($ip, ':') === false ? $ip : "[{$ip}]", + $ips + )); + + $this->curl = curl_init($url); $this->settings = $settings; $this->streamFactory = $streamFactory ?? FactoryDiscovery::getStreamFactory(); @@ -133,14 +151,21 @@ private function __construct(array $settings, RequestInterface $request, ?Stream CURLOPT_ENCODING => '', CURLOPT_CAINFO => CaBundle::getSystemCaRootBundlePath(), CURLOPT_AUTOREFERER => true, - CURLOPT_FOLLOWLOCATION => $settings['follow_location'] ?? true, - CURLOPT_IPRESOLVE => CURL_IPRESOLVE_V4, + CURLOPT_FOLLOWLOCATION => false, + CURLOPT_PROTOCOLS => CURLPROTO_HTTP | CURLPROTO_HTTPS, + CURLOPT_REDIR_PROTOCOLS => CURLPROTO_HTTP | CURLPROTO_HTTPS, CURLOPT_USERAGENT => $settings['user_agent'] ?? $request->getHeaderLine('User-Agent'), CURLOPT_COOKIEJAR => $cookies, CURLOPT_COOKIEFILE => $cookies, CURLOPT_HEADERFUNCTION => [$this, 'writeHeader'], CURLOPT_WRITEFUNCTION => [$this, 'writeBody'], ]); + + curl_setopt( + $this->curl, + CURLOPT_RESOLVE, + [sprintf('%s:%d:%s', $resolveHost, $port, implode(',', $resolveAddresses))] + ); } private function getResponse(ResponseFactoryInterface $responseFactory): ResponseInterface diff --git a/src/functions.php b/src/functions.php index 9965ee63..856d496f 100644 --- a/src/functions.php +++ b/src/functions.php @@ -73,6 +73,97 @@ function resolveUri(UriInterface $base, UriInterface $uri): UriInterface ->withFragment(''); } +/** + * Return the public IP addresses associated with an HTTP(S) URL. + * + * @return string[] An empty array means that the URL is not safe to fetch. + */ +function getValidUrlIps(string $url): array +{ + if (filter_var($url, FILTER_VALIDATE_URL) === false) { + return []; + } + + $parts = parse_url($url); + if ($parts === false) { + return []; + } + + $scheme = strtolower($parts['scheme'] ?? ''); + if (!in_array($scheme, ['http', 'https'], true)) { + return []; + } + + $host = $parts['host'] ?? ''; + // Normalize IPv6 literal formatting wrapping (e.g., [::1] -> ::1) + if (strpos($host, '[') === 0 && strpos($host, ']') === (strlen($host) - 1)) { + $host = substr($host, 1, -1); + } + // Collect all IPs the host resolves to. + $ips = []; + $ips = []; + if (filter_var($host, FILTER_VALIDATE_IP) !== false) { + // The host is already a direct IP address literal. + $ips[] = $host; + } + else { + // Resolve DNS records for both IPv4 (A) and IPv6 (AAAA). + $dnsA = @dns_get_record($host, DNS_A); + $dnsAAAA = @dns_get_record($host, DNS_AAAA); + if (is_array($dnsA)) { + foreach ($dnsA as $record) { + if (isset($record['ip']) && is_string($record['ip'])) { + $ips[] = $record['ip']; + } + } + } + if (is_array($dnsAAAA)) { + foreach ($dnsAAAA as $record) { + if (isset($record['ipv6']) && is_string($record['ipv6'])) { + $ips[] = $record['ipv6']; + } + } + } + // Fallback. If dns_get_record fails but gethostbyname finds something. + if ($ips === []) { + $fallbackIp = @gethostbyname($host); + if ($fallbackIp !== $host) { + $ips[] = $fallbackIp; + } + else { + // If DNS resolution fails, treat URL as invalid. + return []; + } + } + } + // 4. Validate resolved IPs against standard restricted ranges + foreach ($ips as $ip) { + // Check if the IP is valid and falls outside reserved/private scopes + // FILTER_FLAG_NO_PRIV_RANGE: Blocks RFC1918 (10/8, 172.16/12, 192.168/16) + // FILTER_FLAG_NO_RES_RANGE: Blocks Loopback (127.0.0.0/8, ::1) + // and Link-Local (169.254.0.0/16). + $isPublic = filter_var( + $ip, + FILTER_VALIDATE_IP, + FILTER_FLAG_NO_PRIV_RANGE | FILTER_FLAG_NO_RES_RANGE | FILTER_FLAG_GLOBAL_RANGE + ); + if ($isPublic === false) { + // The IP belongs to a restricted/private range. + return []; + } + } + + return array_values(array_unique($ips)); +} + +/** + * Check if the DNS associated with the URL is valid (SSRF). + */ +function isValidUrl(string $url): bool +{ + return getValidUrlIps($url) !== []; +} + function isHttp(string $uri): bool { $result = preg_match('/^(\w+):/', $uri, $matches); diff --git a/tests/FunctionsTest.php b/tests/FunctionsTest.php index bb5e959d..72e495df 100644 --- a/tests/FunctionsTest.php +++ b/tests/FunctionsTest.php @@ -4,8 +4,10 @@ namespace Embed\Tests; use function Embed\isHttp; +use function Embed\isValidUrl; use PHPUnit\Framework\TestCase; + class FunctionsTest extends TestCase { public function urlsProvider(): array @@ -24,6 +26,40 @@ public function urlsProvider(): array ]; } + public function invalidUrlsProvider(): array { + return [ + ['https://169.254.0.0/SSRF_PATH', false], + ['169.254.0.0/SSRF_PATH', false], + ['https://169.254.169.254/latest/meta-data/iam/security-credentials/', false], + ['https://127.0.0.1:9999/SSRF_PATH', false], + ['http://127.0.0.1:9999/SSRF_PATH', false], + ['https://10.0.0.0/SSRF_PATH', false], + ['10.0.0.0/SSRF_PATH', false], + ['https://172.16.0.0/SSRF_PATH', false], + ['172.16.0.0/SSRF_PATH', false], + ['https://192.168.0.0/SSRF_PATH', false], + ['192.168.0.0/SSRF_PATH', false], + ['http://localhost:8080/admin', false], + ['https://100.100.100.200', false], // Alibaba metadata + ['100.100.100.200', false], // Alibaba metadata + ['64:ff9b::192.0.2.1', false], // embedded IPv4: 192.0.2.1 + ['64:ff9b::192.0.2.1', false], // embedded IPv4: 192.0.2.1 + ['https://64:ff9b::198.51.100.1', false], // embedded IPv4: 198.51.100.1 + ['64:ff9b::198.51.100.1', false], // embedded IPv4: 198.51.100.1 + ['https://198.18.50.25', false], + ['198.18.50.25', false], + ['https://198.19.200.10', false], + ['198.19.200.10', false], + ['https://100.64.0.1', false], + ['100.64.0.1', false], + ['https://100.100.50.25', false], + ['100.100.50.25', false], + ['./foo', false], + ['/foo', false], + ['../foo', false], + ]; + } + /** * @dataProvider urlsProvider */ @@ -32,4 +68,13 @@ public function testIsHttp(string $url, bool $expected) $result = isHttp($url); $this->assertSame($expected, $result); } + + /** + * @dataProvider invalidUrlsProvider + */ + public function testIsValidUrl(string $url, bool $expected) + { + $result = isValidUrl($url); + $this->assertSame($expected, $result); + } } diff --git a/tests/MultipleRequestsTest.php b/tests/MultipleRequestsTest.php index 42348449..af9a4478 100644 --- a/tests/MultipleRequestsTest.php +++ b/tests/MultipleRequestsTest.php @@ -4,10 +4,12 @@ namespace Embed\Tests; use Embed\Embed; +use InvalidArgumentException; use PHPUnit\Framework\TestCase; class MultipleRequestsTest extends TestCase { + public function testParallel() { $embed = new Embed(); @@ -27,4 +29,13 @@ public function testParallel() $this->assertEquals('https://x.com/misteroom', (string) $infos[2]->url); $this->assertEquals('en', $infos[2]->language); } + + public function testInvalid(): void { + $this->expectException(InvalidArgumentException::class); + + (new Embed())->getMulti( + 'https://twitter.com/misteroom', + 'https://64:ff9b::198.51.100.1' + ); + } } diff --git a/tests/PagesTest.php b/tests/PagesTest.php index 441b15b5..59e04ced 100644 --- a/tests/PagesTest.php +++ b/tests/PagesTest.php @@ -3,6 +3,9 @@ namespace Embed\Tests; +use Embed\Embed; +use InvalidArgumentException; + class PagesTest extends PagesTestCase { /** @@ -210,4 +213,11 @@ public function testBBCNews() { $this->assertEmbed('https://www.bbc.co.uk/news/uk-54222286'); } + + public function testInvalid(): void + { + $this->expectException(InvalidArgumentException::class); + $embed = new Embed(); + $embed->get('https://64:ff9b::198.51.100.1'); + } }