573 - Check URLs are valid before attempting to perform HTTP request

src/Embed.php

@@ -4,11 +4,15 @@
 namespace Embed;
 
 use Embed\Http\Crawler;
+use InvalidArgumentException;
 use Psr\Http\Message\RequestInterface;
 use Psr\Http\Message\ResponseInterface;
+use Psr\Http\Message\UriInterface;
 
 class Embed
 {
+    private const MAX_HTTP_REDIRECTS = 10;
+
     private Crawler $crawler;
     private ExtractorFactory $extractorFactory;
 
@@ -20,6 +24,9 @@ public function __construct(?Crawler $crawler = null, ?ExtractorFactory $extract
 
     public function get(string $url): Extractor
     {
+        if (!isValidUrl($url)) {
+            throw new InvalidArgumentException(sprintf('Access to this URL is blocked for security reasons (%s)', $url));
+        }
         $request = $this->crawler->createRequest('GET', $url);
         $response = $this->crawler->sendRequest($request);
 
@@ -32,7 +39,15 @@ public function get(string $url): Extractor
     public function getMulti(string ...$urls): array
     {
         $requests = array_map(
-            fn ($url) => $this->crawler->createRequest('GET', $url),
+            function ($url): RequestInterface {
+                if (!isValidUrl($url)) {
+                    throw new InvalidArgumentException(sprintf(
+                        'Access to this URL is blocked for security reasons (%s)',
+                        $url
+                    ));
+                }
+                return $this->crawler->createRequest('GET', $url);
+            },
             $urls
         );
 
@@ -68,8 +83,20 @@ public function setSettings(array $settings): void
         $this->extractorFactory->setSettings($settings);
     }
 
-    private function extract(RequestInterface $request, ResponseInterface $response, bool $redirect = true): Extractor
+    private function extract(RequestInterface $request, ResponseInterface $response, bool $redirect = true, int $httpRedirects = 0): Extractor
     {
+        $httpRedirectUri = $this->getHttpRedirectUri($request, $response);
+        if ($httpRedirectUri !== null) {
+            if ($httpRedirects >= self::MAX_HTTP_REDIRECTS) {
+                throw new InvalidArgumentException('Maximum number of HTTP redirects exceeded');
+            }
+
+            $request = $this->createSafeRequest($httpRedirectUri);
+            $response = $this->crawler->sendRequest($request);
+
+            return $this->extract($request, $response, $redirect, $httpRedirects + 1);
+        }
+
         $uri = $this->crawler->getResponseUri($response);
         if ($uri === null) {
             $uri = $request->getUri();
@@ -87,7 +114,7 @@ private function extract(RequestInterface $request, ResponseInterface $response,
             return $extractor;
         }
 
-        $request = $this->crawler->createRequest('GET', (string) $redirectUri);
+        $request = $this->createSafeRequest($redirectUri);
         $response = $this->crawler->sendRequest($request);
 
         return $this->extract($request, $response, false);
@@ -103,4 +130,26 @@ private function mustRedirect(Extractor $extractor): bool
         $redirectUri = $extractor->redirect;
         return $redirectUri instanceof \Psr\Http\Message\UriInterface;
     }
+
+    private function getHttpRedirectUri(RequestInterface $request, ResponseInterface $response): ?UriInterface
+    {
+        $status = $response->getStatusCode();
+        $location = $response->getHeaderLine('Location');
+
+        if ($status < 300 || $status >= 400 || $location === '') {
+            return null;
+        }
+
+        return resolveUri($request->getUri(), $this->crawler->createUri($location));
+    }
+
+    private function createSafeRequest(UriInterface $uri): RequestInterface
+    {
+        $url = (string) $uri;
+        if (!isValidUrl($url)) {
+            throw new InvalidArgumentException(sprintf('Access to this URL is blocked for security reasons (%s)', $url));
+        }
+
+        return $this->crawler->createRequest('GET', $url);
+    }
 }

src/Extractor.php

@@ -253,7 +253,9 @@ public function resolveUri($uri): UriInterface
             if (!isHttp($uri)) {
                 throw new InvalidArgumentException(sprintf('Uri string must use http or https scheme (%s)', $uri));
             }
-
+            if (!isValidUrl($uri)) {
+                throw new InvalidArgumentException(sprintf('Access to this URL is blocked for security reasons (%s)', $uri));
+            }
             $uri = $this->crawler->createUri($uri);
         }
 

src/Http/CurlDispatcher.php

@@ -9,6 +9,8 @@
 use Psr\Http\Message\ResponseInterface;
 use Psr\Http\Message\StreamFactoryInterface;
 use Psr\Http\Message\StreamInterface;
+use function Embed\getValidUrlIps;
+use InvalidArgumentException;
 
 /**
  * Class to fetch html pages
@@ -115,7 +117,23 @@ public static function fetch(array $settings, ResponseFactoryInterface $response
     private function __construct(array $settings, RequestInterface $request, ?StreamFactoryInterface $streamFactory = null)
     {
         $this->request = $request;
-        $this->curl = curl_init((string) $request->getUri());
+        $uri = $request->getUri();
+        $url = (string) $uri;
+        $ips = getValidUrlIps($url);
+
+        if ($ips === []) {
+            throw new InvalidArgumentException(sprintf('Access to this URL is blocked for security reasons (%s)', $url));
+        }
+
+        $host = $uri->getHost();
+        $port = $uri->getPort() ?? ($uri->getScheme() === 'https' ? 443 : 80);
+        $resolveHost = strpos($host, ':') === false ? $host : "[{$host}]";
+        $resolveAddresses = array_values(array_map(
+            static fn (string $ip): string => strpos($ip, ':') === false ? $ip : "[{$ip}]",
+            $ips
+        ));
+
+        $this->curl = curl_init($url);
         $this->settings = $settings;
         $this->streamFactory = $streamFactory ?? FactoryDiscovery::getStreamFactory();
 
@@ -133,14 +151,21 @@ private function __construct(array $settings, RequestInterface $request, ?Stream
             CURLOPT_ENCODING => '',
             CURLOPT_CAINFO => CaBundle::getSystemCaRootBundlePath(),
             CURLOPT_AUTOREFERER => true,
-            CURLOPT_FOLLOWLOCATION => $settings['follow_location'] ?? true,
-            CURLOPT_IPRESOLVE => CURL_IPRESOLVE_V4,
+            CURLOPT_FOLLOWLOCATION => false,
+            CURLOPT_PROTOCOLS => CURLPROTO_HTTP | CURLPROTO_HTTPS,
+            CURLOPT_REDIR_PROTOCOLS => CURLPROTO_HTTP | CURLPROTO_HTTPS,
             CURLOPT_USERAGENT => $settings['user_agent'] ?? $request->getHeaderLine('User-Agent'),
             CURLOPT_COOKIEJAR => $cookies,
             CURLOPT_COOKIEFILE => $cookies,
             CURLOPT_HEADERFUNCTION => [$this, 'writeHeader'],
             CURLOPT_WRITEFUNCTION => [$this, 'writeBody'],
         ]);
+
+        curl_setopt(
+            $this->curl,
+            CURLOPT_RESOLVE,
+            [sprintf('%s:%d:%s', $resolveHost, $port, implode(',', $resolveAddresses))]
+        );
     }
 
     private function getResponse(ResponseFactoryInterface $responseFactory): ResponseInterface

src/functions.php

@@ -73,6 +73,97 @@ function resolveUri(UriInterface $base, UriInterface $uri): UriInterface
         ->withFragment('');
 }
 
+/**
+ * Return the public IP addresses associated with an HTTP(S) URL.
+ *
+ * @return string[] An empty array means that the URL is not safe to fetch.
+ */
+function getValidUrlIps(string $url): array
+{
+    if (filter_var($url, FILTER_VALIDATE_URL) === false) {
+        return [];
+    }
+
+    $parts = parse_url($url);
+    if ($parts === false) {
+        return [];
+    }
+
+    $scheme = strtolower($parts['scheme'] ?? '');
+    if (!in_array($scheme, ['http', 'https'], true)) {
+        return [];
+    }
+
+    $host = $parts['host'] ?? '';
+    // Normalize IPv6 literal formatting wrapping (e.g., [::1] -> ::1)
+    if (strpos($host, '[') === 0 && strpos($host, ']') === (strlen($host) - 1)) {
+        $host = substr($host, 1, -1);
+    }
+    // Collect all IPs the host resolves to.
+    $ips = [];
+    $ips = [];
+    if (filter_var($host, FILTER_VALIDATE_IP) !== false) {
+        // The host is already a direct IP address literal.
+        $ips[] = $host;
+    }
+    else {
+        // Resolve DNS records for both IPv4 (A) and IPv6 (AAAA).
+        $dnsA = @dns_get_record($host, DNS_A);
+        $dnsAAAA = @dns_get_record($host, DNS_AAAA);
+        if (is_array($dnsA)) {
+            foreach ($dnsA as $record) {
+                if (isset($record['ip']) && is_string($record['ip'])) {
+                    $ips[] = $record['ip'];
+                }
+            }
+        }
+        if (is_array($dnsAAAA)) {
+            foreach ($dnsAAAA as $record) {
+                if (isset($record['ipv6']) && is_string($record['ipv6'])) {
+                    $ips[] = $record['ipv6'];
+                }
+            }
+        }
+        // Fallback. If dns_get_record fails but gethostbyname finds something.
+        if ($ips === []) {
+            $fallbackIp = @gethostbyname($host);
+            if ($fallbackIp !== $host) {
+                $ips[] = $fallbackIp;
+            }
+            else {
+                // If DNS resolution fails, treat URL as invalid.
+                return [];
+            }
+        }
+    }
+    // 4. Validate resolved IPs against standard restricted ranges
+    foreach ($ips as $ip) {
+        // Check if the IP is valid and falls outside reserved/private scopes
+        // FILTER_FLAG_NO_PRIV_RANGE: Blocks RFC1918 (10/8, 172.16/12, 192.168/16)
+        // FILTER_FLAG_NO_RES_RANGE: Blocks Loopback (127.0.0.0/8, ::1)
+        // and Link-Local (169.254.0.0/16).
+        $isPublic = filter_var(
+            $ip,
+            FILTER_VALIDATE_IP,
+            FILTER_FLAG_NO_PRIV_RANGE | FILTER_FLAG_NO_RES_RANGE | FILTER_FLAG_GLOBAL_RANGE
+        );
+        if ($isPublic === false) {
+            // The IP belongs to a restricted/private range.
+            return [];
+        }
+    }
+
+    return array_values(array_unique($ips));
+}
+
+/**
+ * Check if the DNS associated with the URL is valid (SSRF).
+ */
+function isValidUrl(string $url): bool
+{
+    return getValidUrlIps($url) !== [];
+}
+
 function isHttp(string $uri): bool
 {
     $result = preg_match('/^(\w+):/', $uri, $matches);

tests/FunctionsTest.php

@@ -4,8 +4,10 @@
 namespace Embed\Tests;
 
 use function Embed\isHttp;
+use function Embed\isValidUrl;
 use PHPUnit\Framework\TestCase;
 
+
 class FunctionsTest extends TestCase
 {
     public function urlsProvider(): array
@@ -24,6 +26,40 @@ public function urlsProvider(): array
         ];
     }
 
+    public function invalidUrlsProvider(): array {
+        return [
+            ['https://169.254.0.0/SSRF_PATH', false],
+            ['169.254.0.0/SSRF_PATH', false],
+            ['https://169.254.169.254/latest/meta-data/iam/security-credentials/', false],
+            ['https://127.0.0.1:9999/SSRF_PATH', false],
+            ['http://127.0.0.1:9999/SSRF_PATH', false],
+            ['https://10.0.0.0/SSRF_PATH', false],
+            ['10.0.0.0/SSRF_PATH', false],
+            ['https://172.16.0.0/SSRF_PATH', false],
+            ['172.16.0.0/SSRF_PATH', false],
+            ['https://192.168.0.0/SSRF_PATH', false],
+            ['192.168.0.0/SSRF_PATH', false],
+            ['http://localhost:8080/admin', false],
+            ['https://100.100.100.200', false], // Alibaba metadata
+            ['100.100.100.200', false], // Alibaba metadata
+            ['64:ff9b::192.0.2.1', false], // embedded IPv4: 192.0.2.1
+            ['64:ff9b::192.0.2.1', false], // embedded IPv4: 192.0.2.1
+            ['https://64:ff9b::198.51.100.1', false], // embedded IPv4: 198.51.100.1
+            ['64:ff9b::198.51.100.1', false], // embedded IPv4: 198.51.100.1
+            ['https://198.18.50.25', false],
+            ['198.18.50.25', false],
+            ['https://198.19.200.10', false],
+            ['198.19.200.10', false],
+            ['https://100.64.0.1', false],
+            ['100.64.0.1', false],
+            ['https://100.100.50.25', false],
+            ['100.100.50.25', false],
+            ['./foo', false],
+            ['/foo', false],
+            ['../foo', false],
+        ];
+    }
+
     /**
      * @dataProvider urlsProvider
      */
@@ -32,4 +68,13 @@ public function testIsHttp(string $url, bool $expected)
         $result = isHttp($url);
         $this->assertSame($expected, $result);
     }
+
+    /**
+     * @dataProvider invalidUrlsProvider
+     */
+    public function testIsValidUrl(string $url, bool $expected)
+    {
+        $result = isValidUrl($url);
+        $this->assertSame($expected, $result);
+    }
 }

tests/MultipleRequestsTest.php

@@ -4,10 +4,12 @@
 namespace Embed\Tests;
 
 use Embed\Embed;
+use InvalidArgumentException;
 use PHPUnit\Framework\TestCase;
 
 class MultipleRequestsTest extends TestCase
 {
+
     public function testParallel()
     {
         $embed = new Embed();
@@ -27,4 +29,13 @@ public function testParallel()
         $this->assertEquals('https://x.com/misteroom', (string) $infos[2]->url);
         $this->assertEquals('en', $infos[2]->language);
     }
+
+    public function testInvalid(): void {
+        $this->expectException(InvalidArgumentException::class);
+
+        (new Embed())->getMulti(
+            'https://twitter.com/misteroom',
+            'https://64:ff9b::198.51.100.1'
+        );
+    }
 }

tests/PagesTest.php

@@ -3,6 +3,9 @@
 
 namespace Embed\Tests;
 
+use Embed\Embed;
+use InvalidArgumentException;
+
 class PagesTest extends PagesTestCase
 {
     /**
@@ -210,4 +213,11 @@ public function testBBCNews()
     {
         $this->assertEmbed('https://www.bbc.co.uk/news/uk-54222286');
     }
+
+    public function testInvalid(): void
+    {
+        $this->expectException(InvalidArgumentException::class);
+        $embed = new Embed();
+        $embed->get('https://64:ff9b::198.51.100.1');
+    }
 }