Add support for private-network upstreams by exempting hosts from the SSRF dial gate

fetch/fetcher.go

@@ -12,6 +12,7 @@ import (
 	"net"
 	"net/http"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/rs/dnscache"
@@ -60,12 +61,13 @@ type FetcherInterface interface {
 
 // Fetcher downloads artifacts from upstream registries.
 type Fetcher struct {
-	client     *http.Client
-	userAgent  string
-	maxRetries int
-	baseDelay  time.Duration
-	authFn     func(url string) (headerName, headerValue string)
-	stop       chan struct{}
+	client       *http.Client
+	userAgent    string
+	maxRetries   int
+	baseDelay    time.Duration
+	authFn       func(url string) (headerName, headerValue string)
+	allowPrivate map[string]bool
+	stop         chan struct{}
 }
 
 // Option configures a Fetcher.
@@ -108,6 +110,40 @@ func WithAuthFunc(fn func(url string) (headerName, headerValue string)) Option {
 	}
 }
 
+// WithAllowPrivateHosts permits the named hosts to resolve to private IP addresses.
+// Loopback and link-local addresses remain blocked.
+func WithAllowPrivateHosts(hosts ...string) Option {
+	return func(f *Fetcher) {
+		if f.allowPrivate == nil {
+			f.allowPrivate = make(map[string]bool, len(hosts))
+		}
+		for _, h := range hosts {
+			if h = normalizeHost(h); h != "" {
+				f.allowPrivate[h] = true
+			}
+		}
+	}
+}
+
+func normalizeHost(host string) string {
+	host = strings.TrimSpace(host)
+	if h, _, err := net.SplitHostPort(host); err == nil {
+		host = h
+	} else if strings.HasPrefix(host, "[") && strings.HasSuffix(host, "]") {
+		host = host[1 : len(host)-1]
+	}
+	return strings.ToLower(strings.TrimSuffix(host, "."))
+}
+
+// gateOptions returns the safehttp options for a dial to host.
+// Zero-value strict gate unless the host was whitelisted via WithAllowPrivateHosts.
+func (f *Fetcher) gateOptions(host string) safehttp.Options {
+	if f.allowPrivate[normalizeHost(host)] {
+		return safehttp.Options{AllowPrivate: true}
+	}
+	return safehttp.Options{}
+}
+
 // NewFetcher creates a new Fetcher with the given options.
 // Callers should invoke Close when done to release the DNS refresh goroutine.
 func NewFetcher(opts ...Option) *Fetcher {
@@ -131,7 +167,8 @@ func NewFetcher(opts ...Option) *Fetcher {
 		KeepAlive: dialKeepAlive,
 	}
 
-	f := &Fetcher{
+	var f *Fetcher
+	f = &Fetcher{
 		client: &http.Client{
 			Timeout: httpClientTimeout,
 			Transport: &http.Transport{
@@ -153,7 +190,7 @@ func NewFetcher(opts ...Option) *Fetcher {
 					var lastErr error
 					for _, ip := range ips {
 						if parsed := net.ParseIP(ip); parsed != nil {
-							if err := safehttp.CheckIP(parsed, safehttp.Options{}); err != nil {
+							if err := safehttp.CheckIP(parsed, f.gateOptions(host)); err != nil {
 								lastErr = err
 								continue
 							}

fetch/fetcher_test.go

@@ -409,3 +409,41 @@ func TestFetcherCloseStopsGoroutine(t *testing.T) {
 		t.Errorf("second Close returned error: %v", err)
 	}
 }
+
+func TestWithAllowPrivateHosts(t *testing.T) {
+	f := NewFetcher(WithAllowPrivateHosts(
+		" Registry.Internal.svc ",
+		"registry-with-port.internal:8080",
+		"registry-with-dot.internal.",
+		"[fd00::1]",
+		"",
+	))
+	defer func() { _ = f.Close() }()
+
+	for _, host := range []string{
+		"registry.internal.svc",
+		"REGISTRY-WITH-PORT.INTERNAL",
+		"registry-with-dot.internal",
+		"fd00::1",
+	} {
+		opts := f.gateOptions(host)
+		if !opts.AllowPrivate {
+			t.Errorf("gateOptions(%q).AllowPrivate = false, want true", host)
+		}
+		if opts.AllowLoopback {
+			t.Errorf("gateOptions(%q).AllowLoopback = true, want false", host)
+		}
+	}
+
+	opts := f.gateOptions("other.example.com")
+	if opts.AllowPrivate || opts.AllowLoopback {
+		t.Errorf("non-whitelisted host exempted: %+v", opts)
+	}
+
+	strict := NewFetcher()
+	defer func() { _ = strict.Close() }()
+	opts = strict.gateOptions("registry.internal.svc")
+	if opts.AllowPrivate || opts.AllowLoopback {
+		t.Errorf("default fetcher not strict: %+v", opts)
+	}
+}

safehttp/safehttp.go

@@ -41,13 +41,15 @@ const (
 )
 
 // Options configures a safehttp client. The zero value gives the
-// production-strict gate; tests can opt parts of it off explicitly.
+// production-strict gate; tests and explicit operator allowlists can
+// opt parts of it off.
 type Options struct {
 	// AllowLoopback disables the loopback (127.0.0.0/8, ::1) check.
-	// Test-only; never set in production paths.
+	// Only set for tests or explicit operator allowlists.
 	AllowLoopback bool
 
-	// AllowPrivate disables the RFC1918 / ULA / CGNAT checks. Test-only.
+	// AllowPrivate disables the RFC1918 / ULA / CGNAT checks.
+	// Only set for tests or explicit operator allowlists.
 	AllowPrivate bool
 }