From 43f9e961a6e4d14776ae880993dbee04df120d3d Mon Sep 17 00:00:00 2001 From: Haiyan Meng Date: Thu, 13 Aug 2026 10:40:31 -0400 Subject: [PATCH 1/2] egress: MITM tunneled TLS with per-SNI certificate minting The egress gateway terminated the actor's CONNECT tunnel and forwarded the bytes inside it opaquely, so nothing could be said about the TLS session an actor established through it. This intercepts that session: Envoy terminates the tunnelled TLS with a leaf minted for the SNI the client asked for, which puts the plaintext request on a filter chain where policy can be applied later. The data path, all inside the atenet-egress pod: atunnel client -> gateway front door (mTLS, socket listener) -> CONNECT to an IP:port -> mitm_internal cluster (envoy_internal_address) -> mitm_listener (internal, no socket anywhere) -> tls_inspector reads the SNI -> DownstreamTlsContext, on_demand_secret selector -> DELTA_GRPC SDS over a unix socket to sdsmint -> leaf minted for that SNI, handshake resumes -> dynamic forward proxy to the real destination Supporting changes: internal/localca: CA.SigningKey narrows from crypto.PrivateKey to crypto.Signer, so a key that parses but cannot sign (X25519 out of PKCS#8) is refused at load rather than at the first handshake. Adds CA.Validate, and GenerateCA/GenerateOptions with a key type, common name and lifetime. GenerateED25519CA stays as a wrapper -- every existing caller wants exactly what it produced. kubectl-ate admin make-ca-pool: --key-type and --common-name. hack/install-ate.sh: creates the egress-mitm-ca-pool secret, ecdsa-p256 rather than the ed25519 default. These leaves are validated by arbitrary clients inside actor sandboxes, where Ed25519 support cannot be assumed. internal/atunnel: ErrGatewayHandshake and ConnectRejectedError replace formatted strings, so a caller can tell a front-door TLS rejection from an authorization denial without matching on message text. A gateway that resets instead of alerting is folded into the former, since which of the two happens is a race. e2e: internal/e2e/suites/sdsmint covers the tunnelled handshake, the leaf Envoy serves, and actor identity across the MITM, against the new egressprobe fixture. --- cmd/atenet/internal/root.go | 2 + .../internal/sdsmint/certauth/certauth.go | 181 ++++++ .../sdsmint/certauth/certauth_test.go | 470 ++++++++++++++++ cmd/atenet/internal/sdsmint/cmd.go | 73 +++ cmd/atenet/internal/sdsmint/cmd_test.go | 69 +++ cmd/atenet/internal/sdsmint/deltastream.go | 246 ++++++++ .../internal/sdsmint/deltastream_test.go | 530 ++++++++++++++++++ cmd/atenet/internal/sdsmint/doc.go | 19 + cmd/atenet/internal/sdsmint/listen.go | 50 ++ cmd/atenet/internal/sdsmint/minter.go | 145 +++++ cmd/atenet/internal/sdsmint/minter_test.go | 194 +++++++ cmd/atenet/internal/sdsmint/run.go | 141 +++++ cmd/atenet/internal/sdsmint/server.go | 123 ++++ .../internal/cmd/admin_make_ca_pool.go | 12 +- hack/install-ate.sh | 23 +- internal/atunnel/client.go | 64 ++- internal/atunnel/client_test.go | 159 ++++++ .../egressprobe/egressprobe.yaml.tmpl | 113 ++++ internal/e2e/fixtures/egressprobe/main.go | 232 ++++++++ .../e2e/suites/sdsmint/actoridentity_test.go | 275 +++++++++ internal/e2e/suites/sdsmint/sdsmint_test.go | 515 +++++++++++++++++ internal/e2e/suites/sdsmint/testmain_test.go | 41 ++ internal/e2e/testmain.go | 24 + internal/localca/localca.go | 144 ++++- internal/localca/localca_test.go | 185 ++++++ manifests/ate-install/atenet-egress.yaml | 416 +++++++++++++- 26 files changed, 4424 insertions(+), 22 deletions(-) create mode 100644 cmd/atenet/internal/sdsmint/certauth/certauth.go create mode 100644 cmd/atenet/internal/sdsmint/certauth/certauth_test.go create mode 100644 cmd/atenet/internal/sdsmint/cmd.go create mode 100644 cmd/atenet/internal/sdsmint/cmd_test.go create mode 100644 cmd/atenet/internal/sdsmint/deltastream.go create mode 100644 cmd/atenet/internal/sdsmint/deltastream_test.go create mode 100644 cmd/atenet/internal/sdsmint/doc.go create mode 100644 cmd/atenet/internal/sdsmint/listen.go create mode 100644 cmd/atenet/internal/sdsmint/minter.go create mode 100644 cmd/atenet/internal/sdsmint/minter_test.go create mode 100644 cmd/atenet/internal/sdsmint/run.go create mode 100644 cmd/atenet/internal/sdsmint/server.go create mode 100644 internal/e2e/fixtures/egressprobe/egressprobe.yaml.tmpl create mode 100644 internal/e2e/fixtures/egressprobe/main.go create mode 100644 internal/e2e/suites/sdsmint/actoridentity_test.go create mode 100644 internal/e2e/suites/sdsmint/sdsmint_test.go create mode 100644 internal/e2e/suites/sdsmint/testmain_test.go diff --git a/cmd/atenet/internal/root.go b/cmd/atenet/internal/root.go index 53ae85d13..1ab52398a 100644 --- a/cmd/atenet/internal/root.go +++ b/cmd/atenet/internal/root.go @@ -19,6 +19,7 @@ import ( "os" "github.com/agent-substrate/substrate/cmd/atenet/internal/router" + "github.com/agent-substrate/substrate/cmd/atenet/internal/sdsmint" "github.com/agent-substrate/substrate/internal/version" "github.com/spf13/cobra" ) @@ -40,4 +41,5 @@ func Execute() { func init() { rootCmd.AddCommand(router.NewRouterCmd()) rootCmd.AddCommand(NewDnsCmd()) + rootCmd.AddCommand(sdsmint.NewSdsmintCmd()) } diff --git a/cmd/atenet/internal/sdsmint/certauth/certauth.go b/cmd/atenet/internal/sdsmint/certauth/certauth.go new file mode 100644 index 000000000..0c02a94ee --- /dev/null +++ b/cmd/atenet/internal/sdsmint/certauth/certauth.go @@ -0,0 +1,181 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package certauth holds everything in sdsmint that touches the MITM signing +// key: the key itself, the leaf keypair certificates are bound to, and the one +// function that turns a hostname into a certificate for it. +package certauth + +import ( + "crypto" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "encoding/pem" + "errors" + "fmt" + "net" + "time" + + "github.com/agent-substrate/substrate/internal/localca" +) + +// Signer issues leaves for arbitrary hostnames from the MITM CA. +type Signer struct { + pool *localca.Pool + active *localca.CA + + // key is the keypair every leaf this Signer issues is bound to. + key crypto.Signer + keyPEM []byte +} + +// New builds a Signer over pool, signing with the CA named by id. An empty id +// takes the first entry. +// +// The whole pool is retained, not just the selected entry: see the Signer +// fields. +func New(pool *localca.Pool, id string) (*Signer, error) { + active, err := selectCA(pool, id) + if err != nil { + return nil, err + } + if err := active.Validate(); err != nil { + return nil, fmt.Errorf("ca pool: CA %q: %w", active.ID, err) + } + // Generated at startup rather than on first use, so that a mint never pays + // for a keygen and one that cannot succeed fails before the server binds. + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return nil, fmt.Errorf("generating leaf key: %w", err) + } + der, err := x509.MarshalPKCS8PrivateKey(key) + if err != nil { + return nil, fmt.Errorf("marshalling leaf key: %w", err) + } + return &Signer{ + pool: pool, + active: active, + key: key, + keyPEM: pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: der}), + }, nil +} + +// selectCA picks the entry to sign with. An empty id takes the first. +// TODO(haiyanmeng): move this function to internal/localca. +func selectCA(pool *localca.Pool, id string) (*localca.CA, error) { + if pool == nil || len(pool.CAs) == 0 { + return nil, errors.New("ca pool: empty") + } + if id == "" { + return pool.CAs[0], nil + } + ids := make([]string, 0, len(pool.CAs)) + for _, candidate := range pool.CAs { + ids = append(ids, candidate.ID) + if candidate.ID == id { + return candidate, nil + } + } + return nil, fmt.Errorf("ca pool: no CA with ID %q (have %q)", id, ids) +} + +// Issuer returns the certificate this Signer's key belongs to, the one it puts +// directly above each leaf in the chain. +func (s *Signer) Issuer() *x509.Certificate { + return s.active.RootCertificate +} + +// Anchors returns every root in the pool: the full set a client has to trust +// for this Signer's leaves to verify across a rotation, not merely the one +// currently signing. This is what belongs in a trust store or a published +// bundle. +func (s *Signer) Anchors() []*x509.Certificate { + roots := make([]*x509.Certificate, 0, len(s.pool.CAs)) + for _, ca := range s.pool.CAs { + roots = append(roots, ca.RootCertificate) + } + return roots +} + +// MintedCert is a freshly issued leaf plus its private key, in the PEM form +// Envoy's Secret proto expects. +type MintedCert struct { + CertChainPEM []byte // leaf, then the root, PEM + PrivateKeyPEM []byte // leaf private key, PEM + NotAfter time.Time + Serial string // hex, for the issuance audit log +} + +// Sign issues a leaf certificate for host, signed by the CA's root key and +// bound to the Signer's leaf keypair. +func (s *Signer) Sign(host string, ttl time.Duration) (*MintedCert, error) { + if host == "" { + return nil, errors.New("empty host") + } + + ca := s.active + now := time.Now() + + notAfter := now.Add(ttl) + if notAfter.After(ca.RootCertificate.NotAfter) { + notAfter = ca.RootCertificate.NotAfter + } + tmpl := &x509.Certificate{ + NotBefore: now.Add(-5 * time.Minute), + NotAfter: notAfter, + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + IsCA: false, + } + // A literal IP in the SNI position has to land in IPAddresses, not + // DNSNames, or clients will reject the leaf. + // Envoy will hand us whatever was in the SNI, and although SNI is + // not supposed to carry IP literals, some clients send them anyway. + if ip := net.ParseIP(host); ip != nil { + tmpl.IPAddresses = append(tmpl.IPAddresses, ip) + } else { + tmpl.DNSNames = []string{host} + } + + leafDER, err := x509.CreateCertificate(rand.Reader, tmpl, ca.RootCertificate, s.key.Public(), ca.SigningKey) + if err != nil { + return nil, fmt.Errorf("signing leaf for %q: %w", host, err) + } + leaf, err := x509.ParseCertificate(leafDER) + if err != nil { + return nil, fmt.Errorf("parsing the leaf just signed for %q: %w", host, err) + } + + // Leaf first, then its issuer, then whatever climbs from the issuer toward + // the certificate a client is actually configured to trust. RootCertificate + // is the issuer and not necessarily the anchor: localca.Validate pins the + // signing key to it, and IntermediateCertificates is what sits above. + chain := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: leafDER}) + chain = append(chain, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: ca.RootCertificate.Raw})...) + for _, intermediate := range ca.IntermediateCertificates { + chain = append(chain, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: intermediate.Raw})...) + } + + return &MintedCert{ + CertChainPEM: chain, + // Shared with every other leaf this Signer issues, rendered once in + // New. Read-only, like the rest of MintedCert. + PrivateKeyPEM: s.keyPEM, + NotAfter: notAfter, + Serial: leaf.SerialNumber.Text(16), + }, nil +} diff --git a/cmd/atenet/internal/sdsmint/certauth/certauth_test.go b/cmd/atenet/internal/sdsmint/certauth/certauth_test.go new file mode 100644 index 000000000..39363d551 --- /dev/null +++ b/cmd/atenet/internal/sdsmint/certauth/certauth_test.go @@ -0,0 +1,470 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package certauth + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/asn1" + "encoding/pem" + "math/big" + "testing" + "testing/synctest" + "time" + + "github.com/agent-substrate/substrate/internal/localca" +) + +func testCA(t *testing.T) *localca.CA { + t.Helper() + return testRoot(t, "sdsmint test CA", time.Hour) +} + +// testRoot builds a pool entry in the shape sdsmint expects to be handed one. +// P-256 rather than substrate's usual Ed25519: the leaves signed under it are +// validated by whatever HTTP client an actor happens to run, and Ed25519 in a +// chain needs OpenSSL 1.1.1+. +func testRoot(t *testing.T, commonName string, lifetime time.Duration) *localca.CA { + t.Helper() + ca, err := localca.GenerateCA(localca.GenerateOptions{ + ID: "mitm", + CommonName: commonName, + KeyType: localca.KeyTypeECDSAP256, + Lifetime: lifetime, + }) + if err != nil { + t.Fatalf("generating test CA %q: %v", commonName, err) + } + return ca +} + +// inPool wraps entries the way a mounted pool Secret presents them. +func inPool(t *testing.T, id string, entries ...*localca.CA) *Signer { + t.Helper() + signer, err := New(&localca.Pool{CAs: entries}, id) + if err != nil { + t.Fatalf("New(%q): %v", id, err) + } + return signer +} + +// testSigner builds a Signer over a single-entry pool. +func testSigner(t *testing.T, ca *localca.CA) *Signer { + t.Helper() + signer, err := New(&localca.Pool{CAs: []*localca.CA{ca}}, "") + if err != nil { + t.Fatalf("New: %v", err) + } + return signer +} + +// oidSubjectAltName is id-ce-subjectAltName, RFC 5280 section 4.2.1.6. +var oidSubjectAltName = asn1.ObjectIdentifier{2, 5, 29, 17} + +func hasCriticalSAN(t *testing.T, cert *x509.Certificate) bool { + t.Helper() + for _, ext := range cert.Extensions { + if ext.Id.Equal(oidSubjectAltName) { + return ext.Critical + } + } + return false +} + +// parseChain splits a PEM chain into leaf and root. +func parseChain(t *testing.T, chainPEM []byte) []*x509.Certificate { + t.Helper() + var certs []*x509.Certificate + rest := chainPEM + for { + var block *pem.Block + block, rest = pem.Decode(rest) + if block == nil { + break + } + c, err := x509.ParseCertificate(block.Bytes) + if err != nil { + t.Fatalf("parsing chain: %v", err) + } + certs = append(certs, c) + } + return certs +} + +func TestSignProducesUsableLeaf(t *testing.T) { + ca := testCA(t) + + minted, err := testSigner(t, ca).Sign("foo.example", 5*time.Minute) + if err != nil { + t.Fatalf("Sign: %v", err) + } + + chain := parseChain(t, minted.CertChainPEM) + if len(chain) != 2 { + t.Fatalf("chain length = %d, want 2 (leaf + CA)", len(chain)) + } + leaf, root := chain[0], chain[1] + + if got := leaf.DNSNames; len(got) != 1 || got[0] != "foo.example" { + t.Errorf("leaf DNSNames = %v, want [foo.example]", got) + } + // No subject at all: the name lives in the SAN, which is the only place + // clients read it from. An empty subject obliges the SAN to be critical. + if got := leaf.Subject.String(); got != "" { + t.Errorf("leaf subject = %q, want empty", got) + } + if !hasCriticalSAN(t, leaf) { + t.Error("leaf has an empty subject but a non-critical SAN, which RFC 5280 forbids") + } + if leaf.SerialNumber == nil || leaf.SerialNumber.Sign() <= 0 { + t.Errorf("leaf serial = %v, want a positive number", leaf.SerialNumber) + } + if leaf.IsCA { + t.Error("leaf is marked as a CA") + } + if leaf.KeyUsage != x509.KeyUsageDigitalSignature { + t.Errorf("leaf KeyUsage = %v, want DigitalSignature only", leaf.KeyUsage) + } + if len(leaf.ExtKeyUsage) != 1 || leaf.ExtKeyUsage[0] != x509.ExtKeyUsageServerAuth { + t.Errorf("leaf EKU = %v, want [ServerAuth]", leaf.ExtKeyUsage) + } + if !root.Equal(ca.RootCertificate) { + t.Error("chain does not end in the CA certificate") + } + + // The whole point is that a normal TLS client accepts this, so verify the + // way one would. + pool := x509.NewCertPool() + pool.AddCert(ca.RootCertificate) + if _, err := leaf.Verify(x509.VerifyOptions{ + DNSName: "foo.example", + Roots: pool, + KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + }); err != nil { + t.Errorf("leaf does not verify against the CA: %v", err) + } + + // And the key must actually match the certificate. + if _, err := tls.X509KeyPair(minted.CertChainPEM, minted.PrivateKeyPEM); err != nil { + t.Errorf("minted chain/key is not a usable TLS keypair: %v", err) + } +} + +func TestSignIsUniquePerCall(t *testing.T) { + signer := testSigner(t, testCA(t)) + + a, err := signer.Sign("a.example", time.Minute) + if err != nil { + t.Fatalf("Sign a: %v", err) + } + b, err := signer.Sign("a.example", time.Minute) + if err != nil { + t.Fatalf("Sign a again: %v", err) + } + + // The certificate is new every time, which is what the SDS layer reads as + // a version and what the audit log names the issuance by. + if a.Serial == b.Serial { + t.Error("two mints for the same host reused a serial number") + } + // The keypair, by contrast, is shared on purpose. Asserted + // rather than left implicit, because the saving disappears silently the + // moment a mint starts generating its own again. + if string(a.PrivateKeyPEM) != string(b.PrivateKeyPEM) { + t.Error("two mints generated separate private keys; leaves are meant to share one") + } +} + +func TestSignHonoursTTL(t *testing.T) { + // On the fake clock nothing advances between the read and the mint, so the + // deadline is exact rather than a tolerance around real elapsed time. + synctest.Test(t, func(t *testing.T) { + ca := testCA(t) + before := time.Now() + + minted, err := testSigner(t, ca).Sign("ttl.example", 90*time.Second) + if err != nil { + t.Fatalf("Sign: %v", err) + } + + if want := before.Add(90 * time.Second); !minted.NotAfter.Equal(want) { + t.Errorf("NotAfter = %v, want %v", minted.NotAfter, want) + } + }) +} + +func TestSignIPLiteralGoesInSANIPAddresses(t *testing.T) { + ca := testCA(t) + + minted, err := testSigner(t, ca).Sign("10.1.2.3", time.Minute) + if err != nil { + t.Fatalf("Sign: %v", err) + } + leaf := parseChain(t, minted.CertChainPEM)[0] + + if len(leaf.IPAddresses) != 1 || leaf.IPAddresses[0].String() != "10.1.2.3" { + t.Errorf("leaf IPAddresses = %v, want [10.1.2.3]", leaf.IPAddresses) + } + if len(leaf.DNSNames) != 0 { + t.Errorf("leaf DNSNames = %v, want empty for an IP literal", leaf.DNSNames) + } +} + +func TestSignRejectsEmptyHost(t *testing.T) { + if _, err := testSigner(t, testCA(t)).Sign("", time.Minute); err == nil { + t.Fatal("Sign(\"\") succeeded, want an error") + } +} + +func TestNewRejectsANonCACertificate(t *testing.T) { + ca := testCA(t) + // A leaf is not a CA; a pool carrying one must be refused rather than + // silently produce a signer that emits certificates nothing will chain. + minted, err := testSigner(t, ca).Sign("leaf.example", time.Minute) + if err != nil { + t.Fatalf("Sign: %v", err) + } + leaf := parseChain(t, minted.CertChainPEM)[0] + + if _, err := New(&localca.Pool{CAs: []*localca.CA{ + {ID: "mitm", RootCertificate: leaf, SigningKey: ca.SigningKey}, + }}, ""); err == nil { + t.Fatal("New accepted a non-CA certificate") + } +} + +func TestNewAcceptsAnUnconstrainedCA(t *testing.T) { + signer := inPool(t, "", testRoot(t, "wide open", time.Hour)) + + if got := signer.Issuer().PermittedDNSDomains; len(got) != 0 { + t.Errorf("PermittedDNSDomains = %v, want none", got) + } + + // And it signs. Refusing at load and then failing at the first handshake + // would be the same outage with a worse error message. + if _, err := signer.Sign("anything.example", time.Minute); err != nil { + t.Errorf("Sign under an unconstrained root: %v", err) + } +} + +func TestNewRejectsAMismatchedKey(t *testing.T) { + a := testRoot(t, "a", time.Hour) + b := testRoot(t, "b", time.Hour) + + // Signing with the wrong key produces a chain nothing can verify, and the + // failure otherwise surfaces at the first handshake rather than at load. + // localca.CA.Validate is what catches it; this pins that New asks. + mismatched := &localca.CA{ID: "mismatched", RootCertificate: a.RootCertificate, SigningKey: b.SigningKey} + if _, err := New(&localca.Pool{CAs: []*localca.CA{mismatched}}, ""); err == nil { + t.Fatal("New accepted a key that does not match the certificate") + } +} + +// A pool entry may carry certificates that climb from its signing certificate +// up to whatever a client is configured to trust. Those have to travel with the +// leaf: a client holding only the top of that chain cannot build a path to a +// leaf issued two levels down, so omitting them fails the handshake. +func TestSignEmitsIntermediatesInTheChain(t *testing.T) { + anchor := testRoot(t, "anchor", 24*time.Hour) + + // An issuing CA under the anchor, with its own key. This is the shape + // RootCertificate names: the certificate whose key signs leaves, which here + // is not the trust anchor. + issuerKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("generating issuer key: %v", err) + } + issuerTmpl := &x509.Certificate{ + SerialNumber: big.NewInt(2), + Subject: pkix.Name{CommonName: "issuing CA"}, + NotBefore: time.Now().Add(-time.Minute), + NotAfter: time.Now().Add(12 * time.Hour), + IsCA: true, + BasicConstraintsValid: true, + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature, + } + issuerDER, err := x509.CreateCertificate(rand.Reader, issuerTmpl, anchor.RootCertificate, issuerKey.Public(), anchor.SigningKey) + if err != nil { + t.Fatalf("signing issuing CA: %v", err) + } + issuer, err := x509.ParseCertificate(issuerDER) + if err != nil { + t.Fatalf("parsing issuing CA: %v", err) + } + + signer := testSigner(t, &localca.CA{ + ID: "delegated", + RootCertificate: issuer, + SigningKey: issuerKey, + IntermediateCertificates: []*x509.Certificate{anchor.RootCertificate}, + }) + + minted, err := signer.Sign("host.example", time.Hour) + if err != nil { + t.Fatalf("Sign: %v", err) + } + + chain := parseChain(t, minted.CertChainPEM) + if len(chain) != 3 { + t.Fatalf("chain holds %d certificates, want leaf, issuer and anchor", len(chain)) + } + if got := chain[1].Subject.CommonName; got != "issuing CA" { + t.Errorf("chain[1] CN = %q, want the issuing CA", got) + } + if got := chain[2].Subject.CommonName; got != "anchor" { + t.Errorf("chain[2] CN = %q, want the anchor", got) + } + + // The point of carrying them: a client that trusts only the anchor can + // still build a path to the leaf out of what the handshake delivered. + roots := x509.NewCertPool() + roots.AddCert(anchor.RootCertificate) + intermediates := x509.NewCertPool() + for _, c := range chain[1:] { + intermediates.AddCert(c) + } + if _, err := chain[0].Verify(x509.VerifyOptions{ + DNSName: "host.example", + Roots: roots, + Intermediates: intermediates, + KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + }); err != nil { + t.Errorf("verifying the leaf against the anchor alone: %v", err) + } +} + +func TestNewRoundTrip(t *testing.T) { + entry := testRoot(t, "pooled CA", time.Hour) + + // Through the same serialization podcertcontroller's CAs use. + poolBytes, err := localca.Marshal(&localca.Pool{CAs: []*localca.CA{entry}}) + if err != nil { + t.Fatalf("Marshal: %v", err) + } + pool, err := localca.Unmarshal(poolBytes) + if err != nil { + t.Fatalf("Unmarshal: %v", err) + } + + signer, err := New(pool, "") + if err != nil { + t.Fatalf("New: %v", err) + } + minted, err := signer.Sign("host.example", time.Minute) + if err != nil { + t.Fatalf("Sign: %v", err) + } + + roots := x509.NewCertPool() + roots.AddCert(signer.Issuer()) + if _, err := parseChain(t, minted.CertChainPEM)[0].Verify(x509.VerifyOptions{ + DNSName: "host.example", Roots: roots, + KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + }); err != nil { + t.Errorf("leaf from a pooled CA does not verify: %v", err) + } +} + +func TestNewSelectsByID(t *testing.T) { + var entries []*localca.CA + for _, id := range []string{"first", "second"} { + entry := testRoot(t, id, time.Hour) + entry.ID = id + entries = append(entries, entry) + } + pool := &localca.Pool{CAs: entries} + + signer, err := New(pool, "second") + if err != nil { + t.Fatalf("New: %v", err) + } + if got := signer.Issuer().Subject.CommonName; got != "second" { + t.Errorf("selected CA CN = %q, want %q", got, "second") + } + + // An empty ID takes the first, which is what a single-CA pool relies on. + signer, err = New(pool, "") + if err != nil { + t.Fatalf("New(''): %v", err) + } + if got := signer.Issuer().Subject.CommonName; got != "first" { + t.Errorf("default CA CN = %q, want %q", got, "first") + } + + // A typo in --ca-id must not silently fall back to some other CA. + if _, err := New(pool, "third"); err == nil { + t.Fatal("New accepted an unknown CA ID") + } +} + +// TestAnchorsCoversTheWholePool is the distinction the two accessors exist to +// make. During a rotation the pool holds the outgoing and incoming CA at once +// while only one of them signs, so a trust store built from Issuer would reject +// every leaf minted the moment --ca-id moved. +func TestAnchorsCoversTheWholePool(t *testing.T) { + var entries []*localca.CA + for _, id := range []string{"outgoing", "incoming"} { + entry := testRoot(t, id, time.Hour) + entry.ID = id + entries = append(entries, entry) + } + signer := inPool(t, "outgoing", entries...) + + anchors := signer.Anchors() + if len(anchors) != 2 { + t.Fatalf("Anchors returned %d certificates, want one per CA in the pool", len(anchors)) + } + for i, entry := range entries { + if !anchors[i].Equal(entry.RootCertificate) { + t.Errorf("anchor %d is not the root of CA %q", i, entry.ID) + } + } + + // And the CA not signing is still in there, which is the whole point. + if anchors[1].Equal(signer.Issuer()) { + t.Error("the incoming CA was reported as the issuer; only the outgoing one signs") + } +} + +func TestNewRejectsAnEmptyPool(t *testing.T) { + if _, err := New(&localca.Pool{}, ""); err == nil { + t.Fatal("New accepted an empty pool") + } + if _, err := New(nil, ""); err == nil { + t.Fatal("New accepted a nil pool") + } +} + +func TestSignClampsLeafLifetimeToTheRoot(t *testing.T) { + // A leaf outliving its issuer is accepted at handshake time and rejected + // later, with an error that points at the leaf rather than at the CA. + ca := testRoot(t, "short-lived CA", 2*time.Minute) + rootNotAfter := ca.RootCertificate.NotAfter + + minted, err := testSigner(t, ca).Sign("long.example", time.Hour) + if err != nil { + t.Fatalf("Sign: %v", err) + } + if minted.NotAfter.After(rootNotAfter) { + t.Errorf("leaf NotAfter = %v, outlives its issuer at %v", minted.NotAfter, rootNotAfter) + } + if leaf := parseChain(t, minted.CertChainPEM)[0]; leaf.NotAfter.After(rootNotAfter) { + t.Errorf("encoded leaf NotAfter = %v, outlives its issuer at %v", leaf.NotAfter, rootNotAfter) + } +} diff --git a/cmd/atenet/internal/sdsmint/cmd.go b/cmd/atenet/internal/sdsmint/cmd.go new file mode 100644 index 000000000..84b9591d6 --- /dev/null +++ b/cmd/atenet/internal/sdsmint/cmd.go @@ -0,0 +1,73 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sdsmint + +import ( + "fmt" + "time" + + "github.com/spf13/cobra" +) + +type config struct { + UDSPath string + CAPoolPath string + CAID string + LeafCertTTL time.Duration + LogLevel string +} + +func NewSdsmintCmd() *cobra.Command { + var cfg config + + cmd := &cobra.Command{ + Use: "sdsmint", + Short: "Minting SDS server that issues a TLS leaf for the SNI Envoy asks for", + RunE: func(cmd *cobra.Command, _ []string) error { + return run(cmd.Context(), cfg) + }, + } + + cmd.Flags().StringVar(&cfg.UDSPath, "uds-path", "", "unix socket to listen on; required, and the only transport offered, because leaf private keys transit this channel") + cmd.Flags().StringVar(&cfg.CAPoolPath, "ca-pool-path", "", "path to a localca pool JSON holding the MITM CA, the format substrate mounts its other CAs in") + cmd.Flags().StringVar(&cfg.CAID, "ca-id", "", "which CA in the pool to sign with; empty takes the first") + cmd.Flags().DurationVar(&cfg.LeafCertTTL, "leaf-cert-ttl", defaultTTL, "leaf certificate lifetime; the xDS resource TTL is derived from it at half its length, so an actively used name is re-minted about twice per lifetime and an idle one is dropped and not minted again") + cmd.Flags().StringVar(&cfg.LogLevel, "log-level", "info", "one of debug, info, warn, error") + + return cmd +} + +func (c config) validateTTL() error { + ttl := c.LeafCertTTL + if ttl <= 0 { + return fmt.Errorf("--leaf-cert-ttl must be positive, got %s", ttl) + } + + // The band outside which --leaf-cert-ttl is refused. Both edges are cases + // where the flag still starts a server but stops describing what it does, + // which is worse than not starting: the operator has no signal that the + // number they set is not the number in effect. + const ( + minSensibleTTL = time.Minute + maxSensibleTTL = 24 * time.Hour + ) + switch { + case ttl < minSensibleTTL: + return fmt.Errorf("--leaf-cert-ttl %s is below %s; leaves are back-dated 5m for clock skew, so most of each certificate's validity would already be in the past and --leaf-cert-ttl would not describe how long clients accept it", ttl, minSensibleTTL) + case ttl > maxSensibleTTL: + return fmt.Errorf("--leaf-cert-ttl %s is above %s; a leaf is served until the resource TTL derived from this drops it, so a name in steady use would carry the same certificate for half of that, which is not what short-lived MITM leaves are for", ttl, maxSensibleTTL) + } + return nil +} diff --git a/cmd/atenet/internal/sdsmint/cmd_test.go b/cmd/atenet/internal/sdsmint/cmd_test.go new file mode 100644 index 000000000..df8ddfebe --- /dev/null +++ b/cmd/atenet/internal/sdsmint/cmd_test.go @@ -0,0 +1,69 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sdsmint + +import ( + "testing" + "time" +) + +func TestValidateTTL(t *testing.T) { + for _, tc := range []struct { + name string + ttl time.Duration + wantErr bool + }{ + {name: "default", ttl: defaultTTL}, + {name: "deployed value", ttl: 15 * time.Minute}, + {name: "the load-test value", ttl: 5 * time.Minute}, + // The edges of validateTTL's accepted band, spelled out because the + // bounds are local to it. Both are inclusive. + {name: "at the band floor", ttl: time.Minute}, + {name: "at the band ceiling", ttl: 24 * time.Hour}, + + // The whole point of the exercise: --leaf-cert-ttl=0 used to start a server + // that logged 0 and issued defaultTTL leaves. + {name: "zero", ttl: 0, wantErr: true}, + {name: "negative", ttl: -time.Minute, wantErr: true}, + + // Outside the band. These used to start the server with a warning; a + // TTL that no longer means what it says is now refused outright. + {name: "short", ttl: 30 * time.Second, wantErr: true}, + {name: "long", ttl: 48 * time.Hour, wantErr: true}, + } { + t.Run(tc.name, func(t *testing.T) { + err := config{LeafCertTTL: tc.ttl}.validateTTL() + if gotErr := err != nil; gotErr != tc.wantErr { + t.Errorf("validateTTL(ttl=%s) = %v; want an error = %v", tc.ttl, err, tc.wantErr) + } + }) + } +} + +// TestDefaultTTLIsTheFlagDefault guards the trap this validation was added +// for: a fallback and a flag default that disagree, so the lifetime depends on +// which path you came in through. +func TestDefaultTTLIsTheFlagDefault(t *testing.T) { + flag := NewSdsmintCmd().Flags().Lookup("leaf-cert-ttl") + if flag == nil { + t.Fatal("no --leaf-cert-ttl flag") + } + if got, want := flag.DefValue, defaultTTL.String(); got != want { + t.Errorf("--leaf-cert-ttl default = %q; want %q, the same constant newMinter and newServer fall back to", got, want) + } + if err := (config{LeafCertTTL: defaultTTL}).validateTTL(); err != nil { + t.Errorf("validateTTL(defaultTTL) = %v; the default has to be a value run will start with", err) + } +} diff --git a/cmd/atenet/internal/sdsmint/deltastream.go b/cmd/atenet/internal/sdsmint/deltastream.go new file mode 100644 index 000000000..caeddfbfd --- /dev/null +++ b/cmd/atenet/internal/sdsmint/deltastream.go @@ -0,0 +1,246 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Delta SDS: the stateful, per-connection half of the server. +// - https://www.envoyproxy.io/docs/envoy/latest/api-docs/xds_protocol#xds-protocol-delta +// - https://www.envoyproxy.io/docs/envoy/latest/configuration/security/secret +package sdsmint + +import ( + "context" + "errors" + "fmt" + "io" + "log/slog" + + discovery "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + secretservice "github.com/envoyproxy/go-control-plane/envoy/service/secret/v3" + "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/durationpb" + + "github.com/agent-substrate/substrate/cmd/atenet/internal/sdsmint/certauth" +) + +// DeltaSecrets is what DELTA_GRPC drives. It is a long-lived loop that mints +// incrementally rather than serving a fixed snapshot. +func (s *server) DeltaSecrets(stream secretservice.SecretDiscoveryService_DeltaSecretsServer) error { + ctx := stream.Context() + + st := &deltaStream{ + srv: s, + stream: stream, + sendCh: make(chan *discovery.DeltaDiscoveryResponse, 8), + sendErr: make(chan error, 1), + sendDone: make(chan struct{}), + } + + go st.sendLoop(ctx) + + recvCh := make(chan *discovery.DeltaDiscoveryRequest) + recvErrCh := make(chan error, 1) + go func() { + for { + req, err := stream.Recv() + if err != nil { + recvErrCh <- err + return + } + select { + case recvCh <- req: + case <-ctx.Done(): + return + } + } + }() + + // Wait for send thread to finish before exiting. + defer func() { + close(st.sendCh) + <-st.sendDone + }() + + for { + select { + case <-ctx.Done(): + return nil + case err := <-st.sendErr: + return err + case err := <-recvErrCh: + if errors.Is(err, io.EOF) { + return nil + } + return err + case req := <-recvCh: + if err := st.handle(ctx, req); err != nil { + return err + } + } + } +} + +// deltaStream is the per-connection plumbing for a DeltaSecrets stream. +// +// Despite delta xDS being the stateful variant, there is no subscription set +// here. This server tracked one until the resource TTL made it pointless: +// refresh is Envoy's timer now, so nothing on this side ever needs to know what +// a stream is holding. What is left is a send queue, and no shared mutable +// state at all -- handle runs on one goroutine and sendLoop owns the stream. +type deltaStream struct { + srv *server + stream secretservice.SecretDiscoveryService_DeltaSecretsServer + + // sendCh queues responses for sendLoop. sendErr carries the first send + // failure back to DeltaSecrets, and sendDone closes once the loop has + // stopped touching the stream. + sendCh chan *discovery.DeltaDiscoveryResponse + sendErr chan error + sendDone chan struct{} +} + +// handle applies one request from the client. +func (d *deltaStream) handle(ctx context.Context, req *discovery.DeltaDiscoveryRequest) error { + if url := req.GetTypeUrl(); url != "" && url != secretTypeURL { + return fmt.Errorf("unexpected type_url %q on the SDS stream", url) + } + + // A request carrying error_detail is a NACK of whatever we last sent, and + // only that: it brings no subscription changes to apply. + if req.GetErrorDetail() != nil { + d.logNACK(ctx, req) + return nil + } + + // initial_resource_versions and resource_names_unsubscribe are read off the + // wire and dropped; see the package comment for why neither needs acting on. + return d.handleSubscribe(ctx, req.GetResourceNamesSubscribe()) +} + +// logNACK records that Envoy rejected the last response. Nothing is resent: the +// server has no second thing to offer for the name, and a retry loop against a +// client that is rejecting on principle is worse than the failure. +func (d *deltaStream) logNACK(ctx context.Context, req *discovery.DeltaDiscoveryRequest) { + ed := req.GetErrorDetail() + d.srv.log.ErrorContext(ctx, "envoy NACKed an SDS response", + slog.String("message", ed.GetMessage()), + slog.Int("code", int(ed.GetCode())), + slog.String("nonce", req.GetResponseNonce()), + ) +} + +// handleSubscribe mints a leaf for every name the client asked for and sends +// the batch. A name this stream already holds is minted again rather than +// skipped. Envoy re-subscribes in two cases and both want a certificate: after +// a resource TTL dropped the secret and a handshake needs it back, and on the +// first request of a new stream, where it re-subscribes to everything it holds. +// Suppressing the repeat would leave the second case with nothing. +func (d *deltaStream) handleSubscribe(ctx context.Context, names []string) error { + if len(names) == 0 { + // A bare ACK, or an unsubscribe-only request. Nothing to send. + return nil + } + + var resources []*discovery.Resource + var removed []string + + for _, name := range names { + cert, err := d.srv.minter.certificate(ctx, name) + if err != nil { + // Refused. Tell Envoy the name does not exist; the paused + // handshake for that SNI then fails, which is the intended + // outcome for something that is not a hostname. + removed = append(removed, name) + continue + } + res, err := d.pack(name, cert) + if err != nil { + return err + } + resources = append(resources, res) + } + + if len(resources) == 0 && len(removed) == 0 { + return nil + } + return d.send(ctx, resources, removed) +} + +// pack wraps a minted cert as a versioned delta Resource. +func (d *deltaStream) pack(name string, cert *certauth.MintedCert) (*discovery.Resource, error) { + secret := toSecret(name, cert) + body, err := anypb.New(secret) + if err != nil { + return nil, fmt.Errorf("marshalling secret for %q: %w", name, err) + } + // The serial changes on every mint, so it is a natural resource version: + // every mint looks like a new version to Envoy. + version := cert.Serial + + return &discovery.Resource{ + Name: name, + Version: version, + Resource: body, + // Envoy starts a timer per resource when it receives one and drops the + // resource when that timer fires. That drop is the whole refresh + // mechanism: the next handshake for the name finds nothing cached and + // re-subscribes, and this server mints again. Stamped on every response + // rather than the first, because a resource that arrives with no ttl + // has its timer cleared and is then held for good. + Ttl: durationpb.New(d.srv.resourceTTL), + }, nil +} + +func (d *deltaStream) send(ctx context.Context, resources []*discovery.Resource, removed []string) error { + resp := &discovery.DeltaDiscoveryResponse{ + TypeUrl: secretTypeURL, + Resources: resources, + RemovedResources: removed, + Nonce: d.srv.nextNonce(), + } + + select { + case d.sendCh <- resp: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +// sendLoop drains sendCh onto the stream. gRPC forbids concurrent Send on a +// stream, so everything is funneled through this one goroutine; sendCh's +// buffer is what keeps a slow client from stalling the select loop in +// DeltaSecrets, which would hold up incoming requests behind a write. +// +// It stops on the first send failure and hands it to sendErr, which is what +// DeltaSecrets returns; a full sendErr means a failure is already on its way +// back, so the second one is dropped rather than blocking the exit. +func (d *deltaStream) sendLoop(ctx context.Context) { + defer close(d.sendDone) + for { + select { + case <-ctx.Done(): + return + case resp, ok := <-d.sendCh: + if !ok { + return + } + if err := d.stream.Send(resp); err != nil { + select { + case d.sendErr <- err: + default: + } + return + } + } + } +} diff --git a/cmd/atenet/internal/sdsmint/deltastream_test.go b/cmd/atenet/internal/sdsmint/deltastream_test.go new file mode 100644 index 000000000..7300f4642 --- /dev/null +++ b/cmd/atenet/internal/sdsmint/deltastream_test.go @@ -0,0 +1,530 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sdsmint + +import ( + "context" + "crypto/tls" + "crypto/x509" + "encoding/pem" + "io" + "testing" + "testing/synctest" + "time" + + tlsv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" + discovery "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + rpcstatus "google.golang.org/genproto/googleapis/rpc/status" + "google.golang.org/grpc/metadata" +) + +// fakeDeltaStream stands in for the gRPC stream Envoy would be on the other +// end of. Requests are fed in from a slice; responses are collected. +type fakeDeltaStream struct { + ctx context.Context + requests chan *discovery.DeltaDiscoveryRequest + sent chan *discovery.DeltaDiscoveryResponse +} + +func newFakeDeltaStream(ctx context.Context) *fakeDeltaStream { + return &fakeDeltaStream{ + ctx: ctx, + requests: make(chan *discovery.DeltaDiscoveryRequest, 8), + sent: make(chan *discovery.DeltaDiscoveryResponse, 8), + } +} + +func (f *fakeDeltaStream) Send(resp *discovery.DeltaDiscoveryResponse) error { + select { + case f.sent <- resp: + return nil + case <-f.ctx.Done(): + return f.ctx.Err() + } +} + +func (f *fakeDeltaStream) Recv() (*discovery.DeltaDiscoveryRequest, error) { + select { + case req, ok := <-f.requests: + if !ok { + return nil, io.EOF + } + return req, nil + case <-f.ctx.Done(): + return nil, io.EOF + } +} + +func (f *fakeDeltaStream) Context() context.Context { return f.ctx } +func (f *fakeDeltaStream) SetHeader(metadata.MD) error { return nil } +func (f *fakeDeltaStream) SendHeader(metadata.MD) error { return nil } +func (f *fakeDeltaStream) SetTrailer(metadata.MD) {} +func (f *fakeDeltaStream) SendMsg(any) error { return nil } +func (f *fakeDeltaStream) RecvMsg(any) error { return nil } + +// respondWait bounds a wait for something the server should do promptly. Every +// caller runs inside a synctest bubble, so this is fake time: only a run that +// is already failing ever spends it. +const respondWait = time.Minute + +// nextResponse waits for one response, failing the test if none arrives. +func (f *fakeDeltaStream) nextResponse(t *testing.T) *discovery.DeltaDiscoveryResponse { + t.Helper() + select { + case resp := <-f.sent: + return resp + case <-time.After(respondWait): + t.Fatal("timed out waiting for a DeltaDiscoveryResponse") + return nil + } +} + +// quiet blocks until the server has nothing left to do, then fails if it sent +// anything. This is the negative assertion the whole file used to spell as a +// sleep: synctest.Wait returns once every goroutine is durably blocked, so +// anything the server meant to send is already in the channel by then. +func (f *fakeDeltaStream) quiet(t *testing.T, whileDoing string) { + t.Helper() + synctest.Wait() + select { + case resp := <-f.sent: + t.Fatalf("server sent %v %s", resourceNames(resp), whileDoing) + default: + } +} + +func testServer(t *testing.T, opts serverOptions) *server { + t.Helper() + if opts.Logger == nil { + opts.Logger = quietLogger() + } + m := testMinter(t, minterOptions{TTL: defaultTTL}) + return newServer(m, opts) +} + +// startServer runs DeltaSecrets against a fake stream and returns the stream +// plus a func that stops it and reports the server's error. +func startServer(t *testing.T, srv *server) (*fakeDeltaStream, func() error) { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + stream := newFakeDeltaStream(ctx) + + done := make(chan error, 1) + go func() { done <- srv.DeltaSecrets(stream) }() + + return stream, func() error { + cancel() + select { + case err := <-done: + return err + case <-time.After(respondWait): + t.Fatal("DeltaSecrets did not return after the stream was cancelled") + return nil + } + } +} + +func resourceNames(resp *discovery.DeltaDiscoveryResponse) []string { + names := make([]string, 0, len(resp.GetResources())) + for _, r := range resp.GetResources() { + names = append(names, r.GetName()) + } + return names +} + +// unpackSecret pulls the Secret proto out of a delta Resource. +func unpackSecret(t *testing.T, res *discovery.Resource) *tlsv3.Secret { + t.Helper() + msg, err := res.GetResource().UnmarshalNew() + if err != nil { + t.Fatalf("unmarshalling resource %q: %v", res.GetName(), err) + } + secret, ok := msg.(*tlsv3.Secret) + if !ok { + t.Fatalf("resource %q is a %T, want *tlsv3.Secret", res.GetName(), msg) + } + return secret +} + +// leafFromResource pulls the leaf x509 out of a delta Resource carrying a +// Secret, so a test can reason about the certificate Envoy would actually +// serve rather than just the xDS version string. +func leafFromResource(t *testing.T, res *discovery.Resource) *x509.Certificate { + t.Helper() + secret := unpackSecret(t, res) + chain := secret.GetTlsCertificate().GetCertificateChain().GetInlineBytes() + block, _ := pem.Decode(chain) + if block == nil { + t.Fatalf("resource %q: certificate chain is not PEM", res.GetName()) + } + leaf, err := x509.ParseCertificate(block.Bytes) + if err != nil { + t.Fatalf("resource %q: parsing leaf: %v", res.GetName(), err) + } + return leaf +} + +func TestDeltaSecretsMintsSubscribedName(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + srv := testServer(t, serverOptions{}) + stream, stop := startServer(t, srv) + defer func() { + if err := stop(); err != nil { + t.Errorf("DeltaSecrets returned %v", err) + } + }() + + stream.requests <- &discovery.DeltaDiscoveryRequest{ + TypeUrl: secretTypeURL, + ResourceNamesSubscribe: []string{"a.example"}, + } + + resp := stream.nextResponse(t) + if resp.GetTypeUrl() != secretTypeURL { + t.Errorf("type_url = %q, want %q", resp.GetTypeUrl(), secretTypeURL) + } + if resp.GetNonce() == "" { + t.Error("response has no nonce; Envoy needs one to ACK") + } + if len(resp.GetResources()) != 1 { + t.Fatalf("got %d resources, want 1", len(resp.GetResources())) + } + + res := resp.GetResources()[0] + if res.GetName() != "a.example" { + t.Errorf("resource name = %q, want a.example", res.GetName()) + } + if res.GetVersion() == "" { + t.Error("resource has no version; delta xDS needs one per resource") + } + + secret := unpackSecret(t, res) + // This is the invariant the whole design rests on: Envoy matches the + // response to its on-demand subscription by secret name, which is the SNI. + if secret.GetName() != "a.example" { + t.Errorf("secret name = %q, want it to equal the requested resource name", secret.GetName()) + } + + chain := secret.GetTlsCertificate().GetCertificateChain().GetInlineBytes() + key := secret.GetTlsCertificate().GetPrivateKey().GetInlineBytes() + if _, err := tls.X509KeyPair(chain, key); err != nil { + t.Errorf("secret does not contain a usable TLS keypair: %v", err) + } + }) +} + +// TestDeltaSecretsStampsResourceTTL covers the only thing that gets a name +// re-minted. Envoy drops a resource when its TTL fires and re-subscribes on the +// next handshake; nothing on this side pushes. A resource sent without a ttl is +// held by Envoy for good, and the leaf inside it goes on being served long past +// its notAfter -- see poc/sdsmint/expiry, which measured both. +func TestDeltaSecretsStampsResourceTTL(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + srv := testServer(t, serverOptions{}) + stream, stop := startServer(t, srv) + defer func() { + if err := stop(); err != nil { + t.Errorf("DeltaSecrets returned %v", err) + } + }() + + stream.requests <- &discovery.DeltaDiscoveryRequest{ + TypeUrl: secretTypeURL, + ResourceNamesSubscribe: []string{"a.example"}, + } + + res := stream.nextResponse(t).GetResources()[0] + ttl := res.GetTtl() + if ttl == nil { + t.Fatal("resource carries no ttl; Envoy would hold this secret forever and serve the leaf past its notAfter") + } + + // The invariant, rather than the exact fraction: the secret has to be + // dropped while the leaf it carries is still valid, so the handshake + // that re-subscribes is never the one served an expired leaf. + remaining := time.Until(leafFromResource(t, res).NotAfter) + if ttl.AsDuration() >= remaining { + t.Errorf("ttl %s is not shorter than the leaf's remaining validity %s; a handshake could land after the leaf expires but before Envoy drops it", + ttl.AsDuration(), remaining) + } + }) +} + +func TestDeltaSecretsWithdrawsRefusedName(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + srv := testServer(t, serverOptions{}) + stream, stop := startServer(t, srv) + defer func() { + if err := stop(); err != nil { + t.Errorf("DeltaSecrets returned %v", err) + } + }() + + // The minter refuses names, not destinations: "*.evil.test" is turned away + // for being a wildcard rather than for being anyone in particular. What + // this pins is the partial response -- one name refused must not cost the + // other name in the same subscription its certificate. + stream.requests <- &discovery.DeltaDiscoveryRequest{ + TypeUrl: secretTypeURL, + ResourceNamesSubscribe: []string{"ok.allowed", "*.evil.test"}, + } + + resp := stream.nextResponse(t) + + if len(resp.GetResources()) != 1 || resp.GetResources()[0].GetName() != "ok.allowed" { + t.Errorf("resources = %v, want just ok.allowed", resourceNames(resp)) + } + // A server cannot NACK in xDS. Withdrawing the name is how it says "this + // will not be issued", and per the Envoy docs it also cancels the + // data-plane subscription for that name. + if got := resp.GetRemovedResources(); len(got) != 1 || got[0] != "*.evil.test" { + t.Errorf("removed_resources = %v, want [*.evil.test]", got) + } + }) +} + +func TestDeltaSecretsBareAckSendsNothing(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + srv := testServer(t, serverOptions{}) + stream, stop := startServer(t, srv) + defer func() { + if err := stop(); err != nil { + t.Errorf("DeltaSecrets returned %v", err) + } + }() + + stream.requests <- &discovery.DeltaDiscoveryRequest{ + TypeUrl: secretTypeURL, + ResourceNamesSubscribe: []string{"a.example"}, + } + first := stream.nextResponse(t) + + // Envoy's ACK carries the nonce and no subscriptions. Replying to it + // would start an infinite ACK loop. + stream.requests <- &discovery.DeltaDiscoveryRequest{ + TypeUrl: secretTypeURL, + ResponseNonce: first.GetNonce(), + } + stream.quiet(t, "in reply to a bare ACK") + }) +} + +// TestDeltaSecretsNeverPushesUnprompted pins the shape of the server after +// rotation and the idle sweep were both removed: a subscribe is answered once +// and then the stream is silent for the whole life of the leaf and beyond. The +// server never speaks first. Nothing re-mints a name in place, so a leaf +// expires under a live subscription and Envoy goes on serving it -- that is a +// known consequence of the removals, not an accident, and this is where it is +// written down. +func TestDeltaSecretsNeverPushesUnprompted(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + // The TTL testServer's minter is built with. + const ttl = defaultTTL + srv := testServer(t, serverOptions{}) + stream, stop := startServer(t, srv) + defer func() { + if err := stop(); err != nil { + t.Errorf("DeltaSecrets returned %v", err) + } + }() + + stream.requests <- &discovery.DeltaDiscoveryRequest{ + TypeUrl: secretTypeURL, + ResourceNamesSubscribe: []string{"a.example"}, + } + stream.nextResponse(t) + + // Well past the point where the leaf has expired. Free on a fake clock. + time.Sleep(2 * ttl) + stream.quiet(t, "after the only subscribed leaf had expired") + }) +} + +// TestDeltaSecretsIgnoresUnsubscribe pins that the server holds no subscription +// set to unsubscribe from. Envoy volunteers an unsubscribe when the +// configuration referencing a secret goes away; there is nothing on this side +// to forget, so the request must be absorbed silently rather than answered or +// treated as an error that tears down the stream. +func TestDeltaSecretsIgnoresUnsubscribe(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + srv := testServer(t, serverOptions{}) + stream, stop := startServer(t, srv) + defer func() { + if err := stop(); err != nil { + t.Errorf("DeltaSecrets returned %v", err) + } + }() + + stream.requests <- &discovery.DeltaDiscoveryRequest{ + TypeUrl: secretTypeURL, + ResourceNamesSubscribe: []string{"a.example"}, + } + stream.nextResponse(t) + + stream.requests <- &discovery.DeltaDiscoveryRequest{ + TypeUrl: secretTypeURL, + ResourceNamesUnsubscribe: []string{"a.example"}, + } + stream.quiet(t, "in reply to an unsubscribe") + + // The stream is still usable afterwards, and a name it just unsubscribed + // from is minted again on request like any other -- the server draws no + // distinction, because it kept no record to draw one from. + stream.requests <- &discovery.DeltaDiscoveryRequest{ + TypeUrl: secretTypeURL, + ResourceNamesSubscribe: []string{"a.example"}, + } + if names := resourceNames(stream.nextResponse(t)); len(names) != 1 || names[0] != "a.example" { + t.Errorf("after an unsubscribe the server served %v, want [a.example]", names) + } + }) +} + +// TestDeltaSecretsReplayOnlyRequestIsSilent covers the other half of what handle +// ignores. A request carrying nothing but initial_resource_versions says what +// Envoy already holds; answering it would push leaves nobody asked for. The +// re-subscribe that accompanies a real reconnect is what prompts the re-mint, +// and it arrives as an ordinary subscribe. +func TestDeltaSecretsReplayOnlyRequestIsSilent(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + srv := testServer(t, serverOptions{}) + stream, stop := startServer(t, srv) + defer func() { + if err := stop(); err != nil { + t.Errorf("DeltaSecrets returned %v", err) + } + }() + + stream.requests <- &discovery.DeltaDiscoveryRequest{ + TypeUrl: secretTypeURL, + InitialResourceVersions: map[string]string{"resumed.example": "old-version"}, + } + + stream.quiet(t, "in reply to a replay-only request") + }) +} + +func TestDeltaSecretsRejectsWrongTypeURL(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + srv := testServer(t, serverOptions{}) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + stream := newFakeDeltaStream(ctx) + + done := make(chan error, 1) + go func() { done <- srv.DeltaSecrets(stream) }() + + stream.requests <- &discovery.DeltaDiscoveryRequest{ + TypeUrl: "type.googleapis.com/envoy.config.cluster.v3.Cluster", + ResourceNamesSubscribe: []string{"a.example"}, + } + + // Wait for the server to fail on its own rather than canceling, which + // would race the context-done branch of the stream loop. + select { + case err := <-done: + if err == nil { + t.Fatal("DeltaSecrets accepted a non-SDS type_url") + } + case <-time.After(respondWait): + t.Fatal("DeltaSecrets did not reject a non-SDS type_url") + } + }) +} + +func TestDeltaSecretsSurvivesNack(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + srv := testServer(t, serverOptions{}) + stream, stop := startServer(t, srv) + defer func() { + if err := stop(); err != nil { + t.Errorf("DeltaSecrets returned %v", err) + } + }() + + stream.requests <- &discovery.DeltaDiscoveryRequest{ + TypeUrl: secretTypeURL, + ResourceNamesSubscribe: []string{"a.example"}, + } + first := stream.nextResponse(t) + + // A NACK must not tear down the stream; Envoy would just reconnect and + // we would lose every live subscription. + stream.requests <- &discovery.DeltaDiscoveryRequest{ + TypeUrl: secretTypeURL, + ResponseNonce: first.GetNonce(), + ErrorDetail: &rpcstatus.Status{Code: 3, Message: "bad certificate"}, + } + stream.requests <- &discovery.DeltaDiscoveryRequest{ + TypeUrl: secretTypeURL, + ResourceNamesSubscribe: []string{"b.example"}, + } + + resp := stream.nextResponse(t) + if names := resourceNames(resp); len(names) != 1 || names[0] != "b.example" { + t.Errorf("after a NACK the server served %v, want [b.example]", names) + } + }) +} + +// TestResubscribeIsMintedAgain is what is left of the refresh path. Envoy only +// re-subscribes to a name it has dropped, so a repeat subscribe is a request +// for a certificate and must be answered with a freshly minted one rather than +// suppressed as a duplicate. With rotation and the idle sweep both gone this is +// the only way a name ever gets a new leaf. +func TestResubscribeIsMintedAgain(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + srv := testServer(t, serverOptions{}) + stream, stop := startServer(t, srv) + defer func() { + if err := stop(); err != nil { + t.Errorf("DeltaSecrets returned %v", err) + } + }() + + subscribe := &discovery.DeltaDiscoveryRequest{ + TypeUrl: secretTypeURL, + ResourceNamesSubscribe: []string{"a.example"}, + } + + stream.requests <- subscribe + first := stream.nextResponse(t) + if len(first.GetResources()) != 1 { + t.Fatalf("initial response carried %d resources, want 1", len(first.GetResources())) + } + + stream.requests <- subscribe + second := stream.nextResponse(t) + if len(second.GetResources()) != 1 { + t.Fatalf("re-subscribe was answered with %d resources, want 1", len(second.GetResources())) + } + + if got := second.GetResources()[0].GetName(); got != "a.example" { + t.Fatalf("re-subscribe returned %q, want a.example", got) + } + leaf := leafFromResource(t, second.GetResources()[0]) + if err := leaf.VerifyHostname("a.example"); err != nil { + t.Fatalf("re-minted leaf does not cover a.example: %v", err) + } + + // A new leaf, not the first one handed back. The version is the serial, + // so an unchanged version here would mean Envoy sees no update and goes + // on serving whatever it already had. + if v1, v2 := first.GetResources()[0].GetVersion(), second.GetResources()[0].GetVersion(); v1 == v2 { + t.Errorf("re-subscribe returned the same version %s; the name was not minted again", v1) + } + }) +} diff --git a/cmd/atenet/internal/sdsmint/doc.go b/cmd/atenet/internal/sdsmint/doc.go new file mode 100644 index 000000000..28314dd98 --- /dev/null +++ b/cmd/atenet/internal/sdsmint/doc.go @@ -0,0 +1,19 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package sdsmint implements `atenet sdsmint`, a minting SDS server: an Envoy +// Secret Discovery Service that mints a TLS leaf certificate on demand for +// whatever hostname Envoy asks for, where the requested SDS resource name is +// the SNI from the client. +package sdsmint diff --git a/cmd/atenet/internal/sdsmint/listen.go b/cmd/atenet/internal/sdsmint/listen.go new file mode 100644 index 000000000..f016aace5 --- /dev/null +++ b/cmd/atenet/internal/sdsmint/listen.go @@ -0,0 +1,50 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file holds the only socket the process opens: the SDS one, which is +// always a unix domain socket because leaf private keys transit it. + +package sdsmint + +import ( + "errors" + "fmt" + "net" + "os" + "path/filepath" +) + +// listen binds the SDS socket. There is no TCP alternative on purpose: leaf +// private keys transit this channel, and a unix socket restricted to the +// proxy's UID is the only reach that is ever wanted. +func listen(uds string) (net.Listener, error) { + // A stale socket from a previous run would make Listen fail with + // "address already in use" even though nothing is running. + if err := os.Remove(uds); err != nil && !errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("removing stale socket %s: %w", uds, err) + } + if err := os.MkdirAll(filepath.Dir(uds), 0o755); err != nil { + return nil, fmt.Errorf("creating socket directory: %w", err) + } + lis, err := net.Listen("unix", uds) + if err != nil { + return nil, fmt.Errorf("listening on %s: %w", uds, err) + } + // Only the proxy should be able to ask for certificates. + if err := os.Chmod(uds, 0o600); err != nil { + lis.Close() + return nil, fmt.Errorf("restricting socket permissions: %w", err) + } + return lis, nil +} diff --git a/cmd/atenet/internal/sdsmint/minter.go b/cmd/atenet/internal/sdsmint/minter.go new file mode 100644 index 000000000..b300f00a6 --- /dev/null +++ b/cmd/atenet/internal/sdsmint/minter.go @@ -0,0 +1,145 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sdsmint + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net" + "strings" + "time" + + "github.com/agent-substrate/substrate/cmd/atenet/internal/sdsmint/certauth" +) + +// errHostNotAllowed is returned when a requested hostname will not be minted. +var errHostNotAllowed = errors.New("host not allowed") + +// minter returns a leaf certificate for a hostname. +type minter struct { + signer *certauth.Signer + ttl time.Duration + log *slog.Logger +} + +// minterOptions configures newMinter. +type minterOptions struct { + // TTL is the leaf lifetime. + TTL time.Duration + Logger *slog.Logger +} + +// defaultTTL for leaf cert lifetime. +const defaultTTL = 15 * time.Minute + +// newMinter builds a minter over signer. +func newMinter(signer *certauth.Signer, opts minterOptions) (*minter, error) { + if signer == nil { + return nil, errors.New("nil signer") + } + if opts.TTL <= 0 { + opts.TTL = defaultTTL + } + if opts.Logger == nil { + opts.Logger = slog.Default() + } + return &minter{ + signer: signer, + ttl: opts.TTL, + log: opts.Logger, + }, nil +} + +// certificate mints a leaf for host. It returns an error wrapping +// errHostNotAllowed if host is not a name this will mint for. +func (m *minter) certificate(ctx context.Context, host string) (*certauth.MintedCert, error) { + if err := checkHostSyntax(host); err != nil { + m.log.WarnContext(ctx, "certificate request denied", + slog.String("host", host), + slog.String("reason", err.Error()), + ) + // checkHostSyntax already quotes the host, so this does not repeat it. + return nil, fmt.Errorf("%w: %w", errHostNotAllowed, err) + } + + cert, err := m.signer.Sign(host, m.ttl) + if err != nil { + return nil, err + } + + if m.log.Enabled(ctx, slog.LevelInfo) { + m.log.InfoContext(ctx, "certificate issued", + slog.String("host", host), + slog.String("serial", cert.Serial), + slog.Time("not_after", cert.NotAfter), + ) + } + return cert, nil +} + +// checkHostSyntax checks whether host is a valid DNS name or IP address. +func checkHostSyntax(host string) error { + if isValidDNSName(host) || isValidIPAddress(host) { + return nil + } + return fmt.Errorf("invalid host name %q", host) +} + +// isValidDNSName reports whether host is a syntactically valid DNS name. +func isValidDNSName(host string) bool { + // A trailing dot names the root explicitly. It is legal in a DNS name and + // not in SNI, but Envoy passes on whatever it was given, so it is dropped + // here rather than making the final label look empty. + host = strings.TrimSuffix(host, ".") + if host == "" || len(host) > 253 { + return false + } + for _, label := range strings.Split(host, ".") { + if !isValidDNSLabel(label) { + return false + } + } + return true +} + +// isValidDNSLabel reports whether one dot-separated component is a legal label: +// letters, digits, and interior hyphens, up to 63 bytes. +func isValidDNSLabel(label string) bool { + if label == "" || len(label) > 63 { + return false + } + for i := 0; i < len(label); i++ { + switch c := label[i]; { + case c >= 'a' && c <= 'z', c >= 'A' && c <= 'Z', c >= '0' && c <= '9': + // A hyphen may not open or close a label. + case c == '-' && i > 0 && i < len(label)-1: + // Not legal in a hostname, but real names carry it -- service records + // and a fair number of internal names -- and it smuggles nothing into a + // certificate. Refusing it would fail handshakes for no gain. + case c == '_': + default: + return false + } + } + return true +} + +// isValidIPAddress reports whether host is an IP literal, v4 or v6. SNI is not +// supposed to carry an IP addresss, but some clients send it anyway. +func isValidIPAddress(host string) bool { + return net.ParseIP(host) != nil +} diff --git a/cmd/atenet/internal/sdsmint/minter_test.go b/cmd/atenet/internal/sdsmint/minter_test.go new file mode 100644 index 000000000..3ca48df4f --- /dev/null +++ b/cmd/atenet/internal/sdsmint/minter_test.go @@ -0,0 +1,194 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sdsmint + +import ( + "context" + "errors" + "fmt" + "io" + "log/slog" + "strings" + "sync" + "testing" + "time" + + "github.com/agent-substrate/substrate/cmd/atenet/internal/sdsmint/certauth" + "github.com/agent-substrate/substrate/internal/localca" +) + +func quietLogger() *slog.Logger { + return slog.New(slog.NewTextHandler(io.Discard, nil)) +} + +// testSigner builds a signer over a throwaway CA. +func testSigner(t *testing.T) *certauth.Signer { + t.Helper() + ca, err := localca.GenerateCA(localca.GenerateOptions{ + ID: "mitm", + CommonName: "sdsmint test CA", + KeyType: localca.KeyTypeECDSAP256, + Lifetime: time.Hour, + }) + if err != nil { + t.Fatalf("generating test CA: %v", err) + } + signer, err := certauth.New(&localca.Pool{CAs: []*localca.CA{ca}}, "") + if err != nil { + t.Fatalf("certauth.New: %v", err) + } + return signer +} + +func testMinter(t *testing.T, opts minterOptions) *minter { + t.Helper() + if opts.Logger == nil { + opts.Logger = quietLogger() + } + m, err := newMinter(testSigner(t), opts) + if err != nil { + t.Fatalf("newMinter: %v", err) + } + return m +} + +// TestMinterMintsPerCall pins that the minter holds nothing between calls. It +// used to cache by host, and the SDS layer is written on the assumption that +// it no longer does: pack reads cert.Serial as the resource version, so two +// mints of one name have to look like two versions to Envoy. +func TestMinterMintsPerCall(t *testing.T) { + m := testMinter(t, minterOptions{TTL: time.Minute}) + ctx := context.Background() + + first, err := m.certificate(ctx, "a.example") + if err != nil { + t.Fatalf("first certificate: %v", err) + } + second, err := m.certificate(ctx, "a.example") + if err != nil { + t.Fatalf("second certificate: %v", err) + } + if first.Serial == second.Serial { + t.Errorf("both calls returned serial %s; the minter is holding onto leaves", first.Serial) + } + + other, err := m.certificate(ctx, "b.example") + if err != nil { + t.Fatalf("certificate for a different host: %v", err) + } + if other.Serial == first.Serial { + t.Error("different hosts were served the same certificate") + } +} + +// TestMinterRefusesNonHostnames pins the one thing that can still make the +// minter say no. There is no destination allowlist any more, so the SDS +// server's withdraw path is only ever reached through checkHostSyntax, and it +// is reached through a wrapped errHostNotAllowed. +func TestMinterRefusesNonHostnames(t *testing.T) { + m := testMinter(t, minterOptions{TTL: time.Minute}) + ctx := context.Background() + + // Nothing about this name is on a list; it just is a name. + if _, err := m.certificate(ctx, "anything.at.all.test"); err != nil { + t.Fatalf("ordinary hostname was rejected: %v", err) + } + + _, err := m.certificate(ctx, "*.evil.test") + if err == nil { + t.Fatal("a wildcard SNI was minted") + } + if !errors.Is(err, errHostNotAllowed) { + t.Errorf("error = %v, want it to wrap errHostNotAllowed", err) + } +} + +func TestMinterIsConcurrencySafe(t *testing.T) { + m := testMinter(t, minterOptions{TTL: time.Minute}) + ctx := context.Background() + + var wg sync.WaitGroup + errs := make(chan error, 64) + for i := range 64 { + wg.Add(1) + go func() { + defer wg.Done() + if _, err := m.certificate(ctx, fmt.Sprintf("h%d.example", i%16)); err != nil { + errs <- err + } + }() + } + wg.Wait() + close(errs) + + for err := range errs { + t.Fatalf("concurrent GetCertificate: %v", err) + } +} + +func TestCheckHostSyntax(t *testing.T) { + validate := checkHostSyntax + + allowed := []string{ + "example.com", + "a.example.com", + "a.b.c.d.example.com", // any depth, unlike a "*" label + "EXAMPLE.COM", // case is not a hostname's business + "anything.test", // any TLD + "localhost", // a single label is still a name + "xn--80ak6aa92e.com", // punycode + "a.example.com.", // trailing dot + "my-host.example.com", // interior hyphen + "_dmarc.example.com", // underscore; not a hostname, but a real name + + strings.Repeat("a", 63) + ".example.com", // a label at the 63-byte limit + + // IP literals. SNI is not supposed to carry one, but clients send them, + // and Sign puts them in IPAddresses rather than DNSNames. + "192.0.2.1", + "127.0.0.1", + "2001:db8::1", + "::1", + "::ffff:192.0.2.1", // v4-mapped v6 + } + for _, host := range allowed { + if err := validate(host); err != nil { + t.Errorf("validate(%q) = %v, want nil", host, err) + } + } + + denied := []string{ + "", // empty + ".", // the root alone is not a name to mint for + "*.example.com", // a wildcard in the SNI itself + "a.example.com/../x", // path separator smuggling + "..example.com", // empty label + ".example.com", // leading dot + "a.example.com\nfoo", // embedded newline + "-example.com", // label opens with a hyphen + "example-.com", // label closes with a hyphen + "[::1]", // bracketed; Sign would read it as a DNS name + "fe80::1%eth0", // a zone is not part of an address in a SAN + "192.0.2.1:8443", // a port is not part of a host here + + strings.Repeat("a", 64) + ".example.com", // label over 63 bytes + strings.Repeat("a.", 200) + "example.com", // over 253 bytes + } + for _, host := range denied { + if err := validate(host); err == nil { + t.Errorf("validate(%q) = nil, want an error", host) + } + } +} diff --git a/cmd/atenet/internal/sdsmint/run.go b/cmd/atenet/internal/sdsmint/run.go new file mode 100644 index 000000000..3693353ca --- /dev/null +++ b/cmd/atenet/internal/sdsmint/run.go @@ -0,0 +1,141 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file turns a parsed config into a running process: signer, minter, +// listeners, gRPC server, shutdown. The flags it reads are cmd.go's. + +package sdsmint + +import ( + "context" + "errors" + "fmt" + "log/slog" + "os" + "os/signal" + "syscall" + "time" + + secretservice "github.com/envoyproxy/go-control-plane/envoy/service/secret/v3" + "google.golang.org/grpc" + + "github.com/agent-substrate/substrate/cmd/atenet/internal/sdsmint/certauth" + "github.com/agent-substrate/substrate/internal/localca" +) + +func run(ctx context.Context, cfg config) error { + logger, err := newLogger(cfg.LogLevel) + if err != nil { + return err + } + slog.SetDefault(logger) + + if cfg.UDSPath == "" { + return errors.New("--uds-path is required") + } + if cfg.CAPoolPath == "" { + return errors.New("--ca-pool-path is required") + } + if err := cfg.validateTTL(); err != nil { + return err + } + + signer, err := loadSigner(cfg.CAPoolPath, cfg.CAID) + if err != nil { + return err + } + + // Named m because minter is the type. + m, err := newMinter(signer, minterOptions{ + TTL: cfg.LeafCertTTL, + Logger: logger, + }) + if err != nil { + return fmt.Errorf("building minter: %w", err) + } + + lis, err := listen(cfg.UDSPath) + if err != nil { + return err + } + defer lis.Close() + + grpcServer := grpc.NewServer() + secretservice.RegisterSecretDiscoveryServiceServer(grpcServer, newServer(m, serverOptions{ + Logger: logger, + })) + + logger.Info("sdsmint listening", + slog.String("network", lis.Addr().Network()), + slog.String("address", lis.Addr().String()), + slog.Any("config", cfg), + ) + + ctx, stop := signal.NotifyContext(ctx, syscall.SIGINT, syscall.SIGTERM) + defer stop() + go func() { + <-ctx.Done() + logger.Info("shutting down") + // GracefulStop waits for in-flight RPCs to finish, but an xDS stream + // is long-lived by design and only ends when Envoy closes it. Waiting + // on it unconditionally deadlocks shutdown, so fall back to a hard + // stop after a grace period. + done := make(chan struct{}) + go func() { + grpcServer.GracefulStop() + close(done) + }() + + // shutdownGrace is how long a signaled server waits for in-flight RPCs before + // tearing open streams down. Envoy's SDS stream never ends on its own, so this + // is the normal path, not the exceptional one. + const shutdownGrace = 2 * time.Second + select { + case <-done: + case <-time.After(shutdownGrace): + logger.Warn("graceful shutdown timed out; closing open streams", + slog.Duration("grace", shutdownGrace)) + grpcServer.Stop() + } + }() + + if err := grpcServer.Serve(lis); err != nil && !errors.Is(err, grpc.ErrServerStopped) { + return fmt.Errorf("serving: %w", err) + } + return nil +} + +func newLogger(level string) (*slog.Logger, error) { + var lvl slog.Level + if err := lvl.UnmarshalText([]byte(level)); err != nil { + return nil, fmt.Errorf("--log-level %q: %w", level, err) + } + return slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: lvl})), nil +} + +func loadSigner(poolPath, id string) (*certauth.Signer, error) { + poolBytes, err := os.ReadFile(poolPath) + if err != nil { + return nil, fmt.Errorf("reading CA pool %s: %w", poolPath, err) + } + pool, err := localca.Unmarshal(poolBytes) + if err != nil { + return nil, fmt.Errorf("parsing CA pool %s: %w", poolPath, err) + } + signer, err := certauth.New(pool, id) + if err != nil { + return nil, fmt.Errorf("loading CA from %s: %w", poolPath, err) + } + return signer, nil +} diff --git a/cmd/atenet/internal/sdsmint/server.go b/cmd/atenet/internal/sdsmint/server.go new file mode 100644 index 000000000..ea305593f --- /dev/null +++ b/cmd/atenet/internal/sdsmint/server.go @@ -0,0 +1,123 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sdsmint + +import ( + "log/slog" + "strconv" + "sync/atomic" + "time" + + corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + tlsv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" + secretservice "github.com/envoyproxy/go-control-plane/envoy/service/secret/v3" + + "github.com/agent-substrate/substrate/cmd/atenet/internal/sdsmint/certauth" +) + +// secretTypeURL is the xDS type URL for SDS resources. +const secretTypeURL = "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret" + +// server implements Envoy's Secret Discovery Service, minting a certificate +// per requested resource name. +// +// DeltaSecrets is the only method of that service this server implements. +// State-of-the-world SDS -- StreamSecrets and FetchSecrets -- is deliberately +// left to the embedded Unimplemented, so an Envoy configured with anything +// other than DELTA_GRPC fails immediately and visibly. +type server struct { + secretservice.UnimplementedSecretDiscoveryServiceServer + + minter *minter + log *slog.Logger + + // resourceTTL is the xDS TTL stamped on every resource this server sends: + // how long Envoy holds a secret before dropping it of its own accord. It is + // derived from the leaf lifetime rather than configured separately, so the + // two cannot be set into an order that does not work. See pack, which + // stamps it, for what Envoy does with it. + resourceTTL time.Duration + + // nonce numbers the responses this server sends. Every xDS response needs + // one: the client echoes it back as response_nonce, which is what makes a + // later request recognizable as an ACK or NACK of a specific response + // rather than a fresh subscription. A response that carries none cannot be + // ACKed at all. + // + // One counter for the whole server rather than one per stream. A client + // only ever compares a nonce against the last one it received on its own + // stream, so the sequence being sparse there costs nothing, and this + // server does not correlate them either -- an incoming response_nonce is + // read only for the NACK log line in deltaStream.handle. Atomic because + // streams are served concurrently. + nonce atomic.Uint64 +} + +// serverOptions configures newServer. +type serverOptions struct { + Logger *slog.Logger +} + +// newServer builds an SDS server over m. +func newServer(m *minter, opts serverOptions) *server { + if opts.Logger == nil { + opts.Logger = slog.Default() + } + + // Half the leaf lifetime. Envoy drops the secret when the TTL fires and the + // next handshake for that name re-subscribes, so a replacement is minted + // while the leaf it replaces is still valid. A TTL equal to the leaf + // lifetime would drop the secret at the moment the leaf died, leaving a + // window for a handshake to land on one already past its notAfter. + const refreshFraction = 2 + + return &server{ + minter: m, + log: opts.Logger, + // TODO(haiyanmeng): tune resourceTTL to be more efficient. + resourceTTL: m.ttl / refreshFraction, + } +} + +// nextNonce returns the nonce to stamp on the next response. It increments +// before formatting, so the first response is "1" and none ever goes out with +// the empty nonce. +func (s *server) nextNonce() string { + return strconv.FormatUint(s.nonce.Add(1), 10) +} + +// inlineBytes wraps PEM bytes as an inline Envoy DataSource. Leaf material is +// inlined rather than written to a path because it is per-connection and +// short-lived; putting it on a filesystem would only widen exposure. +func inlineBytes(b []byte) *corev3.DataSource { + return &corev3.DataSource{ + Specifier: &corev3.DataSource_InlineBytes{InlineBytes: b}, + } +} + +// toSecret packs a minted cert into the Secret proto Envoy expects back. The +// secret's name MUST equal the requested resource name (the SNI), or Envoy +// will not match the response to its subscription. +func toSecret(name string, c *certauth.MintedCert) *tlsv3.Secret { + return &tlsv3.Secret{ + Name: name, + Type: &tlsv3.Secret_TlsCertificate{ + TlsCertificate: &tlsv3.TlsCertificate{ + CertificateChain: inlineBytes(c.CertChainPEM), + PrivateKey: inlineBytes(c.PrivateKeyPEM), + }, + }, + } +} diff --git a/cmd/kubectl-ate/internal/cmd/admin_make_ca_pool.go b/cmd/kubectl-ate/internal/cmd/admin_make_ca_pool.go index ab42f2690..d7388c376 100644 --- a/cmd/kubectl-ate/internal/cmd/admin_make_ca_pool.go +++ b/cmd/kubectl-ate/internal/cmd/admin_make_ca_pool.go @@ -28,6 +28,8 @@ import ( var caID string var targetSecretNamespace string var targetSecretName string +var caKeyType string +var caCommonName string var makeCaPoolCmd = &cobra.Command{ Use: "make-ca-pool", @@ -45,7 +47,11 @@ var makeCaPoolCmd = &cobra.Command{ return fmt.Errorf("while creating Kubernetes client: %w", err) } - ca, err := localca.GenerateED25519CA(caID) + ca, err := localca.GenerateCA(localca.GenerateOptions{ + ID: caID, + CommonName: caCommonName, + KeyType: localca.KeyType(caKeyType), + }) if err != nil { return fmt.Errorf("while generating CA: %w", err) } @@ -84,5 +90,9 @@ func init() { makeCaPoolCmd.Flags().StringVar(&caID, "ca-id", "", "The ID of the initial CA in the Pool") makeCaPoolCmd.Flags().StringVar(&targetSecretNamespace, "secret-namespace", "default", "Create the secret in this namespace") makeCaPoolCmd.Flags().StringVar(&targetSecretName, "name", "", "Create the secret with this name") + makeCaPoolCmd.Flags().StringVar(&caKeyType, "key-type", string(localca.KeyTypeED25519), + fmt.Sprintf("Signing key algorithm, %q or %q. Prefer %s for a CA whose certificates are validated by clients outside substrate, where Ed25519 support cannot be assumed.", + localca.KeyTypeED25519, localca.KeyTypeECDSAP256, localca.KeyTypeECDSAP256)) + makeCaPoolCmd.Flags().StringVar(&caCommonName, "common-name", "", "Subject common name of the CA certificate. Cosmetic; nothing authenticates on it.") makeCaPoolCmd.MarkFlagRequired("name") } diff --git a/hack/install-ate.sh b/hack/install-ate.sh index 5ace949c2..3d22b118a 100755 --- a/hack/install-ate.sh +++ b/hack/install-ate.sh @@ -82,6 +82,7 @@ function usage() { echo " --create-jwt-authority-pool-secret Create JWT authority pool secret" echo " --create-actor-id-ca-pool-secret Create actor ID CA pool secret" echo " --create-actor-id-ca-certs-secret Create actor ID CA certs secret" + echo " --create-egress-mitm-ca-pool-secret Create egress MITM CA pool secret" echo " --create-podcertificate-controller-cas Create podcertificate controller CAs" echo " --create-valkey-ca-certs-secret Create Valkey's combined client/server CA bundle" echo " --create-api-server-env-vars Create ate-api-server env vars" @@ -351,6 +352,20 @@ create_actor_id_ca_certs_secret() { | run_kubectl apply -f - } +# The MITM CA the egress gateway's sdsmint sidecar signs per-SNI leaves with. +# ecdsa-p256 rather than the ed25519 default: these leaves are validated by +# arbitrary clients inside actor sandboxes, where Ed25519 support cannot be +# assumed. +create_egress_mitm_ca_pool_secret() { + log_step "create_egress_mitm_ca_pool_secret" + run_kubectl_ate admin make-ca-pool \ + --ca-id="mitm" \ + --name="egress-mitm-ca-pool" \ + --secret-namespace=ate-system \ + --key-type=ecdsa-p256 \ + --common-name="substrate egress MITM CA" +} + create_podcertificate_controller_cas() { log_step "create_podcertificate_controller_cas" run_kubectl create namespace podcertificate-controller-system || true @@ -548,6 +563,8 @@ ensure_apiserver_prerequisites() { || create_podcertificate_controller_cas run_kubectl get secret -n ate-system valkey-ca-certs >/dev/null 2>&1 \ || create_valkey_ca_certs_secret + run_kubectl get secret -n ate-system egress-mitm-ca-pool >/dev/null 2>&1 \ + || create_egress_mitm_ca_pool_secret # This ConfigMap carries the selected store backend, so always reconcile it # to make switching --store-backend update an existing installation. create_api_server_env_vars @@ -607,8 +624,11 @@ deploy_atenet() { router_manifest="$(render_atenet_router_manifest)" echo "${router_manifest}" | run_kubectl apply -f - - run_ko apply -f manifests/ate-install/atenet-egress.yaml + run_kubectl get secret -n ate-system egress-mitm-ca-pool >/dev/null 2>&1 \ + || create_egress_mitm_ca_pool_secret + run_ko apply -f manifests/ate-install/atenet-dns.yaml + run_ko apply -f manifests/ate-install/atenet-egress.yaml run_kubectl rollout status deployment/atenet-router -n ate-system --timeout=120s run_kubectl rollout status deployment/atenet-egress -n ate-system --timeout=120s run_kubectl rollout status deployment/dns -n ate-system --timeout=120s @@ -920,6 +940,7 @@ while [[ "$#" -gt 0 ]]; do --create-jwt-authority-pool-secret) create_jwt_authority_pool_secret ;; --create-actor-id-ca-pool-secret) create_actor_id_ca_pool_secret ;; --create-actor-id-ca-certs-secret) create_actor_id_ca_certs_secret ;; + --create-egress-mitm-ca-pool-secret) create_egress_mitm_ca_pool_secret ;; --create-podcertificate-controller-cas) create_podcertificate_controller_cas ;; --create-valkey-ca-certs-secret) create_valkey_ca_certs_secret ;; --create-api-server-env-vars) create_api_server_env_vars ;; diff --git a/internal/atunnel/client.go b/internal/atunnel/client.go index 666431bbf..ba0189198 100644 --- a/internal/atunnel/client.go +++ b/internal/atunnel/client.go @@ -19,6 +19,7 @@ import ( "context" "crypto/tls" "crypto/x509" + "errors" "fmt" "io" "net" @@ -27,6 +28,7 @@ import ( "os" "strconv" "strings" + "syscall" ) // TODO(liorlieberman): support/use CONNECT on Ingress as well. @@ -41,6 +43,28 @@ type ClientConfig struct { // DialFunc dials a network address. It matches net.Dialer.DialContext. type DialFunc func(ctx context.Context, network, address string) (net.Conn, error) +// ErrGatewayHandshake reports that the gateway's front door refused the +// connection at TLS: it rejected the client certificate, or its own +// certificate did not verify. +var ErrGatewayHandshake = errors.New("atunnel: egress gateway TLS handshake") + +// ConnectRejectedError reports a CONNECT the gateway answered with a non-2xx +// status. The caller authenticated successfully and the request was declined +// anyway, which is what an authorization denial looks like from here. The +// status code is carried separately from the message because it is the part a +// caller can act on. +type ConnectRejectedError struct { + StatusCode int + // Status is the full status line, e.g. "403 Forbidden". + Status string + // Message is the response body, or the status text when the body is empty. + Message string +} + +func (e *ConnectRejectedError) Error() string { + return fmt.Sprintf("atunnel: egress gateway rejected CONNECT with %s: %s", e.Status, e.Message) +} + // ClientOption customizes a Client beyond its configuration. Production // callers need none of these. type ClientOption func(*Client) @@ -117,7 +141,7 @@ func (c *Client) DialContext(ctx context.Context, destination string) (net.Conn, tlsConn := tls.Client(rawConn, c.tlsConfig.Clone()) if err := tlsConn.HandshakeContext(ctx); err != nil { _ = rawConn.Close() - return nil, fmt.Errorf("atunnel: egress gateway TLS handshake: %w", err) + return nil, fmt.Errorf("%w: %w", ErrGatewayHandshake, err) } req := &http.Request{ @@ -127,14 +151,14 @@ func (c *Client) DialContext(ctx context.Context, destination string) (net.Conn, } if err := req.Write(tlsConn); err != nil { _ = tlsConn.Close() - return nil, fmt.Errorf("atunnel: writing CONNECT request: %w", err) + return nil, connectExchangeError("writing CONNECT request", err) } reader := bufio.NewReader(tlsConn) resp, err := http.ReadResponse(reader, req) if err != nil { _ = tlsConn.Close() - return nil, fmt.Errorf("atunnel: reading CONNECT response: %w", err) + return nil, connectExchangeError("reading CONNECT response", err) } if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { body, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10)) @@ -144,12 +168,44 @@ func (c *Client) DialContext(ctx context.Context, destination string) (net.Conn, if message == "" { message = http.StatusText(resp.StatusCode) } - return nil, fmt.Errorf("atunnel: egress gateway rejected CONNECT with %s: %s", resp.Status, message) + return nil, &ConnectRejectedError{StatusCode: resp.StatusCode, Status: resp.Status, Message: message} } return &bufferedConn{Conn: tlsConn, reader: reader}, nil } +// connectExchangeError wraps a failure that happened after the TLS handshake +// returned but before the gateway answered CONNECT, naming the front door as +// the cause when the connection was torn down rather than answered. +func connectExchangeError(op string, err error) error { + if gatewayHungUp(err) { + return fmt.Errorf("%w: %s: %w", ErrGatewayHandshake, op, err) + } + return fmt.Errorf("atunnel: %s: %w", op, err) +} + +// gatewayHungUp reports whether err is the peer tearing the connection down, +// as opposed to a local failure such as a context deadline or a malformed +// response. +func gatewayHungUp(err error) bool { + // A TLS alert from the peer -- "unknown certificate authority" is the one + // a wrong client CA produces. crypto/tls reports an alert as a net.OpError + // whose Op is "remote error" and whose Err is an unexported alert type, so + // the Op is the only part of it that can be matched without matching on + // the message text this package deliberately avoids. + var opErr *net.OpError + if errors.As(err, &opErr) && opErr.Op == "remote error" { + return true + } + // Or the gateway closed or reset instead of alerting -- including a reset + // that lands while the CONNECT request is still going out, which is what + // makes the alert-versus-EPIPE outcome a race rather than a distinction. + return errors.Is(err, io.EOF) || + errors.Is(err, io.ErrUnexpectedEOF) || + errors.Is(err, syscall.EPIPE) || + errors.Is(err, syscall.ECONNRESET) +} + func validateDestination(destination string) error { host, port, err := net.SplitHostPort(destination) if err != nil { diff --git a/internal/atunnel/client_test.go b/internal/atunnel/client_test.go index 4f9dc78e4..a7ddb1724 100644 --- a/internal/atunnel/client_test.go +++ b/internal/atunnel/client_test.go @@ -23,6 +23,7 @@ import ( "crypto/tls" "crypto/x509" "encoding/pem" + "errors" "fmt" "io" "math/big" @@ -31,6 +32,7 @@ import ( "os" "path/filepath" "strings" + "syscall" "testing" "time" ) @@ -103,6 +105,124 @@ func TestClientDialContextRejected(t *testing.T) { } } +// TestClientDialContextGatewayRefusesClientCertificate pins the classification +// of a front-door refusal to ErrGatewayHandshake under both TLS versions, which +// is not one behavior but two. +// +// Under 1.2 the server rejects the certificate mid-handshake and +// HandshakeContext returns the error. Under 1.3 the client's handshake has +// already returned nil by the time the server looks at the certificate, so the +// refusal lands on the CONNECT exchange instead -- and 1.3 is what a real +// gateway negotiates. A caller asking "did the door turn me away?" has to get +// the same answer either way, or it is really asking which version was +// negotiated. +func TestClientDialContextGatewayRefusesClientCertificate(t *testing.T) { + for _, tt := range []struct { + name string + maxTLSVersion uint16 + }{ + {name: "TLS 1.3 refuses after the client handshake completes", maxTLSVersion: tls.VersionTLS13}, + {name: "TLS 1.2 refuses during the handshake", maxTLSVersion: tls.VersionTLS12}, + } { + t.Run(tt.name, func(t *testing.T) { + ca := newTestCA(t) + // The gateway trusts a different CA than the one that issued the + // client's certificate, which is the shape of an actor presenting a + // podidentity credential to a door that only accepts actor identity. + gatewayAddress := serveTestRefusingGateway(t, ca, newTestCA(t), tt.maxTLSVersion) + client := newTestClient(t, ca, WithDialer(dialFixedAddress(gatewayAddress))) + + _, err := client.DialContext(context.Background(), "192.0.2.10:443") + if err == nil { + t.Fatal("DialContext succeeded against a gateway that refuses the client certificate") + } + if !errors.Is(err, ErrGatewayHandshake) { + t.Errorf("DialContext error = %v, want it to wrap ErrGatewayHandshake", err) + } + var rejected *ConnectRejectedError + if errors.As(err, &rejected) { + // A refusal at the door is not a CONNECT the gateway answered: + // conflating them would let an authorization denial read as an + // authentication failure. + t.Errorf("DialContext error = %v, want no ConnectRejectedError", err) + } + }) + } +} + +// TestClientDialContextGatewayHangsUpBeforeResponding covers the same window +// without an alert in it: the gateway closes the connection after the handshake +// and says nothing. There is still nobody but the front door on the other end, +// since no CONNECT response means no upstream was ever dialed. +func TestClientDialContextGatewayHangsUpBeforeResponding(t *testing.T) { + ca := newTestCA(t) + gatewayAddress := serveTestConnectGateway(t, ca, func(conn net.Conn, _ *http.Request) { + _ = conn.Close() + }) + client := newTestClient(t, ca, WithDialer(dialFixedAddress(gatewayAddress))) + + _, err := client.DialContext(context.Background(), "192.0.2.10:443") + if err == nil { + t.Fatal("DialContext succeeded against a gateway that hung up") + } + if !errors.Is(err, ErrGatewayHandshake) { + t.Errorf("DialContext error = %v, want it to wrap ErrGatewayHandshake", err) + } +} + +// TestConnectExchangeError is the other half of gatewayHungUp: a failure on +// this side of the connection must not be reported as the door refusing, or +// "the gateway rejected my certificate" stops meaning anything. The cases are +// exercised directly because the ones worth pinning -- a deadline, a response +// the gateway did send but malformed -- are awkward to provoke through a real +// listener and trivial to state as errors. +func TestConnectExchangeError(t *testing.T) { + for _, tt := range []struct { + name string + err error + wantFrontDoor bool + }{ + { + name: "peer TLS alert", + err: &net.OpError{Op: "remote error", Err: errors.New("tls: unknown certificate authority")}, + wantFrontDoor: true, + }, + { + name: "peer closed without alerting", + err: io.EOF, + wantFrontDoor: true, + }, + { + name: "peer reset mid-response", + err: io.ErrUnexpectedEOF, + wantFrontDoor: true, + }, + { + name: "peer reset while the CONNECT was going out", + err: &net.OpError{Op: "write", Err: syscall.EPIPE}, + wantFrontDoor: true, + }, + { + name: "our own deadline expired", + err: os.ErrDeadlineExceeded, + }, + { + name: "the gateway answered, but not with HTTP", + err: errors.New("malformed HTTP response \"garbage\""), + }, + } { + t.Run(tt.name, func(t *testing.T) { + err := connectExchangeError("reading CONNECT response", tt.err) + if got := errors.Is(err, ErrGatewayHandshake); got != tt.wantFrontDoor { + t.Errorf("errors.Is(%v, ErrGatewayHandshake) = %t, want %t", err, got, tt.wantFrontDoor) + } + if !errors.Is(err, tt.err) { + t.Errorf("%v no longer unwraps to the underlying %v", err, tt.err) + } + }) + } +} + func TestClientDialContextValidatesInput(t *testing.T) { ca := newTestCA(t) client := newTestClient(t, ca) @@ -192,6 +312,45 @@ func serveTestConnectGateway(t *testing.T, ca *testCA, handle func(net.Conn, *ht return listener.Addr().String() } +// serveTestRefusingGateway serves a front door that presents a certificate from +// serverCA and requires a client certificate from clientCA, so a client holding +// one from anywhere else is turned away. maxVersion caps the negotiated TLS +// version, which is what decides whether the refusal reaches the client during +// its handshake or after it. +// +// It handshakes and closes rather than reading a request: a refused client +// never gets to send one, and any error here is the refusal working. +func serveTestRefusingGateway(t *testing.T, serverCA, clientCA *testCA, maxVersion uint16) string { + t.Helper() + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = listener.Close() }) + + clientCAs := x509.NewCertPool() + clientCAs.AppendCertsFromPEM(clientCA.certPEM) + config := &tls.Config{ + MinVersion: tls.VersionTLS12, + MaxVersion: maxVersion, + Certificates: []tls.Certificate{issueDNSCertificate(t, serverCA, "egress.test")}, + ClientAuth: tls.RequireAndVerifyClientCert, + ClientCAs: clientCAs, + } + + go func() { + conn, err := listener.Accept() + if err != nil { + return + } + defer conn.Close() + tlsConn := tls.Server(conn, config) + _ = tlsConn.Handshake() + _ = tlsConn.Close() + }() + return listener.Addr().String() +} + func issueDNSCertificate(t *testing.T, ca *testCA, dnsName string) tls.Certificate { t.Helper() key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) diff --git a/internal/e2e/fixtures/egressprobe/egressprobe.yaml.tmpl b/internal/e2e/fixtures/egressprobe/egressprobe.yaml.tmpl new file mode 100644 index 000000000..a07f09629 --- /dev/null +++ b/internal/e2e/fixtures/egressprobe/egressprobe.yaml.tmpl @@ -0,0 +1,113 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The egress probe used by internal/e2e/suites/sdsmint. ${NAMESPACE} is +# substituted by the suite with the randomized namespace it created, so the +# probe is torn down with that namespace and leaves nothing behind. +# +# A plain Pod, not an actor: the suite is testing the gateway's MITM leg, and +# running the probe itself inside an actor would make a snapshot or restore +# failure look like an sdsmint failure. The suite does create one actor, but +# only for its identity -- the certificate mounted below is minted for it, and +# the actor's own workload is never contacted. + +apiVersion: v1 +kind: Pod +metadata: + name: egressprobe + namespace: ${NAMESPACE} + labels: + app: egressprobe +spec: + restartPolicy: Never + containers: + - name: egressprobe + image: ko://github.com/agent-substrate/substrate/internal/e2e/fixtures/egressprobe + args: + - "--listen=:8080" + ports: + - name: http + containerPort: 8080 + # The suite port-forwards to this pod, and port-forward targets the first + # READY pod behind the Service. Without a readiness gate the forward can + # attach before the listener exists and the first handshake fails as a + # connection refused that looks like a gateway problem. + readinessProbe: + httpGet: + path: /healthz + port: 8080 + periodSeconds: 2 + resources: + requests: + cpu: 10m + memory: 32Mi + # runAsUser must be spelled out: ko's distroless static base declares no + # USER, so runAsNonRoot on its own makes kubelet refuse to start the + # container ("image will run as root") rather than pick a uid. 65532 is + # distroless's nonroot uid, and the same one the egress gateway's pod uses. + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + capabilities: + drop: ["ALL"] + volumeMounts: + - name: "actor-identity" + mountPath: "/run/actor-identity" + - name: "actor-identity-unknown" + mountPath: "/run/actor-identity-unknown" + - name: "podidentity" + mountPath: "/run/podidentity.podcert.ate.dev" + - name: "servicedns-ca" + mountPath: "/run/servicedns.podcert.ate.dev" + volumes: + - name: "actor-identity" + secret: + secretName: egressprobe-actor-identity + - name: "actor-identity-unknown" + secret: + secretName: egressprobe-unknown-actor + - name: "podidentity" + projected: + sources: + - podCertificate: + signerName: podidentity.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - name: "servicedns-ca" + projected: + sources: + - clusterTrustBundle: + signerName: servicedns.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem + +--- + +apiVersion: v1 +kind: Service +metadata: + name: egressprobe + namespace: ${NAMESPACE} +spec: + selector: + app: egressprobe + ports: + - name: http + port: 8080 + targetPort: 8080 diff --git a/internal/e2e/fixtures/egressprobe/main.go b/internal/e2e/fixtures/egressprobe/main.go new file mode 100644 index 000000000..a5e424eb0 --- /dev/null +++ b/internal/e2e/fixtures/egressprobe/main.go @@ -0,0 +1,232 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Command egressprobe drives the egress gateway's MITM leg from inside the +// cluster and reports the certificate it was served, so an e2e suite can assert +// on what sdsmint actually minted. +package main + +import ( + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "encoding/pem" + "errors" + "flag" + "fmt" + "log" + "net/http" + "time" + + "github.com/agent-substrate/substrate/internal/atunnel" + "github.com/agent-substrate/substrate/internal/credbundle" +) + +var ( + listenAddress = flag.String("listen", ":8080", "Address the probe's HTTP API listens on.") + gatewayAddress = flag.String("gateway-address", "atenet-egress.ate-system.svc:443", "host:port of the egress gateway's CONNECT front door.") + credentialBundlePath = flag.String("credential-bundle", "/run/actor-identity/credential-bundle.pem", "PEM credential bundle presented as the client certificate to the gateway.") + trustBundlePath = flag.String("trust-bundle", "/run/servicedns.podcert.ate.dev/trust-bundle.pem", "PEM trust bundle used to verify the gateway's serving certificate.") + handshakeTimeout = flag.Duration("handshake-timeout", 20*time.Second, "Budget for one CONNECT plus inner handshake.") +) + +// tunnelDestination is the CONNECT authority. atunnel takes this from +// SO_ORIGINAL_DST and rejects hostnames, so it must be a literal IP -- and the +// gateway routes every CONNECT to the MITM listener regardless of authority, +// which is why a documentation address that resolves nowhere is enough. The +// name being tested travels in the tunneled ClientHello, not here. +const tunnelDestination = "192.0.2.1:443" + +// Stages a handshake can fail at, reported in handshakeResult.Stage. They name +// the hop, so a test can say which of the gateway's several ways of saying no +// it expected without matching on prose. +const ( + // stageClient is a local failure before anything was dialed: a bad flag, an + // unreadable credential bundle. Never a statement about the gateway. + stageClient = "client" + // stageGatewayTLS is the front door's mTLS. Reaching here and failing means + // the gateway refused the client certificate -- or presented one the probe + // would not accept. + stageGatewayTLS = "gateway_tls" + // stageConnect is the CONNECT exchange. A failure here means the + // certificate was accepted and the request was declined anyway, which is + // where the ext_proc authorization check lives. ConnectStatus carries the + // status code. + stageConnect = "connect" + // stageTunnel is any other failure opening the tunnel: DNS, TCP, a + // truncated response. + stageTunnel = "tunnel" + // stageInnerHandshake is the tunneled TLS handshake -- the one sdsmint + // serves. A failure here is the minter's, not the front door's. + stageInnerHandshake = "inner_handshake" +) + +// handshakeResult is the probe's response body. +type handshakeResult struct { + SNI string `json:"sni"` + // Credential is the bundle path this handshake presented, echoed back so a + // test cannot mistake a result for one taken with a different identity. + Credential string `json:"credential"` + // OK reports whether the inner TLS handshake completed. A denied SNI is a + // normal outcome, not a probe failure, so it comes back as OK=false with + // the reason rather than as an HTTP error. + OK bool `json:"ok"` + // Stage is where a failed handshake stopped, one of the stage constants + // above. Empty when OK. + Stage string `json:"stage,omitempty"` + // ConnectStatus is the status the gateway answered the CONNECT with, set + // only when Stage is stageConnect and the gateway actually replied. + ConnectStatus int `json:"connect_status,omitempty"` + Error string `json:"error,omitempty"` + // ChainPEM is the chain the gateway presented, leaf first. + ChainPEM string `json:"chain_pem,omitempty"` +} + +// handshake opens a tunnel through the gateway and completes an inner TLS +// handshake for the requested SNI, which is what makes Envoy ask sdsmint for a +// secret under that name. +func handshake(w http.ResponseWriter, r *http.Request) { + sni := r.URL.Query().Get("sni") + if sni == "" { + http.Error(w, "missing sni query parameter", http.StatusBadRequest) + return + } + + // Which credential to present. The default is the actor identity the suite + // minted; a test that is asserting how the gateway treats some OTHER + // credential names its path here rather than needing a second pod, since + // the choice is made per handshake and nothing is cached across them. + credential := r.URL.Query().Get("credential-bundle") + if credential == "" { + credential = *credentialBundlePath + } + + ctx, cancel := context.WithTimeout(r.Context(), *handshakeTimeout) + defer cancel() + + result := handshakeResult{SNI: sni, Credential: credential} + chain, stage, err := fetchChain(ctx, sni, credential) + if err != nil { + result.Stage = stage + result.Error = err.Error() + var rejected *atunnel.ConnectRejectedError + if errors.As(err, &rejected) { + result.ConnectStatus = rejected.StatusCode + } + } else { + result.OK = true + result.ChainPEM = encodeChain(chain) + } + writeJSON(w, result) +} + +func encodeChain(chain []*x509.Certificate) string { + var out []byte + for _, cert := range chain { + out = append(out, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: cert.Raw})...) + } + return string(out) +} + +// fetchChain opens a tunnel and completes the inner handshake, returning the +// chain it was served. On failure it also returns the stage that failed, since +// which hop said no is the assertion most callers are actually making. +func fetchChain(ctx context.Context, sni, credentialBundle string) ([]*x509.Certificate, string, error) { + client, err := atunnel.NewClient(atunnel.ClientConfig{ + GatewayAddress: *gatewayAddress, + ServerName: serverName(*gatewayAddress), + GetClientCertificate: credbundle.ClientLoader(credentialBundle), + TrustBundlePath: *trustBundlePath, + }) + if err != nil { + return nil, stageClient, fmt.Errorf("building egress client: %w", err) + } + + conn, err := client.DialContext(ctx, tunnelDestination) + if err != nil { + return nil, dialStage(err), fmt.Errorf("opening tunnel: %w", err) + } + defer conn.Close() + + //nolint:gosec // G402: verification is the caller's assertion; see the package comment. + tlsConn := tls.Client(conn, &tls.Config{ + ServerName: sni, + InsecureSkipVerify: true, + MinVersion: tls.VersionTLS12, + }) + if err := tlsConn.HandshakeContext(ctx); err != nil { + return nil, stageInnerHandshake, fmt.Errorf("inner TLS handshake for %q: %w", sni, err) + } + defer tlsConn.Close() + + out := tlsConn.ConnectionState().PeerCertificates + if len(out) == 0 { + return nil, stageInnerHandshake, fmt.Errorf("handshake for %q completed with no peer certificates", sni) + } + return out, "", nil +} + +// dialStage names the hop a DialContext failure stopped at. It reads atunnel's +// typed errors rather than its messages, so the mapping survives a reworded +// error. +func dialStage(err error) string { + var rejected *atunnel.ConnectRejectedError + switch { + case errors.Is(err, atunnel.ErrGatewayHandshake): + return stageGatewayTLS + case errors.As(err, &rejected): + return stageConnect + default: + return stageTunnel + } +} + +// serverName is the host half of a host:port address. It is written by hand +// rather than with net.SplitHostPort so that a malformed flag surfaces at +// handshake time with the address in the message, instead of at startup. +func serverName(address string) string { + for i := len(address) - 1; i >= 0; i-- { + if address[i] == ':' { + return address[:i] + } + } + return address +} + +func writeJSON(w http.ResponseWriter, v any) { + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(v); err != nil { + log.Printf("egressprobe: encoding response: %v", err) + } +} + +func main() { + flag.Parse() + + mux := http.NewServeMux() + mux.HandleFunc("/handshake", handshake) + mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + }) + + server := &http.Server{ + Addr: *listenAddress, + Handler: mux, + ReadHeaderTimeout: 10 * time.Second, + WriteTimeout: 2 * time.Minute, + } + log.Printf("egressprobe: listening on %s, gateway %s", *listenAddress, *gatewayAddress) + log.Fatal(server.ListenAndServe()) +} diff --git a/internal/e2e/suites/sdsmint/actoridentity_test.go b/internal/e2e/suites/sdsmint/actoridentity_test.go new file mode 100644 index 000000000..62faecb70 --- /dev/null +++ b/internal/e2e/suites/sdsmint/actoridentity_test.go @@ -0,0 +1,275 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sdsmint + +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "net/url" + "os" + "path" + "sync" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/agent-substrate/substrate/internal/e2e" + "github.com/agent-substrate/substrate/internal/localca" + "github.com/agent-substrate/substrate/internal/substratex509" + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" +) + +// The gateway's front door requires a client certificate signed by the +// actor-identity CA, and its ext_proc sidecar then looks the certified actor up +// in the control plane. Getting through it therefore needs both halves: a leaf +// this pool signs, and an actor the ate API agrees is running. +const ( + actorIDCASecret = "actor-id-ca-pool" + actorIDCASecretKey = "pool" + + // The atespace the suite's actor lives in. Fixed rather than randomized so + // a leaked actor from an aborted run is easy to find and delete. + probeAtespace = "ate-sdsmint-e2e" + + // actorCertificateLifetime matches what ateapi's MintCert issues. Nothing + // here depends on the exact value -- the suite runs in minutes -- but a + // credential that outlives the real one would hide an expiry bug in the + // gateway rather than reproduce it. + actorCertificateLifetime = time.Hour + + // Where the probe pod finds the credentials the suite mints for it. Kept in + // step with egressprobe.yaml.tmpl and the --credential-bundle default in + // the probe. + actorCredentialSecret = "egressprobe-actor-identity" + + unknownActorCredentialSecret = "egressprobe-unknown-actor" + unknownActorCredentialPath = "/run/actor-identity-unknown/credential-bundle.pem" + + credentialBundleKey = "credential-bundle.pem" + + // podIdentityCredentialPath is the probe's own workload identity: a valid + // substrate credential that is not an actor. It is mounted so the suite can + // show the gateway refusing it. + podIdentityCredentialPath = "/run/podidentity.podcert.ate.dev/credential-bundle.pem" +) + +// templateRef is the ActorTemplate the suite's actor is created from. The +// default is the one every other suite uses; the environment overrides exist +// for clusters that install the fixtures elsewhere. +func templateRef() (namespace, name string) { + namespace, name = "ate-demo-counter", "counter" + if v := os.Getenv("E2E_TEMPLATE_NAMESPACE"); v != "" { + namespace = v + } + if v := os.Getenv("E2E_TEMPLATE_NAME"); v != "" { + name = v + } + return namespace, name +} + +// probeActor is the identity the probe authenticates to the gateway as. +type probeActor struct { + atespace string + name string + uid string +} + +func (a *probeActor) identity() *substratex509.ActorIdentity { + return &substratex509.ActorIdentity{ + Atespace: a.atespace, + ActorName: a.name, + ActorUid: a.uid, + Purpose: substratex509.ActorIdentityPurposeAtunnel, + } +} + +// liveActor returns the one actor the whole suite shares. +var liveActor = sync.OnceValues(createLiveActor) + +func createLiveActor() (*probeActor, error) { + ctx, cancel := context.WithTimeout(context.Background(), 6*time.Minute) + defer cancel() + + clients := e2e.GetClients() + tmplNS, tmplName := templateRef() + name := fmt.Sprintf("sdsmint-probe-%d", time.Now().UnixNano()) + ref := &ateapipb.ObjectRef{Atespace: probeAtespace, Name: name} + + // CreateActor requires the atespace to exist first; a second run finds it + // already there, which is not an error. + _, _ = clients.SubstrateAPI.CreateAtespace(ctx, &ateapipb.CreateAtespaceRequest{ + Atespace: &ateapipb.Atespace{Metadata: &ateapipb.ResourceMetadata{Name: probeAtespace}}, + }) + if _, err := clients.SubstrateAPI.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: probeAtespace, Name: name}, + ActorTemplateNamespace: tmplNS, + ActorTemplateName: tmplName, + }}); err != nil { + return nil, fmt.Errorf("creating actor %s/%s from template %s/%s: %w (install the fixture with hack/install-demo-counter.sh, or point E2E_TEMPLATE_NAMESPACE/E2E_TEMPLATE_NAME at another one)", + probeAtespace, name, tmplNS, tmplName, err) + } + e2e.RegisterSuiteCleanup(func() { + // DeleteActor requires the actor to be suspended first. Both are + // best-effort: a run that could not reach the API here has already + // failed louder somewhere else. + cleanupCtx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + _, _ = clients.SubstrateAPI.SuspendActor(cleanupCtx, &ateapipb.SuspendActorRequest{Actor: ref}) + _, _ = clients.SubstrateAPI.DeleteActor(cleanupCtx, &ateapipb.DeleteActorRequest{Actor: ref}) + }) + + if _, err := clients.SubstrateAPI.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: ref, Boot: true}); err != nil { + return nil, fmt.Errorf("resuming actor %s/%s: %w", probeAtespace, name, err) + } + + deadline := time.Now().Add(4 * time.Minute) + var lastStatus ateapipb.Actor_Status + for time.Now().Before(deadline) { + actor, err := clients.SubstrateAPI.GetActor(ctx, &ateapipb.GetActorRequest{Actor: ref}) + if err == nil { + lastStatus = actor.GetStatus() + if lastStatus == ateapipb.Actor_STATUS_RUNNING { + uid := actor.GetMetadata().GetUid() + if uid == "" { + return nil, fmt.Errorf("actor %s/%s is running but has no UID", probeAtespace, name) + } + return &probeActor{atespace: probeAtespace, name: name, uid: uid}, nil + } + } + select { + case <-ctx.Done(): + return nil, fmt.Errorf("waiting for actor %s/%s to run: %w", probeAtespace, name, ctx.Err()) + case <-time.After(2 * time.Second): + } + } + return nil, fmt.Errorf("actor %s/%s never reached STATUS_RUNNING (last status %v); a saturated worker pool is the usual cause", + probeAtespace, name, lastStatus) +} + +// actorIdentityCA returns the CA that signs actor certificates, straight from +// the secret ateapi signs with. +func actorIdentityCA(t *testing.T, ctx context.Context) *localca.CA { + t.Helper() + secret, err := e2e.GetClients().K8s.CoreV1().Secrets(egressNamespace).Get(ctx, actorIDCASecret, metav1.GetOptions{}) + if err != nil { + t.Fatalf("reading actor-identity CA pool secret %s/%s: %v", egressNamespace, actorIDCASecret, err) + } + pool, err := localca.Unmarshal(secret.Data[actorIDCASecretKey]) + if err != nil { + t.Fatalf("parsing actor-identity CA pool from %s/%s key %q: %v", egressNamespace, actorIDCASecret, actorIDCASecretKey, err) + } + if len(pool.CAs) == 0 { + t.Fatalf("actor-identity CA pool %s/%s contains no CA", egressNamespace, actorIDCASecret) + } + // CAs[0] is the one that signs: ateapi's MintCert makes the same choice. + return pool.CAs[0] +} + +// mintActorCredential issues a client credential for identity, in the shape +// atunnel gets from ateapi. +func mintActorCredential(t *testing.T, ca *localca.CA, identity *substratex509.ActorIdentity) []byte { + t.Helper() + + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("generating actor key: %v", err) + } + + template := &x509.Certificate{ + URIs: []*url.URL{{ + Scheme: "spiffe", + Host: "substrate-actor.local", + Path: path.Join("atespace", identity.Atespace, "actor", identity.ActorName), + }}, + NotBefore: time.Now().Add(-5 * time.Minute), + NotAfter: time.Now().Add(actorCertificateLifetime), + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + BasicConstraintsValid: true, + IsCA: false, + Issuer: pkix.Name{CommonName: "api.ate-system.svc.cluster.local"}, + } + if err := substratex509.AddActorIdentityToCertificate(identity, template); err != nil { + t.Fatalf("adding the ActorIdentity extension for %s/%s: %v", identity.Atespace, identity.ActorName, err) + } + + der, err := x509.CreateCertificate(rand.Reader, template, ca.RootCertificate, key.Public(), ca.SigningKey) + if err != nil { + t.Fatalf("signing the actor certificate for %s/%s: %v", identity.Atespace, identity.ActorName, err) + } + keyDER, err := x509.MarshalPKCS8PrivateKey(key) + if err != nil { + t.Fatalf("marshaling the actor key: %v", err) + } + + // A credential bundle as internal/credbundle parses it: the PKCS#8 key + // first, then the chain leaf-first. + bundle := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER}) + bundle = append(bundle, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})...) + for _, intermediate := range ca.IntermediateCertificates { + bundle = append(bundle, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: intermediate.Raw})...) + } + return bundle +} + +// writeCredentialSecret puts a minted bundle where the probe pod can mount it. +func writeCredentialSecret(t *testing.T, ctx context.Context, ns, name string, bundle []byte) { + t.Helper() + _, err := e2e.GetClients().K8s.CoreV1().Secrets(ns).Create(ctx, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns}, + Data: map[string][]byte{credentialBundleKey: bundle}, + }, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("creating credential secret %s/%s: %v", ns, name, err) + } +} + +// provisionProbeCredentials mints everything the probe pod mounts: the +// credential of the live actor, and one for an actor that does not exist. Both +// are signed by the real CA, so the difference between them is exactly the +// control-plane check and nothing else. Which one a handshake presents is +// chosen per request, which is what lets the whole suite share one pod. +func provisionProbeCredentials(t *testing.T, ctx context.Context, ns string) *probeActor { + t.Helper() + + actor, err := liveActor() + if err != nil { + t.Fatalf("preparing the actor the probe authenticates as: %v", err) + } + t.Logf("probe authenticates as actor %s/%s (uid %s)", actor.atespace, actor.name, actor.uid) + + ca := actorIdentityCA(t, ctx) + writeCredentialSecret(t, ctx, ns, actorCredentialSecret, mintActorCredential(t, ca, actor.identity())) + + // Same CA, same shape, an actor the control plane has never heard of. The + // name is scoped to the probe's namespace so a stray record cannot collide + // with anything. + writeCredentialSecret(t, ctx, ns, unknownActorCredentialSecret, mintActorCredential(t, ca, &substratex509.ActorIdentity{ + Atespace: probeAtespace, + ActorName: "no-such-actor-" + ns, + ActorUid: "00000000-0000-0000-0000-000000000000", + Purpose: substratex509.ActorIdentityPurposeAtunnel, + })) + + return actor +} diff --git a/internal/e2e/suites/sdsmint/sdsmint_test.go b/internal/e2e/suites/sdsmint/sdsmint_test.go new file mode 100644 index 000000000..40e1b4e2e --- /dev/null +++ b/internal/e2e/suites/sdsmint/sdsmint_test.go @@ -0,0 +1,515 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package sdsmint e2e-tests the egress gateway's certificate minter. +// +// sdsmint is an SDS server that mints a leaf certificate on demand for the +// SNI Envoy was asked for. Its unit tests cover the SDS protocol against a fake +// Envoy; what they cannot cover is the part that has actually broken in +// practice -- whether the deployed pod's Envoy, CA pool secret and unix socket +// line up. Every assertion here is made on a certificate that came +// off a real handshake through the real gateway. +// +// Actors do reach the gateway. hack/install-ate.sh applies +// manifests/ate-install/ate-api-server.yaml unconditionally, and that sets +// --egress-gateway-address, so every resume arms the egress redirect +// (prepareActorEgress in cmd/ateom-gvisor/main.go). What that traffic proves is +// that the path carries bytes. It never inspects the certificate it was served, +// because an actor cannot: nothing in the cluster trusts the MITM anchor, so an +// actor speaking TLS through the gateway disables verification and takes +// whatever it is handed. A leaf minted for the wrong name, one that outlives +// its --leaf-cert-ttl by an order of magnitude, one chained to a CA nobody +// installed -- every one of those carries traffic as well as a correct one, and the +// pod stays Running throughout. Those are the assertions below, and no amount +// of actor traffic makes them. +// +// Reaching sdsmint at all now means getting past the front door, which +// authenticates the caller as a named actor: an actor-identity client +// certificate at the TLS layer, and an ext_proc check that the certified actor +// is one the control plane says is running. So the suite creates one actor and +// mints a credential for it (actoridentity_test.go), and the two tests at the +// bottom assert the two ways that door says no. That makes this suite the only +// end-to-end coverage of the egress authorization leg as well as of the minter. +// +// One actor and one probe pod serve the whole suite, because standing them up +// costs more than every handshake here put together and neither is what any +// test is about. Both are torn down with the suite rather than with the test +// that triggered them; see liveActor and sharedProbe. +package sdsmint + +import ( + "context" + "crypto/x509" + "encoding/json" + "encoding/pem" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + + "github.com/agent-substrate/substrate/internal/ateclient" + "github.com/agent-substrate/substrate/internal/e2e" + "github.com/agent-substrate/substrate/internal/localca" + "github.com/agent-substrate/substrate/internal/portforward" +) + +const ( + // Where the gateway and its CA live. Both are fixed by + // manifests/ate-install/atenet-egress.yaml and hack/install-ate.sh. + egressNamespace = "ate-system" + mitmCASecret = "egress-mitm-ca-pool" + mitmCASecretKey = "pool" + mitmCAID = "mitm" + + // leafTTL is --leaf-cert-ttl on the sdsmint sidecar, and leafSkew is the + // backdating sdsmint/certauth applies to NotBefore. Their sum is the validity + // span every leaf should carry. Keep both in step with the manifest: a leaf + // that suddenly lasts hours is the failure this pair is here to catch. + leafTTL = 15 * time.Minute + leafSkew = 5 * time.Minute + + probeName = "egressprobe" +) + +// TestSdsmintMintsALeafPerSNI is the core functional assertion, and the only +// test here that exercises the minter rather than the door in front of it: +// several SNIs through one gateway come back as that many distinct +// certificates, each issued for the name that was asked for, each chaining +// directly to the MITM CA, each short-lived. +// +// Two of the names are ordinary and three deliberately are not, which is what +// pins "mint for every name" as deployed: the gateway mints for names nobody +// enumerated, at depths and in TLDs no allowlist pattern would have covered. +// This test used to assert the opposite -- that an SNI outside an allowlist +// was refused -- and its inversion is the whole content of that change. If the +// gateway is ever given an allowlist again, this test failing is the intended +// signal, not a regression: restore the refusal assertion rather than +// narrowing the names below. +// +// Every SNI is qualified with the probe's namespace, which is fresh per run, so +// each is a name Envoy has never subscribed to. Reusing one would be served +// from Envoy's live secret set and the test would pass without sdsmint having +// minted anything. +func TestSdsmintMintsALeafPerSNI(t *testing.T) { + ctx := context.Background() + + root := mitmRootCertificate(t, ctx) + probe := sharedProbe(t, ctx) + + // Nothing has to resolve: the mint happens during the inner handshake, + // before the gateway looks for an upstream. .invalid can never be delegated + // (RFC 6761), and neither the depth nor the TLD of the last three was + // reachable under the old "example.com *.example.com" allowlist, so a + // half-reverted config cannot pass this by accident. + snis := []string{ + probe.uniqueSNI("a.example.com"), + probe.uniqueSNI("b.example.com"), + probe.uniqueSNI("notallowed.invalid"), // a TLD no pattern mentioned + probe.uniqueSNI("a.b.deep.example.com"), // deeper than one "*" label + probe.uniqueSNI("UPPER.Notallowed.Invalid"), // and case-insensitively + } + + serials := map[string]string{} + for _, sni := range snis { + result := probe.handshake(t, ctx, sni) + if !result.OK { + t.Errorf("gateway refused to mint for %q at stage %q, so it is not minting for every name: %s", sni, result.Stage, result.Error) + continue + } + chain := parseChain(t, sni, result.ChainPEM) + leaf := chain[0] + + // Minted for the name that was asked for, and only that name. A leaf + // carrying anything else means the SNI Envoy policed is not the name + // the certificate authorizes. Case-insensitively, because a ClientHello + // SNI is a DNS name and Envoy is free to normalize it. + if got := leaf.DNSNames; len(got) != 1 || !strings.EqualFold(got[0], sni) { + t.Errorf("leaf for %q has DNSNames %v, want exactly [%q]", sni, got, sni) + } + // And carrying no subject: the SAN above is the whole of the leaf's + // identity, so there is no second name for the two to disagree on. + if got := leaf.Subject.String(); got != "" { + t.Errorf("leaf for %q has subject %q, want empty", sni, got) + } + + // Chains to the MITM root the installer created, verified for the SNI + // itself so the root's dNSName constraint is applied too. + opts := x509.VerifyOptions{ + DNSName: strings.ToLower(sni), + Roots: certPool(root), + Intermediates: certPool(chain[1:]...), + KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + } + if _, err := leaf.Verify(opts); err != nil { + t.Errorf("leaf for %q does not verify against the %s/%s MITM root: %v", sni, egressNamespace, mitmCASecret, err) + } + + // Short-lived, which is what bounds the damage from a leaked leaf key + // and what makes the MITM CA tolerable at all. The window is generous + // because the point is to catch a TTL that is wrong by an order of + // magnitude, not clock skew. + validity := leaf.NotAfter.Sub(leaf.NotBefore) + if want := leafTTL + leafSkew; validity < want-time.Minute || validity > want+time.Minute { + t.Errorf("leaf for %q is valid for %v, want about %v (--leaf-cert-ttl in atenet-egress.yaml)", sni, validity, want) + } + if time.Now().After(leaf.NotAfter) { + t.Errorf("leaf for %q was already expired when served (NotAfter %s)", sni, leaf.NotAfter) + } + + // One certificate per name, not one certificate reused under many + // names. Serials are compared across the whole set rather than pairwise + // so a minter that caches on something other than the SNI is caught + // wherever the collision happens. + serial := leaf.SerialNumber.Text(16) + if other, dup := serials[serial]; dup { + t.Errorf("%q and %q were served the same certificate (serial %s); the gateway is not minting per name", other, sni, serial) + } + serials[serial] = sni + + // Signed by the root itself, and the chain carries nothing but the leaf + // and that root. sdsmint has no delegated-intermediate mode any more, + // so an extra certificate in the middle means the deployed image is + // not the one this suite describes. + if len(chain) != 2 { + t.Errorf("chain for %q has %d certificates, want 2 (leaf + root)", sni, len(chain)) + continue + } + if !chain[1].Equal(root) { + t.Errorf("leaf for %q is not chained to the %s/%s root; issuer is %q", + sni, egressNamespace, mitmCASecret, chain[1].Subject) + } + + t.Logf("%s: served a %d-cert chain, leaf serial %s valid %v, issued by %q constrained to %v", + sni, len(chain), serial, validity, root.Subject.CommonName, root.PermittedDNSDomains) + } +} + +// TestGatewayRefusesANonActorWorkload is the front door's half of the egress +// story: the gateway trusts only the actor-identity CA, so a credential that +// proves "some substrate workload" no longer opens a tunnel. The probe's own +// podidentity certificate is exactly that credential, and it is the one this +// suite presented before actors had to authenticate at all. +// +// The refusal has to come from the gateway's mTLS specifically. A probe that +// failed to dial, or that got as far as the CONNECT, would be reporting +// something other than the front door turning it away. +func TestGatewayRefusesANonActorWorkload(t *testing.T) { + ctx := context.Background() + + probe := sharedProbe(t, ctx) + + sni := probe.uniqueSNI("podidentity.example.com") + result := probe.handshakeAs(t, ctx, sni, podIdentityCredentialPath) + if result.OK { + t.Fatalf("the gateway opened a tunnel for the probe's podidentity credential; its front door is accepting workloads that are not actors") + } + if result.Stage != stageGatewayTLS { + t.Fatalf("podidentity credential failed at stage %q, want %q -- the probe was stopped by something other than the front door, so this test is not checking it: %s", + result.Stage, stageGatewayTLS, result.Error) + } + t.Logf("gateway refused the non-actor credential at its front door as expected: %s", result.Error) +} + +// TestGatewayRefusesAnUnknownActor covers the check that only ext_proc can +// make. The credential here is cryptographically perfect -- signed by the real +// actor-identity CA, correct extension, correct purpose -- so Envoy completes +// the handshake, and the CONNECT is denied only because the control plane has +// no such actor. Without this, nothing distinguishes a gateway that authorizes +// on the certificate alone from one that authorizes on control-plane state, and +// the difference is whether a deleted actor's credential still works. +func TestGatewayRefusesAnUnknownActor(t *testing.T) { + ctx := context.Background() + + probe := sharedProbe(t, ctx) + + sni := probe.uniqueSNI("unknown.example.com") + result := probe.handshakeAs(t, ctx, sni, unknownActorCredentialPath) + if result.OK { + t.Fatalf("the gateway tunneled for an actor the control plane has never heard of; the ext_proc identity check is not running") + } + // A 403 on the CONNECT, not a TLS failure: the certificate was accepted and + // the identity it carries was rejected. A failure at stageGatewayTLS would + // mean the request never reached ext_proc, and the denial would prove + // nothing about the control-plane lookup. + if result.Stage != stageConnect || result.ConnectStatus != http.StatusForbidden { + t.Fatalf("unknown actor was refused at stage %q with CONNECT status %d, want %q and %d -- something other than the ext_proc identity check turned it away: %s", + result.Stage, result.ConnectStatus, stageConnect, http.StatusForbidden, result.Error) + } + t.Logf("gateway denied the unknown actor at CONNECT as expected: %s", result.Error) +} + +// mitmRootCertificate reads the trust anchor sdsmint signs under, straight +// from the secret the sidecar mounts, so the test is checking the chain against +// the CA that is actually deployed rather than one it was told about. +func mitmRootCertificate(t *testing.T, ctx context.Context) *x509.Certificate { + t.Helper() + secret, err := e2e.GetClients().K8s.CoreV1().Secrets(egressNamespace).Get(ctx, mitmCASecret, metav1.GetOptions{}) + if err != nil { + t.Fatalf("reading MITM CA pool secret %s/%s: %v", egressNamespace, mitmCASecret, err) + } + // This unmarshals the signing key along with the certificate. That is + // unavoidable -- the pool is one blob -- and acceptable only because this + // runs against a test cluster with a kubeconfig that could read the secret + // anyway. Nothing below touches the key. + pool, err := localca.Unmarshal(secret.Data[mitmCASecretKey]) + if err != nil { + t.Fatalf("parsing MITM CA pool from %s/%s key %q: %v", egressNamespace, mitmCASecret, mitmCASecretKey, err) + } + for _, ca := range pool.CAs { + if ca.ID == mitmCAID { + return ca.RootCertificate + } + } + t.Fatalf("MITM CA pool %s/%s has no CA with id %q", egressNamespace, mitmCASecret, mitmCAID) + return nil +} + +// probeClient talks to the in-cluster egress probe over a port-forward. +type probeClient struct { + // ns is the probe's namespace, randomized per run. Tests qualify their + // SNIs with it so that no name is one Envoy already holds a secret for. + ns string + baseURL string + http *http.Client +} + +// uniqueSNI qualifies suffix with the probe's namespace, producing a name no +// earlier run has asked the gateway for. Nothing withdraws a secret any more, +// so Envoy holds every name it has ever been given for the life of its process: +// an SNI a previous run used comes back from its secret set without sdsmint +// minting anything, and the test passes having tested nothing. +func (c *probeClient) uniqueSNI(suffix string) string { + return c.ns + "-" + suffix +} + +var ( + probeOnce sync.Once + probeVal *probeClient + probeErr error +) + +// sharedProbe returns the one probe pod the whole suite uses. +func sharedProbe(t *testing.T, ctx context.Context) *probeClient { + t.Helper() + probeOnce.Do(func() { + // startProbe reports failures through t, which unwinds this goroutine + // without returning. Leave something behind so the tests that run + // afterwards fail pointing at the first one instead of dereferencing + // nil. + defer func() { + if probeVal == nil && probeErr == nil { + probeErr = errors.New("setup did not complete; see the failure reported by the first test that needed the probe") + } + }() + probeVal = startProbe(t, ctx) + }) + if probeErr != nil { + t.Fatalf("starting the shared egress probe: %v", probeErr) + } + return probeVal +} + +// startProbe creates the probe's namespace, mints its credentials there, builds +// and deploys the probe, waits for it to be ready, and returns a client for it. +func startProbe(t *testing.T, ctx context.Context) *probeClient { + t.Helper() + if _, err := e2e.CheckEnv("KO_DOCKER_REPO"); err != nil { + t.Fatalf("CheckEnv failed: %v", err) + } + ns := e2e.CreateNamespace(t).Name + + provisionProbeCredentials(t, ctx, ns) + root, err := e2e.FindRepoRoot() + if err != nil { + t.Fatalf("FindRepoRoot: %v", err) + } + + tmpl, err := os.ReadFile(filepath.Join(root, "internal/e2e/fixtures/egressprobe/egressprobe.yaml.tmpl")) + if err != nil { + t.Fatalf("reading egressprobe manifest template: %v", err) + } + manifest := filepath.Join(t.TempDir(), "egressprobe.yaml") + rendered := strings.ReplaceAll(string(tmpl), "${NAMESPACE}", ns) + if err := os.WriteFile(manifest, []byte(rendered), 0o644); err != nil { + t.Fatalf("writing rendered egressprobe manifest: %v", err) + } + + applyArgs := []string{"ko", "apply", "-f", manifest} + if e2e.KubeContext != "" { + applyArgs = append(applyArgs, "--", "--context="+e2e.KubeContext) + } + e2e.RunCmdWithEnv(t, []string{"KO_CONFIG_PATH=" + root}, filepath.Join(root, "hack/run-tool.sh"), applyArgs...) + + waitForProbeReady(t, ctx, ns) + + config, err := ateclient.LoadConfig(e2e.KubeConfig, e2e.KubeContext) + if err != nil { + t.Fatalf("loading kubeconfig: %v", err) + } + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + t.Fatalf("creating k8s client for port-forward: %v", err) + } + localPort, stop, err := portforward.ServicePortForward(ctx, config, clientset, ns, probeName, 8080) + if err != nil { + t.Fatalf("port-forwarding %s/%s: %v", ns, probeName, err) + } + e2e.RegisterSuiteCleanup(stop) + + return &probeClient{ + ns: ns, + baseURL: fmt.Sprintf("http://127.0.0.1:%d", localPort), + http: &http.Client{Timeout: 90 * time.Second}, + } +} + +func waitForProbeReady(t *testing.T, ctx context.Context, ns string) { + t.Helper() + const timeout = 3 * time.Minute + deadline := time.Now().Add(timeout) + var lastState string + for time.Now().Before(deadline) { + pod, err := e2e.GetClients().K8s.CoreV1().Pods(ns).Get(ctx, probeName, metav1.GetOptions{}) + switch { + case err != nil: + lastState = err.Error() + case portforward.IsPodReady(pod): + t.Logf("probe pod %s/%s is ready", ns, probeName) + return + default: + lastState = describeProbeState(pod) + } + time.Sleep(2 * time.Second) + } + t.Fatalf("timed out after %v waiting for probe pod %s/%s to become ready: %s", timeout, ns, probeName, lastState) +} + +func describeProbeState(pod *corev1.Pod) string { + parts := []string{"phase=" + string(pod.Status.Phase)} + for _, cs := range pod.Status.ContainerStatuses { + switch { + case cs.State.Waiting != nil: + parts = append(parts, fmt.Sprintf("%s waiting: %s: %s", cs.Name, cs.State.Waiting.Reason, cs.State.Waiting.Message)) + case cs.State.Terminated != nil: + parts = append(parts, fmt.Sprintf("%s terminated: %s: %s", cs.Name, cs.State.Terminated.Reason, cs.State.Terminated.Message)) + default: + parts = append(parts, fmt.Sprintf("%s running, ready=%t", cs.Name, cs.Ready)) + } + } + return strings.Join(parts, "; ") +} + +// handshake asks the probe to complete one inner TLS handshake for sni, +// presenting the actor credential the suite minted. A refused SNI comes back as +// a result with OK false, not as an error: refusal is one of the outcomes under +// test. +func (c *probeClient) handshake(t *testing.T, ctx context.Context, sni string) handshakeResult { + t.Helper() + return c.handshakeAs(t, ctx, sni, "") +} + +// handshakeAs is handshake with a credential other than the probe's default. +// An empty credential means the default. +func (c *probeClient) handshakeAs(t *testing.T, ctx context.Context, sni, credential string) handshakeResult { + t.Helper() + endpoint := c.baseURL + "/handshake?sni=" + url.QueryEscape(sni) + if credential != "" { + endpoint += "&credential-bundle=" + url.QueryEscape(credential) + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + t.Fatalf("building probe request for %q: %v", sni, err) + } + resp, err := c.http.Do(req) + if err != nil { + t.Fatalf("calling probe for %q: %v", sni, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10)) + t.Fatalf("probe returned %d for %q: %s", resp.StatusCode, sni, strings.TrimSpace(string(body))) + } + var out handshakeResult + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + t.Fatalf("decoding probe response for %q: %v", sni, err) + } + return out +} + +// Stages a handshake can fail at, mirroring the probe's stage constants. Only +// the two the tests assert on are named here; the rest arrive as whatever +// string the probe sent and are printed in the failure. +const ( + stageGatewayTLS = "gateway_tls" + stageConnect = "connect" +) + +// handshakeResult mirrors the probe's response body. It is duplicated rather +// than imported because the probe is package main. +type handshakeResult struct { + SNI string `json:"sni"` + Credential string `json:"credential"` + OK bool `json:"ok"` + // Stage is where a failed handshake stopped. Asserting on it rather than + // on Error is what keeps "the front door refused the certificate" and "the + // door opened and ext_proc said no" from being the same test: they are + // different hops, and their messages are only incidentally different. + Stage string `json:"stage"` + // ConnectStatus is the status the gateway answered the CONNECT with, set + // only when Stage is stageConnect. + ConnectStatus int `json:"connect_status"` + Error string `json:"error"` + ChainPEM string `json:"chain_pem"` +} + +func parseChain(t *testing.T, sni, chainPEM string) []*x509.Certificate { + t.Helper() + var chain []*x509.Certificate + rest := []byte(chainPEM) + for { + var block *pem.Block + block, rest = pem.Decode(rest) + if block == nil { + break + } + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + t.Fatalf("parsing certificate served for %q: %v", sni, err) + } + chain = append(chain, cert) + } + if len(chain) == 0 { + t.Fatalf("no certificates in the chain served for %q", sni) + } + return chain +} + +func certPool(certs ...*x509.Certificate) *x509.CertPool { + pool := x509.NewCertPool() + for _, cert := range certs { + pool.AddCert(cert) + } + return pool +} diff --git a/internal/e2e/suites/sdsmint/testmain_test.go b/internal/e2e/suites/sdsmint/testmain_test.go new file mode 100644 index 000000000..ca19d4b95 --- /dev/null +++ b/internal/e2e/suites/sdsmint/testmain_test.go @@ -0,0 +1,41 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sdsmint + +import ( + "os" + "testing" + + "github.com/agent-substrate/substrate/internal/e2e" +) + +// Setup runs before the suite. The probe is per-test, because it lives in the +// randomized namespace the test creates, so there is nothing to do here. +func Setup() {} + +// Teardown runs after the suite. +func Teardown() {} + +func run(m *testing.M) int { + Setup() + defer Teardown() + + // return allows the deferred Teardown to run. + return e2e.RunTestMain(m) +} + +func TestMain(m *testing.M) { + os.Exit(run(m)) +} diff --git a/internal/e2e/testmain.go b/internal/e2e/testmain.go index a2afad798..9c66c1f49 100644 --- a/internal/e2e/testmain.go +++ b/internal/e2e/testmain.go @@ -19,6 +19,7 @@ import ( goflag "flag" "fmt" "os" + "sync" "testing" "github.com/spf13/pflag" @@ -30,6 +31,28 @@ var ( KubeContext string ) +var ( + suiteCleanupsMu sync.Mutex + suiteCleanups []func() +) + +func RegisterSuiteCleanup(fn func()) { + suiteCleanupsMu.Lock() + defer suiteCleanupsMu.Unlock() + suiteCleanups = append(suiteCleanups, fn) +} + +func runSuiteCleanups() { + suiteCleanupsMu.Lock() + fns := suiteCleanups + suiteCleanups = nil + suiteCleanupsMu.Unlock() + + for i := len(fns) - 1; i >= 0; i-- { + fns[i]() + } +} + func bindFlags() { pflag.BoolVar(&RunE2E, "e2e", false, "run e2e tests") pflag.BoolVar(&NoColor, "no-color", false, "disable colors in output") @@ -82,6 +105,7 @@ func runAndCleanup(m *testing.M) int { // namespace takes those pods with it before anyone — a developer or CI's // post-failure log dump — can read them. code := m.Run() + runSuiteCleanups() if code != 0 { RetainNamespaces() return code diff --git a/internal/localca/localca.go b/internal/localca/localca.go index 3078435ed..72da352c4 100644 --- a/internal/localca/localca.go +++ b/internal/localca/localca.go @@ -18,9 +18,12 @@ package localca import ( "crypto" + "crypto/ecdsa" "crypto/ed25519" + "crypto/elliptic" "crypto/rand" "crypto/x509" + "crypto/x509/pkix" "encoding/json" "encoding/pem" "fmt" @@ -33,11 +36,42 @@ type Pool struct { type CA struct { ID string - SigningKey crypto.PrivateKey + SigningKey crypto.Signer RootCertificate *x509.Certificate IntermediateCertificates []*x509.Certificate } +// Validate reports whether the CA is well formed enough to sign with. +func (ca *CA) Validate() error { + if ca == nil { + return fmt.Errorf("ca: is nil") + } + if ca.RootCertificate == nil { + return fmt.Errorf("ca cert: missing") + } + if ca.SigningKey == nil { + return fmt.Errorf("ca key: missing") + } + if !ca.RootCertificate.IsCA { + return fmt.Errorf("ca cert: %q is not a CA certificate", ca.RootCertificate.Subject) + } + if !keyMatchesCert(ca.RootCertificate, ca.SigningKey) { + return fmt.Errorf("ca key: public key does not match the certificate for %q", ca.RootCertificate.Subject) + } + return nil +} + +// keyMatchesCert catches a mismatched cert/key pair at load rather than at the +// first handshake. +func keyMatchesCert(cert *x509.Certificate, key crypto.Signer) bool { + type equaler interface{ Equal(crypto.PublicKey) bool } + pub, ok := cert.PublicKey.(equaler) + if !ok { + return true + } + return pub.Equal(key.Public()) +} + type serializedPool struct { CAs []*serializedCA } @@ -58,9 +92,15 @@ func Marshal(ca *Pool) ([]byte, error) { caWire.ID = ca.ID + // An external signer has no exportable key material, so this is the + // point where "the key lives in a KMS" stops being marshalable. Name + // that case, because x509's own error ("unknown key type") reads like a + // bug in this code rather than a deliberate property of the signer. signingKeyPKCS8, err := x509.MarshalPKCS8PrivateKey(ca.SigningKey) if err != nil { - return nil, fmt.Errorf("while serializing signing key to PKCS#8: %w", err) + return nil, fmt.Errorf("while serializing signing key for CA %q to PKCS#8: %w "+ + "(a signer that holds no exportable key material, such as a KMS or HSM signer, "+ + "cannot be written to a pool file; keep it in its own store)", ca.ID, err) } caWire.SigningKeyPKCS8 = signingKeyPKCS8 @@ -119,9 +159,13 @@ func Unmarshal(wireBytes []byte) (*Pool, error) { return pool, nil } -func parsePrivateKey(pkcs8 []byte, pemData string) (crypto.PrivateKey, error) { +func parsePrivateKey(pkcs8 []byte, pemData string) (crypto.Signer, error) { if len(pkcs8) != 0 { - return x509.ParsePKCS8PrivateKey(pkcs8) + key, err := x509.ParsePKCS8PrivateKey(pkcs8) + if err != nil { + return nil, err + } + return asSigner(key) } block, _ := pem.Decode([]byte(pemData)) @@ -130,17 +174,28 @@ func parsePrivateKey(pkcs8 []byte, pemData string) (crypto.PrivateKey, error) { } if key, err := x509.ParsePKCS8PrivateKey(block.Bytes); err == nil { - return key, nil + return asSigner(key) } if key, err := x509.ParseECPrivateKey(block.Bytes); err == nil { - return key, nil + return asSigner(key) } if key, err := x509.ParsePKCS1PrivateKey(block.Bytes); err == nil { - return key, nil + return asSigner(key) } return nil, fmt.Errorf("unsupported private key PEM type %q", block.Type) } +// asSigner narrows a parsed key to the signing interface the CAs actually use. +// X25519 keys parse cleanly out of PKCS#8 and cannot sign, so this is a real +// case and not a defensive assertion. +func asSigner(key any) (crypto.Signer, error) { + signer, ok := key.(crypto.Signer) + if !ok { + return nil, fmt.Errorf("private key of type %T cannot sign", key) + } + return signer, nil +} + func parseCertificate(der []byte, pemData string) (*x509.Certificate, error) { if len(der) != 0 { return x509.ParseCertificate(der) @@ -156,14 +211,57 @@ func parseCertificate(der []byte, pemData string) (*x509.Certificate, error) { return x509.ParseCertificate(block.Bytes) } +// KeyType selects the algorithm of a generated CA's signing key. +type KeyType string + +const ( + // KeyTypeED25519 is the default. It is the smallest and fastest option and + // is what substrate's internal CAs have always used. + KeyTypeED25519 KeyType = "ed25519" + // KeyTypeECDSAP256 exists for CAs whose certificates are validated by + // clients outside substrate's control. Ed25519 in a chain needs OpenSSL + // 1.1.1+ or Go 1.13+, which is fine for anything substrate ships and not + // something to assume of an arbitrary process running inside an actor. + KeyTypeECDSAP256 KeyType = "ecdsa-p256" +) + +// GenerateOptions configures GenerateCA. The zero value, apart from ID, +// reproduces what GenerateED25519CA has always produced. +type GenerateOptions struct { + // ID names the CA within its Pool. + ID string + // CommonName is the subject CN. Empty leaves the subject empty, which is + // what the internal CAs do -- nothing authenticates on their name. + CommonName string + // KeyType defaults to KeyTypeED25519. + KeyType KeyType + // Lifetime defaults to 365 days. + Lifetime time.Duration +} + +// GenerateED25519CA creates an unconstrained 365-day Ed25519 CA. It is the +// long-standing shape of substrate's internal CAs, kept as its own function +// because every existing caller wants exactly this. func GenerateED25519CA(id string) (*CA, error) { - rootPubKey, rootPrivKey, err := ed25519.GenerateKey(rand.Reader) + return GenerateCA(GenerateOptions{ID: id}) +} + +// GenerateCA creates a self-signed CA with its own freshly generated key. +func GenerateCA(opts GenerateOptions) (*CA, error) { + if opts.Lifetime == 0 { + opts.Lifetime = 365 * 24 * time.Hour + } + if opts.KeyType == "" { + opts.KeyType = KeyTypeED25519 + } + + rootPrivKey, err := generateKey(opts.KeyType) if err != nil { - return nil, fmt.Errorf("while generating root key: %w", err) + return nil, err } notBefore := time.Now() - notAfter := notBefore.Add(365 * 24 * time.Hour) + notAfter := notBefore.Add(opts.Lifetime) rootTemplate := &x509.Certificate{ NotBefore: notBefore, @@ -172,8 +270,11 @@ func GenerateED25519CA(id string) (*CA, error) { BasicConstraintsValid: true, KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, } + if opts.CommonName != "" { + rootTemplate.Subject = pkix.Name{CommonName: opts.CommonName} + } - rootDER, err := x509.CreateCertificate(rand.Reader, rootTemplate, rootTemplate, rootPubKey, rootPrivKey) + rootDER, err := x509.CreateCertificate(rand.Reader, rootTemplate, rootTemplate, rootPrivKey.Public(), rootPrivKey) if err != nil { return nil, fmt.Errorf("while generating root certificate: %w", err) } @@ -184,9 +285,28 @@ func GenerateED25519CA(id string) (*CA, error) { } return &CA{ - ID: id, + ID: opts.ID, SigningKey: rootPrivKey, RootCertificate: rootCert, // No intermediates. }, nil } + +func generateKey(kt KeyType) (crypto.Signer, error) { + switch kt { + case KeyTypeED25519: + _, key, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + return nil, fmt.Errorf("while generating root key: %w", err) + } + return key, nil + case KeyTypeECDSAP256: + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return nil, fmt.Errorf("while generating root key: %w", err) + } + return key, nil + default: + return nil, fmt.Errorf("unsupported key type %q, want one of %q or %q", kt, KeyTypeED25519, KeyTypeECDSAP256) + } +} diff --git a/internal/localca/localca_test.go b/internal/localca/localca_test.go index 0cca4ef6f..39d92c96c 100644 --- a/internal/localca/localca_test.go +++ b/internal/localca/localca_test.go @@ -16,13 +16,17 @@ package localca import ( "bytes" + "crypto" + "crypto/ecdsa" "crypto/ed25519" + "crypto/elliptic" "crypto/rand" "crypto/rsa" "crypto/x509" "crypto/x509/pkix" "encoding/json" "encoding/pem" + "io" "math/big" "strings" "testing" @@ -298,3 +302,184 @@ func TestUnmarshalErrors(t *testing.T) { }) } } + +// externalSigner stands in for a KMS or HSM signer: it can sign, but it holds +// no exportable key material. +type externalSigner struct{ inner ed25519.PrivateKey } + +func (e externalSigner) Public() crypto.PublicKey { return e.inner.Public() } +func (e externalSigner) Sign(r io.Reader, digest []byte, opts crypto.SignerOpts) ([]byte, error) { + return e.inner.Sign(r, digest, opts) +} + +// The point of typing SigningKey as crypto.Signer is that a key living outside +// the process can be substituted. Verify that actually works end to end: such +// a signer can issue certificates, and Marshal refuses it with an explanation +// rather than x509's "unknown key type". +func TestExternalSignerCanIssueButCannotBeMarshalled(t *testing.T) { + base, err := GenerateED25519CA("external") + if err != nil { + t.Fatalf("GenerateED25519CA: %v", err) + } + ca := &CA{ + ID: base.ID, + SigningKey: externalSigner{inner: base.SigningKey.(ed25519.PrivateKey)}, + RootCertificate: base.RootCertificate, + } + + // Issuing works: x509.CreateCertificate only needs Public and Sign. + leafPub, _, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatalf("generating leaf key: %v", err) + } + leafDER, err := x509.CreateCertificate(rand.Reader, &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "leaf"}, + NotBefore: time.Now().Add(-time.Minute), + NotAfter: time.Now().Add(time.Hour), + }, ca.RootCertificate, leafPub, ca.SigningKey) + if err != nil { + t.Fatalf("signing with an external signer: %v", err) + } + if _, err := x509.ParseCertificate(leafDER); err != nil { + t.Fatalf("parsing the issued leaf: %v", err) + } + + // Serializing does not, and must say why. + _, err = Marshal(&Pool{CAs: []*CA{ca}}) + if err == nil { + t.Fatal("Marshal serialized a signer with no exportable key material") + } + if !strings.Contains(err.Error(), "KMS") { + t.Errorf("error does not explain the external-signer case: %v", err) + } +} + +func TestGenerateCAKeyTypes(t *testing.T) { + for _, tc := range []struct { + keyType KeyType + check func(*testing.T, crypto.Signer) + }{ + {KeyTypeED25519, func(t *testing.T, k crypto.Signer) { + if _, ok := k.(ed25519.PrivateKey); !ok { + t.Errorf("key type = %T, want ed25519.PrivateKey", k) + } + }}, + {KeyTypeECDSAP256, func(t *testing.T, k crypto.Signer) { + ec, ok := k.(*ecdsa.PrivateKey) + if !ok { + t.Fatalf("key type = %T, want *ecdsa.PrivateKey", k) + } + if ec.Curve != elliptic.P256() { + t.Errorf("curve = %v, want P-256", ec.Curve.Params().Name) + } + }}, + } { + t.Run(string(tc.keyType), func(t *testing.T) { + ca, err := GenerateCA(GenerateOptions{ID: "k", KeyType: tc.keyType}) + if err != nil { + t.Fatalf("GenerateCA: %v", err) + } + tc.check(t, ca.SigningKey) + + // Whatever the algorithm, the result has to survive the pool. + data, err := Marshal(&Pool{CAs: []*CA{ca}}) + if err != nil { + t.Fatalf("Marshal: %v", err) + } + restored, err := Unmarshal(data) + if err != nil { + t.Fatalf("Unmarshal: %v", err) + } + tc.check(t, restored.CAs[0].SigningKey) + }) + } + + if _, err := GenerateCA(GenerateOptions{ID: "k", KeyType: "rsa-8192"}); err == nil { + t.Error("GenerateCA accepted an unknown key type") + } +} + +func TestGenerateCALifetimeAndCommonName(t *testing.T) { + ca, err := GenerateCA(GenerateOptions{ID: "x", CommonName: "my ca", Lifetime: 2 * time.Hour}) + if err != nil { + t.Fatalf("GenerateCA: %v", err) + } + if got := ca.RootCertificate.Subject.CommonName; got != "my ca" { + t.Errorf("CN = %q, want %q", got, "my ca") + } + if got := ca.RootCertificate.NotAfter.Sub(ca.RootCertificate.NotBefore); got != 2*time.Hour { + t.Errorf("lifetime = %v, want 2h", got) + } +} + +func TestValidateAcceptsAGeneratedCA(t *testing.T) { + ca, err := GenerateED25519CA("generated") + if err != nil { + t.Fatalf("GenerateED25519CA: %v", err) + } + // GenerateCA is the one path that satisfies Validate by construction. If + // this ever fails the two have drifted apart. + if err := ca.Validate(); err != nil { + t.Errorf("Validate on a freshly generated CA: %v", err) + } + // Intermediates are a supported shape here, whatever individual consumers + // do about them. + ca.IntermediateCertificates = []*x509.Certificate{ca.RootCertificate} + if err := ca.Validate(); err != nil { + t.Errorf("Validate on a CA carrying intermediates: %v", err) + } +} + +func TestValidateRejectsAMalformedCA(t *testing.T) { + good, err := GenerateED25519CA("good") + if err != nil { + t.Fatalf("GenerateED25519CA(good): %v", err) + } + other, err := GenerateED25519CA("other") + if err != nil { + t.Fatalf("GenerateED25519CA(other): %v", err) + } + + // A cert that is not a CA. Signing leaves off it produces a chain that + // nothing will accept as an issuer. + leafKeyPub, leafKeyPriv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatalf("ed25519.GenerateKey(): %v", err) + } + leafTemplate := &x509.Certificate{ + SerialNumber: big.NewInt(1), + NotBefore: time.Now(), + NotAfter: time.Now().Add(24 * time.Hour), + IsCA: false, + BasicConstraintsValid: true, + } + leafDER, err := x509.CreateCertificate(rand.Reader, leafTemplate, good.RootCertificate, leafKeyPub, good.SigningKey) + if err != nil { + t.Fatalf("CreateCertificate(): %v", err) + } + leafCert, err := x509.ParseCertificate(leafDER) + if err != nil { + t.Fatalf("ParseCertificate(): %v", err) + } + + tests := []struct { + name string + ca *CA + }{ + {"nil", nil}, + {"no root certificate", &CA{ID: "x", SigningKey: good.SigningKey}}, + {"no signing key", &CA{ID: "x", RootCertificate: good.RootCertificate}}, + {"root is not a CA", &CA{ID: "x", RootCertificate: leafCert, SigningKey: leafKeyPriv}}, + // The case Unmarshal cannot catch: key and certificate are each well + // formed and are simply not a pair. + {"key does not match the certificate", &CA{ID: "x", RootCertificate: good.RootCertificate, SigningKey: other.SigningKey}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := tt.ca.Validate(); err == nil { + t.Error("Validate() = nil, want error") + } + }) + } +} diff --git a/manifests/ate-install/atenet-egress.yaml b/manifests/ate-install/atenet-egress.yaml index 591ef31fc..72b28588a 100644 --- a/manifests/ate-install/atenet-egress.yaml +++ b/manifests/ate-install/atenet-egress.yaml @@ -35,9 +35,25 @@ metadata: namespace: ate-system data: envoy.yaml: | + # Requires Envoy 1.37 or newer: on_demand_secret and cert_mappers.sni do not + # exist in earlier releases. + # SDS refuses to start without a node id and cluster. + node: + id: atenet-egress + cluster: atenet-egress + admin: address: socket_address: { address: 0.0.0.0, port_value: 15000 } + + # Required by the mitm_listener below. An internal listener has no socket, + # so it is reachable only through a cluster with an envoy_internal_address; + # without this extension registered, that listener silently fails to load. + bootstrap_extensions: + - name: envoy.bootstrap.internal_listener + typed_config: + "@type": type.googleapis.com/envoy.extensions.bootstrap.internal_listener.v3.InternalListener + static_resources: listeners: - name: egress @@ -100,7 +116,12 @@ data: routes: - match: { connect_matcher: {} } route: - cluster: egress_forward_proxy + # the tunnel is handed to the MITM + # listener below, which terminates the tunnelled TLS and + # re-originates it. Sending it straight to + # egress_forward_proxy here would be a raw TCP passthrough + # with no per-destination visibility at all. + cluster: mitm_internal upgrade_configs: - upgrade_type: CONNECT connect_config: {} @@ -137,6 +158,79 @@ data: response_body_mode: NONE request_trailer_mode: SKIP response_trailer_mode: SKIP + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + + # --------------------------------------------------------------------- + # Listener B: the MITM leg. No socket of its own -- it is reachable only + # through the mitm_internal cluster's envoy_internal_address, so it + # cannot be dialled from outside this pod. + # + # It exists because the CONNECT authority is an IP:port (atunnel takes it + # from SO_ORIGINAL_DST and rejects hostnames), so the destination + # hostname is visible only inside the tunnel. This leg terminates the + # tunnelled TLS with a leaf sdsmint mints for the SNI, reads the real + # Host, and re-originates. + # + # Two chains, selected by what the tunnel actually carries. tls_inspector + # tags a ClientHello "tls" and anything else "raw_buffer", so a cleartext + # HTTP tunnel gets an HTTP chain instead of being fed to a TLS transport + # socket, which would fail the handshake and close the tunnel. + # --------------------------------------------------------------------- + - name: mitm_listener + stat_prefix: mitm + internal_listener: {} + listener_filters: + - name: envoy.filters.listener.tls_inspector + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.listener.tls_inspector.v3.TlsInspector + filter_chains: + - filter_chain_match: + transport_protocol: tls + filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: mitm_http + # This is the leg that knows where the traffic actually went. + # REQUESTED_SERVER_NAME is the SNI the leaf was minted for and it + # should always equal the authority -- dynamic_forward_proxy + # resolves from the authority, so a mismatch means the name that + # was policed is not the name that was dialled. + access_log: + - name: envoy.access_loggers.file + typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/stdout + log_format: + json_format: + leg: mitm + time: "%START_TIME%" + sni: "%REQUESTED_SERVER_NAME%" + authority: "%REQ(:AUTHORITY)%" + method: "%REQ(:METHOD)%" + path: "%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%" + protocol: "%PROTOCOL%" + status: "%RESPONSE_CODE%" + flags: "%RESPONSE_FLAGS%" + duration_ms: "%DURATION%" + bytes_sent: "%BYTES_SENT%" + bytes_rcvd: "%BYTES_RECEIVED%" + upstream: "%UPSTREAM_HOST%" + upstream_failure: "%UPSTREAM_TRANSPORT_FAILURE_REASON%" + termination: "%CONNECTION_TERMINATION_DETAILS%" + route_config: + name: local + virtual_hosts: + - name: all + domains: ["*"] + routes: + - match: { prefix: "/" } + route: + cluster: egress_forward_proxy + timeout: 30s + http_filters: - name: envoy.filters.http.dynamic_forward_proxy typed_config: "@type": type.googleapis.com/envoy.extensions.filters.http.dynamic_forward_proxy.v3.FilterConfig @@ -146,7 +240,143 @@ data: - name: envoy.filters.http.router typed_config: "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + # REQUIRED. Without it an sdsmint outage does not fail a + # first-contact handshake, it pauses it forever. + transport_socket_connect_timeout: 5s + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext + common_tls_context: + custom_tls_certificate_selector: + name: on-demand + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.cert_selectors.on_demand_secret.v3.Config + config_source: + api_config_source: + api_type: DELTA_GRPC + transport_api_version: V3 + grpc_services: + - envoy_grpc: + cluster_name: sds_mint + certificate_mapper: + name: sni + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.cert_mappers.sni.v3.SNI + # Reached only by a client that sent no SNI, i.e. one + # dialing by IP. It gets a leaf for this name, which + # resolves nowhere and so matches no destination it + # could have meant -- and since the client prints the + # name in its verification error, the name says why. + default_value: "sni-required.egress.ate.invalid" # default_value is required. + # A resumed session skips certificate selection entirely, which + # would hand a client a secret minted for a different name. + disable_stateless_session_resumption: true + disable_stateful_session_resumption: true + + # The cleartext chain. Nothing to terminate and nothing to mint: the + # Host header is already in the clear, so this leg reads the + # destination directly rather than from a certificate it issued. + - filter_chain_match: + transport_protocol: raw_buffer + filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: mitm_cleartext + access_log: + - name: envoy.access_loggers.file + typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/stdout + log_format: + json_format: + leg: cleartext + time: "%START_TIME%" + authority: "%REQ(:AUTHORITY)%" + method: "%REQ(:METHOD)%" + path: "%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%" + protocol: "%PROTOCOL%" + status: "%RESPONSE_CODE%" + flags: "%RESPONSE_FLAGS%" + duration_ms: "%DURATION%" + bytes_sent: "%BYTES_SENT%" + bytes_rcvd: "%BYTES_RECEIVED%" + upstream: "%UPSTREAM_HOST%" + upstream_failure: "%UPSTREAM_TRANSPORT_FAILURE_REASON%" + termination: "%CONNECTION_TERMINATION_DETAILS%" + route_config: + name: cleartext + virtual_hosts: + - name: all + domains: ["*"] + routes: + - match: { prefix: "/" } + route: + cluster: egress_forward_proxy_cleartext + timeout: 30s + http_filters: + # Same reasoning as the TLS chain: resolve from the request's own + # Host, so the name that was policed is the name that is dialled. + - name: envoy.filters.http.dynamic_forward_proxy + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.dynamic_forward_proxy.v3.FilterConfig + dns_cache_config: + name: egress_dns_cache + dns_lookup_family: V4_ONLY + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + clusters: + # The only way into mitm_listener. An internal address, not a socket, so + # the MITM leg has no listening port anywhere in the pod's netns. + - name: mitm_internal + connect_timeout: 1s + load_assignment: + cluster_name: mitm_internal + endpoints: + - lb_endpoints: + - endpoint: + address: + envoy_internal_address: + server_listener_name: mitm_listener + + - name: sds_mint + type: STATIC + connect_timeout: 1s + # One HTTP/2 connection carrying one DELTA_GRPC stream per name in + # flight. max_requests is the one that matters, and it is subtler than + # a burst limit: Envoy does not close the stream once the secret + # arrives, it holds it open for as long as the secret is live. So this + # is not "concurrent mints", it is a hard cap on the size of the live + # secret set. Past the cap Envoy is refused a mint rather than queued + # for one, which surfaces as a handshake failure and looks exactly like + # the allowlist denying the name -- silently, at a threshold nothing + # else in this file mentions. 32768 is deliberately ~3x the + # 10,000-name target rather than equal to it: the memory guards above + # are what should bind first, at ~13,200, and a limit that trips before + # them would replace a diagnosable overload with a mystery. Raise it + # when you raise the target. + circuit_breakers: + thresholds: + - priority: DEFAULT + max_connections: 128 + max_pending_requests: 32768 + max_requests: 32768 + max_retries: 8 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + explicit_http_config: + http2_protocol_options: {} + load_assignment: + cluster_name: sds_mint + endpoints: + - lb_endpoints: + - endpoint: + address: + pipe: {path: /var/run/sdsmint/sdsmint.sock} # ext_proc gRPC server = the atenet router, co-located in this pod as a # sidecar and called over localhost (same topology the ingress gateway uses # for its dataplane + ext_proc). @@ -168,8 +398,9 @@ data: socket_address: address: 127.0.0.1 port_value: 50051 - # Dials the terminated-CONNECT target (IP:port from the authority). atunnel - # always sends an IP:port, so DNS resolution is effectively a passthrough. + # The only cluster that dials the internet. Reached from the MITM leg, so + # the host it resolves comes from the decrypted request's own Host header + # rather than from the CONNECT authority. - name: egress_forward_proxy lb_policy: CLUSTER_PROVIDED connect_timeout: 5s @@ -180,6 +411,63 @@ data: dns_cache_config: name: egress_dns_cache dns_lookup_family: V4_ONLY + # The MITM must not weaken upstream authentication. Envoy decrypted the + # actor's TLS with a leaf of its own; it still sends the real SNI here + # and still validates the real origin's certificate against the public + # roots. Without this the re-originated request would leave the pod in + # plaintext. + # + # Spelled through typed_extension_protocol_options because the + # cluster-level upstream_http_protocol_options field is deprecated. + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + upstream_http_protocol_options: + auto_sni: true + auto_san_validation: true + explicit_http_config: + http_protocol_options: {} + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + validation_context: + trusted_ca: + filename: /etc/ssl/certs/ca-certificates.crt + + # The cleartext chain's upstream. Identical to egress_forward_proxy but + # without the TLS transport socket: the actor sent plaintext and this + # gateway does not upgrade it. Re-originating over TLS here would mean + # answering a request the actor made in the clear with a connection it + # never asked for and cannot see the peer of. + # + # It shares egress_dns_cache with egress_forward_proxy -- same name, same + # config, so both legs resolve through one cache. + - name: egress_forward_proxy_cleartext + lb_policy: CLUSTER_PROVIDED + connect_timeout: 5s + cluster_type: + name: envoy.clusters.dynamic_forward_proxy + typed_config: + "@type": type.googleapis.com/envoy.extensions.clusters.dynamic_forward_proxy.v3.ClusterConfig + dns_cache_config: + name: egress_dns_cache + dns_lookup_family: V4_ONLY + # Envoy refuses to build a dynamic forward proxy cluster without + # auto_sni and auto_san_validation unless this is set, because for + # the usual TLS case resolving the host from a header and then not + # validating against it is a real hole. There is no upstream TLS on + # this cluster, so there is no SNI to send and no SAN to check -- + # the two settings it is asking for would be no-ops. Setting this + # on egress_forward_proxy, which does re-originate TLS, would be + # the hole it is named after. + allow_insecure_cluster_options: true + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + explicit_http_config: + http_protocol_options: {} --- apiVersion: apps/v1 kind: Deployment @@ -199,15 +487,123 @@ spec: app: atenet-egress spec: serviceAccountName: atenet-egress + # sdsmint and envoy containers must share a UID: sdsmint creates the SDS socket mode + # 0600, and Envoy has to be able to open it. The stock Envoy image runs as + # UID 101 and ko's distroless base as 65532, so left alone they would not + # agree and Envoy would fail every handshake with a connection error on + # the sds_mint cluster. Neither listening port is privileged. securityContext: + runAsUser: 65532 + runAsGroup: 65532 + runAsNonRoot: true + fsGroup: 65532 # Allow the non-root envoy user to bind :443. sysctls: - name: net.ipv4.ip_unprivileged_port_start value: "0" terminationGracePeriodSeconds: 60 + initContainers: + # A native sidecar, not a plain init container. Ordering matters in both + # directions: sdsmint is serving before Envoy loads its config, and it + # outlives Envoy on shutdown, so a drain does not pull the SDS server out + # from under handshakes that are still in flight. + - name: sdsmint + # The same image as the ext-proc container below, run as a different + # subcommand. Sharing the binary is what keeps the two from drifting; + # staying two CONTAINERS is what keeps the MITM signing key off the + # data plane and gives the minter the separate memory limit the header + # of this file depends on. + image: ko://github.com/agent-substrate/substrate/cmd/atenet + restartPolicy: Always + args: + - "sdsmint" + - "--uds-path=/var/run/sdsmint/sdsmint.sock" + - "--ca-pool-path=/run/ca-state/mitm-pool.json" + - "--ca-id=mitm" + # --leaf-cert-ttl sets two things at once. It stamps the leaf's notAfter, + # and sdsmint derives the xDS resource TTL from it at half its length -- + # so at 15m, Envoy drops each cached secret after 7m30s and the next + # handshake for that name re-mints. That drop is the only thing that + # replaces a leaf: nothing pushes, and rotation and the idle sweep are + # both gone. Cost follows traffic, since a name nobody asks for is + # dropped and never minted again. + # + # Do not reach for a longer TTL to cut minting. The two move together, + # and the failure mode at the far end is silent: without the resource + # TTL, Envoy holds a secret indefinitely, serves the leaf past its + # notAfter, and the handshake still completes -- actors cannot object + # because they reach the gateway with verification off, nothing in the + # cluster trusting the MITM anchor. Measured against Envoy 1.37.5 in + # poc/sdsmint/expiry, both with the TTL and without it. + # + # It does not bound validity exactly: leaves are back-dated 5m for + # clock skew and NotAfter is ttl past issuance, so the window a client + # would accept is ttl+5m. See Signer.Sign in + # cmd/atenet/internal/sdsmint/certauth. + - "--leaf-cert-ttl=15m" + - "--log-level=info" + env: + # Go's nearest thing to Envoy's fixed_heap, and it is not as close as + # it looks. It bounds the GARBAGE, not the live set: almost everything + # resident here is reachable -- three goroutine stacks per subscription + # and their timers -- and no amount of GC pressure reclaims a live + # subscription. What it does buy is real though, because minting + # allocates hard, and a soft ceiling meaningfully trims the peak RSS + # that churn produces. Set below the container limit so the + # collector starts working before the kernel starts killing. If sdsmint + # ever reaches this figure with a genuinely live heap it will thrash the + # GC and then OOM anyway; that is the missing shed path described in the + # header, not a misconfiguration here. + - name: GOMEMLIMIT + value: "640MiB" + # Both of these scale with the live secret count, not with request rate. + # + # memory ~44,750 B per live secret x 10,000 => ~427 MiB, against a + # 768Mi limit. That limit is 0.85 + # x Envoy's fixed_heap, which is the ordering invariant from the + # header reduced to one number: at the ~13,200 names where Envoy + # starts refusing, this container is at 73% and still alive. See + # the header for why this is stream overhead and not + # certificates. The request is measured steady state rather than + # a token floor, on purpose -- the scheduler places pods on + # requests, so a 64Mi request on a container that grows to 427 + # MiB does not fail to schedule, it schedules onto a node it + # will later destabilise. Understating it is how this pod takes + # its neighbours down with it. + # cpu ~126 mcore was the measured floor at 10k with no traffic at + # all, when 50 re-mints a second ran on a rotation ticker. That + # steady-state cost is now zero: nothing re-mints, so a name + # costs one signature ever and an idle gateway signs nothing. + # The request is sized for the burst instead. A single mint is + # the ~600us the PoC measured, so a 500/s wave of cold names is + # ~0.3 core -- which is why this is requested and not limited. + # Throttling the signer turns a CPU shortage into handshake + # timeouts across every cold SNI at once. Left at 200m rather + # than trimmed to the new floor because the burst, not the + # steady state, is what this has to absorb. + resources: + requests: + cpu: 200m + memory: 512Mi + limits: + memory: 768Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumeMounts: + - name: sdsmint-sock + mountPath: /var/run/sdsmint + # The MITM CA. Mounted here and nowhere else -- in particular not on + # the Envoy container. Keeping the signing key out of the data plane is + # the reason this is an SDS server rather than a file on disk. + - name: ca-state + mountPath: /run/ca-state + readOnly: true containers: - name: envoy - image: envoyproxy/envoy:v1.34-latest + image: envoyproxy/envoy:v1.37-latest@sha256:1c2b79776c6e3b38e8b0113b825e6a599f9bfc08d680c199d80bf8964856c529 securityContext: allowPrivilegeEscalation: false capabilities: @@ -274,6 +670,8 @@ spec: - name: drain-signal mountPath: /var/run/atenet readOnly: true + - name: sdsmint-sock + mountPath: /var/run/sdsmint # Co-located ext_proc server: the same atenet router binary the ingress # gateway runs, started with --mode=egress so it serves only the egress # ext_proc handler (no xDS server, no ActorTemplate controller, no @@ -342,6 +740,8 @@ spec: name: atenet-egress - name: drain-signal emptyDir: {} + - name: sdsmint-sock + emptyDir: {} - name: servicedns projected: sources: @@ -371,6 +771,14 @@ spec: - name: actor-id-ca-certs secret: secretName: actor-id-ca-certs + - name: ca-state + projected: + sources: + - secret: + name: egress-mitm-ca-pool + items: + - key: pool + path: mitm-pool.json --- apiVersion: v1 kind: Service From fbef52037338c99475fa12025e1ea0b3bcf50599 Mon Sep 17 00:00:00 2001 From: Haiyan Meng Date: Thu, 13 Aug 2026 22:21:54 -0400 Subject: [PATCH 2/2] Address comments --- cmd/atenet/internal/sdsmint/deltastream.go | 49 +++++++++++----------- cmd/atenet/internal/sdsmint/doc.go | 5 +-- cmd/atenet/internal/sdsmint/listen.go | 2 - cmd/atenet/internal/sdsmint/minter.go | 1 + 4 files changed, 26 insertions(+), 31 deletions(-) diff --git a/cmd/atenet/internal/sdsmint/deltastream.go b/cmd/atenet/internal/sdsmint/deltastream.go index caeddfbfd..3e42ca9a4 100644 --- a/cmd/atenet/internal/sdsmint/deltastream.go +++ b/cmd/atenet/internal/sdsmint/deltastream.go @@ -37,10 +37,23 @@ import ( func (s *server) DeltaSecrets(stream secretservice.SecretDiscoveryService_DeltaSecretsServer) error { ctx := stream.Context() + // An arbitrary depth, and not a tuned one. One request yields at most one + // response -- handleSubscribe batches every name in a request into a single + // send -- and the producer signs a leaf per name before it queues anything, + // which costs far more than handing a proto to gRPC's own buffered write + // path. So the queue sits at 0 or 1 and any small number does the same job. + // Only the extremes would change behavior: 0 makes every response a + // synchronous handoff to sendLoop and stalls the select loop on each write, + // which is what the buffer exists to avoid, and unbounded lets a wedged + // stream accumulate every certificate it ever minted. Filling this is not a + // failure either -- send blocks, with a ctx.Done escape, which is the stall + // the buffer defers rather than prevents. + const sendDepth = 8 + st := &deltaStream{ srv: s, stream: stream, - sendCh: make(chan *discovery.DeltaDiscoveryResponse, 8), + sendCh: make(chan *discovery.DeltaDiscoveryResponse, sendDepth), sendErr: make(chan error, 1), sendDone: make(chan struct{}), } @@ -90,21 +103,15 @@ func (s *server) DeltaSecrets(stream secretservice.SecretDiscoveryService_DeltaS } // deltaStream is the per-connection plumbing for a DeltaSecrets stream. -// -// Despite delta xDS being the stateful variant, there is no subscription set -// here. This server tracked one until the resource TTL made it pointless: -// refresh is Envoy's timer now, so nothing on this side ever needs to know what -// a stream is holding. What is left is a send queue, and no shared mutable -// state at all -- handle runs on one goroutine and sendLoop owns the stream. type deltaStream struct { srv *server stream secretservice.SecretDiscoveryService_DeltaSecretsServer - // sendCh queues responses for sendLoop. sendErr carries the first send - // failure back to DeltaSecrets, and sendDone closes once the loop has - // stopped touching the stream. - sendCh chan *discovery.DeltaDiscoveryResponse - sendErr chan error + // sendCh queues responses for sendLoop. + sendCh chan *discovery.DeltaDiscoveryResponse + // sendErr carries the first send failure back to DeltaSecrets. + sendErr chan error + // sendDone closes once the loop has stopped touching the stream. sendDone chan struct{} } @@ -121,8 +128,6 @@ func (d *deltaStream) handle(ctx context.Context, req *discovery.DeltaDiscoveryR return nil } - // initial_resource_versions and resource_names_unsubscribe are read off the - // wire and dropped; see the package comment for why neither needs acting on. return d.handleSubscribe(ctx, req.GetResourceNamesSubscribe()) } @@ -143,7 +148,6 @@ func (d *deltaStream) logNACK(ctx context.Context, req *discovery.DeltaDiscovery // skipped. Envoy re-subscribes in two cases and both want a certificate: after // a resource TTL dropped the secret and a handshake needs it back, and on the // first request of a new stream, where it re-subscribes to everything it holds. -// Suppressing the repeat would leave the second case with nothing. func (d *deltaStream) handleSubscribe(ctx context.Context, names []string) error { if len(names) == 0 { // A bare ACK, or an unsubscribe-only request. Nothing to send. @@ -191,11 +195,10 @@ func (d *deltaStream) pack(name string, cert *certauth.MintedCert) (*discovery.R Version: version, Resource: body, // Envoy starts a timer per resource when it receives one and drops the - // resource when that timer fires. That drop is the whole refresh - // mechanism: the next handshake for the name finds nothing cached and - // re-subscribes, and this server mints again. Stamped on every response - // rather than the first, because a resource that arrives with no ttl - // has its timer cleared and is then held for good. + // resource when that timer fires. The next handshake for the name + // finds nothing cached and re-subscribes, and this server mints again. + // Stamped on every response rather than the first, because a resource + // that arrives with no ttl has its timer cleared and is then held for good. Ttl: durationpb.New(d.srv.resourceTTL), }, nil } @@ -216,11 +219,7 @@ func (d *deltaStream) send(ctx context.Context, resources []*discovery.Resource, } } -// sendLoop drains sendCh onto the stream. gRPC forbids concurrent Send on a -// stream, so everything is funneled through this one goroutine; sendCh's -// buffer is what keeps a slow client from stalling the select loop in -// DeltaSecrets, which would hold up incoming requests behind a write. -// +// sendLoop drains sendCh onto the stream. // It stops on the first send failure and hands it to sendErr, which is what // DeltaSecrets returns; a full sendErr means a failure is already on its way // back, so the second one is dropped rather than blocking the exit. diff --git a/cmd/atenet/internal/sdsmint/doc.go b/cmd/atenet/internal/sdsmint/doc.go index 28314dd98..5930f6efa 100644 --- a/cmd/atenet/internal/sdsmint/doc.go +++ b/cmd/atenet/internal/sdsmint/doc.go @@ -12,8 +12,5 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package sdsmint implements `atenet sdsmint`, a minting SDS server: an Envoy -// Secret Discovery Service that mints a TLS leaf certificate on demand for -// whatever hostname Envoy asks for, where the requested SDS resource name is -// the SNI from the client. +// Package sdsmint implements `atenet sdsmint`, a minting SDS server. package sdsmint diff --git a/cmd/atenet/internal/sdsmint/listen.go b/cmd/atenet/internal/sdsmint/listen.go index f016aace5..0f4dc60bc 100644 --- a/cmd/atenet/internal/sdsmint/listen.go +++ b/cmd/atenet/internal/sdsmint/listen.go @@ -29,8 +29,6 @@ import ( // private keys transit this channel, and a unix socket restricted to the // proxy's UID is the only reach that is ever wanted. func listen(uds string) (net.Listener, error) { - // A stale socket from a previous run would make Listen fail with - // "address already in use" even though nothing is running. if err := os.Remove(uds); err != nil && !errors.Is(err, os.ErrNotExist) { return nil, fmt.Errorf("removing stale socket %s: %w", uds, err) } diff --git a/cmd/atenet/internal/sdsmint/minter.go b/cmd/atenet/internal/sdsmint/minter.go index b300f00a6..8bec1b54d 100644 --- a/cmd/atenet/internal/sdsmint/minter.go +++ b/cmd/atenet/internal/sdsmint/minter.go @@ -118,6 +118,7 @@ func isValidDNSName(host string) bool { // isValidDNSLabel reports whether one dot-separated component is a legal label: // letters, digits, and interior hyphens, up to 63 bytes. +// https://datatracker.ietf.org/doc/html/rfc1035 func isValidDNSLabel(label string) bool { if label == "" || len(label) > 63 { return false