diff --git a/cmd/atenet/internal/root.go b/cmd/atenet/internal/root.go index 53ae85d13..1ab52398a 100644 --- a/cmd/atenet/internal/root.go +++ b/cmd/atenet/internal/root.go @@ -19,6 +19,7 @@ import ( "os" "github.com/agent-substrate/substrate/cmd/atenet/internal/router" + "github.com/agent-substrate/substrate/cmd/atenet/internal/sdsmint" "github.com/agent-substrate/substrate/internal/version" "github.com/spf13/cobra" ) @@ -40,4 +41,5 @@ func Execute() { func init() { rootCmd.AddCommand(router.NewRouterCmd()) rootCmd.AddCommand(NewDnsCmd()) + rootCmd.AddCommand(sdsmint.NewSdsmintCmd()) } diff --git a/cmd/atenet/internal/sdsmint/certauth/certauth.go b/cmd/atenet/internal/sdsmint/certauth/certauth.go new file mode 100644 index 000000000..0c02a94ee --- /dev/null +++ b/cmd/atenet/internal/sdsmint/certauth/certauth.go @@ -0,0 +1,181 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package certauth holds everything in sdsmint that touches the MITM signing +// key: the key itself, the leaf keypair certificates are bound to, and the one +// function that turns a hostname into a certificate for it. +package certauth + +import ( + "crypto" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "encoding/pem" + "errors" + "fmt" + "net" + "time" + + "github.com/agent-substrate/substrate/internal/localca" +) + +// Signer issues leaves for arbitrary hostnames from the MITM CA. +type Signer struct { + pool *localca.Pool + active *localca.CA + + // key is the keypair every leaf this Signer issues is bound to. + key crypto.Signer + keyPEM []byte +} + +// New builds a Signer over pool, signing with the CA named by id. An empty id +// takes the first entry. +// +// The whole pool is retained, not just the selected entry: see the Signer +// fields. +func New(pool *localca.Pool, id string) (*Signer, error) { + active, err := selectCA(pool, id) + if err != nil { + return nil, err + } + if err := active.Validate(); err != nil { + return nil, fmt.Errorf("ca pool: CA %q: %w", active.ID, err) + } + // Generated at startup rather than on first use, so that a mint never pays + // for a keygen and one that cannot succeed fails before the server binds. + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return nil, fmt.Errorf("generating leaf key: %w", err) + } + der, err := x509.MarshalPKCS8PrivateKey(key) + if err != nil { + return nil, fmt.Errorf("marshalling leaf key: %w", err) + } + return &Signer{ + pool: pool, + active: active, + key: key, + keyPEM: pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: der}), + }, nil +} + +// selectCA picks the entry to sign with. An empty id takes the first. +// TODO(haiyanmeng): move this function to internal/localca. +func selectCA(pool *localca.Pool, id string) (*localca.CA, error) { + if pool == nil || len(pool.CAs) == 0 { + return nil, errors.New("ca pool: empty") + } + if id == "" { + return pool.CAs[0], nil + } + ids := make([]string, 0, len(pool.CAs)) + for _, candidate := range pool.CAs { + ids = append(ids, candidate.ID) + if candidate.ID == id { + return candidate, nil + } + } + return nil, fmt.Errorf("ca pool: no CA with ID %q (have %q)", id, ids) +} + +// Issuer returns the certificate this Signer's key belongs to, the one it puts +// directly above each leaf in the chain. +func (s *Signer) Issuer() *x509.Certificate { + return s.active.RootCertificate +} + +// Anchors returns every root in the pool: the full set a client has to trust +// for this Signer's leaves to verify across a rotation, not merely the one +// currently signing. This is what belongs in a trust store or a published +// bundle. +func (s *Signer) Anchors() []*x509.Certificate { + roots := make([]*x509.Certificate, 0, len(s.pool.CAs)) + for _, ca := range s.pool.CAs { + roots = append(roots, ca.RootCertificate) + } + return roots +} + +// MintedCert is a freshly issued leaf plus its private key, in the PEM form +// Envoy's Secret proto expects. +type MintedCert struct { + CertChainPEM []byte // leaf, then the root, PEM + PrivateKeyPEM []byte // leaf private key, PEM + NotAfter time.Time + Serial string // hex, for the issuance audit log +} + +// Sign issues a leaf certificate for host, signed by the CA's root key and +// bound to the Signer's leaf keypair. +func (s *Signer) Sign(host string, ttl time.Duration) (*MintedCert, error) { + if host == "" { + return nil, errors.New("empty host") + } + + ca := s.active + now := time.Now() + + notAfter := now.Add(ttl) + if notAfter.After(ca.RootCertificate.NotAfter) { + notAfter = ca.RootCertificate.NotAfter + } + tmpl := &x509.Certificate{ + NotBefore: now.Add(-5 * time.Minute), + NotAfter: notAfter, + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + IsCA: false, + } + // A literal IP in the SNI position has to land in IPAddresses, not + // DNSNames, or clients will reject the leaf. + // Envoy will hand us whatever was in the SNI, and although SNI is + // not supposed to carry IP literals, some clients send them anyway. + if ip := net.ParseIP(host); ip != nil { + tmpl.IPAddresses = append(tmpl.IPAddresses, ip) + } else { + tmpl.DNSNames = []string{host} + } + + leafDER, err := x509.CreateCertificate(rand.Reader, tmpl, ca.RootCertificate, s.key.Public(), ca.SigningKey) + if err != nil { + return nil, fmt.Errorf("signing leaf for %q: %w", host, err) + } + leaf, err := x509.ParseCertificate(leafDER) + if err != nil { + return nil, fmt.Errorf("parsing the leaf just signed for %q: %w", host, err) + } + + // Leaf first, then its issuer, then whatever climbs from the issuer toward + // the certificate a client is actually configured to trust. RootCertificate + // is the issuer and not necessarily the anchor: localca.Validate pins the + // signing key to it, and IntermediateCertificates is what sits above. + chain := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: leafDER}) + chain = append(chain, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: ca.RootCertificate.Raw})...) + for _, intermediate := range ca.IntermediateCertificates { + chain = append(chain, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: intermediate.Raw})...) + } + + return &MintedCert{ + CertChainPEM: chain, + // Shared with every other leaf this Signer issues, rendered once in + // New. Read-only, like the rest of MintedCert. + PrivateKeyPEM: s.keyPEM, + NotAfter: notAfter, + Serial: leaf.SerialNumber.Text(16), + }, nil +} diff --git a/cmd/atenet/internal/sdsmint/certauth/certauth_test.go b/cmd/atenet/internal/sdsmint/certauth/certauth_test.go new file mode 100644 index 000000000..39363d551 --- /dev/null +++ b/cmd/atenet/internal/sdsmint/certauth/certauth_test.go @@ -0,0 +1,470 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package certauth + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/asn1" + "encoding/pem" + "math/big" + "testing" + "testing/synctest" + "time" + + "github.com/agent-substrate/substrate/internal/localca" +) + +func testCA(t *testing.T) *localca.CA { + t.Helper() + return testRoot(t, "sdsmint test CA", time.Hour) +} + +// testRoot builds a pool entry in the shape sdsmint expects to be handed one. +// P-256 rather than substrate's usual Ed25519: the leaves signed under it are +// validated by whatever HTTP client an actor happens to run, and Ed25519 in a +// chain needs OpenSSL 1.1.1+. +func testRoot(t *testing.T, commonName string, lifetime time.Duration) *localca.CA { + t.Helper() + ca, err := localca.GenerateCA(localca.GenerateOptions{ + ID: "mitm", + CommonName: commonName, + KeyType: localca.KeyTypeECDSAP256, + Lifetime: lifetime, + }) + if err != nil { + t.Fatalf("generating test CA %q: %v", commonName, err) + } + return ca +} + +// inPool wraps entries the way a mounted pool Secret presents them. +func inPool(t *testing.T, id string, entries ...*localca.CA) *Signer { + t.Helper() + signer, err := New(&localca.Pool{CAs: entries}, id) + if err != nil { + t.Fatalf("New(%q): %v", id, err) + } + return signer +} + +// testSigner builds a Signer over a single-entry pool. +func testSigner(t *testing.T, ca *localca.CA) *Signer { + t.Helper() + signer, err := New(&localca.Pool{CAs: []*localca.CA{ca}}, "") + if err != nil { + t.Fatalf("New: %v", err) + } + return signer +} + +// oidSubjectAltName is id-ce-subjectAltName, RFC 5280 section 4.2.1.6. +var oidSubjectAltName = asn1.ObjectIdentifier{2, 5, 29, 17} + +func hasCriticalSAN(t *testing.T, cert *x509.Certificate) bool { + t.Helper() + for _, ext := range cert.Extensions { + if ext.Id.Equal(oidSubjectAltName) { + return ext.Critical + } + } + return false +} + +// parseChain splits a PEM chain into leaf and root. +func parseChain(t *testing.T, chainPEM []byte) []*x509.Certificate { + t.Helper() + var certs []*x509.Certificate + rest := chainPEM + for { + var block *pem.Block + block, rest = pem.Decode(rest) + if block == nil { + break + } + c, err := x509.ParseCertificate(block.Bytes) + if err != nil { + t.Fatalf("parsing chain: %v", err) + } + certs = append(certs, c) + } + return certs +} + +func TestSignProducesUsableLeaf(t *testing.T) { + ca := testCA(t) + + minted, err := testSigner(t, ca).Sign("foo.example", 5*time.Minute) + if err != nil { + t.Fatalf("Sign: %v", err) + } + + chain := parseChain(t, minted.CertChainPEM) + if len(chain) != 2 { + t.Fatalf("chain length = %d, want 2 (leaf + CA)", len(chain)) + } + leaf, root := chain[0], chain[1] + + if got := leaf.DNSNames; len(got) != 1 || got[0] != "foo.example" { + t.Errorf("leaf DNSNames = %v, want [foo.example]", got) + } + // No subject at all: the name lives in the SAN, which is the only place + // clients read it from. An empty subject obliges the SAN to be critical. + if got := leaf.Subject.String(); got != "" { + t.Errorf("leaf subject = %q, want empty", got) + } + if !hasCriticalSAN(t, leaf) { + t.Error("leaf has an empty subject but a non-critical SAN, which RFC 5280 forbids") + } + if leaf.SerialNumber == nil || leaf.SerialNumber.Sign() <= 0 { + t.Errorf("leaf serial = %v, want a positive number", leaf.SerialNumber) + } + if leaf.IsCA { + t.Error("leaf is marked as a CA") + } + if leaf.KeyUsage != x509.KeyUsageDigitalSignature { + t.Errorf("leaf KeyUsage = %v, want DigitalSignature only", leaf.KeyUsage) + } + if len(leaf.ExtKeyUsage) != 1 || leaf.ExtKeyUsage[0] != x509.ExtKeyUsageServerAuth { + t.Errorf("leaf EKU = %v, want [ServerAuth]", leaf.ExtKeyUsage) + } + if !root.Equal(ca.RootCertificate) { + t.Error("chain does not end in the CA certificate") + } + + // The whole point is that a normal TLS client accepts this, so verify the + // way one would. + pool := x509.NewCertPool() + pool.AddCert(ca.RootCertificate) + if _, err := leaf.Verify(x509.VerifyOptions{ + DNSName: "foo.example", + Roots: pool, + KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + }); err != nil { + t.Errorf("leaf does not verify against the CA: %v", err) + } + + // And the key must actually match the certificate. + if _, err := tls.X509KeyPair(minted.CertChainPEM, minted.PrivateKeyPEM); err != nil { + t.Errorf("minted chain/key is not a usable TLS keypair: %v", err) + } +} + +func TestSignIsUniquePerCall(t *testing.T) { + signer := testSigner(t, testCA(t)) + + a, err := signer.Sign("a.example", time.Minute) + if err != nil { + t.Fatalf("Sign a: %v", err) + } + b, err := signer.Sign("a.example", time.Minute) + if err != nil { + t.Fatalf("Sign a again: %v", err) + } + + // The certificate is new every time, which is what the SDS layer reads as + // a version and what the audit log names the issuance by. + if a.Serial == b.Serial { + t.Error("two mints for the same host reused a serial number") + } + // The keypair, by contrast, is shared on purpose. Asserted + // rather than left implicit, because the saving disappears silently the + // moment a mint starts generating its own again. + if string(a.PrivateKeyPEM) != string(b.PrivateKeyPEM) { + t.Error("two mints generated separate private keys; leaves are meant to share one") + } +} + +func TestSignHonoursTTL(t *testing.T) { + // On the fake clock nothing advances between the read and the mint, so the + // deadline is exact rather than a tolerance around real elapsed time. + synctest.Test(t, func(t *testing.T) { + ca := testCA(t) + before := time.Now() + + minted, err := testSigner(t, ca).Sign("ttl.example", 90*time.Second) + if err != nil { + t.Fatalf("Sign: %v", err) + } + + if want := before.Add(90 * time.Second); !minted.NotAfter.Equal(want) { + t.Errorf("NotAfter = %v, want %v", minted.NotAfter, want) + } + }) +} + +func TestSignIPLiteralGoesInSANIPAddresses(t *testing.T) { + ca := testCA(t) + + minted, err := testSigner(t, ca).Sign("10.1.2.3", time.Minute) + if err != nil { + t.Fatalf("Sign: %v", err) + } + leaf := parseChain(t, minted.CertChainPEM)[0] + + if len(leaf.IPAddresses) != 1 || leaf.IPAddresses[0].String() != "10.1.2.3" { + t.Errorf("leaf IPAddresses = %v, want [10.1.2.3]", leaf.IPAddresses) + } + if len(leaf.DNSNames) != 0 { + t.Errorf("leaf DNSNames = %v, want empty for an IP literal", leaf.DNSNames) + } +} + +func TestSignRejectsEmptyHost(t *testing.T) { + if _, err := testSigner(t, testCA(t)).Sign("", time.Minute); err == nil { + t.Fatal("Sign(\"\") succeeded, want an error") + } +} + +func TestNewRejectsANonCACertificate(t *testing.T) { + ca := testCA(t) + // A leaf is not a CA; a pool carrying one must be refused rather than + // silently produce a signer that emits certificates nothing will chain. + minted, err := testSigner(t, ca).Sign("leaf.example", time.Minute) + if err != nil { + t.Fatalf("Sign: %v", err) + } + leaf := parseChain(t, minted.CertChainPEM)[0] + + if _, err := New(&localca.Pool{CAs: []*localca.CA{ + {ID: "mitm", RootCertificate: leaf, SigningKey: ca.SigningKey}, + }}, ""); err == nil { + t.Fatal("New accepted a non-CA certificate") + } +} + +func TestNewAcceptsAnUnconstrainedCA(t *testing.T) { + signer := inPool(t, "", testRoot(t, "wide open", time.Hour)) + + if got := signer.Issuer().PermittedDNSDomains; len(got) != 0 { + t.Errorf("PermittedDNSDomains = %v, want none", got) + } + + // And it signs. Refusing at load and then failing at the first handshake + // would be the same outage with a worse error message. + if _, err := signer.Sign("anything.example", time.Minute); err != nil { + t.Errorf("Sign under an unconstrained root: %v", err) + } +} + +func TestNewRejectsAMismatchedKey(t *testing.T) { + a := testRoot(t, "a", time.Hour) + b := testRoot(t, "b", time.Hour) + + // Signing with the wrong key produces a chain nothing can verify, and the + // failure otherwise surfaces at the first handshake rather than at load. + // localca.CA.Validate is what catches it; this pins that New asks. + mismatched := &localca.CA{ID: "mismatched", RootCertificate: a.RootCertificate, SigningKey: b.SigningKey} + if _, err := New(&localca.Pool{CAs: []*localca.CA{mismatched}}, ""); err == nil { + t.Fatal("New accepted a key that does not match the certificate") + } +} + +// A pool entry may carry certificates that climb from its signing certificate +// up to whatever a client is configured to trust. Those have to travel with the +// leaf: a client holding only the top of that chain cannot build a path to a +// leaf issued two levels down, so omitting them fails the handshake. +func TestSignEmitsIntermediatesInTheChain(t *testing.T) { + anchor := testRoot(t, "anchor", 24*time.Hour) + + // An issuing CA under the anchor, with its own key. This is the shape + // RootCertificate names: the certificate whose key signs leaves, which here + // is not the trust anchor. + issuerKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("generating issuer key: %v", err) + } + issuerTmpl := &x509.Certificate{ + SerialNumber: big.NewInt(2), + Subject: pkix.Name{CommonName: "issuing CA"}, + NotBefore: time.Now().Add(-time.Minute), + NotAfter: time.Now().Add(12 * time.Hour), + IsCA: true, + BasicConstraintsValid: true, + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature, + } + issuerDER, err := x509.CreateCertificate(rand.Reader, issuerTmpl, anchor.RootCertificate, issuerKey.Public(), anchor.SigningKey) + if err != nil { + t.Fatalf("signing issuing CA: %v", err) + } + issuer, err := x509.ParseCertificate(issuerDER) + if err != nil { + t.Fatalf("parsing issuing CA: %v", err) + } + + signer := testSigner(t, &localca.CA{ + ID: "delegated", + RootCertificate: issuer, + SigningKey: issuerKey, + IntermediateCertificates: []*x509.Certificate{anchor.RootCertificate}, + }) + + minted, err := signer.Sign("host.example", time.Hour) + if err != nil { + t.Fatalf("Sign: %v", err) + } + + chain := parseChain(t, minted.CertChainPEM) + if len(chain) != 3 { + t.Fatalf("chain holds %d certificates, want leaf, issuer and anchor", len(chain)) + } + if got := chain[1].Subject.CommonName; got != "issuing CA" { + t.Errorf("chain[1] CN = %q, want the issuing CA", got) + } + if got := chain[2].Subject.CommonName; got != "anchor" { + t.Errorf("chain[2] CN = %q, want the anchor", got) + } + + // The point of carrying them: a client that trusts only the anchor can + // still build a path to the leaf out of what the handshake delivered. + roots := x509.NewCertPool() + roots.AddCert(anchor.RootCertificate) + intermediates := x509.NewCertPool() + for _, c := range chain[1:] { + intermediates.AddCert(c) + } + if _, err := chain[0].Verify(x509.VerifyOptions{ + DNSName: "host.example", + Roots: roots, + Intermediates: intermediates, + KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + }); err != nil { + t.Errorf("verifying the leaf against the anchor alone: %v", err) + } +} + +func TestNewRoundTrip(t *testing.T) { + entry := testRoot(t, "pooled CA", time.Hour) + + // Through the same serialization podcertcontroller's CAs use. + poolBytes, err := localca.Marshal(&localca.Pool{CAs: []*localca.CA{entry}}) + if err != nil { + t.Fatalf("Marshal: %v", err) + } + pool, err := localca.Unmarshal(poolBytes) + if err != nil { + t.Fatalf("Unmarshal: %v", err) + } + + signer, err := New(pool, "") + if err != nil { + t.Fatalf("New: %v", err) + } + minted, err := signer.Sign("host.example", time.Minute) + if err != nil { + t.Fatalf("Sign: %v", err) + } + + roots := x509.NewCertPool() + roots.AddCert(signer.Issuer()) + if _, err := parseChain(t, minted.CertChainPEM)[0].Verify(x509.VerifyOptions{ + DNSName: "host.example", Roots: roots, + KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + }); err != nil { + t.Errorf("leaf from a pooled CA does not verify: %v", err) + } +} + +func TestNewSelectsByID(t *testing.T) { + var entries []*localca.CA + for _, id := range []string{"first", "second"} { + entry := testRoot(t, id, time.Hour) + entry.ID = id + entries = append(entries, entry) + } + pool := &localca.Pool{CAs: entries} + + signer, err := New(pool, "second") + if err != nil { + t.Fatalf("New: %v", err) + } + if got := signer.Issuer().Subject.CommonName; got != "second" { + t.Errorf("selected CA CN = %q, want %q", got, "second") + } + + // An empty ID takes the first, which is what a single-CA pool relies on. + signer, err = New(pool, "") + if err != nil { + t.Fatalf("New(''): %v", err) + } + if got := signer.Issuer().Subject.CommonName; got != "first" { + t.Errorf("default CA CN = %q, want %q", got, "first") + } + + // A typo in --ca-id must not silently fall back to some other CA. + if _, err := New(pool, "third"); err == nil { + t.Fatal("New accepted an unknown CA ID") + } +} + +// TestAnchorsCoversTheWholePool is the distinction the two accessors exist to +// make. During a rotation the pool holds the outgoing and incoming CA at once +// while only one of them signs, so a trust store built from Issuer would reject +// every leaf minted the moment --ca-id moved. +func TestAnchorsCoversTheWholePool(t *testing.T) { + var entries []*localca.CA + for _, id := range []string{"outgoing", "incoming"} { + entry := testRoot(t, id, time.Hour) + entry.ID = id + entries = append(entries, entry) + } + signer := inPool(t, "outgoing", entries...) + + anchors := signer.Anchors() + if len(anchors) != 2 { + t.Fatalf("Anchors returned %d certificates, want one per CA in the pool", len(anchors)) + } + for i, entry := range entries { + if !anchors[i].Equal(entry.RootCertificate) { + t.Errorf("anchor %d is not the root of CA %q", i, entry.ID) + } + } + + // And the CA not signing is still in there, which is the whole point. + if anchors[1].Equal(signer.Issuer()) { + t.Error("the incoming CA was reported as the issuer; only the outgoing one signs") + } +} + +func TestNewRejectsAnEmptyPool(t *testing.T) { + if _, err := New(&localca.Pool{}, ""); err == nil { + t.Fatal("New accepted an empty pool") + } + if _, err := New(nil, ""); err == nil { + t.Fatal("New accepted a nil pool") + } +} + +func TestSignClampsLeafLifetimeToTheRoot(t *testing.T) { + // A leaf outliving its issuer is accepted at handshake time and rejected + // later, with an error that points at the leaf rather than at the CA. + ca := testRoot(t, "short-lived CA", 2*time.Minute) + rootNotAfter := ca.RootCertificate.NotAfter + + minted, err := testSigner(t, ca).Sign("long.example", time.Hour) + if err != nil { + t.Fatalf("Sign: %v", err) + } + if minted.NotAfter.After(rootNotAfter) { + t.Errorf("leaf NotAfter = %v, outlives its issuer at %v", minted.NotAfter, rootNotAfter) + } + if leaf := parseChain(t, minted.CertChainPEM)[0]; leaf.NotAfter.After(rootNotAfter) { + t.Errorf("encoded leaf NotAfter = %v, outlives its issuer at %v", leaf.NotAfter, rootNotAfter) + } +} diff --git a/cmd/atenet/internal/sdsmint/cmd.go b/cmd/atenet/internal/sdsmint/cmd.go new file mode 100644 index 000000000..84b9591d6 --- /dev/null +++ b/cmd/atenet/internal/sdsmint/cmd.go @@ -0,0 +1,73 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sdsmint + +import ( + "fmt" + "time" + + "github.com/spf13/cobra" +) + +type config struct { + UDSPath string + CAPoolPath string + CAID string + LeafCertTTL time.Duration + LogLevel string +} + +func NewSdsmintCmd() *cobra.Command { + var cfg config + + cmd := &cobra.Command{ + Use: "sdsmint", + Short: "Minting SDS server that issues a TLS leaf for the SNI Envoy asks for", + RunE: func(cmd *cobra.Command, _ []string) error { + return run(cmd.Context(), cfg) + }, + } + + cmd.Flags().StringVar(&cfg.UDSPath, "uds-path", "", "unix socket to listen on; required, and the only transport offered, because leaf private keys transit this channel") + cmd.Flags().StringVar(&cfg.CAPoolPath, "ca-pool-path", "", "path to a localca pool JSON holding the MITM CA, the format substrate mounts its other CAs in") + cmd.Flags().StringVar(&cfg.CAID, "ca-id", "", "which CA in the pool to sign with; empty takes the first") + cmd.Flags().DurationVar(&cfg.LeafCertTTL, "leaf-cert-ttl", defaultTTL, "leaf certificate lifetime; the xDS resource TTL is derived from it at half its length, so an actively used name is re-minted about twice per lifetime and an idle one is dropped and not minted again") + cmd.Flags().StringVar(&cfg.LogLevel, "log-level", "info", "one of debug, info, warn, error") + + return cmd +} + +func (c config) validateTTL() error { + ttl := c.LeafCertTTL + if ttl <= 0 { + return fmt.Errorf("--leaf-cert-ttl must be positive, got %s", ttl) + } + + // The band outside which --leaf-cert-ttl is refused. Both edges are cases + // where the flag still starts a server but stops describing what it does, + // which is worse than not starting: the operator has no signal that the + // number they set is not the number in effect. + const ( + minSensibleTTL = time.Minute + maxSensibleTTL = 24 * time.Hour + ) + switch { + case ttl < minSensibleTTL: + return fmt.Errorf("--leaf-cert-ttl %s is below %s; leaves are back-dated 5m for clock skew, so most of each certificate's validity would already be in the past and --leaf-cert-ttl would not describe how long clients accept it", ttl, minSensibleTTL) + case ttl > maxSensibleTTL: + return fmt.Errorf("--leaf-cert-ttl %s is above %s; a leaf is served until the resource TTL derived from this drops it, so a name in steady use would carry the same certificate for half of that, which is not what short-lived MITM leaves are for", ttl, maxSensibleTTL) + } + return nil +} diff --git a/cmd/atenet/internal/sdsmint/cmd_test.go b/cmd/atenet/internal/sdsmint/cmd_test.go new file mode 100644 index 000000000..df8ddfebe --- /dev/null +++ b/cmd/atenet/internal/sdsmint/cmd_test.go @@ -0,0 +1,69 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sdsmint + +import ( + "testing" + "time" +) + +func TestValidateTTL(t *testing.T) { + for _, tc := range []struct { + name string + ttl time.Duration + wantErr bool + }{ + {name: "default", ttl: defaultTTL}, + {name: "deployed value", ttl: 15 * time.Minute}, + {name: "the load-test value", ttl: 5 * time.Minute}, + // The edges of validateTTL's accepted band, spelled out because the + // bounds are local to it. Both are inclusive. + {name: "at the band floor", ttl: time.Minute}, + {name: "at the band ceiling", ttl: 24 * time.Hour}, + + // The whole point of the exercise: --leaf-cert-ttl=0 used to start a server + // that logged 0 and issued defaultTTL leaves. + {name: "zero", ttl: 0, wantErr: true}, + {name: "negative", ttl: -time.Minute, wantErr: true}, + + // Outside the band. These used to start the server with a warning; a + // TTL that no longer means what it says is now refused outright. + {name: "short", ttl: 30 * time.Second, wantErr: true}, + {name: "long", ttl: 48 * time.Hour, wantErr: true}, + } { + t.Run(tc.name, func(t *testing.T) { + err := config{LeafCertTTL: tc.ttl}.validateTTL() + if gotErr := err != nil; gotErr != tc.wantErr { + t.Errorf("validateTTL(ttl=%s) = %v; want an error = %v", tc.ttl, err, tc.wantErr) + } + }) + } +} + +// TestDefaultTTLIsTheFlagDefault guards the trap this validation was added +// for: a fallback and a flag default that disagree, so the lifetime depends on +// which path you came in through. +func TestDefaultTTLIsTheFlagDefault(t *testing.T) { + flag := NewSdsmintCmd().Flags().Lookup("leaf-cert-ttl") + if flag == nil { + t.Fatal("no --leaf-cert-ttl flag") + } + if got, want := flag.DefValue, defaultTTL.String(); got != want { + t.Errorf("--leaf-cert-ttl default = %q; want %q, the same constant newMinter and newServer fall back to", got, want) + } + if err := (config{LeafCertTTL: defaultTTL}).validateTTL(); err != nil { + t.Errorf("validateTTL(defaultTTL) = %v; the default has to be a value run will start with", err) + } +} diff --git a/cmd/atenet/internal/sdsmint/deltastream.go b/cmd/atenet/internal/sdsmint/deltastream.go new file mode 100644 index 000000000..3e42ca9a4 --- /dev/null +++ b/cmd/atenet/internal/sdsmint/deltastream.go @@ -0,0 +1,245 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Delta SDS: the stateful, per-connection half of the server. +// - https://www.envoyproxy.io/docs/envoy/latest/api-docs/xds_protocol#xds-protocol-delta +// - https://www.envoyproxy.io/docs/envoy/latest/configuration/security/secret +package sdsmint + +import ( + "context" + "errors" + "fmt" + "io" + "log/slog" + + discovery "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + secretservice "github.com/envoyproxy/go-control-plane/envoy/service/secret/v3" + "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/durationpb" + + "github.com/agent-substrate/substrate/cmd/atenet/internal/sdsmint/certauth" +) + +// DeltaSecrets is what DELTA_GRPC drives. It is a long-lived loop that mints +// incrementally rather than serving a fixed snapshot. +func (s *server) DeltaSecrets(stream secretservice.SecretDiscoveryService_DeltaSecretsServer) error { + ctx := stream.Context() + + // An arbitrary depth, and not a tuned one. One request yields at most one + // response -- handleSubscribe batches every name in a request into a single + // send -- and the producer signs a leaf per name before it queues anything, + // which costs far more than handing a proto to gRPC's own buffered write + // path. So the queue sits at 0 or 1 and any small number does the same job. + // Only the extremes would change behavior: 0 makes every response a + // synchronous handoff to sendLoop and stalls the select loop on each write, + // which is what the buffer exists to avoid, and unbounded lets a wedged + // stream accumulate every certificate it ever minted. Filling this is not a + // failure either -- send blocks, with a ctx.Done escape, which is the stall + // the buffer defers rather than prevents. + const sendDepth = 8 + + st := &deltaStream{ + srv: s, + stream: stream, + sendCh: make(chan *discovery.DeltaDiscoveryResponse, sendDepth), + sendErr: make(chan error, 1), + sendDone: make(chan struct{}), + } + + go st.sendLoop(ctx) + + recvCh := make(chan *discovery.DeltaDiscoveryRequest) + recvErrCh := make(chan error, 1) + go func() { + for { + req, err := stream.Recv() + if err != nil { + recvErrCh <- err + return + } + select { + case recvCh <- req: + case <-ctx.Done(): + return + } + } + }() + + // Wait for send thread to finish before exiting. + defer func() { + close(st.sendCh) + <-st.sendDone + }() + + for { + select { + case <-ctx.Done(): + return nil + case err := <-st.sendErr: + return err + case err := <-recvErrCh: + if errors.Is(err, io.EOF) { + return nil + } + return err + case req := <-recvCh: + if err := st.handle(ctx, req); err != nil { + return err + } + } + } +} + +// deltaStream is the per-connection plumbing for a DeltaSecrets stream. +type deltaStream struct { + srv *server + stream secretservice.SecretDiscoveryService_DeltaSecretsServer + + // sendCh queues responses for sendLoop. + sendCh chan *discovery.DeltaDiscoveryResponse + // sendErr carries the first send failure back to DeltaSecrets. + sendErr chan error + // sendDone closes once the loop has stopped touching the stream. + sendDone chan struct{} +} + +// handle applies one request from the client. +func (d *deltaStream) handle(ctx context.Context, req *discovery.DeltaDiscoveryRequest) error { + if url := req.GetTypeUrl(); url != "" && url != secretTypeURL { + return fmt.Errorf("unexpected type_url %q on the SDS stream", url) + } + + // A request carrying error_detail is a NACK of whatever we last sent, and + // only that: it brings no subscription changes to apply. + if req.GetErrorDetail() != nil { + d.logNACK(ctx, req) + return nil + } + + return d.handleSubscribe(ctx, req.GetResourceNamesSubscribe()) +} + +// logNACK records that Envoy rejected the last response. Nothing is resent: the +// server has no second thing to offer for the name, and a retry loop against a +// client that is rejecting on principle is worse than the failure. +func (d *deltaStream) logNACK(ctx context.Context, req *discovery.DeltaDiscoveryRequest) { + ed := req.GetErrorDetail() + d.srv.log.ErrorContext(ctx, "envoy NACKed an SDS response", + slog.String("message", ed.GetMessage()), + slog.Int("code", int(ed.GetCode())), + slog.String("nonce", req.GetResponseNonce()), + ) +} + +// handleSubscribe mints a leaf for every name the client asked for and sends +// the batch. A name this stream already holds is minted again rather than +// skipped. Envoy re-subscribes in two cases and both want a certificate: after +// a resource TTL dropped the secret and a handshake needs it back, and on the +// first request of a new stream, where it re-subscribes to everything it holds. +func (d *deltaStream) handleSubscribe(ctx context.Context, names []string) error { + if len(names) == 0 { + // A bare ACK, or an unsubscribe-only request. Nothing to send. + return nil + } + + var resources []*discovery.Resource + var removed []string + + for _, name := range names { + cert, err := d.srv.minter.certificate(ctx, name) + if err != nil { + // Refused. Tell Envoy the name does not exist; the paused + // handshake for that SNI then fails, which is the intended + // outcome for something that is not a hostname. + removed = append(removed, name) + continue + } + res, err := d.pack(name, cert) + if err != nil { + return err + } + resources = append(resources, res) + } + + if len(resources) == 0 && len(removed) == 0 { + return nil + } + return d.send(ctx, resources, removed) +} + +// pack wraps a minted cert as a versioned delta Resource. +func (d *deltaStream) pack(name string, cert *certauth.MintedCert) (*discovery.Resource, error) { + secret := toSecret(name, cert) + body, err := anypb.New(secret) + if err != nil { + return nil, fmt.Errorf("marshalling secret for %q: %w", name, err) + } + // The serial changes on every mint, so it is a natural resource version: + // every mint looks like a new version to Envoy. + version := cert.Serial + + return &discovery.Resource{ + Name: name, + Version: version, + Resource: body, + // Envoy starts a timer per resource when it receives one and drops the + // resource when that timer fires. The next handshake for the name + // finds nothing cached and re-subscribes, and this server mints again. + // Stamped on every response rather than the first, because a resource + // that arrives with no ttl has its timer cleared and is then held for good. + Ttl: durationpb.New(d.srv.resourceTTL), + }, nil +} + +func (d *deltaStream) send(ctx context.Context, resources []*discovery.Resource, removed []string) error { + resp := &discovery.DeltaDiscoveryResponse{ + TypeUrl: secretTypeURL, + Resources: resources, + RemovedResources: removed, + Nonce: d.srv.nextNonce(), + } + + select { + case d.sendCh <- resp: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +// sendLoop drains sendCh onto the stream. +// It stops on the first send failure and hands it to sendErr, which is what +// DeltaSecrets returns; a full sendErr means a failure is already on its way +// back, so the second one is dropped rather than blocking the exit. +func (d *deltaStream) sendLoop(ctx context.Context) { + defer close(d.sendDone) + for { + select { + case <-ctx.Done(): + return + case resp, ok := <-d.sendCh: + if !ok { + return + } + if err := d.stream.Send(resp); err != nil { + select { + case d.sendErr <- err: + default: + } + return + } + } + } +} diff --git a/cmd/atenet/internal/sdsmint/deltastream_test.go b/cmd/atenet/internal/sdsmint/deltastream_test.go new file mode 100644 index 000000000..7300f4642 --- /dev/null +++ b/cmd/atenet/internal/sdsmint/deltastream_test.go @@ -0,0 +1,530 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sdsmint + +import ( + "context" + "crypto/tls" + "crypto/x509" + "encoding/pem" + "io" + "testing" + "testing/synctest" + "time" + + tlsv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" + discovery "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + rpcstatus "google.golang.org/genproto/googleapis/rpc/status" + "google.golang.org/grpc/metadata" +) + +// fakeDeltaStream stands in for the gRPC stream Envoy would be on the other +// end of. Requests are fed in from a slice; responses are collected. +type fakeDeltaStream struct { + ctx context.Context + requests chan *discovery.DeltaDiscoveryRequest + sent chan *discovery.DeltaDiscoveryResponse +} + +func newFakeDeltaStream(ctx context.Context) *fakeDeltaStream { + return &fakeDeltaStream{ + ctx: ctx, + requests: make(chan *discovery.DeltaDiscoveryRequest, 8), + sent: make(chan *discovery.DeltaDiscoveryResponse, 8), + } +} + +func (f *fakeDeltaStream) Send(resp *discovery.DeltaDiscoveryResponse) error { + select { + case f.sent <- resp: + return nil + case <-f.ctx.Done(): + return f.ctx.Err() + } +} + +func (f *fakeDeltaStream) Recv() (*discovery.DeltaDiscoveryRequest, error) { + select { + case req, ok := <-f.requests: + if !ok { + return nil, io.EOF + } + return req, nil + case <-f.ctx.Done(): + return nil, io.EOF + } +} + +func (f *fakeDeltaStream) Context() context.Context { return f.ctx } +func (f *fakeDeltaStream) SetHeader(metadata.MD) error { return nil } +func (f *fakeDeltaStream) SendHeader(metadata.MD) error { return nil } +func (f *fakeDeltaStream) SetTrailer(metadata.MD) {} +func (f *fakeDeltaStream) SendMsg(any) error { return nil } +func (f *fakeDeltaStream) RecvMsg(any) error { return nil } + +// respondWait bounds a wait for something the server should do promptly. Every +// caller runs inside a synctest bubble, so this is fake time: only a run that +// is already failing ever spends it. +const respondWait = time.Minute + +// nextResponse waits for one response, failing the test if none arrives. +func (f *fakeDeltaStream) nextResponse(t *testing.T) *discovery.DeltaDiscoveryResponse { + t.Helper() + select { + case resp := <-f.sent: + return resp + case <-time.After(respondWait): + t.Fatal("timed out waiting for a DeltaDiscoveryResponse") + return nil + } +} + +// quiet blocks until the server has nothing left to do, then fails if it sent +// anything. This is the negative assertion the whole file used to spell as a +// sleep: synctest.Wait returns once every goroutine is durably blocked, so +// anything the server meant to send is already in the channel by then. +func (f *fakeDeltaStream) quiet(t *testing.T, whileDoing string) { + t.Helper() + synctest.Wait() + select { + case resp := <-f.sent: + t.Fatalf("server sent %v %s", resourceNames(resp), whileDoing) + default: + } +} + +func testServer(t *testing.T, opts serverOptions) *server { + t.Helper() + if opts.Logger == nil { + opts.Logger = quietLogger() + } + m := testMinter(t, minterOptions{TTL: defaultTTL}) + return newServer(m, opts) +} + +// startServer runs DeltaSecrets against a fake stream and returns the stream +// plus a func that stops it and reports the server's error. +func startServer(t *testing.T, srv *server) (*fakeDeltaStream, func() error) { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + stream := newFakeDeltaStream(ctx) + + done := make(chan error, 1) + go func() { done <- srv.DeltaSecrets(stream) }() + + return stream, func() error { + cancel() + select { + case err := <-done: + return err + case <-time.After(respondWait): + t.Fatal("DeltaSecrets did not return after the stream was cancelled") + return nil + } + } +} + +func resourceNames(resp *discovery.DeltaDiscoveryResponse) []string { + names := make([]string, 0, len(resp.GetResources())) + for _, r := range resp.GetResources() { + names = append(names, r.GetName()) + } + return names +} + +// unpackSecret pulls the Secret proto out of a delta Resource. +func unpackSecret(t *testing.T, res *discovery.Resource) *tlsv3.Secret { + t.Helper() + msg, err := res.GetResource().UnmarshalNew() + if err != nil { + t.Fatalf("unmarshalling resource %q: %v", res.GetName(), err) + } + secret, ok := msg.(*tlsv3.Secret) + if !ok { + t.Fatalf("resource %q is a %T, want *tlsv3.Secret", res.GetName(), msg) + } + return secret +} + +// leafFromResource pulls the leaf x509 out of a delta Resource carrying a +// Secret, so a test can reason about the certificate Envoy would actually +// serve rather than just the xDS version string. +func leafFromResource(t *testing.T, res *discovery.Resource) *x509.Certificate { + t.Helper() + secret := unpackSecret(t, res) + chain := secret.GetTlsCertificate().GetCertificateChain().GetInlineBytes() + block, _ := pem.Decode(chain) + if block == nil { + t.Fatalf("resource %q: certificate chain is not PEM", res.GetName()) + } + leaf, err := x509.ParseCertificate(block.Bytes) + if err != nil { + t.Fatalf("resource %q: parsing leaf: %v", res.GetName(), err) + } + return leaf +} + +func TestDeltaSecretsMintsSubscribedName(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + srv := testServer(t, serverOptions{}) + stream, stop := startServer(t, srv) + defer func() { + if err := stop(); err != nil { + t.Errorf("DeltaSecrets returned %v", err) + } + }() + + stream.requests <- &discovery.DeltaDiscoveryRequest{ + TypeUrl: secretTypeURL, + ResourceNamesSubscribe: []string{"a.example"}, + } + + resp := stream.nextResponse(t) + if resp.GetTypeUrl() != secretTypeURL { + t.Errorf("type_url = %q, want %q", resp.GetTypeUrl(), secretTypeURL) + } + if resp.GetNonce() == "" { + t.Error("response has no nonce; Envoy needs one to ACK") + } + if len(resp.GetResources()) != 1 { + t.Fatalf("got %d resources, want 1", len(resp.GetResources())) + } + + res := resp.GetResources()[0] + if res.GetName() != "a.example" { + t.Errorf("resource name = %q, want a.example", res.GetName()) + } + if res.GetVersion() == "" { + t.Error("resource has no version; delta xDS needs one per resource") + } + + secret := unpackSecret(t, res) + // This is the invariant the whole design rests on: Envoy matches the + // response to its on-demand subscription by secret name, which is the SNI. + if secret.GetName() != "a.example" { + t.Errorf("secret name = %q, want it to equal the requested resource name", secret.GetName()) + } + + chain := secret.GetTlsCertificate().GetCertificateChain().GetInlineBytes() + key := secret.GetTlsCertificate().GetPrivateKey().GetInlineBytes() + if _, err := tls.X509KeyPair(chain, key); err != nil { + t.Errorf("secret does not contain a usable TLS keypair: %v", err) + } + }) +} + +// TestDeltaSecretsStampsResourceTTL covers the only thing that gets a name +// re-minted. Envoy drops a resource when its TTL fires and re-subscribes on the +// next handshake; nothing on this side pushes. A resource sent without a ttl is +// held by Envoy for good, and the leaf inside it goes on being served long past +// its notAfter -- see poc/sdsmint/expiry, which measured both. +func TestDeltaSecretsStampsResourceTTL(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + srv := testServer(t, serverOptions{}) + stream, stop := startServer(t, srv) + defer func() { + if err := stop(); err != nil { + t.Errorf("DeltaSecrets returned %v", err) + } + }() + + stream.requests <- &discovery.DeltaDiscoveryRequest{ + TypeUrl: secretTypeURL, + ResourceNamesSubscribe: []string{"a.example"}, + } + + res := stream.nextResponse(t).GetResources()[0] + ttl := res.GetTtl() + if ttl == nil { + t.Fatal("resource carries no ttl; Envoy would hold this secret forever and serve the leaf past its notAfter") + } + + // The invariant, rather than the exact fraction: the secret has to be + // dropped while the leaf it carries is still valid, so the handshake + // that re-subscribes is never the one served an expired leaf. + remaining := time.Until(leafFromResource(t, res).NotAfter) + if ttl.AsDuration() >= remaining { + t.Errorf("ttl %s is not shorter than the leaf's remaining validity %s; a handshake could land after the leaf expires but before Envoy drops it", + ttl.AsDuration(), remaining) + } + }) +} + +func TestDeltaSecretsWithdrawsRefusedName(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + srv := testServer(t, serverOptions{}) + stream, stop := startServer(t, srv) + defer func() { + if err := stop(); err != nil { + t.Errorf("DeltaSecrets returned %v", err) + } + }() + + // The minter refuses names, not destinations: "*.evil.test" is turned away + // for being a wildcard rather than for being anyone in particular. What + // this pins is the partial response -- one name refused must not cost the + // other name in the same subscription its certificate. + stream.requests <- &discovery.DeltaDiscoveryRequest{ + TypeUrl: secretTypeURL, + ResourceNamesSubscribe: []string{"ok.allowed", "*.evil.test"}, + } + + resp := stream.nextResponse(t) + + if len(resp.GetResources()) != 1 || resp.GetResources()[0].GetName() != "ok.allowed" { + t.Errorf("resources = %v, want just ok.allowed", resourceNames(resp)) + } + // A server cannot NACK in xDS. Withdrawing the name is how it says "this + // will not be issued", and per the Envoy docs it also cancels the + // data-plane subscription for that name. + if got := resp.GetRemovedResources(); len(got) != 1 || got[0] != "*.evil.test" { + t.Errorf("removed_resources = %v, want [*.evil.test]", got) + } + }) +} + +func TestDeltaSecretsBareAckSendsNothing(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + srv := testServer(t, serverOptions{}) + stream, stop := startServer(t, srv) + defer func() { + if err := stop(); err != nil { + t.Errorf("DeltaSecrets returned %v", err) + } + }() + + stream.requests <- &discovery.DeltaDiscoveryRequest{ + TypeUrl: secretTypeURL, + ResourceNamesSubscribe: []string{"a.example"}, + } + first := stream.nextResponse(t) + + // Envoy's ACK carries the nonce and no subscriptions. Replying to it + // would start an infinite ACK loop. + stream.requests <- &discovery.DeltaDiscoveryRequest{ + TypeUrl: secretTypeURL, + ResponseNonce: first.GetNonce(), + } + stream.quiet(t, "in reply to a bare ACK") + }) +} + +// TestDeltaSecretsNeverPushesUnprompted pins the shape of the server after +// rotation and the idle sweep were both removed: a subscribe is answered once +// and then the stream is silent for the whole life of the leaf and beyond. The +// server never speaks first. Nothing re-mints a name in place, so a leaf +// expires under a live subscription and Envoy goes on serving it -- that is a +// known consequence of the removals, not an accident, and this is where it is +// written down. +func TestDeltaSecretsNeverPushesUnprompted(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + // The TTL testServer's minter is built with. + const ttl = defaultTTL + srv := testServer(t, serverOptions{}) + stream, stop := startServer(t, srv) + defer func() { + if err := stop(); err != nil { + t.Errorf("DeltaSecrets returned %v", err) + } + }() + + stream.requests <- &discovery.DeltaDiscoveryRequest{ + TypeUrl: secretTypeURL, + ResourceNamesSubscribe: []string{"a.example"}, + } + stream.nextResponse(t) + + // Well past the point where the leaf has expired. Free on a fake clock. + time.Sleep(2 * ttl) + stream.quiet(t, "after the only subscribed leaf had expired") + }) +} + +// TestDeltaSecretsIgnoresUnsubscribe pins that the server holds no subscription +// set to unsubscribe from. Envoy volunteers an unsubscribe when the +// configuration referencing a secret goes away; there is nothing on this side +// to forget, so the request must be absorbed silently rather than answered or +// treated as an error that tears down the stream. +func TestDeltaSecretsIgnoresUnsubscribe(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + srv := testServer(t, serverOptions{}) + stream, stop := startServer(t, srv) + defer func() { + if err := stop(); err != nil { + t.Errorf("DeltaSecrets returned %v", err) + } + }() + + stream.requests <- &discovery.DeltaDiscoveryRequest{ + TypeUrl: secretTypeURL, + ResourceNamesSubscribe: []string{"a.example"}, + } + stream.nextResponse(t) + + stream.requests <- &discovery.DeltaDiscoveryRequest{ + TypeUrl: secretTypeURL, + ResourceNamesUnsubscribe: []string{"a.example"}, + } + stream.quiet(t, "in reply to an unsubscribe") + + // The stream is still usable afterwards, and a name it just unsubscribed + // from is minted again on request like any other -- the server draws no + // distinction, because it kept no record to draw one from. + stream.requests <- &discovery.DeltaDiscoveryRequest{ + TypeUrl: secretTypeURL, + ResourceNamesSubscribe: []string{"a.example"}, + } + if names := resourceNames(stream.nextResponse(t)); len(names) != 1 || names[0] != "a.example" { + t.Errorf("after an unsubscribe the server served %v, want [a.example]", names) + } + }) +} + +// TestDeltaSecretsReplayOnlyRequestIsSilent covers the other half of what handle +// ignores. A request carrying nothing but initial_resource_versions says what +// Envoy already holds; answering it would push leaves nobody asked for. The +// re-subscribe that accompanies a real reconnect is what prompts the re-mint, +// and it arrives as an ordinary subscribe. +func TestDeltaSecretsReplayOnlyRequestIsSilent(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + srv := testServer(t, serverOptions{}) + stream, stop := startServer(t, srv) + defer func() { + if err := stop(); err != nil { + t.Errorf("DeltaSecrets returned %v", err) + } + }() + + stream.requests <- &discovery.DeltaDiscoveryRequest{ + TypeUrl: secretTypeURL, + InitialResourceVersions: map[string]string{"resumed.example": "old-version"}, + } + + stream.quiet(t, "in reply to a replay-only request") + }) +} + +func TestDeltaSecretsRejectsWrongTypeURL(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + srv := testServer(t, serverOptions{}) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + stream := newFakeDeltaStream(ctx) + + done := make(chan error, 1) + go func() { done <- srv.DeltaSecrets(stream) }() + + stream.requests <- &discovery.DeltaDiscoveryRequest{ + TypeUrl: "type.googleapis.com/envoy.config.cluster.v3.Cluster", + ResourceNamesSubscribe: []string{"a.example"}, + } + + // Wait for the server to fail on its own rather than canceling, which + // would race the context-done branch of the stream loop. + select { + case err := <-done: + if err == nil { + t.Fatal("DeltaSecrets accepted a non-SDS type_url") + } + case <-time.After(respondWait): + t.Fatal("DeltaSecrets did not reject a non-SDS type_url") + } + }) +} + +func TestDeltaSecretsSurvivesNack(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + srv := testServer(t, serverOptions{}) + stream, stop := startServer(t, srv) + defer func() { + if err := stop(); err != nil { + t.Errorf("DeltaSecrets returned %v", err) + } + }() + + stream.requests <- &discovery.DeltaDiscoveryRequest{ + TypeUrl: secretTypeURL, + ResourceNamesSubscribe: []string{"a.example"}, + } + first := stream.nextResponse(t) + + // A NACK must not tear down the stream; Envoy would just reconnect and + // we would lose every live subscription. + stream.requests <- &discovery.DeltaDiscoveryRequest{ + TypeUrl: secretTypeURL, + ResponseNonce: first.GetNonce(), + ErrorDetail: &rpcstatus.Status{Code: 3, Message: "bad certificate"}, + } + stream.requests <- &discovery.DeltaDiscoveryRequest{ + TypeUrl: secretTypeURL, + ResourceNamesSubscribe: []string{"b.example"}, + } + + resp := stream.nextResponse(t) + if names := resourceNames(resp); len(names) != 1 || names[0] != "b.example" { + t.Errorf("after a NACK the server served %v, want [b.example]", names) + } + }) +} + +// TestResubscribeIsMintedAgain is what is left of the refresh path. Envoy only +// re-subscribes to a name it has dropped, so a repeat subscribe is a request +// for a certificate and must be answered with a freshly minted one rather than +// suppressed as a duplicate. With rotation and the idle sweep both gone this is +// the only way a name ever gets a new leaf. +func TestResubscribeIsMintedAgain(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + srv := testServer(t, serverOptions{}) + stream, stop := startServer(t, srv) + defer func() { + if err := stop(); err != nil { + t.Errorf("DeltaSecrets returned %v", err) + } + }() + + subscribe := &discovery.DeltaDiscoveryRequest{ + TypeUrl: secretTypeURL, + ResourceNamesSubscribe: []string{"a.example"}, + } + + stream.requests <- subscribe + first := stream.nextResponse(t) + if len(first.GetResources()) != 1 { + t.Fatalf("initial response carried %d resources, want 1", len(first.GetResources())) + } + + stream.requests <- subscribe + second := stream.nextResponse(t) + if len(second.GetResources()) != 1 { + t.Fatalf("re-subscribe was answered with %d resources, want 1", len(second.GetResources())) + } + + if got := second.GetResources()[0].GetName(); got != "a.example" { + t.Fatalf("re-subscribe returned %q, want a.example", got) + } + leaf := leafFromResource(t, second.GetResources()[0]) + if err := leaf.VerifyHostname("a.example"); err != nil { + t.Fatalf("re-minted leaf does not cover a.example: %v", err) + } + + // A new leaf, not the first one handed back. The version is the serial, + // so an unchanged version here would mean Envoy sees no update and goes + // on serving whatever it already had. + if v1, v2 := first.GetResources()[0].GetVersion(), second.GetResources()[0].GetVersion(); v1 == v2 { + t.Errorf("re-subscribe returned the same version %s; the name was not minted again", v1) + } + }) +} diff --git a/cmd/atenet/internal/sdsmint/doc.go b/cmd/atenet/internal/sdsmint/doc.go new file mode 100644 index 000000000..5930f6efa --- /dev/null +++ b/cmd/atenet/internal/sdsmint/doc.go @@ -0,0 +1,16 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package sdsmint implements `atenet sdsmint`, a minting SDS server. +package sdsmint diff --git a/cmd/atenet/internal/sdsmint/listen.go b/cmd/atenet/internal/sdsmint/listen.go new file mode 100644 index 000000000..0f4dc60bc --- /dev/null +++ b/cmd/atenet/internal/sdsmint/listen.go @@ -0,0 +1,48 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file holds the only socket the process opens: the SDS one, which is +// always a unix domain socket because leaf private keys transit it. + +package sdsmint + +import ( + "errors" + "fmt" + "net" + "os" + "path/filepath" +) + +// listen binds the SDS socket. There is no TCP alternative on purpose: leaf +// private keys transit this channel, and a unix socket restricted to the +// proxy's UID is the only reach that is ever wanted. +func listen(uds string) (net.Listener, error) { + if err := os.Remove(uds); err != nil && !errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("removing stale socket %s: %w", uds, err) + } + if err := os.MkdirAll(filepath.Dir(uds), 0o755); err != nil { + return nil, fmt.Errorf("creating socket directory: %w", err) + } + lis, err := net.Listen("unix", uds) + if err != nil { + return nil, fmt.Errorf("listening on %s: %w", uds, err) + } + // Only the proxy should be able to ask for certificates. + if err := os.Chmod(uds, 0o600); err != nil { + lis.Close() + return nil, fmt.Errorf("restricting socket permissions: %w", err) + } + return lis, nil +} diff --git a/cmd/atenet/internal/sdsmint/minter.go b/cmd/atenet/internal/sdsmint/minter.go new file mode 100644 index 000000000..8bec1b54d --- /dev/null +++ b/cmd/atenet/internal/sdsmint/minter.go @@ -0,0 +1,146 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sdsmint + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net" + "strings" + "time" + + "github.com/agent-substrate/substrate/cmd/atenet/internal/sdsmint/certauth" +) + +// errHostNotAllowed is returned when a requested hostname will not be minted. +var errHostNotAllowed = errors.New("host not allowed") + +// minter returns a leaf certificate for a hostname. +type minter struct { + signer *certauth.Signer + ttl time.Duration + log *slog.Logger +} + +// minterOptions configures newMinter. +type minterOptions struct { + // TTL is the leaf lifetime. + TTL time.Duration + Logger *slog.Logger +} + +// defaultTTL for leaf cert lifetime. +const defaultTTL = 15 * time.Minute + +// newMinter builds a minter over signer. +func newMinter(signer *certauth.Signer, opts minterOptions) (*minter, error) { + if signer == nil { + return nil, errors.New("nil signer") + } + if opts.TTL <= 0 { + opts.TTL = defaultTTL + } + if opts.Logger == nil { + opts.Logger = slog.Default() + } + return &minter{ + signer: signer, + ttl: opts.TTL, + log: opts.Logger, + }, nil +} + +// certificate mints a leaf for host. It returns an error wrapping +// errHostNotAllowed if host is not a name this will mint for. +func (m *minter) certificate(ctx context.Context, host string) (*certauth.MintedCert, error) { + if err := checkHostSyntax(host); err != nil { + m.log.WarnContext(ctx, "certificate request denied", + slog.String("host", host), + slog.String("reason", err.Error()), + ) + // checkHostSyntax already quotes the host, so this does not repeat it. + return nil, fmt.Errorf("%w: %w", errHostNotAllowed, err) + } + + cert, err := m.signer.Sign(host, m.ttl) + if err != nil { + return nil, err + } + + if m.log.Enabled(ctx, slog.LevelInfo) { + m.log.InfoContext(ctx, "certificate issued", + slog.String("host", host), + slog.String("serial", cert.Serial), + slog.Time("not_after", cert.NotAfter), + ) + } + return cert, nil +} + +// checkHostSyntax checks whether host is a valid DNS name or IP address. +func checkHostSyntax(host string) error { + if isValidDNSName(host) || isValidIPAddress(host) { + return nil + } + return fmt.Errorf("invalid host name %q", host) +} + +// isValidDNSName reports whether host is a syntactically valid DNS name. +func isValidDNSName(host string) bool { + // A trailing dot names the root explicitly. It is legal in a DNS name and + // not in SNI, but Envoy passes on whatever it was given, so it is dropped + // here rather than making the final label look empty. + host = strings.TrimSuffix(host, ".") + if host == "" || len(host) > 253 { + return false + } + for _, label := range strings.Split(host, ".") { + if !isValidDNSLabel(label) { + return false + } + } + return true +} + +// isValidDNSLabel reports whether one dot-separated component is a legal label: +// letters, digits, and interior hyphens, up to 63 bytes. +// https://datatracker.ietf.org/doc/html/rfc1035 +func isValidDNSLabel(label string) bool { + if label == "" || len(label) > 63 { + return false + } + for i := 0; i < len(label); i++ { + switch c := label[i]; { + case c >= 'a' && c <= 'z', c >= 'A' && c <= 'Z', c >= '0' && c <= '9': + // A hyphen may not open or close a label. + case c == '-' && i > 0 && i < len(label)-1: + // Not legal in a hostname, but real names carry it -- service records + // and a fair number of internal names -- and it smuggles nothing into a + // certificate. Refusing it would fail handshakes for no gain. + case c == '_': + default: + return false + } + } + return true +} + +// isValidIPAddress reports whether host is an IP literal, v4 or v6. SNI is not +// supposed to carry an IP addresss, but some clients send it anyway. +func isValidIPAddress(host string) bool { + return net.ParseIP(host) != nil +} diff --git a/cmd/atenet/internal/sdsmint/minter_test.go b/cmd/atenet/internal/sdsmint/minter_test.go new file mode 100644 index 000000000..3ca48df4f --- /dev/null +++ b/cmd/atenet/internal/sdsmint/minter_test.go @@ -0,0 +1,194 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sdsmint + +import ( + "context" + "errors" + "fmt" + "io" + "log/slog" + "strings" + "sync" + "testing" + "time" + + "github.com/agent-substrate/substrate/cmd/atenet/internal/sdsmint/certauth" + "github.com/agent-substrate/substrate/internal/localca" +) + +func quietLogger() *slog.Logger { + return slog.New(slog.NewTextHandler(io.Discard, nil)) +} + +// testSigner builds a signer over a throwaway CA. +func testSigner(t *testing.T) *certauth.Signer { + t.Helper() + ca, err := localca.GenerateCA(localca.GenerateOptions{ + ID: "mitm", + CommonName: "sdsmint test CA", + KeyType: localca.KeyTypeECDSAP256, + Lifetime: time.Hour, + }) + if err != nil { + t.Fatalf("generating test CA: %v", err) + } + signer, err := certauth.New(&localca.Pool{CAs: []*localca.CA{ca}}, "") + if err != nil { + t.Fatalf("certauth.New: %v", err) + } + return signer +} + +func testMinter(t *testing.T, opts minterOptions) *minter { + t.Helper() + if opts.Logger == nil { + opts.Logger = quietLogger() + } + m, err := newMinter(testSigner(t), opts) + if err != nil { + t.Fatalf("newMinter: %v", err) + } + return m +} + +// TestMinterMintsPerCall pins that the minter holds nothing between calls. It +// used to cache by host, and the SDS layer is written on the assumption that +// it no longer does: pack reads cert.Serial as the resource version, so two +// mints of one name have to look like two versions to Envoy. +func TestMinterMintsPerCall(t *testing.T) { + m := testMinter(t, minterOptions{TTL: time.Minute}) + ctx := context.Background() + + first, err := m.certificate(ctx, "a.example") + if err != nil { + t.Fatalf("first certificate: %v", err) + } + second, err := m.certificate(ctx, "a.example") + if err != nil { + t.Fatalf("second certificate: %v", err) + } + if first.Serial == second.Serial { + t.Errorf("both calls returned serial %s; the minter is holding onto leaves", first.Serial) + } + + other, err := m.certificate(ctx, "b.example") + if err != nil { + t.Fatalf("certificate for a different host: %v", err) + } + if other.Serial == first.Serial { + t.Error("different hosts were served the same certificate") + } +} + +// TestMinterRefusesNonHostnames pins the one thing that can still make the +// minter say no. There is no destination allowlist any more, so the SDS +// server's withdraw path is only ever reached through checkHostSyntax, and it +// is reached through a wrapped errHostNotAllowed. +func TestMinterRefusesNonHostnames(t *testing.T) { + m := testMinter(t, minterOptions{TTL: time.Minute}) + ctx := context.Background() + + // Nothing about this name is on a list; it just is a name. + if _, err := m.certificate(ctx, "anything.at.all.test"); err != nil { + t.Fatalf("ordinary hostname was rejected: %v", err) + } + + _, err := m.certificate(ctx, "*.evil.test") + if err == nil { + t.Fatal("a wildcard SNI was minted") + } + if !errors.Is(err, errHostNotAllowed) { + t.Errorf("error = %v, want it to wrap errHostNotAllowed", err) + } +} + +func TestMinterIsConcurrencySafe(t *testing.T) { + m := testMinter(t, minterOptions{TTL: time.Minute}) + ctx := context.Background() + + var wg sync.WaitGroup + errs := make(chan error, 64) + for i := range 64 { + wg.Add(1) + go func() { + defer wg.Done() + if _, err := m.certificate(ctx, fmt.Sprintf("h%d.example", i%16)); err != nil { + errs <- err + } + }() + } + wg.Wait() + close(errs) + + for err := range errs { + t.Fatalf("concurrent GetCertificate: %v", err) + } +} + +func TestCheckHostSyntax(t *testing.T) { + validate := checkHostSyntax + + allowed := []string{ + "example.com", + "a.example.com", + "a.b.c.d.example.com", // any depth, unlike a "*" label + "EXAMPLE.COM", // case is not a hostname's business + "anything.test", // any TLD + "localhost", // a single label is still a name + "xn--80ak6aa92e.com", // punycode + "a.example.com.", // trailing dot + "my-host.example.com", // interior hyphen + "_dmarc.example.com", // underscore; not a hostname, but a real name + + strings.Repeat("a", 63) + ".example.com", // a label at the 63-byte limit + + // IP literals. SNI is not supposed to carry one, but clients send them, + // and Sign puts them in IPAddresses rather than DNSNames. + "192.0.2.1", + "127.0.0.1", + "2001:db8::1", + "::1", + "::ffff:192.0.2.1", // v4-mapped v6 + } + for _, host := range allowed { + if err := validate(host); err != nil { + t.Errorf("validate(%q) = %v, want nil", host, err) + } + } + + denied := []string{ + "", // empty + ".", // the root alone is not a name to mint for + "*.example.com", // a wildcard in the SNI itself + "a.example.com/../x", // path separator smuggling + "..example.com", // empty label + ".example.com", // leading dot + "a.example.com\nfoo", // embedded newline + "-example.com", // label opens with a hyphen + "example-.com", // label closes with a hyphen + "[::1]", // bracketed; Sign would read it as a DNS name + "fe80::1%eth0", // a zone is not part of an address in a SAN + "192.0.2.1:8443", // a port is not part of a host here + + strings.Repeat("a", 64) + ".example.com", // label over 63 bytes + strings.Repeat("a.", 200) + "example.com", // over 253 bytes + } + for _, host := range denied { + if err := validate(host); err == nil { + t.Errorf("validate(%q) = nil, want an error", host) + } + } +} diff --git a/cmd/atenet/internal/sdsmint/run.go b/cmd/atenet/internal/sdsmint/run.go new file mode 100644 index 000000000..3693353ca --- /dev/null +++ b/cmd/atenet/internal/sdsmint/run.go @@ -0,0 +1,141 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file turns a parsed config into a running process: signer, minter, +// listeners, gRPC server, shutdown. The flags it reads are cmd.go's. + +package sdsmint + +import ( + "context" + "errors" + "fmt" + "log/slog" + "os" + "os/signal" + "syscall" + "time" + + secretservice "github.com/envoyproxy/go-control-plane/envoy/service/secret/v3" + "google.golang.org/grpc" + + "github.com/agent-substrate/substrate/cmd/atenet/internal/sdsmint/certauth" + "github.com/agent-substrate/substrate/internal/localca" +) + +func run(ctx context.Context, cfg config) error { + logger, err := newLogger(cfg.LogLevel) + if err != nil { + return err + } + slog.SetDefault(logger) + + if cfg.UDSPath == "" { + return errors.New("--uds-path is required") + } + if cfg.CAPoolPath == "" { + return errors.New("--ca-pool-path is required") + } + if err := cfg.validateTTL(); err != nil { + return err + } + + signer, err := loadSigner(cfg.CAPoolPath, cfg.CAID) + if err != nil { + return err + } + + // Named m because minter is the type. + m, err := newMinter(signer, minterOptions{ + TTL: cfg.LeafCertTTL, + Logger: logger, + }) + if err != nil { + return fmt.Errorf("building minter: %w", err) + } + + lis, err := listen(cfg.UDSPath) + if err != nil { + return err + } + defer lis.Close() + + grpcServer := grpc.NewServer() + secretservice.RegisterSecretDiscoveryServiceServer(grpcServer, newServer(m, serverOptions{ + Logger: logger, + })) + + logger.Info("sdsmint listening", + slog.String("network", lis.Addr().Network()), + slog.String("address", lis.Addr().String()), + slog.Any("config", cfg), + ) + + ctx, stop := signal.NotifyContext(ctx, syscall.SIGINT, syscall.SIGTERM) + defer stop() + go func() { + <-ctx.Done() + logger.Info("shutting down") + // GracefulStop waits for in-flight RPCs to finish, but an xDS stream + // is long-lived by design and only ends when Envoy closes it. Waiting + // on it unconditionally deadlocks shutdown, so fall back to a hard + // stop after a grace period. + done := make(chan struct{}) + go func() { + grpcServer.GracefulStop() + close(done) + }() + + // shutdownGrace is how long a signaled server waits for in-flight RPCs before + // tearing open streams down. Envoy's SDS stream never ends on its own, so this + // is the normal path, not the exceptional one. + const shutdownGrace = 2 * time.Second + select { + case <-done: + case <-time.After(shutdownGrace): + logger.Warn("graceful shutdown timed out; closing open streams", + slog.Duration("grace", shutdownGrace)) + grpcServer.Stop() + } + }() + + if err := grpcServer.Serve(lis); err != nil && !errors.Is(err, grpc.ErrServerStopped) { + return fmt.Errorf("serving: %w", err) + } + return nil +} + +func newLogger(level string) (*slog.Logger, error) { + var lvl slog.Level + if err := lvl.UnmarshalText([]byte(level)); err != nil { + return nil, fmt.Errorf("--log-level %q: %w", level, err) + } + return slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: lvl})), nil +} + +func loadSigner(poolPath, id string) (*certauth.Signer, error) { + poolBytes, err := os.ReadFile(poolPath) + if err != nil { + return nil, fmt.Errorf("reading CA pool %s: %w", poolPath, err) + } + pool, err := localca.Unmarshal(poolBytes) + if err != nil { + return nil, fmt.Errorf("parsing CA pool %s: %w", poolPath, err) + } + signer, err := certauth.New(pool, id) + if err != nil { + return nil, fmt.Errorf("loading CA from %s: %w", poolPath, err) + } + return signer, nil +} diff --git a/cmd/atenet/internal/sdsmint/server.go b/cmd/atenet/internal/sdsmint/server.go new file mode 100644 index 000000000..ea305593f --- /dev/null +++ b/cmd/atenet/internal/sdsmint/server.go @@ -0,0 +1,123 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sdsmint + +import ( + "log/slog" + "strconv" + "sync/atomic" + "time" + + corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + tlsv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" + secretservice "github.com/envoyproxy/go-control-plane/envoy/service/secret/v3" + + "github.com/agent-substrate/substrate/cmd/atenet/internal/sdsmint/certauth" +) + +// secretTypeURL is the xDS type URL for SDS resources. +const secretTypeURL = "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret" + +// server implements Envoy's Secret Discovery Service, minting a certificate +// per requested resource name. +// +// DeltaSecrets is the only method of that service this server implements. +// State-of-the-world SDS -- StreamSecrets and FetchSecrets -- is deliberately +// left to the embedded Unimplemented, so an Envoy configured with anything +// other than DELTA_GRPC fails immediately and visibly. +type server struct { + secretservice.UnimplementedSecretDiscoveryServiceServer + + minter *minter + log *slog.Logger + + // resourceTTL is the xDS TTL stamped on every resource this server sends: + // how long Envoy holds a secret before dropping it of its own accord. It is + // derived from the leaf lifetime rather than configured separately, so the + // two cannot be set into an order that does not work. See pack, which + // stamps it, for what Envoy does with it. + resourceTTL time.Duration + + // nonce numbers the responses this server sends. Every xDS response needs + // one: the client echoes it back as response_nonce, which is what makes a + // later request recognizable as an ACK or NACK of a specific response + // rather than a fresh subscription. A response that carries none cannot be + // ACKed at all. + // + // One counter for the whole server rather than one per stream. A client + // only ever compares a nonce against the last one it received on its own + // stream, so the sequence being sparse there costs nothing, and this + // server does not correlate them either -- an incoming response_nonce is + // read only for the NACK log line in deltaStream.handle. Atomic because + // streams are served concurrently. + nonce atomic.Uint64 +} + +// serverOptions configures newServer. +type serverOptions struct { + Logger *slog.Logger +} + +// newServer builds an SDS server over m. +func newServer(m *minter, opts serverOptions) *server { + if opts.Logger == nil { + opts.Logger = slog.Default() + } + + // Half the leaf lifetime. Envoy drops the secret when the TTL fires and the + // next handshake for that name re-subscribes, so a replacement is minted + // while the leaf it replaces is still valid. A TTL equal to the leaf + // lifetime would drop the secret at the moment the leaf died, leaving a + // window for a handshake to land on one already past its notAfter. + const refreshFraction = 2 + + return &server{ + minter: m, + log: opts.Logger, + // TODO(haiyanmeng): tune resourceTTL to be more efficient. + resourceTTL: m.ttl / refreshFraction, + } +} + +// nextNonce returns the nonce to stamp on the next response. It increments +// before formatting, so the first response is "1" and none ever goes out with +// the empty nonce. +func (s *server) nextNonce() string { + return strconv.FormatUint(s.nonce.Add(1), 10) +} + +// inlineBytes wraps PEM bytes as an inline Envoy DataSource. Leaf material is +// inlined rather than written to a path because it is per-connection and +// short-lived; putting it on a filesystem would only widen exposure. +func inlineBytes(b []byte) *corev3.DataSource { + return &corev3.DataSource{ + Specifier: &corev3.DataSource_InlineBytes{InlineBytes: b}, + } +} + +// toSecret packs a minted cert into the Secret proto Envoy expects back. The +// secret's name MUST equal the requested resource name (the SNI), or Envoy +// will not match the response to its subscription. +func toSecret(name string, c *certauth.MintedCert) *tlsv3.Secret { + return &tlsv3.Secret{ + Name: name, + Type: &tlsv3.Secret_TlsCertificate{ + TlsCertificate: &tlsv3.TlsCertificate{ + CertificateChain: inlineBytes(c.CertChainPEM), + PrivateKey: inlineBytes(c.PrivateKeyPEM), + }, + }, + } +} diff --git a/cmd/kubectl-ate/internal/cmd/admin_make_ca_pool.go b/cmd/kubectl-ate/internal/cmd/admin_make_ca_pool.go index ab42f2690..d7388c376 100644 --- a/cmd/kubectl-ate/internal/cmd/admin_make_ca_pool.go +++ b/cmd/kubectl-ate/internal/cmd/admin_make_ca_pool.go @@ -28,6 +28,8 @@ import ( var caID string var targetSecretNamespace string var targetSecretName string +var caKeyType string +var caCommonName string var makeCaPoolCmd = &cobra.Command{ Use: "make-ca-pool", @@ -45,7 +47,11 @@ var makeCaPoolCmd = &cobra.Command{ return fmt.Errorf("while creating Kubernetes client: %w", err) } - ca, err := localca.GenerateED25519CA(caID) + ca, err := localca.GenerateCA(localca.GenerateOptions{ + ID: caID, + CommonName: caCommonName, + KeyType: localca.KeyType(caKeyType), + }) if err != nil { return fmt.Errorf("while generating CA: %w", err) } @@ -84,5 +90,9 @@ func init() { makeCaPoolCmd.Flags().StringVar(&caID, "ca-id", "", "The ID of the initial CA in the Pool") makeCaPoolCmd.Flags().StringVar(&targetSecretNamespace, "secret-namespace", "default", "Create the secret in this namespace") makeCaPoolCmd.Flags().StringVar(&targetSecretName, "name", "", "Create the secret with this name") + makeCaPoolCmd.Flags().StringVar(&caKeyType, "key-type", string(localca.KeyTypeED25519), + fmt.Sprintf("Signing key algorithm, %q or %q. Prefer %s for a CA whose certificates are validated by clients outside substrate, where Ed25519 support cannot be assumed.", + localca.KeyTypeED25519, localca.KeyTypeECDSAP256, localca.KeyTypeECDSAP256)) + makeCaPoolCmd.Flags().StringVar(&caCommonName, "common-name", "", "Subject common name of the CA certificate. Cosmetic; nothing authenticates on it.") makeCaPoolCmd.MarkFlagRequired("name") } diff --git a/hack/install-ate.sh b/hack/install-ate.sh index 5ace949c2..3d22b118a 100755 --- a/hack/install-ate.sh +++ b/hack/install-ate.sh @@ -82,6 +82,7 @@ function usage() { echo " --create-jwt-authority-pool-secret Create JWT authority pool secret" echo " --create-actor-id-ca-pool-secret Create actor ID CA pool secret" echo " --create-actor-id-ca-certs-secret Create actor ID CA certs secret" + echo " --create-egress-mitm-ca-pool-secret Create egress MITM CA pool secret" echo " --create-podcertificate-controller-cas Create podcertificate controller CAs" echo " --create-valkey-ca-certs-secret Create Valkey's combined client/server CA bundle" echo " --create-api-server-env-vars Create ate-api-server env vars" @@ -351,6 +352,20 @@ create_actor_id_ca_certs_secret() { | run_kubectl apply -f - } +# The MITM CA the egress gateway's sdsmint sidecar signs per-SNI leaves with. +# ecdsa-p256 rather than the ed25519 default: these leaves are validated by +# arbitrary clients inside actor sandboxes, where Ed25519 support cannot be +# assumed. +create_egress_mitm_ca_pool_secret() { + log_step "create_egress_mitm_ca_pool_secret" + run_kubectl_ate admin make-ca-pool \ + --ca-id="mitm" \ + --name="egress-mitm-ca-pool" \ + --secret-namespace=ate-system \ + --key-type=ecdsa-p256 \ + --common-name="substrate egress MITM CA" +} + create_podcertificate_controller_cas() { log_step "create_podcertificate_controller_cas" run_kubectl create namespace podcertificate-controller-system || true @@ -548,6 +563,8 @@ ensure_apiserver_prerequisites() { || create_podcertificate_controller_cas run_kubectl get secret -n ate-system valkey-ca-certs >/dev/null 2>&1 \ || create_valkey_ca_certs_secret + run_kubectl get secret -n ate-system egress-mitm-ca-pool >/dev/null 2>&1 \ + || create_egress_mitm_ca_pool_secret # This ConfigMap carries the selected store backend, so always reconcile it # to make switching --store-backend update an existing installation. create_api_server_env_vars @@ -607,8 +624,11 @@ deploy_atenet() { router_manifest="$(render_atenet_router_manifest)" echo "${router_manifest}" | run_kubectl apply -f - - run_ko apply -f manifests/ate-install/atenet-egress.yaml + run_kubectl get secret -n ate-system egress-mitm-ca-pool >/dev/null 2>&1 \ + || create_egress_mitm_ca_pool_secret + run_ko apply -f manifests/ate-install/atenet-dns.yaml + run_ko apply -f manifests/ate-install/atenet-egress.yaml run_kubectl rollout status deployment/atenet-router -n ate-system --timeout=120s run_kubectl rollout status deployment/atenet-egress -n ate-system --timeout=120s run_kubectl rollout status deployment/dns -n ate-system --timeout=120s @@ -920,6 +940,7 @@ while [[ "$#" -gt 0 ]]; do --create-jwt-authority-pool-secret) create_jwt_authority_pool_secret ;; --create-actor-id-ca-pool-secret) create_actor_id_ca_pool_secret ;; --create-actor-id-ca-certs-secret) create_actor_id_ca_certs_secret ;; + --create-egress-mitm-ca-pool-secret) create_egress_mitm_ca_pool_secret ;; --create-podcertificate-controller-cas) create_podcertificate_controller_cas ;; --create-valkey-ca-certs-secret) create_valkey_ca_certs_secret ;; --create-api-server-env-vars) create_api_server_env_vars ;; diff --git a/internal/atunnel/client.go b/internal/atunnel/client.go index 666431bbf..ba0189198 100644 --- a/internal/atunnel/client.go +++ b/internal/atunnel/client.go @@ -19,6 +19,7 @@ import ( "context" "crypto/tls" "crypto/x509" + "errors" "fmt" "io" "net" @@ -27,6 +28,7 @@ import ( "os" "strconv" "strings" + "syscall" ) // TODO(liorlieberman): support/use CONNECT on Ingress as well. @@ -41,6 +43,28 @@ type ClientConfig struct { // DialFunc dials a network address. It matches net.Dialer.DialContext. type DialFunc func(ctx context.Context, network, address string) (net.Conn, error) +// ErrGatewayHandshake reports that the gateway's front door refused the +// connection at TLS: it rejected the client certificate, or its own +// certificate did not verify. +var ErrGatewayHandshake = errors.New("atunnel: egress gateway TLS handshake") + +// ConnectRejectedError reports a CONNECT the gateway answered with a non-2xx +// status. The caller authenticated successfully and the request was declined +// anyway, which is what an authorization denial looks like from here. The +// status code is carried separately from the message because it is the part a +// caller can act on. +type ConnectRejectedError struct { + StatusCode int + // Status is the full status line, e.g. "403 Forbidden". + Status string + // Message is the response body, or the status text when the body is empty. + Message string +} + +func (e *ConnectRejectedError) Error() string { + return fmt.Sprintf("atunnel: egress gateway rejected CONNECT with %s: %s", e.Status, e.Message) +} + // ClientOption customizes a Client beyond its configuration. Production // callers need none of these. type ClientOption func(*Client) @@ -117,7 +141,7 @@ func (c *Client) DialContext(ctx context.Context, destination string) (net.Conn, tlsConn := tls.Client(rawConn, c.tlsConfig.Clone()) if err := tlsConn.HandshakeContext(ctx); err != nil { _ = rawConn.Close() - return nil, fmt.Errorf("atunnel: egress gateway TLS handshake: %w", err) + return nil, fmt.Errorf("%w: %w", ErrGatewayHandshake, err) } req := &http.Request{ @@ -127,14 +151,14 @@ func (c *Client) DialContext(ctx context.Context, destination string) (net.Conn, } if err := req.Write(tlsConn); err != nil { _ = tlsConn.Close() - return nil, fmt.Errorf("atunnel: writing CONNECT request: %w", err) + return nil, connectExchangeError("writing CONNECT request", err) } reader := bufio.NewReader(tlsConn) resp, err := http.ReadResponse(reader, req) if err != nil { _ = tlsConn.Close() - return nil, fmt.Errorf("atunnel: reading CONNECT response: %w", err) + return nil, connectExchangeError("reading CONNECT response", err) } if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { body, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10)) @@ -144,12 +168,44 @@ func (c *Client) DialContext(ctx context.Context, destination string) (net.Conn, if message == "" { message = http.StatusText(resp.StatusCode) } - return nil, fmt.Errorf("atunnel: egress gateway rejected CONNECT with %s: %s", resp.Status, message) + return nil, &ConnectRejectedError{StatusCode: resp.StatusCode, Status: resp.Status, Message: message} } return &bufferedConn{Conn: tlsConn, reader: reader}, nil } +// connectExchangeError wraps a failure that happened after the TLS handshake +// returned but before the gateway answered CONNECT, naming the front door as +// the cause when the connection was torn down rather than answered. +func connectExchangeError(op string, err error) error { + if gatewayHungUp(err) { + return fmt.Errorf("%w: %s: %w", ErrGatewayHandshake, op, err) + } + return fmt.Errorf("atunnel: %s: %w", op, err) +} + +// gatewayHungUp reports whether err is the peer tearing the connection down, +// as opposed to a local failure such as a context deadline or a malformed +// response. +func gatewayHungUp(err error) bool { + // A TLS alert from the peer -- "unknown certificate authority" is the one + // a wrong client CA produces. crypto/tls reports an alert as a net.OpError + // whose Op is "remote error" and whose Err is an unexported alert type, so + // the Op is the only part of it that can be matched without matching on + // the message text this package deliberately avoids. + var opErr *net.OpError + if errors.As(err, &opErr) && opErr.Op == "remote error" { + return true + } + // Or the gateway closed or reset instead of alerting -- including a reset + // that lands while the CONNECT request is still going out, which is what + // makes the alert-versus-EPIPE outcome a race rather than a distinction. + return errors.Is(err, io.EOF) || + errors.Is(err, io.ErrUnexpectedEOF) || + errors.Is(err, syscall.EPIPE) || + errors.Is(err, syscall.ECONNRESET) +} + func validateDestination(destination string) error { host, port, err := net.SplitHostPort(destination) if err != nil { diff --git a/internal/atunnel/client_test.go b/internal/atunnel/client_test.go index 4f9dc78e4..a7ddb1724 100644 --- a/internal/atunnel/client_test.go +++ b/internal/atunnel/client_test.go @@ -23,6 +23,7 @@ import ( "crypto/tls" "crypto/x509" "encoding/pem" + "errors" "fmt" "io" "math/big" @@ -31,6 +32,7 @@ import ( "os" "path/filepath" "strings" + "syscall" "testing" "time" ) @@ -103,6 +105,124 @@ func TestClientDialContextRejected(t *testing.T) { } } +// TestClientDialContextGatewayRefusesClientCertificate pins the classification +// of a front-door refusal to ErrGatewayHandshake under both TLS versions, which +// is not one behavior but two. +// +// Under 1.2 the server rejects the certificate mid-handshake and +// HandshakeContext returns the error. Under 1.3 the client's handshake has +// already returned nil by the time the server looks at the certificate, so the +// refusal lands on the CONNECT exchange instead -- and 1.3 is what a real +// gateway negotiates. A caller asking "did the door turn me away?" has to get +// the same answer either way, or it is really asking which version was +// negotiated. +func TestClientDialContextGatewayRefusesClientCertificate(t *testing.T) { + for _, tt := range []struct { + name string + maxTLSVersion uint16 + }{ + {name: "TLS 1.3 refuses after the client handshake completes", maxTLSVersion: tls.VersionTLS13}, + {name: "TLS 1.2 refuses during the handshake", maxTLSVersion: tls.VersionTLS12}, + } { + t.Run(tt.name, func(t *testing.T) { + ca := newTestCA(t) + // The gateway trusts a different CA than the one that issued the + // client's certificate, which is the shape of an actor presenting a + // podidentity credential to a door that only accepts actor identity. + gatewayAddress := serveTestRefusingGateway(t, ca, newTestCA(t), tt.maxTLSVersion) + client := newTestClient(t, ca, WithDialer(dialFixedAddress(gatewayAddress))) + + _, err := client.DialContext(context.Background(), "192.0.2.10:443") + if err == nil { + t.Fatal("DialContext succeeded against a gateway that refuses the client certificate") + } + if !errors.Is(err, ErrGatewayHandshake) { + t.Errorf("DialContext error = %v, want it to wrap ErrGatewayHandshake", err) + } + var rejected *ConnectRejectedError + if errors.As(err, &rejected) { + // A refusal at the door is not a CONNECT the gateway answered: + // conflating them would let an authorization denial read as an + // authentication failure. + t.Errorf("DialContext error = %v, want no ConnectRejectedError", err) + } + }) + } +} + +// TestClientDialContextGatewayHangsUpBeforeResponding covers the same window +// without an alert in it: the gateway closes the connection after the handshake +// and says nothing. There is still nobody but the front door on the other end, +// since no CONNECT response means no upstream was ever dialed. +func TestClientDialContextGatewayHangsUpBeforeResponding(t *testing.T) { + ca := newTestCA(t) + gatewayAddress := serveTestConnectGateway(t, ca, func(conn net.Conn, _ *http.Request) { + _ = conn.Close() + }) + client := newTestClient(t, ca, WithDialer(dialFixedAddress(gatewayAddress))) + + _, err := client.DialContext(context.Background(), "192.0.2.10:443") + if err == nil { + t.Fatal("DialContext succeeded against a gateway that hung up") + } + if !errors.Is(err, ErrGatewayHandshake) { + t.Errorf("DialContext error = %v, want it to wrap ErrGatewayHandshake", err) + } +} + +// TestConnectExchangeError is the other half of gatewayHungUp: a failure on +// this side of the connection must not be reported as the door refusing, or +// "the gateway rejected my certificate" stops meaning anything. The cases are +// exercised directly because the ones worth pinning -- a deadline, a response +// the gateway did send but malformed -- are awkward to provoke through a real +// listener and trivial to state as errors. +func TestConnectExchangeError(t *testing.T) { + for _, tt := range []struct { + name string + err error + wantFrontDoor bool + }{ + { + name: "peer TLS alert", + err: &net.OpError{Op: "remote error", Err: errors.New("tls: unknown certificate authority")}, + wantFrontDoor: true, + }, + { + name: "peer closed without alerting", + err: io.EOF, + wantFrontDoor: true, + }, + { + name: "peer reset mid-response", + err: io.ErrUnexpectedEOF, + wantFrontDoor: true, + }, + { + name: "peer reset while the CONNECT was going out", + err: &net.OpError{Op: "write", Err: syscall.EPIPE}, + wantFrontDoor: true, + }, + { + name: "our own deadline expired", + err: os.ErrDeadlineExceeded, + }, + { + name: "the gateway answered, but not with HTTP", + err: errors.New("malformed HTTP response \"garbage\""), + }, + } { + t.Run(tt.name, func(t *testing.T) { + err := connectExchangeError("reading CONNECT response", tt.err) + if got := errors.Is(err, ErrGatewayHandshake); got != tt.wantFrontDoor { + t.Errorf("errors.Is(%v, ErrGatewayHandshake) = %t, want %t", err, got, tt.wantFrontDoor) + } + if !errors.Is(err, tt.err) { + t.Errorf("%v no longer unwraps to the underlying %v", err, tt.err) + } + }) + } +} + func TestClientDialContextValidatesInput(t *testing.T) { ca := newTestCA(t) client := newTestClient(t, ca) @@ -192,6 +312,45 @@ func serveTestConnectGateway(t *testing.T, ca *testCA, handle func(net.Conn, *ht return listener.Addr().String() } +// serveTestRefusingGateway serves a front door that presents a certificate from +// serverCA and requires a client certificate from clientCA, so a client holding +// one from anywhere else is turned away. maxVersion caps the negotiated TLS +// version, which is what decides whether the refusal reaches the client during +// its handshake or after it. +// +// It handshakes and closes rather than reading a request: a refused client +// never gets to send one, and any error here is the refusal working. +func serveTestRefusingGateway(t *testing.T, serverCA, clientCA *testCA, maxVersion uint16) string { + t.Helper() + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = listener.Close() }) + + clientCAs := x509.NewCertPool() + clientCAs.AppendCertsFromPEM(clientCA.certPEM) + config := &tls.Config{ + MinVersion: tls.VersionTLS12, + MaxVersion: maxVersion, + Certificates: []tls.Certificate{issueDNSCertificate(t, serverCA, "egress.test")}, + ClientAuth: tls.RequireAndVerifyClientCert, + ClientCAs: clientCAs, + } + + go func() { + conn, err := listener.Accept() + if err != nil { + return + } + defer conn.Close() + tlsConn := tls.Server(conn, config) + _ = tlsConn.Handshake() + _ = tlsConn.Close() + }() + return listener.Addr().String() +} + func issueDNSCertificate(t *testing.T, ca *testCA, dnsName string) tls.Certificate { t.Helper() key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) diff --git a/internal/e2e/fixtures/egressprobe/egressprobe.yaml.tmpl b/internal/e2e/fixtures/egressprobe/egressprobe.yaml.tmpl new file mode 100644 index 000000000..a07f09629 --- /dev/null +++ b/internal/e2e/fixtures/egressprobe/egressprobe.yaml.tmpl @@ -0,0 +1,113 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The egress probe used by internal/e2e/suites/sdsmint. ${NAMESPACE} is +# substituted by the suite with the randomized namespace it created, so the +# probe is torn down with that namespace and leaves nothing behind. +# +# A plain Pod, not an actor: the suite is testing the gateway's MITM leg, and +# running the probe itself inside an actor would make a snapshot or restore +# failure look like an sdsmint failure. The suite does create one actor, but +# only for its identity -- the certificate mounted below is minted for it, and +# the actor's own workload is never contacted. + +apiVersion: v1 +kind: Pod +metadata: + name: egressprobe + namespace: ${NAMESPACE} + labels: + app: egressprobe +spec: + restartPolicy: Never + containers: + - name: egressprobe + image: ko://github.com/agent-substrate/substrate/internal/e2e/fixtures/egressprobe + args: + - "--listen=:8080" + ports: + - name: http + containerPort: 8080 + # The suite port-forwards to this pod, and port-forward targets the first + # READY pod behind the Service. Without a readiness gate the forward can + # attach before the listener exists and the first handshake fails as a + # connection refused that looks like a gateway problem. + readinessProbe: + httpGet: + path: /healthz + port: 8080 + periodSeconds: 2 + resources: + requests: + cpu: 10m + memory: 32Mi + # runAsUser must be spelled out: ko's distroless static base declares no + # USER, so runAsNonRoot on its own makes kubelet refuse to start the + # container ("image will run as root") rather than pick a uid. 65532 is + # distroless's nonroot uid, and the same one the egress gateway's pod uses. + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + capabilities: + drop: ["ALL"] + volumeMounts: + - name: "actor-identity" + mountPath: "/run/actor-identity" + - name: "actor-identity-unknown" + mountPath: "/run/actor-identity-unknown" + - name: "podidentity" + mountPath: "/run/podidentity.podcert.ate.dev" + - name: "servicedns-ca" + mountPath: "/run/servicedns.podcert.ate.dev" + volumes: + - name: "actor-identity" + secret: + secretName: egressprobe-actor-identity + - name: "actor-identity-unknown" + secret: + secretName: egressprobe-unknown-actor + - name: "podidentity" + projected: + sources: + - podCertificate: + signerName: podidentity.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - name: "servicedns-ca" + projected: + sources: + - clusterTrustBundle: + signerName: servicedns.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem + +--- + +apiVersion: v1 +kind: Service +metadata: + name: egressprobe + namespace: ${NAMESPACE} +spec: + selector: + app: egressprobe + ports: + - name: http + port: 8080 + targetPort: 8080 diff --git a/internal/e2e/fixtures/egressprobe/main.go b/internal/e2e/fixtures/egressprobe/main.go new file mode 100644 index 000000000..a5e424eb0 --- /dev/null +++ b/internal/e2e/fixtures/egressprobe/main.go @@ -0,0 +1,232 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Command egressprobe drives the egress gateway's MITM leg from inside the +// cluster and reports the certificate it was served, so an e2e suite can assert +// on what sdsmint actually minted. +package main + +import ( + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "encoding/pem" + "errors" + "flag" + "fmt" + "log" + "net/http" + "time" + + "github.com/agent-substrate/substrate/internal/atunnel" + "github.com/agent-substrate/substrate/internal/credbundle" +) + +var ( + listenAddress = flag.String("listen", ":8080", "Address the probe's HTTP API listens on.") + gatewayAddress = flag.String("gateway-address", "atenet-egress.ate-system.svc:443", "host:port of the egress gateway's CONNECT front door.") + credentialBundlePath = flag.String("credential-bundle", "/run/actor-identity/credential-bundle.pem", "PEM credential bundle presented as the client certificate to the gateway.") + trustBundlePath = flag.String("trust-bundle", "/run/servicedns.podcert.ate.dev/trust-bundle.pem", "PEM trust bundle used to verify the gateway's serving certificate.") + handshakeTimeout = flag.Duration("handshake-timeout", 20*time.Second, "Budget for one CONNECT plus inner handshake.") +) + +// tunnelDestination is the CONNECT authority. atunnel takes this from +// SO_ORIGINAL_DST and rejects hostnames, so it must be a literal IP -- and the +// gateway routes every CONNECT to the MITM listener regardless of authority, +// which is why a documentation address that resolves nowhere is enough. The +// name being tested travels in the tunneled ClientHello, not here. +const tunnelDestination = "192.0.2.1:443" + +// Stages a handshake can fail at, reported in handshakeResult.Stage. They name +// the hop, so a test can say which of the gateway's several ways of saying no +// it expected without matching on prose. +const ( + // stageClient is a local failure before anything was dialed: a bad flag, an + // unreadable credential bundle. Never a statement about the gateway. + stageClient = "client" + // stageGatewayTLS is the front door's mTLS. Reaching here and failing means + // the gateway refused the client certificate -- or presented one the probe + // would not accept. + stageGatewayTLS = "gateway_tls" + // stageConnect is the CONNECT exchange. A failure here means the + // certificate was accepted and the request was declined anyway, which is + // where the ext_proc authorization check lives. ConnectStatus carries the + // status code. + stageConnect = "connect" + // stageTunnel is any other failure opening the tunnel: DNS, TCP, a + // truncated response. + stageTunnel = "tunnel" + // stageInnerHandshake is the tunneled TLS handshake -- the one sdsmint + // serves. A failure here is the minter's, not the front door's. + stageInnerHandshake = "inner_handshake" +) + +// handshakeResult is the probe's response body. +type handshakeResult struct { + SNI string `json:"sni"` + // Credential is the bundle path this handshake presented, echoed back so a + // test cannot mistake a result for one taken with a different identity. + Credential string `json:"credential"` + // OK reports whether the inner TLS handshake completed. A denied SNI is a + // normal outcome, not a probe failure, so it comes back as OK=false with + // the reason rather than as an HTTP error. + OK bool `json:"ok"` + // Stage is where a failed handshake stopped, one of the stage constants + // above. Empty when OK. + Stage string `json:"stage,omitempty"` + // ConnectStatus is the status the gateway answered the CONNECT with, set + // only when Stage is stageConnect and the gateway actually replied. + ConnectStatus int `json:"connect_status,omitempty"` + Error string `json:"error,omitempty"` + // ChainPEM is the chain the gateway presented, leaf first. + ChainPEM string `json:"chain_pem,omitempty"` +} + +// handshake opens a tunnel through the gateway and completes an inner TLS +// handshake for the requested SNI, which is what makes Envoy ask sdsmint for a +// secret under that name. +func handshake(w http.ResponseWriter, r *http.Request) { + sni := r.URL.Query().Get("sni") + if sni == "" { + http.Error(w, "missing sni query parameter", http.StatusBadRequest) + return + } + + // Which credential to present. The default is the actor identity the suite + // minted; a test that is asserting how the gateway treats some OTHER + // credential names its path here rather than needing a second pod, since + // the choice is made per handshake and nothing is cached across them. + credential := r.URL.Query().Get("credential-bundle") + if credential == "" { + credential = *credentialBundlePath + } + + ctx, cancel := context.WithTimeout(r.Context(), *handshakeTimeout) + defer cancel() + + result := handshakeResult{SNI: sni, Credential: credential} + chain, stage, err := fetchChain(ctx, sni, credential) + if err != nil { + result.Stage = stage + result.Error = err.Error() + var rejected *atunnel.ConnectRejectedError + if errors.As(err, &rejected) { + result.ConnectStatus = rejected.StatusCode + } + } else { + result.OK = true + result.ChainPEM = encodeChain(chain) + } + writeJSON(w, result) +} + +func encodeChain(chain []*x509.Certificate) string { + var out []byte + for _, cert := range chain { + out = append(out, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: cert.Raw})...) + } + return string(out) +} + +// fetchChain opens a tunnel and completes the inner handshake, returning the +// chain it was served. On failure it also returns the stage that failed, since +// which hop said no is the assertion most callers are actually making. +func fetchChain(ctx context.Context, sni, credentialBundle string) ([]*x509.Certificate, string, error) { + client, err := atunnel.NewClient(atunnel.ClientConfig{ + GatewayAddress: *gatewayAddress, + ServerName: serverName(*gatewayAddress), + GetClientCertificate: credbundle.ClientLoader(credentialBundle), + TrustBundlePath: *trustBundlePath, + }) + if err != nil { + return nil, stageClient, fmt.Errorf("building egress client: %w", err) + } + + conn, err := client.DialContext(ctx, tunnelDestination) + if err != nil { + return nil, dialStage(err), fmt.Errorf("opening tunnel: %w", err) + } + defer conn.Close() + + //nolint:gosec // G402: verification is the caller's assertion; see the package comment. + tlsConn := tls.Client(conn, &tls.Config{ + ServerName: sni, + InsecureSkipVerify: true, + MinVersion: tls.VersionTLS12, + }) + if err := tlsConn.HandshakeContext(ctx); err != nil { + return nil, stageInnerHandshake, fmt.Errorf("inner TLS handshake for %q: %w", sni, err) + } + defer tlsConn.Close() + + out := tlsConn.ConnectionState().PeerCertificates + if len(out) == 0 { + return nil, stageInnerHandshake, fmt.Errorf("handshake for %q completed with no peer certificates", sni) + } + return out, "", nil +} + +// dialStage names the hop a DialContext failure stopped at. It reads atunnel's +// typed errors rather than its messages, so the mapping survives a reworded +// error. +func dialStage(err error) string { + var rejected *atunnel.ConnectRejectedError + switch { + case errors.Is(err, atunnel.ErrGatewayHandshake): + return stageGatewayTLS + case errors.As(err, &rejected): + return stageConnect + default: + return stageTunnel + } +} + +// serverName is the host half of a host:port address. It is written by hand +// rather than with net.SplitHostPort so that a malformed flag surfaces at +// handshake time with the address in the message, instead of at startup. +func serverName(address string) string { + for i := len(address) - 1; i >= 0; i-- { + if address[i] == ':' { + return address[:i] + } + } + return address +} + +func writeJSON(w http.ResponseWriter, v any) { + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(v); err != nil { + log.Printf("egressprobe: encoding response: %v", err) + } +} + +func main() { + flag.Parse() + + mux := http.NewServeMux() + mux.HandleFunc("/handshake", handshake) + mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + }) + + server := &http.Server{ + Addr: *listenAddress, + Handler: mux, + ReadHeaderTimeout: 10 * time.Second, + WriteTimeout: 2 * time.Minute, + } + log.Printf("egressprobe: listening on %s, gateway %s", *listenAddress, *gatewayAddress) + log.Fatal(server.ListenAndServe()) +} diff --git a/internal/e2e/suites/sdsmint/actoridentity_test.go b/internal/e2e/suites/sdsmint/actoridentity_test.go new file mode 100644 index 000000000..62faecb70 --- /dev/null +++ b/internal/e2e/suites/sdsmint/actoridentity_test.go @@ -0,0 +1,275 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sdsmint + +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "net/url" + "os" + "path" + "sync" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/agent-substrate/substrate/internal/e2e" + "github.com/agent-substrate/substrate/internal/localca" + "github.com/agent-substrate/substrate/internal/substratex509" + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" +) + +// The gateway's front door requires a client certificate signed by the +// actor-identity CA, and its ext_proc sidecar then looks the certified actor up +// in the control plane. Getting through it therefore needs both halves: a leaf +// this pool signs, and an actor the ate API agrees is running. +const ( + actorIDCASecret = "actor-id-ca-pool" + actorIDCASecretKey = "pool" + + // The atespace the suite's actor lives in. Fixed rather than randomized so + // a leaked actor from an aborted run is easy to find and delete. + probeAtespace = "ate-sdsmint-e2e" + + // actorCertificateLifetime matches what ateapi's MintCert issues. Nothing + // here depends on the exact value -- the suite runs in minutes -- but a + // credential that outlives the real one would hide an expiry bug in the + // gateway rather than reproduce it. + actorCertificateLifetime = time.Hour + + // Where the probe pod finds the credentials the suite mints for it. Kept in + // step with egressprobe.yaml.tmpl and the --credential-bundle default in + // the probe. + actorCredentialSecret = "egressprobe-actor-identity" + + unknownActorCredentialSecret = "egressprobe-unknown-actor" + unknownActorCredentialPath = "/run/actor-identity-unknown/credential-bundle.pem" + + credentialBundleKey = "credential-bundle.pem" + + // podIdentityCredentialPath is the probe's own workload identity: a valid + // substrate credential that is not an actor. It is mounted so the suite can + // show the gateway refusing it. + podIdentityCredentialPath = "/run/podidentity.podcert.ate.dev/credential-bundle.pem" +) + +// templateRef is the ActorTemplate the suite's actor is created from. The +// default is the one every other suite uses; the environment overrides exist +// for clusters that install the fixtures elsewhere. +func templateRef() (namespace, name string) { + namespace, name = "ate-demo-counter", "counter" + if v := os.Getenv("E2E_TEMPLATE_NAMESPACE"); v != "" { + namespace = v + } + if v := os.Getenv("E2E_TEMPLATE_NAME"); v != "" { + name = v + } + return namespace, name +} + +// probeActor is the identity the probe authenticates to the gateway as. +type probeActor struct { + atespace string + name string + uid string +} + +func (a *probeActor) identity() *substratex509.ActorIdentity { + return &substratex509.ActorIdentity{ + Atespace: a.atespace, + ActorName: a.name, + ActorUid: a.uid, + Purpose: substratex509.ActorIdentityPurposeAtunnel, + } +} + +// liveActor returns the one actor the whole suite shares. +var liveActor = sync.OnceValues(createLiveActor) + +func createLiveActor() (*probeActor, error) { + ctx, cancel := context.WithTimeout(context.Background(), 6*time.Minute) + defer cancel() + + clients := e2e.GetClients() + tmplNS, tmplName := templateRef() + name := fmt.Sprintf("sdsmint-probe-%d", time.Now().UnixNano()) + ref := &ateapipb.ObjectRef{Atespace: probeAtespace, Name: name} + + // CreateActor requires the atespace to exist first; a second run finds it + // already there, which is not an error. + _, _ = clients.SubstrateAPI.CreateAtespace(ctx, &ateapipb.CreateAtespaceRequest{ + Atespace: &ateapipb.Atespace{Metadata: &ateapipb.ResourceMetadata{Name: probeAtespace}}, + }) + if _, err := clients.SubstrateAPI.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: probeAtespace, Name: name}, + ActorTemplateNamespace: tmplNS, + ActorTemplateName: tmplName, + }}); err != nil { + return nil, fmt.Errorf("creating actor %s/%s from template %s/%s: %w (install the fixture with hack/install-demo-counter.sh, or point E2E_TEMPLATE_NAMESPACE/E2E_TEMPLATE_NAME at another one)", + probeAtespace, name, tmplNS, tmplName, err) + } + e2e.RegisterSuiteCleanup(func() { + // DeleteActor requires the actor to be suspended first. Both are + // best-effort: a run that could not reach the API here has already + // failed louder somewhere else. + cleanupCtx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + _, _ = clients.SubstrateAPI.SuspendActor(cleanupCtx, &ateapipb.SuspendActorRequest{Actor: ref}) + _, _ = clients.SubstrateAPI.DeleteActor(cleanupCtx, &ateapipb.DeleteActorRequest{Actor: ref}) + }) + + if _, err := clients.SubstrateAPI.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: ref, Boot: true}); err != nil { + return nil, fmt.Errorf("resuming actor %s/%s: %w", probeAtespace, name, err) + } + + deadline := time.Now().Add(4 * time.Minute) + var lastStatus ateapipb.Actor_Status + for time.Now().Before(deadline) { + actor, err := clients.SubstrateAPI.GetActor(ctx, &ateapipb.GetActorRequest{Actor: ref}) + if err == nil { + lastStatus = actor.GetStatus() + if lastStatus == ateapipb.Actor_STATUS_RUNNING { + uid := actor.GetMetadata().GetUid() + if uid == "" { + return nil, fmt.Errorf("actor %s/%s is running but has no UID", probeAtespace, name) + } + return &probeActor{atespace: probeAtespace, name: name, uid: uid}, nil + } + } + select { + case <-ctx.Done(): + return nil, fmt.Errorf("waiting for actor %s/%s to run: %w", probeAtespace, name, ctx.Err()) + case <-time.After(2 * time.Second): + } + } + return nil, fmt.Errorf("actor %s/%s never reached STATUS_RUNNING (last status %v); a saturated worker pool is the usual cause", + probeAtespace, name, lastStatus) +} + +// actorIdentityCA returns the CA that signs actor certificates, straight from +// the secret ateapi signs with. +func actorIdentityCA(t *testing.T, ctx context.Context) *localca.CA { + t.Helper() + secret, err := e2e.GetClients().K8s.CoreV1().Secrets(egressNamespace).Get(ctx, actorIDCASecret, metav1.GetOptions{}) + if err != nil { + t.Fatalf("reading actor-identity CA pool secret %s/%s: %v", egressNamespace, actorIDCASecret, err) + } + pool, err := localca.Unmarshal(secret.Data[actorIDCASecretKey]) + if err != nil { + t.Fatalf("parsing actor-identity CA pool from %s/%s key %q: %v", egressNamespace, actorIDCASecret, actorIDCASecretKey, err) + } + if len(pool.CAs) == 0 { + t.Fatalf("actor-identity CA pool %s/%s contains no CA", egressNamespace, actorIDCASecret) + } + // CAs[0] is the one that signs: ateapi's MintCert makes the same choice. + return pool.CAs[0] +} + +// mintActorCredential issues a client credential for identity, in the shape +// atunnel gets from ateapi. +func mintActorCredential(t *testing.T, ca *localca.CA, identity *substratex509.ActorIdentity) []byte { + t.Helper() + + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("generating actor key: %v", err) + } + + template := &x509.Certificate{ + URIs: []*url.URL{{ + Scheme: "spiffe", + Host: "substrate-actor.local", + Path: path.Join("atespace", identity.Atespace, "actor", identity.ActorName), + }}, + NotBefore: time.Now().Add(-5 * time.Minute), + NotAfter: time.Now().Add(actorCertificateLifetime), + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + BasicConstraintsValid: true, + IsCA: false, + Issuer: pkix.Name{CommonName: "api.ate-system.svc.cluster.local"}, + } + if err := substratex509.AddActorIdentityToCertificate(identity, template); err != nil { + t.Fatalf("adding the ActorIdentity extension for %s/%s: %v", identity.Atespace, identity.ActorName, err) + } + + der, err := x509.CreateCertificate(rand.Reader, template, ca.RootCertificate, key.Public(), ca.SigningKey) + if err != nil { + t.Fatalf("signing the actor certificate for %s/%s: %v", identity.Atespace, identity.ActorName, err) + } + keyDER, err := x509.MarshalPKCS8PrivateKey(key) + if err != nil { + t.Fatalf("marshaling the actor key: %v", err) + } + + // A credential bundle as internal/credbundle parses it: the PKCS#8 key + // first, then the chain leaf-first. + bundle := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER}) + bundle = append(bundle, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})...) + for _, intermediate := range ca.IntermediateCertificates { + bundle = append(bundle, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: intermediate.Raw})...) + } + return bundle +} + +// writeCredentialSecret puts a minted bundle where the probe pod can mount it. +func writeCredentialSecret(t *testing.T, ctx context.Context, ns, name string, bundle []byte) { + t.Helper() + _, err := e2e.GetClients().K8s.CoreV1().Secrets(ns).Create(ctx, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns}, + Data: map[string][]byte{credentialBundleKey: bundle}, + }, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("creating credential secret %s/%s: %v", ns, name, err) + } +} + +// provisionProbeCredentials mints everything the probe pod mounts: the +// credential of the live actor, and one for an actor that does not exist. Both +// are signed by the real CA, so the difference between them is exactly the +// control-plane check and nothing else. Which one a handshake presents is +// chosen per request, which is what lets the whole suite share one pod. +func provisionProbeCredentials(t *testing.T, ctx context.Context, ns string) *probeActor { + t.Helper() + + actor, err := liveActor() + if err != nil { + t.Fatalf("preparing the actor the probe authenticates as: %v", err) + } + t.Logf("probe authenticates as actor %s/%s (uid %s)", actor.atespace, actor.name, actor.uid) + + ca := actorIdentityCA(t, ctx) + writeCredentialSecret(t, ctx, ns, actorCredentialSecret, mintActorCredential(t, ca, actor.identity())) + + // Same CA, same shape, an actor the control plane has never heard of. The + // name is scoped to the probe's namespace so a stray record cannot collide + // with anything. + writeCredentialSecret(t, ctx, ns, unknownActorCredentialSecret, mintActorCredential(t, ca, &substratex509.ActorIdentity{ + Atespace: probeAtespace, + ActorName: "no-such-actor-" + ns, + ActorUid: "00000000-0000-0000-0000-000000000000", + Purpose: substratex509.ActorIdentityPurposeAtunnel, + })) + + return actor +} diff --git a/internal/e2e/suites/sdsmint/sdsmint_test.go b/internal/e2e/suites/sdsmint/sdsmint_test.go new file mode 100644 index 000000000..40e1b4e2e --- /dev/null +++ b/internal/e2e/suites/sdsmint/sdsmint_test.go @@ -0,0 +1,515 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package sdsmint e2e-tests the egress gateway's certificate minter. +// +// sdsmint is an SDS server that mints a leaf certificate on demand for the +// SNI Envoy was asked for. Its unit tests cover the SDS protocol against a fake +// Envoy; what they cannot cover is the part that has actually broken in +// practice -- whether the deployed pod's Envoy, CA pool secret and unix socket +// line up. Every assertion here is made on a certificate that came +// off a real handshake through the real gateway. +// +// Actors do reach the gateway. hack/install-ate.sh applies +// manifests/ate-install/ate-api-server.yaml unconditionally, and that sets +// --egress-gateway-address, so every resume arms the egress redirect +// (prepareActorEgress in cmd/ateom-gvisor/main.go). What that traffic proves is +// that the path carries bytes. It never inspects the certificate it was served, +// because an actor cannot: nothing in the cluster trusts the MITM anchor, so an +// actor speaking TLS through the gateway disables verification and takes +// whatever it is handed. A leaf minted for the wrong name, one that outlives +// its --leaf-cert-ttl by an order of magnitude, one chained to a CA nobody +// installed -- every one of those carries traffic as well as a correct one, and the +// pod stays Running throughout. Those are the assertions below, and no amount +// of actor traffic makes them. +// +// Reaching sdsmint at all now means getting past the front door, which +// authenticates the caller as a named actor: an actor-identity client +// certificate at the TLS layer, and an ext_proc check that the certified actor +// is one the control plane says is running. So the suite creates one actor and +// mints a credential for it (actoridentity_test.go), and the two tests at the +// bottom assert the two ways that door says no. That makes this suite the only +// end-to-end coverage of the egress authorization leg as well as of the minter. +// +// One actor and one probe pod serve the whole suite, because standing them up +// costs more than every handshake here put together and neither is what any +// test is about. Both are torn down with the suite rather than with the test +// that triggered them; see liveActor and sharedProbe. +package sdsmint + +import ( + "context" + "crypto/x509" + "encoding/json" + "encoding/pem" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + + "github.com/agent-substrate/substrate/internal/ateclient" + "github.com/agent-substrate/substrate/internal/e2e" + "github.com/agent-substrate/substrate/internal/localca" + "github.com/agent-substrate/substrate/internal/portforward" +) + +const ( + // Where the gateway and its CA live. Both are fixed by + // manifests/ate-install/atenet-egress.yaml and hack/install-ate.sh. + egressNamespace = "ate-system" + mitmCASecret = "egress-mitm-ca-pool" + mitmCASecretKey = "pool" + mitmCAID = "mitm" + + // leafTTL is --leaf-cert-ttl on the sdsmint sidecar, and leafSkew is the + // backdating sdsmint/certauth applies to NotBefore. Their sum is the validity + // span every leaf should carry. Keep both in step with the manifest: a leaf + // that suddenly lasts hours is the failure this pair is here to catch. + leafTTL = 15 * time.Minute + leafSkew = 5 * time.Minute + + probeName = "egressprobe" +) + +// TestSdsmintMintsALeafPerSNI is the core functional assertion, and the only +// test here that exercises the minter rather than the door in front of it: +// several SNIs through one gateway come back as that many distinct +// certificates, each issued for the name that was asked for, each chaining +// directly to the MITM CA, each short-lived. +// +// Two of the names are ordinary and three deliberately are not, which is what +// pins "mint for every name" as deployed: the gateway mints for names nobody +// enumerated, at depths and in TLDs no allowlist pattern would have covered. +// This test used to assert the opposite -- that an SNI outside an allowlist +// was refused -- and its inversion is the whole content of that change. If the +// gateway is ever given an allowlist again, this test failing is the intended +// signal, not a regression: restore the refusal assertion rather than +// narrowing the names below. +// +// Every SNI is qualified with the probe's namespace, which is fresh per run, so +// each is a name Envoy has never subscribed to. Reusing one would be served +// from Envoy's live secret set and the test would pass without sdsmint having +// minted anything. +func TestSdsmintMintsALeafPerSNI(t *testing.T) { + ctx := context.Background() + + root := mitmRootCertificate(t, ctx) + probe := sharedProbe(t, ctx) + + // Nothing has to resolve: the mint happens during the inner handshake, + // before the gateway looks for an upstream. .invalid can never be delegated + // (RFC 6761), and neither the depth nor the TLD of the last three was + // reachable under the old "example.com *.example.com" allowlist, so a + // half-reverted config cannot pass this by accident. + snis := []string{ + probe.uniqueSNI("a.example.com"), + probe.uniqueSNI("b.example.com"), + probe.uniqueSNI("notallowed.invalid"), // a TLD no pattern mentioned + probe.uniqueSNI("a.b.deep.example.com"), // deeper than one "*" label + probe.uniqueSNI("UPPER.Notallowed.Invalid"), // and case-insensitively + } + + serials := map[string]string{} + for _, sni := range snis { + result := probe.handshake(t, ctx, sni) + if !result.OK { + t.Errorf("gateway refused to mint for %q at stage %q, so it is not minting for every name: %s", sni, result.Stage, result.Error) + continue + } + chain := parseChain(t, sni, result.ChainPEM) + leaf := chain[0] + + // Minted for the name that was asked for, and only that name. A leaf + // carrying anything else means the SNI Envoy policed is not the name + // the certificate authorizes. Case-insensitively, because a ClientHello + // SNI is a DNS name and Envoy is free to normalize it. + if got := leaf.DNSNames; len(got) != 1 || !strings.EqualFold(got[0], sni) { + t.Errorf("leaf for %q has DNSNames %v, want exactly [%q]", sni, got, sni) + } + // And carrying no subject: the SAN above is the whole of the leaf's + // identity, so there is no second name for the two to disagree on. + if got := leaf.Subject.String(); got != "" { + t.Errorf("leaf for %q has subject %q, want empty", sni, got) + } + + // Chains to the MITM root the installer created, verified for the SNI + // itself so the root's dNSName constraint is applied too. + opts := x509.VerifyOptions{ + DNSName: strings.ToLower(sni), + Roots: certPool(root), + Intermediates: certPool(chain[1:]...), + KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + } + if _, err := leaf.Verify(opts); err != nil { + t.Errorf("leaf for %q does not verify against the %s/%s MITM root: %v", sni, egressNamespace, mitmCASecret, err) + } + + // Short-lived, which is what bounds the damage from a leaked leaf key + // and what makes the MITM CA tolerable at all. The window is generous + // because the point is to catch a TTL that is wrong by an order of + // magnitude, not clock skew. + validity := leaf.NotAfter.Sub(leaf.NotBefore) + if want := leafTTL + leafSkew; validity < want-time.Minute || validity > want+time.Minute { + t.Errorf("leaf for %q is valid for %v, want about %v (--leaf-cert-ttl in atenet-egress.yaml)", sni, validity, want) + } + if time.Now().After(leaf.NotAfter) { + t.Errorf("leaf for %q was already expired when served (NotAfter %s)", sni, leaf.NotAfter) + } + + // One certificate per name, not one certificate reused under many + // names. Serials are compared across the whole set rather than pairwise + // so a minter that caches on something other than the SNI is caught + // wherever the collision happens. + serial := leaf.SerialNumber.Text(16) + if other, dup := serials[serial]; dup { + t.Errorf("%q and %q were served the same certificate (serial %s); the gateway is not minting per name", other, sni, serial) + } + serials[serial] = sni + + // Signed by the root itself, and the chain carries nothing but the leaf + // and that root. sdsmint has no delegated-intermediate mode any more, + // so an extra certificate in the middle means the deployed image is + // not the one this suite describes. + if len(chain) != 2 { + t.Errorf("chain for %q has %d certificates, want 2 (leaf + root)", sni, len(chain)) + continue + } + if !chain[1].Equal(root) { + t.Errorf("leaf for %q is not chained to the %s/%s root; issuer is %q", + sni, egressNamespace, mitmCASecret, chain[1].Subject) + } + + t.Logf("%s: served a %d-cert chain, leaf serial %s valid %v, issued by %q constrained to %v", + sni, len(chain), serial, validity, root.Subject.CommonName, root.PermittedDNSDomains) + } +} + +// TestGatewayRefusesANonActorWorkload is the front door's half of the egress +// story: the gateway trusts only the actor-identity CA, so a credential that +// proves "some substrate workload" no longer opens a tunnel. The probe's own +// podidentity certificate is exactly that credential, and it is the one this +// suite presented before actors had to authenticate at all. +// +// The refusal has to come from the gateway's mTLS specifically. A probe that +// failed to dial, or that got as far as the CONNECT, would be reporting +// something other than the front door turning it away. +func TestGatewayRefusesANonActorWorkload(t *testing.T) { + ctx := context.Background() + + probe := sharedProbe(t, ctx) + + sni := probe.uniqueSNI("podidentity.example.com") + result := probe.handshakeAs(t, ctx, sni, podIdentityCredentialPath) + if result.OK { + t.Fatalf("the gateway opened a tunnel for the probe's podidentity credential; its front door is accepting workloads that are not actors") + } + if result.Stage != stageGatewayTLS { + t.Fatalf("podidentity credential failed at stage %q, want %q -- the probe was stopped by something other than the front door, so this test is not checking it: %s", + result.Stage, stageGatewayTLS, result.Error) + } + t.Logf("gateway refused the non-actor credential at its front door as expected: %s", result.Error) +} + +// TestGatewayRefusesAnUnknownActor covers the check that only ext_proc can +// make. The credential here is cryptographically perfect -- signed by the real +// actor-identity CA, correct extension, correct purpose -- so Envoy completes +// the handshake, and the CONNECT is denied only because the control plane has +// no such actor. Without this, nothing distinguishes a gateway that authorizes +// on the certificate alone from one that authorizes on control-plane state, and +// the difference is whether a deleted actor's credential still works. +func TestGatewayRefusesAnUnknownActor(t *testing.T) { + ctx := context.Background() + + probe := sharedProbe(t, ctx) + + sni := probe.uniqueSNI("unknown.example.com") + result := probe.handshakeAs(t, ctx, sni, unknownActorCredentialPath) + if result.OK { + t.Fatalf("the gateway tunneled for an actor the control plane has never heard of; the ext_proc identity check is not running") + } + // A 403 on the CONNECT, not a TLS failure: the certificate was accepted and + // the identity it carries was rejected. A failure at stageGatewayTLS would + // mean the request never reached ext_proc, and the denial would prove + // nothing about the control-plane lookup. + if result.Stage != stageConnect || result.ConnectStatus != http.StatusForbidden { + t.Fatalf("unknown actor was refused at stage %q with CONNECT status %d, want %q and %d -- something other than the ext_proc identity check turned it away: %s", + result.Stage, result.ConnectStatus, stageConnect, http.StatusForbidden, result.Error) + } + t.Logf("gateway denied the unknown actor at CONNECT as expected: %s", result.Error) +} + +// mitmRootCertificate reads the trust anchor sdsmint signs under, straight +// from the secret the sidecar mounts, so the test is checking the chain against +// the CA that is actually deployed rather than one it was told about. +func mitmRootCertificate(t *testing.T, ctx context.Context) *x509.Certificate { + t.Helper() + secret, err := e2e.GetClients().K8s.CoreV1().Secrets(egressNamespace).Get(ctx, mitmCASecret, metav1.GetOptions{}) + if err != nil { + t.Fatalf("reading MITM CA pool secret %s/%s: %v", egressNamespace, mitmCASecret, err) + } + // This unmarshals the signing key along with the certificate. That is + // unavoidable -- the pool is one blob -- and acceptable only because this + // runs against a test cluster with a kubeconfig that could read the secret + // anyway. Nothing below touches the key. + pool, err := localca.Unmarshal(secret.Data[mitmCASecretKey]) + if err != nil { + t.Fatalf("parsing MITM CA pool from %s/%s key %q: %v", egressNamespace, mitmCASecret, mitmCASecretKey, err) + } + for _, ca := range pool.CAs { + if ca.ID == mitmCAID { + return ca.RootCertificate + } + } + t.Fatalf("MITM CA pool %s/%s has no CA with id %q", egressNamespace, mitmCASecret, mitmCAID) + return nil +} + +// probeClient talks to the in-cluster egress probe over a port-forward. +type probeClient struct { + // ns is the probe's namespace, randomized per run. Tests qualify their + // SNIs with it so that no name is one Envoy already holds a secret for. + ns string + baseURL string + http *http.Client +} + +// uniqueSNI qualifies suffix with the probe's namespace, producing a name no +// earlier run has asked the gateway for. Nothing withdraws a secret any more, +// so Envoy holds every name it has ever been given for the life of its process: +// an SNI a previous run used comes back from its secret set without sdsmint +// minting anything, and the test passes having tested nothing. +func (c *probeClient) uniqueSNI(suffix string) string { + return c.ns + "-" + suffix +} + +var ( + probeOnce sync.Once + probeVal *probeClient + probeErr error +) + +// sharedProbe returns the one probe pod the whole suite uses. +func sharedProbe(t *testing.T, ctx context.Context) *probeClient { + t.Helper() + probeOnce.Do(func() { + // startProbe reports failures through t, which unwinds this goroutine + // without returning. Leave something behind so the tests that run + // afterwards fail pointing at the first one instead of dereferencing + // nil. + defer func() { + if probeVal == nil && probeErr == nil { + probeErr = errors.New("setup did not complete; see the failure reported by the first test that needed the probe") + } + }() + probeVal = startProbe(t, ctx) + }) + if probeErr != nil { + t.Fatalf("starting the shared egress probe: %v", probeErr) + } + return probeVal +} + +// startProbe creates the probe's namespace, mints its credentials there, builds +// and deploys the probe, waits for it to be ready, and returns a client for it. +func startProbe(t *testing.T, ctx context.Context) *probeClient { + t.Helper() + if _, err := e2e.CheckEnv("KO_DOCKER_REPO"); err != nil { + t.Fatalf("CheckEnv failed: %v", err) + } + ns := e2e.CreateNamespace(t).Name + + provisionProbeCredentials(t, ctx, ns) + root, err := e2e.FindRepoRoot() + if err != nil { + t.Fatalf("FindRepoRoot: %v", err) + } + + tmpl, err := os.ReadFile(filepath.Join(root, "internal/e2e/fixtures/egressprobe/egressprobe.yaml.tmpl")) + if err != nil { + t.Fatalf("reading egressprobe manifest template: %v", err) + } + manifest := filepath.Join(t.TempDir(), "egressprobe.yaml") + rendered := strings.ReplaceAll(string(tmpl), "${NAMESPACE}", ns) + if err := os.WriteFile(manifest, []byte(rendered), 0o644); err != nil { + t.Fatalf("writing rendered egressprobe manifest: %v", err) + } + + applyArgs := []string{"ko", "apply", "-f", manifest} + if e2e.KubeContext != "" { + applyArgs = append(applyArgs, "--", "--context="+e2e.KubeContext) + } + e2e.RunCmdWithEnv(t, []string{"KO_CONFIG_PATH=" + root}, filepath.Join(root, "hack/run-tool.sh"), applyArgs...) + + waitForProbeReady(t, ctx, ns) + + config, err := ateclient.LoadConfig(e2e.KubeConfig, e2e.KubeContext) + if err != nil { + t.Fatalf("loading kubeconfig: %v", err) + } + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + t.Fatalf("creating k8s client for port-forward: %v", err) + } + localPort, stop, err := portforward.ServicePortForward(ctx, config, clientset, ns, probeName, 8080) + if err != nil { + t.Fatalf("port-forwarding %s/%s: %v", ns, probeName, err) + } + e2e.RegisterSuiteCleanup(stop) + + return &probeClient{ + ns: ns, + baseURL: fmt.Sprintf("http://127.0.0.1:%d", localPort), + http: &http.Client{Timeout: 90 * time.Second}, + } +} + +func waitForProbeReady(t *testing.T, ctx context.Context, ns string) { + t.Helper() + const timeout = 3 * time.Minute + deadline := time.Now().Add(timeout) + var lastState string + for time.Now().Before(deadline) { + pod, err := e2e.GetClients().K8s.CoreV1().Pods(ns).Get(ctx, probeName, metav1.GetOptions{}) + switch { + case err != nil: + lastState = err.Error() + case portforward.IsPodReady(pod): + t.Logf("probe pod %s/%s is ready", ns, probeName) + return + default: + lastState = describeProbeState(pod) + } + time.Sleep(2 * time.Second) + } + t.Fatalf("timed out after %v waiting for probe pod %s/%s to become ready: %s", timeout, ns, probeName, lastState) +} + +func describeProbeState(pod *corev1.Pod) string { + parts := []string{"phase=" + string(pod.Status.Phase)} + for _, cs := range pod.Status.ContainerStatuses { + switch { + case cs.State.Waiting != nil: + parts = append(parts, fmt.Sprintf("%s waiting: %s: %s", cs.Name, cs.State.Waiting.Reason, cs.State.Waiting.Message)) + case cs.State.Terminated != nil: + parts = append(parts, fmt.Sprintf("%s terminated: %s: %s", cs.Name, cs.State.Terminated.Reason, cs.State.Terminated.Message)) + default: + parts = append(parts, fmt.Sprintf("%s running, ready=%t", cs.Name, cs.Ready)) + } + } + return strings.Join(parts, "; ") +} + +// handshake asks the probe to complete one inner TLS handshake for sni, +// presenting the actor credential the suite minted. A refused SNI comes back as +// a result with OK false, not as an error: refusal is one of the outcomes under +// test. +func (c *probeClient) handshake(t *testing.T, ctx context.Context, sni string) handshakeResult { + t.Helper() + return c.handshakeAs(t, ctx, sni, "") +} + +// handshakeAs is handshake with a credential other than the probe's default. +// An empty credential means the default. +func (c *probeClient) handshakeAs(t *testing.T, ctx context.Context, sni, credential string) handshakeResult { + t.Helper() + endpoint := c.baseURL + "/handshake?sni=" + url.QueryEscape(sni) + if credential != "" { + endpoint += "&credential-bundle=" + url.QueryEscape(credential) + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + t.Fatalf("building probe request for %q: %v", sni, err) + } + resp, err := c.http.Do(req) + if err != nil { + t.Fatalf("calling probe for %q: %v", sni, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10)) + t.Fatalf("probe returned %d for %q: %s", resp.StatusCode, sni, strings.TrimSpace(string(body))) + } + var out handshakeResult + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + t.Fatalf("decoding probe response for %q: %v", sni, err) + } + return out +} + +// Stages a handshake can fail at, mirroring the probe's stage constants. Only +// the two the tests assert on are named here; the rest arrive as whatever +// string the probe sent and are printed in the failure. +const ( + stageGatewayTLS = "gateway_tls" + stageConnect = "connect" +) + +// handshakeResult mirrors the probe's response body. It is duplicated rather +// than imported because the probe is package main. +type handshakeResult struct { + SNI string `json:"sni"` + Credential string `json:"credential"` + OK bool `json:"ok"` + // Stage is where a failed handshake stopped. Asserting on it rather than + // on Error is what keeps "the front door refused the certificate" and "the + // door opened and ext_proc said no" from being the same test: they are + // different hops, and their messages are only incidentally different. + Stage string `json:"stage"` + // ConnectStatus is the status the gateway answered the CONNECT with, set + // only when Stage is stageConnect. + ConnectStatus int `json:"connect_status"` + Error string `json:"error"` + ChainPEM string `json:"chain_pem"` +} + +func parseChain(t *testing.T, sni, chainPEM string) []*x509.Certificate { + t.Helper() + var chain []*x509.Certificate + rest := []byte(chainPEM) + for { + var block *pem.Block + block, rest = pem.Decode(rest) + if block == nil { + break + } + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + t.Fatalf("parsing certificate served for %q: %v", sni, err) + } + chain = append(chain, cert) + } + if len(chain) == 0 { + t.Fatalf("no certificates in the chain served for %q", sni) + } + return chain +} + +func certPool(certs ...*x509.Certificate) *x509.CertPool { + pool := x509.NewCertPool() + for _, cert := range certs { + pool.AddCert(cert) + } + return pool +} diff --git a/internal/e2e/suites/sdsmint/testmain_test.go b/internal/e2e/suites/sdsmint/testmain_test.go new file mode 100644 index 000000000..ca19d4b95 --- /dev/null +++ b/internal/e2e/suites/sdsmint/testmain_test.go @@ -0,0 +1,41 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sdsmint + +import ( + "os" + "testing" + + "github.com/agent-substrate/substrate/internal/e2e" +) + +// Setup runs before the suite. The probe is per-test, because it lives in the +// randomized namespace the test creates, so there is nothing to do here. +func Setup() {} + +// Teardown runs after the suite. +func Teardown() {} + +func run(m *testing.M) int { + Setup() + defer Teardown() + + // return allows the deferred Teardown to run. + return e2e.RunTestMain(m) +} + +func TestMain(m *testing.M) { + os.Exit(run(m)) +} diff --git a/internal/e2e/testmain.go b/internal/e2e/testmain.go index a2afad798..9c66c1f49 100644 --- a/internal/e2e/testmain.go +++ b/internal/e2e/testmain.go @@ -19,6 +19,7 @@ import ( goflag "flag" "fmt" "os" + "sync" "testing" "github.com/spf13/pflag" @@ -30,6 +31,28 @@ var ( KubeContext string ) +var ( + suiteCleanupsMu sync.Mutex + suiteCleanups []func() +) + +func RegisterSuiteCleanup(fn func()) { + suiteCleanupsMu.Lock() + defer suiteCleanupsMu.Unlock() + suiteCleanups = append(suiteCleanups, fn) +} + +func runSuiteCleanups() { + suiteCleanupsMu.Lock() + fns := suiteCleanups + suiteCleanups = nil + suiteCleanupsMu.Unlock() + + for i := len(fns) - 1; i >= 0; i-- { + fns[i]() + } +} + func bindFlags() { pflag.BoolVar(&RunE2E, "e2e", false, "run e2e tests") pflag.BoolVar(&NoColor, "no-color", false, "disable colors in output") @@ -82,6 +105,7 @@ func runAndCleanup(m *testing.M) int { // namespace takes those pods with it before anyone — a developer or CI's // post-failure log dump — can read them. code := m.Run() + runSuiteCleanups() if code != 0 { RetainNamespaces() return code diff --git a/internal/localca/localca.go b/internal/localca/localca.go index 3078435ed..72da352c4 100644 --- a/internal/localca/localca.go +++ b/internal/localca/localca.go @@ -18,9 +18,12 @@ package localca import ( "crypto" + "crypto/ecdsa" "crypto/ed25519" + "crypto/elliptic" "crypto/rand" "crypto/x509" + "crypto/x509/pkix" "encoding/json" "encoding/pem" "fmt" @@ -33,11 +36,42 @@ type Pool struct { type CA struct { ID string - SigningKey crypto.PrivateKey + SigningKey crypto.Signer RootCertificate *x509.Certificate IntermediateCertificates []*x509.Certificate } +// Validate reports whether the CA is well formed enough to sign with. +func (ca *CA) Validate() error { + if ca == nil { + return fmt.Errorf("ca: is nil") + } + if ca.RootCertificate == nil { + return fmt.Errorf("ca cert: missing") + } + if ca.SigningKey == nil { + return fmt.Errorf("ca key: missing") + } + if !ca.RootCertificate.IsCA { + return fmt.Errorf("ca cert: %q is not a CA certificate", ca.RootCertificate.Subject) + } + if !keyMatchesCert(ca.RootCertificate, ca.SigningKey) { + return fmt.Errorf("ca key: public key does not match the certificate for %q", ca.RootCertificate.Subject) + } + return nil +} + +// keyMatchesCert catches a mismatched cert/key pair at load rather than at the +// first handshake. +func keyMatchesCert(cert *x509.Certificate, key crypto.Signer) bool { + type equaler interface{ Equal(crypto.PublicKey) bool } + pub, ok := cert.PublicKey.(equaler) + if !ok { + return true + } + return pub.Equal(key.Public()) +} + type serializedPool struct { CAs []*serializedCA } @@ -58,9 +92,15 @@ func Marshal(ca *Pool) ([]byte, error) { caWire.ID = ca.ID + // An external signer has no exportable key material, so this is the + // point where "the key lives in a KMS" stops being marshalable. Name + // that case, because x509's own error ("unknown key type") reads like a + // bug in this code rather than a deliberate property of the signer. signingKeyPKCS8, err := x509.MarshalPKCS8PrivateKey(ca.SigningKey) if err != nil { - return nil, fmt.Errorf("while serializing signing key to PKCS#8: %w", err) + return nil, fmt.Errorf("while serializing signing key for CA %q to PKCS#8: %w "+ + "(a signer that holds no exportable key material, such as a KMS or HSM signer, "+ + "cannot be written to a pool file; keep it in its own store)", ca.ID, err) } caWire.SigningKeyPKCS8 = signingKeyPKCS8 @@ -119,9 +159,13 @@ func Unmarshal(wireBytes []byte) (*Pool, error) { return pool, nil } -func parsePrivateKey(pkcs8 []byte, pemData string) (crypto.PrivateKey, error) { +func parsePrivateKey(pkcs8 []byte, pemData string) (crypto.Signer, error) { if len(pkcs8) != 0 { - return x509.ParsePKCS8PrivateKey(pkcs8) + key, err := x509.ParsePKCS8PrivateKey(pkcs8) + if err != nil { + return nil, err + } + return asSigner(key) } block, _ := pem.Decode([]byte(pemData)) @@ -130,17 +174,28 @@ func parsePrivateKey(pkcs8 []byte, pemData string) (crypto.PrivateKey, error) { } if key, err := x509.ParsePKCS8PrivateKey(block.Bytes); err == nil { - return key, nil + return asSigner(key) } if key, err := x509.ParseECPrivateKey(block.Bytes); err == nil { - return key, nil + return asSigner(key) } if key, err := x509.ParsePKCS1PrivateKey(block.Bytes); err == nil { - return key, nil + return asSigner(key) } return nil, fmt.Errorf("unsupported private key PEM type %q", block.Type) } +// asSigner narrows a parsed key to the signing interface the CAs actually use. +// X25519 keys parse cleanly out of PKCS#8 and cannot sign, so this is a real +// case and not a defensive assertion. +func asSigner(key any) (crypto.Signer, error) { + signer, ok := key.(crypto.Signer) + if !ok { + return nil, fmt.Errorf("private key of type %T cannot sign", key) + } + return signer, nil +} + func parseCertificate(der []byte, pemData string) (*x509.Certificate, error) { if len(der) != 0 { return x509.ParseCertificate(der) @@ -156,14 +211,57 @@ func parseCertificate(der []byte, pemData string) (*x509.Certificate, error) { return x509.ParseCertificate(block.Bytes) } +// KeyType selects the algorithm of a generated CA's signing key. +type KeyType string + +const ( + // KeyTypeED25519 is the default. It is the smallest and fastest option and + // is what substrate's internal CAs have always used. + KeyTypeED25519 KeyType = "ed25519" + // KeyTypeECDSAP256 exists for CAs whose certificates are validated by + // clients outside substrate's control. Ed25519 in a chain needs OpenSSL + // 1.1.1+ or Go 1.13+, which is fine for anything substrate ships and not + // something to assume of an arbitrary process running inside an actor. + KeyTypeECDSAP256 KeyType = "ecdsa-p256" +) + +// GenerateOptions configures GenerateCA. The zero value, apart from ID, +// reproduces what GenerateED25519CA has always produced. +type GenerateOptions struct { + // ID names the CA within its Pool. + ID string + // CommonName is the subject CN. Empty leaves the subject empty, which is + // what the internal CAs do -- nothing authenticates on their name. + CommonName string + // KeyType defaults to KeyTypeED25519. + KeyType KeyType + // Lifetime defaults to 365 days. + Lifetime time.Duration +} + +// GenerateED25519CA creates an unconstrained 365-day Ed25519 CA. It is the +// long-standing shape of substrate's internal CAs, kept as its own function +// because every existing caller wants exactly this. func GenerateED25519CA(id string) (*CA, error) { - rootPubKey, rootPrivKey, err := ed25519.GenerateKey(rand.Reader) + return GenerateCA(GenerateOptions{ID: id}) +} + +// GenerateCA creates a self-signed CA with its own freshly generated key. +func GenerateCA(opts GenerateOptions) (*CA, error) { + if opts.Lifetime == 0 { + opts.Lifetime = 365 * 24 * time.Hour + } + if opts.KeyType == "" { + opts.KeyType = KeyTypeED25519 + } + + rootPrivKey, err := generateKey(opts.KeyType) if err != nil { - return nil, fmt.Errorf("while generating root key: %w", err) + return nil, err } notBefore := time.Now() - notAfter := notBefore.Add(365 * 24 * time.Hour) + notAfter := notBefore.Add(opts.Lifetime) rootTemplate := &x509.Certificate{ NotBefore: notBefore, @@ -172,8 +270,11 @@ func GenerateED25519CA(id string) (*CA, error) { BasicConstraintsValid: true, KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, } + if opts.CommonName != "" { + rootTemplate.Subject = pkix.Name{CommonName: opts.CommonName} + } - rootDER, err := x509.CreateCertificate(rand.Reader, rootTemplate, rootTemplate, rootPubKey, rootPrivKey) + rootDER, err := x509.CreateCertificate(rand.Reader, rootTemplate, rootTemplate, rootPrivKey.Public(), rootPrivKey) if err != nil { return nil, fmt.Errorf("while generating root certificate: %w", err) } @@ -184,9 +285,28 @@ func GenerateED25519CA(id string) (*CA, error) { } return &CA{ - ID: id, + ID: opts.ID, SigningKey: rootPrivKey, RootCertificate: rootCert, // No intermediates. }, nil } + +func generateKey(kt KeyType) (crypto.Signer, error) { + switch kt { + case KeyTypeED25519: + _, key, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + return nil, fmt.Errorf("while generating root key: %w", err) + } + return key, nil + case KeyTypeECDSAP256: + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return nil, fmt.Errorf("while generating root key: %w", err) + } + return key, nil + default: + return nil, fmt.Errorf("unsupported key type %q, want one of %q or %q", kt, KeyTypeED25519, KeyTypeECDSAP256) + } +} diff --git a/internal/localca/localca_test.go b/internal/localca/localca_test.go index 0cca4ef6f..39d92c96c 100644 --- a/internal/localca/localca_test.go +++ b/internal/localca/localca_test.go @@ -16,13 +16,17 @@ package localca import ( "bytes" + "crypto" + "crypto/ecdsa" "crypto/ed25519" + "crypto/elliptic" "crypto/rand" "crypto/rsa" "crypto/x509" "crypto/x509/pkix" "encoding/json" "encoding/pem" + "io" "math/big" "strings" "testing" @@ -298,3 +302,184 @@ func TestUnmarshalErrors(t *testing.T) { }) } } + +// externalSigner stands in for a KMS or HSM signer: it can sign, but it holds +// no exportable key material. +type externalSigner struct{ inner ed25519.PrivateKey } + +func (e externalSigner) Public() crypto.PublicKey { return e.inner.Public() } +func (e externalSigner) Sign(r io.Reader, digest []byte, opts crypto.SignerOpts) ([]byte, error) { + return e.inner.Sign(r, digest, opts) +} + +// The point of typing SigningKey as crypto.Signer is that a key living outside +// the process can be substituted. Verify that actually works end to end: such +// a signer can issue certificates, and Marshal refuses it with an explanation +// rather than x509's "unknown key type". +func TestExternalSignerCanIssueButCannotBeMarshalled(t *testing.T) { + base, err := GenerateED25519CA("external") + if err != nil { + t.Fatalf("GenerateED25519CA: %v", err) + } + ca := &CA{ + ID: base.ID, + SigningKey: externalSigner{inner: base.SigningKey.(ed25519.PrivateKey)}, + RootCertificate: base.RootCertificate, + } + + // Issuing works: x509.CreateCertificate only needs Public and Sign. + leafPub, _, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatalf("generating leaf key: %v", err) + } + leafDER, err := x509.CreateCertificate(rand.Reader, &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "leaf"}, + NotBefore: time.Now().Add(-time.Minute), + NotAfter: time.Now().Add(time.Hour), + }, ca.RootCertificate, leafPub, ca.SigningKey) + if err != nil { + t.Fatalf("signing with an external signer: %v", err) + } + if _, err := x509.ParseCertificate(leafDER); err != nil { + t.Fatalf("parsing the issued leaf: %v", err) + } + + // Serializing does not, and must say why. + _, err = Marshal(&Pool{CAs: []*CA{ca}}) + if err == nil { + t.Fatal("Marshal serialized a signer with no exportable key material") + } + if !strings.Contains(err.Error(), "KMS") { + t.Errorf("error does not explain the external-signer case: %v", err) + } +} + +func TestGenerateCAKeyTypes(t *testing.T) { + for _, tc := range []struct { + keyType KeyType + check func(*testing.T, crypto.Signer) + }{ + {KeyTypeED25519, func(t *testing.T, k crypto.Signer) { + if _, ok := k.(ed25519.PrivateKey); !ok { + t.Errorf("key type = %T, want ed25519.PrivateKey", k) + } + }}, + {KeyTypeECDSAP256, func(t *testing.T, k crypto.Signer) { + ec, ok := k.(*ecdsa.PrivateKey) + if !ok { + t.Fatalf("key type = %T, want *ecdsa.PrivateKey", k) + } + if ec.Curve != elliptic.P256() { + t.Errorf("curve = %v, want P-256", ec.Curve.Params().Name) + } + }}, + } { + t.Run(string(tc.keyType), func(t *testing.T) { + ca, err := GenerateCA(GenerateOptions{ID: "k", KeyType: tc.keyType}) + if err != nil { + t.Fatalf("GenerateCA: %v", err) + } + tc.check(t, ca.SigningKey) + + // Whatever the algorithm, the result has to survive the pool. + data, err := Marshal(&Pool{CAs: []*CA{ca}}) + if err != nil { + t.Fatalf("Marshal: %v", err) + } + restored, err := Unmarshal(data) + if err != nil { + t.Fatalf("Unmarshal: %v", err) + } + tc.check(t, restored.CAs[0].SigningKey) + }) + } + + if _, err := GenerateCA(GenerateOptions{ID: "k", KeyType: "rsa-8192"}); err == nil { + t.Error("GenerateCA accepted an unknown key type") + } +} + +func TestGenerateCALifetimeAndCommonName(t *testing.T) { + ca, err := GenerateCA(GenerateOptions{ID: "x", CommonName: "my ca", Lifetime: 2 * time.Hour}) + if err != nil { + t.Fatalf("GenerateCA: %v", err) + } + if got := ca.RootCertificate.Subject.CommonName; got != "my ca" { + t.Errorf("CN = %q, want %q", got, "my ca") + } + if got := ca.RootCertificate.NotAfter.Sub(ca.RootCertificate.NotBefore); got != 2*time.Hour { + t.Errorf("lifetime = %v, want 2h", got) + } +} + +func TestValidateAcceptsAGeneratedCA(t *testing.T) { + ca, err := GenerateED25519CA("generated") + if err != nil { + t.Fatalf("GenerateED25519CA: %v", err) + } + // GenerateCA is the one path that satisfies Validate by construction. If + // this ever fails the two have drifted apart. + if err := ca.Validate(); err != nil { + t.Errorf("Validate on a freshly generated CA: %v", err) + } + // Intermediates are a supported shape here, whatever individual consumers + // do about them. + ca.IntermediateCertificates = []*x509.Certificate{ca.RootCertificate} + if err := ca.Validate(); err != nil { + t.Errorf("Validate on a CA carrying intermediates: %v", err) + } +} + +func TestValidateRejectsAMalformedCA(t *testing.T) { + good, err := GenerateED25519CA("good") + if err != nil { + t.Fatalf("GenerateED25519CA(good): %v", err) + } + other, err := GenerateED25519CA("other") + if err != nil { + t.Fatalf("GenerateED25519CA(other): %v", err) + } + + // A cert that is not a CA. Signing leaves off it produces a chain that + // nothing will accept as an issuer. + leafKeyPub, leafKeyPriv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatalf("ed25519.GenerateKey(): %v", err) + } + leafTemplate := &x509.Certificate{ + SerialNumber: big.NewInt(1), + NotBefore: time.Now(), + NotAfter: time.Now().Add(24 * time.Hour), + IsCA: false, + BasicConstraintsValid: true, + } + leafDER, err := x509.CreateCertificate(rand.Reader, leafTemplate, good.RootCertificate, leafKeyPub, good.SigningKey) + if err != nil { + t.Fatalf("CreateCertificate(): %v", err) + } + leafCert, err := x509.ParseCertificate(leafDER) + if err != nil { + t.Fatalf("ParseCertificate(): %v", err) + } + + tests := []struct { + name string + ca *CA + }{ + {"nil", nil}, + {"no root certificate", &CA{ID: "x", SigningKey: good.SigningKey}}, + {"no signing key", &CA{ID: "x", RootCertificate: good.RootCertificate}}, + {"root is not a CA", &CA{ID: "x", RootCertificate: leafCert, SigningKey: leafKeyPriv}}, + // The case Unmarshal cannot catch: key and certificate are each well + // formed and are simply not a pair. + {"key does not match the certificate", &CA{ID: "x", RootCertificate: good.RootCertificate, SigningKey: other.SigningKey}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := tt.ca.Validate(); err == nil { + t.Error("Validate() = nil, want error") + } + }) + } +} diff --git a/manifests/ate-install/atenet-egress.yaml b/manifests/ate-install/atenet-egress.yaml index 591ef31fc..72b28588a 100644 --- a/manifests/ate-install/atenet-egress.yaml +++ b/manifests/ate-install/atenet-egress.yaml @@ -35,9 +35,25 @@ metadata: namespace: ate-system data: envoy.yaml: | + # Requires Envoy 1.37 or newer: on_demand_secret and cert_mappers.sni do not + # exist in earlier releases. + # SDS refuses to start without a node id and cluster. + node: + id: atenet-egress + cluster: atenet-egress + admin: address: socket_address: { address: 0.0.0.0, port_value: 15000 } + + # Required by the mitm_listener below. An internal listener has no socket, + # so it is reachable only through a cluster with an envoy_internal_address; + # without this extension registered, that listener silently fails to load. + bootstrap_extensions: + - name: envoy.bootstrap.internal_listener + typed_config: + "@type": type.googleapis.com/envoy.extensions.bootstrap.internal_listener.v3.InternalListener + static_resources: listeners: - name: egress @@ -100,7 +116,12 @@ data: routes: - match: { connect_matcher: {} } route: - cluster: egress_forward_proxy + # the tunnel is handed to the MITM + # listener below, which terminates the tunnelled TLS and + # re-originates it. Sending it straight to + # egress_forward_proxy here would be a raw TCP passthrough + # with no per-destination visibility at all. + cluster: mitm_internal upgrade_configs: - upgrade_type: CONNECT connect_config: {} @@ -137,6 +158,79 @@ data: response_body_mode: NONE request_trailer_mode: SKIP response_trailer_mode: SKIP + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + + # --------------------------------------------------------------------- + # Listener B: the MITM leg. No socket of its own -- it is reachable only + # through the mitm_internal cluster's envoy_internal_address, so it + # cannot be dialled from outside this pod. + # + # It exists because the CONNECT authority is an IP:port (atunnel takes it + # from SO_ORIGINAL_DST and rejects hostnames), so the destination + # hostname is visible only inside the tunnel. This leg terminates the + # tunnelled TLS with a leaf sdsmint mints for the SNI, reads the real + # Host, and re-originates. + # + # Two chains, selected by what the tunnel actually carries. tls_inspector + # tags a ClientHello "tls" and anything else "raw_buffer", so a cleartext + # HTTP tunnel gets an HTTP chain instead of being fed to a TLS transport + # socket, which would fail the handshake and close the tunnel. + # --------------------------------------------------------------------- + - name: mitm_listener + stat_prefix: mitm + internal_listener: {} + listener_filters: + - name: envoy.filters.listener.tls_inspector + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.listener.tls_inspector.v3.TlsInspector + filter_chains: + - filter_chain_match: + transport_protocol: tls + filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: mitm_http + # This is the leg that knows where the traffic actually went. + # REQUESTED_SERVER_NAME is the SNI the leaf was minted for and it + # should always equal the authority -- dynamic_forward_proxy + # resolves from the authority, so a mismatch means the name that + # was policed is not the name that was dialled. + access_log: + - name: envoy.access_loggers.file + typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/stdout + log_format: + json_format: + leg: mitm + time: "%START_TIME%" + sni: "%REQUESTED_SERVER_NAME%" + authority: "%REQ(:AUTHORITY)%" + method: "%REQ(:METHOD)%" + path: "%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%" + protocol: "%PROTOCOL%" + status: "%RESPONSE_CODE%" + flags: "%RESPONSE_FLAGS%" + duration_ms: "%DURATION%" + bytes_sent: "%BYTES_SENT%" + bytes_rcvd: "%BYTES_RECEIVED%" + upstream: "%UPSTREAM_HOST%" + upstream_failure: "%UPSTREAM_TRANSPORT_FAILURE_REASON%" + termination: "%CONNECTION_TERMINATION_DETAILS%" + route_config: + name: local + virtual_hosts: + - name: all + domains: ["*"] + routes: + - match: { prefix: "/" } + route: + cluster: egress_forward_proxy + timeout: 30s + http_filters: - name: envoy.filters.http.dynamic_forward_proxy typed_config: "@type": type.googleapis.com/envoy.extensions.filters.http.dynamic_forward_proxy.v3.FilterConfig @@ -146,7 +240,143 @@ data: - name: envoy.filters.http.router typed_config: "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + # REQUIRED. Without it an sdsmint outage does not fail a + # first-contact handshake, it pauses it forever. + transport_socket_connect_timeout: 5s + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext + common_tls_context: + custom_tls_certificate_selector: + name: on-demand + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.cert_selectors.on_demand_secret.v3.Config + config_source: + api_config_source: + api_type: DELTA_GRPC + transport_api_version: V3 + grpc_services: + - envoy_grpc: + cluster_name: sds_mint + certificate_mapper: + name: sni + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.cert_mappers.sni.v3.SNI + # Reached only by a client that sent no SNI, i.e. one + # dialing by IP. It gets a leaf for this name, which + # resolves nowhere and so matches no destination it + # could have meant -- and since the client prints the + # name in its verification error, the name says why. + default_value: "sni-required.egress.ate.invalid" # default_value is required. + # A resumed session skips certificate selection entirely, which + # would hand a client a secret minted for a different name. + disable_stateless_session_resumption: true + disable_stateful_session_resumption: true + + # The cleartext chain. Nothing to terminate and nothing to mint: the + # Host header is already in the clear, so this leg reads the + # destination directly rather than from a certificate it issued. + - filter_chain_match: + transport_protocol: raw_buffer + filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: mitm_cleartext + access_log: + - name: envoy.access_loggers.file + typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/stdout + log_format: + json_format: + leg: cleartext + time: "%START_TIME%" + authority: "%REQ(:AUTHORITY)%" + method: "%REQ(:METHOD)%" + path: "%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%" + protocol: "%PROTOCOL%" + status: "%RESPONSE_CODE%" + flags: "%RESPONSE_FLAGS%" + duration_ms: "%DURATION%" + bytes_sent: "%BYTES_SENT%" + bytes_rcvd: "%BYTES_RECEIVED%" + upstream: "%UPSTREAM_HOST%" + upstream_failure: "%UPSTREAM_TRANSPORT_FAILURE_REASON%" + termination: "%CONNECTION_TERMINATION_DETAILS%" + route_config: + name: cleartext + virtual_hosts: + - name: all + domains: ["*"] + routes: + - match: { prefix: "/" } + route: + cluster: egress_forward_proxy_cleartext + timeout: 30s + http_filters: + # Same reasoning as the TLS chain: resolve from the request's own + # Host, so the name that was policed is the name that is dialled. + - name: envoy.filters.http.dynamic_forward_proxy + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.dynamic_forward_proxy.v3.FilterConfig + dns_cache_config: + name: egress_dns_cache + dns_lookup_family: V4_ONLY + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + clusters: + # The only way into mitm_listener. An internal address, not a socket, so + # the MITM leg has no listening port anywhere in the pod's netns. + - name: mitm_internal + connect_timeout: 1s + load_assignment: + cluster_name: mitm_internal + endpoints: + - lb_endpoints: + - endpoint: + address: + envoy_internal_address: + server_listener_name: mitm_listener + + - name: sds_mint + type: STATIC + connect_timeout: 1s + # One HTTP/2 connection carrying one DELTA_GRPC stream per name in + # flight. max_requests is the one that matters, and it is subtler than + # a burst limit: Envoy does not close the stream once the secret + # arrives, it holds it open for as long as the secret is live. So this + # is not "concurrent mints", it is a hard cap on the size of the live + # secret set. Past the cap Envoy is refused a mint rather than queued + # for one, which surfaces as a handshake failure and looks exactly like + # the allowlist denying the name -- silently, at a threshold nothing + # else in this file mentions. 32768 is deliberately ~3x the + # 10,000-name target rather than equal to it: the memory guards above + # are what should bind first, at ~13,200, and a limit that trips before + # them would replace a diagnosable overload with a mystery. Raise it + # when you raise the target. + circuit_breakers: + thresholds: + - priority: DEFAULT + max_connections: 128 + max_pending_requests: 32768 + max_requests: 32768 + max_retries: 8 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + explicit_http_config: + http2_protocol_options: {} + load_assignment: + cluster_name: sds_mint + endpoints: + - lb_endpoints: + - endpoint: + address: + pipe: {path: /var/run/sdsmint/sdsmint.sock} # ext_proc gRPC server = the atenet router, co-located in this pod as a # sidecar and called over localhost (same topology the ingress gateway uses # for its dataplane + ext_proc). @@ -168,8 +398,9 @@ data: socket_address: address: 127.0.0.1 port_value: 50051 - # Dials the terminated-CONNECT target (IP:port from the authority). atunnel - # always sends an IP:port, so DNS resolution is effectively a passthrough. + # The only cluster that dials the internet. Reached from the MITM leg, so + # the host it resolves comes from the decrypted request's own Host header + # rather than from the CONNECT authority. - name: egress_forward_proxy lb_policy: CLUSTER_PROVIDED connect_timeout: 5s @@ -180,6 +411,63 @@ data: dns_cache_config: name: egress_dns_cache dns_lookup_family: V4_ONLY + # The MITM must not weaken upstream authentication. Envoy decrypted the + # actor's TLS with a leaf of its own; it still sends the real SNI here + # and still validates the real origin's certificate against the public + # roots. Without this the re-originated request would leave the pod in + # plaintext. + # + # Spelled through typed_extension_protocol_options because the + # cluster-level upstream_http_protocol_options field is deprecated. + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + upstream_http_protocol_options: + auto_sni: true + auto_san_validation: true + explicit_http_config: + http_protocol_options: {} + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + validation_context: + trusted_ca: + filename: /etc/ssl/certs/ca-certificates.crt + + # The cleartext chain's upstream. Identical to egress_forward_proxy but + # without the TLS transport socket: the actor sent plaintext and this + # gateway does not upgrade it. Re-originating over TLS here would mean + # answering a request the actor made in the clear with a connection it + # never asked for and cannot see the peer of. + # + # It shares egress_dns_cache with egress_forward_proxy -- same name, same + # config, so both legs resolve through one cache. + - name: egress_forward_proxy_cleartext + lb_policy: CLUSTER_PROVIDED + connect_timeout: 5s + cluster_type: + name: envoy.clusters.dynamic_forward_proxy + typed_config: + "@type": type.googleapis.com/envoy.extensions.clusters.dynamic_forward_proxy.v3.ClusterConfig + dns_cache_config: + name: egress_dns_cache + dns_lookup_family: V4_ONLY + # Envoy refuses to build a dynamic forward proxy cluster without + # auto_sni and auto_san_validation unless this is set, because for + # the usual TLS case resolving the host from a header and then not + # validating against it is a real hole. There is no upstream TLS on + # this cluster, so there is no SNI to send and no SAN to check -- + # the two settings it is asking for would be no-ops. Setting this + # on egress_forward_proxy, which does re-originate TLS, would be + # the hole it is named after. + allow_insecure_cluster_options: true + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + explicit_http_config: + http_protocol_options: {} --- apiVersion: apps/v1 kind: Deployment @@ -199,15 +487,123 @@ spec: app: atenet-egress spec: serviceAccountName: atenet-egress + # sdsmint and envoy containers must share a UID: sdsmint creates the SDS socket mode + # 0600, and Envoy has to be able to open it. The stock Envoy image runs as + # UID 101 and ko's distroless base as 65532, so left alone they would not + # agree and Envoy would fail every handshake with a connection error on + # the sds_mint cluster. Neither listening port is privileged. securityContext: + runAsUser: 65532 + runAsGroup: 65532 + runAsNonRoot: true + fsGroup: 65532 # Allow the non-root envoy user to bind :443. sysctls: - name: net.ipv4.ip_unprivileged_port_start value: "0" terminationGracePeriodSeconds: 60 + initContainers: + # A native sidecar, not a plain init container. Ordering matters in both + # directions: sdsmint is serving before Envoy loads its config, and it + # outlives Envoy on shutdown, so a drain does not pull the SDS server out + # from under handshakes that are still in flight. + - name: sdsmint + # The same image as the ext-proc container below, run as a different + # subcommand. Sharing the binary is what keeps the two from drifting; + # staying two CONTAINERS is what keeps the MITM signing key off the + # data plane and gives the minter the separate memory limit the header + # of this file depends on. + image: ko://github.com/agent-substrate/substrate/cmd/atenet + restartPolicy: Always + args: + - "sdsmint" + - "--uds-path=/var/run/sdsmint/sdsmint.sock" + - "--ca-pool-path=/run/ca-state/mitm-pool.json" + - "--ca-id=mitm" + # --leaf-cert-ttl sets two things at once. It stamps the leaf's notAfter, + # and sdsmint derives the xDS resource TTL from it at half its length -- + # so at 15m, Envoy drops each cached secret after 7m30s and the next + # handshake for that name re-mints. That drop is the only thing that + # replaces a leaf: nothing pushes, and rotation and the idle sweep are + # both gone. Cost follows traffic, since a name nobody asks for is + # dropped and never minted again. + # + # Do not reach for a longer TTL to cut minting. The two move together, + # and the failure mode at the far end is silent: without the resource + # TTL, Envoy holds a secret indefinitely, serves the leaf past its + # notAfter, and the handshake still completes -- actors cannot object + # because they reach the gateway with verification off, nothing in the + # cluster trusting the MITM anchor. Measured against Envoy 1.37.5 in + # poc/sdsmint/expiry, both with the TTL and without it. + # + # It does not bound validity exactly: leaves are back-dated 5m for + # clock skew and NotAfter is ttl past issuance, so the window a client + # would accept is ttl+5m. See Signer.Sign in + # cmd/atenet/internal/sdsmint/certauth. + - "--leaf-cert-ttl=15m" + - "--log-level=info" + env: + # Go's nearest thing to Envoy's fixed_heap, and it is not as close as + # it looks. It bounds the GARBAGE, not the live set: almost everything + # resident here is reachable -- three goroutine stacks per subscription + # and their timers -- and no amount of GC pressure reclaims a live + # subscription. What it does buy is real though, because minting + # allocates hard, and a soft ceiling meaningfully trims the peak RSS + # that churn produces. Set below the container limit so the + # collector starts working before the kernel starts killing. If sdsmint + # ever reaches this figure with a genuinely live heap it will thrash the + # GC and then OOM anyway; that is the missing shed path described in the + # header, not a misconfiguration here. + - name: GOMEMLIMIT + value: "640MiB" + # Both of these scale with the live secret count, not with request rate. + # + # memory ~44,750 B per live secret x 10,000 => ~427 MiB, against a + # 768Mi limit. That limit is 0.85 + # x Envoy's fixed_heap, which is the ordering invariant from the + # header reduced to one number: at the ~13,200 names where Envoy + # starts refusing, this container is at 73% and still alive. See + # the header for why this is stream overhead and not + # certificates. The request is measured steady state rather than + # a token floor, on purpose -- the scheduler places pods on + # requests, so a 64Mi request on a container that grows to 427 + # MiB does not fail to schedule, it schedules onto a node it + # will later destabilise. Understating it is how this pod takes + # its neighbours down with it. + # cpu ~126 mcore was the measured floor at 10k with no traffic at + # all, when 50 re-mints a second ran on a rotation ticker. That + # steady-state cost is now zero: nothing re-mints, so a name + # costs one signature ever and an idle gateway signs nothing. + # The request is sized for the burst instead. A single mint is + # the ~600us the PoC measured, so a 500/s wave of cold names is + # ~0.3 core -- which is why this is requested and not limited. + # Throttling the signer turns a CPU shortage into handshake + # timeouts across every cold SNI at once. Left at 200m rather + # than trimmed to the new floor because the burst, not the + # steady state, is what this has to absorb. + resources: + requests: + cpu: 200m + memory: 512Mi + limits: + memory: 768Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumeMounts: + - name: sdsmint-sock + mountPath: /var/run/sdsmint + # The MITM CA. Mounted here and nowhere else -- in particular not on + # the Envoy container. Keeping the signing key out of the data plane is + # the reason this is an SDS server rather than a file on disk. + - name: ca-state + mountPath: /run/ca-state + readOnly: true containers: - name: envoy - image: envoyproxy/envoy:v1.34-latest + image: envoyproxy/envoy:v1.37-latest@sha256:1c2b79776c6e3b38e8b0113b825e6a599f9bfc08d680c199d80bf8964856c529 securityContext: allowPrivilegeEscalation: false capabilities: @@ -274,6 +670,8 @@ spec: - name: drain-signal mountPath: /var/run/atenet readOnly: true + - name: sdsmint-sock + mountPath: /var/run/sdsmint # Co-located ext_proc server: the same atenet router binary the ingress # gateway runs, started with --mode=egress so it serves only the egress # ext_proc handler (no xDS server, no ActorTemplate controller, no @@ -342,6 +740,8 @@ spec: name: atenet-egress - name: drain-signal emptyDir: {} + - name: sdsmint-sock + emptyDir: {} - name: servicedns projected: sources: @@ -371,6 +771,14 @@ spec: - name: actor-id-ca-certs secret: secretName: actor-id-ca-certs + - name: ca-state + projected: + sources: + - secret: + name: egress-mitm-ca-pool + items: + - key: pool + path: mitm-pool.json --- apiVersion: v1 kind: Service