localca: Rotation support part 1

cmd/ateapi/internal/actoridentity/actoridentity.go

@@ -50,7 +50,7 @@ type Server struct {
 
 	// TODO: Cache the signing keys in memory, so we don't read from a file every time.
 	actorIDJWTPoolFile string
-	actorIDCAPoolFile  string
+	actorIDCAPool      localca.Pool
 
 	// store is the actor database. MintCert consults it to confirm the caller
 	// is entitled to the actor it is asking for a credential for.
@@ -60,11 +60,11 @@ type Server struct {
 
 var _ ateapipb.ActorIdentityServer = (*Server)(nil)
 
-func New(actorIdentityJWTIssuer, actorIDJWTPoolFile, actorIDCAPoolFile string, store store.Interface, workers *workercache.Cache) *Server {
+func New(actorIdentityJWTIssuer, actorIDJWTPoolFile string, actorIDCAPool localca.Pool, store store.Interface, workers *workercache.Cache) *Server {
 	return &Server{
 		actorIdentityJWTIssuer: actorIdentityJWTIssuer,
 		actorIDJWTPoolFile:     actorIDJWTPoolFile,
-		actorIDCAPoolFile:      actorIDCAPoolFile,
+		actorIDCAPool:          actorIDCAPool,
 		store:                  store,
 		workers:                workers,
 	}
@@ -175,18 +175,6 @@ func (s *Server) MintCert(ctx context.Context, req *ateapipb.MintCertRequest) (*
 		return nil, status.Error(codes.FailedPrecondition, "worker assignment changed while minting actor certificate")
 	}
 
-	// Load the CA pool for signing
-	poolBytes, err := os.ReadFile(s.actorIDCAPoolFile)
-	if err != nil {
-		slog.ErrorContext(ctx, "Failed to read actor CA pool file", slog.Any("err", err))
-		return nil, status.Errorf(codes.Internal, "Failed to load actor CA")
-	}
-	caPool, err := localca.Unmarshal(poolBytes)
-	if err != nil || len(caPool.CAs) == 0 {
-		slog.ErrorContext(ctx, "Failed to load actor CA", slog.Any("err", err))
-		return nil, status.Errorf(codes.Internal, "Failed to load actor CA")
-	}
-
 	// Parse the CSR
 	csr, err := x509.ParseCertificateRequest(req.GetCertificateSigningRequest())
 	if err != nil {
@@ -227,20 +215,14 @@ func (s *Server) MintCert(ctx context.Context, req *ateapipb.MintCertRequest) (*
 	}
 
 	// Sign and return the actor cert.
-	ca := caPool.CAs[0]
-	derBytes, err := x509.CreateCertificate(rand.Reader, template, ca.RootCertificate, csr.PublicKey, ca.SigningKey)
+	chain, err := s.actorIDCAPool.CreateCertificate(template, csr.PublicKey)
 	if err != nil {
 		slog.ErrorContext(ctx, "Failed to sign certificate", slog.Any("err", err))
 		return nil, status.Errorf(codes.Internal, "Failed to sign certificate")
 	}
 
-	certificates := [][]byte{derBytes}
-	for _, intermed := range ca.IntermediateCertificates {
-		certificates = append(certificates, intermed.Raw)
-	}
-
 	return &ateapipb.MintCertResponse{
-		ActorCertificates: certificates,
+		ActorCertificates: chain,
 	}, nil
 }
 

cmd/ateapi/internal/actoridentity/actoridentity_test.go

@@ -23,9 +23,7 @@ import (
 	"crypto/x509/pkix"
 	"math/big"
 	"net/url"
-	"os"
 	"path"
-	"path/filepath"
 	"testing"
 	"time"
 
@@ -140,13 +138,9 @@ func newTestServer(t *testing.T, st store.Interface) *Server {
 	if err != nil {
 		t.Fatalf("generate CA: %v", err)
 	}
-	poolBytes, err := localca.Marshal(&localca.Pool{CAs: []*localca.CA{ca}})
-	if err != nil {
-		t.Fatalf("marshal CA pool: %v", err)
-	}
-	poolFile := filepath.Join(t.TempDir(), "actor-ca-pool.json")
-	if err := os.WriteFile(poolFile, poolBytes, 0o600); err != nil {
-		t.Fatalf("write CA pool: %v", err)
+	pool := &localca.ConcretePool{
+		CAs:              []*localca.CA{ca},
+		ActiveForSigning: "test-actor-ca",
 	}
 
 	var workers *workercache.Cache
@@ -158,7 +152,7 @@ func newTestServer(t *testing.T, st store.Interface) *Server {
 			t.Fatalf("start worker cache: %v", err)
 		}
 	}
-	return New("issuer", "", poolFile, st, workers)
+	return New("issuer", "", pool, st, workers)
 }
 
 func TestMintJWTRequiresConfiguredJWTProvider(t *testing.T) {
@@ -726,7 +720,17 @@ func TestMintCertAuthorizesBeforeSigning(t *testing.T) {
 	if err := workers.Start(cacheCtx); err != nil {
 		t.Fatal(err)
 	}
-	srv := New("issuer", "", filepath.Join(t.TempDir(), "missing.json"), st, workers)
+
+	ca, err := localca.GenerateED25519CA("test-actor-ca")
+	if err != nil {
+		t.Fatalf("generate CA: %v", err)
+	}
+	pool := &localca.ConcretePool{
+		CAs:              []*localca.CA{ca},
+		ActiveForSigning: "test-actor-ca",
+	}
+
+	srv := New("issuer", "", pool, st, workers)
 
 	actor, err := st.GetActor(ctx, resources.ActorRef{Atespace: testAtespace, Name: testActorName})
 	if err != nil {

cmd/ateapi/main.go

@@ -37,6 +37,7 @@ import (
 	"github.com/agent-substrate/substrate/internal/ateapiauth"
 	"github.com/agent-substrate/substrate/internal/ateinterceptors"
 	"github.com/agent-substrate/substrate/internal/credbundle"
+	"github.com/agent-substrate/substrate/internal/localca"
 	"github.com/agent-substrate/substrate/internal/serverboot"
 	"github.com/agent-substrate/substrate/internal/version"
 	"github.com/agent-substrate/substrate/internal/volume"
@@ -195,7 +196,12 @@ func main() {
 	ateletDialer := controlapi.NewAteletDialer(workerPodInformer.GetIndexer(), ateletPodInformer.GetIndexer(), *ateletClientCredBundle, *podIdentityCACerts)
 	sm := controlapi.NewService(persistence, workerCache, actorTemplateLister, workerPoolLister, sandboxConfigLister, csiDriverConfigLister, storageClassLister, ateletDialer, instruments, *egressGatewayAddress, volPlugins)
 
-	actorIdentitySrv := actoridentity.New(actorIdentityJWTIssuer, *actorIDJWTPoolFile, *actorIDCAPoolFile, persistence, workerCache)
+	actorIDCAPool, err := localca.NewRefreshingPool(*actorIDCAPoolFile)
+	if err != nil {
+		serverboot.Fatal(ctx, "while loading the Actor ID CA", err)
+	}
+
+	actorIdentitySrv := actoridentity.New(actorIdentityJWTIssuer, *actorIDJWTPoolFile, actorIDCAPool, persistence, workerCache)
 	debugSrv := debugapi.NewService(persistence)
 
 	lisCfg := &net.ListenConfig{}

cmd/kubectl-ate/internal/cmd/admin_make_ca_pool.go

@@ -50,7 +50,7 @@ var makeCaPoolCmd = &cobra.Command{
 			return fmt.Errorf("while generating CA: %w", err)
 		}
 
-		pool := &localca.Pool{
+		pool := &localca.ConcretePool{
 			CAs: []*localca.CA{ca},
 		}
 

cmd/podcertcontroller/internal/podidentitysigner/podidentitysigner.go

@@ -17,7 +17,6 @@ package podidentitysigner
 import (
 	"bytes"
 	"context"
-	"crypto/rand"
 	"crypto/x509"
 	"encoding/pem"
 	"fmt"
@@ -67,12 +66,12 @@ func extKeyUsages(pod *corev1.Pod, namespace, serviceAccount string) []x509.ExtK
 
 type Impl struct {
 	kc     kubernetes.Interface
-	caPool *localca.Pool
+	caPool localca.Pool
 
 	clock clock.PassiveClock
 }
 
-func NewImpl(kc kubernetes.Interface, caPool *localca.Pool, clock clock.PassiveClock) *Impl {
+func NewImpl(kc kubernetes.Interface, caPool localca.Pool, clock clock.PassiveClock) *Impl {
 	return &Impl{
 		kc:     kc,
 		caPool: caPool,
@@ -86,14 +85,19 @@ func (h *Impl) SignerName() string {
 	return Name
 }
 
-func (h *Impl) DesiredClusterTrustBundles() []*certsv1beta1.ClusterTrustBundle {
+func (h *Impl) DesiredClusterTrustBundles() ([]*certsv1beta1.ClusterTrustBundle, error) {
 	name := CTBPrefix + "primary-bundle"
 
+	trustAnchors, err := h.caPool.TrustAnchors()
+	if err != nil {
+		return nil, fmt.Errorf("while retrieving CA pool trust anchors: %w", err)
+	}
+
 	wantTrustBundle := bytes.Buffer{}
-	for _, ca := range h.caPool.CAs {
+	for _, anchor := range trustAnchors {
 		block := pem.EncodeToMemory(&pem.Block{
 			Type:  "CERTIFICATE",
-			Bytes: ca.RootCertificate.Raw,
+			Bytes: anchor.Raw,
 		})
 		_, _ = wantTrustBundle.Write(block)
 	}
@@ -113,7 +117,7 @@ func (h *Impl) DesiredClusterTrustBundles() []*certsv1beta1.ClusterTrustBundle {
 
 	return []*certsv1beta1.ClusterTrustBundle{
 		wantCTB,
-	}
+	}, nil
 }
 
 func (h *Impl) MakeCert(ctx context.Context, pcr *certsv1beta1.PodCertificateRequest) error {
@@ -148,20 +152,15 @@ func (h *Impl) MakeCert(ctx context.Context, pcr *certsv1beta1.PodCertificateReq
 		Path:   path.Join("ns", pcr.ObjectMeta.Namespace, "sa", pcr.Spec.ServiceAccountName),
 	}
 
-	parent := h.caPool.CAs[0].RootCertificate
-
 	template := &x509.Certificate{
 		BasicConstraintsValid: true,
 		NotBefore:             notBefore,
 		NotAfter:              notAfter,
 		URIs:                  []*url.URL{spiffeURI},
 		KeyUsage:              x509.KeyUsageDigitalSignature,
 		ExtKeyUsage:           extKeyUsages(pod, pcr.ObjectMeta.Namespace, pcr.Spec.ServiceAccountName),
-		// Link the leaf to its issuing CA by key id so verifiers can disambiguate
-		// a multi-CA trust bundle (e.g. valkey trusts both the servicedns and
-		// podidentity CAs).
-		// https://datatracker.ietf.org/doc/html/rfc5280#section-4.2.1.1
-		AuthorityKeyId: parent.SubjectKeyId,
+		// AuthorityKeyID is automatically set to the SubjectKeyID of the parent
+		// certificate.
 	}
 
 	// Fields are sourced from the PCR spec (attested by kube-apiserver) rather
@@ -179,14 +178,9 @@ func (h *Impl) MakeCert(ctx context.Context, pcr *certsv1beta1.PodCertificateReq
 		return fmt.Errorf("while adding pod identity to certificate: %w", err)
 	}
 
-	subjectCertDER, err := x509.CreateCertificate(rand.Reader, template, parent, subjectPublicKey, h.caPool.CAs[0].SigningKey)
+	chainDER, err := h.caPool.CreateCertificate(template, subjectPublicKey)
 	if err != nil {
-		return fmt.Errorf("while signing subject cert: %w", err)
-	}
-
-	chainDER := [][]byte{subjectCertDER}
-	for _, intermed := range h.caPool.CAs[0].IntermediateCertificates {
-		chainDER = append(chainDER, intermed.Raw)
+		return fmt.Errorf("while signing certificate: %w", err)
 	}
 
 	chainPEM := &bytes.Buffer{}

cmd/podcertcontroller/internal/podidentitysigner/podidentitysigner_test.go

@@ -212,7 +212,7 @@ func TestMakeCert(t *testing.T) {
 			if err != nil {
 				t.Fatalf("while generating CA: %v", err)
 			}
-			caPool := &localca.Pool{CAs: []*localca.CA{ca}}
+			caPool := &localca.ConcretePool{CAs: []*localca.CA{ca}}
 
 			subjectPub, subjectPriv, err := ed25519.GenerateKey(rand.Reader)
 			if err != nil {
@@ -340,7 +340,7 @@ func TestMakeCertErrors(t *testing.T) {
 			if err != nil {
 				t.Fatalf("while generating CA: %v", err)
 			}
-			caPool := &localca.Pool{CAs: []*localca.CA{ca}}
+			caPool := &localca.ConcretePool{CAs: []*localca.CA{ca}}
 
 			pod, pcr := makePodAndPCR("ate-system", "atelet-abcde", "atelet", 86400)
 			pod.ObjectMeta.UID = tc.podUID
@@ -389,7 +389,7 @@ func TestMakeCertChainIncludesIntermediates(t *testing.T) {
 		t.Fatalf("while generating intermediate CA: %v", err)
 	}
 	ca.IntermediateCertificates = []*x509.Certificate{intermediateCA.RootCertificate}
-	caPool := &localca.Pool{CAs: []*localca.CA{ca}}
+	caPool := &localca.ConcretePool{CAs: []*localca.CA{ca}}
 
 	_, subjectPriv, err := ed25519.GenerateKey(rand.Reader)
 	if err != nil {
@@ -444,10 +444,13 @@ func TestDesiredClusterTrustBundles(t *testing.T) {
 	if err != nil {
 		t.Fatalf("while generating CA 2: %v", err)
 	}
-	caPool := &localca.Pool{CAs: []*localca.CA{ca1, ca2}}
+	caPool := &localca.ConcretePool{CAs: []*localca.CA{ca1, ca2}}
 	impl := NewImpl(nil, caPool, fixedClock{now: testNow})
 
-	ctbs := impl.DesiredClusterTrustBundles()
+	ctbs, err := impl.DesiredClusterTrustBundles()
+	if err != nil {
+		t.Fatalf("Error while getting desired ClusterTrustBundles: %v", err)
+	}
 	if len(ctbs) != 1 {
 		t.Fatalf("got %d ClusterTrustBundles, want 1", len(ctbs))
 	}

cmd/podcertcontroller/internal/servicednssigner/servicednssigner.go

@@ -17,7 +17,6 @@ package servicednssigner
 import (
 	"bytes"
 	"context"
-	"crypto/rand"
 	"crypto/x509"
 	"encoding/pem"
 	"fmt"
@@ -40,12 +39,12 @@ const CTBPrefix = "servicedns.podcert.ate.dev:identity:"
 
 type Impl struct {
 	kc     kubernetes.Interface
-	caPool *localca.Pool
+	caPool localca.Pool
 
 	clock clock.PassiveClock
 }
 
-func NewImpl(kc kubernetes.Interface, caPool *localca.Pool, clock clock.PassiveClock) *Impl {
+func NewImpl(kc kubernetes.Interface, caPool localca.Pool, clock clock.PassiveClock) *Impl {
 	return &Impl{
 		kc:     kc,
 		caPool: caPool,
@@ -59,14 +58,19 @@ func (h *Impl) SignerName() string {
 	return Name
 }
 
-func (h *Impl) DesiredClusterTrustBundles() []*certsv1beta1.ClusterTrustBundle {
+func (h *Impl) DesiredClusterTrustBundles() ([]*certsv1beta1.ClusterTrustBundle, error) {
 	name := CTBPrefix + "primary-bundle"
 
+	trustAnchors, err := h.caPool.TrustAnchors()
+	if err != nil {
+		return nil, fmt.Errorf("while retrieving CA pool trust anchors: %w", err)
+	}
+
 	wantTrustBundle := bytes.Buffer{}
-	for _, ca := range h.caPool.CAs {
+	for _, anchor := range trustAnchors {
 		block := pem.EncodeToMemory(&pem.Block{
 			Type:  "CERTIFICATE",
-			Bytes: ca.RootCertificate.Raw,
+			Bytes: anchor.Raw,
 		})
 		_, _ = wantTrustBundle.Write(block)
 	}
@@ -86,7 +90,7 @@ func (h *Impl) DesiredClusterTrustBundles() []*certsv1beta1.ClusterTrustBundle {
 
 	return []*certsv1beta1.ClusterTrustBundle{
 		wantCTB,
-	}
+	}, nil
 }
 
 func (h *Impl) MakeCert(ctx context.Context, pcr *certsv1beta1.PodCertificateRequest) error {
@@ -163,27 +167,20 @@ func (h *Impl) MakeCert(ctx context.Context, pcr *certsv1beta1.PodCertificateReq
 	notAfter := notBefore.Add(lifetime)
 	beginRefreshAt := notAfter.Add(-30 * time.Minute)
 
-	parent := h.caPool.CAs[0].RootCertificate
 	template := &x509.Certificate{
 		BasicConstraintsValid: true,
 		NotBefore:             notBefore,
 		NotAfter:              notAfter,
 		DNSNames:              dnsNames,
 		KeyUsage:              x509.KeyUsageDigitalSignature,
 		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
-		// Link the leaf to its issuing CA by key id. Needed this for Valkey
-		// to understand which CA to use when validating a client cert.
-		AuthorityKeyId: parent.SubjectKeyId,
+		// AuthorityKeyID is automatically set to the SubjectKeyID of the parent
+		// certificate.
 	}
 
-	subjectCertDER, err := x509.CreateCertificate(rand.Reader, template, parent, subjectPublicKey, h.caPool.CAs[0].SigningKey)
+	chainDER, err := h.caPool.CreateCertificate(template, subjectPublicKey)
 	if err != nil {
-		return fmt.Errorf("while signing subject cert: %w", err)
-	}
-
-	chainDER := [][]byte{subjectCertDER}
-	for _, intermed := range h.caPool.CAs[0].IntermediateCertificates {
-		chainDER = append(chainDER, intermed.Raw)
+		return fmt.Errorf("while signing certificate: %w", err)
 	}
 
 	chainPEM := &bytes.Buffer{}