diff --git a/docs/design/2026_04_29_partial_data_at_rest_encryption.md b/docs/design/2026_04_29_partial_data_at_rest_encryption.md index a842d6391..210365bba 100644 --- a/docs/design/2026_04_29_partial_data_at_rest_encryption.md +++ b/docs/design/2026_04_29_partial_data_at_rest_encryption.md @@ -36,7 +36,8 @@ Date: 2026-04-29 | 9B | AWS KMS, GCP KMS, Vault Transit, and test/CI env KEK providers; mutually-exclusive source loader and loaded-provider mutator gate (§5.1, §6.1, §6.5) | shipped | `2026_07_18_implemented_9b_kek_providers.md` | | 9C-1 | Storage-envelope observability: `decrypt_failures_total`, `writes_per_dek`, `value_overhead_bytes`, wired from the storage envelope path through `monitoring.Registry` (§9.2) | shipped | — | | 9C-2 | Sidecar/KEK observability: `active_dek_id{purpose}`, `sidecar_raft_index`, `kek_unwrap_seconds` (§9.2) | shipped | — | -| 9C+ | Rotation budget/rewrap/retire/rewrite, `last_proposed_index_per_raft_dek` (needs the §5.4 raft-DEK Wrap path), remaining benchmarks and encrypted Jepsen (§5.2, §5.4, §6.5, §8) | open | — | +| 9C-4 | §5.2 writes-per-DEK rotation budget: the 2^32 per-(DEK, process-load) ceiling, the 90% refuse-and-rotate threshold, and per-DEK accounting. The §9.2 `writes_per_dek` counter ships separately in 9C-1, from the storage-envelope path; wiring this budget to the same accounting is part of the open row below. | shipped | — | +| 9C+ | Rotation rewrap/retire/rewrite, admission-control and auto-propose wiring for the budget above, `last_proposed_index_per_raft_dek` (needs the §5.4 raft-DEK Wrap path), remaining benchmarks and encrypted Jepsen (§5.2, §5.4, §6.5, §8, §9.2) | open | — | Stages 0–4 ship the entire byte-tag pipeline (storage envelope, raft envelope, FSM dispatch, halt-on-error) but leave it **production diff --git a/internal/encryption/write_budget.go b/internal/encryption/write_budget.go new file mode 100644 index 000000000..78aa074ab --- /dev/null +++ b/internal/encryption/write_budget.go @@ -0,0 +1,274 @@ +package encryption + +import ( + "sync" + "sync/atomic" +) + +// §5.2 writes-per-DEK rotation trigger. +// +// The design bounds rotation cadence by two triggers, whichever fires +// first: 90 days, or a hard ceiling of 2^32 writes per +// (DEK, process-load) pair, in line with NIST SP 800-38D §8.3 for +// authenticated encryption. This is the second one. +// +// The ceiling is deliberately conservative. With the §4.1 +// counter-based nonces the actual cryptographic safety budget is far +// higher, but the design keeps 2^32 so the system does not depend on a +// single number being right everywhere in the codebase — and this +// tracker inherits that posture: it would rather rotate early than +// reason its way to a larger bound. +const ( + // DefaultWriteBudgetCeiling is the §5.2 hard ceiling per + // (DEK, process-load). + DefaultWriteBudgetCeiling uint64 = 1 << 32 + + // writeBudgetRefusePercent is the fraction of the ceiling at which + // admission control starts refusing new writes and the cluster + // auto-proposes a rotate-dek entry. Refusing BEFORE the ceiling is + // the point: rotation needs a Raft round trip, so waiting until + // the budget is actually spent would mean either blocking writes + // while it commits or issuing writes past the ceiling. + writeBudgetRefusePercent = 90 + + // writeBudgetPercentBase is the denominator writeBudgetRefusePercent + // is expressed against. + writeBudgetPercentBase = 100 +) + +// WriteBudgetVerdict is what a write attempt is permitted to do. +type WriteBudgetVerdict int + +const ( + // WriteBudgetAllow — under the refusal threshold, proceed. + WriteBudgetAllow WriteBudgetVerdict = iota + + // WriteBudgetRotate — at or past 90% of the ceiling. The write is + // refused and the caller should propose a rotation. The write is + // NOT counted, so a caller that retries cannot drive the counter + // past the ceiling while rotation commits. + WriteBudgetRotate + + // WriteBudgetExhausted — at or past the ceiling itself. + // + // Because Record reserves a slot with CAS, the counter stops at + // the refusal threshold, so a ceiling above that threshold is + // never reached through Record. This verdict is the fail-closed + // guard for the cases that can still land on it: a degenerate + // ceiling whose threshold equals it, and any future path that + // raises the counter without going through admission control (a + // count restored from disk, say). It must never silently become + // Allow. + WriteBudgetExhausted +) + +func (v WriteBudgetVerdict) String() string { + switch v { + case WriteBudgetAllow: + return "allow" + case WriteBudgetRotate: + return "rotate" + case WriteBudgetExhausted: + return "exhausted" + default: + return "unknown" + } +} + +// Allowed reports whether the write may proceed. +func (v WriteBudgetVerdict) Allowed() bool { return v == WriteBudgetAllow } + +// WriteBudget tracks writes per DEK for this process load. +// +// Scope is deliberately per-load, matching §5.2's "(DEK, +// process-load)" pair and the §4.1 nonce construction, whose +// local_epoch bumps on every process start. A restart therefore begins +// a fresh budget — which is correct, because it also begins a fresh +// nonce epoch, so the (key, nonce) space the ceiling protects is +// itself fresh. +type WriteBudget struct { + ceiling uint64 + threshold uint64 + + mu sync.RWMutex + counters map[uint32]*atomic.Uint64 +} + +// NewWriteBudget returns a budget with the given ceiling; a +// non-positive ceiling uses the §5.2 default. +func NewWriteBudget(ceiling uint64) *WriteBudget { + if ceiling == 0 { + ceiling = DefaultWriteBudgetCeiling + } + return &WriteBudget{ + ceiling: ceiling, + threshold: refusalThreshold(ceiling), + counters: make(map[uint32]*atomic.Uint64), + } +} + +// refusalThreshold returns floor(ceiling * writeBudgetRefusePercent / +// writeBudgetPercentBase) without overflowing and without discarding +// the remainder. +// +// Dividing first and multiplying after loses up to 99 ceiling-units of +// precision, which is invisible at the 2^32 default but severe for a +// smaller configured ceiling: a ceiling of 199 would refuse at 90 +// (about 45%), and anything under 100 would refuse at 0 -- i.e. refuse +// the very first write and wedge the DEK. Splitting the quotient and +// the remainder keeps the exact floor: with ceiling = 100q + r, the +// result is 90q + floor(9r/10), and r < 100 bounds the remainder term +// at 8910 so neither term can overflow. +func refusalThreshold(ceiling uint64) uint64 { + exact := ceiling/writeBudgetPercentBase*writeBudgetRefusePercent + + ceiling%writeBudgetPercentBase*writeBudgetRefusePercent/writeBudgetPercentBase + if exact == 0 { + // A ceiling below 2 rounds the 90% point down to zero. Refusing + // every write is worse than refusing slightly late: it would + // wedge the DEK and loop on rotation proposals that can never + // make progress. Allow exactly one write instead. + return 1 + } + return exact +} + +// Record accounts for one write under keyID and returns whether it may +// proceed. +// +// A refused write is not counted. Counting it would let a caller that +// retries on refusal walk the counter past the ceiling, which is the +// one thing the ceiling exists to prevent. +func (b *WriteBudget) Record(keyID uint32) WriteBudgetVerdict { + if b == nil { + return WriteBudgetAllow + } + ceiling := b.ceilingOrDefault() + threshold := b.refusalThresholdOrDefault() + counter := b.counterFor(keyID) + // The slot is RESERVED with CAS rather than incremented after the + // fact. Load-then-Add lets every writer that read a count below the + // threshold increment it, so writers that are then refused still + // consume budget: with a ceiling of 100 and the count at 89, + // eleven concurrent calls leave the counter at 100 having permitted + // one write, and the next call reports Exhausted for a DEK that + // issued 90 writes. CAS makes the decision and the increment one + // step, so the counter only ever records writes that were allowed. + for { + used := counter.Load() + if used >= ceiling { + return WriteBudgetExhausted + } + if used >= threshold { + return WriteBudgetRotate + } + if counter.CompareAndSwap(used, used+1) { + return WriteBudgetAllow + } + } +} + +// Used reports the writes recorded under keyID this process load. +func (b *WriteBudget) Used(keyID uint32) uint64 { + if b == nil { + return 0 + } + b.mu.RLock() + counter, ok := b.counters[keyID] + b.mu.RUnlock() + if !ok { + return 0 + } + return counter.Load() +} + +// RemainingUnlimited is what Remaining reports for a budget that is not +// wired, so a caller testing `Remaining(k) == 0` cannot read "no budget +// configured" as "rotation due". +// +// Zero was wrong for that state, and wrong in the dangerous direction: +// Record on a nil budget returns Allow, so the two APIs disagreed and any +// admission or dashboard check keyed on Remaining would demand rotation on +// every unconfigured node forever. +const RemainingUnlimited = ^uint64(0) + +// Remaining reports how many writes keyID may still issue before the refusal +// threshold. Zero means rotation is due; RemainingUnlimited means no budget +// is configured, matching Record's Allow on the same receiver. +func (b *WriteBudget) Remaining(keyID uint32) uint64 { + if b == nil { + return RemainingUnlimited + } + threshold := b.refusalThresholdOrDefault() + used := b.Used(keyID) + if used >= threshold { + return 0 + } + return threshold - used +} + +// refusalThresholdOrDefault is the threshold, derived on demand for a +// zero-value budget. +// +// A WriteBudget declared or embedded without NewWriteBudget has a zero +// threshold and a zero ceiling, which would refuse the first write. Treating +// the zero value as "the §5.2 default" matches what every other method on this +// type does with an unconfigured receiver: behave sanely rather than punish the +// caller for a construction detail. +func (b *WriteBudget) refusalThresholdOrDefault() uint64 { + if b.threshold != 0 { + return b.threshold + } + return refusalThreshold(b.ceilingOrDefault()) +} + +// ceilingOrDefault is the ceiling, defaulted for a zero-value budget. +func (b *WriteBudget) ceilingOrDefault() uint64 { + if b.ceiling != 0 { + return b.ceiling + } + return DefaultWriteBudgetCeiling +} + +// Forget drops the counter for a retired DEK. +// +// Called after a rotation retires keyID, so a long-lived process that +// rotates repeatedly does not accumulate a counter per historical DEK. +// It is NOT a way to reset a live DEK's budget: doing that would +// discard the very accounting the ceiling depends on. +func (b *WriteBudget) Forget(keyID uint32) { + if b == nil { + return + } + b.mu.Lock() + defer b.mu.Unlock() + delete(b.counters, keyID) +} + +// counterFor returns keyID's counter, creating it on first use. The +// read path takes only an RLock, so the steady state on a hot write +// path is an uncontended read plus an atomic add. +func (b *WriteBudget) counterFor(keyID uint32) *atomic.Uint64 { + b.mu.RLock() + counter, ok := b.counters[keyID] + b.mu.RUnlock() + if ok { + return counter + } + + b.mu.Lock() + defer b.mu.Unlock() + if counter, ok := b.counters[keyID]; ok { + return counter + } + if b.counters == nil { + // A zero-value budget -- declared or embedded rather than built by + // NewWriteBudget -- reaches here with a nil map, and assigning into + // one panics. Surprising precisely because every other method on this + // type tolerates an unconfigured receiver, so it is initialised here + // instead of panicking on a construction detail. + b.counters = make(map[uint32]*atomic.Uint64, 1) + } + counter = &atomic.Uint64{} + b.counters[keyID] = counter + return counter +} diff --git a/internal/encryption/write_budget_test.go b/internal/encryption/write_budget_test.go new file mode 100644 index 000000000..28736b6bc --- /dev/null +++ b/internal/encryption/write_budget_test.go @@ -0,0 +1,347 @@ +package encryption_test + +import ( + "sync" + "sync/atomic" + "testing" + + "github.com/bootjp/elastickv/internal/encryption" + "github.com/stretchr/testify/require" +) + +// TestWriteBudgetAllowsUpToTheRefusalThreshold pins the ordinary path: +// writes proceed until 90% of the ceiling. +func TestWriteBudgetAllowsUpToTheRefusalThreshold(t *testing.T) { + t.Parallel() + + // Ceiling 100 -> threshold 90. + b := encryption.NewWriteBudget(100) + for i := range 90 { + require.True(t, b.Record(1).Allowed(), "write %d must be allowed", i) + } + require.Zero(t, b.Remaining(1)) +} + +// TestWriteBudgetRefusesAndSignalsRotationAtNinetyPercent is the §5.2 +// trigger. Refusing BEFORE the ceiling is the point: rotation needs a +// Raft round trip, so waiting until the budget is actually spent would +// mean either blocking writes while it commits or issuing writes past +// the ceiling. +func TestWriteBudgetRefusesAndSignalsRotationAtNinetyPercent(t *testing.T) { + t.Parallel() + + b := encryption.NewWriteBudget(100) + for range 90 { + require.True(t, b.Record(1).Allowed()) + } + v := b.Record(1) + require.Equal(t, encryption.WriteBudgetRotate, v) + require.False(t, v.Allowed()) +} + +// TestWriteBudgetDoesNotCountRefusedWrites is the property that keeps +// the ceiling meaningful. A caller that retries on refusal must not be +// able to walk the counter past the ceiling. +func TestWriteBudgetDoesNotCountRefusedWrites(t *testing.T) { + t.Parallel() + + b := encryption.NewWriteBudget(100) + for range 90 { + b.Record(1) + } + used := b.Used(1) + + for range 1000 { + require.False(t, b.Record(1).Allowed()) + } + require.Equal(t, used, b.Used(1), + "a refused write must not consume budget, or retries would breach the ceiling") +} + +// TestWriteBudgetFailsClosedPastTheCeiling covers a caller that ignored +// the rotate signal: it must still stop rather than keep encrypting +// under a DEK whose budget is spent. +func TestWriteBudgetFailsClosedPastTheCeiling(t *testing.T) { + t.Parallel() + + b := encryption.NewWriteBudget(10) + // Threshold is 9, so nine writes are allowed and the tenth is + // refused. (Before the threshold arithmetic was fixed this + // ceiling produced a threshold of 0 and the comment was false: + // every write here was refused and the test asserted nothing.) + for range 10 { + b.Record(1) + } + require.Equal(t, uint64(9), b.Used(1)) + // Even having crossed into rotate territory, the verdict must + // never become Allow again. + for range 50 { + require.False(t, b.Record(1).Allowed()) + } +} + +// TestWriteBudgetIsPerDEK pins that one DEK's exhaustion does not +// refuse writes under another — rotation installs a new key_id, and +// that new key must start with a full budget or the cluster would be +// wedged the moment it rotated. +func TestWriteBudgetIsPerDEK(t *testing.T) { + t.Parallel() + + b := encryption.NewWriteBudget(100) + for range 95 { + b.Record(1) + } + require.False(t, b.Record(1).Allowed()) + + require.True(t, b.Record(2).Allowed(), "a freshly rotated DEK starts with a full budget") + require.Equal(t, uint64(89), b.Remaining(2)) +} + +// TestWriteBudgetForgetDropsARetiredDEK pins the cleanup path: a +// long-lived process that rotates repeatedly must not accumulate a +// counter per historical DEK. +func TestWriteBudgetForgetDropsARetiredDEK(t *testing.T) { + t.Parallel() + + b := encryption.NewWriteBudget(100) + b.Record(1) + require.Equal(t, uint64(1), b.Used(1)) + + b.Forget(1) + require.Zero(t, b.Used(1)) +} + +// TestWriteBudgetDefaultCeilingMatchesTheDesign pins the §5.2 number +// itself: 2^32 per (DEK, process-load), per NIST SP 800-38D §8.3. +func TestWriteBudgetDefaultCeilingMatchesTheDesign(t *testing.T) { + t.Parallel() + + require.Equal(t, uint64(1)<<32, encryption.DefaultWriteBudgetCeiling) + + b := encryption.NewWriteBudget(0) + // Threshold is 90% of 2^32, stated as the specification rather than + // as the implementation's expression. The previous form here was + // `1<<32/100*90`, which mirrored the production division order and + // therefore could not catch an error in it: it asserted the lossy + // 3865470480 instead of the exact 3865470566. + require.Equal(t, uint64(1)<<32*9/10, b.Remaining(1)) + require.Equal(t, uint64(3865470566), b.Remaining(1)) +} + +// TestWriteBudgetIsRaceFree exercises the hot path from many +// goroutines: this sits on every encrypted write, so a torn counter +// would be a live data race in production. +func TestWriteBudgetIsRaceFree(t *testing.T) { + t.Parallel() + + const goroutines = uint32(16) + const perGoroutine = 64 + const distinctKeys = uint32(4) + + b := encryption.NewWriteBudget(1 << 20) + var wg sync.WaitGroup + for g := range goroutines { + wg.Add(1) + go func() { + defer wg.Done() + for range perGoroutine { + b.Record(g % distinctKeys) + } + }() + } + wg.Wait() + + total := uint64(0) + for keyID := range distinctKeys { + total += b.Used(keyID) + } + require.Equal(t, uint64(goroutines)*uint64(perGoroutine), total) +} + +// TestWriteBudgetNilReceiverAllows covers a node with no budget wired: +// it must not refuse every write. +func TestWriteBudgetNilReceiverAllows(t *testing.T) { + t.Parallel() + + var b *encryption.WriteBudget + require.True(t, b.Record(1).Allowed()) + require.Zero(t, b.Used(1)) + require.NotPanics(t, func() { b.Forget(1) }) +} + +func TestWriteBudgetVerdictStringsAreStable(t *testing.T) { + t.Parallel() + + require.Equal(t, "allow", encryption.WriteBudgetAllow.String()) + require.Equal(t, "rotate", encryption.WriteBudgetRotate.String()) + require.Equal(t, "exhausted", encryption.WriteBudgetExhausted.String()) +} + +// TestWriteBudgetRefusedWritesDoNotConsumeBudgetUnderConcurrency is the +// concurrent form of TestWriteBudgetDoesNotCountRefusedWrites, which +// only ever exercised the sequential path and so passed while refused +// writes were being counted. +// +// Load-then-Add admits every writer that read a count below the +// threshold; they all increment, so writers that are subsequently +// refused still spend budget. Sequentially the bug is invisible -- the +// Load sees the threshold and returns first -- so only sustained +// contention at the boundary exposes it. +// +// The invariant asserted here is the one that makes the overshoot +// impossible: the counter can never exceed the refusal threshold, +// because only a write that won the CAS is recorded. That also means +// the number of permitted writes equals the threshold exactly, no +// matter how many writers raced. +func TestWriteBudgetRefusedWritesDoNotConsumeBudgetUnderConcurrency(t *testing.T) { + t.Parallel() + + const ( + ceiling = uint64(10000) + threshold = uint64(9000) + racers = 64 + trials = 20 + ) + + for trial := range trials { + b := encryption.NewWriteBudget(ceiling) + + // Every racer hammers Record until it is refused, so the + // writers contending at the boundary are many and arrive + // continuously rather than in one staged burst. + var allowed atomic.Uint64 + var wg sync.WaitGroup + for range racers { + wg.Add(1) + go func() { + defer wg.Done() + for b.Record(1).Allowed() { + allowed.Add(1) + } + }() + } + wg.Wait() + + require.LessOrEqual(t, b.Used(1), threshold, + "trial %d: the counter passed the refusal threshold, so refused "+ + "writes consumed budget", trial) + require.Equal(t, threshold, allowed.Load(), + "trial %d: exactly the threshold many writes may be permitted", trial) + require.Equal(t, allowed.Load(), b.Used(1), + "trial %d: the counter must record permitted writes and nothing else", trial) + } +} + +// TestWriteBudgetThresholdIsNinetyPercent pins the threshold arithmetic +// across ceilings that are not multiples of 100, where dividing before +// multiplying silently collapsed the budget. +func TestWriteBudgetThresholdIsNinetyPercent(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + ceiling uint64 + want uint64 + }{ + {"design default 2^32", encryption.DefaultWriteBudgetCeiling, 3865470566}, + {"exact multiple of 100", 100, 90}, + {"not a multiple of 100", 199, 179}, + {"just under 100", 99, 89}, + {"single digit", 10, 9}, + {"smallest with a nonzero 90%", 2, 1}, + {"degenerate ceiling of 1 still allows one write", 1, 1}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + b := encryption.NewWriteBudget(tc.ceiling) + // Remaining from a fresh budget is the threshold. + require.Equal(t, tc.want, b.Remaining(1)) + }) + } +} + +// TestWriteBudgetTinyCeilingDoesNotWedgeTheDEK covers the anti-wedge +// guard. A ceiling whose 90% point floors to zero would refuse the very +// first write, so the cluster would propose rotations forever and never +// issue a write under the new DEK either. +func TestWriteBudgetTinyCeilingDoesNotWedgeTheDEK(t *testing.T) { + t.Parallel() + + for ceiling := uint64(1); ceiling <= 9; ceiling++ { + b := encryption.NewWriteBudget(ceiling) + require.True(t, b.Record(1).Allowed(), + "ceiling %d must allow at least one write", ceiling) + } +} + +// TestWriteBudgetReachesExhaustedAtADegenerateCeiling covers the +// ceiling branch, which atomic reservation otherwise makes unreachable: +// with a ceiling of 1 the threshold equals the ceiling, so the counter +// does land on it. +func TestWriteBudgetReachesExhaustedAtADegenerateCeiling(t *testing.T) { + t.Parallel() + + b := encryption.NewWriteBudget(1) + require.Equal(t, encryption.WriteBudgetAllow, b.Record(1)) + require.Equal(t, encryption.WriteBudgetExhausted, b.Record(1)) +} + +// TestWriteBudgetNilReceiverAgreesAcrossItsAPIs pins that the unwired state +// reads the same way from every method. +// +// Record returns Allow on a nil budget, but Remaining returned 0 — which this +// type documents as "rotation is due". Any admission check or dashboard keyed +// on `Remaining(k) == 0` therefore demanded rotation on every unconfigured node +// forever, while the write verdict said Allow. The two APIs disagreed about the +// same receiver. +func TestWriteBudgetNilReceiverAgreesAcrossItsAPIs(t *testing.T) { + t.Parallel() + + var b *encryption.WriteBudget + + require.True(t, b.Record(1).Allowed(), "an unwired budget does not refuse writes") + require.Equal(t, encryption.RemainingUnlimited, b.Remaining(1), + "so it must not report rotation due either") + require.NotZero(t, b.Remaining(1), + "zero is the rotation-due signal and must not double as not-configured") + require.Zero(t, b.Used(1)) +} + +// TestWriteBudgetZeroValueIsUsable covers a WriteBudget declared or embedded +// without NewWriteBudget. +// +// Record assigned into a nil counters map and panicked, which is surprising +// precisely because this type tolerates a nil receiver and its other methods +// tolerate a zero value. The zero value now behaves as the §5.2 default rather +// than as a ceiling of zero, which would have refused the first write. +func TestWriteBudgetZeroValueIsUsable(t *testing.T) { + t.Parallel() + + t.Run("Record does not panic and applies the default ceiling", func(t *testing.T) { + t.Parallel() + + var b encryption.WriteBudget + require.True(t, b.Record(1).Allowed(), + "a zero-value budget must not refuse the first write") + require.Equal(t, uint64(1), b.Used(1)) + }) + + t.Run("Remaining reports the default threshold", func(t *testing.T) { + t.Parallel() + + var b encryption.WriteBudget + var configured = encryption.NewWriteBudget(0) + require.Equal(t, configured.Remaining(1), b.Remaining(1), + "the zero value must agree with an explicitly default-constructed budget") + }) + + t.Run("an embedded zero value works too", func(t *testing.T) { + t.Parallel() + + type holder struct{ budget encryption.WriteBudget } + var h holder + require.True(t, h.budget.Record(9).Allowed()) + require.Equal(t, uint64(1), h.budget.Used(9)) + }) +}