From 98bb0bf190711f4ca927c6795e862de7e2a33a61 Mon Sep 17 00:00:00 2001 From: Josh Allmann Date: Thu, 12 Nov 2026 08:39:52 -0800 Subject: [PATCH 1/3] Wait for durable redeemer handoff before runner retirement --- docs/orchestrator.md | 4 ++ orchestrator/handoff_drain_test.go | 68 ++++++++++++++++++++++++++++++ orchestrator/payment_engine.go | 19 ++++++++- orchestrator/registry.go | 22 +++++++--- redemption/handoff_test.go | 42 ++++++++++++++++++ redemption/state.go | 17 +++++++- 6 files changed, 164 insertions(+), 8 deletions(-) create mode 100644 orchestrator/handoff_drain_test.go create mode 100644 redemption/handoff_test.go diff --git a/docs/orchestrator.md b/docs/orchestrator.md index 587f10d..a9ad0cd 100644 --- a/docs/orchestrator.md +++ b/docs/orchestrator.md @@ -91,6 +91,10 @@ Set it in TOML or with `LIVEPEER_ORCHESTRATOR_DRAIN_TIMEOUT`. Unregistering a ru or letting its heartbeat expire ends its sessions. An orchestrator restart ends all sessions. See the [heartbeat contract](http-api.md#live-runner-http) for removal confirmation. +With an external redeemer, unregister acknowledges completion only after every +outstanding winner has a durable handoff acknowledgement. Retry a 503 response, +including after a Node restart. Transaction confirmation continues at the redeemer. + Enabling an app's proxy creates an automatic proxy belonging to that app; use its returned proxy ID and URL. Without a URL template, generated URLs use `ServiceURL/run/{proxy}`. The earlier diff --git a/orchestrator/handoff_drain_test.go b/orchestrator/handoff_drain_test.go new file mode 100644 index 0000000..a45f492 --- /dev/null +++ b/orchestrator/handoff_drain_test.go @@ -0,0 +1,68 @@ +package orchestrator + +import ( + "context" + "math/big" + "net/http" + "testing" + + "github.com/livepeer/node/pm" + "github.com/stretchr/testify/require" +) + +func TestRunnerRetirementWaitsForDurableHandoffEvenAfterRegistrationLoss(t *testing.T) { + _, store, ticket, _, _ := newRecoveryTest(t, 19) + engine, err := NewPaymentEngine(store, paymentTestChain{}, ticket.Recipient, big.NewInt(1), big.NewInt(1)) + require.NoError(t, err) + registry, server := reviewServer(t, "http://127.0.0.1:1") + defer server.Close() + server.SetPayment(engine) + owner := appHeartbeat(t, registry, "owner", "bootstrap", registeredApp("app")) + lifetime := registry.apps["app"].lifetime + status, _ := registry.Unregister("owner", "wrong") + require.Equal(t, http.StatusUnauthorized, status) + require.False(t, lifetime.closed.Load()) + status, _ = registry.Unregister("owner", owner.HeartbeatSecret) + require.Equal(t, http.StatusServiceUnavailable, status) + require.True(t, lifetime.closed.Load()) + // A retry, expiration or Node restart can leave no in-memory runner. + status, _ = registry.Unregister("missing-owner", "bootstrap") + require.Equal(t, http.StatusServiceUnavailable, status) + _, err = store.db.Exec(`UPDATE ticket_state SET delivery='acknowledged' WHERE hash=?`, ticket.Hash().Bytes()) + require.NoError(t, err) + status, _ = registry.Unregister("missing-owner", "bootstrap") + require.Equal(t, http.StatusNotFound, status) + var settled int + require.NoError(t, store.db.QueryRow(`SELECT settled FROM ticket_state`).Scan(&settled)) + require.Zero(t, settled, "runner retirement does not wait for chain confirmation") +} + +func TestRetiredAppRejectsPaymentReceiptStillWaitingOnRPC(t *testing.T) { + engine, chain, key, _ := newRecipientTestEngine(t) + registry, server := reviewServer(t, "http://127.0.0.1:1") + defer server.Close() + server.SetPayment(engine) + owner := appHeartbeat(t, registry, "owner", "bootstrap", registeredApp("runner")) + info := issueRecipientParams(t, engine, key) + entered, release := make(chan struct{}), make(chan struct{}) + chain.funds = func(ctx context.Context, funds pm.PayerFunds) (pm.PayerFunds, error) { + close(entered) + select { + case <-release: + return funds, nil + case <-ctx.Done(): + return funds, ctx.Err() + } + } + done := make(chan error, 1) + go func() { done <- receiveRecipientPayment(t, engine, key, info, 1) }() + <-entered + status, err := registry.Unregister("owner", owner.HeartbeatSecret) + close(release) + require.NoError(t, err) + require.Equal(t, http.StatusOK, status) + require.ErrorIs(t, <-done, ErrMissingChallenge) + var count int + require.NoError(t, engine.store.db.QueryRow(`SELECT count(*) FROM winning_tickets`).Scan(&count)) + require.Zero(t, count, "no receipt can commit after the normal drain acknowledgement") +} diff --git a/orchestrator/payment_engine.go b/orchestrator/payment_engine.go index 3d3142d..707deb3 100644 --- a/orchestrator/payment_engine.go +++ b/orchestrator/payment_engine.go @@ -313,7 +313,7 @@ func (e *PaymentEngine) Receive(ctx context.Context, appID, manifest, paymentHea defer e.mu.Unlock() // Store winning tickets before publishing in-memory replay guards and credit. // The lock serializes receipt and accounting; failed batches change neither. - if e.sessions[manifest] != session { + if e.sessions[manifest] != session || (session.registration != nil && session.registration.closed.Load()) { return ethcommon.Address{}, nil, ErrMissingChallenge } if err := snapshot.Validate(); err != nil { @@ -349,6 +349,23 @@ func (e *PaymentEngine) Receive(ctx context.Context, appID, manifest, paymentHea return payer, new(big.Rat).Set(balance), nil } +// Unregister fences new receipts before taking this lock, so any receipt +// already committing is included in the durable handoff check. +func (e *PaymentEngine) handoffDrained() error { + e.mu.Lock() + defer e.mu.Unlock() + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + drained, err := e.store.HandoffDrained(ctx) + if err != nil { + return errors.New("ticket handoff status unavailable") + } + if !drained { + return errors.New("ticket handoff pending") + } + return nil +} + func (e *PaymentEngine) Charge(ctx context.Context, manifest string, now time.Time) (result error) { labels := e.paymentLabels("", manifest) observe := true diff --git a/orchestrator/registry.go b/orchestrator/registry.go index 2a4e657..feadeab 100644 --- a/orchestrator/registry.go +++ b/orchestrator/registry.go @@ -17,6 +17,7 @@ import ( "slices" "strings" "sync" + "sync/atomic" "time" "github.com/livepeer/node/destination" @@ -163,8 +164,8 @@ type seqConflict uint64 func (seqConflict) Error() string { return "seq must increase after owner opt-in" } -// appLifetime distinguishes registration instances without retaining their state. -type appLifetime struct{ _ byte } +// appLifetime fences receipts after their app registration is removed. +type appLifetime struct{ closed atomic.Bool } type app struct { lifetime *appLifetime @@ -562,6 +563,7 @@ func (item *app) response() appResponse { // removeApp requires r.mu and an app with no remaining sessions. func (r *Registry) removeApp(item *app) { + item.lifetime.closed.Store(true) r.runnerEvent(item.ID, item, "removed") delete(item.owner.Apps, item.ID) delete(r.apps, item.ID) @@ -571,19 +573,27 @@ func (r *Registry) Unregister(id, auth string) (int, error) { r.mu.Lock() defer r.mu.Unlock() current := r.runners[id] + if current != nil && (current.Static || !equalSecret(current.Credential, auth)) { + return http.StatusUnauthorized, errors.New("invalid runner credential") + } + if current != nil { + r.removeRunner(current, nil, "runner_unregistered", "unregistered") + } + if r.server != nil && r.server.payment != nil { + if err := r.server.payment.handoffDrained(); err != nil { + return http.StatusServiceUnavailable, err + } + } if current == nil { return http.StatusNotFound, errors.New("runner not found") } - if current.Static || !equalSecret(current.Credential, auth) { - return http.StatusUnauthorized, errors.New("invalid runner credential") - } - r.removeRunner(current, nil, "runner_unregistered", "unregistered") return http.StatusOK, nil } // removeRunner requires r.mu. Session teardown precedes owner/channel removal. func (r *Registry) removeRunner(current *runner, cause error, reason, event string) { for id, item := range current.Apps { + item.lifetime.closed.Store(true) item.Removing = false for sid := range item.Sessions { r.releaseLocked(id, sid, cause, reason) diff --git a/redemption/handoff_test.go b/redemption/handoff_test.go new file mode 100644 index 0000000..290c0f7 --- /dev/null +++ b/redemption/handoff_test.go @@ -0,0 +1,42 @@ +package redemption + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestHandoffDrainRequiresDurableAcknowledgementNotSettlement(t *testing.T) { + s, binding := boundStore(t, "orchestrator", "https://redeemer.example", DefaultLimits) + ticket, _ := winningTicket(t, 1) + require.NoError(t, s.Admit(t.Context(), ticket)) + binding.Destination = "https://replacement.example" + check := func(want bool) { + t.Helper() + drained, err := s.HandoffDrained(t.Context()) + require.NoError(t, err) + require.Equal(t, want, drained) + } + for _, phase := range []string{"queued", "sending", "retryable", "rejected"} { + require.NoError(t, s.delivery(t.Context(), ticket, phase, 0)) + check(false) + require.ErrorContains(t, s.Bind(t.Context(), binding, DefaultLimits, nil), "drain") + } + require.NoError(t, s.delivery(t.Context(), ticket, "acknowledged", 0)) + check(true) + var settled int + require.NoError(t, s.db.QueryRow(`SELECT settled FROM ticket_state`).Scan(&settled)) + require.Zero(t, settled) + require.NoError(t, s.Bind(t.Context(), binding, DefaultLimits, nil), "destination changes need handoff, not onchain confirmation") + path := s.path + require.NoError(t, s.Close()) + reopened, err := OpenRedeemerDB(path) + require.NoError(t, err) + defer reopened.Close() + drained, err := reopened.HandoffDrained(t.Context()) + require.NoError(t, err) + require.True(t, drained) + require.NoError(t, reopened.Close()) + _, err = reopened.HandoffDrained(t.Context()) + require.Error(t, err, "unavailable storage cannot prove a completed handoff") +} diff --git a/redemption/state.go b/redemption/state.go index d920f07..90797f8 100644 --- a/redemption/state.go +++ b/redemption/state.go @@ -20,6 +20,17 @@ var ErrConflict = errors.New("ticket scheduling metadata conflicts with stored t type Limits struct{ Pending, PerRecipient int } +// HandoffDrained reports whether the external redeemer has durably accepted +// every outstanding winner. It does not consult the chain or wait for finality. +func (s *RedeemerDB) HandoffDrained(ctx context.Context) (bool, error) { + if s.binding != nil && s.binding.Destination == "chain" { + return true, nil + } + var drained bool + err := s.db.QueryRowContext(ctx, `SELECT NOT EXISTS(SELECT 1 FROM ticket_state WHERE settled=0 AND delivery!='acknowledged')`).Scan(&drained) + return drained, err +} + var DefaultLimits = Limits{Pending: 100000, PerRecipient: 10000} type Binding struct { @@ -55,7 +66,11 @@ func (s *RedeemerDB) Bind(ctx context.Context, b Binding, limits Limits, chain N } if previous.Destination != b.Destination { var count int - if err = tx.QueryRowContext(ctx, "SELECT count(*) FROM ticket_state WHERE settled=0").Scan(&count); err != nil { + pending := "SELECT count(*) FROM ticket_state WHERE settled=0" + if previous.Role == "orchestrator" && previous.Destination != "chain" { + pending += " AND delivery!='acknowledged'" + } + if err = tx.QueryRowContext(ctx, pending).Scan(&count); err != nil { return err } if count != 0 { From 6c3846db192e1cf6bf5d025c335742a2289f6615 Mon Sep 17 00:00:00 2001 From: Josh Allmann Date: Thu, 12 Nov 2026 09:10:01 -0800 Subject: [PATCH 2/3] Simplify durable handoff drain --- docs/orchestrator.md | 4 -- orchestrator/drain_test.go | 59 ++++++++++++++++++++++++++ orchestrator/handoff_drain_test.go | 68 ------------------------------ orchestrator/payment_engine.go | 17 -------- orchestrator/registry.go | 13 ++++-- redemption/handoff_test.go | 42 ------------------ redemption/http_test.go | 26 ++++++++++++ redemption/state.go | 7 ++- 8 files changed, 98 insertions(+), 138 deletions(-) delete mode 100644 orchestrator/handoff_drain_test.go delete mode 100644 redemption/handoff_test.go diff --git a/docs/orchestrator.md b/docs/orchestrator.md index a9ad0cd..587f10d 100644 --- a/docs/orchestrator.md +++ b/docs/orchestrator.md @@ -91,10 +91,6 @@ Set it in TOML or with `LIVEPEER_ORCHESTRATOR_DRAIN_TIMEOUT`. Unregistering a ru or letting its heartbeat expire ends its sessions. An orchestrator restart ends all sessions. See the [heartbeat contract](http-api.md#live-runner-http) for removal confirmation. -With an external redeemer, unregister acknowledges completion only after every -outstanding winner has a durable handoff acknowledgement. Retry a 503 response, -including after a Node restart. Transaction confirmation continues at the redeemer. - Enabling an app's proxy creates an automatic proxy belonging to that app; use its returned proxy ID and URL. Without a URL template, generated URLs use `ServiceURL/run/{proxy}`. The earlier diff --git a/orchestrator/drain_test.go b/orchestrator/drain_test.go index 47ab7ba..7887dff 100644 --- a/orchestrator/drain_test.go +++ b/orchestrator/drain_test.go @@ -4,12 +4,14 @@ import ( "context" "encoding/json" "fmt" + "math/big" "net/http" "testing" "testing/synctest" "time" "github.com/livepeer/node/internal/test" + "github.com/livepeer/node/pm" "github.com/stretchr/testify/require" ) @@ -196,3 +198,60 @@ func TestAppDrainReplacementAtCapacity(t *testing.T) { } } } + +func TestRunnerRetirementWaitsForDurableHandoffEvenAfterRegistrationLoss(t *testing.T) { + _, store, ticket, _, _ := newRecoveryTest(t, 19) + engine, err := NewPaymentEngine(store, paymentTestChain{}, ticket.Recipient, big.NewInt(1), big.NewInt(1)) + require.NoError(t, err) + registry, server := reviewServer(t, "http://127.0.0.1:1") + defer server.Close() + server.SetPayment(engine) + owner := appHeartbeat(t, registry, "owner", "bootstrap", registeredApp("app")) + lifetime := registry.apps["app"].lifetime + status, _ := registry.Unregister("owner", "wrong") + require.Equal(t, http.StatusUnauthorized, status) + require.False(t, lifetime.closed.Load()) + status, _ = registry.Unregister("owner", owner.HeartbeatSecret) + require.Equal(t, http.StatusServiceUnavailable, status) + require.True(t, lifetime.closed.Load()) + // A retry, expiration or Node restart can leave no in-memory runner. + status, _ = registry.Unregister("missing-owner", "bootstrap") + require.Equal(t, http.StatusServiceUnavailable, status) + _, err = store.db.Exec(`UPDATE ticket_state SET delivery='acknowledged' WHERE hash=?`, ticket.Hash().Bytes()) + require.NoError(t, err) + status, _ = registry.Unregister("missing-owner", "bootstrap") + require.Equal(t, http.StatusNotFound, status) + require.NoError(t, store.Close()) + status, _ = registry.Unregister("missing-owner", "bootstrap") + require.Equal(t, http.StatusServiceUnavailable, status) +} + +func TestRetiredAppRejectsPaymentReceiptStillWaitingOnRPC(t *testing.T) { + engine, chain, key, _ := newRecipientTestEngine(t) + registry, server := reviewServer(t, "http://127.0.0.1:1") + defer server.Close() + server.SetPayment(engine) + owner := appHeartbeat(t, registry, "owner", "bootstrap", registeredApp("runner")) + info := issueRecipientParams(t, engine, key) + entered, release := make(chan struct{}), make(chan struct{}) + chain.funds = func(ctx context.Context, funds pm.PayerFunds) (pm.PayerFunds, error) { + close(entered) + select { + case <-release: + return funds, nil + case <-ctx.Done(): + return funds, ctx.Err() + } + } + done := make(chan error, 1) + go func() { done <- receiveRecipientPayment(t, engine, key, info, 1) }() + <-entered + status, err := registry.Unregister("owner", owner.HeartbeatSecret) + close(release) + require.NoError(t, err) + require.Equal(t, http.StatusOK, status) + require.ErrorIs(t, <-done, ErrMissingChallenge) + var count int + require.NoError(t, engine.store.db.QueryRow(`SELECT count(*) FROM winning_tickets`).Scan(&count)) + require.Zero(t, count, "no receipt can commit after the normal drain acknowledgement") +} diff --git a/orchestrator/handoff_drain_test.go b/orchestrator/handoff_drain_test.go deleted file mode 100644 index a45f492..0000000 --- a/orchestrator/handoff_drain_test.go +++ /dev/null @@ -1,68 +0,0 @@ -package orchestrator - -import ( - "context" - "math/big" - "net/http" - "testing" - - "github.com/livepeer/node/pm" - "github.com/stretchr/testify/require" -) - -func TestRunnerRetirementWaitsForDurableHandoffEvenAfterRegistrationLoss(t *testing.T) { - _, store, ticket, _, _ := newRecoveryTest(t, 19) - engine, err := NewPaymentEngine(store, paymentTestChain{}, ticket.Recipient, big.NewInt(1), big.NewInt(1)) - require.NoError(t, err) - registry, server := reviewServer(t, "http://127.0.0.1:1") - defer server.Close() - server.SetPayment(engine) - owner := appHeartbeat(t, registry, "owner", "bootstrap", registeredApp("app")) - lifetime := registry.apps["app"].lifetime - status, _ := registry.Unregister("owner", "wrong") - require.Equal(t, http.StatusUnauthorized, status) - require.False(t, lifetime.closed.Load()) - status, _ = registry.Unregister("owner", owner.HeartbeatSecret) - require.Equal(t, http.StatusServiceUnavailable, status) - require.True(t, lifetime.closed.Load()) - // A retry, expiration or Node restart can leave no in-memory runner. - status, _ = registry.Unregister("missing-owner", "bootstrap") - require.Equal(t, http.StatusServiceUnavailable, status) - _, err = store.db.Exec(`UPDATE ticket_state SET delivery='acknowledged' WHERE hash=?`, ticket.Hash().Bytes()) - require.NoError(t, err) - status, _ = registry.Unregister("missing-owner", "bootstrap") - require.Equal(t, http.StatusNotFound, status) - var settled int - require.NoError(t, store.db.QueryRow(`SELECT settled FROM ticket_state`).Scan(&settled)) - require.Zero(t, settled, "runner retirement does not wait for chain confirmation") -} - -func TestRetiredAppRejectsPaymentReceiptStillWaitingOnRPC(t *testing.T) { - engine, chain, key, _ := newRecipientTestEngine(t) - registry, server := reviewServer(t, "http://127.0.0.1:1") - defer server.Close() - server.SetPayment(engine) - owner := appHeartbeat(t, registry, "owner", "bootstrap", registeredApp("runner")) - info := issueRecipientParams(t, engine, key) - entered, release := make(chan struct{}), make(chan struct{}) - chain.funds = func(ctx context.Context, funds pm.PayerFunds) (pm.PayerFunds, error) { - close(entered) - select { - case <-release: - return funds, nil - case <-ctx.Done(): - return funds, ctx.Err() - } - } - done := make(chan error, 1) - go func() { done <- receiveRecipientPayment(t, engine, key, info, 1) }() - <-entered - status, err := registry.Unregister("owner", owner.HeartbeatSecret) - close(release) - require.NoError(t, err) - require.Equal(t, http.StatusOK, status) - require.ErrorIs(t, <-done, ErrMissingChallenge) - var count int - require.NoError(t, engine.store.db.QueryRow(`SELECT count(*) FROM winning_tickets`).Scan(&count)) - require.Zero(t, count, "no receipt can commit after the normal drain acknowledgement") -} diff --git a/orchestrator/payment_engine.go b/orchestrator/payment_engine.go index 707deb3..d818be2 100644 --- a/orchestrator/payment_engine.go +++ b/orchestrator/payment_engine.go @@ -349,23 +349,6 @@ func (e *PaymentEngine) Receive(ctx context.Context, appID, manifest, paymentHea return payer, new(big.Rat).Set(balance), nil } -// Unregister fences new receipts before taking this lock, so any receipt -// already committing is included in the durable handoff check. -func (e *PaymentEngine) handoffDrained() error { - e.mu.Lock() - defer e.mu.Unlock() - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() - drained, err := e.store.HandoffDrained(ctx) - if err != nil { - return errors.New("ticket handoff status unavailable") - } - if !drained { - return errors.New("ticket handoff pending") - } - return nil -} - func (e *PaymentEngine) Charge(ctx context.Context, manifest string, now time.Time) (result error) { labels := e.paymentLabels("", manifest) observe := true diff --git a/orchestrator/registry.go b/orchestrator/registry.go index feadeab..fcd748d 100644 --- a/orchestrator/registry.go +++ b/orchestrator/registry.go @@ -571,17 +571,24 @@ func (r *Registry) removeApp(item *app) { func (r *Registry) Unregister(id, auth string) (int, error) { r.mu.Lock() - defer r.mu.Unlock() current := r.runners[id] if current != nil && (current.Static || !equalSecret(current.Credential, auth)) { + r.mu.Unlock() return http.StatusUnauthorized, errors.New("invalid runner credential") } if current != nil { r.removeRunner(current, nil, "runner_unregistered", "unregistered") } + r.mu.Unlock() if r.server != nil && r.server.payment != nil { - if err := r.server.payment.handoffDrained(); err != nil { - return http.StatusServiceUnavailable, err + payment := r.server.payment + payment.mu.Lock() + //lint:ignore SA2001 Wait for receipts that passed the registration fence to commit. + payment.mu.Unlock() + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if drained, err := payment.store.HandoffDrained(ctx); err != nil || !drained { + return http.StatusServiceUnavailable, errors.New("ticket handoff pending or unavailable") } } if current == nil { diff --git a/redemption/handoff_test.go b/redemption/handoff_test.go deleted file mode 100644 index 290c0f7..0000000 --- a/redemption/handoff_test.go +++ /dev/null @@ -1,42 +0,0 @@ -package redemption - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func TestHandoffDrainRequiresDurableAcknowledgementNotSettlement(t *testing.T) { - s, binding := boundStore(t, "orchestrator", "https://redeemer.example", DefaultLimits) - ticket, _ := winningTicket(t, 1) - require.NoError(t, s.Admit(t.Context(), ticket)) - binding.Destination = "https://replacement.example" - check := func(want bool) { - t.Helper() - drained, err := s.HandoffDrained(t.Context()) - require.NoError(t, err) - require.Equal(t, want, drained) - } - for _, phase := range []string{"queued", "sending", "retryable", "rejected"} { - require.NoError(t, s.delivery(t.Context(), ticket, phase, 0)) - check(false) - require.ErrorContains(t, s.Bind(t.Context(), binding, DefaultLimits, nil), "drain") - } - require.NoError(t, s.delivery(t.Context(), ticket, "acknowledged", 0)) - check(true) - var settled int - require.NoError(t, s.db.QueryRow(`SELECT settled FROM ticket_state`).Scan(&settled)) - require.Zero(t, settled) - require.NoError(t, s.Bind(t.Context(), binding, DefaultLimits, nil), "destination changes need handoff, not onchain confirmation") - path := s.path - require.NoError(t, s.Close()) - reopened, err := OpenRedeemerDB(path) - require.NoError(t, err) - defer reopened.Close() - drained, err := reopened.HandoffDrained(t.Context()) - require.NoError(t, err) - require.True(t, drained) - require.NoError(t, reopened.Close()) - _, err = reopened.HandoffDrained(t.Context()) - require.Error(t, err, "unavailable storage cannot prove a completed handoff") -} diff --git a/redemption/http_test.go b/redemption/http_test.go index 5a436c0..68da5c1 100644 --- a/redemption/http_test.go +++ b/redemption/http_test.go @@ -403,3 +403,29 @@ func TestDestinationSubmissionHeadersAndRedirectIsolation(t *testing.T) { require.Error(t, d2.Submit(t.Context(), ticket)) require.False(t, redirected) } + +func TestHandoffDrainRequiresDurableAcknowledgementNotSettlement(t *testing.T) { + s, binding := boundStore(t, "orchestrator", "https://redeemer.example", DefaultLimits) + ticket, _ := winningTicket(t, 1) + require.NoError(t, s.Admit(t.Context(), ticket)) + binding.Destination = "https://replacement.example" + for _, phase := range []string{"queued", "sending", "retryable", "rejected"} { + require.NoError(t, s.delivery(t.Context(), ticket, phase, 0)) + require.ErrorContains(t, s.Bind(t.Context(), binding, DefaultLimits, nil), "drain") + } + require.NoError(t, s.delivery(t.Context(), ticket, "acknowledged", 0)) + require.NoError(t, s.Bind(t.Context(), binding, DefaultLimits, nil), "destination changes need handoff, not onchain confirmation") + path := s.path + require.NoError(t, s.Close()) + reopened, err := OpenRedeemerDB(path) + require.NoError(t, err) + defer reopened.Close() + binding.Destination = "https://third.example" + require.NoError(t, reopened.Bind(t.Context(), binding, DefaultLimits, nil), "handoff acknowledgement survives restart") + binding.Destination = "chain" + require.NoError(t, reopened.Bind(t.Context(), binding, DefaultLimits, nil)) + require.NoError(t, reopened.Close()) + drained, err := reopened.HandoffDrained(t.Context()) + require.NoError(t, err) + require.True(t, drained, "local redemption bypasses handoff even with unavailable storage") +} diff --git a/redemption/state.go b/redemption/state.go index 90797f8..0c512bd 100644 --- a/redemption/state.go +++ b/redemption/state.go @@ -20,8 +20,9 @@ var ErrConflict = errors.New("ticket scheduling metadata conflicts with stored t type Limits struct{ Pending, PerRecipient int } -// HandoffDrained reports whether the external redeemer has durably accepted -// every outstanding winner. It does not consult the chain or wait for finality. +var DefaultLimits = Limits{Pending: 100000, PerRecipient: 10000} + +// HandoffDrained reports whether all unsettled tickets have been handed off or delivery is local. func (s *RedeemerDB) HandoffDrained(ctx context.Context) (bool, error) { if s.binding != nil && s.binding.Destination == "chain" { return true, nil @@ -31,8 +32,6 @@ func (s *RedeemerDB) HandoffDrained(ctx context.Context) (bool, error) { return drained, err } -var DefaultLimits = Limits{Pending: 100000, PerRecipient: 10000} - type Binding struct { ChainID string Controller common.Address From 386aa8f6ffc84079d105040a239197bbcbf431b1 Mon Sep 17 00:00:00 2001 From: Josh Allmann Date: Thu, 12 Nov 2026 09:31:10 -0800 Subject: [PATCH 3/3] Avoid blocking unregister on payment accounting --- orchestrator/drain_test.go | 14 +++++++------- orchestrator/registry.go | 10 +++++----- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/orchestrator/drain_test.go b/orchestrator/drain_test.go index 7887dff..30da9ec 100644 --- a/orchestrator/drain_test.go +++ b/orchestrator/drain_test.go @@ -211,7 +211,11 @@ func TestRunnerRetirementWaitsForDurableHandoffEvenAfterRegistrationLoss(t *test status, _ := registry.Unregister("owner", "wrong") require.Equal(t, http.StatusUnauthorized, status) require.False(t, lifetime.closed.Load()) + engine.mu.Lock() + release := time.AfterFunc(time.Second, engine.mu.Unlock) status, _ = registry.Unregister("owner", owner.HeartbeatSecret) + require.True(t, release.Stop(), "busy accounting must return promptly") + engine.mu.Unlock() require.Equal(t, http.StatusServiceUnavailable, status) require.True(t, lifetime.closed.Load()) // A retry, expiration or Node restart can leave no in-memory runner. @@ -234,14 +238,10 @@ func TestRetiredAppRejectsPaymentReceiptStillWaitingOnRPC(t *testing.T) { owner := appHeartbeat(t, registry, "owner", "bootstrap", registeredApp("runner")) info := issueRecipientParams(t, engine, key) entered, release := make(chan struct{}), make(chan struct{}) - chain.funds = func(ctx context.Context, funds pm.PayerFunds) (pm.PayerFunds, error) { + chain.funds = func(_ context.Context, funds pm.PayerFunds) (pm.PayerFunds, error) { close(entered) - select { - case <-release: - return funds, nil - case <-ctx.Done(): - return funds, ctx.Err() - } + <-release + return funds, nil } done := make(chan error, 1) go func() { done <- receiveRecipientPayment(t, engine, key, info, 1) }() diff --git a/orchestrator/registry.go b/orchestrator/registry.go index fcd748d..6fc995f 100644 --- a/orchestrator/registry.go +++ b/orchestrator/registry.go @@ -581,13 +581,13 @@ func (r *Registry) Unregister(id, auth string) (int, error) { } r.mu.Unlock() if r.server != nil && r.server.payment != nil { - payment := r.server.payment - payment.mu.Lock() - //lint:ignore SA2001 Wait for receipts that passed the registration fence to commit. - payment.mu.Unlock() + if !r.server.payment.mu.TryLock() { + return http.StatusServiceUnavailable, errors.New("payment accounting busy") + } + r.server.payment.mu.Unlock() ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() - if drained, err := payment.store.HandoffDrained(ctx); err != nil || !drained { + if drained, err := r.server.payment.store.HandoffDrained(ctx); err != nil || !drained { return http.StatusServiceUnavailable, errors.New("ticket handoff pending or unavailable") } }