diff --git a/orchestrator/drain_test.go b/orchestrator/drain_test.go index 47ab7ba..30da9ec 100644 --- a/orchestrator/drain_test.go +++ b/orchestrator/drain_test.go @@ -4,12 +4,14 @@ import ( "context" "encoding/json" "fmt" + "math/big" "net/http" "testing" "testing/synctest" "time" "github.com/livepeer/node/internal/test" + "github.com/livepeer/node/pm" "github.com/stretchr/testify/require" ) @@ -196,3 +198,60 @@ func TestAppDrainReplacementAtCapacity(t *testing.T) { } } } + +func TestRunnerRetirementWaitsForDurableHandoffEvenAfterRegistrationLoss(t *testing.T) { + _, store, ticket, _, _ := newRecoveryTest(t, 19) + engine, err := NewPaymentEngine(store, paymentTestChain{}, ticket.Recipient, big.NewInt(1), big.NewInt(1)) + require.NoError(t, err) + registry, server := reviewServer(t, "http://127.0.0.1:1") + defer server.Close() + server.SetPayment(engine) + owner := appHeartbeat(t, registry, "owner", "bootstrap", registeredApp("app")) + lifetime := registry.apps["app"].lifetime + status, _ := registry.Unregister("owner", "wrong") + require.Equal(t, http.StatusUnauthorized, status) + require.False(t, lifetime.closed.Load()) + engine.mu.Lock() + release := time.AfterFunc(time.Second, engine.mu.Unlock) + status, _ = registry.Unregister("owner", owner.HeartbeatSecret) + require.True(t, release.Stop(), "busy accounting must return promptly") + engine.mu.Unlock() + require.Equal(t, http.StatusServiceUnavailable, status) + require.True(t, lifetime.closed.Load()) + // A retry, expiration or Node restart can leave no in-memory runner. + status, _ = registry.Unregister("missing-owner", "bootstrap") + require.Equal(t, http.StatusServiceUnavailable, status) + _, err = store.db.Exec(`UPDATE ticket_state SET delivery='acknowledged' WHERE hash=?`, ticket.Hash().Bytes()) + require.NoError(t, err) + status, _ = registry.Unregister("missing-owner", "bootstrap") + require.Equal(t, http.StatusNotFound, status) + require.NoError(t, store.Close()) + status, _ = registry.Unregister("missing-owner", "bootstrap") + require.Equal(t, http.StatusServiceUnavailable, status) +} + +func TestRetiredAppRejectsPaymentReceiptStillWaitingOnRPC(t *testing.T) { + engine, chain, key, _ := newRecipientTestEngine(t) + registry, server := reviewServer(t, "http://127.0.0.1:1") + defer server.Close() + server.SetPayment(engine) + owner := appHeartbeat(t, registry, "owner", "bootstrap", registeredApp("runner")) + info := issueRecipientParams(t, engine, key) + entered, release := make(chan struct{}), make(chan struct{}) + chain.funds = func(_ context.Context, funds pm.PayerFunds) (pm.PayerFunds, error) { + close(entered) + <-release + return funds, nil + } + done := make(chan error, 1) + go func() { done <- receiveRecipientPayment(t, engine, key, info, 1) }() + <-entered + status, err := registry.Unregister("owner", owner.HeartbeatSecret) + close(release) + require.NoError(t, err) + require.Equal(t, http.StatusOK, status) + require.ErrorIs(t, <-done, ErrMissingChallenge) + var count int + require.NoError(t, engine.store.db.QueryRow(`SELECT count(*) FROM winning_tickets`).Scan(&count)) + require.Zero(t, count, "no receipt can commit after the normal drain acknowledgement") +} diff --git a/orchestrator/payment_engine.go b/orchestrator/payment_engine.go index 3d3142d..d818be2 100644 --- a/orchestrator/payment_engine.go +++ b/orchestrator/payment_engine.go @@ -313,7 +313,7 @@ func (e *PaymentEngine) Receive(ctx context.Context, appID, manifest, paymentHea defer e.mu.Unlock() // Store winning tickets before publishing in-memory replay guards and credit. // The lock serializes receipt and accounting; failed batches change neither. - if e.sessions[manifest] != session { + if e.sessions[manifest] != session || (session.registration != nil && session.registration.closed.Load()) { return ethcommon.Address{}, nil, ErrMissingChallenge } if err := snapshot.Validate(); err != nil { diff --git a/orchestrator/registry.go b/orchestrator/registry.go index 2a4e657..6fc995f 100644 --- a/orchestrator/registry.go +++ b/orchestrator/registry.go @@ -17,6 +17,7 @@ import ( "slices" "strings" "sync" + "sync/atomic" "time" "github.com/livepeer/node/destination" @@ -163,8 +164,8 @@ type seqConflict uint64 func (seqConflict) Error() string { return "seq must increase after owner opt-in" } -// appLifetime distinguishes registration instances without retaining their state. -type appLifetime struct{ _ byte } +// appLifetime fences receipts after their app registration is removed. +type appLifetime struct{ closed atomic.Bool } type app struct { lifetime *appLifetime @@ -562,6 +563,7 @@ func (item *app) response() appResponse { // removeApp requires r.mu and an app with no remaining sessions. func (r *Registry) removeApp(item *app) { + item.lifetime.closed.Store(true) r.runnerEvent(item.ID, item, "removed") delete(item.owner.Apps, item.ID) delete(r.apps, item.ID) @@ -569,21 +571,36 @@ func (r *Registry) removeApp(item *app) { func (r *Registry) Unregister(id, auth string) (int, error) { r.mu.Lock() - defer r.mu.Unlock() current := r.runners[id] + if current != nil && (current.Static || !equalSecret(current.Credential, auth)) { + r.mu.Unlock() + return http.StatusUnauthorized, errors.New("invalid runner credential") + } + if current != nil { + r.removeRunner(current, nil, "runner_unregistered", "unregistered") + } + r.mu.Unlock() + if r.server != nil && r.server.payment != nil { + if !r.server.payment.mu.TryLock() { + return http.StatusServiceUnavailable, errors.New("payment accounting busy") + } + r.server.payment.mu.Unlock() + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if drained, err := r.server.payment.store.HandoffDrained(ctx); err != nil || !drained { + return http.StatusServiceUnavailable, errors.New("ticket handoff pending or unavailable") + } + } if current == nil { return http.StatusNotFound, errors.New("runner not found") } - if current.Static || !equalSecret(current.Credential, auth) { - return http.StatusUnauthorized, errors.New("invalid runner credential") - } - r.removeRunner(current, nil, "runner_unregistered", "unregistered") return http.StatusOK, nil } // removeRunner requires r.mu. Session teardown precedes owner/channel removal. func (r *Registry) removeRunner(current *runner, cause error, reason, event string) { for id, item := range current.Apps { + item.lifetime.closed.Store(true) item.Removing = false for sid := range item.Sessions { r.releaseLocked(id, sid, cause, reason) diff --git a/redemption/http_test.go b/redemption/http_test.go index 5a436c0..68da5c1 100644 --- a/redemption/http_test.go +++ b/redemption/http_test.go @@ -403,3 +403,29 @@ func TestDestinationSubmissionHeadersAndRedirectIsolation(t *testing.T) { require.Error(t, d2.Submit(t.Context(), ticket)) require.False(t, redirected) } + +func TestHandoffDrainRequiresDurableAcknowledgementNotSettlement(t *testing.T) { + s, binding := boundStore(t, "orchestrator", "https://redeemer.example", DefaultLimits) + ticket, _ := winningTicket(t, 1) + require.NoError(t, s.Admit(t.Context(), ticket)) + binding.Destination = "https://replacement.example" + for _, phase := range []string{"queued", "sending", "retryable", "rejected"} { + require.NoError(t, s.delivery(t.Context(), ticket, phase, 0)) + require.ErrorContains(t, s.Bind(t.Context(), binding, DefaultLimits, nil), "drain") + } + require.NoError(t, s.delivery(t.Context(), ticket, "acknowledged", 0)) + require.NoError(t, s.Bind(t.Context(), binding, DefaultLimits, nil), "destination changes need handoff, not onchain confirmation") + path := s.path + require.NoError(t, s.Close()) + reopened, err := OpenRedeemerDB(path) + require.NoError(t, err) + defer reopened.Close() + binding.Destination = "https://third.example" + require.NoError(t, reopened.Bind(t.Context(), binding, DefaultLimits, nil), "handoff acknowledgement survives restart") + binding.Destination = "chain" + require.NoError(t, reopened.Bind(t.Context(), binding, DefaultLimits, nil)) + require.NoError(t, reopened.Close()) + drained, err := reopened.HandoffDrained(t.Context()) + require.NoError(t, err) + require.True(t, drained, "local redemption bypasses handoff even with unavailable storage") +} diff --git a/redemption/state.go b/redemption/state.go index d920f07..0c512bd 100644 --- a/redemption/state.go +++ b/redemption/state.go @@ -22,6 +22,16 @@ type Limits struct{ Pending, PerRecipient int } var DefaultLimits = Limits{Pending: 100000, PerRecipient: 10000} +// HandoffDrained reports whether all unsettled tickets have been handed off or delivery is local. +func (s *RedeemerDB) HandoffDrained(ctx context.Context) (bool, error) { + if s.binding != nil && s.binding.Destination == "chain" { + return true, nil + } + var drained bool + err := s.db.QueryRowContext(ctx, `SELECT NOT EXISTS(SELECT 1 FROM ticket_state WHERE settled=0 AND delivery!='acknowledged')`).Scan(&drained) + return drained, err +} + type Binding struct { ChainID string Controller common.Address @@ -55,7 +65,11 @@ func (s *RedeemerDB) Bind(ctx context.Context, b Binding, limits Limits, chain N } if previous.Destination != b.Destination { var count int - if err = tx.QueryRowContext(ctx, "SELECT count(*) FROM ticket_state WHERE settled=0").Scan(&count); err != nil { + pending := "SELECT count(*) FROM ticket_state WHERE settled=0" + if previous.Role == "orchestrator" && previous.Destination != "chain" { + pending += " AND delivery!='acknowledged'" + } + if err = tx.QueryRowContext(ctx, pending).Scan(&count); err != nil { return err } if count != 0 {