Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 59 additions & 0 deletions orchestrator/drain_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,14 @@ import (
"context"
"encoding/json"
"fmt"
"math/big"
"net/http"
"testing"
"testing/synctest"
"time"

"github.com/livepeer/node/internal/test"
"github.com/livepeer/node/pm"
"github.com/stretchr/testify/require"
)

Expand Down Expand Up @@ -196,3 +198,60 @@ func TestAppDrainReplacementAtCapacity(t *testing.T) {
}
}
}

func TestRunnerRetirementWaitsForDurableHandoffEvenAfterRegistrationLoss(t *testing.T) {
_, store, ticket, _, _ := newRecoveryTest(t, 19)
engine, err := NewPaymentEngine(store, paymentTestChain{}, ticket.Recipient, big.NewInt(1), big.NewInt(1))
require.NoError(t, err)
registry, server := reviewServer(t, "http://127.0.0.1:1")
defer server.Close()
server.SetPayment(engine)
owner := appHeartbeat(t, registry, "owner", "bootstrap", registeredApp("app"))
lifetime := registry.apps["app"].lifetime
status, _ := registry.Unregister("owner", "wrong")
require.Equal(t, http.StatusUnauthorized, status)
require.False(t, lifetime.closed.Load())
engine.mu.Lock()
release := time.AfterFunc(time.Second, engine.mu.Unlock)
status, _ = registry.Unregister("owner", owner.HeartbeatSecret)
require.True(t, release.Stop(), "busy accounting must return promptly")
engine.mu.Unlock()
require.Equal(t, http.StatusServiceUnavailable, status)
require.True(t, lifetime.closed.Load())
// A retry, expiration or Node restart can leave no in-memory runner.
status, _ = registry.Unregister("missing-owner", "bootstrap")
require.Equal(t, http.StatusServiceUnavailable, status)
_, err = store.db.Exec(`UPDATE ticket_state SET delivery='acknowledged' WHERE hash=?`, ticket.Hash().Bytes())
require.NoError(t, err)
status, _ = registry.Unregister("missing-owner", "bootstrap")
require.Equal(t, http.StatusNotFound, status)
require.NoError(t, store.Close())
status, _ = registry.Unregister("missing-owner", "bootstrap")
require.Equal(t, http.StatusServiceUnavailable, status)
}

func TestRetiredAppRejectsPaymentReceiptStillWaitingOnRPC(t *testing.T) {
engine, chain, key, _ := newRecipientTestEngine(t)
registry, server := reviewServer(t, "http://127.0.0.1:1")
defer server.Close()
server.SetPayment(engine)
owner := appHeartbeat(t, registry, "owner", "bootstrap", registeredApp("runner"))
info := issueRecipientParams(t, engine, key)
entered, release := make(chan struct{}), make(chan struct{})
chain.funds = func(_ context.Context, funds pm.PayerFunds) (pm.PayerFunds, error) {
close(entered)
<-release
return funds, nil
}
done := make(chan error, 1)
go func() { done <- receiveRecipientPayment(t, engine, key, info, 1) }()
<-entered
status, err := registry.Unregister("owner", owner.HeartbeatSecret)
close(release)
require.NoError(t, err)
require.Equal(t, http.StatusOK, status)
require.ErrorIs(t, <-done, ErrMissingChallenge)
var count int
require.NoError(t, engine.store.db.QueryRow(`SELECT count(*) FROM winning_tickets`).Scan(&count))
require.Zero(t, count, "no receipt can commit after the normal drain acknowledgement")
}
2 changes: 1 addition & 1 deletion orchestrator/payment_engine.go
Original file line number Diff line number Diff line change
Expand Up @@ -313,7 +313,7 @@ func (e *PaymentEngine) Receive(ctx context.Context, appID, manifest, paymentHea
defer e.mu.Unlock()
// Store winning tickets before publishing in-memory replay guards and credit.
// The lock serializes receipt and accounting; failed batches change neither.
if e.sessions[manifest] != session {
if e.sessions[manifest] != session || (session.registration != nil && session.registration.closed.Load()) {
return ethcommon.Address{}, nil, ErrMissingChallenge
}
if err := snapshot.Validate(); err != nil {
Expand Down
31 changes: 24 additions & 7 deletions orchestrator/registry.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (
"slices"
"strings"
"sync"
"sync/atomic"
"time"

"github.com/livepeer/node/destination"
Expand Down Expand Up @@ -163,8 +164,8 @@ type seqConflict uint64

func (seqConflict) Error() string { return "seq must increase after owner opt-in" }

// appLifetime distinguishes registration instances without retaining their state.
type appLifetime struct{ _ byte }
// appLifetime fences receipts after their app registration is removed.
type appLifetime struct{ closed atomic.Bool }

type app struct {
lifetime *appLifetime
Expand Down Expand Up @@ -562,28 +563,44 @@ func (item *app) response() appResponse {

// removeApp requires r.mu and an app with no remaining sessions.
func (r *Registry) removeApp(item *app) {
item.lifetime.closed.Store(true)
r.runnerEvent(item.ID, item, "removed")
delete(item.owner.Apps, item.ID)
delete(r.apps, item.ID)
}

func (r *Registry) Unregister(id, auth string) (int, error) {
r.mu.Lock()
defer r.mu.Unlock()
current := r.runners[id]
if current != nil && (current.Static || !equalSecret(current.Credential, auth)) {

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Reviewed with Astra. We’re keeping this behavior in this PR to preserve retirement retries after removal, expiry, or restart, when the scoped credential is no longer available to authenticate against. Requiring bootstrap authentication would reject retries using the original scoped credential; retaining and authenticating it across restart needs a separate recovery protocol.

The absent-runner path reports coarse busy/clear status and cannot remove a live runner without its credential. The mutex-wait issue is fixed in 386aa8f.

r.mu.Unlock()
return http.StatusUnauthorized, errors.New("invalid runner credential")
}
if current != nil {
r.removeRunner(current, nil, "runner_unregistered", "unregistered")
}
r.mu.Unlock()
if r.server != nil && r.server.payment != nil {
if !r.server.payment.mu.TryLock() {
return http.StatusServiceUnavailable, errors.New("payment accounting busy")
}
r.server.payment.mu.Unlock()
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
if drained, err := r.server.payment.store.HandoffDrained(ctx); err != nil || !drained {
return http.StatusServiceUnavailable, errors.New("ticket handoff pending or unavailable")
}
}
if current == nil {
return http.StatusNotFound, errors.New("runner not found")
}
if current.Static || !equalSecret(current.Credential, auth) {
return http.StatusUnauthorized, errors.New("invalid runner credential")
}
r.removeRunner(current, nil, "runner_unregistered", "unregistered")
return http.StatusOK, nil
}

// removeRunner requires r.mu. Session teardown precedes owner/channel removal.
func (r *Registry) removeRunner(current *runner, cause error, reason, event string) {
for id, item := range current.Apps {
item.lifetime.closed.Store(true)
item.Removing = false
for sid := range item.Sessions {
r.releaseLocked(id, sid, cause, reason)
Expand Down
26 changes: 26 additions & 0 deletions redemption/http_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -403,3 +403,29 @@ func TestDestinationSubmissionHeadersAndRedirectIsolation(t *testing.T) {
require.Error(t, d2.Submit(t.Context(), ticket))
require.False(t, redirected)
}

func TestHandoffDrainRequiresDurableAcknowledgementNotSettlement(t *testing.T) {
s, binding := boundStore(t, "orchestrator", "https://redeemer.example", DefaultLimits)
ticket, _ := winningTicket(t, 1)
require.NoError(t, s.Admit(t.Context(), ticket))
binding.Destination = "https://replacement.example"
for _, phase := range []string{"queued", "sending", "retryable", "rejected"} {
require.NoError(t, s.delivery(t.Context(), ticket, phase, 0))
require.ErrorContains(t, s.Bind(t.Context(), binding, DefaultLimits, nil), "drain")
}
require.NoError(t, s.delivery(t.Context(), ticket, "acknowledged", 0))
require.NoError(t, s.Bind(t.Context(), binding, DefaultLimits, nil), "destination changes need handoff, not onchain confirmation")
path := s.path
require.NoError(t, s.Close())
reopened, err := OpenRedeemerDB(path)
require.NoError(t, err)
defer reopened.Close()
binding.Destination = "https://third.example"
require.NoError(t, reopened.Bind(t.Context(), binding, DefaultLimits, nil), "handoff acknowledgement survives restart")
binding.Destination = "chain"
require.NoError(t, reopened.Bind(t.Context(), binding, DefaultLimits, nil))
require.NoError(t, reopened.Close())
drained, err := reopened.HandoffDrained(t.Context())
require.NoError(t, err)
require.True(t, drained, "local redemption bypasses handoff even with unavailable storage")
}
16 changes: 15 additions & 1 deletion redemption/state.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,16 @@ type Limits struct{ Pending, PerRecipient int }

var DefaultLimits = Limits{Pending: 100000, PerRecipient: 10000}

// HandoffDrained reports whether all unsettled tickets have been handed off or delivery is local.
func (s *RedeemerDB) HandoffDrained(ctx context.Context) (bool, error) {
if s.binding != nil && s.binding.Destination == "chain" {
return true, nil
}
var drained bool
err := s.db.QueryRowContext(ctx, `SELECT NOT EXISTS(SELECT 1 FROM ticket_state WHERE settled=0 AND delivery!='acknowledged')`).Scan(&drained)
return drained, err
}

type Binding struct {
ChainID string
Controller common.Address
Expand Down Expand Up @@ -55,7 +65,11 @@ func (s *RedeemerDB) Bind(ctx context.Context, b Binding, limits Limits, chain N
}
if previous.Destination != b.Destination {
var count int
if err = tx.QueryRowContext(ctx, "SELECT count(*) FROM ticket_state WHERE settled=0").Scan(&count); err != nil {
pending := "SELECT count(*) FROM ticket_state WHERE settled=0"
if previous.Role == "orchestrator" && previous.Destination != "chain" {
pending += " AND delivery!='acknowledged'"
}
if err = tx.QueryRowContext(ctx, pending).Scan(&count); err != nil {
return err
}
if count != 0 {
Expand Down
Loading