From 39b55ecffebb9f7cd627c80b2592a0cbed204327 Mon Sep 17 00:00:00 2001 From: Josh Allmann Date: Wed, 11 Nov 2026 18:08:10 -0800 Subject: [PATCH 1/4] Add redeemer authorization webhook --- .../testdata/cli/redeemer-help.golden | 6 +- configs/redeemer/config.example.toml | 3 + docs/http-api.md | 21 ++ docs/metrics.md | 1 + docs/redeemer.md | 17 + nodeconfig/config.go | 3 +- nodeconfig/headers.go | 91 +++++ {signer => nodeconfig}/headers_test.go | 2 +- redeemer/app.go | 61 +++- redeemer/app_test.go | 163 ++++++++- redeemer/auth.go | 121 +++++++ redeemer/auth_test.go | 326 ++++++++++++++++++ redemption/http.go | 20 +- redemption/http_test.go | 79 ++++- signer/headers.go | 91 +---- 15 files changed, 888 insertions(+), 117 deletions(-) create mode 100644 nodeconfig/headers.go rename {signer => nodeconfig}/headers_test.go (98%) create mode 100644 redeemer/auth.go create mode 100644 redeemer/auth_test.go diff --git a/cmd/livepeer/testdata/cli/redeemer-help.golden b/cmd/livepeer/testdata/cli/redeemer-help.golden index 2a820da..9f07064 100644 --- a/cmd/livepeer/testdata/cli/redeemer-help.golden +++ b/cmd/livepeer/testdata/cli/redeemer-help.golden @@ -21,6 +21,8 @@ Flags: -k, --keystore-file string Encrypted geth redemption account JSON file (env: LIVEPEER_REDEEMER_KEYSTORE_FILE) --keystore-password-file string Owner-only file containing exact password bytes (env: LIVEPEER_REDEEMER_KEYSTORE_PASSWORD_FILE) --rpc-url-file string (env: LIVEPEER_REDEEMER_RPC_URL_FILE) + --auth-webhook-file string (env: LIVEPEER_REDEEMER_AUTH_WEBHOOK_FILE) + --auth-webhook-headers-file string (env: LIVEPEER_REDEEMER_AUTH_WEBHOOK_HEADERS_FILE) --chain-id uint (env: LIVEPEER_REDEEMER_CHAIN_ID) --controller-address string (env: LIVEPEER_REDEEMER_CONTROLLER_ADDRESS) --redeemer-tx-timeout duration Transaction inclusion timeout (env: LIVEPEER_REDEEMER_REDEEMER_TX_TIMEOUT) (default 5m0s) @@ -35,5 +37,7 @@ Flags: Use "livepeer-redeemer [command] --help" for more information about a command. Environment Variables: - LIVEPEER_REDEEMER_KEYSTORE_PASSWORD Keystore decryption password + LIVEPEER_REDEEMER_KEYSTORE_PASSWORD Keystore decryption password LIVEPEER_REDEEMER_RPC_URL + LIVEPEER_REDEEMER_AUTH_WEBHOOK + LIVEPEER_REDEEMER_AUTH_WEBHOOK_HEADERS diff --git a/configs/redeemer/config.example.toml b/configs/redeemer/config.example.toml index cd1a897..f314898 100644 --- a/configs/redeemer/config.example.toml +++ b/configs/redeemer/config.example.toml @@ -8,6 +8,9 @@ RedeemerDB = "tickets.sqlite" KeystoreFile = "/run/secrets/livepeer-redeemer-keystore.json" KeystorePasswordFile = "/run/secrets/livepeer-redeemer-password" RPCURLFile = "/run/secrets/livepeer-rpc-url" +# Optional per-request authorization; sends request headers and the ticket. +# AuthWebhookFile = "/run/secrets/livepeer-redeemer-auth-url" +# AuthWebhookHeadersFile = "/run/secrets/livepeer-redeemer-auth-headers" # Files containing credentials must be owner-only. Password bytes are exact. # ChainID = 42161 # Controller = "0xD8E8328501E9645d16Cf49539efC04f734606ee4" diff --git a/docs/http-api.md b/docs/http-api.md index b1ab736..979f7ae 100644 --- a/docs/http-api.md +++ b/docs/http-api.md @@ -274,6 +274,27 @@ parameters. This field is not covered by the payer's signature; submitters must preserve the value issued by the orchestrator. Protect ticket submissions with TLS and endpoint authentication in your deployment; see [redeemer setup](redeemer.md). +### Redeemer authorization webhook + +When [configured](redeemer.md#authorization-webhook), the redeemer authorizes +each admitted `POST /tickets`, including duplicates, without caching decisions. +It sends a JSON POST with `headers` (incoming headers as arrays of values) and +`ticket` (the submitted ticket fields above, including `sig` and +`recipient_rand_hash`, but omitting `recipient_rand`). The ticket is null if the +submission cannot be decoded or exceeds the size limit; otherwise it may still +be invalid, as validation follows authorization. Configured webhook credentials +are sent as HTTP headers, with `Content-Type: application/json`. + +The webhook must return HTTP 200 with a JSON integer `status`: 200 approves and +400–599 denies with that status. A decision of 409 returns 403, preserving 409 +for ticket conflicts. An optional `reason` string or null is ignored and never +exposed. Redirects, timeouts, non-200 responses, and invalid decisions return 502; +the webhook timeout is five seconds. + +Overloaded requests return 503 before authorization. Accepted tickets continue +through redemption, recovery, and transaction retries independently of later +webhook decisions or outages. + ## Accounting events Optional Kafka accounting emits `create_signed_ticket` JSON events with diff --git a/docs/metrics.md b/docs/metrics.md index 6b061fd..2c49a04 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -85,6 +85,7 @@ gauges; timestamps use Unix seconds. Most event families appear on first use. | Payments | `payment_operations_total{operation,outcome}` for challenge/receive/refresh/charge; `payment_operation_duration_seconds{operation}` for challenge/receive/refresh; `payment_tickets_total{kind="accepted\|winning"}`, `payment_credit_eth_total`, `payment_charge_eth_total`, `payment_billed_seconds_total`, `payment_fixed_charges_total`, with app identity. Credit is accepted expected ticket value; charges are successfully debited service usage. | | Signer | `signer_payments_total{type,outcome}`, `signer_payment_duration_seconds{type}`, `signer_tickets_total{type}`, `signer_sign_operations_total{kind,outcome}`. Signing kinds are ticket, segment, state, and orchestrator_info. Partial signing before a failed request remains visible. Successful payment/ticket totals follow required outbox persistence, even if response delivery is interrupted. | | Authorization | `signer_auth_webhook_requests_total{status,outcome}`, `signer_auth_webhook_duration_seconds`, `signer_auth_cache_hits_total`. Status zero means no webhook response; outcomes distinguish success, denied, and error. | +| Redeemer authorization | `redeemer_auth_webhook_requests_total{status,outcome}`, `redeemer_auth_webhook_duration_seconds`. Status is the upstream HTTP status, or zero without a response; outcomes are `success`, `denied`, and `error`. | | Outbox | Existing `signer_kafka_pending_events`, `signer_kafka_pending_bytes`, `signer_kafka_oldest_event_age_seconds`, `signer_kafka_outbox_healthy`, `signer_kafka_publish_errors_total`, `signer_kafka_enqueue_errors_total`, and `signer_kafka_storage_errors_total`. New `signer_kafka_delivered_events_total` and `signer_kafka_last_delivery_timestamp_seconds` follow successful local acknowledgement; delivery remains at least once. | | Redemption queue | `redemption_queue_tickets{state}`, `redemption_oldest_ticket_timestamp_seconds{state}`: parameter-expiry `waiting`, clock-eligible `eligible`, `in_flight`, `awaiting_finality`, rejected remote submissions (`rejected`), invalid or reverted local attempts (`blocked`), or `unknown` without a fresh clock observation. Each logical ticket counts once, regardless of signatures or replacements. | | Transactions | `redemption_unresolved_transactions{phase}`, `redemption_oldest_transaction_timestamp_seconds{phase}` include historical retries/replacements until the identity is confirmed, reverted, or superseded, or its ticket is settled (`confirmed`/`used`). Ticket expiry or invalidation alone does not remove an unresolved identity. `sending` appears as `broadcast`. | diff --git a/docs/redeemer.md b/docs/redeemer.md index 73b435d..46429e8 100644 --- a/docs/redeemer.md +++ b/docs/redeemer.md @@ -17,6 +17,23 @@ through your deployment or a proxy. Use a separate persistent database for each redeemer; see [configuration](configuration.md) and [recovery](payment-recovery.md). `--config` takes a local TOML or JSON file path, as with other components. +## Authorization webhook + +To authorize ticket submissions through a policy service, configure: + +```toml +AuthWebhookFile = "/run/secrets/livepeer-redeemer-auth-url" +# AuthWebhookHeadersFile = "/run/secrets/livepeer-redeemer-auth-headers" +``` + +Both files must be owner-only regular files. The URL must be absolute HTTP or +HTTPS without a username, password, or fragment. The optional headers file uses +comma-separated `Header: value` entries to authenticate the redeemer to the +webhook. If the policy relies on caller identity headers, configure the proxy to +authenticate callers and replace untrusted identity headers. See the +[HTTP contract](http-api.md#redeemer-authorization-webhook) and +[metrics](metrics.md). + ## Connect an orchestrator Add these settings to an onchain orchestrator's configuration, using its diff --git a/nodeconfig/config.go b/nodeconfig/config.go index 373f892..df23d41 100644 --- a/nodeconfig/config.go +++ b/nodeconfig/config.go @@ -1,4 +1,5 @@ -// Package nodeconfig supplies shared CLI policy, data-directory defaults, and storage paths. +// Package nodeconfig supplies shared configuration types, CLI policy, +// data-directory defaults, and storage paths. package nodeconfig import ( diff --git a/nodeconfig/headers.go b/nodeconfig/headers.go new file mode 100644 index 0000000..d87e7e6 --- /dev/null +++ b/nodeconfig/headers.go @@ -0,0 +1,91 @@ +package nodeconfig + +import ( + "bytes" + "encoding/csv" + "errors" + "io" + "maps" + "net/http" + "slices" + "strings" +) + +// Headers is a text scalar for configuration and secret files. It accepts +// comma-separated Header: value entries, with CSV quoting for values containing +// commas. Header names are case-insensitive and repeated names retain all values. +type Headers http.Header + +func validHeader(name, value string) bool { + if name == "" { + return false + } + for _, c := range name { + if !((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || strings.ContainsRune("!#$%&'*+-.^_`|~", c)) { + return false + } + } + for _, c := range value { + if (c < 32 && c != '\t') || c == 127 { + return false + } + } + return true +} + +func (h *Headers) UnmarshalText(text []byte) error { + parsed := make(http.Header) + input := strings.TrimSpace(string(text)) + if input != "" { + reader := csv.NewReader(strings.NewReader(input)) + reader.TrimLeadingSpace = true + reader.FieldsPerRecord = -1 + entries, err := reader.Read() + if err != nil { + return errors.New("invalid comma-separated webhook headers") + } + if _, err := reader.Read(); err != io.EOF { + return errors.New("webhook headers must be one comma-separated record") + } + for _, entry := range entries { + name, value, ok := strings.Cut(entry, ":") + name, value = strings.TrimSpace(name), strings.TrimSpace(value) + if !ok || !validHeader(name, value) { + return errors.New("expected webhook headers in Header: value form") + } + parsed.Add(name, value) + } + } + *h = Headers(parsed) + return nil +} + +// MarshalText sorts names for deterministic secret-file comparisons and auth +// policy fingerprints, and quotes entries containing commas or quotation marks. +func (h Headers) MarshalText() ([]byte, error) { + var entries []string + for _, name := range slices.Sorted(maps.Keys(h)) { + if !validHeader(name, "") { + return nil, errors.New("invalid webhook header name") + } + for _, value := range h[name] { + if !validHeader(name, value) { + return nil, errors.New("invalid webhook header name or value") + } + entries = append(entries, name+": "+value) + } + } + if len(entries) == 0 { + return nil, nil + } + var buffer bytes.Buffer + writer := csv.NewWriter(&buffer) + if err := writer.Write(entries); err != nil { + return nil, err + } + writer.Flush() + if err := writer.Error(); err != nil { + return nil, err + } + return bytes.TrimSuffix(buffer.Bytes(), []byte("\n")), nil +} diff --git a/signer/headers_test.go b/nodeconfig/headers_test.go similarity index 98% rename from signer/headers_test.go rename to nodeconfig/headers_test.go index b933b58..faf1bc4 100644 --- a/signer/headers_test.go +++ b/nodeconfig/headers_test.go @@ -1,4 +1,4 @@ -package signer +package nodeconfig import ( "net/http" diff --git a/redeemer/app.go b/redeemer/app.go index a3efec3..6eafe63 100644 --- a/redeemer/app.go +++ b/redeemer/app.go @@ -11,6 +11,7 @@ import ( "net" "net/http" "net/url" + "os" "os/signal" "sync" "syscall" @@ -33,23 +34,27 @@ import ( type Params struct { ConfigFile string `name:"config" configfile:"optional-default" basepath:"source" default:"config.toml" boa:"noconfig" descr:"Configuration file; empty disables discovery"` nodeconfig.Settings - Listen boa.HTTPBind `default:"127.0.0.1:8941" descr:"Ticket submission listener"` - MetricsListen boa.HTTPBind `default:"127.0.0.1:8942" descr:"Metrics and probe listener"` - Account *common.Address `descr:"Dedicated redemption wallet; must not be an orchestrator"` - RedeemerDB string `file:"optional" descr:"Winning-ticket SQLite database"` - KeystoreFile string `basepath:"source" optional:"true" descr:"Encrypted geth redemption account JSON file"` - KeystorePassword *string `secret:"true" optional:"true" descr:"Keystore decryption password"` - KeystorePasswordFile string `basepath:"source" secretfor:"KeystorePassword" descr:"Owner-only file containing exact password bytes"` - RPCURL *url.URL `name:"rpc-url" secret:"true" optional:"true"` - RPCURLFile string `basepath:"source" name:"rpc-url-file" secretfor:"RPCURL"` - ChainID *uint64 `min:"1"` - Controller *common.Address `name:"controller-address"` - RedeemerTxTimeout time.Duration `default:"5m" descr:"Transaction inclusion timeout"` - RedeemerMaxTxReplacements int `default:"1" min:"0" descr:"Maximum fee replacements per transaction"` - RedeemerMaxFeePerGas *uint256.Int `descr:"Optional maximum fee in wei per gas"` - MaxPendingTickets int `default:"100000" min:"1" descr:"Maximum unsettled winning tickets"` - MaxTicketsPerRecipient int `default:"10000" min:"1" descr:"Maximum unsettled tickets per recipient"` - PrintConfig bool `optional:"true" boa:"noconfig,noenv" descr:"Print audited redacted TOML configuration"` + Listen boa.HTTPBind `default:"127.0.0.1:8941" descr:"Ticket submission listener"` + MetricsListen boa.HTTPBind `default:"127.0.0.1:8942" descr:"Metrics and probe listener"` + Account *common.Address `descr:"Dedicated redemption wallet; must not be an orchestrator"` + RedeemerDB string `file:"optional" descr:"Winning-ticket SQLite database"` + KeystoreFile string `basepath:"source" optional:"true" descr:"Encrypted geth redemption account JSON file"` + KeystorePassword *string `secret:"true" optional:"true" descr:"Keystore decryption password"` + KeystorePasswordFile string `basepath:"source" secretfor:"KeystorePassword" descr:"Owner-only file containing exact password bytes"` + RPCURL *url.URL `name:"rpc-url" secret:"true" optional:"true"` + RPCURLFile string `basepath:"source" name:"rpc-url-file" secretfor:"RPCURL"` + AuthWebhook *url.URL `name:"auth-webhook" optional:"true" secret:"true"` + AuthWebhookFile string `basepath:"source" name:"auth-webhook-file" secretfor:"AuthWebhook"` + AuthWebhookHeaders nodeconfig.Headers `name:"auth-webhook-headers" optional:"true" secret:"true"` + AuthWebhookHeadersFile string `basepath:"source" name:"auth-webhook-headers-file" secretfor:"AuthWebhookHeaders"` + ChainID *uint64 `min:"1"` + Controller *common.Address `name:"controller-address"` + RedeemerTxTimeout time.Duration `default:"5m" descr:"Transaction inclusion timeout"` + RedeemerMaxTxReplacements int `default:"1" min:"0" descr:"Maximum fee replacements per transaction"` + RedeemerMaxFeePerGas *uint256.Int `descr:"Optional maximum fee in wei per gas"` + MaxPendingTickets int `default:"100000" min:"1" descr:"Maximum unsettled winning tickets"` + MaxTicketsPerRecipient int `default:"10000" min:"1" descr:"Maximum unsettled tickets per recipient"` + PrintConfig bool `optional:"true" boa:"noconfig,noenv" descr:"Print audited redacted TOML configuration"` } func (p Params) Validate() error { @@ -68,6 +73,23 @@ func (p Params) Validate() error { if err := destination.ValidateURL(p.RPCURL); err != nil { return errors.New("redeemer requires a valid RPC URL") } + if p.AuthWebhook != nil || p.AuthWebhookFile != "" || len(p.AuthWebhookHeaders) > 0 || p.AuthWebhookHeadersFile != "" { + if err := destination.ValidateURL(p.AuthWebhook); err != nil || p.AuthWebhook.User != nil || p.AuthWebhook.Fragment != "" { + return errors.New("auth webhook must be an absolute HTTP or HTTPS URL without credentials or fragment") + } + } + if _, err := p.AuthWebhookHeaders.MarshalText(); err != nil { + return fmt.Errorf("invalid auth webhook headers: %w", err) + } + for _, path := range []string{p.AuthWebhookFile, p.AuthWebhookHeadersFile} { + if path == "" { + continue + } + info, err := os.Stat(path) + if err != nil || !info.Mode().IsRegular() || info.Mode().Perm()&0077 != 0 { + return errors.New("auth webhook secret files must be owner-only regular files") + } + } if p.RedeemerTxTimeout <= 0 || p.RedeemerMaxTxReplacements < 0 || p.MaxPendingTickets <= 0 || p.MaxTicketsPerRecipient <= 0 { return errors.New("invalid redemption timeout, replacement count or queue capacity") } @@ -198,6 +220,11 @@ func Serve(parent context.Context, p Params, logOut io.Writer) (result error) { }) probe.Handle("GET /metrics", metrics.Handler()) handler := redemption.Handler(store) + if p.AuthWebhook != nil { + auth := newAuthWebhook(p.AuthWebhook, p.AuthWebhookHeaders, logger, metrics) + defer auth.client.CloseIdleConnections() + handler = redemption.HandlerWithAuthorization(store, auth.authorize) + } servers := []*http.Server{ {Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { metrics.HTTP(w, r, handler.ServeHTTP) }), BaseContext: func(net.Listener) context.Context { return ctx }, ReadHeaderTimeout: 5 * time.Second, ReadTimeout: 15 * time.Second, WriteTimeout: 15 * time.Second, IdleTimeout: 30 * time.Second, MaxHeaderBytes: 1 << 16, MaxHeaderValueCount: 128}, {Handler: probe, ReadHeaderTimeout: 5 * time.Second, WriteTimeout: 15 * time.Second, IdleTimeout: 30 * time.Second, MaxHeaderBytes: 1 << 16, MaxHeaderValueCount: 128}, diff --git a/redeemer/app_test.go b/redeemer/app_test.go index 33c2340..0e4849b 100644 --- a/redeemer/app_test.go +++ b/redeemer/app_test.go @@ -9,10 +9,12 @@ import ( "math/big" "net" "net/http" + "net/http/httptest" "net/url" "os" "path/filepath" "strings" + "sync/atomic" "testing" "time" @@ -24,6 +26,7 @@ import ( "github.com/livepeer/node/nodeconfig" "github.com/livepeer/node/pm" "github.com/livepeer/node/redemption" + "github.com/spf13/cobra" "github.com/stretchr/testify/require" ) @@ -77,11 +80,20 @@ func TestDedicatedWalletStartupPolicy(t *testing.T) { } func TestServeProbesAndShutdown(t *testing.T) { - for _, busy := range []bool{false, true} { - t.Run(fmt.Sprintf("busy=%t", busy), func(t *testing.T) { + for _, tc := range []struct{ busy, auth bool }{{}, {busy: true}, {auth: true}} { + t.Run(fmt.Sprintf("busy=%t/auth=%t", tc.busy, tc.auth), func(t *testing.T) { f := test.NewRedemptionRPC(t) p := serviceParams(t, f) - if busy { + var calls atomic.Int32 + if tc.auth { + webhook := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + _, _ = io.WriteString(w, `{"status":403}`) + })) + defer webhook.Close() + p.AuthWebhook, _ = url.Parse(webhook.URL) + } + if tc.busy { seedBusyQueue(t, p) handler := f.Server.Config.Handler selector := fmt.Sprintf("0x%x", crypto.Keccak256([]byte("usedTickets(bytes32)"))[:4]) @@ -106,7 +118,7 @@ func TestServeProbesAndShutdown(t *testing.T) { }) } wait := 5 * time.Second - if busy { + if tc.busy { wait = 40 * time.Second } ctx, cancel := context.WithCancel(t.Context()) @@ -123,14 +135,15 @@ func TestServeProbesAndShutdown(t *testing.T) { }) client := &http.Client{Timeout: time.Second} defer client.CloseIdleConnections() - require.Eventually(t, func() bool { + ready := func() bool { response, err := client.Get("http://" + p.MetricsListen.String() + "/readyz") if err != nil { return false } defer response.Body.Close() return response.StatusCode == 200 - }, wait, 10*time.Millisecond) + } + require.Eventually(t, ready, wait, 10*time.Millisecond) response, err := client.Get("http://" + p.MetricsListen.String() + "/metrics") require.NoError(t, err) defer response.Body.Close() @@ -141,6 +154,15 @@ func TestServeProbesAndShutdown(t *testing.T) { require.NoError(t, err) response.Body.Close() require.Equal(t, 405, response.StatusCode) + if tc.auth { + require.Zero(t, calls.Load(), "probes and other methods do not authorize") + response, err = client.Post("http://"+p.Listen.String()+"/tickets", "application/json", strings.NewReader("invalid ticket")) + require.NoError(t, err) + response.Body.Close() + require.Equal(t, 403, response.StatusCode, "invalid submissions are still authorized") + require.True(t, ready(), "authorization denial does not change readiness") + require.EqualValues(t, 1, calls.Load()) + } info, err := os.Stat(p.RedeemerDB) require.NoError(t, err) require.Equal(t, os.FileMode(0600), info.Mode().Perm()) @@ -169,6 +191,8 @@ func TestPrintConfigRedactsCredentials(t *testing.T) { t.Setenv("HOME", t.TempDir()) t.Setenv("LIVEPEER_REDEEMER_RPC_URL", "http://private.example/secret-token") t.Setenv("LIVEPEER_REDEEMER_KEYSTORE_PASSWORD", "wallet-secret") + t.Setenv("LIVEPEER_REDEEMER_AUTH_WEBHOOK", "https://example.com/?auth-url-secret") + t.Setenv("LIVEPEER_REDEEMER_AUTH_WEBHOOK_HEADERS", "Authorization: Bearer auth-header-secret") var out, errOut bytes.Buffer cmd := Root(&out, &errOut) cmd.SetArgs([]string{"--config=", "--print-config"}) @@ -176,4 +200,131 @@ func TestPrintConfigRedactsCredentials(t *testing.T) { require.Contains(t, out.String(), "8941") require.NotContains(t, out.String(), "secret-token") require.NotContains(t, out.String(), "wallet-secret") + require.NotContains(t, out.String(), "auth-url-secret") + require.NotContains(t, out.String(), "auth-header-secret") +} + +func loadAuthConfig(t *testing.T, p Params, args ...string) (Params, error) { + t.Helper() + cmd := nodeconfig.Command("redeemer", nodeconfig.Mainnet, boa.Cmd[Params]{Params: &p, RejectUnknown: true, + RunFuncCtxE: func(ctx *boa.HookContext, p *Params, _ *cobra.Command, _ []string) error { + dump, err := ctx.DumpBytes(".toml", nil) + require.NoError(t, err) + require.NotContains(t, string(dump), "private-detail") + return p.Validate() + }, + }, &p.ConfigFile) + err := cmd.RunArgsE(args) + return p, err +} + +func TestAuthWebhookConfiguration(t *testing.T) { + base := serviceParams(t, test.NewRedemptionRPC(t)) + const endpoint = "https://webhook.example/auth?token=private-detail" + const headers = `Authorization: Bearer private-detail, X-Tag: one, x-tag: two, "X-List: a,b"` + for _, tc := range []struct { + name, field, content string + withURL bool + }{ + {"empty URL file", "AUTH_WEBHOOK_FILE", "", false}, + {"empty orphan headers file", "AUTH_WEBHOOK_HEADERS_FILE", "", false}, + {"blank orphan headers file", "AUTH_WEBHOOK_HEADERS_FILE", " \t\n", false}, + {"empty headers with URL", "AUTH_WEBHOOK_HEADERS_FILE", "", true}, + } { + t.Run(tc.name, func(t *testing.T) { + file := filepath.Join(t.TempDir(), "secret") + require.NoError(t, os.WriteFile(file, []byte(tc.content), 0600)) + t.Setenv("LIVEPEER_REDEEMER_AUTH_WEBHOOK", "") + t.Setenv("LIVEPEER_REDEEMER_"+tc.field, file) + if tc.withURL { + t.Setenv("LIVEPEER_REDEEMER_AUTH_WEBHOOK", endpoint) + } + _, err := loadAuthConfig(t, base, "--config=") + if tc.withURL { + require.NoError(t, err) + } else { + require.ErrorContains(t, err, "auth webhook must be an absolute HTTP or HTTPS URL") + require.NotContains(t, err.Error(), "private-detail") + } + }) + } + for _, source := range []string{"environment", "TOML", "JSON", "file flags"} { + t.Run(source, func(t *testing.T) { + dir := t.TempDir() + uFile, hFile := filepath.Join(dir, "url"), filepath.Join(dir, "headers") + require.NoError(t, os.WriteFile(uFile, []byte(endpoint), 0600)) + require.NoError(t, os.WriteFile(hFile, []byte(headers), 0600)) + args := []string{"--config="} + switch source { + case "environment": + t.Setenv("LIVEPEER_REDEEMER_AUTH_WEBHOOK", endpoint) + t.Setenv("LIVEPEER_REDEEMER_AUTH_WEBHOOK_HEADERS", headers) + case "file flags": + args = append(args, "--auth-webhook-file", uFile, "--auth-webhook-headers-file", hFile) + default: + ext := ".toml" + if source == "JSON" { + ext = ".json" + } + path := filepath.Join(dir, "config"+ext) + test.WriteConfig(t, path, "AuthWebhookFile = 'url'\nAuthWebhookHeadersFile = 'headers'\n") + args = []string{"--config", path} + } + p, err := loadAuthConfig(t, base, args...) + require.NoError(t, err) + require.Equal(t, endpoint, p.AuthWebhook.String()) + require.Equal(t, "Bearer private-detail", http.Header(p.AuthWebhookHeaders).Get("Authorization")) + require.Equal(t, []string{"one", "two"}, http.Header(p.AuthWebhookHeaders).Values("X-Tag")) + require.Equal(t, "a,b", http.Header(p.AuthWebhookHeaders).Get("X-List")) + }) + } + for _, flag := range []string{"--auth-webhook", "--auth-webhook-headers"} { + _, err := loadAuthConfig(t, base, "--config=", flag, "private-detail") + require.ErrorContains(t, err, "unknown flag") + require.NotContains(t, err.Error(), "private-detail") + } + for _, field := range []string{"AUTH_WEBHOOK", "AUTH_WEBHOOK_HEADERS"} { + t.Run(field+" file protection", func(t *testing.T) { + file := filepath.Join(t.TempDir(), "secret") + value := endpoint + if field == "AUTH_WEBHOOK_HEADERS" { + value = headers + } + require.NoError(t, os.WriteFile(file, []byte(value), 0600)) + t.Setenv("LIVEPEER_REDEEMER_AUTH_WEBHOOK", endpoint) + t.Setenv("LIVEPEER_REDEEMER_"+field+"_FILE", file) + if field == "AUTH_WEBHOOK_HEADERS" { + t.Setenv("LIVEPEER_REDEEMER_"+field, headers) + } + _, err := loadAuthConfig(t, base, "--config=") + require.ErrorContains(t, err, "cannot both be set") + require.NotContains(t, err.Error(), "private-detail") + t.Setenv("LIVEPEER_REDEEMER_"+field, "") + require.NoError(t, os.Chmod(file, 0644)) + _, err = loadAuthConfig(t, base, "--config=") + require.Error(t, err) + require.NotContains(t, err.Error(), "private-detail") + }) + } +} + +func TestAuthWebhookConfigurationValidation(t *testing.T) { + p := serviceParams(t, test.NewRedemptionRPC(t)) + require.NoError(t, p.Validate(), "omitted webhook is disabled") + for _, endpoint := range []string{"/private-detail", "ftp://private-detail", "https://user:private-detail@example.com", "https://example.com/#private-detail"} { + p.AuthWebhook, _ = url.Parse(endpoint) + err := p.Validate() + require.Error(t, err) + require.NotContains(t, err.Error(), "private-detail") + } + p.AuthWebhook = nil + p.AuthWebhookHeaders = nodeconfig.Headers{"Authorization": {"private-detail"}} + require.Error(t, p.Validate(), "headers require an endpoint") + p.AuthWebhook, _ = url.Parse("http://127.0.0.1/private?token=private-detail") + require.NoError(t, p.Validate()) + p.AuthWebhookHeaders = nodeconfig.Headers{"Bad Name": {"private-detail"}} + err := p.Validate() + require.ErrorContains(t, err, "invalid auth webhook headers") + require.NotContains(t, err.Error(), "private-detail") + } diff --git a/redeemer/auth.go b/redeemer/auth.go new file mode 100644 index 0000000..c8c7cfe --- /dev/null +++ b/redeemer/auth.go @@ -0,0 +1,121 @@ +package redeemer + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "net/url" + "strconv" + "time" + + "github.com/livepeer/node/internal/observability" + "github.com/livepeer/node/nodeconfig" + "github.com/livepeer/node/redemption" +) + +type authWebhook struct { + endpoint string + headers http.Header + client *http.Client + logger *slog.Logger + metrics *observability.Metrics +} + +// Configuration is validated by Params before constructing the immutable client. +func newAuthWebhook(endpoint *url.URL, headers nodeconfig.Headers, logger *slog.Logger, metrics *observability.Metrics) *authWebhook { + return &authWebhook{ + endpoint: endpoint.String(), + headers: http.Header(headers).Clone(), + client: &http.Client{ + Transport: http.DefaultTransport.(*http.Transport).Clone(), + Timeout: 5 * time.Second, + CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, + }, + logger: logger.With("event", "authorization", "operation", "authorization"), + metrics: metrics, + } +} + +func (a *authWebhook) authorize(w http.ResponseWriter, r *http.Request, ticket *redemption.TicketJSON) bool { + started := time.Now() + status, outcome := 0, "error" + defer func() { + a.metrics.Add("redeemer_auth_webhook_requests_total", 1, observability.Labels{"status": strconv.Itoa(status), "outcome": outcome}) + a.metrics.Observe("redeemer_auth_webhook_duration_seconds", time.Since(started).Seconds(), nil) + }() + fail := func(message, kind string) bool { + a.logger.ErrorContext(r.Context(), message, "outcome", "error", "error_kind", kind) + http.Error(w, message, http.StatusBadGateway) + return false + } + type webhookTicket struct { + *redemption.TicketJSON + // Shadow the JSON field without changing the ticket used for redemption. + Randomness *string `json:"recipient_rand,omitempty"` + } + var outgoing *webhookTicket + if ticket != nil { + outgoing = &webhookTicket{TicketJSON: ticket} + } + body, err := json.Marshal(struct { + Headers http.Header `json:"headers"` + Ticket *webhookTicket `json:"ticket"` + }{r.Header, outgoing}) + if err != nil { + return fail("redeemer auth payload encoding failed", observability.ErrorKind(err)) + } + request, err := http.NewRequestWithContext(r.Context(), http.MethodPost, a.endpoint, bytes.NewReader(body)) + if err != nil { + return fail("redeemer auth request construction failed", observability.ErrorKind(err)) + } + // A received authorization request must not be replayed after a connection + // failure, even when configured service headers include an idempotency key. + request.GetBody = nil + request.Header = a.headers.Clone() + if request.Header == nil { + request.Header = make(http.Header) + } + request.Header.Set("Content-Type", "application/json") + response, err := a.client.Do(request) + if err != nil { + return fail("redeemer auth webhook unavailable", observability.ErrorKind(err)) + } + defer response.Body.Close() + status = response.StatusCode + if status != http.StatusOK { + return fail("redeemer auth webhook failed", "http_status") + } + data, err := io.ReadAll(io.LimitReader(response.Body, (1<<20)+1)) + if err != nil { + return fail("invalid redeemer auth response", observability.ErrorKind(err)) + } + if len(data) > 1<<20 { + return fail("invalid redeemer auth response", "response_limit") + } + var decision struct { + Status int `json:"status"` + Reason string `json:"reason"` + } + if err := json.Unmarshal(data, &decision); err != nil { + return fail("invalid redeemer auth response", observability.ErrorKind(err)) + } + if decision.Status == http.StatusOK { + outcome = "success" + return true + } + if decision.Status < 400 || decision.Status > 599 { + return fail("invalid redeemer auth response", "invalid_decision") + } + outcome = "denied" + a.logger.WarnContext(r.Context(), "redeemer authorization rejected", "outcome", outcome, "error_kind", "policy_rejected", "decision_status", decision.Status) + deniedStatus := decision.Status + if deniedStatus == http.StatusConflict { + // Forwarders permanently reject 409 as conflicting ticket metadata. + deniedStatus = http.StatusForbidden + } + http.Error(w, fmt.Sprintf("redeemer auth rejected request with status %d", decision.Status), deniedStatus) + return false +} diff --git a/redeemer/auth_test.go b/redeemer/auth_test.go new file mode 100644 index 0000000..ef3e058 --- /dev/null +++ b/redeemer/auth_test.go @@ -0,0 +1,326 @@ +package redeemer + +import ( + "bytes" + "context" + "database/sql" + "encoding/json" + "errors" + "io" + "log/slog" + "math/big" + "net/http" + "net/http/httptest" + "net/url" + "path/filepath" + "strconv" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/ethereum/go-ethereum/accounts" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" + "github.com/livepeer/node/destination" + "github.com/livepeer/node/eth" + "github.com/livepeer/node/internal/observability" + "github.com/livepeer/node/internal/test" + "github.com/livepeer/node/nodeconfig" + "github.com/livepeer/node/pm" + "github.com/livepeer/node/redemption" + "github.com/stretchr/testify/require" +) + +func testAuthWebhook(t *testing.T, endpoint string, headers nodeconfig.Headers, logs io.Writer) *authWebhook { + t.Helper() + u, err := url.Parse(endpoint) + require.NoError(t, err) + auth := newAuthWebhook(u, headers, slog.New(slog.NewTextHandler(logs, nil)), observability.New("redeemer")) + t.Cleanup(auth.client.CloseIdleConnections) + return auth +} + +func TestAuthWebhookHeadersAndRequests(t *testing.T) { + type webhookRequest struct { + Headers http.Header `json:"headers"` + Ticket json.RawMessage `json:"ticket"` + } + received := make(chan webhookRequest, 1) + var calls atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + require.Equal(t, "POST", r.Method) + require.Equal(t, "token=private-url", r.URL.RawQuery) + require.Equal(t, "application/json", r.Header.Get("Content-Type")) + require.Equal(t, "Bearer service-secret", r.Header.Get("Authorization")) + require.Empty(t, r.Header.Values("X-Caller")) + var payload webhookRequest + decoder := json.NewDecoder(r.Body) + decoder.DisallowUnknownFields() + require.NoError(t, decoder.Decode(&payload)) + received <- payload + _, _ = io.WriteString(w, `{"status":200,"expiry":9999999999,"auth_id":"ignored","maxPrice":{}}`) + })) + defer server.Close() + headers := nodeconfig.Headers{"Authorization": {"Bearer service-secret"}, "Content-Type": {"text/plain"}} + auth := testAuthWebhook(t, server.URL+"?token=private-url", headers, io.Discard) + headers["Authorization"][0] = "mutated" + require.Equal(t, 5*time.Second, auth.client.Timeout) + store, err := redemption.OpenRedeemerDB(filepath.Join(t.TempDir(), "tickets.sqlite")) + require.NoError(t, err) + defer store.Close() + binding := redemption.Binding{ChainID: "1", Controller: common.HexToAddress("0x1000"), Role: "redeemer", Destination: "chain"} + require.NoError(t, store.Bind(t.Context(), binding, redemption.DefaultLimits, nil)) + ticket := redemption.TicketJSON{ + Payer: common.HexToAddress("0x1234"), Recipient: common.HexToAddress("0x5678"), + FaceValue: "1000", WinProb: "42", Nonce: 7, + RandHash: common.HexToHash("0x1234"), CreationRound: "1", RoundHash: common.HexToHash("0x5678"), + Expiration: "10", Signature: []byte{1, 2, 3}, Randomness: "9", + } + body, err := json.Marshal(redemption.TicketRequest{ChainID: binding.ChainID, Controller: binding.Controller, Ticket: ticket}) + require.NoError(t, err) + handler := redemption.HandlerWithAuthorization(store, auth.authorize) + for _, tc := range []struct { + body string + ticket *redemption.TicketJSON + }{ + {"bad ticket", nil}, + {strings.Repeat(" ", redemption.MaxTicketBody) + string(body), nil}, + {string(body), &ticket}, + } { + r := httptest.NewRequest("POST", "/tickets", strings.NewReader(tc.body)) + r.Header = http.Header{"Authorization": {"Bearer caller-secret"}, "X-Caller": {"one", "two"}} + w := httptest.NewRecorder() + handler.ServeHTTP(w, r) + require.Equal(t, 400, w.Code, "approval does not bypass ticket decoding or validation") + payload := <-received + require.Equal(t, r.Header, payload.Headers) + if tc.ticket == nil { + require.JSONEq(t, "null", string(payload.Ticket)) + continue + } + var fields map[string]json.RawMessage + require.NoError(t, json.Unmarshal(payload.Ticket, &fields)) + require.NotContains(t, fields, "recipient_rand", "the secret field must be absent, not empty or null") + require.Contains(t, fields, "sig") + require.Contains(t, fields, "recipient_rand_hash") + var actual redemption.TicketJSON + require.NoError(t, json.Unmarshal(payload.Ticket, &actual)) + expected := *tc.ticket + expected.Randomness = "" + require.Equal(t, expected, actual, "all other ticket fields reach the webhook unchanged") + } + for _, target := range []string{"/healthz", "/readyz", "/metrics", "/other"} { + handler.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", target, nil)) + } + handler.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "/tickets", nil)) + require.EqualValues(t, 3, calls.Load(), "approvals are never cached") +} + +func TestAuthWebhookDecisions(t *testing.T) { + for _, tc := range []struct { + name, body string + upstream, want int + outcome string + }{ + {"approval", `{"status":200}`, 200, 200, "success"}, + {"nullable reason", `{"status":200,"reason":null}`, 200, 200, "success"}, + {"denial", `{"status":403,"reason":"private-detail"}`, 200, 403, "denied"}, + {"conflict denial", `{"status":409}`, 200, 403, "denied"}, + {"service denial", `{"status":503}`, 200, 503, "denied"}, + {"last denial", `{"status":599}`, 200, 599, "denied"}, + {"non-200", "private-detail", 500, 502, "error"}, + {"redirect", "private-detail", 307, 502, "error"}, + {"malformed", "private-detail", 200, 502, "error"}, + {"trailing JSON", `{"status":200}{}`, 200, 502, "error"}, + {"missing status", `{}`, 200, 502, "error"}, + {"null status", `{"status":null}`, 200, 502, "error"}, + {"string status", `{"status":"200"}`, 200, 502, "error"}, + {"fractional status", `{"status":200.5}`, 200, 502, "error"}, + {"invalid status", `{"status":201}`, 200, 502, "error"}, + {"invalid denial", `{"status":600}`, 200, 502, "error"}, + {"invalid reason", `{"status":200,"reason":123}`, 200, 502, "error"}, + {"oversized", strings.Repeat(" ", (1<<20)+1), 200, 502, "error"}, + {"read failure", `{"status":200}`, 200, 502, "error"}, + } { + t.Run(tc.name, func(t *testing.T) { + var calls atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + w.Header().Set("Location", "/redirected") + if tc.name == "read failure" { + w.Header().Set("Content-Length", strconv.Itoa(len(tc.body)+10)) + } + w.WriteHeader(tc.upstream) + _, _ = io.WriteString(w, tc.body) + })) + defer server.Close() + var logs bytes.Buffer + auth := testAuthWebhook(t, server.URL+"?private-detail", nodeconfig.Headers{"Authorization": {"private-detail"}}, &logs) + r := httptest.NewRequest("POST", "/tickets", nil) + w := httptest.NewRecorder() + allowed := auth.authorize(w, r, nil) + require.Equal(t, tc.want == 200, allowed) + require.Equal(t, tc.want, w.Code) + require.EqualValues(t, 1, calls.Load()) + require.NotContains(t, logs.String(), "private-detail") + require.NotContains(t, w.Body.String(), "private-detail") + require.EqualValues(t, 1, test.MetricSum(t, auth.metrics.Registry, "redeemer_auth_webhook_requests_total", map[string]string{"status": strconv.Itoa(tc.upstream), "outcome": tc.outcome})) + require.EqualValues(t, 1, test.MetricSum(t, auth.metrics.Registry, "redeemer_auth_webhook_duration_seconds", nil)) + }) + } +} + +func TestAuthWebhookUnavailableTimeoutAndCancellation(t *testing.T) { + for _, kind := range []string{"unavailable", "timeout", "cancelled"} { + t.Run(kind, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + <-r.Context().Done() + })) + defer server.Close() + if kind == "unavailable" { + server.Close() + } + var logs bytes.Buffer + auth := testAuthWebhook(t, server.URL+"?private-detail", nil, &logs) + if kind == "timeout" { + auth.client.Timeout = 20 * time.Millisecond + } + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + if kind == "cancelled" { + cancel() + } + r := httptest.NewRequest("POST", "/tickets", nil).WithContext(ctx) + w := httptest.NewRecorder() + require.False(t, auth.authorize(w, r, nil)) + require.Equal(t, 502, w.Code) + require.NotContains(t, logs.String(), "private-detail") + require.EqualValues(t, 1, test.MetricSum(t, auth.metrics.Registry, "redeemer_auth_webhook_requests_total", map[string]string{"status": "0", "outcome": "error"})) + }) + } +} + +func TestAuthWebhookDoesNotReplayReceivedPOST(t *testing.T) { + var calls atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + if calls.Add(1) == 2 { + conn, _, err := w.(http.Hijacker).Hijack() + require.NoError(t, err) + _ = conn.Close() // Lose the response on an already pooled connection. + return + } + _, _ = io.WriteString(w, `{"status":200}`) + })) + defer server.Close() + auth := testAuthWebhook(t, server.URL, nodeconfig.Headers{"Idempotency-Key": {"configured"}}, io.Discard) + require.True(t, auth.authorize(httptest.NewRecorder(), httptest.NewRequest("POST", "/tickets", nil), nil)) + w := httptest.NewRecorder() + require.False(t, auth.authorize(w, httptest.NewRequest("POST", "/tickets", nil), nil)) + require.Equal(t, 502, w.Code) + require.EqualValues(t, 2, calls.Load(), "the received second POST must not be replayed") +} + +func TestAuthorizationForwardingAndQueuedRedemption(t *testing.T) { + f := test.NewRedemptionRPC(t) + p := serviceParams(t, f) + rpc, err := eth.NewRPC(p.RPCURL, nil) + require.NoError(t, err) + defer rpc.Close() + contracts, err := eth.NewContracts(rpc, *p.Controller) + require.NoError(t, err) + contracts.CheckRedemptionCost = true + chain := eth.PaymentChain{Contracts: contracts} + contracts.BeforeBroadcast = redemption.StandalonePolicy(chain, big.NewInt(1)) + newStore := func(role, target string) (*redemption.RedeemerDB, *sql.DB) { + db, err := redemption.OpenDatabase(t.Context(), filepath.Join(t.TempDir(), "tickets.sqlite"), true) + require.NoError(t, err) + s, err := redemption.NewStore(db) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, s.Close()) }) + require.NoError(t, s.Bind(t.Context(), redemption.Binding{ChainID: "1", Controller: *p.Controller, Role: role, Destination: target}, redemption.Limits{Pending: 1, PerRecipient: 1}, chain)) + return s, db + } + serverStore, serverDB := newStore("redeemer", "chain") + var decision, calls atomic.Int32 + decision.Store(409) + webhook := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + var payload struct { + Ticket redemption.TicketJSON `json:"ticket"` + } + require.NoError(t, json.NewDecoder(r.Body).Decode(&payload)) + require.Empty(t, payload.Ticket.Randomness) + // Signature verification needs only the commitment, not its preimage. + payload.Ticket.Randomness = "0" + unsigned, err := payload.Ticket.SignedTicket() + require.NoError(t, err) + require.True(t, (pm.DefaultSigVerifier{}).Verify(unsigned.PayerAddress, unsigned.Hash().Bytes(), unsigned.Sig)) + _, _ = io.WriteString(w, `{"status":`+big.NewInt(int64(decision.Load())).String()+`}`) + })) + defer webhook.Close() + auth := testAuthWebhook(t, webhook.URL, nil, io.Discard) + server := httptest.NewServer(redemption.HandlerWithAuthorization(serverStore, auth.authorize)) + defer server.Close() + clientStore, clientDB := newStore("orchestrator", server.URL) + policy, err := destination.New("redeemer", []string{"127.0.0.1"}) + require.NoError(t, err) + u, err := url.Parse(server.URL) + require.NoError(t, err) + remote, err := redemption.NewHTTPDestination(u, policy, redemption.Binding{ChainID: "1", Controller: *p.Controller}) + require.NoError(t, err) + defer remote.Close() + payer, err := crypto.GenerateKey() + require.NoError(t, err) + random := big.NewInt(1) + ticket := &pm.SignedTicket{Ticket: &pm.Ticket{PayerAddress: crypto.PubkeyToAddress(payer.PublicKey), Recipient: common.HexToAddress("0x5678"), FaceValue: big.NewInt(1e12), WinProb: new(big.Int).Sub(new(big.Int).Lsh(big.NewInt(1), 256), big.NewInt(1)), TicketNonce: 1, RecipientRandHash: crypto.Keccak256Hash(common.LeftPadBytes(random.Bytes(), 32)), CreationRound: 1, CreationRoundBlockHash: common.HexToHash("0x1"), ParamsExpirationBlock: big.NewInt(10)}, RecipientRand: random} + ticket.Sig, err = crypto.Sign(accounts.TextHash(ticket.Hash().Bytes()), payer) + require.NoError(t, err) + ticket.Sig[64] += 27 + f.Mu.Lock() + f.Tickets[ticket.Hash()] = ticket.RecipientRandHash + f.Finalized = false + f.Mu.Unlock() + require.NoError(t, clientStore.Admit(t.Context(), ticket)) + forwarder, err := redemption.NewWorker(clientStore, chain, remote) + require.NoError(t, err) + require.NotEmpty(t, forwarder.Process(t.Context())) + var delivery string + require.NoError(t, clientDB.QueryRow("SELECT delivery FROM ticket_state").Scan(&delivery)) + require.Equal(t, "retryable", delivery, "webhook 409 must not permanently reject delivery") + var count int + require.NoError(t, serverDB.QueryRow("SELECT count(*) FROM winning_tickets").Scan(&count)) + require.Zero(t, count, "denied requests do not write ticket state") + decision.Store(200) + _, err = clientDB.Exec("UPDATE ticket_state SET next_attempt=0") + require.NoError(t, err) + require.Empty(t, forwarder.Process(t.Context())) + require.NoError(t, remote.Submit(t.Context(), ticket), "approved duplicates succeed at queue capacity") + ticket.ParamsExpirationBlock = big.NewInt(11) + err = remote.Submit(t.Context(), ticket) + var rejection *redemption.Rejection + require.ErrorAs(t, err, &rejection, "actual scheduling conflicts remain permanent") + ticket.ParamsExpirationBlock = big.NewInt(10) + decision.Store(403) + err = remote.Submit(t.Context(), ticket) + require.Error(t, err, "duplicates are authorized again") + require.False(t, errors.As(err, &rejection)) + require.EqualValues(t, 5, calls.Load()) + webhook.Close() + require.Error(t, remote.Submit(t.Context(), ticket), "outage withholds duplicate acknowledgment") + key, err := eth.OpenKeystoreFile(p.KeystoreFile, p.KeystorePasswordFile) + require.NoError(t, err) + chainDest, err := redemption.NewChainDestination(t.Context(), serverStore, chain, key, big.NewInt(1), redemption.RedemptionOptions{Timeout: 30 * time.Millisecond, MaxReplacements: 1}) + require.NoError(t, err) + worker, err := redemption.NewWorker(serverStore, chain, chainDest) + require.NoError(t, err) + require.Empty(t, worker.Process(t.Context())) + require.Eventually(t, func() bool { return len(f.Transactions()) == 2 }, 3*time.Second, 5*time.Millisecond) + sent := f.Transactions() + require.Equal(t, sent[0].Nonce(), sent[1].Nonce()) + require.Equal(t, sent[0].Data(), sent[1].Data(), "transaction replacement proceeds during authorization outage") + require.EqualValues(t, 5, calls.Load(), "background redemption never authorizes again") +} diff --git a/redemption/http.go b/redemption/http.go index f61be5d..abbd81e 100644 --- a/redemption/http.go +++ b/redemption/http.go @@ -87,6 +87,14 @@ func decodeJSON(r io.Reader, value any) error { } func Handler(s *RedeemerDB) http.Handler { + return HandlerWithAuthorization(s, nil) +} + +// HandlerWithAuthorization checks each admitted POST /tickets after decoding +// its bounded body, before validation or storage. A decode failure passes a nil +// ticket. authorize must write a failure response when returning false. +// Accepted tickets need no further authorization during background redemption. +func HandlerWithAuthorization(s *RedeemerDB, authorize func(http.ResponseWriter, *http.Request, *TicketJSON) bool) http.Handler { mux := http.NewServeMux() slots := make(chan struct{}, 32) mux.HandleFunc("POST /tickets", func(w http.ResponseWriter, r *http.Request) { @@ -98,7 +106,15 @@ func Handler(s *RedeemerDB) http.Handler { return } var req TicketRequest - if err := decodeJSON(http.MaxBytesReader(w, r.Body, MaxTicketBody), &req); err != nil { + decodeErr := decodeJSON(http.MaxBytesReader(w, r.Body, MaxTicketBody), &req) + var ticket *TicketJSON + if decodeErr == nil { + ticket = &req.Ticket + } + if authorize != nil && !authorize(w, r, ticket) { + return + } + if decodeErr != nil { http.Error(w, "invalid ticket request", 400) return } @@ -168,7 +184,7 @@ func NewHTTPDestination(u *url.URL, policy destination.Policy, b Binding) (*HTTP if err != nil { return nil, err } - client := &http.Client{Transport: policy.Transport(10 * time.Second), Timeout: 10 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} + client := &http.Client{Transport: policy.Transport(15 * time.Second), Timeout: 20 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} return &HTTPDestination{endpoint, client, b}, nil } diff --git a/redemption/http_test.go b/redemption/http_test.go index 7d0361a..b1d8692 100644 --- a/redemption/http_test.go +++ b/redemption/http_test.go @@ -2,6 +2,7 @@ package redemption import ( "bytes" + "context" "crypto/ecdsa" "crypto/rand" "encoding/json" @@ -12,6 +13,8 @@ import ( "net/http/httptest" "net/url" "path/filepath" + "strings" + "sync" "sync/atomic" "testing" "time" @@ -54,7 +57,8 @@ func boundStore(t *testing.T, role, target string, limits Limits) (*RedeemerDB, func TestHTTPAdmissionIdentityCapacityAndScope(t *testing.T) { s, b := boundStore(t, "redeemer", "chain", Limits{Pending: 2, PerRecipient: 1}) - handler := Handler(s) + calls := 0 + handler := HandlerWithAuthorization(s, func(http.ResponseWriter, *http.Request, *TicketJSON) bool { calls++; return true }) submit := func(req TicketRequest) *httptest.ResponseRecorder { raw, err := json.Marshal(req) require.NoError(t, err) @@ -100,6 +104,79 @@ func TestHTTPAdmissionIdentityCapacityAndScope(t *testing.T) { changed = b changed.Destination = "http://elsewhere.example" require.ErrorContains(t, s.Bind(t.Context(), changed, DefaultLimits, nil), "drain") + require.Equal(t, 7, calls, "every submission, including duplicates and invalid tickets, is authorized") +} + +func TestHTTPAuthorizationConcurrency(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + entered := make(chan struct{}, 32) + var requests sync.WaitGroup + t.Cleanup(func() { cancel(); requests.Wait() }) + var calls atomic.Int32 + handler := HandlerWithAuthorization(nil, func(w http.ResponseWriter, r *http.Request, ticket *TicketJSON) bool { + calls.Add(1) + entered <- struct{}{} + <-ctx.Done() + http.Error(w, "denied", 403) + return false + }) + for range 32 { + requests.Go(func() { + handler.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("POST", "/tickets", strings.NewReader("invalid"))) + }) + } + for range 32 { + select { + case <-entered: + case <-time.After(5 * time.Second): + t.Fatal("authorization did not fill admission slots") + } + } + w := httptest.NewRecorder() + handler.ServeHTTP(w, httptest.NewRequest("POST", "/tickets", nil)) + require.Equal(t, 503, w.Code) + handler.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "/tickets", nil)) + require.EqualValues(t, 32, calls.Load(), "overloaded requests and other methods do not authorize") +} + +func TestHTTPDestinationAuthorizationAndAdmissionBudgets(t *testing.T) { + s, b := boundStore(t, "redeemer", "chain", DefaultLimits) + s.db.SetMaxOpenConns(1) + conn, err := s.db.Conn(t.Context()) + require.NoError(t, err) + defer conn.Close() + authorized := make(chan struct{}) + handler := HandlerWithAuthorization(s, func(http.ResponseWriter, *http.Request, *TicketJSON) bool { + time.Sleep(50 * time.Millisecond) + close(authorized) + return true + }) + server := httptest.NewServer(handler) + defer server.Close() + policy, err := destination.New("redeemer", []string{"127.0.0.1"}) + require.NoError(t, err) + u, err := url.Parse(server.URL) + require.NoError(t, err) + remote, err := NewHTTPDestination(u, policy, b) + require.NoError(t, err) + defer remote.Close() + require.Equal(t, 20*time.Second, remote.client.Timeout) + require.Equal(t, 15*time.Second, remote.client.Transport.(*http.Transport).ResponseHeaderTimeout) + done := make(chan struct{}) + go func() { + defer close(done) + select { + case <-authorized: + time.Sleep(50 * time.Millisecond) // Admission waits for storage after approval. + _ = conn.Close() + case <-t.Context().Done(): + } + }() + ticket, _ := winningTicket(t, 1) + started := time.Now() + require.NoError(t, remote.Submit(t.Context(), ticket)) + <-done + require.GreaterOrEqual(t, time.Since(started), 100*time.Millisecond) } func signAlternativeTicket(t *testing.T, ticket *pm.SignedTicket, key *ecdsa.PrivateKey) { diff --git a/signer/headers.go b/signer/headers.go index c039782..586ffc6 100644 --- a/signer/headers.go +++ b/signer/headers.go @@ -1,91 +1,6 @@ package signer -import ( - "bytes" - "encoding/csv" - "errors" - "io" - "maps" - "net/http" - "slices" - "strings" -) +import "github.com/livepeer/node/nodeconfig" -// Headers is a text scalar for configuration and secret files. It accepts -// comma-separated Header: value entries, with CSV quoting for values containing -// commas. Header names are case-insensitive and repeated names retain all values. -type Headers http.Header - -func validHeader(name, value string) bool { - if name == "" { - return false - } - for _, c := range name { - if !((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || strings.ContainsRune("!#$%&'*+-.^_`|~", c)) { - return false - } - } - for _, c := range value { - if (c < 32 && c != '\t') || c == 127 { - return false - } - } - return true -} - -func (h *Headers) UnmarshalText(text []byte) error { - parsed := make(http.Header) - input := strings.TrimSpace(string(text)) - if input != "" { - reader := csv.NewReader(strings.NewReader(input)) - reader.TrimLeadingSpace = true - reader.FieldsPerRecord = -1 - entries, err := reader.Read() - if err != nil { - return errors.New("invalid comma-separated webhook headers") - } - if _, err := reader.Read(); err != io.EOF { - return errors.New("webhook headers must be one comma-separated record") - } - for _, entry := range entries { - name, value, ok := strings.Cut(entry, ":") - name, value = strings.TrimSpace(name), strings.TrimSpace(value) - if !ok || !validHeader(name, value) { - return errors.New("expected webhook headers in Header: value form") - } - parsed.Add(name, value) - } - } - *h = Headers(parsed) - return nil -} - -// MarshalText sorts names for deterministic secret-file comparisons and auth -// policy fingerprints, and quotes entries containing commas or quotation marks. -func (h Headers) MarshalText() ([]byte, error) { - var entries []string - for _, name := range slices.Sorted(maps.Keys(h)) { - if !validHeader(name, "") { - return nil, errors.New("invalid webhook header name") - } - for _, value := range h[name] { - if !validHeader(name, value) { - return nil, errors.New("invalid webhook header name or value") - } - entries = append(entries, name+": "+value) - } - } - if len(entries) == 0 { - return nil, nil - } - var buffer bytes.Buffer - writer := csv.NewWriter(&buffer) - if err := writer.Write(entries); err != nil { - return nil, err - } - writer.Flush() - if err := writer.Error(); err != nil { - return nil, err - } - return bytes.TrimSuffix(buffer.Bytes(), []byte("\n")), nil -} +// Headers is the shared text scalar for webhook configuration and secret files. +type Headers = nodeconfig.Headers From 6999ff17a468d522f783953b3c1db80bb34bf191 Mon Sep 17 00:00:00 2001 From: Josh Allmann Date: Wed, 11 Nov 2026 19:46:44 -0800 Subject: [PATCH 2/4] Clarify webhook docs and make timing tests deterministic --- docs/configuration.md | 1 + docs/http-api.md | 45 +++++++------ docs/redeemer.md | 13 ++-- nodeconfig/headers.go | 8 +-- redeemer/auth.go | 4 +- redeemer/auth_test.go | 92 +++++++++++++++----------- redemption/http.go | 7 +- redemption/http_test.go | 139 ++++++++++++++++++++++------------------ signer/headers.go | 2 +- 9 files changed, 171 insertions(+), 140 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 925c7e6..3624988 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -92,6 +92,7 @@ openssl rand -hex 32 | tr -d '\n' > runner-bootstrap.secret Set `BootstrapSecretFile` to its absolute path and use the same credential for initial runner registration. Secret-manager mounts must preserve exact bytes; URL files require a valid URL without a trailing newline. +Redeemer webhook URL and headers files must be owner-only regular files. ## Ethereum keystores diff --git a/docs/http-api.md b/docs/http-api.md index 979f7ae..400a601 100644 --- a/docs/http-api.md +++ b/docs/http-api.md @@ -276,24 +276,33 @@ TLS and endpoint authentication in your deployment; see [redeemer setup](redeeme ### Redeemer authorization webhook -When [configured](redeemer.md#authorization-webhook), the redeemer authorizes -each admitted `POST /tickets`, including duplicates, without caching decisions. -It sends a JSON POST with `headers` (incoming headers as arrays of values) and -`ticket` (the submitted ticket fields above, including `sig` and -`recipient_rand_hash`, but omitting `recipient_rand`). The ticket is null if the -submission cannot be decoded or exceeds the size limit; otherwise it may still -be invalid, as validation follows authorization. Configured webhook credentials -are sent as HTTP headers, with `Content-Type: application/json`. - -The webhook must return HTTP 200 with a JSON integer `status`: 200 approves and -400–599 denies with that status. A decision of 409 returns 403, preserving 409 -for ticket conflicts. An optional `reason` string or null is ignored and never -exposed. Redirects, timeouts, non-200 responses, and invalid decisions return 502; -the webhook timeout is five seconds. - -Overloaded requests return 503 before authorization. Accepted tickets continue -through redemption, recovery, and transaction retries independently of later -webhook decisions or outages. +When [configured](redeemer.md#authorization-webhook), the redeemer checks each submitted ticket. +It sends the webhook a JSON POST, for example: + +```json +{ + "headers": {"Authorization": ["Bearer caller-credential"]}, + "ticket": {"sig": "0x...", "recipient_rand_hash": "0x..."} +} +``` + +The example shows selected [ticket fields](#redeemer-http). The webhook receives +all those fields except `recipient_rand`. + +The webhook must return HTTP 200 with a JSON decision, for example: + +```json +{"status": 200} +``` + +| Decision `status` | Result | Client response | +| --- | --- | --- | +| 200 | Approve the ticket. | 202 Accepted if validation and storage succeed. | +| 409 | Deny the ticket. | 403 Forbidden | +| Other 400–599 | Deny the ticket. | Same status | + +A redirect, timeout, non-200 response, or invalid decision returns 502. The +timeout is five seconds. ## Accounting events diff --git a/docs/redeemer.md b/docs/redeemer.md index 46429e8..ab69abb 100644 --- a/docs/redeemer.md +++ b/docs/redeemer.md @@ -19,20 +19,17 @@ redeemer; see [configuration](configuration.md) and [recovery](payment-recovery. ## Authorization webhook -To authorize ticket submissions through a policy service, configure: +To authorize ticket submissions: ```toml AuthWebhookFile = "/run/secrets/livepeer-redeemer-auth-url" # AuthWebhookHeadersFile = "/run/secrets/livepeer-redeemer-auth-headers" ``` -Both files must be owner-only regular files. The URL must be absolute HTTP or -HTTPS without a username, password, or fragment. The optional headers file uses -comma-separated `Header: value` entries to authenticate the redeemer to the -webhook. If the policy relies on caller identity headers, configure the proxy to -authenticate callers and replace untrusted identity headers. See the -[HTTP contract](http-api.md#redeemer-authorization-webhook) and -[metrics](metrics.md). +The URL must be absolute HTTP or HTTPS without a username, password, or fragment. +The optional headers file contains comma-separated `Header: value` entries. See +[credential files](configuration.md#credential-files) and the +[HTTP contract](http-api.md#redeemer-authorization-webhook). ## Connect an orchestrator diff --git a/nodeconfig/headers.go b/nodeconfig/headers.go index d87e7e6..b360465 100644 --- a/nodeconfig/headers.go +++ b/nodeconfig/headers.go @@ -11,9 +11,8 @@ import ( "strings" ) -// Headers is a text scalar for configuration and secret files. It accepts -// comma-separated Header: value entries, with CSV quoting for values containing -// commas. Header names are case-insensitive and repeated names retain all values. +// Headers parses comma-separated Header: value entries, with CSV quoting for +// commas. Names are case-insensitive; repeated names retain all values. type Headers http.Header func validHeader(name, value string) bool { @@ -60,8 +59,7 @@ func (h *Headers) UnmarshalText(text []byte) error { return nil } -// MarshalText sorts names for deterministic secret-file comparisons and auth -// policy fingerprints, and quotes entries containing commas or quotation marks. +// MarshalText returns CSV entries sorted by header name. func (h Headers) MarshalText() ([]byte, error) { var entries []string for _, name := range slices.Sorted(maps.Keys(h)) { diff --git a/redeemer/auth.go b/redeemer/auth.go index c8c7cfe..35513ef 100644 --- a/redeemer/auth.go +++ b/redeemer/auth.go @@ -24,7 +24,6 @@ type authWebhook struct { metrics *observability.Metrics } -// Configuration is validated by Params before constructing the immutable client. func newAuthWebhook(endpoint *url.URL, headers nodeconfig.Headers, logger *slog.Logger, metrics *observability.Metrics) *authWebhook { return &authWebhook{ endpoint: endpoint.String(), @@ -71,8 +70,7 @@ func (a *authWebhook) authorize(w http.ResponseWriter, r *http.Request, ticket * if err != nil { return fail("redeemer auth request construction failed", observability.ErrorKind(err)) } - // A received authorization request must not be replayed after a connection - // failure, even when configured service headers include an idempotency key. + // Prevent transport retries, including with a configured idempotency key. request.GetBody = nil request.Header = a.headers.Clone() if request.Header == nil { diff --git a/redeemer/auth_test.go b/redeemer/auth_test.go index ef3e058..8de83eb 100644 --- a/redeemer/auth_test.go +++ b/redeemer/auth_test.go @@ -17,6 +17,7 @@ import ( "strings" "sync/atomic" "testing" + "testing/synctest" "time" "github.com/ethereum/go-ethereum/accounts" @@ -32,6 +33,10 @@ import ( "github.com/stretchr/testify/require" ) +type authTransport func(*http.Request) (*http.Response, error) + +func (f authTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } + func testAuthWebhook(t *testing.T, endpoint string, headers nodeconfig.Headers, logs io.Writer) *authWebhook { t.Helper() u, err := url.Parse(endpoint) @@ -175,30 +180,32 @@ func TestAuthWebhookDecisions(t *testing.T) { func TestAuthWebhookUnavailableTimeoutAndCancellation(t *testing.T) { for _, kind := range []string{"unavailable", "timeout", "cancelled"} { t.Run(kind, func(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - _, _ = io.Copy(io.Discard, r.Body) - <-r.Context().Done() - })) - defer server.Close() - if kind == "unavailable" { - server.Close() - } - var logs bytes.Buffer - auth := testAuthWebhook(t, server.URL+"?private-detail", nil, &logs) - if kind == "timeout" { - auth.client.Timeout = 20 * time.Millisecond - } - ctx, cancel := context.WithCancel(t.Context()) - defer cancel() - if kind == "cancelled" { - cancel() - } - r := httptest.NewRequest("POST", "/tickets", nil).WithContext(ctx) - w := httptest.NewRecorder() - require.False(t, auth.authorize(w, r, nil)) - require.Equal(t, 502, w.Code) - require.NotContains(t, logs.String(), "private-detail") - require.EqualValues(t, 1, test.MetricSum(t, auth.metrics.Registry, "redeemer_auth_webhook_requests_total", map[string]string{"status": "0", "outcome": "error"})) + synctest.Test(t, func(t *testing.T) { + var logs bytes.Buffer + auth := testAuthWebhook(t, "http://webhook.example?private-detail", nil, &logs) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + auth.client.Transport = authTransport(func(r *http.Request) (*http.Response, error) { + if kind == "unavailable" { + return nil, errors.New("private-detail") + } + if kind == "cancelled" { + cancel() + } + <-r.Context().Done() + return nil, r.Context().Err() + }) + r := httptest.NewRequest("POST", "/tickets", nil).WithContext(ctx) + w := httptest.NewRecorder() + started := time.Now() + require.False(t, auth.authorize(w, r, nil)) + if kind == "timeout" { + require.Equal(t, 5*time.Second, time.Since(started)) + } + require.Equal(t, 502, w.Code) + require.NotContains(t, logs.String(), "private-detail") + require.EqualValues(t, 1, test.MetricSum(t, auth.metrics.Registry, "redeemer_auth_webhook_requests_total", map[string]string{"status": "0", "outcome": "error"})) + }) }) } } @@ -227,7 +234,11 @@ func TestAuthWebhookDoesNotReplayReceivedPOST(t *testing.T) { func TestAuthorizationForwardingAndQueuedRedemption(t *testing.T) { f := test.NewRedemptionRPC(t) p := serviceParams(t, f) - rpc, err := eth.NewRPC(p.RPCURL, nil) + rpc, err := eth.NewRPC(p.RPCURL, authTransport(func(r *http.Request) (*http.Response, error) { + w := httptest.NewRecorder() + f.Serve(w, r) + return w.Result(), nil + })) require.NoError(t, err) defer rpc.Close() contracts, err := eth.NewContracts(rpc, *p.Controller) @@ -311,16 +322,23 @@ func TestAuthorizationForwardingAndQueuedRedemption(t *testing.T) { require.EqualValues(t, 5, calls.Load()) webhook.Close() require.Error(t, remote.Submit(t.Context(), ticket), "outage withholds duplicate acknowledgment") - key, err := eth.OpenKeystoreFile(p.KeystoreFile, p.KeystorePasswordFile) - require.NoError(t, err) - chainDest, err := redemption.NewChainDestination(t.Context(), serverStore, chain, key, big.NewInt(1), redemption.RedemptionOptions{Timeout: 30 * time.Millisecond, MaxReplacements: 1}) - require.NoError(t, err) - worker, err := redemption.NewWorker(serverStore, chain, chainDest) - require.NoError(t, err) - require.Empty(t, worker.Process(t.Context())) - require.Eventually(t, func() bool { return len(f.Transactions()) == 2 }, 3*time.Second, 5*time.Millisecond) - sent := f.Transactions() - require.Equal(t, sent[0].Nonce(), sent[1].Nonce()) - require.Equal(t, sent[0].Data(), sent[1].Data(), "transaction replacement proceeds during authorization outage") - require.EqualValues(t, 5, calls.Load(), "background redemption never authorizes again") + synctest.Test(t, func(t *testing.T) { + defer serverStore.Close() + key, err := eth.OpenKeystoreFile(p.KeystoreFile, p.KeystorePasswordFile) + require.NoError(t, err) + chainDest, err := redemption.NewChainDestination(t.Context(), serverStore, chain, key, big.NewInt(1), redemption.RedemptionOptions{Timeout: 5 * time.Second, MaxReplacements: 1}) + require.NoError(t, err) + worker, err := redemption.NewWorker(serverStore, chain, chainDest) + require.NoError(t, err) + require.Empty(t, worker.Process(t.Context())) + synctest.Wait() + require.Len(t, f.Transactions(), 1) + time.Sleep(5 * time.Second) + synctest.Wait() + require.Len(t, f.Transactions(), 2) + sent := f.Transactions() + require.Equal(t, sent[0].Nonce(), sent[1].Nonce()) + require.Equal(t, sent[0].Data(), sent[1].Data(), "transaction replacement proceeds during authorization outage") + require.EqualValues(t, 5, calls.Load(), "background redemption never authorizes again") + }) } diff --git a/redemption/http.go b/redemption/http.go index abbd81e..ba083c9 100644 --- a/redemption/http.go +++ b/redemption/http.go @@ -1,7 +1,6 @@ // QueueTicket admission and ticket conversion adapt Nico Vergauwen's and // Yondon Fu's go-livepeer/server/redeemer.go at afaaa36e743d39e6f460e7993d516d52b3a81341. // HTTP transport, deployment binding and validation are implemented for Node. -// MIT notice: LICENSE. No max-float RPCs are included. package redemption import ( @@ -90,10 +89,8 @@ func Handler(s *RedeemerDB) http.Handler { return HandlerWithAuthorization(s, nil) } -// HandlerWithAuthorization checks each admitted POST /tickets after decoding -// its bounded body, before validation or storage. A decode failure passes a nil -// ticket. authorize must write a failure response when returning false. -// Accepted tickets need no further authorization during background redemption. +// HandlerWithAuthorization calls authorize before validating or storing a ticket. +// The ticket is nil if decoding fails. authorize must write any failure response. func HandlerWithAuthorization(s *RedeemerDB, authorize func(http.ResponseWriter, *http.Request, *TicketJSON) bool) http.Handler { mux := http.NewServeMux() slots := make(chan struct{}, 32) diff --git a/redemption/http_test.go b/redemption/http_test.go index b1d8692..7add8d7 100644 --- a/redemption/http_test.go +++ b/redemption/http_test.go @@ -1,6 +1,7 @@ package redemption import ( + "bufio" "bytes" "context" "crypto/ecdsa" @@ -9,6 +10,7 @@ import ( "errors" "io" "math/big" + "net" "net/http" "net/http/httptest" "net/url" @@ -17,6 +19,7 @@ import ( "sync" "sync/atomic" "testing" + "testing/synctest" "time" "github.com/ethereum/go-ethereum/accounts" @@ -108,75 +111,85 @@ func TestHTTPAdmissionIdentityCapacityAndScope(t *testing.T) { } func TestHTTPAuthorizationConcurrency(t *testing.T) { - ctx, cancel := context.WithCancel(t.Context()) - entered := make(chan struct{}, 32) - var requests sync.WaitGroup - t.Cleanup(func() { cancel(); requests.Wait() }) - var calls atomic.Int32 - handler := HandlerWithAuthorization(nil, func(w http.ResponseWriter, r *http.Request, ticket *TicketJSON) bool { - calls.Add(1) - entered <- struct{}{} - <-ctx.Done() - http.Error(w, "denied", 403) - return false - }) - for range 32 { - requests.Go(func() { - handler.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("POST", "/tickets", strings.NewReader("invalid"))) + synctest.Test(t, func(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + var requests sync.WaitGroup + defer func() { cancel(); requests.Wait() }() + var calls atomic.Int32 + handler := HandlerWithAuthorization(nil, func(w http.ResponseWriter, r *http.Request, ticket *TicketJSON) bool { + calls.Add(1) + <-ctx.Done() + http.Error(w, "denied", 403) + return false }) - } - for range 32 { - select { - case <-entered: - case <-time.After(5 * time.Second): - t.Fatal("authorization did not fill admission slots") + for range 32 { + requests.Go(func() { + handler.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("POST", "/tickets", strings.NewReader("invalid"))) + }) } - } - w := httptest.NewRecorder() - handler.ServeHTTP(w, httptest.NewRequest("POST", "/tickets", nil)) - require.Equal(t, 503, w.Code) - handler.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "/tickets", nil)) - require.EqualValues(t, 32, calls.Load(), "overloaded requests and other methods do not authorize") + synctest.Wait() + require.EqualValues(t, 32, calls.Load()) + w := httptest.NewRecorder() + handler.ServeHTTP(w, httptest.NewRequest("POST", "/tickets", nil)) + require.Equal(t, 503, w.Code) + handler.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "/tickets", nil)) + require.EqualValues(t, 32, calls.Load(), "overloaded requests and other methods do not authorize") + }) } func TestHTTPDestinationAuthorizationAndAdmissionBudgets(t *testing.T) { - s, b := boundStore(t, "redeemer", "chain", DefaultLimits) - s.db.SetMaxOpenConns(1) - conn, err := s.db.Conn(t.Context()) - require.NoError(t, err) - defer conn.Close() - authorized := make(chan struct{}) - handler := HandlerWithAuthorization(s, func(http.ResponseWriter, *http.Request, *TicketJSON) bool { - time.Sleep(50 * time.Millisecond) - close(authorized) - return true + synctest.Test(t, func(t *testing.T) { + s, b := boundStore(t, "redeemer", "chain", DefaultLimits) + s.db.SetMaxOpenConns(1) + conn, err := s.db.Conn(t.Context()) + require.NoError(t, err) + defer conn.Close() + authorized := make(chan struct{}) + handler := HandlerWithAuthorization(s, func(http.ResponseWriter, *http.Request, *TicketJSON) bool { + time.Sleep(4 * time.Second) + close(authorized) + return true + }) + serverConn, clientConn := net.Pipe() + defer serverConn.Close() + defer clientConn.Close() + go func() { + defer serverConn.Close() + request, err := http.ReadRequest(bufio.NewReader(serverConn)) + require.NoError(t, err) + defer request.Body.Close() + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + require.NoError(t, response.Result().Write(serverConn)) + }() + policy, err := destination.New("redeemer", []string{"127.0.0.1"}) + require.NoError(t, err) + u, err := url.Parse("http://127.0.0.1") + require.NoError(t, err) + remote, err := NewHTTPDestination(u, policy, b) + require.NoError(t, err) + defer remote.Close() + require.Equal(t, 20*time.Second, remote.client.Timeout) + transport := remote.client.Transport.(*http.Transport) + require.Equal(t, 15*time.Second, transport.ResponseHeaderTimeout) + transport.Proxy = nil + transport.DialContext = func(context.Context, string, string) (net.Conn, error) { return clientConn, nil } + done := make(chan struct{}) + go func() { + defer close(done) + select { + case <-authorized: + time.Sleep(4 * time.Second) // Hold storage after approval. + _ = conn.Close() + case <-t.Context().Done(): + } + }() + ticket, _ := winningTicket(t, 1) + started := time.Now() + require.NoError(t, remote.Submit(t.Context(), ticket)) + <-done + require.Equal(t, 8*time.Second, time.Since(started)) }) - server := httptest.NewServer(handler) - defer server.Close() - policy, err := destination.New("redeemer", []string{"127.0.0.1"}) - require.NoError(t, err) - u, err := url.Parse(server.URL) - require.NoError(t, err) - remote, err := NewHTTPDestination(u, policy, b) - require.NoError(t, err) - defer remote.Close() - require.Equal(t, 20*time.Second, remote.client.Timeout) - require.Equal(t, 15*time.Second, remote.client.Transport.(*http.Transport).ResponseHeaderTimeout) - done := make(chan struct{}) - go func() { - defer close(done) - select { - case <-authorized: - time.Sleep(50 * time.Millisecond) // Admission waits for storage after approval. - _ = conn.Close() - case <-t.Context().Done(): - } - }() - ticket, _ := winningTicket(t, 1) - started := time.Now() - require.NoError(t, remote.Submit(t.Context(), ticket)) - <-done - require.GreaterOrEqual(t, time.Since(started), 100*time.Millisecond) } func signAlternativeTicket(t *testing.T, ticket *pm.SignedTicket, key *ecdsa.PrivateKey) { diff --git a/signer/headers.go b/signer/headers.go index 586ffc6..2d72245 100644 --- a/signer/headers.go +++ b/signer/headers.go @@ -2,5 +2,5 @@ package signer import "github.com/livepeer/node/nodeconfig" -// Headers is the shared text scalar for webhook configuration and secret files. +// Headers parses webhook headers from configuration and secret files. type Headers = nodeconfig.Headers From 61e40987b2a23623dda90bf0b32da161f83e066c Mon Sep 17 00:00:00 2001 From: Josh Allmann Date: Wed, 11 Nov 2026 20:01:32 -0800 Subject: [PATCH 3/4] Consolidate redeemer webhook tests --- redeemer/app_test.go | 290 +++++++++++++++++++++++++++++---- redeemer/auth_test.go | 344 ---------------------------------------- redemption/http_test.go | 39 ++--- 3 files changed, 274 insertions(+), 399 deletions(-) delete mode 100644 redeemer/auth_test.go diff --git a/redeemer/app_test.go b/redeemer/app_test.go index 0e4849b..6720912 100644 --- a/redeemer/app_test.go +++ b/redeemer/app_test.go @@ -4,8 +4,10 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "io" + "log/slog" "math/big" "net" "net/http" @@ -13,15 +15,19 @@ import ( "net/url" "os" "path/filepath" + "strconv" "strings" "sync/atomic" "testing" + "testing/iotest" + "testing/synctest" "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" "github.com/j0sh/boa/pkg/boa" "github.com/livepeer/node/eth" + "github.com/livepeer/node/internal/observability" "github.com/livepeer/node/internal/test" "github.com/livepeer/node/nodeconfig" "github.com/livepeer/node/pm" @@ -207,12 +213,7 @@ func TestPrintConfigRedactsCredentials(t *testing.T) { func loadAuthConfig(t *testing.T, p Params, args ...string) (Params, error) { t.Helper() cmd := nodeconfig.Command("redeemer", nodeconfig.Mainnet, boa.Cmd[Params]{Params: &p, RejectUnknown: true, - RunFuncCtxE: func(ctx *boa.HookContext, p *Params, _ *cobra.Command, _ []string) error { - dump, err := ctx.DumpBytes(".toml", nil) - require.NoError(t, err) - require.NotContains(t, string(dump), "private-detail") - return p.Validate() - }, + RunFuncE: func(p *Params, _ *cobra.Command, _ []string) error { return p.Validate() }, }, &p.ConfigFile) err := cmd.RunArgsE(args) return p, err @@ -221,7 +222,7 @@ func loadAuthConfig(t *testing.T, p Params, args ...string) (Params, error) { func TestAuthWebhookConfiguration(t *testing.T) { base := serviceParams(t, test.NewRedemptionRPC(t)) const endpoint = "https://webhook.example/auth?token=private-detail" - const headers = `Authorization: Bearer private-detail, X-Tag: one, x-tag: two, "X-List: a,b"` + const headers = "Authorization: Bearer private-detail" for _, tc := range []struct { name, field, content string withURL bool @@ -248,25 +249,17 @@ func TestAuthWebhookConfiguration(t *testing.T) { } }) } - for _, source := range []string{"environment", "TOML", "JSON", "file flags"} { + for _, source := range []string{"environment", "files"} { t.Run(source, func(t *testing.T) { - dir := t.TempDir() - uFile, hFile := filepath.Join(dir, "url"), filepath.Join(dir, "headers") - require.NoError(t, os.WriteFile(uFile, []byte(endpoint), 0600)) - require.NoError(t, os.WriteFile(hFile, []byte(headers), 0600)) args := []string{"--config="} - switch source { - case "environment": + if source == "environment" { t.Setenv("LIVEPEER_REDEEMER_AUTH_WEBHOOK", endpoint) t.Setenv("LIVEPEER_REDEEMER_AUTH_WEBHOOK_HEADERS", headers) - case "file flags": - args = append(args, "--auth-webhook-file", uFile, "--auth-webhook-headers-file", hFile) - default: - ext := ".toml" - if source == "JSON" { - ext = ".json" - } - path := filepath.Join(dir, "config"+ext) + } else { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "url"), []byte(endpoint), 0600)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "headers"), []byte(headers), 0600)) + path := filepath.Join(dir, "config.toml") test.WriteConfig(t, path, "AuthWebhookFile = 'url'\nAuthWebhookHeadersFile = 'headers'\n") args = []string{"--config", path} } @@ -274,8 +267,6 @@ func TestAuthWebhookConfiguration(t *testing.T) { require.NoError(t, err) require.Equal(t, endpoint, p.AuthWebhook.String()) require.Equal(t, "Bearer private-detail", http.Header(p.AuthWebhookHeaders).Get("Authorization")) - require.Equal(t, []string{"one", "two"}, http.Header(p.AuthWebhookHeaders).Values("X-Tag")) - require.Equal(t, "a,b", http.Header(p.AuthWebhookHeaders).Get("X-List")) }) } for _, flag := range []string{"--auth-webhook", "--auth-webhook-headers"} { @@ -283,19 +274,13 @@ func TestAuthWebhookConfiguration(t *testing.T) { require.ErrorContains(t, err, "unknown flag") require.NotContains(t, err.Error(), "private-detail") } - for _, field := range []string{"AUTH_WEBHOOK", "AUTH_WEBHOOK_HEADERS"} { + for field, value := range map[string]string{"AUTH_WEBHOOK": endpoint, "AUTH_WEBHOOK_HEADERS": headers} { t.Run(field+" file protection", func(t *testing.T) { file := filepath.Join(t.TempDir(), "secret") - value := endpoint - if field == "AUTH_WEBHOOK_HEADERS" { - value = headers - } require.NoError(t, os.WriteFile(file, []byte(value), 0600)) t.Setenv("LIVEPEER_REDEEMER_AUTH_WEBHOOK", endpoint) t.Setenv("LIVEPEER_REDEEMER_"+field+"_FILE", file) - if field == "AUTH_WEBHOOK_HEADERS" { - t.Setenv("LIVEPEER_REDEEMER_"+field, headers) - } + t.Setenv("LIVEPEER_REDEEMER_"+field, value) _, err := loadAuthConfig(t, base, "--config=") require.ErrorContains(t, err, "cannot both be set") require.NotContains(t, err.Error(), "private-detail") @@ -328,3 +313,244 @@ func TestAuthWebhookConfigurationValidation(t *testing.T) { require.NotContains(t, err.Error(), "private-detail") } + +type authTransport func(*http.Request) (*http.Response, error) + +func (f authTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } + +func testAuthWebhook(t *testing.T, endpoint string, headers nodeconfig.Headers, logs io.Writer) *authWebhook { + t.Helper() + u, err := url.Parse(endpoint) + require.NoError(t, err) + auth := newAuthWebhook(u, headers, slog.New(slog.NewTextHandler(logs, nil)), observability.New("redeemer")) + t.Cleanup(auth.client.CloseIdleConnections) + return auth +} + +func TestAuthWebhookHeadersAndRequests(t *testing.T) { + type webhookRequest struct { + Headers http.Header `json:"headers"` + Ticket json.RawMessage `json:"ticket"` + } + received := make(chan webhookRequest, 1) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + require.Equal(t, "POST", r.Method) + require.Equal(t, "token=private-url", r.URL.RawQuery) + require.Equal(t, "application/json", r.Header.Get("Content-Type")) + require.Equal(t, "Bearer service-secret", r.Header.Get("Authorization")) + require.Empty(t, r.Header.Values("X-Caller")) + var payload webhookRequest + decoder := json.NewDecoder(r.Body) + decoder.DisallowUnknownFields() + require.NoError(t, decoder.Decode(&payload)) + received <- payload + _, _ = io.WriteString(w, `{"status":200,"expiry":9999999999,"auth_id":"ignored","maxPrice":{}}`) + })) + defer server.Close() + headers := nodeconfig.Headers{"Authorization": {"Bearer service-secret"}, "Content-Type": {"text/plain"}} + auth := testAuthWebhook(t, server.URL+"?token=private-url", headers, io.Discard) + headers["Authorization"][0] = "mutated" + ticket := redemption.TicketJSON{ + Payer: common.HexToAddress("0x1234"), Recipient: common.HexToAddress("0x5678"), + FaceValue: "1000", WinProb: "42", Nonce: 7, + RandHash: common.HexToHash("0x1234"), CreationRound: "1", RoundHash: common.HexToHash("0x5678"), + Expiration: "10", Signature: []byte{1, 2, 3}, Randomness: "9", + } + for _, submitted := range []*redemption.TicketJSON{nil, &ticket} { + raw, err := json.Marshal(submitted) + require.NoError(t, err) + var expected map[string]any + require.NoError(t, json.Unmarshal(raw, &expected)) + delete(expected, "recipient_rand") + want, err := json.Marshal(expected) + require.NoError(t, err) + r := httptest.NewRequest("POST", "/tickets", nil) + r.Header = http.Header{"Authorization": {"Bearer caller-secret"}, "X-Caller": {"one", "two"}} + require.True(t, auth.authorize(httptest.NewRecorder(), r, submitted)) + payload := <-received + require.Equal(t, r.Header, payload.Headers) + require.JSONEq(t, string(want), string(payload.Ticket), "only recipient_rand is omitted") + } + require.Equal(t, "9", ticket.Randomness, "redemption keeps the original randomness") +} + +func TestAuthWebhookDecisions(t *testing.T) { + for _, tc := range []struct { + name, body string + upstream, want int + outcome string + }{ + {"approval", `{"status":200}`, 200, 200, "success"}, + {"nullable reason", `{"status":200,"reason":null}`, 200, 200, "success"}, + {"denial", `{"status":403,"reason":"private-detail"}`, 200, 403, "denied"}, + {"conflict denial", `{"status":409}`, 200, 403, "denied"}, + {"service denial", `{"status":503}`, 200, 503, "denied"}, + {"last denial", `{"status":599}`, 200, 599, "denied"}, + {"non-200", "private-detail", 500, 502, "error"}, + {"malformed", "private-detail", 200, 502, "error"}, + {"trailing JSON", `{"status":200}{}`, 200, 502, "error"}, + {"missing status", `{}`, 200, 502, "error"}, + {"null status", `{"status":null}`, 200, 502, "error"}, + {"string status", `{"status":"200"}`, 200, 502, "error"}, + {"fractional status", `{"status":200.5}`, 200, 502, "error"}, + {"invalid status", `{"status":201}`, 200, 502, "error"}, + {"invalid denial", `{"status":600}`, 200, 502, "error"}, + {"invalid reason", `{"status":200,"reason":123}`, 200, 502, "error"}, + {"oversized", strings.Repeat(" ", (1<<20)+1), 200, 502, "error"}, + {"read failure", `{"status":200}`, 200, 502, "error"}, + {"unavailable", "", 0, 502, "error"}, + {"timeout", "", 0, 502, "error"}, + {"cancelled", "", 0, 502, "error"}, + } { + t.Run(tc.name, func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + var logs bytes.Buffer + calls := 0 + auth := testAuthWebhook(t, "http://webhook.example?private-detail", nodeconfig.Headers{"Authorization": {"private-detail"}}, &logs) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + auth.client.Transport = authTransport(func(r *http.Request) (*http.Response, error) { + calls++ + switch tc.name { + case "unavailable": + return nil, errors.New("private-detail") + case "cancelled": + cancel() + fallthrough + case "timeout": + <-r.Context().Done() + return nil, r.Context().Err() + } + var body io.Reader = strings.NewReader(tc.body) + if tc.name == "read failure" { + body = io.MultiReader(body, iotest.ErrReader(errors.New("private-detail"))) + } + return &http.Response{StatusCode: tc.upstream, Body: io.NopCloser(body)}, nil + }) + w := httptest.NewRecorder() + started := time.Now() + require.Equal(t, tc.want == 200, auth.authorize(w, httptest.NewRequest("POST", "/tickets", nil).WithContext(ctx), nil)) + if tc.name == "timeout" { + require.Equal(t, 5*time.Second, time.Since(started)) + } + require.Equal(t, tc.want, w.Code) + require.Equal(t, 1, calls, "authorization never retries") + require.NotContains(t, logs.String()+w.Body.String(), "private-detail") + require.EqualValues(t, 1, test.MetricSum(t, auth.metrics.Registry, "redeemer_auth_webhook_requests_total", map[string]string{"status": strconv.Itoa(tc.upstream), "outcome": tc.outcome})) + require.EqualValues(t, 1, test.MetricSum(t, auth.metrics.Registry, "redeemer_auth_webhook_duration_seconds", nil)) + }) + }) + } +} + +func TestAuthWebhookRedirectAndReplay(t *testing.T) { + var calls atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + switch calls.Add(1) { + case 1: + http.Redirect(w, r, "/redirected", http.StatusFound) + return + case 3: + conn, _, err := w.(http.Hijacker).Hijack() + require.NoError(t, err) + _ = conn.Close() // Lose the response on an already pooled connection. + return + } + _, _ = io.WriteString(w, `{"status":200}`) + })) + defer server.Close() + auth := testAuthWebhook(t, server.URL, nodeconfig.Headers{"Idempotency-Key": {"configured"}}, io.Discard) + for i, allowed := range []bool{false, true, false} { + w := httptest.NewRecorder() + require.Equal(t, allowed, auth.authorize(w, httptest.NewRequest("POST", "/tickets", nil), nil)) + if !allowed { + require.Equal(t, 502, w.Code) + } + require.EqualValues(t, i+1, calls.Load(), "no redirects or replays") + } +} + +func TestAuthorizationAndQueuedRedemption(t *testing.T) { + f := test.NewRedemptionRPC(t) + p := serviceParams(t, f) + synctest.Test(t, func(t *testing.T) { + rpc, err := eth.NewRPC(p.RPCURL, authTransport(func(r *http.Request) (*http.Response, error) { + w := httptest.NewRecorder() + f.Serve(w, r) + return w.Result(), nil + })) + require.NoError(t, err) + defer rpc.Close() + contracts, err := eth.NewContracts(rpc, *p.Controller) + require.NoError(t, err) + chain := eth.PaymentChain{Contracts: contracts} + store, err := redemption.OpenRedeemerDB(p.RedeemerDB) + require.NoError(t, err) + defer store.Close() + require.NoError(t, store.Bind(t.Context(), redemption.Binding{ChainID: "1", Controller: *p.Controller, Role: "redeemer", Destination: "chain"}, redemption.Limits{Pending: 1, PerRecipient: 1}, chain)) + key, err := eth.OpenKeystoreFile(p.KeystoreFile, p.KeystorePasswordFile) + require.NoError(t, err) + input := redemption.TicketJSON{ + Payer: key.Address(), Recipient: common.HexToAddress("0x5678"), FaceValue: "1000000000000", + WinProb: new(big.Int).Sub(new(big.Int).Lsh(big.NewInt(1), 256), big.NewInt(1)).String(), Nonce: 1, + RandHash: crypto.Keccak256Hash(common.LeftPadBytes([]byte{1}, 32)), CreationRound: "1", + RoundHash: common.HexToHash("0x1"), Expiration: "10", Randomness: "1", + } + ticket, err := input.SignedTicket() + require.NoError(t, err) + input.Signature, err = key.SignMessage(ticket.Hash().Bytes()) + require.NoError(t, err) + f.Tickets[ticket.Hash()], f.Finalized = ticket.RecipientRandHash, false + decision, calls := 409, 0 + unavailable := false + auth := testAuthWebhook(t, "http://webhook.example", nil, io.Discard) + auth.client.Transport = authTransport(func(r *http.Request) (*http.Response, error) { + calls++ + if unavailable { + return nil, errors.New("unavailable") + } + var payload struct{ Ticket redemption.TicketJSON } + require.NoError(t, json.NewDecoder(r.Body).Decode(&payload)) + require.Empty(t, payload.Ticket.Randomness) + payload.Ticket.Randomness = "0" // Verify the signature without the preimage. + signed, err := payload.Ticket.SignedTicket() + require.NoError(t, err) + require.True(t, (pm.DefaultSigVerifier{}).Verify(signed.PayerAddress, signed.Hash().Bytes(), signed.Sig)) + return &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader(`{"status":` + strconv.Itoa(decision) + `}`))}, nil + }) + body, err := json.Marshal(redemption.TicketRequest{ChainID: "1", Controller: *p.Controller, Ticket: input}) + require.NoError(t, err) + handler := redemption.HandlerWithAuthorization(store, auth.authorize) + submit := func() int { + w := httptest.NewRecorder() + handler.ServeHTTP(w, httptest.NewRequest("POST", "/tickets", bytes.NewReader(body))) + return w.Code + } + require.Equal(t, 403, submit()) + count, err := store.WinningTicketCount(input.Payer) + require.NoError(t, err) + require.Zero(t, count, "denied requests do not store tickets") + decision = 200 + require.Equal(t, 202, submit()) + require.Equal(t, 202, submit(), "duplicates succeed at capacity") + decision = 403 + require.Equal(t, 403, submit(), "duplicates are authorized again") + unavailable = true + require.Equal(t, 502, submit()) + chainDest, err := redemption.NewChainDestination(t.Context(), store, chain, key, big.NewInt(1), redemption.RedemptionOptions{Timeout: 5 * time.Second, MaxReplacements: 1}) + require.NoError(t, err) + worker, err := redemption.NewWorker(store, chain, chainDest) + require.NoError(t, err) + require.Empty(t, worker.Process(t.Context())) + synctest.Wait() + require.Len(t, f.Transactions(), 1) + time.Sleep(5 * time.Second) + synctest.Wait() + sent := f.Transactions() + require.Len(t, sent, 2) + require.Equal(t, sent[0].Nonce(), sent[1].Nonce()) + require.Equal(t, sent[0].Data(), sent[1].Data()) + require.Equal(t, 5, calls, "background redemption never authorizes again") + }) +} diff --git a/redeemer/auth_test.go b/redeemer/auth_test.go deleted file mode 100644 index 8de83eb..0000000 --- a/redeemer/auth_test.go +++ /dev/null @@ -1,344 +0,0 @@ -package redeemer - -import ( - "bytes" - "context" - "database/sql" - "encoding/json" - "errors" - "io" - "log/slog" - "math/big" - "net/http" - "net/http/httptest" - "net/url" - "path/filepath" - "strconv" - "strings" - "sync/atomic" - "testing" - "testing/synctest" - "time" - - "github.com/ethereum/go-ethereum/accounts" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/crypto" - "github.com/livepeer/node/destination" - "github.com/livepeer/node/eth" - "github.com/livepeer/node/internal/observability" - "github.com/livepeer/node/internal/test" - "github.com/livepeer/node/nodeconfig" - "github.com/livepeer/node/pm" - "github.com/livepeer/node/redemption" - "github.com/stretchr/testify/require" -) - -type authTransport func(*http.Request) (*http.Response, error) - -func (f authTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } - -func testAuthWebhook(t *testing.T, endpoint string, headers nodeconfig.Headers, logs io.Writer) *authWebhook { - t.Helper() - u, err := url.Parse(endpoint) - require.NoError(t, err) - auth := newAuthWebhook(u, headers, slog.New(slog.NewTextHandler(logs, nil)), observability.New("redeemer")) - t.Cleanup(auth.client.CloseIdleConnections) - return auth -} - -func TestAuthWebhookHeadersAndRequests(t *testing.T) { - type webhookRequest struct { - Headers http.Header `json:"headers"` - Ticket json.RawMessage `json:"ticket"` - } - received := make(chan webhookRequest, 1) - var calls atomic.Int32 - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - calls.Add(1) - require.Equal(t, "POST", r.Method) - require.Equal(t, "token=private-url", r.URL.RawQuery) - require.Equal(t, "application/json", r.Header.Get("Content-Type")) - require.Equal(t, "Bearer service-secret", r.Header.Get("Authorization")) - require.Empty(t, r.Header.Values("X-Caller")) - var payload webhookRequest - decoder := json.NewDecoder(r.Body) - decoder.DisallowUnknownFields() - require.NoError(t, decoder.Decode(&payload)) - received <- payload - _, _ = io.WriteString(w, `{"status":200,"expiry":9999999999,"auth_id":"ignored","maxPrice":{}}`) - })) - defer server.Close() - headers := nodeconfig.Headers{"Authorization": {"Bearer service-secret"}, "Content-Type": {"text/plain"}} - auth := testAuthWebhook(t, server.URL+"?token=private-url", headers, io.Discard) - headers["Authorization"][0] = "mutated" - require.Equal(t, 5*time.Second, auth.client.Timeout) - store, err := redemption.OpenRedeemerDB(filepath.Join(t.TempDir(), "tickets.sqlite")) - require.NoError(t, err) - defer store.Close() - binding := redemption.Binding{ChainID: "1", Controller: common.HexToAddress("0x1000"), Role: "redeemer", Destination: "chain"} - require.NoError(t, store.Bind(t.Context(), binding, redemption.DefaultLimits, nil)) - ticket := redemption.TicketJSON{ - Payer: common.HexToAddress("0x1234"), Recipient: common.HexToAddress("0x5678"), - FaceValue: "1000", WinProb: "42", Nonce: 7, - RandHash: common.HexToHash("0x1234"), CreationRound: "1", RoundHash: common.HexToHash("0x5678"), - Expiration: "10", Signature: []byte{1, 2, 3}, Randomness: "9", - } - body, err := json.Marshal(redemption.TicketRequest{ChainID: binding.ChainID, Controller: binding.Controller, Ticket: ticket}) - require.NoError(t, err) - handler := redemption.HandlerWithAuthorization(store, auth.authorize) - for _, tc := range []struct { - body string - ticket *redemption.TicketJSON - }{ - {"bad ticket", nil}, - {strings.Repeat(" ", redemption.MaxTicketBody) + string(body), nil}, - {string(body), &ticket}, - } { - r := httptest.NewRequest("POST", "/tickets", strings.NewReader(tc.body)) - r.Header = http.Header{"Authorization": {"Bearer caller-secret"}, "X-Caller": {"one", "two"}} - w := httptest.NewRecorder() - handler.ServeHTTP(w, r) - require.Equal(t, 400, w.Code, "approval does not bypass ticket decoding or validation") - payload := <-received - require.Equal(t, r.Header, payload.Headers) - if tc.ticket == nil { - require.JSONEq(t, "null", string(payload.Ticket)) - continue - } - var fields map[string]json.RawMessage - require.NoError(t, json.Unmarshal(payload.Ticket, &fields)) - require.NotContains(t, fields, "recipient_rand", "the secret field must be absent, not empty or null") - require.Contains(t, fields, "sig") - require.Contains(t, fields, "recipient_rand_hash") - var actual redemption.TicketJSON - require.NoError(t, json.Unmarshal(payload.Ticket, &actual)) - expected := *tc.ticket - expected.Randomness = "" - require.Equal(t, expected, actual, "all other ticket fields reach the webhook unchanged") - } - for _, target := range []string{"/healthz", "/readyz", "/metrics", "/other"} { - handler.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", target, nil)) - } - handler.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "/tickets", nil)) - require.EqualValues(t, 3, calls.Load(), "approvals are never cached") -} - -func TestAuthWebhookDecisions(t *testing.T) { - for _, tc := range []struct { - name, body string - upstream, want int - outcome string - }{ - {"approval", `{"status":200}`, 200, 200, "success"}, - {"nullable reason", `{"status":200,"reason":null}`, 200, 200, "success"}, - {"denial", `{"status":403,"reason":"private-detail"}`, 200, 403, "denied"}, - {"conflict denial", `{"status":409}`, 200, 403, "denied"}, - {"service denial", `{"status":503}`, 200, 503, "denied"}, - {"last denial", `{"status":599}`, 200, 599, "denied"}, - {"non-200", "private-detail", 500, 502, "error"}, - {"redirect", "private-detail", 307, 502, "error"}, - {"malformed", "private-detail", 200, 502, "error"}, - {"trailing JSON", `{"status":200}{}`, 200, 502, "error"}, - {"missing status", `{}`, 200, 502, "error"}, - {"null status", `{"status":null}`, 200, 502, "error"}, - {"string status", `{"status":"200"}`, 200, 502, "error"}, - {"fractional status", `{"status":200.5}`, 200, 502, "error"}, - {"invalid status", `{"status":201}`, 200, 502, "error"}, - {"invalid denial", `{"status":600}`, 200, 502, "error"}, - {"invalid reason", `{"status":200,"reason":123}`, 200, 502, "error"}, - {"oversized", strings.Repeat(" ", (1<<20)+1), 200, 502, "error"}, - {"read failure", `{"status":200}`, 200, 502, "error"}, - } { - t.Run(tc.name, func(t *testing.T) { - var calls atomic.Int32 - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - calls.Add(1) - w.Header().Set("Location", "/redirected") - if tc.name == "read failure" { - w.Header().Set("Content-Length", strconv.Itoa(len(tc.body)+10)) - } - w.WriteHeader(tc.upstream) - _, _ = io.WriteString(w, tc.body) - })) - defer server.Close() - var logs bytes.Buffer - auth := testAuthWebhook(t, server.URL+"?private-detail", nodeconfig.Headers{"Authorization": {"private-detail"}}, &logs) - r := httptest.NewRequest("POST", "/tickets", nil) - w := httptest.NewRecorder() - allowed := auth.authorize(w, r, nil) - require.Equal(t, tc.want == 200, allowed) - require.Equal(t, tc.want, w.Code) - require.EqualValues(t, 1, calls.Load()) - require.NotContains(t, logs.String(), "private-detail") - require.NotContains(t, w.Body.String(), "private-detail") - require.EqualValues(t, 1, test.MetricSum(t, auth.metrics.Registry, "redeemer_auth_webhook_requests_total", map[string]string{"status": strconv.Itoa(tc.upstream), "outcome": tc.outcome})) - require.EqualValues(t, 1, test.MetricSum(t, auth.metrics.Registry, "redeemer_auth_webhook_duration_seconds", nil)) - }) - } -} - -func TestAuthWebhookUnavailableTimeoutAndCancellation(t *testing.T) { - for _, kind := range []string{"unavailable", "timeout", "cancelled"} { - t.Run(kind, func(t *testing.T) { - synctest.Test(t, func(t *testing.T) { - var logs bytes.Buffer - auth := testAuthWebhook(t, "http://webhook.example?private-detail", nil, &logs) - ctx, cancel := context.WithCancel(t.Context()) - defer cancel() - auth.client.Transport = authTransport(func(r *http.Request) (*http.Response, error) { - if kind == "unavailable" { - return nil, errors.New("private-detail") - } - if kind == "cancelled" { - cancel() - } - <-r.Context().Done() - return nil, r.Context().Err() - }) - r := httptest.NewRequest("POST", "/tickets", nil).WithContext(ctx) - w := httptest.NewRecorder() - started := time.Now() - require.False(t, auth.authorize(w, r, nil)) - if kind == "timeout" { - require.Equal(t, 5*time.Second, time.Since(started)) - } - require.Equal(t, 502, w.Code) - require.NotContains(t, logs.String(), "private-detail") - require.EqualValues(t, 1, test.MetricSum(t, auth.metrics.Registry, "redeemer_auth_webhook_requests_total", map[string]string{"status": "0", "outcome": "error"})) - }) - }) - } -} - -func TestAuthWebhookDoesNotReplayReceivedPOST(t *testing.T) { - var calls atomic.Int32 - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - _, _ = io.Copy(io.Discard, r.Body) - if calls.Add(1) == 2 { - conn, _, err := w.(http.Hijacker).Hijack() - require.NoError(t, err) - _ = conn.Close() // Lose the response on an already pooled connection. - return - } - _, _ = io.WriteString(w, `{"status":200}`) - })) - defer server.Close() - auth := testAuthWebhook(t, server.URL, nodeconfig.Headers{"Idempotency-Key": {"configured"}}, io.Discard) - require.True(t, auth.authorize(httptest.NewRecorder(), httptest.NewRequest("POST", "/tickets", nil), nil)) - w := httptest.NewRecorder() - require.False(t, auth.authorize(w, httptest.NewRequest("POST", "/tickets", nil), nil)) - require.Equal(t, 502, w.Code) - require.EqualValues(t, 2, calls.Load(), "the received second POST must not be replayed") -} - -func TestAuthorizationForwardingAndQueuedRedemption(t *testing.T) { - f := test.NewRedemptionRPC(t) - p := serviceParams(t, f) - rpc, err := eth.NewRPC(p.RPCURL, authTransport(func(r *http.Request) (*http.Response, error) { - w := httptest.NewRecorder() - f.Serve(w, r) - return w.Result(), nil - })) - require.NoError(t, err) - defer rpc.Close() - contracts, err := eth.NewContracts(rpc, *p.Controller) - require.NoError(t, err) - contracts.CheckRedemptionCost = true - chain := eth.PaymentChain{Contracts: contracts} - contracts.BeforeBroadcast = redemption.StandalonePolicy(chain, big.NewInt(1)) - newStore := func(role, target string) (*redemption.RedeemerDB, *sql.DB) { - db, err := redemption.OpenDatabase(t.Context(), filepath.Join(t.TempDir(), "tickets.sqlite"), true) - require.NoError(t, err) - s, err := redemption.NewStore(db) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, s.Close()) }) - require.NoError(t, s.Bind(t.Context(), redemption.Binding{ChainID: "1", Controller: *p.Controller, Role: role, Destination: target}, redemption.Limits{Pending: 1, PerRecipient: 1}, chain)) - return s, db - } - serverStore, serverDB := newStore("redeemer", "chain") - var decision, calls atomic.Int32 - decision.Store(409) - webhook := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - calls.Add(1) - var payload struct { - Ticket redemption.TicketJSON `json:"ticket"` - } - require.NoError(t, json.NewDecoder(r.Body).Decode(&payload)) - require.Empty(t, payload.Ticket.Randomness) - // Signature verification needs only the commitment, not its preimage. - payload.Ticket.Randomness = "0" - unsigned, err := payload.Ticket.SignedTicket() - require.NoError(t, err) - require.True(t, (pm.DefaultSigVerifier{}).Verify(unsigned.PayerAddress, unsigned.Hash().Bytes(), unsigned.Sig)) - _, _ = io.WriteString(w, `{"status":`+big.NewInt(int64(decision.Load())).String()+`}`) - })) - defer webhook.Close() - auth := testAuthWebhook(t, webhook.URL, nil, io.Discard) - server := httptest.NewServer(redemption.HandlerWithAuthorization(serverStore, auth.authorize)) - defer server.Close() - clientStore, clientDB := newStore("orchestrator", server.URL) - policy, err := destination.New("redeemer", []string{"127.0.0.1"}) - require.NoError(t, err) - u, err := url.Parse(server.URL) - require.NoError(t, err) - remote, err := redemption.NewHTTPDestination(u, policy, redemption.Binding{ChainID: "1", Controller: *p.Controller}) - require.NoError(t, err) - defer remote.Close() - payer, err := crypto.GenerateKey() - require.NoError(t, err) - random := big.NewInt(1) - ticket := &pm.SignedTicket{Ticket: &pm.Ticket{PayerAddress: crypto.PubkeyToAddress(payer.PublicKey), Recipient: common.HexToAddress("0x5678"), FaceValue: big.NewInt(1e12), WinProb: new(big.Int).Sub(new(big.Int).Lsh(big.NewInt(1), 256), big.NewInt(1)), TicketNonce: 1, RecipientRandHash: crypto.Keccak256Hash(common.LeftPadBytes(random.Bytes(), 32)), CreationRound: 1, CreationRoundBlockHash: common.HexToHash("0x1"), ParamsExpirationBlock: big.NewInt(10)}, RecipientRand: random} - ticket.Sig, err = crypto.Sign(accounts.TextHash(ticket.Hash().Bytes()), payer) - require.NoError(t, err) - ticket.Sig[64] += 27 - f.Mu.Lock() - f.Tickets[ticket.Hash()] = ticket.RecipientRandHash - f.Finalized = false - f.Mu.Unlock() - require.NoError(t, clientStore.Admit(t.Context(), ticket)) - forwarder, err := redemption.NewWorker(clientStore, chain, remote) - require.NoError(t, err) - require.NotEmpty(t, forwarder.Process(t.Context())) - var delivery string - require.NoError(t, clientDB.QueryRow("SELECT delivery FROM ticket_state").Scan(&delivery)) - require.Equal(t, "retryable", delivery, "webhook 409 must not permanently reject delivery") - var count int - require.NoError(t, serverDB.QueryRow("SELECT count(*) FROM winning_tickets").Scan(&count)) - require.Zero(t, count, "denied requests do not write ticket state") - decision.Store(200) - _, err = clientDB.Exec("UPDATE ticket_state SET next_attempt=0") - require.NoError(t, err) - require.Empty(t, forwarder.Process(t.Context())) - require.NoError(t, remote.Submit(t.Context(), ticket), "approved duplicates succeed at queue capacity") - ticket.ParamsExpirationBlock = big.NewInt(11) - err = remote.Submit(t.Context(), ticket) - var rejection *redemption.Rejection - require.ErrorAs(t, err, &rejection, "actual scheduling conflicts remain permanent") - ticket.ParamsExpirationBlock = big.NewInt(10) - decision.Store(403) - err = remote.Submit(t.Context(), ticket) - require.Error(t, err, "duplicates are authorized again") - require.False(t, errors.As(err, &rejection)) - require.EqualValues(t, 5, calls.Load()) - webhook.Close() - require.Error(t, remote.Submit(t.Context(), ticket), "outage withholds duplicate acknowledgment") - synctest.Test(t, func(t *testing.T) { - defer serverStore.Close() - key, err := eth.OpenKeystoreFile(p.KeystoreFile, p.KeystorePasswordFile) - require.NoError(t, err) - chainDest, err := redemption.NewChainDestination(t.Context(), serverStore, chain, key, big.NewInt(1), redemption.RedemptionOptions{Timeout: 5 * time.Second, MaxReplacements: 1}) - require.NoError(t, err) - worker, err := redemption.NewWorker(serverStore, chain, chainDest) - require.NoError(t, err) - require.Empty(t, worker.Process(t.Context())) - synctest.Wait() - require.Len(t, f.Transactions(), 1) - time.Sleep(5 * time.Second) - synctest.Wait() - require.Len(t, f.Transactions(), 2) - sent := f.Transactions() - require.Equal(t, sent[0].Nonce(), sent[1].Nonce()) - require.Equal(t, sent[0].Data(), sent[1].Data(), "transaction replacement proceeds during authorization outage") - require.EqualValues(t, 5, calls.Load(), "background redemption never authorizes again") - }) -} diff --git a/redemption/http_test.go b/redemption/http_test.go index 7add8d7..f33b6e4 100644 --- a/redemption/http_test.go +++ b/redemption/http_test.go @@ -16,7 +16,6 @@ import ( "net/url" "path/filepath" "strings" - "sync" "sync/atomic" "testing" "testing/synctest" @@ -99,7 +98,7 @@ func TestHTTPAdmissionIdentityCapacityAndScope(t *testing.T) { third, _ := winningTicket(t, 3) require.ErrorIs(t, s.Admit(t.Context(), third), ErrCapacity) bad := httptest.NewRecorder() - handler.ServeHTTP(bad, httptest.NewRequest("POST", "/tickets", bytes.NewReader(make([]byte, MaxTicketBody+1)))) + handler.ServeHTTP(bad, httptest.NewRequest("POST", "/tickets", strings.NewReader(strings.Repeat(" ", MaxTicketBody+1)))) require.Equal(t, 400, bad.Code) changed := b changed.Controller = common.HexToAddress("0x999") @@ -112,20 +111,26 @@ func TestHTTPAdmissionIdentityCapacityAndScope(t *testing.T) { func TestHTTPAuthorizationConcurrency(t *testing.T) { synctest.Test(t, func(t *testing.T) { - ctx, cancel := context.WithCancel(t.Context()) - var requests sync.WaitGroup - defer func() { cancel(); requests.Wait() }() + release := make(chan struct{}) + defer close(release) var calls atomic.Int32 handler := HandlerWithAuthorization(nil, func(w http.ResponseWriter, r *http.Request, ticket *TicketJSON) bool { + require.Nil(t, ticket) calls.Add(1) - <-ctx.Done() + <-release http.Error(w, "denied", 403) return false }) - for range 32 { - requests.Go(func() { - handler.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("POST", "/tickets", strings.NewReader("invalid"))) - }) + for i := range 32 { + go func() { + body := "invalid" + if i == 0 { + body = strings.Repeat(" ", MaxTicketBody+1) + } + w := httptest.NewRecorder() + handler.ServeHTTP(w, httptest.NewRequest("POST", "/tickets", strings.NewReader(body))) + require.Equal(t, 403, w.Code) + }() } synctest.Wait() require.EqualValues(t, 32, calls.Load()) @@ -144,10 +149,9 @@ func TestHTTPDestinationAuthorizationAndAdmissionBudgets(t *testing.T) { conn, err := s.db.Conn(t.Context()) require.NoError(t, err) defer conn.Close() - authorized := make(chan struct{}) handler := HandlerWithAuthorization(s, func(http.ResponseWriter, *http.Request, *TicketJSON) bool { time.Sleep(4 * time.Second) - close(authorized) + time.AfterFunc(4*time.Second, func() { _ = conn.Close() }) return true }) serverConn, clientConn := net.Pipe() @@ -174,20 +178,9 @@ func TestHTTPDestinationAuthorizationAndAdmissionBudgets(t *testing.T) { require.Equal(t, 15*time.Second, transport.ResponseHeaderTimeout) transport.Proxy = nil transport.DialContext = func(context.Context, string, string) (net.Conn, error) { return clientConn, nil } - done := make(chan struct{}) - go func() { - defer close(done) - select { - case <-authorized: - time.Sleep(4 * time.Second) // Hold storage after approval. - _ = conn.Close() - case <-t.Context().Done(): - } - }() ticket, _ := winningTicket(t, 1) started := time.Now() require.NoError(t, remote.Submit(t.Context(), ticket)) - <-done require.Equal(t, 8*time.Second, time.Since(started)) }) } From 0ee73c000bed553f5a02e40c972c1b5bd2d0ba80 Mon Sep 17 00:00:00 2001 From: Josh Allmann Date: Wed, 11 Nov 2026 20:05:47 -0800 Subject: [PATCH 4/4] Fix webhook admission Staticcheck warning --- redemption/http_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redemption/http_test.go b/redemption/http_test.go index f33b6e4..1880fc9 100644 --- a/redemption/http_test.go +++ b/redemption/http_test.go @@ -118,7 +118,7 @@ func TestHTTPAuthorizationConcurrency(t *testing.T) { require.Nil(t, ticket) calls.Add(1) <-release - http.Error(w, "denied", 403) + http.Error(w, "denied", http.StatusForbidden) return false }) for i := range 32 {