diff --git a/cmd/livepeer/testdata/cli/redeemer-help.golden b/cmd/livepeer/testdata/cli/redeemer-help.golden index 2a820da..9f07064 100644 --- a/cmd/livepeer/testdata/cli/redeemer-help.golden +++ b/cmd/livepeer/testdata/cli/redeemer-help.golden @@ -21,6 +21,8 @@ Flags: -k, --keystore-file string Encrypted geth redemption account JSON file (env: LIVEPEER_REDEEMER_KEYSTORE_FILE) --keystore-password-file string Owner-only file containing exact password bytes (env: LIVEPEER_REDEEMER_KEYSTORE_PASSWORD_FILE) --rpc-url-file string (env: LIVEPEER_REDEEMER_RPC_URL_FILE) + --auth-webhook-file string (env: LIVEPEER_REDEEMER_AUTH_WEBHOOK_FILE) + --auth-webhook-headers-file string (env: LIVEPEER_REDEEMER_AUTH_WEBHOOK_HEADERS_FILE) --chain-id uint (env: LIVEPEER_REDEEMER_CHAIN_ID) --controller-address string (env: LIVEPEER_REDEEMER_CONTROLLER_ADDRESS) --redeemer-tx-timeout duration Transaction inclusion timeout (env: LIVEPEER_REDEEMER_REDEEMER_TX_TIMEOUT) (default 5m0s) @@ -35,5 +37,7 @@ Flags: Use "livepeer-redeemer [command] --help" for more information about a command. Environment Variables: - LIVEPEER_REDEEMER_KEYSTORE_PASSWORD Keystore decryption password + LIVEPEER_REDEEMER_KEYSTORE_PASSWORD Keystore decryption password LIVEPEER_REDEEMER_RPC_URL + LIVEPEER_REDEEMER_AUTH_WEBHOOK + LIVEPEER_REDEEMER_AUTH_WEBHOOK_HEADERS diff --git a/configs/redeemer/config.example.toml b/configs/redeemer/config.example.toml index cd1a897..f314898 100644 --- a/configs/redeemer/config.example.toml +++ b/configs/redeemer/config.example.toml @@ -8,6 +8,9 @@ RedeemerDB = "tickets.sqlite" KeystoreFile = "/run/secrets/livepeer-redeemer-keystore.json" KeystorePasswordFile = "/run/secrets/livepeer-redeemer-password" RPCURLFile = "/run/secrets/livepeer-rpc-url" +# Optional per-request authorization; sends request headers and the ticket. +# AuthWebhookFile = "/run/secrets/livepeer-redeemer-auth-url" +# AuthWebhookHeadersFile = "/run/secrets/livepeer-redeemer-auth-headers" # Files containing credentials must be owner-only. Password bytes are exact. # ChainID = 42161 # Controller = "0xD8E8328501E9645d16Cf49539efC04f734606ee4" diff --git a/docs/configuration.md b/docs/configuration.md index 925c7e6..3624988 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -92,6 +92,7 @@ openssl rand -hex 32 | tr -d '\n' > runner-bootstrap.secret Set `BootstrapSecretFile` to its absolute path and use the same credential for initial runner registration. Secret-manager mounts must preserve exact bytes; URL files require a valid URL without a trailing newline. +Redeemer webhook URL and headers files must be owner-only regular files. ## Ethereum keystores diff --git a/docs/http-api.md b/docs/http-api.md index b1ab736..400a601 100644 --- a/docs/http-api.md +++ b/docs/http-api.md @@ -274,6 +274,36 @@ parameters. This field is not covered by the payer's signature; submitters must preserve the value issued by the orchestrator. Protect ticket submissions with TLS and endpoint authentication in your deployment; see [redeemer setup](redeemer.md). +### Redeemer authorization webhook + +When [configured](redeemer.md#authorization-webhook), the redeemer checks each submitted ticket. +It sends the webhook a JSON POST, for example: + +```json +{ + "headers": {"Authorization": ["Bearer caller-credential"]}, + "ticket": {"sig": "0x...", "recipient_rand_hash": "0x..."} +} +``` + +The example shows selected [ticket fields](#redeemer-http). The webhook receives +all those fields except `recipient_rand`. + +The webhook must return HTTP 200 with a JSON decision, for example: + +```json +{"status": 200} +``` + +| Decision `status` | Result | Client response | +| --- | --- | --- | +| 200 | Approve the ticket. | 202 Accepted if validation and storage succeed. | +| 409 | Deny the ticket. | 403 Forbidden | +| Other 400–599 | Deny the ticket. | Same status | + +A redirect, timeout, non-200 response, or invalid decision returns 502. The +timeout is five seconds. + ## Accounting events Optional Kafka accounting emits `create_signed_ticket` JSON events with diff --git a/docs/metrics.md b/docs/metrics.md index 6b061fd..2c49a04 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -85,6 +85,7 @@ gauges; timestamps use Unix seconds. Most event families appear on first use. | Payments | `payment_operations_total{operation,outcome}` for challenge/receive/refresh/charge; `payment_operation_duration_seconds{operation}` for challenge/receive/refresh; `payment_tickets_total{kind="accepted\|winning"}`, `payment_credit_eth_total`, `payment_charge_eth_total`, `payment_billed_seconds_total`, `payment_fixed_charges_total`, with app identity. Credit is accepted expected ticket value; charges are successfully debited service usage. | | Signer | `signer_payments_total{type,outcome}`, `signer_payment_duration_seconds{type}`, `signer_tickets_total{type}`, `signer_sign_operations_total{kind,outcome}`. Signing kinds are ticket, segment, state, and orchestrator_info. Partial signing before a failed request remains visible. Successful payment/ticket totals follow required outbox persistence, even if response delivery is interrupted. | | Authorization | `signer_auth_webhook_requests_total{status,outcome}`, `signer_auth_webhook_duration_seconds`, `signer_auth_cache_hits_total`. Status zero means no webhook response; outcomes distinguish success, denied, and error. | +| Redeemer authorization | `redeemer_auth_webhook_requests_total{status,outcome}`, `redeemer_auth_webhook_duration_seconds`. Status is the upstream HTTP status, or zero without a response; outcomes are `success`, `denied`, and `error`. | | Outbox | Existing `signer_kafka_pending_events`, `signer_kafka_pending_bytes`, `signer_kafka_oldest_event_age_seconds`, `signer_kafka_outbox_healthy`, `signer_kafka_publish_errors_total`, `signer_kafka_enqueue_errors_total`, and `signer_kafka_storage_errors_total`. New `signer_kafka_delivered_events_total` and `signer_kafka_last_delivery_timestamp_seconds` follow successful local acknowledgement; delivery remains at least once. | | Redemption queue | `redemption_queue_tickets{state}`, `redemption_oldest_ticket_timestamp_seconds{state}`: parameter-expiry `waiting`, clock-eligible `eligible`, `in_flight`, `awaiting_finality`, rejected remote submissions (`rejected`), invalid or reverted local attempts (`blocked`), or `unknown` without a fresh clock observation. Each logical ticket counts once, regardless of signatures or replacements. | | Transactions | `redemption_unresolved_transactions{phase}`, `redemption_oldest_transaction_timestamp_seconds{phase}` include historical retries/replacements until the identity is confirmed, reverted, or superseded, or its ticket is settled (`confirmed`/`used`). Ticket expiry or invalidation alone does not remove an unresolved identity. `sending` appears as `broadcast`. | diff --git a/docs/redeemer.md b/docs/redeemer.md index 73b435d..ab69abb 100644 --- a/docs/redeemer.md +++ b/docs/redeemer.md @@ -17,6 +17,20 @@ through your deployment or a proxy. Use a separate persistent database for each redeemer; see [configuration](configuration.md) and [recovery](payment-recovery.md). `--config` takes a local TOML or JSON file path, as with other components. +## Authorization webhook + +To authorize ticket submissions: + +```toml +AuthWebhookFile = "/run/secrets/livepeer-redeemer-auth-url" +# AuthWebhookHeadersFile = "/run/secrets/livepeer-redeemer-auth-headers" +``` + +The URL must be absolute HTTP or HTTPS without a username, password, or fragment. +The optional headers file contains comma-separated `Header: value` entries. See +[credential files](configuration.md#credential-files) and the +[HTTP contract](http-api.md#redeemer-authorization-webhook). + ## Connect an orchestrator Add these settings to an onchain orchestrator's configuration, using its diff --git a/nodeconfig/config.go b/nodeconfig/config.go index 373f892..df23d41 100644 --- a/nodeconfig/config.go +++ b/nodeconfig/config.go @@ -1,4 +1,5 @@ -// Package nodeconfig supplies shared CLI policy, data-directory defaults, and storage paths. +// Package nodeconfig supplies shared configuration types, CLI policy, +// data-directory defaults, and storage paths. package nodeconfig import ( diff --git a/nodeconfig/headers.go b/nodeconfig/headers.go new file mode 100644 index 0000000..b360465 --- /dev/null +++ b/nodeconfig/headers.go @@ -0,0 +1,89 @@ +package nodeconfig + +import ( + "bytes" + "encoding/csv" + "errors" + "io" + "maps" + "net/http" + "slices" + "strings" +) + +// Headers parses comma-separated Header: value entries, with CSV quoting for +// commas. Names are case-insensitive; repeated names retain all values. +type Headers http.Header + +func validHeader(name, value string) bool { + if name == "" { + return false + } + for _, c := range name { + if !((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || strings.ContainsRune("!#$%&'*+-.^_`|~", c)) { + return false + } + } + for _, c := range value { + if (c < 32 && c != '\t') || c == 127 { + return false + } + } + return true +} + +func (h *Headers) UnmarshalText(text []byte) error { + parsed := make(http.Header) + input := strings.TrimSpace(string(text)) + if input != "" { + reader := csv.NewReader(strings.NewReader(input)) + reader.TrimLeadingSpace = true + reader.FieldsPerRecord = -1 + entries, err := reader.Read() + if err != nil { + return errors.New("invalid comma-separated webhook headers") + } + if _, err := reader.Read(); err != io.EOF { + return errors.New("webhook headers must be one comma-separated record") + } + for _, entry := range entries { + name, value, ok := strings.Cut(entry, ":") + name, value = strings.TrimSpace(name), strings.TrimSpace(value) + if !ok || !validHeader(name, value) { + return errors.New("expected webhook headers in Header: value form") + } + parsed.Add(name, value) + } + } + *h = Headers(parsed) + return nil +} + +// MarshalText returns CSV entries sorted by header name. +func (h Headers) MarshalText() ([]byte, error) { + var entries []string + for _, name := range slices.Sorted(maps.Keys(h)) { + if !validHeader(name, "") { + return nil, errors.New("invalid webhook header name") + } + for _, value := range h[name] { + if !validHeader(name, value) { + return nil, errors.New("invalid webhook header name or value") + } + entries = append(entries, name+": "+value) + } + } + if len(entries) == 0 { + return nil, nil + } + var buffer bytes.Buffer + writer := csv.NewWriter(&buffer) + if err := writer.Write(entries); err != nil { + return nil, err + } + writer.Flush() + if err := writer.Error(); err != nil { + return nil, err + } + return bytes.TrimSuffix(buffer.Bytes(), []byte("\n")), nil +} diff --git a/signer/headers_test.go b/nodeconfig/headers_test.go similarity index 98% rename from signer/headers_test.go rename to nodeconfig/headers_test.go index b933b58..faf1bc4 100644 --- a/signer/headers_test.go +++ b/nodeconfig/headers_test.go @@ -1,4 +1,4 @@ -package signer +package nodeconfig import ( "net/http" diff --git a/redeemer/app.go b/redeemer/app.go index a3efec3..6eafe63 100644 --- a/redeemer/app.go +++ b/redeemer/app.go @@ -11,6 +11,7 @@ import ( "net" "net/http" "net/url" + "os" "os/signal" "sync" "syscall" @@ -33,23 +34,27 @@ import ( type Params struct { ConfigFile string `name:"config" configfile:"optional-default" basepath:"source" default:"config.toml" boa:"noconfig" descr:"Configuration file; empty disables discovery"` nodeconfig.Settings - Listen boa.HTTPBind `default:"127.0.0.1:8941" descr:"Ticket submission listener"` - MetricsListen boa.HTTPBind `default:"127.0.0.1:8942" descr:"Metrics and probe listener"` - Account *common.Address `descr:"Dedicated redemption wallet; must not be an orchestrator"` - RedeemerDB string `file:"optional" descr:"Winning-ticket SQLite database"` - KeystoreFile string `basepath:"source" optional:"true" descr:"Encrypted geth redemption account JSON file"` - KeystorePassword *string `secret:"true" optional:"true" descr:"Keystore decryption password"` - KeystorePasswordFile string `basepath:"source" secretfor:"KeystorePassword" descr:"Owner-only file containing exact password bytes"` - RPCURL *url.URL `name:"rpc-url" secret:"true" optional:"true"` - RPCURLFile string `basepath:"source" name:"rpc-url-file" secretfor:"RPCURL"` - ChainID *uint64 `min:"1"` - Controller *common.Address `name:"controller-address"` - RedeemerTxTimeout time.Duration `default:"5m" descr:"Transaction inclusion timeout"` - RedeemerMaxTxReplacements int `default:"1" min:"0" descr:"Maximum fee replacements per transaction"` - RedeemerMaxFeePerGas *uint256.Int `descr:"Optional maximum fee in wei per gas"` - MaxPendingTickets int `default:"100000" min:"1" descr:"Maximum unsettled winning tickets"` - MaxTicketsPerRecipient int `default:"10000" min:"1" descr:"Maximum unsettled tickets per recipient"` - PrintConfig bool `optional:"true" boa:"noconfig,noenv" descr:"Print audited redacted TOML configuration"` + Listen boa.HTTPBind `default:"127.0.0.1:8941" descr:"Ticket submission listener"` + MetricsListen boa.HTTPBind `default:"127.0.0.1:8942" descr:"Metrics and probe listener"` + Account *common.Address `descr:"Dedicated redemption wallet; must not be an orchestrator"` + RedeemerDB string `file:"optional" descr:"Winning-ticket SQLite database"` + KeystoreFile string `basepath:"source" optional:"true" descr:"Encrypted geth redemption account JSON file"` + KeystorePassword *string `secret:"true" optional:"true" descr:"Keystore decryption password"` + KeystorePasswordFile string `basepath:"source" secretfor:"KeystorePassword" descr:"Owner-only file containing exact password bytes"` + RPCURL *url.URL `name:"rpc-url" secret:"true" optional:"true"` + RPCURLFile string `basepath:"source" name:"rpc-url-file" secretfor:"RPCURL"` + AuthWebhook *url.URL `name:"auth-webhook" optional:"true" secret:"true"` + AuthWebhookFile string `basepath:"source" name:"auth-webhook-file" secretfor:"AuthWebhook"` + AuthWebhookHeaders nodeconfig.Headers `name:"auth-webhook-headers" optional:"true" secret:"true"` + AuthWebhookHeadersFile string `basepath:"source" name:"auth-webhook-headers-file" secretfor:"AuthWebhookHeaders"` + ChainID *uint64 `min:"1"` + Controller *common.Address `name:"controller-address"` + RedeemerTxTimeout time.Duration `default:"5m" descr:"Transaction inclusion timeout"` + RedeemerMaxTxReplacements int `default:"1" min:"0" descr:"Maximum fee replacements per transaction"` + RedeemerMaxFeePerGas *uint256.Int `descr:"Optional maximum fee in wei per gas"` + MaxPendingTickets int `default:"100000" min:"1" descr:"Maximum unsettled winning tickets"` + MaxTicketsPerRecipient int `default:"10000" min:"1" descr:"Maximum unsettled tickets per recipient"` + PrintConfig bool `optional:"true" boa:"noconfig,noenv" descr:"Print audited redacted TOML configuration"` } func (p Params) Validate() error { @@ -68,6 +73,23 @@ func (p Params) Validate() error { if err := destination.ValidateURL(p.RPCURL); err != nil { return errors.New("redeemer requires a valid RPC URL") } + if p.AuthWebhook != nil || p.AuthWebhookFile != "" || len(p.AuthWebhookHeaders) > 0 || p.AuthWebhookHeadersFile != "" { + if err := destination.ValidateURL(p.AuthWebhook); err != nil || p.AuthWebhook.User != nil || p.AuthWebhook.Fragment != "" { + return errors.New("auth webhook must be an absolute HTTP or HTTPS URL without credentials or fragment") + } + } + if _, err := p.AuthWebhookHeaders.MarshalText(); err != nil { + return fmt.Errorf("invalid auth webhook headers: %w", err) + } + for _, path := range []string{p.AuthWebhookFile, p.AuthWebhookHeadersFile} { + if path == "" { + continue + } + info, err := os.Stat(path) + if err != nil || !info.Mode().IsRegular() || info.Mode().Perm()&0077 != 0 { + return errors.New("auth webhook secret files must be owner-only regular files") + } + } if p.RedeemerTxTimeout <= 0 || p.RedeemerMaxTxReplacements < 0 || p.MaxPendingTickets <= 0 || p.MaxTicketsPerRecipient <= 0 { return errors.New("invalid redemption timeout, replacement count or queue capacity") } @@ -198,6 +220,11 @@ func Serve(parent context.Context, p Params, logOut io.Writer) (result error) { }) probe.Handle("GET /metrics", metrics.Handler()) handler := redemption.Handler(store) + if p.AuthWebhook != nil { + auth := newAuthWebhook(p.AuthWebhook, p.AuthWebhookHeaders, logger, metrics) + defer auth.client.CloseIdleConnections() + handler = redemption.HandlerWithAuthorization(store, auth.authorize) + } servers := []*http.Server{ {Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { metrics.HTTP(w, r, handler.ServeHTTP) }), BaseContext: func(net.Listener) context.Context { return ctx }, ReadHeaderTimeout: 5 * time.Second, ReadTimeout: 15 * time.Second, WriteTimeout: 15 * time.Second, IdleTimeout: 30 * time.Second, MaxHeaderBytes: 1 << 16, MaxHeaderValueCount: 128}, {Handler: probe, ReadHeaderTimeout: 5 * time.Second, WriteTimeout: 15 * time.Second, IdleTimeout: 30 * time.Second, MaxHeaderBytes: 1 << 16, MaxHeaderValueCount: 128}, diff --git a/redeemer/app_test.go b/redeemer/app_test.go index 33c2340..6720912 100644 --- a/redeemer/app_test.go +++ b/redeemer/app_test.go @@ -4,26 +4,35 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "io" + "log/slog" "math/big" "net" "net/http" + "net/http/httptest" "net/url" "os" "path/filepath" + "strconv" "strings" + "sync/atomic" "testing" + "testing/iotest" + "testing/synctest" "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" "github.com/j0sh/boa/pkg/boa" "github.com/livepeer/node/eth" + "github.com/livepeer/node/internal/observability" "github.com/livepeer/node/internal/test" "github.com/livepeer/node/nodeconfig" "github.com/livepeer/node/pm" "github.com/livepeer/node/redemption" + "github.com/spf13/cobra" "github.com/stretchr/testify/require" ) @@ -77,11 +86,20 @@ func TestDedicatedWalletStartupPolicy(t *testing.T) { } func TestServeProbesAndShutdown(t *testing.T) { - for _, busy := range []bool{false, true} { - t.Run(fmt.Sprintf("busy=%t", busy), func(t *testing.T) { + for _, tc := range []struct{ busy, auth bool }{{}, {busy: true}, {auth: true}} { + t.Run(fmt.Sprintf("busy=%t/auth=%t", tc.busy, tc.auth), func(t *testing.T) { f := test.NewRedemptionRPC(t) p := serviceParams(t, f) - if busy { + var calls atomic.Int32 + if tc.auth { + webhook := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + _, _ = io.WriteString(w, `{"status":403}`) + })) + defer webhook.Close() + p.AuthWebhook, _ = url.Parse(webhook.URL) + } + if tc.busy { seedBusyQueue(t, p) handler := f.Server.Config.Handler selector := fmt.Sprintf("0x%x", crypto.Keccak256([]byte("usedTickets(bytes32)"))[:4]) @@ -106,7 +124,7 @@ func TestServeProbesAndShutdown(t *testing.T) { }) } wait := 5 * time.Second - if busy { + if tc.busy { wait = 40 * time.Second } ctx, cancel := context.WithCancel(t.Context()) @@ -123,14 +141,15 @@ func TestServeProbesAndShutdown(t *testing.T) { }) client := &http.Client{Timeout: time.Second} defer client.CloseIdleConnections() - require.Eventually(t, func() bool { + ready := func() bool { response, err := client.Get("http://" + p.MetricsListen.String() + "/readyz") if err != nil { return false } defer response.Body.Close() return response.StatusCode == 200 - }, wait, 10*time.Millisecond) + } + require.Eventually(t, ready, wait, 10*time.Millisecond) response, err := client.Get("http://" + p.MetricsListen.String() + "/metrics") require.NoError(t, err) defer response.Body.Close() @@ -141,6 +160,15 @@ func TestServeProbesAndShutdown(t *testing.T) { require.NoError(t, err) response.Body.Close() require.Equal(t, 405, response.StatusCode) + if tc.auth { + require.Zero(t, calls.Load(), "probes and other methods do not authorize") + response, err = client.Post("http://"+p.Listen.String()+"/tickets", "application/json", strings.NewReader("invalid ticket")) + require.NoError(t, err) + response.Body.Close() + require.Equal(t, 403, response.StatusCode, "invalid submissions are still authorized") + require.True(t, ready(), "authorization denial does not change readiness") + require.EqualValues(t, 1, calls.Load()) + } info, err := os.Stat(p.RedeemerDB) require.NoError(t, err) require.Equal(t, os.FileMode(0600), info.Mode().Perm()) @@ -169,6 +197,8 @@ func TestPrintConfigRedactsCredentials(t *testing.T) { t.Setenv("HOME", t.TempDir()) t.Setenv("LIVEPEER_REDEEMER_RPC_URL", "http://private.example/secret-token") t.Setenv("LIVEPEER_REDEEMER_KEYSTORE_PASSWORD", "wallet-secret") + t.Setenv("LIVEPEER_REDEEMER_AUTH_WEBHOOK", "https://example.com/?auth-url-secret") + t.Setenv("LIVEPEER_REDEEMER_AUTH_WEBHOOK_HEADERS", "Authorization: Bearer auth-header-secret") var out, errOut bytes.Buffer cmd := Root(&out, &errOut) cmd.SetArgs([]string{"--config=", "--print-config"}) @@ -176,4 +206,351 @@ func TestPrintConfigRedactsCredentials(t *testing.T) { require.Contains(t, out.String(), "8941") require.NotContains(t, out.String(), "secret-token") require.NotContains(t, out.String(), "wallet-secret") + require.NotContains(t, out.String(), "auth-url-secret") + require.NotContains(t, out.String(), "auth-header-secret") +} + +func loadAuthConfig(t *testing.T, p Params, args ...string) (Params, error) { + t.Helper() + cmd := nodeconfig.Command("redeemer", nodeconfig.Mainnet, boa.Cmd[Params]{Params: &p, RejectUnknown: true, + RunFuncE: func(p *Params, _ *cobra.Command, _ []string) error { return p.Validate() }, + }, &p.ConfigFile) + err := cmd.RunArgsE(args) + return p, err +} + +func TestAuthWebhookConfiguration(t *testing.T) { + base := serviceParams(t, test.NewRedemptionRPC(t)) + const endpoint = "https://webhook.example/auth?token=private-detail" + const headers = "Authorization: Bearer private-detail" + for _, tc := range []struct { + name, field, content string + withURL bool + }{ + {"empty URL file", "AUTH_WEBHOOK_FILE", "", false}, + {"empty orphan headers file", "AUTH_WEBHOOK_HEADERS_FILE", "", false}, + {"blank orphan headers file", "AUTH_WEBHOOK_HEADERS_FILE", " \t\n", false}, + {"empty headers with URL", "AUTH_WEBHOOK_HEADERS_FILE", "", true}, + } { + t.Run(tc.name, func(t *testing.T) { + file := filepath.Join(t.TempDir(), "secret") + require.NoError(t, os.WriteFile(file, []byte(tc.content), 0600)) + t.Setenv("LIVEPEER_REDEEMER_AUTH_WEBHOOK", "") + t.Setenv("LIVEPEER_REDEEMER_"+tc.field, file) + if tc.withURL { + t.Setenv("LIVEPEER_REDEEMER_AUTH_WEBHOOK", endpoint) + } + _, err := loadAuthConfig(t, base, "--config=") + if tc.withURL { + require.NoError(t, err) + } else { + require.ErrorContains(t, err, "auth webhook must be an absolute HTTP or HTTPS URL") + require.NotContains(t, err.Error(), "private-detail") + } + }) + } + for _, source := range []string{"environment", "files"} { + t.Run(source, func(t *testing.T) { + args := []string{"--config="} + if source == "environment" { + t.Setenv("LIVEPEER_REDEEMER_AUTH_WEBHOOK", endpoint) + t.Setenv("LIVEPEER_REDEEMER_AUTH_WEBHOOK_HEADERS", headers) + } else { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "url"), []byte(endpoint), 0600)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "headers"), []byte(headers), 0600)) + path := filepath.Join(dir, "config.toml") + test.WriteConfig(t, path, "AuthWebhookFile = 'url'\nAuthWebhookHeadersFile = 'headers'\n") + args = []string{"--config", path} + } + p, err := loadAuthConfig(t, base, args...) + require.NoError(t, err) + require.Equal(t, endpoint, p.AuthWebhook.String()) + require.Equal(t, "Bearer private-detail", http.Header(p.AuthWebhookHeaders).Get("Authorization")) + }) + } + for _, flag := range []string{"--auth-webhook", "--auth-webhook-headers"} { + _, err := loadAuthConfig(t, base, "--config=", flag, "private-detail") + require.ErrorContains(t, err, "unknown flag") + require.NotContains(t, err.Error(), "private-detail") + } + for field, value := range map[string]string{"AUTH_WEBHOOK": endpoint, "AUTH_WEBHOOK_HEADERS": headers} { + t.Run(field+" file protection", func(t *testing.T) { + file := filepath.Join(t.TempDir(), "secret") + require.NoError(t, os.WriteFile(file, []byte(value), 0600)) + t.Setenv("LIVEPEER_REDEEMER_AUTH_WEBHOOK", endpoint) + t.Setenv("LIVEPEER_REDEEMER_"+field+"_FILE", file) + t.Setenv("LIVEPEER_REDEEMER_"+field, value) + _, err := loadAuthConfig(t, base, "--config=") + require.ErrorContains(t, err, "cannot both be set") + require.NotContains(t, err.Error(), "private-detail") + t.Setenv("LIVEPEER_REDEEMER_"+field, "") + require.NoError(t, os.Chmod(file, 0644)) + _, err = loadAuthConfig(t, base, "--config=") + require.Error(t, err) + require.NotContains(t, err.Error(), "private-detail") + }) + } +} + +func TestAuthWebhookConfigurationValidation(t *testing.T) { + p := serviceParams(t, test.NewRedemptionRPC(t)) + require.NoError(t, p.Validate(), "omitted webhook is disabled") + for _, endpoint := range []string{"/private-detail", "ftp://private-detail", "https://user:private-detail@example.com", "https://example.com/#private-detail"} { + p.AuthWebhook, _ = url.Parse(endpoint) + err := p.Validate() + require.Error(t, err) + require.NotContains(t, err.Error(), "private-detail") + } + p.AuthWebhook = nil + p.AuthWebhookHeaders = nodeconfig.Headers{"Authorization": {"private-detail"}} + require.Error(t, p.Validate(), "headers require an endpoint") + p.AuthWebhook, _ = url.Parse("http://127.0.0.1/private?token=private-detail") + require.NoError(t, p.Validate()) + p.AuthWebhookHeaders = nodeconfig.Headers{"Bad Name": {"private-detail"}} + err := p.Validate() + require.ErrorContains(t, err, "invalid auth webhook headers") + require.NotContains(t, err.Error(), "private-detail") + +} + +type authTransport func(*http.Request) (*http.Response, error) + +func (f authTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } + +func testAuthWebhook(t *testing.T, endpoint string, headers nodeconfig.Headers, logs io.Writer) *authWebhook { + t.Helper() + u, err := url.Parse(endpoint) + require.NoError(t, err) + auth := newAuthWebhook(u, headers, slog.New(slog.NewTextHandler(logs, nil)), observability.New("redeemer")) + t.Cleanup(auth.client.CloseIdleConnections) + return auth +} + +func TestAuthWebhookHeadersAndRequests(t *testing.T) { + type webhookRequest struct { + Headers http.Header `json:"headers"` + Ticket json.RawMessage `json:"ticket"` + } + received := make(chan webhookRequest, 1) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + require.Equal(t, "POST", r.Method) + require.Equal(t, "token=private-url", r.URL.RawQuery) + require.Equal(t, "application/json", r.Header.Get("Content-Type")) + require.Equal(t, "Bearer service-secret", r.Header.Get("Authorization")) + require.Empty(t, r.Header.Values("X-Caller")) + var payload webhookRequest + decoder := json.NewDecoder(r.Body) + decoder.DisallowUnknownFields() + require.NoError(t, decoder.Decode(&payload)) + received <- payload + _, _ = io.WriteString(w, `{"status":200,"expiry":9999999999,"auth_id":"ignored","maxPrice":{}}`) + })) + defer server.Close() + headers := nodeconfig.Headers{"Authorization": {"Bearer service-secret"}, "Content-Type": {"text/plain"}} + auth := testAuthWebhook(t, server.URL+"?token=private-url", headers, io.Discard) + headers["Authorization"][0] = "mutated" + ticket := redemption.TicketJSON{ + Payer: common.HexToAddress("0x1234"), Recipient: common.HexToAddress("0x5678"), + FaceValue: "1000", WinProb: "42", Nonce: 7, + RandHash: common.HexToHash("0x1234"), CreationRound: "1", RoundHash: common.HexToHash("0x5678"), + Expiration: "10", Signature: []byte{1, 2, 3}, Randomness: "9", + } + for _, submitted := range []*redemption.TicketJSON{nil, &ticket} { + raw, err := json.Marshal(submitted) + require.NoError(t, err) + var expected map[string]any + require.NoError(t, json.Unmarshal(raw, &expected)) + delete(expected, "recipient_rand") + want, err := json.Marshal(expected) + require.NoError(t, err) + r := httptest.NewRequest("POST", "/tickets", nil) + r.Header = http.Header{"Authorization": {"Bearer caller-secret"}, "X-Caller": {"one", "two"}} + require.True(t, auth.authorize(httptest.NewRecorder(), r, submitted)) + payload := <-received + require.Equal(t, r.Header, payload.Headers) + require.JSONEq(t, string(want), string(payload.Ticket), "only recipient_rand is omitted") + } + require.Equal(t, "9", ticket.Randomness, "redemption keeps the original randomness") +} + +func TestAuthWebhookDecisions(t *testing.T) { + for _, tc := range []struct { + name, body string + upstream, want int + outcome string + }{ + {"approval", `{"status":200}`, 200, 200, "success"}, + {"nullable reason", `{"status":200,"reason":null}`, 200, 200, "success"}, + {"denial", `{"status":403,"reason":"private-detail"}`, 200, 403, "denied"}, + {"conflict denial", `{"status":409}`, 200, 403, "denied"}, + {"service denial", `{"status":503}`, 200, 503, "denied"}, + {"last denial", `{"status":599}`, 200, 599, "denied"}, + {"non-200", "private-detail", 500, 502, "error"}, + {"malformed", "private-detail", 200, 502, "error"}, + {"trailing JSON", `{"status":200}{}`, 200, 502, "error"}, + {"missing status", `{}`, 200, 502, "error"}, + {"null status", `{"status":null}`, 200, 502, "error"}, + {"string status", `{"status":"200"}`, 200, 502, "error"}, + {"fractional status", `{"status":200.5}`, 200, 502, "error"}, + {"invalid status", `{"status":201}`, 200, 502, "error"}, + {"invalid denial", `{"status":600}`, 200, 502, "error"}, + {"invalid reason", `{"status":200,"reason":123}`, 200, 502, "error"}, + {"oversized", strings.Repeat(" ", (1<<20)+1), 200, 502, "error"}, + {"read failure", `{"status":200}`, 200, 502, "error"}, + {"unavailable", "", 0, 502, "error"}, + {"timeout", "", 0, 502, "error"}, + {"cancelled", "", 0, 502, "error"}, + } { + t.Run(tc.name, func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + var logs bytes.Buffer + calls := 0 + auth := testAuthWebhook(t, "http://webhook.example?private-detail", nodeconfig.Headers{"Authorization": {"private-detail"}}, &logs) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + auth.client.Transport = authTransport(func(r *http.Request) (*http.Response, error) { + calls++ + switch tc.name { + case "unavailable": + return nil, errors.New("private-detail") + case "cancelled": + cancel() + fallthrough + case "timeout": + <-r.Context().Done() + return nil, r.Context().Err() + } + var body io.Reader = strings.NewReader(tc.body) + if tc.name == "read failure" { + body = io.MultiReader(body, iotest.ErrReader(errors.New("private-detail"))) + } + return &http.Response{StatusCode: tc.upstream, Body: io.NopCloser(body)}, nil + }) + w := httptest.NewRecorder() + started := time.Now() + require.Equal(t, tc.want == 200, auth.authorize(w, httptest.NewRequest("POST", "/tickets", nil).WithContext(ctx), nil)) + if tc.name == "timeout" { + require.Equal(t, 5*time.Second, time.Since(started)) + } + require.Equal(t, tc.want, w.Code) + require.Equal(t, 1, calls, "authorization never retries") + require.NotContains(t, logs.String()+w.Body.String(), "private-detail") + require.EqualValues(t, 1, test.MetricSum(t, auth.metrics.Registry, "redeemer_auth_webhook_requests_total", map[string]string{"status": strconv.Itoa(tc.upstream), "outcome": tc.outcome})) + require.EqualValues(t, 1, test.MetricSum(t, auth.metrics.Registry, "redeemer_auth_webhook_duration_seconds", nil)) + }) + }) + } +} + +func TestAuthWebhookRedirectAndReplay(t *testing.T) { + var calls atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + switch calls.Add(1) { + case 1: + http.Redirect(w, r, "/redirected", http.StatusFound) + return + case 3: + conn, _, err := w.(http.Hijacker).Hijack() + require.NoError(t, err) + _ = conn.Close() // Lose the response on an already pooled connection. + return + } + _, _ = io.WriteString(w, `{"status":200}`) + })) + defer server.Close() + auth := testAuthWebhook(t, server.URL, nodeconfig.Headers{"Idempotency-Key": {"configured"}}, io.Discard) + for i, allowed := range []bool{false, true, false} { + w := httptest.NewRecorder() + require.Equal(t, allowed, auth.authorize(w, httptest.NewRequest("POST", "/tickets", nil), nil)) + if !allowed { + require.Equal(t, 502, w.Code) + } + require.EqualValues(t, i+1, calls.Load(), "no redirects or replays") + } +} + +func TestAuthorizationAndQueuedRedemption(t *testing.T) { + f := test.NewRedemptionRPC(t) + p := serviceParams(t, f) + synctest.Test(t, func(t *testing.T) { + rpc, err := eth.NewRPC(p.RPCURL, authTransport(func(r *http.Request) (*http.Response, error) { + w := httptest.NewRecorder() + f.Serve(w, r) + return w.Result(), nil + })) + require.NoError(t, err) + defer rpc.Close() + contracts, err := eth.NewContracts(rpc, *p.Controller) + require.NoError(t, err) + chain := eth.PaymentChain{Contracts: contracts} + store, err := redemption.OpenRedeemerDB(p.RedeemerDB) + require.NoError(t, err) + defer store.Close() + require.NoError(t, store.Bind(t.Context(), redemption.Binding{ChainID: "1", Controller: *p.Controller, Role: "redeemer", Destination: "chain"}, redemption.Limits{Pending: 1, PerRecipient: 1}, chain)) + key, err := eth.OpenKeystoreFile(p.KeystoreFile, p.KeystorePasswordFile) + require.NoError(t, err) + input := redemption.TicketJSON{ + Payer: key.Address(), Recipient: common.HexToAddress("0x5678"), FaceValue: "1000000000000", + WinProb: new(big.Int).Sub(new(big.Int).Lsh(big.NewInt(1), 256), big.NewInt(1)).String(), Nonce: 1, + RandHash: crypto.Keccak256Hash(common.LeftPadBytes([]byte{1}, 32)), CreationRound: "1", + RoundHash: common.HexToHash("0x1"), Expiration: "10", Randomness: "1", + } + ticket, err := input.SignedTicket() + require.NoError(t, err) + input.Signature, err = key.SignMessage(ticket.Hash().Bytes()) + require.NoError(t, err) + f.Tickets[ticket.Hash()], f.Finalized = ticket.RecipientRandHash, false + decision, calls := 409, 0 + unavailable := false + auth := testAuthWebhook(t, "http://webhook.example", nil, io.Discard) + auth.client.Transport = authTransport(func(r *http.Request) (*http.Response, error) { + calls++ + if unavailable { + return nil, errors.New("unavailable") + } + var payload struct{ Ticket redemption.TicketJSON } + require.NoError(t, json.NewDecoder(r.Body).Decode(&payload)) + require.Empty(t, payload.Ticket.Randomness) + payload.Ticket.Randomness = "0" // Verify the signature without the preimage. + signed, err := payload.Ticket.SignedTicket() + require.NoError(t, err) + require.True(t, (pm.DefaultSigVerifier{}).Verify(signed.PayerAddress, signed.Hash().Bytes(), signed.Sig)) + return &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader(`{"status":` + strconv.Itoa(decision) + `}`))}, nil + }) + body, err := json.Marshal(redemption.TicketRequest{ChainID: "1", Controller: *p.Controller, Ticket: input}) + require.NoError(t, err) + handler := redemption.HandlerWithAuthorization(store, auth.authorize) + submit := func() int { + w := httptest.NewRecorder() + handler.ServeHTTP(w, httptest.NewRequest("POST", "/tickets", bytes.NewReader(body))) + return w.Code + } + require.Equal(t, 403, submit()) + count, err := store.WinningTicketCount(input.Payer) + require.NoError(t, err) + require.Zero(t, count, "denied requests do not store tickets") + decision = 200 + require.Equal(t, 202, submit()) + require.Equal(t, 202, submit(), "duplicates succeed at capacity") + decision = 403 + require.Equal(t, 403, submit(), "duplicates are authorized again") + unavailable = true + require.Equal(t, 502, submit()) + chainDest, err := redemption.NewChainDestination(t.Context(), store, chain, key, big.NewInt(1), redemption.RedemptionOptions{Timeout: 5 * time.Second, MaxReplacements: 1}) + require.NoError(t, err) + worker, err := redemption.NewWorker(store, chain, chainDest) + require.NoError(t, err) + require.Empty(t, worker.Process(t.Context())) + synctest.Wait() + require.Len(t, f.Transactions(), 1) + time.Sleep(5 * time.Second) + synctest.Wait() + sent := f.Transactions() + require.Len(t, sent, 2) + require.Equal(t, sent[0].Nonce(), sent[1].Nonce()) + require.Equal(t, sent[0].Data(), sent[1].Data()) + require.Equal(t, 5, calls, "background redemption never authorizes again") + }) } diff --git a/redeemer/auth.go b/redeemer/auth.go new file mode 100644 index 0000000..35513ef --- /dev/null +++ b/redeemer/auth.go @@ -0,0 +1,119 @@ +package redeemer + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "net/url" + "strconv" + "time" + + "github.com/livepeer/node/internal/observability" + "github.com/livepeer/node/nodeconfig" + "github.com/livepeer/node/redemption" +) + +type authWebhook struct { + endpoint string + headers http.Header + client *http.Client + logger *slog.Logger + metrics *observability.Metrics +} + +func newAuthWebhook(endpoint *url.URL, headers nodeconfig.Headers, logger *slog.Logger, metrics *observability.Metrics) *authWebhook { + return &authWebhook{ + endpoint: endpoint.String(), + headers: http.Header(headers).Clone(), + client: &http.Client{ + Transport: http.DefaultTransport.(*http.Transport).Clone(), + Timeout: 5 * time.Second, + CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, + }, + logger: logger.With("event", "authorization", "operation", "authorization"), + metrics: metrics, + } +} + +func (a *authWebhook) authorize(w http.ResponseWriter, r *http.Request, ticket *redemption.TicketJSON) bool { + started := time.Now() + status, outcome := 0, "error" + defer func() { + a.metrics.Add("redeemer_auth_webhook_requests_total", 1, observability.Labels{"status": strconv.Itoa(status), "outcome": outcome}) + a.metrics.Observe("redeemer_auth_webhook_duration_seconds", time.Since(started).Seconds(), nil) + }() + fail := func(message, kind string) bool { + a.logger.ErrorContext(r.Context(), message, "outcome", "error", "error_kind", kind) + http.Error(w, message, http.StatusBadGateway) + return false + } + type webhookTicket struct { + *redemption.TicketJSON + // Shadow the JSON field without changing the ticket used for redemption. + Randomness *string `json:"recipient_rand,omitempty"` + } + var outgoing *webhookTicket + if ticket != nil { + outgoing = &webhookTicket{TicketJSON: ticket} + } + body, err := json.Marshal(struct { + Headers http.Header `json:"headers"` + Ticket *webhookTicket `json:"ticket"` + }{r.Header, outgoing}) + if err != nil { + return fail("redeemer auth payload encoding failed", observability.ErrorKind(err)) + } + request, err := http.NewRequestWithContext(r.Context(), http.MethodPost, a.endpoint, bytes.NewReader(body)) + if err != nil { + return fail("redeemer auth request construction failed", observability.ErrorKind(err)) + } + // Prevent transport retries, including with a configured idempotency key. + request.GetBody = nil + request.Header = a.headers.Clone() + if request.Header == nil { + request.Header = make(http.Header) + } + request.Header.Set("Content-Type", "application/json") + response, err := a.client.Do(request) + if err != nil { + return fail("redeemer auth webhook unavailable", observability.ErrorKind(err)) + } + defer response.Body.Close() + status = response.StatusCode + if status != http.StatusOK { + return fail("redeemer auth webhook failed", "http_status") + } + data, err := io.ReadAll(io.LimitReader(response.Body, (1<<20)+1)) + if err != nil { + return fail("invalid redeemer auth response", observability.ErrorKind(err)) + } + if len(data) > 1<<20 { + return fail("invalid redeemer auth response", "response_limit") + } + var decision struct { + Status int `json:"status"` + Reason string `json:"reason"` + } + if err := json.Unmarshal(data, &decision); err != nil { + return fail("invalid redeemer auth response", observability.ErrorKind(err)) + } + if decision.Status == http.StatusOK { + outcome = "success" + return true + } + if decision.Status < 400 || decision.Status > 599 { + return fail("invalid redeemer auth response", "invalid_decision") + } + outcome = "denied" + a.logger.WarnContext(r.Context(), "redeemer authorization rejected", "outcome", outcome, "error_kind", "policy_rejected", "decision_status", decision.Status) + deniedStatus := decision.Status + if deniedStatus == http.StatusConflict { + // Forwarders permanently reject 409 as conflicting ticket metadata. + deniedStatus = http.StatusForbidden + } + http.Error(w, fmt.Sprintf("redeemer auth rejected request with status %d", decision.Status), deniedStatus) + return false +} diff --git a/redemption/http.go b/redemption/http.go index f61be5d..ba083c9 100644 --- a/redemption/http.go +++ b/redemption/http.go @@ -1,7 +1,6 @@ // QueueTicket admission and ticket conversion adapt Nico Vergauwen's and // Yondon Fu's go-livepeer/server/redeemer.go at afaaa36e743d39e6f460e7993d516d52b3a81341. // HTTP transport, deployment binding and validation are implemented for Node. -// MIT notice: LICENSE. No max-float RPCs are included. package redemption import ( @@ -87,6 +86,12 @@ func decodeJSON(r io.Reader, value any) error { } func Handler(s *RedeemerDB) http.Handler { + return HandlerWithAuthorization(s, nil) +} + +// HandlerWithAuthorization calls authorize before validating or storing a ticket. +// The ticket is nil if decoding fails. authorize must write any failure response. +func HandlerWithAuthorization(s *RedeemerDB, authorize func(http.ResponseWriter, *http.Request, *TicketJSON) bool) http.Handler { mux := http.NewServeMux() slots := make(chan struct{}, 32) mux.HandleFunc("POST /tickets", func(w http.ResponseWriter, r *http.Request) { @@ -98,7 +103,15 @@ func Handler(s *RedeemerDB) http.Handler { return } var req TicketRequest - if err := decodeJSON(http.MaxBytesReader(w, r.Body, MaxTicketBody), &req); err != nil { + decodeErr := decodeJSON(http.MaxBytesReader(w, r.Body, MaxTicketBody), &req) + var ticket *TicketJSON + if decodeErr == nil { + ticket = &req.Ticket + } + if authorize != nil && !authorize(w, r, ticket) { + return + } + if decodeErr != nil { http.Error(w, "invalid ticket request", 400) return } @@ -168,7 +181,7 @@ func NewHTTPDestination(u *url.URL, policy destination.Policy, b Binding) (*HTTP if err != nil { return nil, err } - client := &http.Client{Transport: policy.Transport(10 * time.Second), Timeout: 10 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} + client := &http.Client{Transport: policy.Transport(15 * time.Second), Timeout: 20 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} return &HTTPDestination{endpoint, client, b}, nil } diff --git a/redemption/http_test.go b/redemption/http_test.go index 7d0361a..1880fc9 100644 --- a/redemption/http_test.go +++ b/redemption/http_test.go @@ -1,19 +1,24 @@ package redemption import ( + "bufio" "bytes" + "context" "crypto/ecdsa" "crypto/rand" "encoding/json" "errors" "io" "math/big" + "net" "net/http" "net/http/httptest" "net/url" "path/filepath" + "strings" "sync/atomic" "testing" + "testing/synctest" "time" "github.com/ethereum/go-ethereum/accounts" @@ -54,7 +59,8 @@ func boundStore(t *testing.T, role, target string, limits Limits) (*RedeemerDB, func TestHTTPAdmissionIdentityCapacityAndScope(t *testing.T) { s, b := boundStore(t, "redeemer", "chain", Limits{Pending: 2, PerRecipient: 1}) - handler := Handler(s) + calls := 0 + handler := HandlerWithAuthorization(s, func(http.ResponseWriter, *http.Request, *TicketJSON) bool { calls++; return true }) submit := func(req TicketRequest) *httptest.ResponseRecorder { raw, err := json.Marshal(req) require.NoError(t, err) @@ -92,7 +98,7 @@ func TestHTTPAdmissionIdentityCapacityAndScope(t *testing.T) { third, _ := winningTicket(t, 3) require.ErrorIs(t, s.Admit(t.Context(), third), ErrCapacity) bad := httptest.NewRecorder() - handler.ServeHTTP(bad, httptest.NewRequest("POST", "/tickets", bytes.NewReader(make([]byte, MaxTicketBody+1)))) + handler.ServeHTTP(bad, httptest.NewRequest("POST", "/tickets", strings.NewReader(strings.Repeat(" ", MaxTicketBody+1)))) require.Equal(t, 400, bad.Code) changed := b changed.Controller = common.HexToAddress("0x999") @@ -100,6 +106,83 @@ func TestHTTPAdmissionIdentityCapacityAndScope(t *testing.T) { changed = b changed.Destination = "http://elsewhere.example" require.ErrorContains(t, s.Bind(t.Context(), changed, DefaultLimits, nil), "drain") + require.Equal(t, 7, calls, "every submission, including duplicates and invalid tickets, is authorized") +} + +func TestHTTPAuthorizationConcurrency(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + release := make(chan struct{}) + defer close(release) + var calls atomic.Int32 + handler := HandlerWithAuthorization(nil, func(w http.ResponseWriter, r *http.Request, ticket *TicketJSON) bool { + require.Nil(t, ticket) + calls.Add(1) + <-release + http.Error(w, "denied", http.StatusForbidden) + return false + }) + for i := range 32 { + go func() { + body := "invalid" + if i == 0 { + body = strings.Repeat(" ", MaxTicketBody+1) + } + w := httptest.NewRecorder() + handler.ServeHTTP(w, httptest.NewRequest("POST", "/tickets", strings.NewReader(body))) + require.Equal(t, 403, w.Code) + }() + } + synctest.Wait() + require.EqualValues(t, 32, calls.Load()) + w := httptest.NewRecorder() + handler.ServeHTTP(w, httptest.NewRequest("POST", "/tickets", nil)) + require.Equal(t, 503, w.Code) + handler.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "/tickets", nil)) + require.EqualValues(t, 32, calls.Load(), "overloaded requests and other methods do not authorize") + }) +} + +func TestHTTPDestinationAuthorizationAndAdmissionBudgets(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + s, b := boundStore(t, "redeemer", "chain", DefaultLimits) + s.db.SetMaxOpenConns(1) + conn, err := s.db.Conn(t.Context()) + require.NoError(t, err) + defer conn.Close() + handler := HandlerWithAuthorization(s, func(http.ResponseWriter, *http.Request, *TicketJSON) bool { + time.Sleep(4 * time.Second) + time.AfterFunc(4*time.Second, func() { _ = conn.Close() }) + return true + }) + serverConn, clientConn := net.Pipe() + defer serverConn.Close() + defer clientConn.Close() + go func() { + defer serverConn.Close() + request, err := http.ReadRequest(bufio.NewReader(serverConn)) + require.NoError(t, err) + defer request.Body.Close() + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + require.NoError(t, response.Result().Write(serverConn)) + }() + policy, err := destination.New("redeemer", []string{"127.0.0.1"}) + require.NoError(t, err) + u, err := url.Parse("http://127.0.0.1") + require.NoError(t, err) + remote, err := NewHTTPDestination(u, policy, b) + require.NoError(t, err) + defer remote.Close() + require.Equal(t, 20*time.Second, remote.client.Timeout) + transport := remote.client.Transport.(*http.Transport) + require.Equal(t, 15*time.Second, transport.ResponseHeaderTimeout) + transport.Proxy = nil + transport.DialContext = func(context.Context, string, string) (net.Conn, error) { return clientConn, nil } + ticket, _ := winningTicket(t, 1) + started := time.Now() + require.NoError(t, remote.Submit(t.Context(), ticket)) + require.Equal(t, 8*time.Second, time.Since(started)) + }) } func signAlternativeTicket(t *testing.T, ticket *pm.SignedTicket, key *ecdsa.PrivateKey) { diff --git a/signer/headers.go b/signer/headers.go index c039782..2d72245 100644 --- a/signer/headers.go +++ b/signer/headers.go @@ -1,91 +1,6 @@ package signer -import ( - "bytes" - "encoding/csv" - "errors" - "io" - "maps" - "net/http" - "slices" - "strings" -) +import "github.com/livepeer/node/nodeconfig" -// Headers is a text scalar for configuration and secret files. It accepts -// comma-separated Header: value entries, with CSV quoting for values containing -// commas. Header names are case-insensitive and repeated names retain all values. -type Headers http.Header - -func validHeader(name, value string) bool { - if name == "" { - return false - } - for _, c := range name { - if !((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || strings.ContainsRune("!#$%&'*+-.^_`|~", c)) { - return false - } - } - for _, c := range value { - if (c < 32 && c != '\t') || c == 127 { - return false - } - } - return true -} - -func (h *Headers) UnmarshalText(text []byte) error { - parsed := make(http.Header) - input := strings.TrimSpace(string(text)) - if input != "" { - reader := csv.NewReader(strings.NewReader(input)) - reader.TrimLeadingSpace = true - reader.FieldsPerRecord = -1 - entries, err := reader.Read() - if err != nil { - return errors.New("invalid comma-separated webhook headers") - } - if _, err := reader.Read(); err != io.EOF { - return errors.New("webhook headers must be one comma-separated record") - } - for _, entry := range entries { - name, value, ok := strings.Cut(entry, ":") - name, value = strings.TrimSpace(name), strings.TrimSpace(value) - if !ok || !validHeader(name, value) { - return errors.New("expected webhook headers in Header: value form") - } - parsed.Add(name, value) - } - } - *h = Headers(parsed) - return nil -} - -// MarshalText sorts names for deterministic secret-file comparisons and auth -// policy fingerprints, and quotes entries containing commas or quotation marks. -func (h Headers) MarshalText() ([]byte, error) { - var entries []string - for _, name := range slices.Sorted(maps.Keys(h)) { - if !validHeader(name, "") { - return nil, errors.New("invalid webhook header name") - } - for _, value := range h[name] { - if !validHeader(name, value) { - return nil, errors.New("invalid webhook header name or value") - } - entries = append(entries, name+": "+value) - } - } - if len(entries) == 0 { - return nil, nil - } - var buffer bytes.Buffer - writer := csv.NewWriter(&buffer) - if err := writer.Write(entries); err != nil { - return nil, err - } - writer.Flush() - if err := writer.Error(); err != nil { - return nil, err - } - return bytes.TrimSuffix(buffer.Bytes(), []byte("\n")), nil -} +// Headers parses webhook headers from configuration and secret files. +type Headers = nodeconfig.Headers