From adbf515c23069b0cf8c2d5a0200c2f2cedae7068 Mon Sep 17 00:00:00 2001 From: Josh Allmann Date: Thu, 12 Nov 2026 08:36:43 -0800 Subject: [PATCH 1/5] Enforce orchestrator API authority without proxy templates --- .../testdata/cli/orchestrator-help.golden | 2 +- configs/orchestrator/config.example.toml | 2 +- docs/orchestrator.md | 10 +-- orchestrator/app.go | 2 +- orchestrator/app_test.go | 83 ++++++++++--------- orchestrator/discovery_test.go | 3 +- orchestrator/http.go | 4 +- orchestrator/http_test.go | 48 +++++++---- orchestrator/lifecycle_test.go | 6 +- orchestrator/metrics_test.go | 8 +- orchestrator/payment_recipient_test.go | 2 +- orchestrator/payment_test.go | 17 ++-- orchestrator/proxy_test.go | 47 +++++++---- orchestrator/python_controls_test.go | 1 + orchestrator/python_media_test.go | 1 + orchestrator/regression_test.go | 14 ++-- orchestrator/runner_address_test.go | 12 +-- orchestrator/seq_test.go | 2 +- orchestrator/trickle_test.go | 20 ++--- 19 files changed, 164 insertions(+), 120 deletions(-) diff --git a/cmd/livepeer/testdata/cli/orchestrator-help.golden b/cmd/livepeer/testdata/cli/orchestrator-help.golden index 81f469b..28a1337 100644 --- a/cmd/livepeer/testdata/cli/orchestrator-help.golden +++ b/cmd/livepeer/testdata/cli/orchestrator-help.golden @@ -21,7 +21,7 @@ Flags: -l, --listen string Public HTTP listener (env: LIVEPEER_ORCHESTRATOR_LISTEN) (default "127.0.0.1:8935") -m, --metrics-listen string Metrics listener (env: LIVEPEER_ORCHESTRATOR_METRICS_LISTEN) (default "127.0.0.1:8936") -s, --service-url string Public orchestrator base URL; defaults to http://listen (env: LIVEPEER_ORCHESTRATOR_SERVICE_URL) - --api-origin-url string API request Host as a URL when proxy-url-template is set; defaults to service-url (env: LIVEPEER_ORCHESTRATOR_API_ORIGIN_URL) + --api-origin-url string Required API request Host as a URL; defaults to service-url (env: LIVEPEER_ORCHESTRATOR_API_ORIGIN_URL) -r, --runner-service-url string Runner-facing base URL for callbacks and trickle; defaults to service-url (env: LIVEPEER_ORCHESTRATOR_RUNNER_SERVICE_URL) --proxy-url-template string Generated proxy URL with {proxy} in a hostname label or final path segment (env: LIVEPEER_ORCHESTRATOR_PROXY_URL_TEMPLATE) --excluded-headers strings Headers to remove from app requests; a trailing * matches a prefix (env: LIVEPEER_ORCHESTRATOR_EXCLUDED_HEADERS) diff --git a/configs/orchestrator/config.example.toml b/configs/orchestrator/config.example.toml index cd93d01..a239ac8 100644 --- a/configs/orchestrator/config.example.toml +++ b/configs/orchestrator/config.example.toml @@ -3,7 +3,7 @@ Listen = "127.0.0.1:8935" MetricsListen = "127.0.0.1:8936" ServiceURL = "http://127.0.0.1:8935" -# APIOriginURL = "http://orchestrator.internal:8935" # Host sent by the reverse proxy when using ProxyURLTemplate; defaults to ServiceURL. +# APIOriginURL = "http://orchestrator.internal:8935" # Required API Host; defaults to ServiceURL, independent of ProxyURLTemplate. # RunnerServiceURL = "http://127.0.0.1:8935" # Runner-facing callbacks and trickle; defaults to ServiceURL. BootstrapSecretFile = "/run/secrets/livepeer-runner-bootstrap" RunnerConfig = "/etc/livepeer/runners.toml" diff --git a/docs/orchestrator.md b/docs/orchestrator.md index 587f10d..2cc2529 100644 --- a/docs/orchestrator.md +++ b/docs/orchestrator.md @@ -103,11 +103,11 @@ DNS, TLS, and forwarding to this listener for those URLs. Hostname routing uses the actual Host header. Single-shot domain proxies require an app registration ID that is a lowercase DNS label of at most 63 characters. -With `ProxyURLTemplate`, API requests must use the Host from `APIOriginURL`, -which defaults to `ServiceURL`. If a reverse proxy changes that Host, set -`APIOriginURL` to an HTTP(S) URL with the Host it sends, without a path. Runner -requests must use the same Host; configure `RunnerServiceURL` or the reverse proxy -accordingly. `APIOriginURL` does not change advertised URLs. +API requests require the `APIOriginURL` Host (default: `ServiceURL`), even +without `ProxyURLTemplate`. Set it to the HTTP(S) origin a reverse proxy +forwards, without a path. Runner, trickle, app, and fallback proxy routes share +this Host; configure `RunnerServiceURL` or the proxy accordingly. Template-matched +proxies use their template Host. Advertised URLs are unchanged. To remove headers from app requests, set `ExcludedHeaders`, for example: `["CF-Access-*", "Cookie"]`. Matching is case-insensitive. diff --git a/orchestrator/app.go b/orchestrator/app.go index 62774a4..c1f2118 100644 --- a/orchestrator/app.go +++ b/orchestrator/app.go @@ -41,7 +41,7 @@ type Params struct { Listen boa.HTTPBind `default:"127.0.0.1:8935" descr:"Public HTTP listener"` MetricsListen boa.HTTPBind `default:"127.0.0.1:8936" descr:"Metrics listener"` ServiceURL boa.Text[*url.URL] `optional:"true" descr:"Public orchestrator base URL; defaults to http://listen"` - APIOriginURL boa.Text[*url.URL] `optional:"true" descr:"API request Host as a URL when proxy-url-template is set; defaults to service-url"` + APIOriginURL boa.Text[*url.URL] `optional:"true" descr:"Required API request Host as a URL; defaults to service-url"` RunnerServiceURL boa.Text[*url.URL] `optional:"true" descr:"Runner-facing base URL for callbacks and trickle; defaults to service-url"` ProxyURLTemplate string `optional:"true" descr:"Generated proxy URL with {proxy} in a hostname label or final path segment"` ExcludedHeaders []string `optional:"true" descr:"Headers to remove from app requests; a trailing * matches a prefix"` diff --git a/orchestrator/app_test.go b/orchestrator/app_test.go index a10d440..a012a0a 100644 --- a/orchestrator/app_test.go +++ b/orchestrator/app_test.go @@ -55,44 +55,50 @@ func freeTCPPort(t *testing.T) int { } func TestAPIOriginURLRouting(t *testing.T) { - address := fmt.Sprintf("127.0.0.1:%d", freeTCPPort(t)) - metrics := fmt.Sprintf("127.0.0.1:%d", freeTCPPort(t)) - const origin = "orchestrator.internal" - t.Setenv("LIVEPEER_ORCHESTRATOR_BOOTSTRAP_SECRET", "bootstrap") - t.Setenv("LIVEPEER_ORCHESTRATOR_API_ORIGIN_URL", "http://wrong.internal:8935") - cmd := Root(io.Discard, io.Discard) - cmd.SetArgs([]string{"--config", "", "--listen", address, "--metrics-listen", metrics, - "--service-url", "https://public.example", - "--api-origin-url", "http://" + origin + ":80", "--runner-service-url", "http://" + origin, - "--proxy-url-template", "https://{proxy}.internal"}) - ctx, cancel := context.WithCancel(t.Context()) - done := make(chan error, 1) - go func() { done <- cmd.ExecuteContext(ctx) }() - t.Cleanup(func() { - cancel() - select { - case err := <-done: - require.NoError(t, err) - case <-time.After(5 * time.Second): - t.Fatal("orchestrator did not stop") - } - }) - client := &http.Client{Timeout: time.Second, Transport: &http.Transport{DisableKeepAlives: true}} - request := func(host string) int { - req, err := http.NewRequest(http.MethodGet, "http://"+address+"/discovery", nil) - require.NoError(t, err) - req.Host = host - response, err := client.Do(req) - if err != nil { - return 0 - } - defer response.Body.Close() - return response.StatusCode - } - require.Eventually(t, func() bool { return request(origin) == http.StatusOK }, 5*time.Second, 10*time.Millisecond) - require.Equal(t, http.StatusOK, request(origin+":80")) - for _, host := range []string{origin + ":443", "public.example", "wrong.internal:8935", "other.example"} { - require.Equal(t, http.StatusNotFound, request(host), host) + for _, template := range []string{"", "https://{proxy}.internal", "https://public.example/custom/{proxy}"} { + t.Run(template, func(t *testing.T) { + address := fmt.Sprintf("127.0.0.1:%d", freeTCPPort(t)) + metrics := fmt.Sprintf("127.0.0.1:%d", freeTCPPort(t)) + const origin = "orchestrator.internal" + t.Setenv("LIVEPEER_ORCHESTRATOR_BOOTSTRAP_SECRET", "bootstrap") + t.Setenv("LIVEPEER_ORCHESTRATOR_API_ORIGIN_URL", "http://wrong.internal:8935") + cmd := Root(io.Discard, io.Discard) + cmd.SetArgs([]string{"--config", "", "--listen", address, "--metrics-listen", metrics, + "--service-url", "https://public.example", + "--api-origin-url", "http://" + origin + ":80", "--runner-service-url", "http://" + origin, + "--proxy-url-template", template}) + ctx, cancel := context.WithCancel(t.Context()) + done := make(chan error, 1) + go func() { done <- cmd.ExecuteContext(ctx) }() + t.Cleanup(func() { + cancel() + select { + case err := <-done: + require.NoError(t, err) + case <-time.After(5 * time.Second): + t.Fatal("orchestrator did not stop") + } + }) + client := &http.Client{Timeout: time.Second, Transport: &http.Transport{DisableKeepAlives: true}} + request := func(host string) int { + req, err := http.NewRequest(http.MethodGet, "http://"+address+"/discovery", nil) + require.NoError(t, err) + req.Host = host + req.Header.Set("X-Forwarded-Host", origin) + req.Header.Set("Forwarded", "host="+origin) + response, err := client.Do(req) + if err != nil { + return 0 + } + defer response.Body.Close() + return response.StatusCode + } + require.Eventually(t, func() bool { return request(origin) == http.StatusOK }, 5*time.Second, 10*time.Millisecond) + require.Equal(t, http.StatusOK, request(origin+":80")) + for _, host := range []string{origin + ":443", "public.example", "wrong.internal:8935", "other.example"} { + require.Equal(t, http.StatusNotFound, request(host), host) + } + }) } } @@ -146,6 +152,7 @@ func TestPaidHTTPStartup(t *testing.T) { require.NoError(t, p.MetricsListen.UnmarshalText(fmt.Appendf(nil, "unsafe@0.0.0.0:%d", metricsPort))) metricsURL := fmt.Sprintf("http://127.0.0.1:%d", metricsPort) p.ServiceURL = boa.Text[*url.URL]{Value: mustURL(t, "https://public.example/external")} + p.APIOriginURL = boa.Text[*url.URL]{Value: mustURL(t, address)} p.RunnerConfig = filepath.Join(t.TempDir(), "runners.toml") require.NoError(t, os.WriteFile(p.RunnerConfig, []byte("[[Runners]]\nID = 'r'\nRunnerURL = 'https://runner.example'\nApp = 'test'\n[Runners.PriceInfo]\nPrice = 1\n"), 0600)) ctx, cancel := context.WithCancel(t.Context()) diff --git a/orchestrator/discovery_test.go b/orchestrator/discovery_test.go index c80c718..f3ae100 100644 --- a/orchestrator/discovery_test.go +++ b/orchestrator/discovery_test.go @@ -7,7 +7,6 @@ import ( "log/slog" "math/big" "net/http" - "net/http/httptest" "testing" "time" @@ -38,7 +37,7 @@ func TestDiscoveryLocalInventory(t *testing.T) { require.NoError(t, err) s := NewServer(registry, policy, policy, slog.New(slog.NewTextHandler(io.Discard, nil))) t.Cleanup(s.Close) - server := httptest.NewServer(s) + server := newTestHTTPServer(t, s) t.Cleanup(server.Close) response, err := server.Client().Get(server.URL + "/deployment/discovery") require.NoError(t, err) diff --git a/orchestrator/http.go b/orchestrator/http.go index 45994d4..6589c20 100644 --- a/orchestrator/http.go +++ b/orchestrator/http.go @@ -100,8 +100,8 @@ func (s *Server) serveHTTP(w http.ResponseWriter, r *http.Request) { } proxyID, appPath := s.registry.matchProxy(r, s.apiOrigin) if proxyID == "" { - // Unknown proxy Hosts must never fall through to the API router. - if s.registry.proxyTemplate != "" && !matchesOrigin(r.Host, s.apiOrigin) { + // Only the API Host may reach the API router. + if !matchesOrigin(r.Host, s.apiOrigin) { fail(w, http.StatusNotFound, "route not found") return } diff --git a/orchestrator/http_test.go b/orchestrator/http_test.go index 37c2c10..c955cb3 100644 --- a/orchestrator/http_test.go +++ b/orchestrator/http_test.go @@ -25,6 +25,20 @@ import ( "github.com/stretchr/testify/require" ) +func apiRequest(server *Server, method, target string, body io.Reader) *http.Request { + req := httptest.NewRequest(method, target, body) + req.Host = server.apiOrigin.Host + return req +} + +func newTestHTTPServer(t *testing.T, app *Server) *httptest.Server { + t.Helper() + server := httptest.NewUnstartedServer(app) + app.apiOrigin = mustURL(t, "http://"+server.Listener.Addr().String()) + server.Start() + return server +} + func TestDynamicRunnerSessionProxyAndCapacity(t *testing.T) { upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { require.Equal(t, "/api/hello", r.URL.Path) @@ -37,7 +51,7 @@ func TestDynamicRunnerSessionProxyAndCapacity(t *testing.T) { policy, err := destination.New("runner", []string{"127.0.0.0/8"}) require.NoError(t, err) registry := NewRegistry("bootstrap", "http://orchestrator.example", 10*time.Millisecond, 50*time.Millisecond) - srv := httptest.NewServer(NewServer(registry, policy, policy, slog.New(slog.NewTextHandler(io.Discard, nil)))) + srv := newTestHTTPServer(t, NewServer(registry, policy, policy, slog.New(slog.NewTextHandler(io.Discard, nil)))) defer srv.Close() body := `{"runner_url":"` + upstream.URL + `/api","app":"test","mode":"persistent","capacity":1,"price_info":{"price":0,"currency":"usd","unit":"hour"}}` @@ -121,7 +135,7 @@ func TestSessionCallbackCredentials(t *testing.T) { } { for name, credential := range map[string]string{"missing": "", "wrong": "wrong", "cross-session": otherToken} { t.Run(operation.method+" "+operation.path+"/"+name, func(t *testing.T) { - req := httptest.NewRequest(operation.method, "/runner/r/session/"+sid+"/"+operation.path, strings.NewReader(`{}`)) + req := apiRequest(server, operation.method, "/runner/r/session/"+sid+"/"+operation.path, strings.NewReader(`{}`)) if credential != "" { req.Header.Set("Livepeer-Session-Token", credential) } @@ -135,7 +149,7 @@ func TestSessionCallbackCredentials(t *testing.T) { _, err := registry.addProxy("r", sid, token, target) require.NoError(t, err) } - req := httptest.NewRequest(http.MethodPost, "/runner/r/session/"+sid+"/proxy", strings.NewReader(`{}`)) + req := apiRequest(server, http.MethodPost, "/runner/r/session/"+sid+"/proxy", strings.NewReader(`{}`)) req.Header.Set("Livepeer-Session-Token", token) w := httptest.NewRecorder() server.ServeHTTP(w, req) @@ -277,7 +291,7 @@ func TestSessionProxyStreamsSSEAndWebSocket(t *testing.T) { app := NewServer(registry, policy, policy, slog.New(slog.NewTextHandler(io.Discard, nil))) app.setExcludedHeaders([]string{"CF-Access-*", "Cookie"}) defer app.Close() - server := httptest.NewServer(app) + server := newTestHTTPServer(t, app) defer server.Close() sid, _, _, _, err := registry.Reserve(registered.RunnerID) require.NoError(t, err) @@ -320,7 +334,7 @@ func TestRunnerDestinationDeniedWithoutGrant(t *testing.T) { policy, err := destination.New("runner", nil) require.NoError(t, err) registry := NewRegistry("bootstrap", "http://orchestrator.example", time.Second, time.Minute) - server := httptest.NewServer(NewServer(registry, policy, policy, slog.New(slog.NewTextHandler(io.Discard, nil)))) + server := newTestHTTPServer(t, NewServer(registry, policy, policy, slog.New(slog.NewTextHandler(io.Discard, nil)))) defer server.Close() body := `{"runner_url":"` + upstream.URL + `","app":"test","mode":"single-shot","capacity":1,"price_info":{"price":0}}` request, err := http.NewRequest(http.MethodPost, server.URL+"/runners/heartbeat", bytes.NewBufferString(body)) @@ -363,12 +377,12 @@ func TestGeneratedProxyHasSeparateDestinationGrant(t *testing.T) { require.Equal(t, http.StatusOK, status) srv := NewServer(registry, runnerPolicy, proxyPolicy, slog.New(slog.NewTextHandler(io.Discard, nil))) path := "/runner/" + registered.RunnerID + "/session/" + sid + "/proxy" - create := httptest.NewRequest(http.MethodPost, path, bytes.NewBufferString(`{"target_url":"`+upstream.URL+`"}`)) + create := apiRequest(srv, http.MethodPost, path, bytes.NewBufferString(`{"target_url":"`+upstream.URL+`"}`)) create.Header.Set("Livepeer-Session-Token", "bad") record := httptest.NewRecorder() srv.ServeHTTP(record, create) require.Equal(t, http.StatusUnauthorized, record.Code) - create = httptest.NewRequest(http.MethodPost, path, bytes.NewBufferString(`{"target_url":"`+upstream.URL+`"}`)) + create = apiRequest(srv, http.MethodPost, path, bytes.NewBufferString(`{"target_url":"`+upstream.URL+`"}`)) create.Header.Set("Livepeer-Session-Token", token) record = httptest.NewRecorder() srv.ServeHTTP(record, create) @@ -381,18 +395,18 @@ func TestGeneratedProxyHasSeparateDestinationGrant(t *testing.T) { require.NotEmpty(t, created.ProxyID) require.Equal(t, "http://orchestrator.example/run/"+created.ProxyID, created.URL) record = httptest.NewRecorder() - srv.ServeHTTP(record, httptest.NewRequest(http.MethodGet, "/run/"+created.ProxyID, nil)) + srv.ServeHTTP(record, apiRequest(srv, http.MethodGet, "/run/"+created.ProxyID, nil)) require.Equal(t, http.StatusBadGateway, record.Code) granted, err := destination.New("session-proxy", []string{grant}) require.NoError(t, err) srv = NewServer(registry, runnerPolicy, granted, slog.New(slog.NewTextHandler(io.Discard, nil))) record = httptest.NewRecorder() - srv.ServeHTTP(record, httptest.NewRequest(http.MethodGet, "/run/"+created.ProxyID, nil)) + srv.ServeHTTP(record, apiRequest(srv, http.MethodGet, "/run/"+created.ProxyID, nil)) require.Equal(t, http.StatusOK, record.Code) record = httptest.NewRecorder() - srv.ServeHTTP(record, httptest.NewRequest(http.MethodGet, "/proxy/"+created.ProxyID, nil)) + srv.ServeHTTP(record, apiRequest(srv, http.MethodGet, "/proxy/"+created.ProxyID, nil)) require.Equal(t, http.StatusOK, record.Code) - create = httptest.NewRequest("POST", "/runner/"+registered.RunnerID+"/session/"+sid+"/channels", strings.NewReader(`{"channels":[{"name":"events"}]}`)) + create = apiRequest(srv, "POST", "/runner/"+registered.RunnerID+"/session/"+sid+"/channels", strings.NewReader(`{"channels":[{"name":"events"}]}`)) create.Header.Set("Livepeer-Session-Token", token) record = httptest.NewRecorder() srv.ServeHTTP(record, create) @@ -438,7 +452,7 @@ func TestTrickleChannelsScopedToSession(t *testing.T) { _, token2, _, err := registry.sessionTarget(registered.RunnerID, sid2) require.NoError(t, err) path1 := "/runner/" + registered.RunnerID + "/session/" + sid1 + "/channels" - create := httptest.NewRequest(http.MethodPost, path1, bytes.NewBufferString(`{"channels":[{"name":"events","mime_type":"application/json"}]}`)) + create := apiRequest(srv, http.MethodPost, path1, bytes.NewBufferString(`{"channels":[{"name":"events","mime_type":"application/json"}]}`)) create.Header.Set("Livepeer-Session-Token", token1) record := httptest.NewRecorder() srv.ServeHTTP(record, create) @@ -452,7 +466,7 @@ func TestTrickleChannelsScopedToSession(t *testing.T) { require.Len(t, created.Channels, 1) channelID := created.Channels[0].ChannelName path2 := "/runner/" + registered.RunnerID + "/session/" + sid2 + "/channels" - deletion := httptest.NewRequest(http.MethodDelete, path2, bytes.NewBufferString(`{"channels":["`+channelID+`"]}`)) + deletion := apiRequest(srv, http.MethodDelete, path2, bytes.NewBufferString(`{"channels":["`+channelID+`"]}`)) deletion.Header.Set("Livepeer-Session-Token", token2) record = httptest.NewRecorder() srv.ServeHTTP(record, deletion) @@ -460,19 +474,19 @@ func TestTrickleChannelsScopedToSession(t *testing.T) { require.JSONEq(t, `{"deleted":[]}`, record.Body.String()) segment := "/ai/trickle/" + channelID + "/0" record = httptest.NewRecorder() - srv.ServeHTTP(record, httptest.NewRequest(http.MethodPost, segment, bytes.NewBufferString(`{"x":1}`))) + srv.ServeHTTP(record, apiRequest(srv, http.MethodPost, segment, bytes.NewBufferString(`{"x":1}`))) require.Equal(t, http.StatusOK, record.Code) record = httptest.NewRecorder() - srv.ServeHTTP(record, httptest.NewRequest(http.MethodGet, segment, nil)) + srv.ServeHTTP(record, apiRequest(srv, http.MethodGet, segment, nil)) require.Equal(t, http.StatusOK, record.Code) require.JSONEq(t, `{"x":1}`, record.Body.String()) - deletion = httptest.NewRequest(http.MethodDelete, path1, bytes.NewBufferString(`{"channels":["`+channelID+`"]}`)) + deletion = apiRequest(srv, http.MethodDelete, path1, bytes.NewBufferString(`{"channels":["`+channelID+`"]}`)) deletion.Header.Set("Livepeer-Session-Token", token1) record = httptest.NewRecorder() srv.ServeHTTP(record, deletion) require.Equal(t, http.StatusOK, record.Code) record = httptest.NewRecorder() - srv.ServeHTTP(record, httptest.NewRequest(http.MethodGet, segment, nil)) + srv.ServeHTTP(record, apiRequest(srv, http.MethodGet, segment, nil)) require.Equal(t, http.StatusNotFound, record.Code) } diff --git a/orchestrator/lifecycle_test.go b/orchestrator/lifecycle_test.go index 02ec1a9..af8ecfb 100644 --- a/orchestrator/lifecycle_test.go +++ b/orchestrator/lifecycle_test.go @@ -67,7 +67,7 @@ func TestRunnerStopDrainsProxy(t *testing.T) { if scenario.name == "client stop" { prefix = "/apps/" } - stop := httptest.NewRequest(http.MethodPost, prefix+"r/session/"+sid+"/stop", nil) + stop := apiRequest(server, http.MethodPost, prefix+"r/session/"+sid+"/stop", nil) stop.Header.Set("Livepeer-Session-Token", token) stopped := httptest.NewRecorder() released := time.Now() @@ -131,7 +131,7 @@ func TestReleaseCancelsActiveProxy(t *testing.T) { require.NoError(t, registry.AddStatic(StaticRunner{ID: "runner", RunnerURL: upstream.URL, App: "test", Mode: mode, Capacity: 1})) } app := NewServer(registry, policy, policy, slog.New(slog.NewTextHandler(io.Discard, nil))) - server := httptest.NewServer(app) + server := newTestHTTPServer(t, app) defer server.Close() defer app.Close() path := "/apps/runner/app" @@ -362,7 +362,7 @@ func TestMultiAppRemovalKeepsStreams(t *testing.T) { })) defer upstream.Close() registry, appServer := reviewServer(t, upstream.URL) - server := httptest.NewServer(appServer) + server := newTestHTTPServer(t, appServer) defer server.Close() defer appServer.Close() a, b := registeredApp("a"), registeredApp("b") diff --git a/orchestrator/metrics_test.go b/orchestrator/metrics_test.go index c6da9e3..cad6dcc 100644 --- a/orchestrator/metrics_test.go +++ b/orchestrator/metrics_test.go @@ -198,7 +198,7 @@ func TestUpgradePreservesHalfClose(t *testing.T) { registry, app := reviewServer(t, upstream.URL) defer app.Close() require.NoError(t, registry.AddStatic(StaticRunner{ID: "runner", App: "model", Mode: "single-shot", RunnerURL: upstream.URL, Capacity: 1})) - server := httptest.NewServer(app) + server := newTestHTTPServer(t, app) defer server.Close() u, err := url.Parse(server.URL) require.NoError(t, err) @@ -253,7 +253,7 @@ func TestApplicationMetricsAndInventory(t *testing.T) { } for _, path := range []string{"/apps/runner/app", "/run/runner", "/proxy/runner"} { response := httptest.NewRecorder() - app.ServeHTTP(response, httptest.NewRequest("POST", path, nil)) + app.ServeHTTP(response, apiRequest(app, "POST", path, nil)) require.Equal(t, 503, response.Code) } require.EqualValues(t, 3, test.MetricSum(t, registry.metrics.Registry, "app_calls_total", map[string]string{"app": "model", "outcome": "error"})) @@ -263,7 +263,7 @@ func TestApplicationMetricsAndInventory(t *testing.T) { for range cap(app.streamSlots) { app.streamSlots <- struct{}{} } - app.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("POST", "/run/runner", nil)) + app.ServeHTTP(httptest.NewRecorder(), apiRequest(app, "POST", "/run/runner", nil)) for range cap(app.streamSlots) { <-app.streamSlots } @@ -366,7 +366,7 @@ func TestUpstreamFailureAfterHeaders(t *testing.T) { defer app.Close() require.NoError(t, registry.AddStatic(StaticRunner{ID: "runner", App: "model", Mode: "single-shot", RunnerURL: upstream.URL, Capacity: 1})) response := httptest.NewRecorder() - app.ServeHTTP(response, httptest.NewRequest("GET", "/apps/runner/app", nil)) + app.ServeHTTP(response, apiRequest(app, "GET", "/apps/runner/app", nil)) require.Equal(t, 200, response.Code, "the upstream status was already forwarded") require.EqualValues(t, 1, test.MetricSum(t, registry.metrics.Registry, "app_calls_total", map[string]string{"outcome": "stream_error"})) require.EqualValues(t, 1, test.MetricSum(t, registry.metrics.Registry, "stream_errors_total", nil)) diff --git a/orchestrator/payment_recipient_test.go b/orchestrator/payment_recipient_test.go index 185f091..86e2689 100644 --- a/orchestrator/payment_recipient_test.go +++ b/orchestrator/payment_recipient_test.go @@ -471,7 +471,7 @@ func TestPaidFailureOutcomes(t *testing.T) { if cause == "capacity" || cause == "queue capacity" { status = 503 } - request := httptest.NewRequest("POST", "/apps/runner/app", nil) + request := apiRequest(server, "POST", "/apps/runner/app", nil) if operation == "receive" { payment, segment := recipientPayment(t, key, info, nonces...) if cause == "unhashed segment" { diff --git a/orchestrator/payment_test.go b/orchestrator/payment_test.go index bea0af4..3d93008 100644 --- a/orchestrator/payment_test.go +++ b/orchestrator/payment_test.go @@ -83,7 +83,7 @@ func TestPaidFixedSessionWithRemoteSigner(t *testing.T) { defer signerService.Close() signerService.SetPaymentChain(paymentTestChain{}) reserveURL := "/apps/runner1/session" - first := httptest.NewRequest(http.MethodPost, reserveURL, nil) + first := apiRequest(server, http.MethodPost, reserveURL, nil) first.Header.Set("Livepeer-Payer-Address", payer.Address().Hex()) w := httptest.NewRecorder() server.ServeHTTP(w, first) @@ -106,7 +106,7 @@ func TestPaidFixedSessionWithRemoteSigner(t *testing.T) { // changes the advertised price before the reservation is paid. _, _, err = registry.Heartbeat(heartbeatRequest{RunnerID: "runner1", RunnerURL: "http://127.0.0.1:9000", App: "demo", Mode: "persistent", Capacity: 1, PriceInfo: priceInfo{Price: "2", Currency: "usd", Unit: "fixed"}}, resp.HeartbeatSecret) require.NoError(t, err) - paid := httptest.NewRequest(http.MethodPost, reserveURL, nil) + paid := apiRequest(server, http.MethodPost, reserveURL, nil) paid.Header.Set("Livepeer-Payment", payment.Payment) paid.Header.Set("Livepeer-Segment", payment.SegCreds) occupied, _, _, _, err := registry.Reserve("runner1") @@ -185,6 +185,7 @@ func TestPinnedPythonPaidSingleShot(t *testing.T) { orch := httptest.NewUnstartedServer(server) defer orch.Close() registry.service = "http://" + orch.Listener.Addr().String() + server.apiOrigin = mustURL(t, registry.service) orch.Start() signerService, err := signer.NewService(payer) require.NoError(t, err) @@ -216,6 +217,7 @@ func TestPinnedPythonSignerDiscovery(t *testing.T) { orch := httptest.NewUnstartedServer(server) defer orch.Close() registry.service = "http://" + orch.Listener.Addr().String() + server.apiOrigin = mustURL(t, registry.service) orch.Start() signerService, err := signer.NewService(paymentTestKey(t)) require.NoError(t, err) @@ -303,6 +305,7 @@ func TestPinnedPythonSustainedPaymentRefresh(t *testing.T) { app.ServeHTTP(w, r) })) registry.service = "http://" + orch.Listener.Addr().String() + app.apiOrigin = mustURL(t, registry.service) orch.Start() defer orch.Close() service, err := signer.NewService(payer) @@ -350,14 +353,14 @@ func TestMultiAppPaidIsolationAndDrain(t *testing.T) { owner := appHeartbeat(t, registry, "owner", "bootstrap", a, b, busy) for _, path := range []string{"/apps/live/session", "/apps/fixed/app"} { w := httptest.NewRecorder() - server.ServeHTTP(w, httptest.NewRequest(http.MethodPost, path, nil)) + server.ServeHTTP(w, apiRequest(server, http.MethodPost, path, nil)) require.Equal(t, http.StatusNotFound, w.Code, w.Body.String()) } require.Empty(t, registry.sessions) appHeartbeat(t, registry, owner.RunnerID, owner.HeartbeatSecret, a, b, busy) payer := crypto.PubkeyToAddress(payerKey.PublicKey) request := func(method, path, payment, segment string) *httptest.ResponseRecorder { - r := httptest.NewRequest(method, path, nil) + r := apiRequest(server, method, path, nil) r.Header.Set("Livepeer-Payer-Address", payer.Hex()) r.Header.Set("Livepeer-Payment", payment) r.Header.Set("Livepeer-Segment", segment) @@ -379,7 +382,7 @@ func TestMultiAppPaidIsolationAndDrain(t *testing.T) { sessions := len(engine.sessions) operations := test.MetricSum(t, registry.metrics.Registry, "payment_operations_total", nil) for _, address := range []string{payer.Hex(), ""} { - r := httptest.NewRequest(method, path, nil) + r := apiRequest(server, method, path, nil) r.Header.Set("Livepeer-Payer-Address", address) w := httptest.NewRecorder() server.ServeHTTP(w, r) @@ -459,7 +462,7 @@ func TestMultiAppPaidIsolationAndDrain(t *testing.T) { // Follow-up payment and refresh remain available for the draining live app. refreshBody := mustJSON(t, map[string]string{"sender": payer.Hex(), "manifest_id": live.ManifestID}) w = httptest.NewRecorder() - server.ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/refresh-payment", bytes.NewReader(refreshBody))) + server.ServeHTTP(w, apiRequest(server, http.MethodPost, "/refresh-payment", bytes.NewReader(refreshBody))) require.Equal(t, 200, w.Code, w.Body.String()) next, segment := signed(live, 4, 5, 6) w = request(http.MethodPost, live.PaymentURL, next, segment) @@ -497,7 +500,7 @@ func TestAppIDReuseRejectsPreviousRegistrationQuotes(t *testing.T) { owner := appHeartbeat(t, registry, "old-owner", "bootstrap", a) appHeartbeat(t, registry, owner.RunnerID, owner.HeartbeatSecret, a) request := func(payment, segment string) *httptest.ResponseRecorder { - req := httptest.NewRequest(http.MethodPost, "/apps/reused/session", nil) + req := apiRequest(server, http.MethodPost, "/apps/reused/session", nil) req.Header.Set("Livepeer-Payer-Address", crypto.PubkeyToAddress(key.PublicKey).Hex()) req.Header.Set("Livepeer-Payment", payment) req.Header.Set("Livepeer-Segment", segment) diff --git a/orchestrator/proxy_test.go b/orchestrator/proxy_test.go index 7cca937..f8a22a4 100644 --- a/orchestrator/proxy_test.go +++ b/orchestrator/proxy_test.go @@ -143,7 +143,6 @@ func TestOriginDefaultPorts(t *testing.T) { policy, err := destination.New("runner", nil) require.NoError(t, err) registry := NewRegistry("bootstrap", tc.origin, time.Second, time.Minute) - registry.proxyTemplate = "https://{proxy}.example" server := NewServer(registry, policy, policy, slog.New(slog.NewTextHandler(io.Discard, nil))) defer server.Close() for _, request := range []struct { @@ -191,35 +190,55 @@ func TestProxyHostCannotReachAPI(t *testing.T) { request := func(method, target string) *httptest.ResponseRecorder { t.Helper() req := httptest.NewRequest(method, target, nil) - req.Header.Set("X-Forwarded-Host", "api-aaaaaaa.machine.example") + req.Header.Set("X-Forwarded-Host", server.apiOrigin.Host) + req.Header.Set("Forwarded", "host="+server.apiOrigin.Host) w := httptest.NewRecorder() server.ServeHTTP(w, req) return w } - if template != "" { - for _, host := range []string{ - "app-echo.machine.example:8443", - "app-echo.machine.example:443", - "app-echo.machine.example.", - "nested.app-echo.machine.example", - "other.example", - } { - w := request(http.MethodPost, "http://"+host+"/apps/app-fixture/session") - require.Equal(t, http.StatusNotFound, w.Code, host+": "+w.Body.String()) - } + for _, host := range []string{ + "app-echo.machine.example:8443", + "app-echo.machine.example:443", + "app-echo.machine.example.", + "nested.app-echo.machine.example", + "other.example", + } { + w := request(http.MethodPost, "http://"+host+"/apps/app-fixture/session") + require.Equal(t, http.StatusNotFound, w.Code, host+": "+w.Body.String()) } for _, origin := range []string{registry.service, registry.runnerService, "http://api-private.machine.example"} { status := http.StatusNotFound u, err := url.Parse(origin) require.NoError(t, err) - if template == "" || u.Host == server.apiOrigin.Host { + if u.Host == server.apiOrigin.Host { status = http.StatusOK } w := request(http.MethodPost, origin+"/apps/app-fixture/session") require.Equal(t, status, w.Code, w.Body.String()) require.Equal(t, status, request(http.MethodGet, origin+"/discovery").Code) + if status == http.StatusNotFound { + for _, route := range []struct{ method, path string }{ + {http.MethodPost, "/runners/heartbeat"}, + {http.MethodPost, "/runner/app-fixture/session/unknown/channels"}, + {http.MethodGet, "/ai/trickle/unknown/0"}, + {http.MethodGet, "/apps/app-echo/app/probe"}, + {http.MethodGet, "/run/app-echo/probe"}, + {http.MethodGet, "/proxy/app-echo/probe"}, + } { + w := request(route.method, origin+route.path) + require.Equal(t, http.StatusNotFound, w.Code, origin+route.path+": "+w.Body.String()) + } + } } w := request(http.MethodGet, registry.proxyURL("app-echo")+"/probe") + if template == "" && apiOrigin != "" { + require.Equal(t, http.StatusNotFound, w.Code, w.Body.String()) + // A reverse proxy forwards the advertised URL with the API Host. + req := httptest.NewRequest(http.MethodGet, registry.proxyURL("app-echo")+"/probe", nil) + req.Host = server.apiOrigin.Host + w = httptest.NewRecorder() + server.ServeHTTP(w, req) + } require.Equal(t, http.StatusOK, w.Code, w.Body.String()) require.Equal(t, "application response", w.Body.String()) }) diff --git a/orchestrator/python_controls_test.go b/orchestrator/python_controls_test.go index 0abea15..9632a88 100644 --- a/orchestrator/python_controls_test.go +++ b/orchestrator/python_controls_test.go @@ -48,6 +48,7 @@ func TestPinnedPythonRequestSessionControls(t *testing.T) { defer orch.Close() defer server.Close() registry.service = "http://" + orch.Listener.Addr().String() + server.apiOrigin = mustURL(t, registry.service) orch.Start() fixture := filepath.Join(root, "cmd", "livepeer", "testdata", "python_runner_session_controls.py") cmd := exec.Command(python, fixture, orch.URL, "bootstrap", "unused", "request-controls", "persistent", "1", "metadata") diff --git a/orchestrator/python_media_test.go b/orchestrator/python_media_test.go index 33ea6f7..6497120 100644 --- a/orchestrator/python_media_test.go +++ b/orchestrator/python_media_test.go @@ -48,6 +48,7 @@ func TestPinnedPythonMultiTrackTrickle(t *testing.T) { orch := httptest.NewUnstartedServer(server) defer orch.Close() registry.service = "http://" + orch.Listener.Addr().String() + server.apiOrigin = mustURL(t, registry.service) orch.Start() manifest, _, _, status, err := registry.Reserve("media-runner") require.NoError(t, err) diff --git a/orchestrator/regression_test.go b/orchestrator/regression_test.go index 709f38f..7e0e851 100644 --- a/orchestrator/regression_test.go +++ b/orchestrator/regression_test.go @@ -87,7 +87,7 @@ func TestRegressionProxySendsContinueBeforeBody(t *testing.T) { registry, server := reviewServer(t, upstream.URL) defer server.Close() require.NoError(t, registry.AddStatic(StaticRunner{ID: "one", RunnerURL: upstream.URL, App: "one", Mode: "single-shot", Capacity: 1})) - orch := httptest.NewServer(server) + orch := newTestHTTPServer(t, server) defer orch.Close() address := orch.Listener.Addr().String() conn, err := net.DialTimeout("tcp", address, time.Second) @@ -111,7 +111,7 @@ func TestRegressionOffchainSingleShotCapacityAndHeaders(t *testing.T) { defer close(release) reg, s := reviewServer(t, upstream.URL) require.NoError(t, reg.AddStatic(StaticRunner{ID: "one", RunnerURL: upstream.URL, App: "one", Mode: "single-shot", Capacity: 1})) - go s.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "/apps/one/app", nil)) + go s.ServeHTTP(httptest.NewRecorder(), apiRequest(s, "GET", "/apps/one/app", nil)) header := <-seen t.Run("session_headers", func(t *testing.T) { require.NotEmpty(t, header.Get("Livepeer-Session-Id"), "off-chain single-shot lost its session control headers") @@ -119,7 +119,7 @@ func TestRegressionOffchainSingleShotCapacityAndHeaders(t *testing.T) { done := make(chan int, 1) go func() { w := httptest.NewRecorder() - s.ServeHTTP(w, httptest.NewRequest("GET", "/apps/one/app", nil)) + s.ServeHTTP(w, apiRequest(s, "GET", "/apps/one/app", nil)) done <- w.Code }() select { @@ -195,12 +195,12 @@ func TestRegressionTrickleNextDeliversNextPublishedPart(t *testing.T) { done := make(chan struct{}) w := httptest.NewRecorder() go func() { - s.ServeHTTP(w, httptest.NewRequest("GET", "/ai/trickle/test/-1", nil).WithContext(ctx)) + s.ServeHTTP(w, apiRequest(s, "GET", "/ai/trickle/test/-1", nil).WithContext(ctx)) close(done) }() synctest.Wait() pub := httptest.NewRecorder() - s.ServeHTTP(pub, httptest.NewRequest("POST", "/ai/trickle/test/0", strings.NewReader("first"))) + s.ServeHTTP(pub, apiRequest(s, "POST", "/ai/trickle/test/0", strings.NewReader("first"))) require.Equal(t, 200, pub.Code) synctest.Wait() select { @@ -238,8 +238,8 @@ func TestRegressionTrickleStreamsBeforePublisherEOF(t *testing.T) { ctx, cancel := context.WithCancel(t.Context()) defer cancel() out := &reviewStreamingWriter{header: http.Header{}, wrote: make(chan struct{}, 1)} - go s.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("POST", "/ai/trickle/test/0", input)) - go s.ServeHTTP(out, httptest.NewRequest("GET", "/ai/trickle/test/0", nil).WithContext(ctx)) + go s.ServeHTTP(httptest.NewRecorder(), apiRequest(s, "POST", "/ai/trickle/test/0", input)) + go s.ServeHTTP(out, apiRequest(s, "GET", "/ai/trickle/test/0", nil).WithContext(ctx)) _, err := writer.Write([]byte("streaming chunk")) require.NoError(t, err) synctest.Wait() diff --git a/orchestrator/runner_address_test.go b/orchestrator/runner_address_test.go index 82b0b77..bb011ed 100644 --- a/orchestrator/runner_address_test.go +++ b/orchestrator/runner_address_test.go @@ -34,7 +34,7 @@ func TestRunnerFacingAddressKeepsPublicAndInternalURLsSeparate(t *testing.T) { s := NewServer(registry, policy, policy, slog.New(slog.NewTextHandler(io.Discard, nil))) defer s.Close() - heartbeat := httptest.NewRequest(http.MethodPost, "http://runner.internal/internal/runners/heartbeat", strings.NewReader(`{"runner_id":"r","runner_url":"`+upstream.URL+`","app":"test","capacity":1}`)) + heartbeat := apiRequest(s, http.MethodPost, "http://runner.internal/internal/runners/heartbeat", strings.NewReader(`{"runner_id":"r","runner_url":"`+upstream.URL+`","app":"test","capacity":1}`)) heartbeat.Header.Set("Authorization", "bootstrap") w := httptest.NewRecorder() s.ServeHTTP(w, heartbeat) @@ -44,7 +44,7 @@ func TestRunnerFacingAddressKeepsPublicAndInternalURLsSeparate(t *testing.T) { require.Equal(t, "http://runner.internal/internal", registration.Orchestrator) require.Equal(t, "http://runner.internal/internal/ai/trickle/"+registration.O2R.ChannelName, registration.O2R.URL) require.Empty(t, registration.O2R.InternalURL) - heartbeat = httptest.NewRequest(http.MethodPost, registration.Orchestrator+"/runners/heartbeat", strings.NewReader(`{"runner_id":"r","runner_url":"`+upstream.URL+`","app":"test","capacity":1}`)) + heartbeat = apiRequest(s, http.MethodPost, registration.Orchestrator+"/runners/heartbeat", strings.NewReader(`{"runner_id":"r","runner_url":"`+upstream.URL+`","app":"test","capacity":1}`)) heartbeat.Header.Set("Authorization", registration.HeartbeatSecret) w = httptest.NewRecorder() s.ServeHTTP(w, heartbeat) @@ -76,7 +76,7 @@ func TestRunnerFacingAddressKeepsPublicAndInternalURLsSeparate(t *testing.T) { require.Equal(t, "http://runner.internal/internal/runner/r/session/"+reservation.SessionID, controlHeader) _, token, _, err := registry.sessionTarget("r", reservation.SessionID) require.NoError(t, err) - create := httptest.NewRequest(http.MethodPost, controlHeader+"/channels", strings.NewReader(`{"channels":[{"name":"events"}]}`)) + create := apiRequest(s, http.MethodPost, controlHeader+"/channels", strings.NewReader(`{"channels":[{"name":"events"}]}`)) create.Header.Set("Livepeer-Session-Token", token) w = httptest.NewRecorder() s.ServeHTTP(w, create) @@ -91,7 +91,7 @@ func TestRunnerFacingAddressKeepsPublicAndInternalURLsSeparate(t *testing.T) { require.Equal(t, "http://runner.internal/internal/ai/trickle/"+channel.ChannelName, channel.InternalURL) w = httptest.NewRecorder() - s.ServeHTTP(w, httptest.NewRequest(http.MethodPost, channel.InternalURL+"/0", bytes.NewBufferString("message"))) + s.ServeHTTP(w, apiRequest(s, http.MethodPost, channel.InternalURL+"/0", bytes.NewBufferString("message"))) require.Equal(t, http.StatusOK, w.Code, w.Body.String()) w = httptest.NewRecorder() s.ServeHTTP(w, httptest.NewRequest(http.MethodGet, channel.URL+"/0", nil)) @@ -106,14 +106,14 @@ func TestO2RKeepaliveRepeatsUntilShutdown(t *testing.T) { s := NewServer(registry, policy, policy, slog.New(slog.NewTextHandler(io.Discard, nil))) defer s.Close() w := httptest.NewRecorder() - heartbeat := httptest.NewRequest(http.MethodPost, "/runners/heartbeat", strings.NewReader(`{"runner_id":"r","runner_url":"https://runner.example","app":"test"}`)) + heartbeat := apiRequest(s, http.MethodPost, "/runners/heartbeat", strings.NewReader(`{"runner_id":"r","runner_url":"https://runner.example","app":"test"}`)) heartbeat.Header.Set("Authorization", "bootstrap") s.ServeHTTP(w, heartbeat) require.Equal(t, http.StatusOK, w.Code, w.Body.String()) var registration heartbeatResponse require.NoError(t, json.Unmarshal(w.Body.Bytes(), ®istration)) - server := httptest.NewServer(s) + server := newTestHTTPServer(t, s) defer server.Close() ctx, cancel := context.WithCancel(t.Context()) done := make(chan struct{}) diff --git a/orchestrator/seq_test.go b/orchestrator/seq_test.go index 89e58d0..c7ff573 100644 --- a/orchestrator/seq_test.go +++ b/orchestrator/seq_test.go @@ -41,7 +41,7 @@ func TestOwnerSeqOrdering(t *testing.T) { } lastLegacy := registry.runners["owner"].Last for _, seq := range []string{`-1`, `1.5`, `"1"`, `18446744073709551616`, `true`, `[]`} { - req := httptest.NewRequest(http.MethodPost, "/runners/heartbeat", strings.NewReader(`{"runner_id":"owner","seq":`+seq+`,"apps":[]}`)) + req := apiRequest(server, http.MethodPost, "/runners/heartbeat", strings.NewReader(`{"runner_id":"owner","seq":`+seq+`,"apps":[]}`)) req.Header.Set("Authorization", owner.HeartbeatSecret) w := httptest.NewRecorder() server.ServeHTTP(w, req) diff --git a/orchestrator/trickle_test.go b/orchestrator/trickle_test.go index 4cb3c17..218443e 100644 --- a/orchestrator/trickle_test.go +++ b/orchestrator/trickle_test.go @@ -24,7 +24,7 @@ func TestDirectTrickleDeleteAllowsNamedChannelRecreation(t *testing.T) { require.NoError(t, err) create := func() string { t.Helper() - req := httptest.NewRequest(http.MethodPost, "/runner/r/session/"+sid+"/channels", strings.NewReader(`{"channels":[{"name":"events"}]}`)) + req := apiRequest(s, http.MethodPost, "/runner/r/session/"+sid+"/channels", strings.NewReader(`{"channels":[{"name":"events"}]}`)) req.Header.Set("Livepeer-Session-Token", token) w := httptest.NewRecorder() s.ServeHTTP(w, req) @@ -40,7 +40,7 @@ func TestDirectTrickleDeleteAllowsNamedChannelRecreation(t *testing.T) { } first := create() w := httptest.NewRecorder() - s.ServeHTTP(w, httptest.NewRequest(http.MethodDelete, "/ai/trickle/"+first, nil)) + s.ServeHTTP(w, apiRequest(s, http.MethodDelete, "/ai/trickle/"+first, nil)) require.Equal(t, http.StatusOK, w.Code) require.NotContains(t, s.channels, first) second := create() @@ -72,14 +72,14 @@ func TestTrickleActiveStreamAndCancellation(t *testing.T) { defer writer.Close() pubDone, subDone := make(chan struct{}), make(chan struct{}) go func() { - s.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("POST", "/ai/trickle/test/0", input)) + s.ServeHTTP(httptest.NewRecorder(), apiRequest(s, "POST", "/ai/trickle/test/0", input)) close(pubDone) }() ctx, cancel := context.WithCancel(t.Context()) defer cancel() out := &reviewStreamingWriter{header: http.Header{}, wrote: make(chan struct{}, 1)} go func() { - s.ServeHTTP(out, httptest.NewRequest("GET", "/ai/trickle/test/0", nil).WithContext(ctx)) + s.ServeHTTP(out, apiRequest(s, "GET", "/ai/trickle/test/0", nil).WithContext(ctx)) close(subDone) }() _, err = writer.Write([]byte("first")) @@ -97,9 +97,9 @@ func TestTrickleActiveStreamAndCancellation(t *testing.T) { } switch action { case "delete": - s.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("DELETE", "/ai/trickle/test/0", nil)) + s.ServeHTTP(httptest.NewRecorder(), apiRequest(s, "DELETE", "/ai/trickle/test/0", nil)) case "reset": - req := httptest.NewRequest("POST", "/ai/trickle/test/1", strings.NewReader("next")) + req := apiRequest(s, "POST", "/ai/trickle/test/1", strings.NewReader("next")) req.Header.Set("Lp-Trickle-Reset", "true") s.ServeHTTP(httptest.NewRecorder(), req) case "release": @@ -178,7 +178,7 @@ func TestNamedChannelsAreIdempotentAndReleased(t *testing.T) { require.NoError(t, err) var previous string for range 2 { - req := httptest.NewRequest("POST", "/runner/r/session/"+sid+"/channels", strings.NewReader(`{"channels":[{"name":"events"}]}`)) + req := apiRequest(s, "POST", "/runner/r/session/"+sid+"/channels", strings.NewReader(`{"channels":[{"name":"events"}]}`)) req.Header.Set("Livepeer-Session-Token", token) w := httptest.NewRecorder() s.ServeHTTP(w, req) @@ -202,8 +202,8 @@ func TestTrickleLargeSegmentDelivered(t *testing.T) { defer writer.Close() pub, sub := httptest.NewRecorder(), httptest.NewRecorder() pubDone, subDone := make(chan struct{}), make(chan struct{}) - go func() { s.ServeHTTP(pub, httptest.NewRequest("POST", "/ai/trickle/large/0", input)); close(pubDone) }() - go func() { s.ServeHTTP(sub, httptest.NewRequest("GET", "/ai/trickle/large/0", nil)); close(subDone) }() + go func() { s.ServeHTTP(pub, apiRequest(s, "POST", "/ai/trickle/large/0", input)); close(pubDone) }() + go func() { s.ServeHTTP(sub, apiRequest(s, "GET", "/ai/trickle/large/0", nil)); close(subDone) }() chunk := strings.Repeat("x", 32<<10) for range (3 << 20) / len(chunk) { _, err := io.WriteString(writer, chunk) @@ -216,7 +216,7 @@ func TestTrickleLargeSegmentDelivered(t *testing.T) { require.Equal(t, http.StatusOK, pub.Code) require.Equal(t, 3<<20, sub.Body.Len()) late := httptest.NewRecorder() - s.ServeHTTP(late, httptest.NewRequest("GET", "/ai/trickle/large/0", nil)) + s.ServeHTTP(late, apiRequest(s, "GET", "/ai/trickle/large/0", nil)) require.Equal(t, http.StatusOK, late.Code) require.Equal(t, 3<<20, late.Body.Len()) }) From 154056635a82230818966bc1e76477c9c8b871c7 Mon Sep 17 00:00:00 2001 From: Josh Allmann Date: Thu, 12 Nov 2026 08:43:52 -0800 Subject: [PATCH 2/5] Reuse authority tests and simplify their fixtures --- configs/orchestrator/config.example.toml | 2 +- docs/orchestrator.md | 8 +-- orchestrator/app_test.go | 81 +++++++++++------------- orchestrator/discovery_test.go | 4 +- orchestrator/http_test.go | 59 ++++++++--------- orchestrator/lifecycle_test.go | 8 ++- orchestrator/metrics_test.go | 9 +-- orchestrator/payment_recipient_test.go | 2 +- orchestrator/payment_test.go | 16 ++--- orchestrator/proxy_test.go | 19 +----- orchestrator/regression_test.go | 17 ++--- orchestrator/runner_address_test.go | 20 ++++-- orchestrator/seq_test.go | 2 +- orchestrator/trickle_test.go | 20 +++--- 14 files changed, 122 insertions(+), 145 deletions(-) diff --git a/configs/orchestrator/config.example.toml b/configs/orchestrator/config.example.toml index a239ac8..035a18b 100644 --- a/configs/orchestrator/config.example.toml +++ b/configs/orchestrator/config.example.toml @@ -3,7 +3,7 @@ Listen = "127.0.0.1:8935" MetricsListen = "127.0.0.1:8936" ServiceURL = "http://127.0.0.1:8935" -# APIOriginURL = "http://orchestrator.internal:8935" # Required API Host; defaults to ServiceURL, independent of ProxyURLTemplate. +# APIOriginURL = "http://orchestrator.internal:8935" # API request Host sent by the reverse proxy; defaults to ServiceURL. # RunnerServiceURL = "http://127.0.0.1:8935" # Runner-facing callbacks and trickle; defaults to ServiceURL. BootstrapSecretFile = "/run/secrets/livepeer-runner-bootstrap" RunnerConfig = "/etc/livepeer/runners.toml" diff --git a/docs/orchestrator.md b/docs/orchestrator.md index 2cc2529..11573c9 100644 --- a/docs/orchestrator.md +++ b/docs/orchestrator.md @@ -103,11 +103,9 @@ DNS, TLS, and forwarding to this listener for those URLs. Hostname routing uses the actual Host header. Single-shot domain proxies require an app registration ID that is a lowercase DNS label of at most 63 characters. -API requests require the `APIOriginURL` Host (default: `ServiceURL`), even -without `ProxyURLTemplate`. Set it to the HTTP(S) origin a reverse proxy -forwards, without a path. Runner, trickle, app, and fallback proxy routes share -this Host; configure `RunnerServiceURL` or the proxy accordingly. Template-matched -proxies use their template Host. Advertised URLs are unchanged. +API requests must use the Host from `APIOriginURL`, which defaults to +`ServiceURL`. If a reverse proxy rewrites Host, set `APIOriginURL` to an +HTTP(S) URL with the Host it sends, without a path. To remove headers from app requests, set `ExcludedHeaders`, for example: `["CF-Access-*", "Cookie"]`. Matching is case-insensitive. diff --git a/orchestrator/app_test.go b/orchestrator/app_test.go index a012a0a..69a1045 100644 --- a/orchestrator/app_test.go +++ b/orchestrator/app_test.go @@ -55,50 +55,43 @@ func freeTCPPort(t *testing.T) int { } func TestAPIOriginURLRouting(t *testing.T) { - for _, template := range []string{"", "https://{proxy}.internal", "https://public.example/custom/{proxy}"} { - t.Run(template, func(t *testing.T) { - address := fmt.Sprintf("127.0.0.1:%d", freeTCPPort(t)) - metrics := fmt.Sprintf("127.0.0.1:%d", freeTCPPort(t)) - const origin = "orchestrator.internal" - t.Setenv("LIVEPEER_ORCHESTRATOR_BOOTSTRAP_SECRET", "bootstrap") - t.Setenv("LIVEPEER_ORCHESTRATOR_API_ORIGIN_URL", "http://wrong.internal:8935") - cmd := Root(io.Discard, io.Discard) - cmd.SetArgs([]string{"--config", "", "--listen", address, "--metrics-listen", metrics, - "--service-url", "https://public.example", - "--api-origin-url", "http://" + origin + ":80", "--runner-service-url", "http://" + origin, - "--proxy-url-template", template}) - ctx, cancel := context.WithCancel(t.Context()) - done := make(chan error, 1) - go func() { done <- cmd.ExecuteContext(ctx) }() - t.Cleanup(func() { - cancel() - select { - case err := <-done: - require.NoError(t, err) - case <-time.After(5 * time.Second): - t.Fatal("orchestrator did not stop") - } - }) - client := &http.Client{Timeout: time.Second, Transport: &http.Transport{DisableKeepAlives: true}} - request := func(host string) int { - req, err := http.NewRequest(http.MethodGet, "http://"+address+"/discovery", nil) - require.NoError(t, err) - req.Host = host - req.Header.Set("X-Forwarded-Host", origin) - req.Header.Set("Forwarded", "host="+origin) - response, err := client.Do(req) - if err != nil { - return 0 - } - defer response.Body.Close() - return response.StatusCode - } - require.Eventually(t, func() bool { return request(origin) == http.StatusOK }, 5*time.Second, 10*time.Millisecond) - require.Equal(t, http.StatusOK, request(origin+":80")) - for _, host := range []string{origin + ":443", "public.example", "wrong.internal:8935", "other.example"} { - require.Equal(t, http.StatusNotFound, request(host), host) - } - }) + address := fmt.Sprintf("127.0.0.1:%d", freeTCPPort(t)) + metrics := fmt.Sprintf("127.0.0.1:%d", freeTCPPort(t)) + const origin = "orchestrator.internal" + t.Setenv("LIVEPEER_ORCHESTRATOR_BOOTSTRAP_SECRET", "bootstrap") + t.Setenv("LIVEPEER_ORCHESTRATOR_API_ORIGIN_URL", "http://wrong.internal:8935") + cmd := Root(io.Discard, io.Discard) + cmd.SetArgs([]string{"--config", "", "--listen", address, "--metrics-listen", metrics, + "--service-url", "https://public.example", + "--api-origin-url", "http://" + origin + ":80", "--runner-service-url", "http://" + origin}) + ctx, cancel := context.WithCancel(t.Context()) + done := make(chan error, 1) + go func() { done <- cmd.ExecuteContext(ctx) }() + t.Cleanup(func() { + cancel() + select { + case err := <-done: + require.NoError(t, err) + case <-time.After(5 * time.Second): + t.Fatal("orchestrator did not stop") + } + }) + client := &http.Client{Timeout: time.Second, Transport: &http.Transport{DisableKeepAlives: true}} + request := func(host string) int { + req, err := http.NewRequest(http.MethodGet, "http://"+address+"/discovery", nil) + require.NoError(t, err) + req.Host = host + response, err := client.Do(req) + if err != nil { + return 0 + } + defer response.Body.Close() + return response.StatusCode + } + require.Eventually(t, func() bool { return request(origin) == http.StatusOK }, 5*time.Second, 10*time.Millisecond) + require.Equal(t, http.StatusOK, request(origin+":80")) + for _, host := range []string{origin + ":443", "public.example", "wrong.internal:8935", "other.example"} { + require.Equal(t, http.StatusNotFound, request(host), host) } } diff --git a/orchestrator/discovery_test.go b/orchestrator/discovery_test.go index f3ae100..9b96812 100644 --- a/orchestrator/discovery_test.go +++ b/orchestrator/discovery_test.go @@ -7,6 +7,7 @@ import ( "log/slog" "math/big" "net/http" + "net/http/httptest" "testing" "time" @@ -37,7 +38,8 @@ func TestDiscoveryLocalInventory(t *testing.T) { require.NoError(t, err) s := NewServer(registry, policy, policy, slog.New(slog.NewTextHandler(io.Discard, nil))) t.Cleanup(s.Close) - server := newTestHTTPServer(t, s) + server := httptest.NewServer(s) + s.apiOrigin = mustURL(t, server.URL) t.Cleanup(server.Close) response, err := server.Client().Get(server.URL + "/deployment/discovery") require.NoError(t, err) diff --git a/orchestrator/http_test.go b/orchestrator/http_test.go index c955cb3..65b2ba2 100644 --- a/orchestrator/http_test.go +++ b/orchestrator/http_test.go @@ -25,20 +25,6 @@ import ( "github.com/stretchr/testify/require" ) -func apiRequest(server *Server, method, target string, body io.Reader) *http.Request { - req := httptest.NewRequest(method, target, body) - req.Host = server.apiOrigin.Host - return req -} - -func newTestHTTPServer(t *testing.T, app *Server) *httptest.Server { - t.Helper() - server := httptest.NewUnstartedServer(app) - app.apiOrigin = mustURL(t, "http://"+server.Listener.Addr().String()) - server.Start() - return server -} - func TestDynamicRunnerSessionProxyAndCapacity(t *testing.T) { upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { require.Equal(t, "/api/hello", r.URL.Path) @@ -51,7 +37,9 @@ func TestDynamicRunnerSessionProxyAndCapacity(t *testing.T) { policy, err := destination.New("runner", []string{"127.0.0.0/8"}) require.NoError(t, err) registry := NewRegistry("bootstrap", "http://orchestrator.example", 10*time.Millisecond, 50*time.Millisecond) - srv := newTestHTTPServer(t, NewServer(registry, policy, policy, slog.New(slog.NewTextHandler(io.Discard, nil)))) + app := NewServer(registry, policy, policy, slog.New(slog.NewTextHandler(io.Discard, nil))) + srv := httptest.NewServer(app) + app.apiOrigin = mustURL(t, srv.URL) defer srv.Close() body := `{"runner_url":"` + upstream.URL + `/api","app":"test","mode":"persistent","capacity":1,"price_info":{"price":0,"currency":"usd","unit":"hour"}}` @@ -135,7 +123,7 @@ func TestSessionCallbackCredentials(t *testing.T) { } { for name, credential := range map[string]string{"missing": "", "wrong": "wrong", "cross-session": otherToken} { t.Run(operation.method+" "+operation.path+"/"+name, func(t *testing.T) { - req := apiRequest(server, operation.method, "/runner/r/session/"+sid+"/"+operation.path, strings.NewReader(`{}`)) + req := httptest.NewRequest(operation.method, "/runner/r/session/"+sid+"/"+operation.path, strings.NewReader(`{}`)) if credential != "" { req.Header.Set("Livepeer-Session-Token", credential) } @@ -149,7 +137,7 @@ func TestSessionCallbackCredentials(t *testing.T) { _, err := registry.addProxy("r", sid, token, target) require.NoError(t, err) } - req := apiRequest(server, http.MethodPost, "/runner/r/session/"+sid+"/proxy", strings.NewReader(`{}`)) + req := httptest.NewRequest(http.MethodPost, "/runner/r/session/"+sid+"/proxy", strings.NewReader(`{}`)) req.Header.Set("Livepeer-Session-Token", token) w := httptest.NewRecorder() server.ServeHTTP(w, req) @@ -291,7 +279,8 @@ func TestSessionProxyStreamsSSEAndWebSocket(t *testing.T) { app := NewServer(registry, policy, policy, slog.New(slog.NewTextHandler(io.Discard, nil))) app.setExcludedHeaders([]string{"CF-Access-*", "Cookie"}) defer app.Close() - server := newTestHTTPServer(t, app) + server := httptest.NewServer(app) + app.apiOrigin = mustURL(t, server.URL) defer server.Close() sid, _, _, _, err := registry.Reserve(registered.RunnerID) require.NoError(t, err) @@ -334,7 +323,9 @@ func TestRunnerDestinationDeniedWithoutGrant(t *testing.T) { policy, err := destination.New("runner", nil) require.NoError(t, err) registry := NewRegistry("bootstrap", "http://orchestrator.example", time.Second, time.Minute) - server := newTestHTTPServer(t, NewServer(registry, policy, policy, slog.New(slog.NewTextHandler(io.Discard, nil)))) + app := NewServer(registry, policy, policy, slog.New(slog.NewTextHandler(io.Discard, nil))) + server := httptest.NewServer(app) + app.apiOrigin = mustURL(t, server.URL) defer server.Close() body := `{"runner_url":"` + upstream.URL + `","app":"test","mode":"single-shot","capacity":1,"price_info":{"price":0}}` request, err := http.NewRequest(http.MethodPost, server.URL+"/runners/heartbeat", bytes.NewBufferString(body)) @@ -361,7 +352,7 @@ func TestGeneratedProxyHasSeparateDestinationGrant(t *testing.T) { require.NoError(t, err) proxyPolicy, err := destination.New("session-proxy", nil) require.NoError(t, err) - registry := NewRegistry("bootstrap", "http://orchestrator.example", time.Second, time.Minute) + registry := NewRegistry("bootstrap", "http://example.com", time.Second, time.Minute) registration := heartbeatRequest{RunnerURL: upstream.URL, App: "test", Mode: "persistent", Capacity: 1} registered, status, err := registry.Heartbeat(registration, "bootstrap") require.NoError(t, err) @@ -377,12 +368,12 @@ func TestGeneratedProxyHasSeparateDestinationGrant(t *testing.T) { require.Equal(t, http.StatusOK, status) srv := NewServer(registry, runnerPolicy, proxyPolicy, slog.New(slog.NewTextHandler(io.Discard, nil))) path := "/runner/" + registered.RunnerID + "/session/" + sid + "/proxy" - create := apiRequest(srv, http.MethodPost, path, bytes.NewBufferString(`{"target_url":"`+upstream.URL+`"}`)) + create := httptest.NewRequest(http.MethodPost, path, bytes.NewBufferString(`{"target_url":"`+upstream.URL+`"}`)) create.Header.Set("Livepeer-Session-Token", "bad") record := httptest.NewRecorder() srv.ServeHTTP(record, create) require.Equal(t, http.StatusUnauthorized, record.Code) - create = apiRequest(srv, http.MethodPost, path, bytes.NewBufferString(`{"target_url":"`+upstream.URL+`"}`)) + create = httptest.NewRequest(http.MethodPost, path, bytes.NewBufferString(`{"target_url":"`+upstream.URL+`"}`)) create.Header.Set("Livepeer-Session-Token", token) record = httptest.NewRecorder() srv.ServeHTTP(record, create) @@ -393,20 +384,20 @@ func TestGeneratedProxyHasSeparateDestinationGrant(t *testing.T) { } require.NoError(t, json.Unmarshal(record.Body.Bytes(), &created)) require.NotEmpty(t, created.ProxyID) - require.Equal(t, "http://orchestrator.example/run/"+created.ProxyID, created.URL) + require.Equal(t, "http://example.com/run/"+created.ProxyID, created.URL) record = httptest.NewRecorder() - srv.ServeHTTP(record, apiRequest(srv, http.MethodGet, "/run/"+created.ProxyID, nil)) + srv.ServeHTTP(record, httptest.NewRequest(http.MethodGet, "/run/"+created.ProxyID, nil)) require.Equal(t, http.StatusBadGateway, record.Code) granted, err := destination.New("session-proxy", []string{grant}) require.NoError(t, err) srv = NewServer(registry, runnerPolicy, granted, slog.New(slog.NewTextHandler(io.Discard, nil))) record = httptest.NewRecorder() - srv.ServeHTTP(record, apiRequest(srv, http.MethodGet, "/run/"+created.ProxyID, nil)) + srv.ServeHTTP(record, httptest.NewRequest(http.MethodGet, "/run/"+created.ProxyID, nil)) require.Equal(t, http.StatusOK, record.Code) record = httptest.NewRecorder() - srv.ServeHTTP(record, apiRequest(srv, http.MethodGet, "/proxy/"+created.ProxyID, nil)) + srv.ServeHTTP(record, httptest.NewRequest(http.MethodGet, "/proxy/"+created.ProxyID, nil)) require.Equal(t, http.StatusOK, record.Code) - create = apiRequest(srv, "POST", "/runner/"+registered.RunnerID+"/session/"+sid+"/channels", strings.NewReader(`{"channels":[{"name":"events"}]}`)) + create = httptest.NewRequest("POST", "/runner/"+registered.RunnerID+"/session/"+sid+"/channels", strings.NewReader(`{"channels":[{"name":"events"}]}`)) create.Header.Set("Livepeer-Session-Token", token) record = httptest.NewRecorder() srv.ServeHTTP(record, create) @@ -435,7 +426,7 @@ func TestRunnerExpires(t *testing.T) { func TestTrickleChannelsScopedToSession(t *testing.T) { policy, err := destination.New("runner", nil) require.NoError(t, err) - registry := NewRegistry("bootstrap", "http://orchestrator.example", time.Second, time.Minute) + registry := NewRegistry("bootstrap", "http://example.com", time.Second, time.Minute) registration := heartbeatRequest{RunnerURL: "https://runner.example", App: "test", Mode: "persistent", Capacity: 2} registered, _, err := registry.Heartbeat(registration, "bootstrap") require.NoError(t, err) @@ -452,7 +443,7 @@ func TestTrickleChannelsScopedToSession(t *testing.T) { _, token2, _, err := registry.sessionTarget(registered.RunnerID, sid2) require.NoError(t, err) path1 := "/runner/" + registered.RunnerID + "/session/" + sid1 + "/channels" - create := apiRequest(srv, http.MethodPost, path1, bytes.NewBufferString(`{"channels":[{"name":"events","mime_type":"application/json"}]}`)) + create := httptest.NewRequest(http.MethodPost, path1, bytes.NewBufferString(`{"channels":[{"name":"events","mime_type":"application/json"}]}`)) create.Header.Set("Livepeer-Session-Token", token1) record := httptest.NewRecorder() srv.ServeHTTP(record, create) @@ -466,7 +457,7 @@ func TestTrickleChannelsScopedToSession(t *testing.T) { require.Len(t, created.Channels, 1) channelID := created.Channels[0].ChannelName path2 := "/runner/" + registered.RunnerID + "/session/" + sid2 + "/channels" - deletion := apiRequest(srv, http.MethodDelete, path2, bytes.NewBufferString(`{"channels":["`+channelID+`"]}`)) + deletion := httptest.NewRequest(http.MethodDelete, path2, bytes.NewBufferString(`{"channels":["`+channelID+`"]}`)) deletion.Header.Set("Livepeer-Session-Token", token2) record = httptest.NewRecorder() srv.ServeHTTP(record, deletion) @@ -474,19 +465,19 @@ func TestTrickleChannelsScopedToSession(t *testing.T) { require.JSONEq(t, `{"deleted":[]}`, record.Body.String()) segment := "/ai/trickle/" + channelID + "/0" record = httptest.NewRecorder() - srv.ServeHTTP(record, apiRequest(srv, http.MethodPost, segment, bytes.NewBufferString(`{"x":1}`))) + srv.ServeHTTP(record, httptest.NewRequest(http.MethodPost, segment, bytes.NewBufferString(`{"x":1}`))) require.Equal(t, http.StatusOK, record.Code) record = httptest.NewRecorder() - srv.ServeHTTP(record, apiRequest(srv, http.MethodGet, segment, nil)) + srv.ServeHTTP(record, httptest.NewRequest(http.MethodGet, segment, nil)) require.Equal(t, http.StatusOK, record.Code) require.JSONEq(t, `{"x":1}`, record.Body.String()) - deletion = apiRequest(srv, http.MethodDelete, path1, bytes.NewBufferString(`{"channels":["`+channelID+`"]}`)) + deletion = httptest.NewRequest(http.MethodDelete, path1, bytes.NewBufferString(`{"channels":["`+channelID+`"]}`)) deletion.Header.Set("Livepeer-Session-Token", token1) record = httptest.NewRecorder() srv.ServeHTTP(record, deletion) require.Equal(t, http.StatusOK, record.Code) record = httptest.NewRecorder() - srv.ServeHTTP(record, apiRequest(srv, http.MethodGet, segment, nil)) + srv.ServeHTTP(record, httptest.NewRequest(http.MethodGet, segment, nil)) require.Equal(t, http.StatusNotFound, record.Code) } diff --git a/orchestrator/lifecycle_test.go b/orchestrator/lifecycle_test.go index af8ecfb..e579b8c 100644 --- a/orchestrator/lifecycle_test.go +++ b/orchestrator/lifecycle_test.go @@ -67,7 +67,7 @@ func TestRunnerStopDrainsProxy(t *testing.T) { if scenario.name == "client stop" { prefix = "/apps/" } - stop := apiRequest(server, http.MethodPost, prefix+"r/session/"+sid+"/stop", nil) + stop := httptest.NewRequest(http.MethodPost, prefix+"r/session/"+sid+"/stop", nil) stop.Header.Set("Livepeer-Session-Token", token) stopped := httptest.NewRecorder() released := time.Now() @@ -131,7 +131,8 @@ func TestReleaseCancelsActiveProxy(t *testing.T) { require.NoError(t, registry.AddStatic(StaticRunner{ID: "runner", RunnerURL: upstream.URL, App: "test", Mode: mode, Capacity: 1})) } app := NewServer(registry, policy, policy, slog.New(slog.NewTextHandler(io.Discard, nil))) - server := newTestHTTPServer(t, app) + server := httptest.NewServer(app) + app.apiOrigin = mustURL(t, server.URL) defer server.Close() defer app.Close() path := "/apps/runner/app" @@ -362,7 +363,8 @@ func TestMultiAppRemovalKeepsStreams(t *testing.T) { })) defer upstream.Close() registry, appServer := reviewServer(t, upstream.URL) - server := newTestHTTPServer(t, appServer) + server := httptest.NewServer(appServer) + appServer.apiOrigin = mustURL(t, server.URL) defer server.Close() defer appServer.Close() a, b := registeredApp("a"), registeredApp("b") diff --git a/orchestrator/metrics_test.go b/orchestrator/metrics_test.go index cad6dcc..ae14e02 100644 --- a/orchestrator/metrics_test.go +++ b/orchestrator/metrics_test.go @@ -198,7 +198,8 @@ func TestUpgradePreservesHalfClose(t *testing.T) { registry, app := reviewServer(t, upstream.URL) defer app.Close() require.NoError(t, registry.AddStatic(StaticRunner{ID: "runner", App: "model", Mode: "single-shot", RunnerURL: upstream.URL, Capacity: 1})) - server := newTestHTTPServer(t, app) + server := httptest.NewServer(app) + app.apiOrigin = mustURL(t, server.URL) defer server.Close() u, err := url.Parse(server.URL) require.NoError(t, err) @@ -253,7 +254,7 @@ func TestApplicationMetricsAndInventory(t *testing.T) { } for _, path := range []string{"/apps/runner/app", "/run/runner", "/proxy/runner"} { response := httptest.NewRecorder() - app.ServeHTTP(response, apiRequest(app, "POST", path, nil)) + app.ServeHTTP(response, httptest.NewRequest("POST", path, nil)) require.Equal(t, 503, response.Code) } require.EqualValues(t, 3, test.MetricSum(t, registry.metrics.Registry, "app_calls_total", map[string]string{"app": "model", "outcome": "error"})) @@ -263,7 +264,7 @@ func TestApplicationMetricsAndInventory(t *testing.T) { for range cap(app.streamSlots) { app.streamSlots <- struct{}{} } - app.ServeHTTP(httptest.NewRecorder(), apiRequest(app, "POST", "/run/runner", nil)) + app.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("POST", "/run/runner", nil)) for range cap(app.streamSlots) { <-app.streamSlots } @@ -366,7 +367,7 @@ func TestUpstreamFailureAfterHeaders(t *testing.T) { defer app.Close() require.NoError(t, registry.AddStatic(StaticRunner{ID: "runner", App: "model", Mode: "single-shot", RunnerURL: upstream.URL, Capacity: 1})) response := httptest.NewRecorder() - app.ServeHTTP(response, apiRequest(app, "GET", "/apps/runner/app", nil)) + app.ServeHTTP(response, httptest.NewRequest("GET", "/apps/runner/app", nil)) require.Equal(t, 200, response.Code, "the upstream status was already forwarded") require.EqualValues(t, 1, test.MetricSum(t, registry.metrics.Registry, "app_calls_total", map[string]string{"outcome": "stream_error"})) require.EqualValues(t, 1, test.MetricSum(t, registry.metrics.Registry, "stream_errors_total", nil)) diff --git a/orchestrator/payment_recipient_test.go b/orchestrator/payment_recipient_test.go index 86e2689..185f091 100644 --- a/orchestrator/payment_recipient_test.go +++ b/orchestrator/payment_recipient_test.go @@ -471,7 +471,7 @@ func TestPaidFailureOutcomes(t *testing.T) { if cause == "capacity" || cause == "queue capacity" { status = 503 } - request := apiRequest(server, "POST", "/apps/runner/app", nil) + request := httptest.NewRequest("POST", "/apps/runner/app", nil) if operation == "receive" { payment, segment := recipientPayment(t, key, info, nonces...) if cause == "unhashed segment" { diff --git a/orchestrator/payment_test.go b/orchestrator/payment_test.go index 3d93008..0f3cb5a 100644 --- a/orchestrator/payment_test.go +++ b/orchestrator/payment_test.go @@ -64,7 +64,7 @@ func TestPaidFixedSessionWithRemoteSigner(t *testing.T) { max := new(big.Int).Sub(new(big.Int).Lsh(big.NewInt(1), 256), big.NewInt(1)) engine, err := NewPaymentEngine(store, paymentTestChain{}, recipient.Address(), big.NewInt(11), new(big.Int).Sub(max, big.NewInt(1))) require.NoError(t, err) - registry := NewRegistry("bootstrap", "http://127.0.0.1:8935", time.Second, 30*time.Second) + registry := NewRegistry("bootstrap", "http://example.com", time.Second, 30*time.Second) registry.SetWeiPerUSD(big.NewRat(10, 1)) registration := heartbeatRequest{RunnerID: "runner1", RunnerURL: "http://127.0.0.1:9000", App: "demo", Mode: "persistent", Capacity: 1, PriceInfo: priceInfo{Price: "1", Currency: "usd", Unit: "fixed"}} resp, status, err := registry.Heartbeat(registration, "bootstrap") @@ -83,7 +83,7 @@ func TestPaidFixedSessionWithRemoteSigner(t *testing.T) { defer signerService.Close() signerService.SetPaymentChain(paymentTestChain{}) reserveURL := "/apps/runner1/session" - first := apiRequest(server, http.MethodPost, reserveURL, nil) + first := httptest.NewRequest(http.MethodPost, reserveURL, nil) first.Header.Set("Livepeer-Payer-Address", payer.Address().Hex()) w := httptest.NewRecorder() server.ServeHTTP(w, first) @@ -106,7 +106,7 @@ func TestPaidFixedSessionWithRemoteSigner(t *testing.T) { // changes the advertised price before the reservation is paid. _, _, err = registry.Heartbeat(heartbeatRequest{RunnerID: "runner1", RunnerURL: "http://127.0.0.1:9000", App: "demo", Mode: "persistent", Capacity: 1, PriceInfo: priceInfo{Price: "2", Currency: "usd", Unit: "fixed"}}, resp.HeartbeatSecret) require.NoError(t, err) - paid := apiRequest(server, http.MethodPost, reserveURL, nil) + paid := httptest.NewRequest(http.MethodPost, reserveURL, nil) paid.Header.Set("Livepeer-Payment", payment.Payment) paid.Header.Set("Livepeer-Segment", payment.SegCreds) occupied, _, _, _, err := registry.Reserve("runner1") @@ -353,14 +353,14 @@ func TestMultiAppPaidIsolationAndDrain(t *testing.T) { owner := appHeartbeat(t, registry, "owner", "bootstrap", a, b, busy) for _, path := range []string{"/apps/live/session", "/apps/fixed/app"} { w := httptest.NewRecorder() - server.ServeHTTP(w, apiRequest(server, http.MethodPost, path, nil)) + server.ServeHTTP(w, httptest.NewRequest(http.MethodPost, path, nil)) require.Equal(t, http.StatusNotFound, w.Code, w.Body.String()) } require.Empty(t, registry.sessions) appHeartbeat(t, registry, owner.RunnerID, owner.HeartbeatSecret, a, b, busy) payer := crypto.PubkeyToAddress(payerKey.PublicKey) request := func(method, path, payment, segment string) *httptest.ResponseRecorder { - r := apiRequest(server, method, path, nil) + r := httptest.NewRequest(method, path, nil) r.Header.Set("Livepeer-Payer-Address", payer.Hex()) r.Header.Set("Livepeer-Payment", payment) r.Header.Set("Livepeer-Segment", segment) @@ -382,7 +382,7 @@ func TestMultiAppPaidIsolationAndDrain(t *testing.T) { sessions := len(engine.sessions) operations := test.MetricSum(t, registry.metrics.Registry, "payment_operations_total", nil) for _, address := range []string{payer.Hex(), ""} { - r := apiRequest(server, method, path, nil) + r := httptest.NewRequest(method, path, nil) r.Header.Set("Livepeer-Payer-Address", address) w := httptest.NewRecorder() server.ServeHTTP(w, r) @@ -462,7 +462,7 @@ func TestMultiAppPaidIsolationAndDrain(t *testing.T) { // Follow-up payment and refresh remain available for the draining live app. refreshBody := mustJSON(t, map[string]string{"sender": payer.Hex(), "manifest_id": live.ManifestID}) w = httptest.NewRecorder() - server.ServeHTTP(w, apiRequest(server, http.MethodPost, "/refresh-payment", bytes.NewReader(refreshBody))) + server.ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/refresh-payment", bytes.NewReader(refreshBody))) require.Equal(t, 200, w.Code, w.Body.String()) next, segment := signed(live, 4, 5, 6) w = request(http.MethodPost, live.PaymentURL, next, segment) @@ -500,7 +500,7 @@ func TestAppIDReuseRejectsPreviousRegistrationQuotes(t *testing.T) { owner := appHeartbeat(t, registry, "old-owner", "bootstrap", a) appHeartbeat(t, registry, owner.RunnerID, owner.HeartbeatSecret, a) request := func(payment, segment string) *httptest.ResponseRecorder { - req := apiRequest(server, http.MethodPost, "/apps/reused/session", nil) + req := httptest.NewRequest(http.MethodPost, "/apps/reused/session", nil) req.Header.Set("Livepeer-Payer-Address", crypto.PubkeyToAddress(key.PublicKey).Hex()) req.Header.Set("Livepeer-Payment", payment) req.Header.Set("Livepeer-Segment", segment) diff --git a/orchestrator/proxy_test.go b/orchestrator/proxy_test.go index f8a22a4..0eeeaaf 100644 --- a/orchestrator/proxy_test.go +++ b/orchestrator/proxy_test.go @@ -216,28 +216,11 @@ func TestProxyHostCannotReachAPI(t *testing.T) { w := request(http.MethodPost, origin+"/apps/app-fixture/session") require.Equal(t, status, w.Code, w.Body.String()) require.Equal(t, status, request(http.MethodGet, origin+"/discovery").Code) - if status == http.StatusNotFound { - for _, route := range []struct{ method, path string }{ - {http.MethodPost, "/runners/heartbeat"}, - {http.MethodPost, "/runner/app-fixture/session/unknown/channels"}, - {http.MethodGet, "/ai/trickle/unknown/0"}, - {http.MethodGet, "/apps/app-echo/app/probe"}, - {http.MethodGet, "/run/app-echo/probe"}, - {http.MethodGet, "/proxy/app-echo/probe"}, - } { - w := request(route.method, origin+route.path) - require.Equal(t, http.StatusNotFound, w.Code, origin+route.path+": "+w.Body.String()) - } - } } w := request(http.MethodGet, registry.proxyURL("app-echo")+"/probe") if template == "" && apiOrigin != "" { require.Equal(t, http.StatusNotFound, w.Code, w.Body.String()) - // A reverse proxy forwards the advertised URL with the API Host. - req := httptest.NewRequest(http.MethodGet, registry.proxyURL("app-echo")+"/probe", nil) - req.Host = server.apiOrigin.Host - w = httptest.NewRecorder() - server.ServeHTTP(w, req) + w = request(http.MethodGet, server.apiOrigin.String()+"/run/app-echo/probe") } require.Equal(t, http.StatusOK, w.Code, w.Body.String()) require.Equal(t, "application response", w.Body.String()) diff --git a/orchestrator/regression_test.go b/orchestrator/regression_test.go index 7e0e851..59ebf64 100644 --- a/orchestrator/regression_test.go +++ b/orchestrator/regression_test.go @@ -74,7 +74,7 @@ func reviewServer(t *testing.T, upstream string) (*Registry, *Server) { t.Helper() p, err := destination.New("runner", []string{strings.TrimPrefix(upstream, "http://")}) require.NoError(t, err) - reg := NewRegistry("bootstrap", "https://orch.example", time.Second, time.Minute) + reg := NewRegistry("bootstrap", "https://example.com", time.Second, time.Minute) return reg, NewServer(reg, p, p, slog.New(slog.NewTextHandler(io.Discard, nil))) } @@ -87,7 +87,8 @@ func TestRegressionProxySendsContinueBeforeBody(t *testing.T) { registry, server := reviewServer(t, upstream.URL) defer server.Close() require.NoError(t, registry.AddStatic(StaticRunner{ID: "one", RunnerURL: upstream.URL, App: "one", Mode: "single-shot", Capacity: 1})) - orch := newTestHTTPServer(t, server) + orch := httptest.NewServer(server) + server.apiOrigin = mustURL(t, orch.URL) defer orch.Close() address := orch.Listener.Addr().String() conn, err := net.DialTimeout("tcp", address, time.Second) @@ -111,7 +112,7 @@ func TestRegressionOffchainSingleShotCapacityAndHeaders(t *testing.T) { defer close(release) reg, s := reviewServer(t, upstream.URL) require.NoError(t, reg.AddStatic(StaticRunner{ID: "one", RunnerURL: upstream.URL, App: "one", Mode: "single-shot", Capacity: 1})) - go s.ServeHTTP(httptest.NewRecorder(), apiRequest(s, "GET", "/apps/one/app", nil)) + go s.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "/apps/one/app", nil)) header := <-seen t.Run("session_headers", func(t *testing.T) { require.NotEmpty(t, header.Get("Livepeer-Session-Id"), "off-chain single-shot lost its session control headers") @@ -119,7 +120,7 @@ func TestRegressionOffchainSingleShotCapacityAndHeaders(t *testing.T) { done := make(chan int, 1) go func() { w := httptest.NewRecorder() - s.ServeHTTP(w, apiRequest(s, "GET", "/apps/one/app", nil)) + s.ServeHTTP(w, httptest.NewRequest("GET", "/apps/one/app", nil)) done <- w.Code }() select { @@ -195,12 +196,12 @@ func TestRegressionTrickleNextDeliversNextPublishedPart(t *testing.T) { done := make(chan struct{}) w := httptest.NewRecorder() go func() { - s.ServeHTTP(w, apiRequest(s, "GET", "/ai/trickle/test/-1", nil).WithContext(ctx)) + s.ServeHTTP(w, httptest.NewRequest("GET", "/ai/trickle/test/-1", nil).WithContext(ctx)) close(done) }() synctest.Wait() pub := httptest.NewRecorder() - s.ServeHTTP(pub, apiRequest(s, "POST", "/ai/trickle/test/0", strings.NewReader("first"))) + s.ServeHTTP(pub, httptest.NewRequest("POST", "/ai/trickle/test/0", strings.NewReader("first"))) require.Equal(t, 200, pub.Code) synctest.Wait() select { @@ -238,8 +239,8 @@ func TestRegressionTrickleStreamsBeforePublisherEOF(t *testing.T) { ctx, cancel := context.WithCancel(t.Context()) defer cancel() out := &reviewStreamingWriter{header: http.Header{}, wrote: make(chan struct{}, 1)} - go s.ServeHTTP(httptest.NewRecorder(), apiRequest(s, "POST", "/ai/trickle/test/0", input)) - go s.ServeHTTP(out, apiRequest(s, "GET", "/ai/trickle/test/0", nil).WithContext(ctx)) + go s.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("POST", "/ai/trickle/test/0", input)) + go s.ServeHTTP(out, httptest.NewRequest("GET", "/ai/trickle/test/0", nil).WithContext(ctx)) _, err := writer.Write([]byte("streaming chunk")) require.NoError(t, err) synctest.Wait() diff --git a/orchestrator/runner_address_test.go b/orchestrator/runner_address_test.go index bb011ed..e620834 100644 --- a/orchestrator/runner_address_test.go +++ b/orchestrator/runner_address_test.go @@ -34,7 +34,8 @@ func TestRunnerFacingAddressKeepsPublicAndInternalURLsSeparate(t *testing.T) { s := NewServer(registry, policy, policy, slog.New(slog.NewTextHandler(io.Discard, nil))) defer s.Close() - heartbeat := apiRequest(s, http.MethodPost, "http://runner.internal/internal/runners/heartbeat", strings.NewReader(`{"runner_id":"r","runner_url":"`+upstream.URL+`","app":"test","capacity":1}`)) + heartbeat := httptest.NewRequest(http.MethodPost, "http://runner.internal/internal/runners/heartbeat", strings.NewReader(`{"runner_id":"r","runner_url":"`+upstream.URL+`","app":"test","capacity":1}`)) + heartbeat.Host = s.apiOrigin.Host heartbeat.Header.Set("Authorization", "bootstrap") w := httptest.NewRecorder() s.ServeHTTP(w, heartbeat) @@ -44,7 +45,8 @@ func TestRunnerFacingAddressKeepsPublicAndInternalURLsSeparate(t *testing.T) { require.Equal(t, "http://runner.internal/internal", registration.Orchestrator) require.Equal(t, "http://runner.internal/internal/ai/trickle/"+registration.O2R.ChannelName, registration.O2R.URL) require.Empty(t, registration.O2R.InternalURL) - heartbeat = apiRequest(s, http.MethodPost, registration.Orchestrator+"/runners/heartbeat", strings.NewReader(`{"runner_id":"r","runner_url":"`+upstream.URL+`","app":"test","capacity":1}`)) + heartbeat = httptest.NewRequest(http.MethodPost, registration.Orchestrator+"/runners/heartbeat", strings.NewReader(`{"runner_id":"r","runner_url":"`+upstream.URL+`","app":"test","capacity":1}`)) + heartbeat.Host = s.apiOrigin.Host heartbeat.Header.Set("Authorization", registration.HeartbeatSecret) w = httptest.NewRecorder() s.ServeHTTP(w, heartbeat) @@ -76,7 +78,8 @@ func TestRunnerFacingAddressKeepsPublicAndInternalURLsSeparate(t *testing.T) { require.Equal(t, "http://runner.internal/internal/runner/r/session/"+reservation.SessionID, controlHeader) _, token, _, err := registry.sessionTarget("r", reservation.SessionID) require.NoError(t, err) - create := apiRequest(s, http.MethodPost, controlHeader+"/channels", strings.NewReader(`{"channels":[{"name":"events"}]}`)) + create := httptest.NewRequest(http.MethodPost, controlHeader+"/channels", strings.NewReader(`{"channels":[{"name":"events"}]}`)) + create.Host = s.apiOrigin.Host create.Header.Set("Livepeer-Session-Token", token) w = httptest.NewRecorder() s.ServeHTTP(w, create) @@ -91,7 +94,9 @@ func TestRunnerFacingAddressKeepsPublicAndInternalURLsSeparate(t *testing.T) { require.Equal(t, "http://runner.internal/internal/ai/trickle/"+channel.ChannelName, channel.InternalURL) w = httptest.NewRecorder() - s.ServeHTTP(w, apiRequest(s, http.MethodPost, channel.InternalURL+"/0", bytes.NewBufferString("message"))) + create = httptest.NewRequest(http.MethodPost, channel.InternalURL+"/0", bytes.NewBufferString("message")) + create.Host = s.apiOrigin.Host + s.ServeHTTP(w, create) require.Equal(t, http.StatusOK, w.Code, w.Body.String()) w = httptest.NewRecorder() s.ServeHTTP(w, httptest.NewRequest(http.MethodGet, channel.URL+"/0", nil)) @@ -102,18 +107,19 @@ func TestRunnerFacingAddressKeepsPublicAndInternalURLsSeparate(t *testing.T) { func TestO2RKeepaliveRepeatsUntilShutdown(t *testing.T) { policy, err := destination.New("runner", nil) require.NoError(t, err) - registry := NewRegistry("bootstrap", "http://orchestrator.example", time.Second, time.Minute) + registry := NewRegistry("bootstrap", "http://example.com", time.Second, time.Minute) s := NewServer(registry, policy, policy, slog.New(slog.NewTextHandler(io.Discard, nil))) defer s.Close() w := httptest.NewRecorder() - heartbeat := apiRequest(s, http.MethodPost, "/runners/heartbeat", strings.NewReader(`{"runner_id":"r","runner_url":"https://runner.example","app":"test"}`)) + heartbeat := httptest.NewRequest(http.MethodPost, "/runners/heartbeat", strings.NewReader(`{"runner_id":"r","runner_url":"https://runner.example","app":"test"}`)) heartbeat.Header.Set("Authorization", "bootstrap") s.ServeHTTP(w, heartbeat) require.Equal(t, http.StatusOK, w.Code, w.Body.String()) var registration heartbeatResponse require.NoError(t, json.Unmarshal(w.Body.Bytes(), ®istration)) - server := newTestHTTPServer(t, s) + server := httptest.NewServer(s) + s.apiOrigin = mustURL(t, server.URL) defer server.Close() ctx, cancel := context.WithCancel(t.Context()) done := make(chan struct{}) diff --git a/orchestrator/seq_test.go b/orchestrator/seq_test.go index c7ff573..89e58d0 100644 --- a/orchestrator/seq_test.go +++ b/orchestrator/seq_test.go @@ -41,7 +41,7 @@ func TestOwnerSeqOrdering(t *testing.T) { } lastLegacy := registry.runners["owner"].Last for _, seq := range []string{`-1`, `1.5`, `"1"`, `18446744073709551616`, `true`, `[]`} { - req := apiRequest(server, http.MethodPost, "/runners/heartbeat", strings.NewReader(`{"runner_id":"owner","seq":`+seq+`,"apps":[]}`)) + req := httptest.NewRequest(http.MethodPost, "/runners/heartbeat", strings.NewReader(`{"runner_id":"owner","seq":`+seq+`,"apps":[]}`)) req.Header.Set("Authorization", owner.HeartbeatSecret) w := httptest.NewRecorder() server.ServeHTTP(w, req) diff --git a/orchestrator/trickle_test.go b/orchestrator/trickle_test.go index 218443e..4cb3c17 100644 --- a/orchestrator/trickle_test.go +++ b/orchestrator/trickle_test.go @@ -24,7 +24,7 @@ func TestDirectTrickleDeleteAllowsNamedChannelRecreation(t *testing.T) { require.NoError(t, err) create := func() string { t.Helper() - req := apiRequest(s, http.MethodPost, "/runner/r/session/"+sid+"/channels", strings.NewReader(`{"channels":[{"name":"events"}]}`)) + req := httptest.NewRequest(http.MethodPost, "/runner/r/session/"+sid+"/channels", strings.NewReader(`{"channels":[{"name":"events"}]}`)) req.Header.Set("Livepeer-Session-Token", token) w := httptest.NewRecorder() s.ServeHTTP(w, req) @@ -40,7 +40,7 @@ func TestDirectTrickleDeleteAllowsNamedChannelRecreation(t *testing.T) { } first := create() w := httptest.NewRecorder() - s.ServeHTTP(w, apiRequest(s, http.MethodDelete, "/ai/trickle/"+first, nil)) + s.ServeHTTP(w, httptest.NewRequest(http.MethodDelete, "/ai/trickle/"+first, nil)) require.Equal(t, http.StatusOK, w.Code) require.NotContains(t, s.channels, first) second := create() @@ -72,14 +72,14 @@ func TestTrickleActiveStreamAndCancellation(t *testing.T) { defer writer.Close() pubDone, subDone := make(chan struct{}), make(chan struct{}) go func() { - s.ServeHTTP(httptest.NewRecorder(), apiRequest(s, "POST", "/ai/trickle/test/0", input)) + s.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("POST", "/ai/trickle/test/0", input)) close(pubDone) }() ctx, cancel := context.WithCancel(t.Context()) defer cancel() out := &reviewStreamingWriter{header: http.Header{}, wrote: make(chan struct{}, 1)} go func() { - s.ServeHTTP(out, apiRequest(s, "GET", "/ai/trickle/test/0", nil).WithContext(ctx)) + s.ServeHTTP(out, httptest.NewRequest("GET", "/ai/trickle/test/0", nil).WithContext(ctx)) close(subDone) }() _, err = writer.Write([]byte("first")) @@ -97,9 +97,9 @@ func TestTrickleActiveStreamAndCancellation(t *testing.T) { } switch action { case "delete": - s.ServeHTTP(httptest.NewRecorder(), apiRequest(s, "DELETE", "/ai/trickle/test/0", nil)) + s.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("DELETE", "/ai/trickle/test/0", nil)) case "reset": - req := apiRequest(s, "POST", "/ai/trickle/test/1", strings.NewReader("next")) + req := httptest.NewRequest("POST", "/ai/trickle/test/1", strings.NewReader("next")) req.Header.Set("Lp-Trickle-Reset", "true") s.ServeHTTP(httptest.NewRecorder(), req) case "release": @@ -178,7 +178,7 @@ func TestNamedChannelsAreIdempotentAndReleased(t *testing.T) { require.NoError(t, err) var previous string for range 2 { - req := apiRequest(s, "POST", "/runner/r/session/"+sid+"/channels", strings.NewReader(`{"channels":[{"name":"events"}]}`)) + req := httptest.NewRequest("POST", "/runner/r/session/"+sid+"/channels", strings.NewReader(`{"channels":[{"name":"events"}]}`)) req.Header.Set("Livepeer-Session-Token", token) w := httptest.NewRecorder() s.ServeHTTP(w, req) @@ -202,8 +202,8 @@ func TestTrickleLargeSegmentDelivered(t *testing.T) { defer writer.Close() pub, sub := httptest.NewRecorder(), httptest.NewRecorder() pubDone, subDone := make(chan struct{}), make(chan struct{}) - go func() { s.ServeHTTP(pub, apiRequest(s, "POST", "/ai/trickle/large/0", input)); close(pubDone) }() - go func() { s.ServeHTTP(sub, apiRequest(s, "GET", "/ai/trickle/large/0", nil)); close(subDone) }() + go func() { s.ServeHTTP(pub, httptest.NewRequest("POST", "/ai/trickle/large/0", input)); close(pubDone) }() + go func() { s.ServeHTTP(sub, httptest.NewRequest("GET", "/ai/trickle/large/0", nil)); close(subDone) }() chunk := strings.Repeat("x", 32<<10) for range (3 << 20) / len(chunk) { _, err := io.WriteString(writer, chunk) @@ -216,7 +216,7 @@ func TestTrickleLargeSegmentDelivered(t *testing.T) { require.Equal(t, http.StatusOK, pub.Code) require.Equal(t, 3<<20, sub.Body.Len()) late := httptest.NewRecorder() - s.ServeHTTP(late, apiRequest(s, "GET", "/ai/trickle/large/0", nil)) + s.ServeHTTP(late, httptest.NewRequest("GET", "/ai/trickle/large/0", nil)) require.Equal(t, http.StatusOK, late.Code) require.Equal(t, 3<<20, late.Body.Len()) }) From efb4bbd87b5fb3a3de32ef088d12d4fb8745319b Mon Sep 17 00:00:00 2001 From: Josh Allmann Date: Thu, 12 Nov 2026 09:01:03 -0800 Subject: [PATCH 3/5] Preserve authority precedence coverage and clarify runner URLs --- docs/orchestrator.md | 7 +++---- orchestrator/app_test.go | 2 +- orchestrator/proxy_test.go | 1 + 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/orchestrator.md b/docs/orchestrator.md index 11573c9..86f4d69 100644 --- a/docs/orchestrator.md +++ b/docs/orchestrator.md @@ -60,11 +60,10 @@ require grants. ## Public URLs and TLS `ServiceURL` is the public base URL advertised to clients, defaulting to -`http://`. Set `RunnerServiceURL` if runners reach the listener through -a different URL such as Docker. That URL is used for heartbeat responses, +`http://`. `RunnerServiceURL` overrides it for heartbeat responses, session callbacks, Trickle `internal_url`, and the orchestrator-to-runner (O2R) -control channel. Client discovery, session, proxy, and channel URLs continue -to use `ServiceURL`. +control channel. Runner requests must use the Host from `APIOriginURL`. +Client discovery, session, proxy, and channel URLs use `ServiceURL`. The application listener serves HTTP and defaults to loopback. Use explicit IP:port addresses for `Listen` and `MetricsListen`, including IPv6 addresses diff --git a/orchestrator/app_test.go b/orchestrator/app_test.go index 69a1045..653fc7f 100644 --- a/orchestrator/app_test.go +++ b/orchestrator/app_test.go @@ -63,7 +63,7 @@ func TestAPIOriginURLRouting(t *testing.T) { cmd := Root(io.Discard, io.Discard) cmd.SetArgs([]string{"--config", "", "--listen", address, "--metrics-listen", metrics, "--service-url", "https://public.example", - "--api-origin-url", "http://" + origin + ":80", "--runner-service-url", "http://" + origin}) + "--api-origin-url", "http://" + origin + ":80"}) ctx, cancel := context.WithCancel(t.Context()) done := make(chan error, 1) go func() { done <- cmd.ExecuteContext(ctx) }() diff --git a/orchestrator/proxy_test.go b/orchestrator/proxy_test.go index 0eeeaaf..da8adfc 100644 --- a/orchestrator/proxy_test.go +++ b/orchestrator/proxy_test.go @@ -143,6 +143,7 @@ func TestOriginDefaultPorts(t *testing.T) { policy, err := destination.New("runner", nil) require.NoError(t, err) registry := NewRegistry("bootstrap", tc.origin, time.Second, time.Minute) + registry.proxyTemplate = "https://{proxy}.example" server := NewServer(registry, policy, policy, slog.New(slog.NewTextHandler(io.Discard, nil))) defer server.Close() for _, request := range []struct { From d26a41aebb558148e81c54fc757f28f4b9c8c742 Mon Sep 17 00:00:00 2001 From: Josh Allmann Date: Thu, 12 Nov 2026 09:13:47 -0800 Subject: [PATCH 4/5] Allow the configured runner Host on runner routes --- .../testdata/cli/orchestrator-help.golden | 2 +- configs/orchestrator/config.example.toml | 2 +- docs/orchestrator.md | 9 +++--- orchestrator/app.go | 2 +- orchestrator/http.go | 24 ++++++++-------- orchestrator/proxy.go | 6 ++-- orchestrator/runner_address_test.go | 28 ++++++++++--------- 7 files changed, 38 insertions(+), 35 deletions(-) diff --git a/cmd/livepeer/testdata/cli/orchestrator-help.golden b/cmd/livepeer/testdata/cli/orchestrator-help.golden index 28a1337..4aea0c1 100644 --- a/cmd/livepeer/testdata/cli/orchestrator-help.golden +++ b/cmd/livepeer/testdata/cli/orchestrator-help.golden @@ -21,7 +21,7 @@ Flags: -l, --listen string Public HTTP listener (env: LIVEPEER_ORCHESTRATOR_LISTEN) (default "127.0.0.1:8935") -m, --metrics-listen string Metrics listener (env: LIVEPEER_ORCHESTRATOR_METRICS_LISTEN) (default "127.0.0.1:8936") -s, --service-url string Public orchestrator base URL; defaults to http://listen (env: LIVEPEER_ORCHESTRATOR_SERVICE_URL) - --api-origin-url string Required API request Host as a URL; defaults to service-url (env: LIVEPEER_ORCHESTRATOR_API_ORIGIN_URL) + --api-origin-url string Client API Host as a URL; defaults to service-url (env: LIVEPEER_ORCHESTRATOR_API_ORIGIN_URL) -r, --runner-service-url string Runner-facing base URL for callbacks and trickle; defaults to service-url (env: LIVEPEER_ORCHESTRATOR_RUNNER_SERVICE_URL) --proxy-url-template string Generated proxy URL with {proxy} in a hostname label or final path segment (env: LIVEPEER_ORCHESTRATOR_PROXY_URL_TEMPLATE) --excluded-headers strings Headers to remove from app requests; a trailing * matches a prefix (env: LIVEPEER_ORCHESTRATOR_EXCLUDED_HEADERS) diff --git a/configs/orchestrator/config.example.toml b/configs/orchestrator/config.example.toml index 035a18b..4eca4c3 100644 --- a/configs/orchestrator/config.example.toml +++ b/configs/orchestrator/config.example.toml @@ -3,7 +3,7 @@ Listen = "127.0.0.1:8935" MetricsListen = "127.0.0.1:8936" ServiceURL = "http://127.0.0.1:8935" -# APIOriginURL = "http://orchestrator.internal:8935" # API request Host sent by the reverse proxy; defaults to ServiceURL. +# APIOriginURL = "http://orchestrator.internal:8935" # Client API Host sent by the reverse proxy; defaults to ServiceURL. # RunnerServiceURL = "http://127.0.0.1:8935" # Runner-facing callbacks and trickle; defaults to ServiceURL. BootstrapSecretFile = "/run/secrets/livepeer-runner-bootstrap" RunnerConfig = "/etc/livepeer/runners.toml" diff --git a/docs/orchestrator.md b/docs/orchestrator.md index 86f4d69..c6c333a 100644 --- a/docs/orchestrator.md +++ b/docs/orchestrator.md @@ -60,10 +60,11 @@ require grants. ## Public URLs and TLS `ServiceURL` is the public base URL advertised to clients, defaulting to -`http://`. `RunnerServiceURL` overrides it for heartbeat responses, +`http://`. Set `RunnerServiceURL` if runners reach the listener through +a different URL such as Docker. That URL is used for heartbeat responses, session callbacks, Trickle `internal_url`, and the orchestrator-to-runner (O2R) -control channel. Runner requests must use the Host from `APIOriginURL`. -Client discovery, session, proxy, and channel URLs use `ServiceURL`. +control channel. Client discovery, session, proxy, and channel URLs continue +to use `ServiceURL`. The application listener serves HTTP and defaults to loopback. Use explicit IP:port addresses for `Listen` and `MetricsListen`, including IPv6 addresses @@ -102,7 +103,7 @@ DNS, TLS, and forwarding to this listener for those URLs. Hostname routing uses the actual Host header. Single-shot domain proxies require an app registration ID that is a lowercase DNS label of at most 63 characters. -API requests must use the Host from `APIOriginURL`, which defaults to +Client API requests use the Host from `APIOriginURL`, which defaults to `ServiceURL`. If a reverse proxy rewrites Host, set `APIOriginURL` to an HTTP(S) URL with the Host it sends, without a path. diff --git a/orchestrator/app.go b/orchestrator/app.go index c1f2118..e1a69fa 100644 --- a/orchestrator/app.go +++ b/orchestrator/app.go @@ -41,7 +41,7 @@ type Params struct { Listen boa.HTTPBind `default:"127.0.0.1:8935" descr:"Public HTTP listener"` MetricsListen boa.HTTPBind `default:"127.0.0.1:8936" descr:"Metrics listener"` ServiceURL boa.Text[*url.URL] `optional:"true" descr:"Public orchestrator base URL; defaults to http://listen"` - APIOriginURL boa.Text[*url.URL] `optional:"true" descr:"Required API request Host as a URL; defaults to service-url"` + APIOriginURL boa.Text[*url.URL] `optional:"true" descr:"Client API Host as a URL; defaults to service-url"` RunnerServiceURL boa.Text[*url.URL] `optional:"true" descr:"Runner-facing base URL for callbacks and trickle; defaults to service-url"` ProxyURLTemplate string `optional:"true" descr:"Generated proxy URL with {proxy} in a hostname label or final path segment"` ExcludedHeaders []string `optional:"true" descr:"Headers to remove from app requests; a trailing * matches a prefix"` diff --git a/orchestrator/http.go b/orchestrator/http.go index 6589c20..857c1da 100644 --- a/orchestrator/http.go +++ b/orchestrator/http.go @@ -29,7 +29,7 @@ const ( type Server struct { basePath string - runnerBasePath string + runnerOrigin *url.URL apiOrigin *url.URL registry *Registry runnerTransport *http.Transport @@ -53,8 +53,8 @@ func NewServer(registry *Registry, runnerPolicy, proxyPolicy destination.Policy, serviceURL, _ := url.Parse(registry.service) s.basePath = strings.TrimRight(serviceURL.Path, "/") s.apiOrigin = serviceURL - runnerURL, _ := url.Parse(registry.runnerServiceURL()) - s.runnerBasePath = strings.TrimRight(runnerURL.Path, "/") + s.runnerOrigin, _ = url.Parse(registry.runnerServiceURL()) + s.runnerOrigin.Path = strings.TrimRight(s.runnerOrigin.Path, "/") s.runnerTransport = runnerPolicy.Transport(time.Minute) s.proxyTransport = proxyPolicy.Transport(time.Minute) // A stalled stream must not take the slot needed to stop or fund it. @@ -98,20 +98,15 @@ func (s *Server) serveHTTP(w http.ResponseWriter, r *http.Request) { fail(w, http.StatusBadRequest, "invalid path") return } - proxyID, appPath := s.registry.matchProxy(r, s.apiOrigin) + proxyID, appPath := s.registry.matchProxy(r, s.apiOrigin, s.runnerOrigin) if proxyID == "" { - // Only the API Host may reach the API router. - if !matchesOrigin(r.Host, s.apiOrigin) { - fail(w, http.StatusNotFound, "route not found") - return - } basePath := "" - for _, candidate := range []string{s.basePath, s.runnerBasePath} { - if len(candidate) > len(basePath) && strings.HasPrefix(r.URL.Path, candidate+"/") { + for _, candidate := range []string{s.basePath, s.runnerOrigin.Path} { + if len(candidate) > len(basePath) && strings.HasPrefix(r.URL.Path, candidate+"/") && (candidate == s.runnerOrigin.Path || matchesOrigin(r.Host, s.apiOrigin)) { basePath = candidate } } - if basePath == "" && s.basePath != "" && s.runnerBasePath != "" { + if basePath == "" && s.basePath != "" && s.runnerOrigin.Path != "" { fail(w, 404, "route not found") return } @@ -120,6 +115,11 @@ func (s *Server) serveHTTP(w http.ResponseWriter, r *http.Request) { r.URL.Path = strings.TrimPrefix(r.URL.Path, basePath) r.URL.RawPath = "" } + runnerRoute := strings.HasPrefix(r.URL.Path, "/runners/") || strings.HasPrefix(r.URL.Path, "/runner/") || strings.HasPrefix(r.URL.Path, "/ai/trickle/") + if !matchesOrigin(r.Host, s.apiOrigin) && !(runnerRoute && basePath == s.runnerOrigin.Path && s.registry.runnerService != "" && matchesOrigin(r.Host, s.runnerOrigin)) { + fail(w, http.StatusNotFound, "route not found") + return + } } stream := proxyID != "" || strings.HasPrefix(r.URL.Path, "/ai/trickle/") || strings.HasPrefix(r.URL.Path, "/proxy/") || strings.HasPrefix(r.URL.Path, "/run/") || strings.HasSuffix(r.URL.Path, "/app") || strings.Contains(r.URL.Path, "/app/") _, pattern := s.mux.Handler(r) diff --git a/orchestrator/proxy.go b/orchestrator/proxy.go index c57f5ce..19c6d66 100644 --- a/orchestrator/proxy.go +++ b/orchestrator/proxy.go @@ -38,14 +38,14 @@ func (r *Registry) proxyURL(id string) string { // The template is fixed before serving. Match the actual Host, never a // forwarded header supplied by a caller, and retain every application path. -func (r *Registry) matchProxy(req *http.Request, apiOrigin *url.URL) (string, string) { +func (r *Registry) matchProxy(req *http.Request, apiOrigin, runnerOrigin *url.URL) (string, string) { if r.proxyTemplate == "" { return "", "" } u, _ := url.Parse(strings.Replace(r.proxyTemplate, "{proxy}", proxyMarker, 1)) if before, after, ok := strings.Cut(strings.ToLower(u.Host), proxyMarker); ok { - // The API Host takes precedence over wildcard proxy Hosts. - if matchesOrigin(req.Host, apiOrigin) { + // API and runner Hosts take precedence over wildcard proxy Hosts. + if matchesOrigin(req.Host, apiOrigin) || r.runnerService != "" && matchesOrigin(req.Host, runnerOrigin) { return "", "" } host := strings.ToLower(req.Host) diff --git a/orchestrator/runner_address_test.go b/orchestrator/runner_address_test.go index e620834..806f0c8 100644 --- a/orchestrator/runner_address_test.go +++ b/orchestrator/runner_address_test.go @@ -29,13 +29,12 @@ func TestRunnerFacingAddressKeepsPublicAndInternalURLsSeparate(t *testing.T) { defer upstream.Close() policy, err := destination.New("runner", []string{strings.TrimPrefix(upstream.URL, "http://")}) require.NoError(t, err) - registry := NewRegistry("bootstrap", "https://public.example/external", time.Second, time.Minute) + registry := NewRegistry("bootstrap", "https://public.example/internal/runners", time.Second, time.Minute) registry.runnerService = "http://runner.internal/internal" s := NewServer(registry, policy, policy, slog.New(slog.NewTextHandler(io.Discard, nil))) defer s.Close() heartbeat := httptest.NewRequest(http.MethodPost, "http://runner.internal/internal/runners/heartbeat", strings.NewReader(`{"runner_id":"r","runner_url":"`+upstream.URL+`","app":"test","capacity":1}`)) - heartbeat.Host = s.apiOrigin.Host heartbeat.Header.Set("Authorization", "bootstrap") w := httptest.NewRecorder() s.ServeHTTP(w, heartbeat) @@ -45,22 +44,28 @@ func TestRunnerFacingAddressKeepsPublicAndInternalURLsSeparate(t *testing.T) { require.Equal(t, "http://runner.internal/internal", registration.Orchestrator) require.Equal(t, "http://runner.internal/internal/ai/trickle/"+registration.O2R.ChannelName, registration.O2R.URL) require.Empty(t, registration.O2R.InternalURL) + registry.proxyTemplate = "https://{proxy}.internal" heartbeat = httptest.NewRequest(http.MethodPost, registration.Orchestrator+"/runners/heartbeat", strings.NewReader(`{"runner_id":"r","runner_url":"`+upstream.URL+`","app":"test","capacity":1}`)) - heartbeat.Host = s.apiOrigin.Host heartbeat.Header.Set("Authorization", registration.HeartbeatSecret) w = httptest.NewRecorder() s.ServeHTTP(w, heartbeat) require.Equal(t, http.StatusOK, w.Code, w.Body.String()) + for _, path := range []string{"/internal/discovery", "/external/runners/heartbeat"} { + w = httptest.NewRecorder() + s.ServeHTTP(w, httptest.NewRequest(http.MethodPost, "http://runner.internal"+path, nil)) + require.Equal(t, http.StatusNotFound, w.Code, path) + } + w = httptest.NewRecorder() - s.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "https://public.example/external/discovery", nil)) + s.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "https://public.example/internal/runners/discovery", nil)) require.Equal(t, http.StatusOK, w.Code) var discovered []discoveryEntry require.NoError(t, json.Unmarshal(w.Body.Bytes(), &discovered)) - require.Equal(t, "https://public.example/external", discovered[0].Address) + require.Equal(t, "https://public.example/internal/runners", discovered[0].Address) w = httptest.NewRecorder() - s.ServeHTTP(w, httptest.NewRequest(http.MethodPost, "https://public.example/external/apps/r/session", nil)) + s.ServeHTTP(w, httptest.NewRequest(http.MethodPost, "https://public.example/internal/runners/apps/r/session", nil)) require.Equal(t, http.StatusOK, w.Code, w.Body.String()) var reservation struct { SessionID string `json:"session_id"` @@ -68,8 +73,8 @@ func TestRunnerFacingAddressKeepsPublicAndInternalURLsSeparate(t *testing.T) { ControlURL string `json:"control_url"` } require.NoError(t, json.Unmarshal(w.Body.Bytes(), &reservation)) - require.Equal(t, "https://public.example/external/apps/r/session/"+reservation.SessionID+"/app", reservation.AppURL) - require.Equal(t, "https://public.example/external/apps/r/session/"+reservation.SessionID, reservation.ControlURL) + require.Equal(t, "https://public.example/internal/runners/apps/r/session/"+reservation.SessionID+"/app", reservation.AppURL) + require.Equal(t, "https://public.example/internal/runners/apps/r/session/"+reservation.SessionID, reservation.ControlURL) w = httptest.NewRecorder() s.ServeHTTP(w, httptest.NewRequest(http.MethodGet, reservation.AppURL+"/hello", nil)) @@ -79,7 +84,6 @@ func TestRunnerFacingAddressKeepsPublicAndInternalURLsSeparate(t *testing.T) { _, token, _, err := registry.sessionTarget("r", reservation.SessionID) require.NoError(t, err) create := httptest.NewRequest(http.MethodPost, controlHeader+"/channels", strings.NewReader(`{"channels":[{"name":"events"}]}`)) - create.Host = s.apiOrigin.Host create.Header.Set("Livepeer-Session-Token", token) w = httptest.NewRecorder() s.ServeHTTP(w, create) @@ -90,13 +94,11 @@ func TestRunnerFacingAddressKeepsPublicAndInternalURLsSeparate(t *testing.T) { require.NoError(t, json.Unmarshal(w.Body.Bytes(), &created)) require.Len(t, created.Channels, 1) channel := created.Channels[0] - require.Equal(t, "https://public.example/external/ai/trickle/"+channel.ChannelName, channel.URL) + require.Equal(t, "https://public.example/internal/runners/ai/trickle/"+channel.ChannelName, channel.URL) require.Equal(t, "http://runner.internal/internal/ai/trickle/"+channel.ChannelName, channel.InternalURL) w = httptest.NewRecorder() - create = httptest.NewRequest(http.MethodPost, channel.InternalURL+"/0", bytes.NewBufferString("message")) - create.Host = s.apiOrigin.Host - s.ServeHTTP(w, create) + s.ServeHTTP(w, httptest.NewRequest(http.MethodPost, channel.InternalURL+"/0", bytes.NewBufferString("message"))) require.Equal(t, http.StatusOK, w.Code, w.Body.String()) w = httptest.NewRecorder() s.ServeHTTP(w, httptest.NewRequest(http.MethodGet, channel.URL+"/0", nil)) From b722fe80a256cbb0478c8ac06d62f70ca82bb068 Mon Sep 17 00:00:00 2001 From: Josh Allmann Date: Thu, 12 Nov 2026 09:19:26 -0800 Subject: [PATCH 5/5] Cover runner base-path rejection in the authority matrix --- orchestrator/proxy_test.go | 5 ++++- orchestrator/runner_address_test.go | 6 ------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/orchestrator/proxy_test.go b/orchestrator/proxy_test.go index da8adfc..8e3c645 100644 --- a/orchestrator/proxy_test.go +++ b/orchestrator/proxy_test.go @@ -178,7 +178,7 @@ func TestProxyHostCannotReachAPI(t *testing.T) { policy, err := destination.New("runner", []string{strings.TrimPrefix(upstream.URL, "http://")}) require.NoError(t, err) registry := NewRegistry("bootstrap", "https://api-aaaaaaa.machine.example", time.Second, time.Minute) - registry.runnerService = "http://runner.internal:8935" + registry.runnerService = "http://runner.internal:8935/internal" registry.proxyTemplate = template require.NoError(t, registry.AddStatic(StaticRunner{ID: "app-fixture", RunnerURL: upstream.URL, App: "fixture", Mode: "persistent", Capacity: 4})) require.NoError(t, registry.AddStatic(StaticRunner{ID: "app-echo", RunnerURL: upstream.URL, App: "echo", Mode: "single-shot", Proxy: true, Capacity: 4})) @@ -197,6 +197,9 @@ func TestProxyHostCannotReachAPI(t *testing.T) { server.ServeHTTP(w, req) return w } + if apiOrigin != "runner.internal:8935" { + require.Equal(t, http.StatusNotFound, request(http.MethodPost, "http://runner.internal:8935/runners/heartbeat").Code) + } for _, host := range []string{ "app-echo.machine.example:8443", "app-echo.machine.example:443", diff --git a/orchestrator/runner_address_test.go b/orchestrator/runner_address_test.go index 806f0c8..319c6ef 100644 --- a/orchestrator/runner_address_test.go +++ b/orchestrator/runner_address_test.go @@ -51,12 +51,6 @@ func TestRunnerFacingAddressKeepsPublicAndInternalURLsSeparate(t *testing.T) { s.ServeHTTP(w, heartbeat) require.Equal(t, http.StatusOK, w.Code, w.Body.String()) - for _, path := range []string{"/internal/discovery", "/external/runners/heartbeat"} { - w = httptest.NewRecorder() - s.ServeHTTP(w, httptest.NewRequest(http.MethodPost, "http://runner.internal"+path, nil)) - require.Equal(t, http.StatusNotFound, w.Code, path) - } - w = httptest.NewRecorder() s.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "https://public.example/internal/runners/discovery", nil)) require.Equal(t, http.StatusOK, w.Code)