Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cmd/livepeer/testdata/cli/orchestrator-help.golden
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ Flags:
-l, --listen string Public HTTP listener (env: LIVEPEER_ORCHESTRATOR_LISTEN) (default "127.0.0.1:8935")
-m, --metrics-listen string Metrics listener (env: LIVEPEER_ORCHESTRATOR_METRICS_LISTEN) (default "127.0.0.1:8936")
-s, --service-url string Public orchestrator base URL; defaults to http://listen (env: LIVEPEER_ORCHESTRATOR_SERVICE_URL)
--api-origin-url string API request Host as a URL when proxy-url-template is set; defaults to service-url (env: LIVEPEER_ORCHESTRATOR_API_ORIGIN_URL)
--api-origin-url string Client API Host as a URL; defaults to service-url (env: LIVEPEER_ORCHESTRATOR_API_ORIGIN_URL)
-r, --runner-service-url string Runner-facing base URL for callbacks and trickle; defaults to service-url (env: LIVEPEER_ORCHESTRATOR_RUNNER_SERVICE_URL)
--proxy-url-template string Generated proxy URL with {proxy} in a hostname label or final path segment (env: LIVEPEER_ORCHESTRATOR_PROXY_URL_TEMPLATE)
--excluded-headers strings Headers to remove from app requests; a trailing * matches a prefix (env: LIVEPEER_ORCHESTRATOR_EXCLUDED_HEADERS)
Expand Down
2 changes: 1 addition & 1 deletion configs/orchestrator/config.example.toml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
Listen = "127.0.0.1:8935"
MetricsListen = "127.0.0.1:8936"
ServiceURL = "http://127.0.0.1:8935"
# APIOriginURL = "http://orchestrator.internal:8935" # Host sent by the reverse proxy when using ProxyURLTemplate; defaults to ServiceURL.
# APIOriginURL = "http://orchestrator.internal:8935" # Client API Host sent by the reverse proxy; defaults to ServiceURL.
# RunnerServiceURL = "http://127.0.0.1:8935" # Runner-facing callbacks and trickle; defaults to ServiceURL.
BootstrapSecretFile = "/run/secrets/livepeer-runner-bootstrap"
RunnerConfig = "/etc/livepeer/runners.toml"
Expand Down
8 changes: 3 additions & 5 deletions docs/orchestrator.md
Original file line number Diff line number Diff line change
Expand Up @@ -103,11 +103,9 @@ DNS, TLS, and forwarding to this listener for those URLs. Hostname routing uses
the actual Host header. Single-shot domain proxies require an app registration ID
that is a lowercase DNS label of at most 63 characters.

With `ProxyURLTemplate`, API requests must use the Host from `APIOriginURL`,
which defaults to `ServiceURL`. If a reverse proxy changes that Host, set
`APIOriginURL` to an HTTP(S) URL with the Host it sends, without a path. Runner
requests must use the same Host; configure `RunnerServiceURL` or the reverse proxy
accordingly. `APIOriginURL` does not change advertised URLs.
Client API requests use the Host from `APIOriginURL`, which defaults to
`ServiceURL`. If a reverse proxy rewrites Host, set `APIOriginURL` to an
HTTP(S) URL with the Host it sends, without a path.

To remove headers from app requests, set `ExcludedHeaders`, for example:
`["CF-Access-*", "Cookie"]`. Matching is case-insensitive.
Expand Down
2 changes: 1 addition & 1 deletion orchestrator/app.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ type Params struct {
Listen boa.HTTPBind `default:"127.0.0.1:8935" descr:"Public HTTP listener"`
MetricsListen boa.HTTPBind `default:"127.0.0.1:8936" descr:"Metrics listener"`
ServiceURL boa.Text[*url.URL] `optional:"true" descr:"Public orchestrator base URL; defaults to http://listen"`
APIOriginURL boa.Text[*url.URL] `optional:"true" descr:"API request Host as a URL when proxy-url-template is set; defaults to service-url"`
APIOriginURL boa.Text[*url.URL] `optional:"true" descr:"Client API Host as a URL; defaults to service-url"`
RunnerServiceURL boa.Text[*url.URL] `optional:"true" descr:"Runner-facing base URL for callbacks and trickle; defaults to service-url"`
ProxyURLTemplate string `optional:"true" descr:"Generated proxy URL with {proxy} in a hostname label or final path segment"`
ExcludedHeaders []string `optional:"true" descr:"Headers to remove from app requests; a trailing * matches a prefix"`
Expand Down
4 changes: 2 additions & 2 deletions orchestrator/app_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -63,8 +63,7 @@ func TestAPIOriginURLRouting(t *testing.T) {
cmd := Root(io.Discard, io.Discard)
cmd.SetArgs([]string{"--config", "", "--listen", address, "--metrics-listen", metrics,
"--service-url", "https://public.example",
"--api-origin-url", "http://" + origin + ":80", "--runner-service-url", "http://" + origin,
"--proxy-url-template", "https://{proxy}.internal"})
"--api-origin-url", "http://" + origin + ":80"})
ctx, cancel := context.WithCancel(t.Context())
done := make(chan error, 1)
go func() { done <- cmd.ExecuteContext(ctx) }()
Expand Down Expand Up @@ -146,6 +145,7 @@ func TestPaidHTTPStartup(t *testing.T) {
require.NoError(t, p.MetricsListen.UnmarshalText(fmt.Appendf(nil, "unsafe@0.0.0.0:%d", metricsPort)))
metricsURL := fmt.Sprintf("http://127.0.0.1:%d", metricsPort)
p.ServiceURL = boa.Text[*url.URL]{Value: mustURL(t, "https://public.example/external")}
p.APIOriginURL = boa.Text[*url.URL]{Value: mustURL(t, address)}
p.RunnerConfig = filepath.Join(t.TempDir(), "runners.toml")
require.NoError(t, os.WriteFile(p.RunnerConfig, []byte("[[Runners]]\nID = 'r'\nRunnerURL = 'https://runner.example'\nApp = 'test'\n[Runners.PriceInfo]\nPrice = 1\n"), 0600))
ctx, cancel := context.WithCancel(t.Context())
Expand Down
1 change: 1 addition & 0 deletions orchestrator/discovery_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ func TestDiscoveryLocalInventory(t *testing.T) {
s := NewServer(registry, policy, policy, slog.New(slog.NewTextHandler(io.Discard, nil)))
t.Cleanup(s.Close)
server := httptest.NewServer(s)
s.apiOrigin = mustURL(t, server.URL)
t.Cleanup(server.Close)
response, err := server.Client().Get(server.URL + "/deployment/discovery")
require.NoError(t, err)
Expand Down
24 changes: 12 additions & 12 deletions orchestrator/http.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ const (

type Server struct {
basePath string
runnerBasePath string
runnerOrigin *url.URL
apiOrigin *url.URL
registry *Registry
runnerTransport *http.Transport
Expand All @@ -53,8 +53,8 @@ func NewServer(registry *Registry, runnerPolicy, proxyPolicy destination.Policy,
serviceURL, _ := url.Parse(registry.service)
s.basePath = strings.TrimRight(serviceURL.Path, "/")
s.apiOrigin = serviceURL
runnerURL, _ := url.Parse(registry.runnerServiceURL())
s.runnerBasePath = strings.TrimRight(runnerURL.Path, "/")
s.runnerOrigin, _ = url.Parse(registry.runnerServiceURL())
s.runnerOrigin.Path = strings.TrimRight(s.runnerOrigin.Path, "/")
s.runnerTransport = runnerPolicy.Transport(time.Minute)
s.proxyTransport = proxyPolicy.Transport(time.Minute)
// A stalled stream must not take the slot needed to stop or fund it.
Expand Down Expand Up @@ -98,20 +98,15 @@ func (s *Server) serveHTTP(w http.ResponseWriter, r *http.Request) {
fail(w, http.StatusBadRequest, "invalid path")
return
}
proxyID, appPath := s.registry.matchProxy(r, s.apiOrigin)
proxyID, appPath := s.registry.matchProxy(r, s.apiOrigin, s.runnerOrigin)
if proxyID == "" {
// Unknown proxy Hosts must never fall through to the API router.
if s.registry.proxyTemplate != "" && !matchesOrigin(r.Host, s.apiOrigin) {
fail(w, http.StatusNotFound, "route not found")
return
}
basePath := ""
for _, candidate := range []string{s.basePath, s.runnerBasePath} {
if len(candidate) > len(basePath) && strings.HasPrefix(r.URL.Path, candidate+"/") {
for _, candidate := range []string{s.basePath, s.runnerOrigin.Path} {
if len(candidate) > len(basePath) && strings.HasPrefix(r.URL.Path, candidate+"/") && (candidate == s.runnerOrigin.Path || matchesOrigin(r.Host, s.apiOrigin)) {
basePath = candidate
}
}
if basePath == "" && s.basePath != "" && s.runnerBasePath != "" {
if basePath == "" && s.basePath != "" && s.runnerOrigin.Path != "" {
fail(w, 404, "route not found")
return
}
Expand All @@ -120,6 +115,11 @@ func (s *Server) serveHTTP(w http.ResponseWriter, r *http.Request) {
r.URL.Path = strings.TrimPrefix(r.URL.Path, basePath)
r.URL.RawPath = ""
}
runnerRoute := strings.HasPrefix(r.URL.Path, "/runners/") || strings.HasPrefix(r.URL.Path, "/runner/") || strings.HasPrefix(r.URL.Path, "/ai/trickle/")
if !matchesOrigin(r.Host, s.apiOrigin) && !(runnerRoute && basePath == s.runnerOrigin.Path && s.registry.runnerService != "" && matchesOrigin(r.Host, s.runnerOrigin)) {
fail(w, http.StatusNotFound, "route not found")
return
}
}
stream := proxyID != "" || strings.HasPrefix(r.URL.Path, "/ai/trickle/") || strings.HasPrefix(r.URL.Path, "/proxy/") || strings.HasPrefix(r.URL.Path, "/run/") || strings.HasSuffix(r.URL.Path, "/app") || strings.Contains(r.URL.Path, "/app/")
_, pattern := s.mux.Handler(r)
Expand Down
15 changes: 10 additions & 5 deletions orchestrator/http_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,9 @@ func TestDynamicRunnerSessionProxyAndCapacity(t *testing.T) {
policy, err := destination.New("runner", []string{"127.0.0.0/8"})
require.NoError(t, err)
registry := NewRegistry("bootstrap", "http://orchestrator.example", 10*time.Millisecond, 50*time.Millisecond)
srv := httptest.NewServer(NewServer(registry, policy, policy, slog.New(slog.NewTextHandler(io.Discard, nil))))
app := NewServer(registry, policy, policy, slog.New(slog.NewTextHandler(io.Discard, nil)))
srv := httptest.NewServer(app)
app.apiOrigin = mustURL(t, srv.URL)
defer srv.Close()

body := `{"runner_url":"` + upstream.URL + `/api","app":"test","mode":"persistent","capacity":1,"price_info":{"price":0,"currency":"usd","unit":"hour"}}`
Expand Down Expand Up @@ -278,6 +280,7 @@ func TestSessionProxyStreamsSSEAndWebSocket(t *testing.T) {
app.setExcludedHeaders([]string{"CF-Access-*", "Cookie"})
defer app.Close()
server := httptest.NewServer(app)
app.apiOrigin = mustURL(t, server.URL)
defer server.Close()
sid, _, _, _, err := registry.Reserve(registered.RunnerID)
require.NoError(t, err)
Expand Down Expand Up @@ -320,7 +323,9 @@ func TestRunnerDestinationDeniedWithoutGrant(t *testing.T) {
policy, err := destination.New("runner", nil)
require.NoError(t, err)
registry := NewRegistry("bootstrap", "http://orchestrator.example", time.Second, time.Minute)
server := httptest.NewServer(NewServer(registry, policy, policy, slog.New(slog.NewTextHandler(io.Discard, nil))))
app := NewServer(registry, policy, policy, slog.New(slog.NewTextHandler(io.Discard, nil)))
server := httptest.NewServer(app)
app.apiOrigin = mustURL(t, server.URL)
defer server.Close()
body := `{"runner_url":"` + upstream.URL + `","app":"test","mode":"single-shot","capacity":1,"price_info":{"price":0}}`
request, err := http.NewRequest(http.MethodPost, server.URL+"/runners/heartbeat", bytes.NewBufferString(body))
Expand All @@ -347,7 +352,7 @@ func TestGeneratedProxyHasSeparateDestinationGrant(t *testing.T) {
require.NoError(t, err)
proxyPolicy, err := destination.New("session-proxy", nil)
require.NoError(t, err)
registry := NewRegistry("bootstrap", "http://orchestrator.example", time.Second, time.Minute)
registry := NewRegistry("bootstrap", "http://example.com", time.Second, time.Minute)
registration := heartbeatRequest{RunnerURL: upstream.URL, App: "test", Mode: "persistent", Capacity: 1}
registered, status, err := registry.Heartbeat(registration, "bootstrap")
require.NoError(t, err)
Expand Down Expand Up @@ -379,7 +384,7 @@ func TestGeneratedProxyHasSeparateDestinationGrant(t *testing.T) {
}
require.NoError(t, json.Unmarshal(record.Body.Bytes(), &created))
require.NotEmpty(t, created.ProxyID)
require.Equal(t, "http://orchestrator.example/run/"+created.ProxyID, created.URL)
require.Equal(t, "http://example.com/run/"+created.ProxyID, created.URL)
record = httptest.NewRecorder()
srv.ServeHTTP(record, httptest.NewRequest(http.MethodGet, "/run/"+created.ProxyID, nil))
require.Equal(t, http.StatusBadGateway, record.Code)
Expand Down Expand Up @@ -421,7 +426,7 @@ func TestRunnerExpires(t *testing.T) {
func TestTrickleChannelsScopedToSession(t *testing.T) {
policy, err := destination.New("runner", nil)
require.NoError(t, err)
registry := NewRegistry("bootstrap", "http://orchestrator.example", time.Second, time.Minute)
registry := NewRegistry("bootstrap", "http://example.com", time.Second, time.Minute)
registration := heartbeatRequest{RunnerURL: "https://runner.example", App: "test", Mode: "persistent", Capacity: 2}
registered, _, err := registry.Heartbeat(registration, "bootstrap")
require.NoError(t, err)
Expand Down
2 changes: 2 additions & 0 deletions orchestrator/lifecycle_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,7 @@ func TestReleaseCancelsActiveProxy(t *testing.T) {
}
app := NewServer(registry, policy, policy, slog.New(slog.NewTextHandler(io.Discard, nil)))
server := httptest.NewServer(app)
app.apiOrigin = mustURL(t, server.URL)
defer server.Close()
defer app.Close()
path := "/apps/runner/app"
Expand Down Expand Up @@ -363,6 +364,7 @@ func TestMultiAppRemovalKeepsStreams(t *testing.T) {
defer upstream.Close()
registry, appServer := reviewServer(t, upstream.URL)
server := httptest.NewServer(appServer)
appServer.apiOrigin = mustURL(t, server.URL)
defer server.Close()
defer appServer.Close()
a, b := registeredApp("a"), registeredApp("b")
Expand Down
1 change: 1 addition & 0 deletions orchestrator/metrics_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,7 @@ func TestUpgradePreservesHalfClose(t *testing.T) {
defer app.Close()
require.NoError(t, registry.AddStatic(StaticRunner{ID: "runner", App: "model", Mode: "single-shot", RunnerURL: upstream.URL, Capacity: 1}))
server := httptest.NewServer(app)
app.apiOrigin = mustURL(t, server.URL)
defer server.Close()
u, err := url.Parse(server.URL)
require.NoError(t, err)
Expand Down
5 changes: 4 additions & 1 deletion orchestrator/payment_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ func TestPaidFixedSessionWithRemoteSigner(t *testing.T) {
max := new(big.Int).Sub(new(big.Int).Lsh(big.NewInt(1), 256), big.NewInt(1))
engine, err := NewPaymentEngine(store, paymentTestChain{}, recipient.Address(), big.NewInt(11), new(big.Int).Sub(max, big.NewInt(1)))
require.NoError(t, err)
registry := NewRegistry("bootstrap", "http://127.0.0.1:8935", time.Second, 30*time.Second)
registry := NewRegistry("bootstrap", "http://example.com", time.Second, 30*time.Second)
registry.SetWeiPerUSD(big.NewRat(10, 1))
registration := heartbeatRequest{RunnerID: "runner1", RunnerURL: "http://127.0.0.1:9000", App: "demo", Mode: "persistent", Capacity: 1, PriceInfo: priceInfo{Price: "1", Currency: "usd", Unit: "fixed"}}
resp, status, err := registry.Heartbeat(registration, "bootstrap")
Expand Down Expand Up @@ -185,6 +185,7 @@ func TestPinnedPythonPaidSingleShot(t *testing.T) {
orch := httptest.NewUnstartedServer(server)
defer orch.Close()
registry.service = "http://" + orch.Listener.Addr().String()
server.apiOrigin = mustURL(t, registry.service)
orch.Start()
signerService, err := signer.NewService(payer)
require.NoError(t, err)
Expand Down Expand Up @@ -216,6 +217,7 @@ func TestPinnedPythonSignerDiscovery(t *testing.T) {
orch := httptest.NewUnstartedServer(server)
defer orch.Close()
registry.service = "http://" + orch.Listener.Addr().String()
server.apiOrigin = mustURL(t, registry.service)
orch.Start()
signerService, err := signer.NewService(paymentTestKey(t))
require.NoError(t, err)
Expand Down Expand Up @@ -303,6 +305,7 @@ func TestPinnedPythonSustainedPaymentRefresh(t *testing.T) {
app.ServeHTTP(w, r)
}))
registry.service = "http://" + orch.Listener.Addr().String()
app.apiOrigin = mustURL(t, registry.service)
orch.Start()
defer orch.Close()
service, err := signer.NewService(payer)
Expand Down
6 changes: 3 additions & 3 deletions orchestrator/proxy.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,14 +38,14 @@ func (r *Registry) proxyURL(id string) string {

// The template is fixed before serving. Match the actual Host, never a
// forwarded header supplied by a caller, and retain every application path.
func (r *Registry) matchProxy(req *http.Request, apiOrigin *url.URL) (string, string) {
func (r *Registry) matchProxy(req *http.Request, apiOrigin, runnerOrigin *url.URL) (string, string) {
if r.proxyTemplate == "" {
return "", ""
}
u, _ := url.Parse(strings.Replace(r.proxyTemplate, "{proxy}", proxyMarker, 1))
if before, after, ok := strings.Cut(strings.ToLower(u.Host), proxyMarker); ok {
// The API Host takes precedence over wildcard proxy Hosts.
if matchesOrigin(req.Host, apiOrigin) {
// API and runner Hosts take precedence over wildcard proxy Hosts.
if matchesOrigin(req.Host, apiOrigin) || r.runnerService != "" && matchesOrigin(req.Host, runnerOrigin) {
return "", ""
}
host := strings.ToLower(req.Host)
Expand Down
34 changes: 20 additions & 14 deletions orchestrator/proxy_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -178,7 +178,7 @@ func TestProxyHostCannotReachAPI(t *testing.T) {
policy, err := destination.New("runner", []string{strings.TrimPrefix(upstream.URL, "http://")})
require.NoError(t, err)
registry := NewRegistry("bootstrap", "https://api-aaaaaaa.machine.example", time.Second, time.Minute)
registry.runnerService = "http://runner.internal:8935"
registry.runnerService = "http://runner.internal:8935/internal"
registry.proxyTemplate = template
require.NoError(t, registry.AddStatic(StaticRunner{ID: "app-fixture", RunnerURL: upstream.URL, App: "fixture", Mode: "persistent", Capacity: 4}))
require.NoError(t, registry.AddStatic(StaticRunner{ID: "app-echo", RunnerURL: upstream.URL, App: "echo", Mode: "single-shot", Proxy: true, Capacity: 4}))
Expand All @@ -191,35 +191,41 @@ func TestProxyHostCannotReachAPI(t *testing.T) {
request := func(method, target string) *httptest.ResponseRecorder {
t.Helper()
req := httptest.NewRequest(method, target, nil)
req.Header.Set("X-Forwarded-Host", "api-aaaaaaa.machine.example")
req.Header.Set("X-Forwarded-Host", server.apiOrigin.Host)
req.Header.Set("Forwarded", "host="+server.apiOrigin.Host)
w := httptest.NewRecorder()
server.ServeHTTP(w, req)
return w
}
if template != "" {
for _, host := range []string{
"app-echo.machine.example:8443",
"app-echo.machine.example:443",
"app-echo.machine.example.",
"nested.app-echo.machine.example",
"other.example",
} {
w := request(http.MethodPost, "http://"+host+"/apps/app-fixture/session")
require.Equal(t, http.StatusNotFound, w.Code, host+": "+w.Body.String())
}
if apiOrigin != "runner.internal:8935" {
require.Equal(t, http.StatusNotFound, request(http.MethodPost, "http://runner.internal:8935/runners/heartbeat").Code)
}
for _, host := range []string{
"app-echo.machine.example:8443",
"app-echo.machine.example:443",
"app-echo.machine.example.",
"nested.app-echo.machine.example",
"other.example",
} {
w := request(http.MethodPost, "http://"+host+"/apps/app-fixture/session")
require.Equal(t, http.StatusNotFound, w.Code, host+": "+w.Body.String())
}
for _, origin := range []string{registry.service, registry.runnerService, "http://api-private.machine.example"} {
status := http.StatusNotFound
u, err := url.Parse(origin)
require.NoError(t, err)
if template == "" || u.Host == server.apiOrigin.Host {
if u.Host == server.apiOrigin.Host {
status = http.StatusOK
}
w := request(http.MethodPost, origin+"/apps/app-fixture/session")
require.Equal(t, status, w.Code, w.Body.String())
require.Equal(t, status, request(http.MethodGet, origin+"/discovery").Code)
}
w := request(http.MethodGet, registry.proxyURL("app-echo")+"/probe")
if template == "" && apiOrigin != "" {
require.Equal(t, http.StatusNotFound, w.Code, w.Body.String())
w = request(http.MethodGet, server.apiOrigin.String()+"/run/app-echo/probe")
}
require.Equal(t, http.StatusOK, w.Code, w.Body.String())
require.Equal(t, "application response", w.Body.String())
})
Expand Down
1 change: 1 addition & 0 deletions orchestrator/python_controls_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ func TestPinnedPythonRequestSessionControls(t *testing.T) {
defer orch.Close()
defer server.Close()
registry.service = "http://" + orch.Listener.Addr().String()
server.apiOrigin = mustURL(t, registry.service)
orch.Start()
fixture := filepath.Join(root, "cmd", "livepeer", "testdata", "python_runner_session_controls.py")
cmd := exec.Command(python, fixture, orch.URL, "bootstrap", "unused", "request-controls", "persistent", "1", "metadata")
Expand Down
1 change: 1 addition & 0 deletions orchestrator/python_media_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ func TestPinnedPythonMultiTrackTrickle(t *testing.T) {
orch := httptest.NewUnstartedServer(server)
defer orch.Close()
registry.service = "http://" + orch.Listener.Addr().String()
server.apiOrigin = mustURL(t, registry.service)
orch.Start()
manifest, _, _, status, err := registry.Reserve("media-runner")
require.NoError(t, err)
Expand Down
Loading
Loading