From 0a27f934b10c54c618f6a1bc3780b33746fee943 Mon Sep 17 00:00:00 2001 From: parsilver Date: Thu, 19 Nov 2026 13:41:41 +0700 Subject: [PATCH 1/2] test(controller): pin that /healthz counts unreadable scopes without names The endpoint is public, so naming an unreadable private repository leaks it. Fails now: the body lists the scope keys. --- internal/controller/healthz_names_test.go | 44 +++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 internal/controller/healthz_names_test.go diff --git a/internal/controller/healthz_names_test.go b/internal/controller/healthz_names_test.go new file mode 100644 index 0000000..f3c9710 --- /dev/null +++ b/internal/controller/healthz_names_test.go @@ -0,0 +1,44 @@ +package controller + +import ( + "bytes" + "context" + "errors" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +// /healthz is public, so it must not name a scope it cannot read: a private repository's name would +// leak to anyone who calls it. It reports how many scopes are unreadable; the names stay in the +// controller's logs and in the failing alert, which only the operator sees. +func TestHealthzCountsUnreadableScopesWithoutNamingThem(t *testing.T) { + gh := &fakeGH{jobsErr: map[string]error{ + repo.Key(): notFound("acme/app"), + org.Key(): errors.New("403 rate limited"), + }} + c, alerts := alerting(gh, &fakeRW{}) + var logs bytes.Buffer + c.cfg.Log = slog.New(slog.NewTextHandler(&logs, nil)) + for i := 0; i < c.cfg.FailAlertAfter; i++ { + _ = c.Cycle(context.Background()) + } + + rec := httptest.NewRecorder() + c.HealthHandler().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/healthz", nil)) + if raw := rec.Body.String(); strings.Contains(raw, "acme") { + t.Fatalf("/healthz body %s names a scope, want only the count", raw) + } + if _, body := health(t, c); body["unreadable_scopes"] != float64(2) { + t.Fatalf("unreadable_scopes %v, want 2", body["unreadable_scopes"]) + } + + if !strings.Contains(logs.String(), "scope not found on GitHub; skipped") || !strings.Contains(logs.String(), "scope="+repo.Key()) { + t.Fatalf("logs %q, want the not-found warn line naming %s", logs.String(), repo.Key()) + } + if len(*alerts) == 0 || !strings.Contains(strings.Join(*alerts, "\n"), "scopes unreadable that long: "+org.Key()) { + t.Fatalf("alerts %q, want the failing alert naming %s", *alerts, org.Key()) + } +} From 3f5a05de6e96ab7db7696a3fd770ba5d54c22106 Mon Sep 17 00:00:00 2001 From: parsilver Date: Thu, 19 Nov 2026 13:46:05 +0700 Subject: [PATCH 2/2] fix(controller): count unreadable scopes on /healthz instead of naming them /healthz is public, so listing an unreadable private repository in unreadable_scopes leaked its name. Status now holds only the count; the names stay in the warn and cycle-failed log lines and in the failing alert. The field was never released, so the CHANGELOG entries are reworded rather than marked breaking. Proved by TestHealthzCountsUnreadableScopesWithoutNamingThem (red run kept in docs/evidence). --- CHANGELOG.md | 4 ++-- README.md | 6 +++--- docs/evidence/red-51-healthz-names.txt | 6 ++++++ internal/controller/controller.go | 18 +++++++----------- internal/controller/health_test.go | 6 +++--- internal/controller/healthz_names_test.go | 1 + internal/controller/notfound_test.go | 12 ++++++------ 7 files changed, 28 insertions(+), 25 deletions(-) create mode 100644 docs/evidence/red-51-healthz-names.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index ad655f0..68f8f63 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ the major version is 0, a minor release may change configuration (each such chan - The GitHub CLI (`gh` 2.102.0, pinned by version and sha256) in every image variant, so a job can open or comment on issues without a GitHub-hosted runner. Nothing is logged in: a job passes `GH_TOKEN`. ### Fixed -- A scope in `RUNNER_SCOPES` that GitHub answers 404 for (a deleted repository, for example) no longer fails every cycle and turns `/healthz` 503: it is listed in `unreadable_scopes`, logged at warn once a cycle and left out of the failing alert, while the other scopes keep getting runners. After `ALERT_FAILED_CYCLES` cycles in a row such a scope raises one alert of its own, and one more when it reads again. When every scope answers 404 (the App uninstalled, for example), the cycle still fails. In an org scope, a repository or run that vanishes mid-cycle is skipped instead of hiding the whole org. Other GitHub errors still fail the cycle. +- A scope in `RUNNER_SCOPES` that GitHub answers 404 for (a deleted repository, for example) no longer fails every cycle and turns `/healthz` 503: it is counted in `unreadable_scopes`, logged at warn once a cycle and left out of the failing alert, while the other scopes keep getting runners. After `ALERT_FAILED_CYCLES` cycles in a row such a scope raises one alert of its own, and one more when it reads again. When every scope answers 404 (the App uninstalled, for example), the cycle still fails. In an org scope, a repository or run that vanishes mid-cycle is skipped instead of hiding the whole org. Other GitHub errors still fail the cycle. - `/healthz` answers at once while a cycle runs, with the previous cycle's status; it waited for the cycle to end (about a minute with many repositories) and health checks timed out. - Jobs queued in a run that GitHub reports as `pending` (for example while it waits on a concurrency group) are served; the controller listed only `queued` and `in_progress` runs and left such jobs waiting forever. @@ -23,7 +23,7 @@ the major version is 0, a minor release may change configuration (each such chan ### Added - A `php` image variant (`php-`): PHP 8.3, 8.4 and 8.5 from the ondrej/php PPA and Composer, with `use-php ` to choose one without root. `shivammathur/setup-php` cannot run on the default images (it needs `sudo` and then waits instead of failing); the README lists which `setup-*` actions work. - `RUNNER_REGION` (optional): the Railway region a controller creates its runners in, set per controller through `multiRegionConfig`. Unset keeps today's placement in the workspace's preferred region. -- `ALERT_FAILED_CYCLES` (default 3): after that many failed cycles in a row the controller alerts once (naming scopes unreadable that long), `/healthz` answers 503, and one "recovered" alert follows the next good cycle. `/healthz` also reports `last_success`, `consecutive_failures` and `unreadable_scopes`. +- `ALERT_FAILED_CYCLES` (default 3): after that many failed cycles in a row the controller alerts once (naming scopes unreadable that long), `/healthz` answers 503, and one "recovered" alert follows the next good cycle. `/healthz` also reports `last_success`, `consecutive_failures` and `unreadable_scopes`, the number of scopes the last cycle could not read. Since `/healthz` is public, it never names them (a private repository's name would leak); the names are in the log lines and the failing alert. ### Fixed - A controller whose every cycle failed kept answering 200 on `/healthz` and raised no alert. diff --git a/README.md b/README.md index e087af6..ba4d544 100644 --- a/README.md +++ b/README.md @@ -104,7 +104,7 @@ RUNNER_LABEL=railway MAX_RUNNERS=2 ``` -`/healthz` answers 200 once a cycle has completed, and 503 when the last cycle is older than three polls or `ALERT_FAILED_CYCLES` cycles in a row have failed. Its JSON: `last_cycle`, `last_success`, `consecutive_failures`, `unreadable_scopes`, `services`, `queued`, `error`. A scope GitHub answers 404 for (a deleted repository, a typo in `RUNNER_SCOPES`, or an App no longer installed there) is listed in `unreadable_scopes` and logged at warn each cycle, but does not fail the cycle; after `ALERT_FAILED_CYCLES` cycles in a row it raises one alert of its own, and one more when it reads again. When every scope answers 404 (the App uninstalled, for example) the cycle fails, since nothing can be served. Any other read error fails the cycle too. `github_rate_limits` shows the requests GitHub last reported left per installation account (a warning is logged once per window below a tenth). The log line `controller up` shows the scopes, +`/healthz` answers 200 once a cycle has completed, and 503 when the last cycle is older than three polls or `ALERT_FAILED_CYCLES` cycles in a row have failed. Its JSON: `last_cycle`, `last_success`, `consecutive_failures`, `unreadable_scopes`, `services`, `queued`, `error`. `/healthz` is public, so `unreadable_scopes` is only how many scopes the last cycle could not read; their names are in the controller's log lines and in the failing alert, which only the operator sees. A scope GitHub answers 404 for (a deleted repository, a typo in `RUNNER_SCOPES`, or an App no longer installed there) is counted in `unreadable_scopes` and logged at warn each cycle, but does not fail the cycle; after `ALERT_FAILED_CYCLES` cycles in a row it raises one alert of its own, and one more when it reads again. When every scope answers 404 (the App uninstalled, for example) the cycle fails, since nothing can be served. Any other read error fails the cycle too. `github_rate_limits` shows the requests GitHub last reported left per installation account (a warning is logged once per window below a tenth). The log line `controller up` shows the scopes, label, prefix, image and limits it runs with. ### 4. Use it in a workflow @@ -293,8 +293,8 @@ pin a version (or a digest, `image@sha256:...`) so a runner never changes under | Symptom | Look at | | --- | --- | | The controller exits at start | its log names the missing or malformed variable (never its value) | -| `/healthz` 503 | no cycle has completed yet, the last one is older than three polls, or cycles keep failing (`consecutive_failures`, `unreadable_scopes`); see the `cycle failed` log lines | -| `/healthz` 200 but a scope in `unreadable_scopes` | GitHub answers 404 for it: the repository was deleted, the name in `RUNNER_SCOPES` is wrong, or the App is no longer installed there. See the `scope not found on GitHub; skipped` warn line and the `not found on GitHub for N cycles in a row` alert, then fix or remove the scope. If every scope answers 404, the cycles fail and `/healthz` goes 503: check the App is still installed. A renamed or transferred repository answers a redirect instead, which fails the cycle until `RUNNER_SCOPES` has the new name | +| `/healthz` 503 | no cycle has completed yet, the last one is older than three polls, or cycles keep failing (`consecutive_failures`, `unreadable_scopes`); the `cycle failed` log lines say why and name the scopes | +| `/healthz` 200 but `unreadable_scopes` above 0 | GitHub answers 404 for a scope (the warn line below names it): the repository was deleted, the name in `RUNNER_SCOPES` is wrong, or the App is no longer installed there. See the `scope not found on GitHub; skipped` warn line and the `not found on GitHub for N cycles in a row` alert, then fix or remove the scope. If every scope answers 404, the cycles fail and `/healthz` goes 503: check the App is still installed. A renamed or transferred repository answers a redirect instead, which fails the cycle until `RUNNER_SCOPES` has the new name | | `/healthz` `"error": true` | the last cycle failed; the `cycle failed` log line says why (often a GitHub or Railway API error, which the next cycle retries) | | A job stays queued | the job's labels must all be among `self-hosted`, `linux`, `x64` and your label; its repository must be in `RUNNER_SCOPES` and the App installed on it; `MAX_RUNNERS` may be reached | | How often cycles run, and what they cost | every cycle logs one `cycle` line: `duration`, `github_calls`, `railway_calls`, `created`, `reaped`, `failed` | diff --git a/docs/evidence/red-51-healthz-names.txt b/docs/evidence/red-51-healthz-names.txt new file mode 100644 index 0000000..ce6ac10 --- /dev/null +++ b/docs/evidence/red-51-healthz-names.txt @@ -0,0 +1,6 @@ +--- FAIL: TestHealthzCountsUnreadableScopesWithoutNamingThem (0.00s) + healthz_names_test.go:32: /healthz body {"consecutive_failures":3,"error":true,"last_cycle":"2026-10-17T12:00:00Z","last_success":null,"queued":0,"services":0,"unreadable_scopes":["org:acme","repo:acme/app"]} + names a scope, want only the count +FAIL +FAIL github.com/farzai/github-runner-railway/internal/controller 0.160s +FAIL diff --git a/internal/controller/controller.go b/internal/controller/controller.go index 4d0968b..d4ac375 100644 --- a/internal/controller/controller.go +++ b/internal/controller/controller.go @@ -89,8 +89,8 @@ type Status struct { Queued int ConsecutiveFailures int LastSuccess time.Time - UnreadableScopes []string // scopes that could not be read in the last cycle - Failing bool // ConsecutiveFailures reached FailAlertAfter + UnreadableScopes int // how many scopes the last cycle could not read; /healthz is public, so no names + Failing bool // ConsecutiveFailures reached FailAlertAfter } // New builds a Controller. @@ -408,13 +408,13 @@ func (c *Controller) record(now time.Time, errs []error, services, queued int) { } } -// unreadableScopes are the scopes the last cycle could not read (a 404 included), and of those the -// ones that failed for FailAlertAfter cycles or more for another reason. -func (c *Controller) unreadableScopes() (unreadable, stuck []string) { +// unreadableScopes counts the scopes the last cycle could not read (a 404 included), and names the +// ones that failed for FailAlertAfter cycles or more for another reason, for the operator's alert. +func (c *Controller) unreadableScopes() (unreadable int, stuck []string) { for _, s := range c.cfg.Scopes { n := c.scopeFails[s.Key()] if n > 0 || c.missing[s.Key()] > 0 { - unreadable = append(unreadable, s.Key()) + unreadable++ } if n >= c.cfg.FailAlertAfter { stuck = append(stuck, s.Key()) @@ -518,13 +518,9 @@ func (c *Controller) HealthHandler() http.Handler { if !st.LastSuccess.IsZero() { lastSuccess = st.LastSuccess } - unreadable := st.UnreadableScopes - if unreadable == nil { - unreadable = []string{} - } body := map[string]any{"last_cycle": st.LastCycle, "services": st.Services, "queued": st.Queued, "error": st.LastError != "", "consecutive_failures": st.ConsecutiveFailures, "last_success": lastSuccess, - "unreadable_scopes": unreadable} + "unreadable_scopes": st.UnreadableScopes} if rl, ok := c.gh.(RateLimiter); ok { body["github_rate_limits"] = rl.RateLimits() } diff --git a/internal/controller/health_test.go b/internal/controller/health_test.go index fc7ad47..d0b7035 100644 --- a/internal/controller/health_test.go +++ b/internal/controller/health_test.go @@ -82,8 +82,8 @@ func TestOneUnreadableScopeIsNamedAndTheOtherKeepsWorking(t *testing.T) { rw := &fakeRW{} c, alerts := alerting(gh, rw) _ = c.Cycle(context.Background()) - if got := c.Status().UnreadableScopes; len(got) != 1 || got[0] != org.Key() { - t.Fatalf("after one failed read: unreadable %v, want [%s]", got, org.Key()) + if got := c.Status().UnreadableScopes; got != 1 { + t.Fatalf("after one failed read: unreadable %d, want 1", got) } for i := 0; i < 3; i++ { _ = c.Cycle(context.Background()) @@ -92,7 +92,7 @@ func TestOneUnreadableScopeIsNamedAndTheOtherKeepsWorking(t *testing.T) { t.Fatalf("alerts %q, want one naming only %s as unreadable that long", *alerts, org.Key()) } _, body := health(t, c) - if got, _ := body["unreadable_scopes"].([]any); len(got) != 1 || got[0] != org.Key() { + if body["unreadable_scopes"] != float64(1) { t.Fatalf("unreadable_scopes %v", body["unreadable_scopes"]) } gh.jobs = map[string][]ghapp.Job{repo.Key(): {{ID: 9, Repo: "acme/app", CreatedAt: now}}} diff --git a/internal/controller/healthz_names_test.go b/internal/controller/healthz_names_test.go index f3c9710..dc1ae25 100644 --- a/internal/controller/healthz_names_test.go +++ b/internal/controller/healthz_names_test.go @@ -28,6 +28,7 @@ func TestHealthzCountsUnreadableScopesWithoutNamingThem(t *testing.T) { rec := httptest.NewRecorder() c.HealthHandler().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/healthz", nil)) + // fakeGH reports no rate limits, so no owner login can reach the body that way either. if raw := rec.Body.String(); strings.Contains(raw, "acme") { t.Fatalf("/healthz body %s names a scope, want only the count", raw) } diff --git a/internal/controller/notfound_test.go b/internal/controller/notfound_test.go index 058aceb..04b204b 100644 --- a/internal/controller/notfound_test.go +++ b/internal/controller/notfound_test.go @@ -37,8 +37,8 @@ func TestAScopeThatIsNotFoundIsWarnedAboutButTheCycleSucceeds(t *testing.T) { if code != http.StatusOK || body["consecutive_failures"] != float64(0) || body["error"] != false { t.Fatalf("health %d %v, want 200 with no failures", code, body) } - if got, _ := body["unreadable_scopes"].([]any); len(got) != 1 || got[0] != repo.Key() { - t.Fatalf("unreadable_scopes %v, want [%s]", body["unreadable_scopes"], repo.Key()) + if body["unreadable_scopes"] != float64(1) { + t.Fatalf("unreadable_scopes %v, want 1", body["unreadable_scopes"]) } if len(rw.created) == 0 { t.Fatalf("created %v: the readable scope must get runners", rw.created) @@ -85,8 +85,8 @@ func TestAScopeWhoseRunnersAreNotFoundDoesNotFailTheCycle(t *testing.T) { if code != http.StatusOK || body["error"] != false { t.Fatalf("health %d %v, want 200", code, body) } - if got, _ := body["unreadable_scopes"].([]any); len(got) != 1 || got[0] != repo.Key() { - t.Fatalf("unreadable_scopes %v after one cycle, want [%s]", body["unreadable_scopes"], repo.Key()) + if body["unreadable_scopes"] != float64(1) { + t.Fatalf("unreadable_scopes %v after one cycle, want 1", body["unreadable_scopes"]) } } @@ -112,7 +112,7 @@ func TestTheFailingAlertDoesNotBlameANotFoundScope(t *testing.T) { t.Fatalf("alerts %q, want one failing alert naming only %s", *alerts, org.Key()) } _, body := health(t, c) - if got, _ := body["unreadable_scopes"].([]any); len(got) != 2 { + if body["unreadable_scopes"] != float64(2) { t.Fatalf("unreadable_scopes %v, want both scopes", body["unreadable_scopes"]) } } @@ -175,7 +175,7 @@ func TestAMissingScopeIsAlertedOnceAndClearedWhenItReadsAgain(t *testing.T) { if len(*alerts) != 2 || (*alerts)[1] != "controller: scope "+repo.Key()+" is readable again" { t.Fatalf("alerts %q, want one readable-again message", *alerts) } - if _, body := health(t, c); len(body["unreadable_scopes"].([]any)) != 0 { + if _, body := health(t, c); body["unreadable_scopes"] != float64(0) { t.Fatalf("unreadable_scopes %v, want none", body["unreadable_scopes"]) } }