From 60777cf87286016e784bf3a61ae01af0bc6c9eb3 Mon Sep 17 00:00:00 2001 From: parsilver Date: Mon, 16 Nov 2026 13:50:00 +0700 Subject: [PATCH 1/2] test(controller): show /healthz waits for a running cycle A cycle holds the controller lock for every GitHub and Railway call, and Status takes the same lock, so a health check that lands during the minute-long cycle times out. The test holds a cycle open with a blocking fake GitHub call and fails today after 100 ms. --- internal/controller/controller_test.go | 4 +++ internal/controller/health_test.go | 41 ++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/internal/controller/controller_test.go b/internal/controller/controller_test.go index 50cad7e..1646051 100644 --- a/internal/controller/controller_test.go +++ b/internal/controller/controller_test.go @@ -28,9 +28,13 @@ type fakeGH struct { deleted []int64 jit []string nextID int64 + hold func() // when set, QueuedJobs calls it first: a test holds a cycle open with it } func (f *fakeGH) QueuedJobs(_ context.Context, s reconcile.Scope) ([]ghapp.Job, error) { + if f.hold != nil { + f.hold() + } return f.jobs[s.Key()], f.jobsErr[s.Key()] } func (f *fakeGH) Runners(_ context.Context, s reconcile.Scope) ([]reconcile.Runner, error) { diff --git a/internal/controller/health_test.go b/internal/controller/health_test.go index 9decc86..8f954cd 100644 --- a/internal/controller/health_test.go +++ b/internal/controller/health_test.go @@ -7,6 +7,7 @@ import ( "net/http" "net/http/httptest" "strings" + "sync" "testing" "time" @@ -121,3 +122,43 @@ func TestAlertsAreSentWithoutHoldingTheLock(t *testing.T) { t.Fatalf("alerts %d, want 1", len(sent)) } } + +// /healthz must not wait for a running cycle: a cycle holds the controller's lock for every GitHub +// and Railway call (about a minute in production), and the health check used to time out behind it. +// While a cycle is held open, /healthz answers at once with the previous cycle's status. +func TestHealthzAnswersDuringACycleWithThePreviousStatus(t *testing.T) { + gh := &fakeGH{jobs: map[string][]ghapp.Job{repo.Key(): {{ID: 7, Repo: "acme/app", CreatedAt: now}}}} + c, _ := alerting(gh, &fakeRW{}) + if err := c.Cycle(context.Background()); err != nil { + t.Fatal(err) + } + entered, release := make(chan struct{}), make(chan struct{}) + var once sync.Once + gh.hold = func() { + once.Do(func() { close(entered) }) + <-release + } + done := make(chan struct{}) + go func() { + _ = c.Cycle(context.Background()) + close(done) + }() + <-entered + answered := make(chan int, 1) + var body map[string]any + go func() { + code, b := health(t, c) + body = b + answered <- code + }() + select { + case code := <-answered: + if code != http.StatusOK || body["queued"] != float64(1) || body["services"] != float64(1) { + t.Errorf("during a cycle: %d %v, want 200 with the previous cycle's queued 1 and services 1", code, body) + } + case <-time.After(100 * time.Millisecond): + t.Error("/healthz did not answer within 100 ms while a cycle was running") + } + close(release) + <-done +} From 7be87b4a8da43a6f7a723695ce8e08112cd46961 Mon Sep 17 00:00:00 2001 From: parsilver Date: Mon, 16 Nov 2026 13:56:12 +0700 Subject: [PATCH 2/2] fix(controller): answer /healthz without waiting for a running cycle Status took the lock a cycle holds for every GitHub and Railway call, so a health check during the minute-long cycle timed out. The cycle now publishes each finished Status through an atomic pointer that Status reads without the lock; the lock still serialises cycles and guards the cycle state. The alert test now takes the lock itself, since Status no longer would catch an alert sent under it. The scope scan moves out of record to keep it under CRAP 10, and its test now kills the two mutants that survived there. Proved with go test -race and a mutation run on the changed lines. --- CHANGELOG.md | 1 + internal/controller/controller.go | 50 ++++++++++++++++++------------ internal/controller/health_test.go | 50 +++++++++++++++++++++--------- 3 files changed, 66 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 352981f..fd6f309 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ the major version is 0, a minor release may change configuration (each such chan - The GitHub CLI (`gh` 2.102.0, pinned by version and sha256) in every image variant, so a job can open or comment on issues without a GitHub-hosted runner. Nothing is logged in: a job passes `GH_TOKEN`. ### Fixed +- `/healthz` answers at once while a cycle runs, with the previous cycle's status; it waited for the cycle to end (about a minute with many repositories) and health checks timed out. - Jobs queued in a run that GitHub reports as `pending` (for example while it waits on a concurrency group) are served; the controller listed only `queued` and `in_progress` runs and left such jobs waiting forever. ### Changed diff --git a/internal/controller/controller.go b/internal/controller/controller.go index c5a635e..5318fe7 100644 --- a/internal/controller/controller.go +++ b/internal/controller/controller.go @@ -14,6 +14,7 @@ import ( "net/http" "strings" "sync" + "sync/atomic" "time" "github.com/farzai/github-runner-railway/internal/ghapp" @@ -61,9 +62,9 @@ type Controller struct { rw RW now func() time.Time wake chan struct{} - mu sync.Mutex - alerted map[int64]string // job id -> scope key - status Status + mu sync.Mutex // held for a whole cycle; guards the cycle state below, not status + alerted map[int64]string // job id -> scope key + status atomic.Pointer[Status] // the last cycle's outcome, replaced whole and never changed after failures int // failed cycles in a row lastSuccess time.Time // the end of the last cycle with no error @@ -130,11 +131,13 @@ func (c *Controller) Nudge() { } } -// Status is the state after the last cycle. +// Status is the state after the last cycle. It never waits for a running cycle, so /healthz +// answers while a cycle is busy with GitHub and Railway. func (c *Controller) Status() Status { - c.mu.Lock() - defer c.mu.Unlock() - return c.status + if st := c.status.Load(); st != nil { + return *st + } + return Status{} } // Run cycles until ctx ends: on every interval and on every nudge. A nudge waits out the debounce @@ -190,7 +193,7 @@ func scopeByKey(ss []reconcile.Scope, key string) reconcile.Scope { // Cycle is one pass. A scope GitHub cannot be read for is left alone entirely: its services // are neither counted nor reaped, since a guess about them could kill a busy runner. func (c *Controller) Cycle(ctx context.Context) (err error) { - // Alerts go out after the lock is released: a slow alert webhook must not stall /healthz. + // Alerts go out after the lock is released: a slow alert webhook must not hold up the next cycle. var alerts []string defer func() { for _, a := range alerts { @@ -296,20 +299,13 @@ func (c *Controller) record(now time.Time, errs []error, services, queued int) { c.failures = 0 c.lastSuccess = now } - var unreadable, stuck []string - for _, s := range c.cfg.Scopes { - if n := c.scopeFails[s.Key()]; n > 0 { - unreadable = append(unreadable, s.Key()) - if n >= c.cfg.FailAlertAfter { - stuck = append(stuck, s.Key()) - } - } - } - c.status = Status{LastCycle: now, Services: services, Queued: queued, ConsecutiveFailures: c.failures, + unreadable, stuck := c.unreadableScopes() + st := &Status{LastCycle: now, Services: services, Queued: queued, ConsecutiveFailures: c.failures, LastSuccess: c.lastSuccess, UnreadableScopes: unreadable, Failing: c.failures >= c.cfg.FailAlertAfter} if len(errs) > 0 { - c.status.LastError = errors.Join(errs...).Error() + st.LastError = errors.Join(errs...).Error() } + c.status.Store(st) switch { case c.failures >= c.cfg.FailAlertAfter && !c.failAlerted: c.failAlerted = true @@ -317,13 +313,27 @@ func (c *Controller) record(now time.Time, errs []error, services, queued int) { if len(stuck) > 0 { msg += "; scopes unreadable that long: " + strings.Join(stuck, ", ") } - c.pending = append(c.pending, msg+"; last error: "+truncate(c.status.LastError, 300)) + c.pending = append(c.pending, msg+"; last error: "+truncate(st.LastError, 300)) case c.failures == 0 && c.failAlerted: c.failAlerted = false c.pending = append(c.pending, "controller: recovered, the last cycle succeeded") } } +// unreadableScopes are the scopes the last cycle could not read, and of those the ones unreadable +// for FailAlertAfter cycles or more. +func (c *Controller) unreadableScopes() (unreadable, stuck []string) { + for _, s := range c.cfg.Scopes { + if n := c.scopeFails[s.Key()]; n > 0 { + unreadable = append(unreadable, s.Key()) + if n >= c.cfg.FailAlertAfter { + stuck = append(stuck, s.Key()) + } + } + } + return unreadable, stuck +} + func truncate(s string, n int) string { if len(s) <= n { return s diff --git a/internal/controller/health_test.go b/internal/controller/health_test.go index 8f954cd..fc7ad47 100644 --- a/internal/controller/health_test.go +++ b/internal/controller/health_test.go @@ -81,11 +81,15 @@ func TestOneUnreadableScopeIsNamedAndTheOtherKeepsWorking(t *testing.T) { gh := &fakeGH{jobsErr: map[string]error{org.Key(): errors.New("403 rate limited")}} rw := &fakeRW{} c, alerts := alerting(gh, rw) - for i := 0; i < 4; i++ { + _ = c.Cycle(context.Background()) + if got := c.Status().UnreadableScopes; len(got) != 1 || got[0] != org.Key() { + t.Fatalf("after one failed read: unreadable %v, want [%s]", got, org.Key()) + } + for i := 0; i < 3; i++ { _ = c.Cycle(context.Background()) } - if len(*alerts) != 1 || !strings.Contains((*alerts)[0], org.Key()) || strings.Contains((*alerts)[0], repo.Key()) { - t.Fatalf("alerts %q, want one naming only %s", *alerts, org.Key()) + if len(*alerts) != 1 || !strings.Contains((*alerts)[0], "scopes unreadable that long: "+org.Key()+";") || strings.Contains((*alerts)[0], repo.Key()) { + t.Fatalf("alerts %q, want one naming only %s as unreadable that long", *alerts, org.Key()) } _, body := health(t, c) if got, _ := body["unreadable_scopes"].([]any); len(got) != 1 || got[0] != org.Key() { @@ -98,14 +102,14 @@ func TestOneUnreadableScopeIsNamedAndTheOtherKeepsWorking(t *testing.T) { } } -// Alerts are sent after the cycle releases its lock: a slow alert webhook must not stall /healthz, -// which reads the status under the same lock. +// Alerts are sent after the cycle releases its lock: a slow alert webhook must not hold up the next +// cycle. The alert takes the lock itself, so it deadlocks if it runs while the cycle holds it. func TestAlertsAreSentWithoutHoldingTheLock(t *testing.T) { boom := errors.New("github down") gh := &fakeGH{jobsErr: map[string]error{org.Key(): boom, repo.Key(): boom}} c, _ := alerting(gh, &fakeRW{}) sent := make(chan struct{}, 4) - c.cfg.Alert = func(string) { _ = c.Status(); sent <- struct{}{} } + c.cfg.Alert = func(string) { c.mu.Lock(); c.mu.Unlock(); sent <- struct{}{} } done := make(chan struct{}) go func() { for i := 0; i < 3; i++ { @@ -144,21 +148,37 @@ func TestHealthzAnswersDuringACycleWithThePreviousStatus(t *testing.T) { close(done) }() <-entered - answered := make(chan int, 1) - var body map[string]any + answered := make(chan *httptest.ResponseRecorder, 1) go func() { - code, b := health(t, c) - body = b - answered <- code + rec := httptest.NewRecorder() + c.HealthHandler().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/healthz", nil)) + answered <- rec }() + var rec *httptest.ResponseRecorder select { - case code := <-answered: - if code != http.StatusOK || body["queued"] != float64(1) || body["services"] != float64(1) { - t.Errorf("during a cycle: %d %v, want 200 with the previous cycle's queued 1 and services 1", code, body) - } + case rec = <-answered: case <-time.After(100 * time.Millisecond): t.Error("/healthz did not answer within 100 ms while a cycle was running") } close(release) <-done + if rec == nil { + <-answered // the late answer, so nothing outlives the test + return + } + var body map[string]any + if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { + t.Fatalf("body %q: %v", rec.Body.String(), err) + } + if rec.Code != http.StatusOK || body["queued"] != float64(1) || body["services"] != float64(1) { + t.Errorf("during a cycle: %d %v, want 200 with the previous cycle's queued 1 and services 1", rec.Code, body) + } +} + +// Before the first cycle has finished there is no status yet, and /healthz answers 503. +func TestHealthzIs503BeforeTheFirstCycle(t *testing.T) { + c, _ := alerting(&fakeGH{}, &fakeRW{}) + if code, body := health(t, c); code != http.StatusServiceUnavailable || body["services"] != float64(0) { + t.Fatalf("before any cycle: %d %v, want 503", code, body) + } }