diff --git a/CHANGELOG.md b/CHANGELOG.md index 352981f..fd6f309 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ the major version is 0, a minor release may change configuration (each such chan - The GitHub CLI (`gh` 2.102.0, pinned by version and sha256) in every image variant, so a job can open or comment on issues without a GitHub-hosted runner. Nothing is logged in: a job passes `GH_TOKEN`. ### Fixed +- `/healthz` answers at once while a cycle runs, with the previous cycle's status; it waited for the cycle to end (about a minute with many repositories) and health checks timed out. - Jobs queued in a run that GitHub reports as `pending` (for example while it waits on a concurrency group) are served; the controller listed only `queued` and `in_progress` runs and left such jobs waiting forever. ### Changed diff --git a/internal/controller/controller.go b/internal/controller/controller.go index c5a635e..5318fe7 100644 --- a/internal/controller/controller.go +++ b/internal/controller/controller.go @@ -14,6 +14,7 @@ import ( "net/http" "strings" "sync" + "sync/atomic" "time" "github.com/farzai/github-runner-railway/internal/ghapp" @@ -61,9 +62,9 @@ type Controller struct { rw RW now func() time.Time wake chan struct{} - mu sync.Mutex - alerted map[int64]string // job id -> scope key - status Status + mu sync.Mutex // held for a whole cycle; guards the cycle state below, not status + alerted map[int64]string // job id -> scope key + status atomic.Pointer[Status] // the last cycle's outcome, replaced whole and never changed after failures int // failed cycles in a row lastSuccess time.Time // the end of the last cycle with no error @@ -130,11 +131,13 @@ func (c *Controller) Nudge() { } } -// Status is the state after the last cycle. +// Status is the state after the last cycle. It never waits for a running cycle, so /healthz +// answers while a cycle is busy with GitHub and Railway. func (c *Controller) Status() Status { - c.mu.Lock() - defer c.mu.Unlock() - return c.status + if st := c.status.Load(); st != nil { + return *st + } + return Status{} } // Run cycles until ctx ends: on every interval and on every nudge. A nudge waits out the debounce @@ -190,7 +193,7 @@ func scopeByKey(ss []reconcile.Scope, key string) reconcile.Scope { // Cycle is one pass. A scope GitHub cannot be read for is left alone entirely: its services // are neither counted nor reaped, since a guess about them could kill a busy runner. func (c *Controller) Cycle(ctx context.Context) (err error) { - // Alerts go out after the lock is released: a slow alert webhook must not stall /healthz. + // Alerts go out after the lock is released: a slow alert webhook must not hold up the next cycle. var alerts []string defer func() { for _, a := range alerts { @@ -296,20 +299,13 @@ func (c *Controller) record(now time.Time, errs []error, services, queued int) { c.failures = 0 c.lastSuccess = now } - var unreadable, stuck []string - for _, s := range c.cfg.Scopes { - if n := c.scopeFails[s.Key()]; n > 0 { - unreadable = append(unreadable, s.Key()) - if n >= c.cfg.FailAlertAfter { - stuck = append(stuck, s.Key()) - } - } - } - c.status = Status{LastCycle: now, Services: services, Queued: queued, ConsecutiveFailures: c.failures, + unreadable, stuck := c.unreadableScopes() + st := &Status{LastCycle: now, Services: services, Queued: queued, ConsecutiveFailures: c.failures, LastSuccess: c.lastSuccess, UnreadableScopes: unreadable, Failing: c.failures >= c.cfg.FailAlertAfter} if len(errs) > 0 { - c.status.LastError = errors.Join(errs...).Error() + st.LastError = errors.Join(errs...).Error() } + c.status.Store(st) switch { case c.failures >= c.cfg.FailAlertAfter && !c.failAlerted: c.failAlerted = true @@ -317,13 +313,27 @@ func (c *Controller) record(now time.Time, errs []error, services, queued int) { if len(stuck) > 0 { msg += "; scopes unreadable that long: " + strings.Join(stuck, ", ") } - c.pending = append(c.pending, msg+"; last error: "+truncate(c.status.LastError, 300)) + c.pending = append(c.pending, msg+"; last error: "+truncate(st.LastError, 300)) case c.failures == 0 && c.failAlerted: c.failAlerted = false c.pending = append(c.pending, "controller: recovered, the last cycle succeeded") } } +// unreadableScopes are the scopes the last cycle could not read, and of those the ones unreadable +// for FailAlertAfter cycles or more. +func (c *Controller) unreadableScopes() (unreadable, stuck []string) { + for _, s := range c.cfg.Scopes { + if n := c.scopeFails[s.Key()]; n > 0 { + unreadable = append(unreadable, s.Key()) + if n >= c.cfg.FailAlertAfter { + stuck = append(stuck, s.Key()) + } + } + } + return unreadable, stuck +} + func truncate(s string, n int) string { if len(s) <= n { return s diff --git a/internal/controller/controller_test.go b/internal/controller/controller_test.go index 50cad7e..1646051 100644 --- a/internal/controller/controller_test.go +++ b/internal/controller/controller_test.go @@ -28,9 +28,13 @@ type fakeGH struct { deleted []int64 jit []string nextID int64 + hold func() // when set, QueuedJobs calls it first: a test holds a cycle open with it } func (f *fakeGH) QueuedJobs(_ context.Context, s reconcile.Scope) ([]ghapp.Job, error) { + if f.hold != nil { + f.hold() + } return f.jobs[s.Key()], f.jobsErr[s.Key()] } func (f *fakeGH) Runners(_ context.Context, s reconcile.Scope) ([]reconcile.Runner, error) { diff --git a/internal/controller/health_test.go b/internal/controller/health_test.go index 9decc86..fc7ad47 100644 --- a/internal/controller/health_test.go +++ b/internal/controller/health_test.go @@ -7,6 +7,7 @@ import ( "net/http" "net/http/httptest" "strings" + "sync" "testing" "time" @@ -80,11 +81,15 @@ func TestOneUnreadableScopeIsNamedAndTheOtherKeepsWorking(t *testing.T) { gh := &fakeGH{jobsErr: map[string]error{org.Key(): errors.New("403 rate limited")}} rw := &fakeRW{} c, alerts := alerting(gh, rw) - for i := 0; i < 4; i++ { + _ = c.Cycle(context.Background()) + if got := c.Status().UnreadableScopes; len(got) != 1 || got[0] != org.Key() { + t.Fatalf("after one failed read: unreadable %v, want [%s]", got, org.Key()) + } + for i := 0; i < 3; i++ { _ = c.Cycle(context.Background()) } - if len(*alerts) != 1 || !strings.Contains((*alerts)[0], org.Key()) || strings.Contains((*alerts)[0], repo.Key()) { - t.Fatalf("alerts %q, want one naming only %s", *alerts, org.Key()) + if len(*alerts) != 1 || !strings.Contains((*alerts)[0], "scopes unreadable that long: "+org.Key()+";") || strings.Contains((*alerts)[0], repo.Key()) { + t.Fatalf("alerts %q, want one naming only %s as unreadable that long", *alerts, org.Key()) } _, body := health(t, c) if got, _ := body["unreadable_scopes"].([]any); len(got) != 1 || got[0] != org.Key() { @@ -97,14 +102,14 @@ func TestOneUnreadableScopeIsNamedAndTheOtherKeepsWorking(t *testing.T) { } } -// Alerts are sent after the cycle releases its lock: a slow alert webhook must not stall /healthz, -// which reads the status under the same lock. +// Alerts are sent after the cycle releases its lock: a slow alert webhook must not hold up the next +// cycle. The alert takes the lock itself, so it deadlocks if it runs while the cycle holds it. func TestAlertsAreSentWithoutHoldingTheLock(t *testing.T) { boom := errors.New("github down") gh := &fakeGH{jobsErr: map[string]error{org.Key(): boom, repo.Key(): boom}} c, _ := alerting(gh, &fakeRW{}) sent := make(chan struct{}, 4) - c.cfg.Alert = func(string) { _ = c.Status(); sent <- struct{}{} } + c.cfg.Alert = func(string) { c.mu.Lock(); c.mu.Unlock(); sent <- struct{}{} } done := make(chan struct{}) go func() { for i := 0; i < 3; i++ { @@ -121,3 +126,59 @@ func TestAlertsAreSentWithoutHoldingTheLock(t *testing.T) { t.Fatalf("alerts %d, want 1", len(sent)) } } + +// /healthz must not wait for a running cycle: a cycle holds the controller's lock for every GitHub +// and Railway call (about a minute in production), and the health check used to time out behind it. +// While a cycle is held open, /healthz answers at once with the previous cycle's status. +func TestHealthzAnswersDuringACycleWithThePreviousStatus(t *testing.T) { + gh := &fakeGH{jobs: map[string][]ghapp.Job{repo.Key(): {{ID: 7, Repo: "acme/app", CreatedAt: now}}}} + c, _ := alerting(gh, &fakeRW{}) + if err := c.Cycle(context.Background()); err != nil { + t.Fatal(err) + } + entered, release := make(chan struct{}), make(chan struct{}) + var once sync.Once + gh.hold = func() { + once.Do(func() { close(entered) }) + <-release + } + done := make(chan struct{}) + go func() { + _ = c.Cycle(context.Background()) + close(done) + }() + <-entered + answered := make(chan *httptest.ResponseRecorder, 1) + go func() { + rec := httptest.NewRecorder() + c.HealthHandler().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/healthz", nil)) + answered <- rec + }() + var rec *httptest.ResponseRecorder + select { + case rec = <-answered: + case <-time.After(100 * time.Millisecond): + t.Error("/healthz did not answer within 100 ms while a cycle was running") + } + close(release) + <-done + if rec == nil { + <-answered // the late answer, so nothing outlives the test + return + } + var body map[string]any + if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { + t.Fatalf("body %q: %v", rec.Body.String(), err) + } + if rec.Code != http.StatusOK || body["queued"] != float64(1) || body["services"] != float64(1) { + t.Errorf("during a cycle: %d %v, want 200 with the previous cycle's queued 1 and services 1", rec.Code, body) + } +} + +// Before the first cycle has finished there is no status yet, and /healthz answers 503. +func TestHealthzIs503BeforeTheFirstCycle(t *testing.T) { + c, _ := alerting(&fakeGH{}, &fakeRW{}) + if code, body := health(t, c); code != http.StatusServiceUnavailable || body["services"] != float64(0) { + t.Fatalf("before any cycle: %d %v, want 503", code, body) + } +}