Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ the major version is 0, a minor release may change configuration (each such chan
- The GitHub CLI (`gh` 2.102.0, pinned by version and sha256) in every image variant, so a job can open or comment on issues without a GitHub-hosted runner. Nothing is logged in: a job passes `GH_TOKEN`.

### Fixed
- `/healthz` answers at once while a cycle runs, with the previous cycle's status; it waited for the cycle to end (about a minute with many repositories) and health checks timed out.
- Jobs queued in a run that GitHub reports as `pending` (for example while it waits on a concurrency group) are served; the controller listed only `queued` and `in_progress` runs and left such jobs waiting forever.

### Changed
Expand Down
50 changes: 30 additions & 20 deletions internal/controller/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
"net/http"
"strings"
"sync"
"sync/atomic"
"time"

"github.com/farzai/github-runner-railway/internal/ghapp"
Expand Down Expand Up @@ -61,9 +62,9 @@ type Controller struct {
rw RW
now func() time.Time
wake chan struct{}
mu sync.Mutex
alerted map[int64]string // job id -> scope key
status Status
mu sync.Mutex // held for a whole cycle; guards the cycle state below, not status
alerted map[int64]string // job id -> scope key
status atomic.Pointer[Status] // the last cycle's outcome, replaced whole and never changed after

failures int // failed cycles in a row
lastSuccess time.Time // the end of the last cycle with no error
Expand Down Expand Up @@ -130,11 +131,13 @@ func (c *Controller) Nudge() {
}
}

// Status is the state after the last cycle.
// Status is the state after the last cycle. It never waits for a running cycle, so /healthz
// answers while a cycle is busy with GitHub and Railway.
func (c *Controller) Status() Status {
c.mu.Lock()
defer c.mu.Unlock()
return c.status
if st := c.status.Load(); st != nil {
return *st
}
return Status{}
}

// Run cycles until ctx ends: on every interval and on every nudge. A nudge waits out the debounce
Expand Down Expand Up @@ -190,7 +193,7 @@ func scopeByKey(ss []reconcile.Scope, key string) reconcile.Scope {
// Cycle is one pass. A scope GitHub cannot be read for is left alone entirely: its services
// are neither counted nor reaped, since a guess about them could kill a busy runner.
func (c *Controller) Cycle(ctx context.Context) (err error) {
// Alerts go out after the lock is released: a slow alert webhook must not stall /healthz.
// Alerts go out after the lock is released: a slow alert webhook must not hold up the next cycle.
var alerts []string
defer func() {
for _, a := range alerts {
Expand Down Expand Up @@ -296,34 +299,41 @@ func (c *Controller) record(now time.Time, errs []error, services, queued int) {
c.failures = 0
c.lastSuccess = now
}
var unreadable, stuck []string
for _, s := range c.cfg.Scopes {
if n := c.scopeFails[s.Key()]; n > 0 {
unreadable = append(unreadable, s.Key())
if n >= c.cfg.FailAlertAfter {
stuck = append(stuck, s.Key())
}
}
}
c.status = Status{LastCycle: now, Services: services, Queued: queued, ConsecutiveFailures: c.failures,
unreadable, stuck := c.unreadableScopes()
st := &Status{LastCycle: now, Services: services, Queued: queued, ConsecutiveFailures: c.failures,
LastSuccess: c.lastSuccess, UnreadableScopes: unreadable, Failing: c.failures >= c.cfg.FailAlertAfter}
if len(errs) > 0 {
c.status.LastError = errors.Join(errs...).Error()
st.LastError = errors.Join(errs...).Error()
}
c.status.Store(st)
switch {
case c.failures >= c.cfg.FailAlertAfter && !c.failAlerted:
c.failAlerted = true
msg := fmt.Sprintf("controller: %d cycles in a row failed", c.failures)
if len(stuck) > 0 {
msg += "; scopes unreadable that long: " + strings.Join(stuck, ", ")
}
c.pending = append(c.pending, msg+"; last error: "+truncate(c.status.LastError, 300))
c.pending = append(c.pending, msg+"; last error: "+truncate(st.LastError, 300))
case c.failures == 0 && c.failAlerted:
c.failAlerted = false
c.pending = append(c.pending, "controller: recovered, the last cycle succeeded")
}
}

// unreadableScopes are the scopes the last cycle could not read, and of those the ones unreadable
// for FailAlertAfter cycles or more.
func (c *Controller) unreadableScopes() (unreadable, stuck []string) {
for _, s := range c.cfg.Scopes {
if n := c.scopeFails[s.Key()]; n > 0 {
unreadable = append(unreadable, s.Key())
if n >= c.cfg.FailAlertAfter {
stuck = append(stuck, s.Key())
}
}
}
return unreadable, stuck
}

func truncate(s string, n int) string {
if len(s) <= n {
return s
Expand Down
4 changes: 4 additions & 0 deletions internal/controller/controller_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,13 @@ type fakeGH struct {
deleted []int64
jit []string
nextID int64
hold func() // when set, QueuedJobs calls it first: a test holds a cycle open with it
}

func (f *fakeGH) QueuedJobs(_ context.Context, s reconcile.Scope) ([]ghapp.Job, error) {
if f.hold != nil {
f.hold()
}
return f.jobs[s.Key()], f.jobsErr[s.Key()]
}
func (f *fakeGH) Runners(_ context.Context, s reconcile.Scope) ([]reconcile.Runner, error) {
Expand Down
73 changes: 67 additions & 6 deletions internal/controller/health_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"net/http"
"net/http/httptest"
"strings"
"sync"
"testing"
"time"

Expand Down Expand Up @@ -80,11 +81,15 @@ func TestOneUnreadableScopeIsNamedAndTheOtherKeepsWorking(t *testing.T) {
gh := &fakeGH{jobsErr: map[string]error{org.Key(): errors.New("403 rate limited")}}
rw := &fakeRW{}
c, alerts := alerting(gh, rw)
for i := 0; i < 4; i++ {
_ = c.Cycle(context.Background())
if got := c.Status().UnreadableScopes; len(got) != 1 || got[0] != org.Key() {
t.Fatalf("after one failed read: unreadable %v, want [%s]", got, org.Key())
}
for i := 0; i < 3; i++ {
_ = c.Cycle(context.Background())
}
if len(*alerts) != 1 || !strings.Contains((*alerts)[0], org.Key()) || strings.Contains((*alerts)[0], repo.Key()) {
t.Fatalf("alerts %q, want one naming only %s", *alerts, org.Key())
if len(*alerts) != 1 || !strings.Contains((*alerts)[0], "scopes unreadable that long: "+org.Key()+";") || strings.Contains((*alerts)[0], repo.Key()) {
t.Fatalf("alerts %q, want one naming only %s as unreadable that long", *alerts, org.Key())
}
_, body := health(t, c)
if got, _ := body["unreadable_scopes"].([]any); len(got) != 1 || got[0] != org.Key() {
Expand All @@ -97,14 +102,14 @@ func TestOneUnreadableScopeIsNamedAndTheOtherKeepsWorking(t *testing.T) {
}
}

// Alerts are sent after the cycle releases its lock: a slow alert webhook must not stall /healthz,
// which reads the status under the same lock.
// Alerts are sent after the cycle releases its lock: a slow alert webhook must not hold up the next
// cycle. The alert takes the lock itself, so it deadlocks if it runs while the cycle holds it.
func TestAlertsAreSentWithoutHoldingTheLock(t *testing.T) {
boom := errors.New("github down")
gh := &fakeGH{jobsErr: map[string]error{org.Key(): boom, repo.Key(): boom}}
c, _ := alerting(gh, &fakeRW{})
sent := make(chan struct{}, 4)
c.cfg.Alert = func(string) { _ = c.Status(); sent <- struct{}{} }
c.cfg.Alert = func(string) { c.mu.Lock(); c.mu.Unlock(); sent <- struct{}{} }
done := make(chan struct{})
go func() {
for i := 0; i < 3; i++ {
Expand All @@ -121,3 +126,59 @@ func TestAlertsAreSentWithoutHoldingTheLock(t *testing.T) {
t.Fatalf("alerts %d, want 1", len(sent))
}
}

// /healthz must not wait for a running cycle: a cycle holds the controller's lock for every GitHub
// and Railway call (about a minute in production), and the health check used to time out behind it.
// While a cycle is held open, /healthz answers at once with the previous cycle's status.
func TestHealthzAnswersDuringACycleWithThePreviousStatus(t *testing.T) {
gh := &fakeGH{jobs: map[string][]ghapp.Job{repo.Key(): {{ID: 7, Repo: "acme/app", CreatedAt: now}}}}
c, _ := alerting(gh, &fakeRW{})
if err := c.Cycle(context.Background()); err != nil {
t.Fatal(err)
}
entered, release := make(chan struct{}), make(chan struct{})
var once sync.Once
gh.hold = func() {
once.Do(func() { close(entered) })
<-release
}
done := make(chan struct{})
go func() {
_ = c.Cycle(context.Background())
close(done)
}()
<-entered
answered := make(chan *httptest.ResponseRecorder, 1)
go func() {
rec := httptest.NewRecorder()
c.HealthHandler().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/healthz", nil))
answered <- rec
}()
var rec *httptest.ResponseRecorder
select {
case rec = <-answered:
case <-time.After(100 * time.Millisecond):
t.Error("/healthz did not answer within 100 ms while a cycle was running")
}
close(release)
<-done
if rec == nil {
<-answered // the late answer, so nothing outlives the test
return
}
var body map[string]any
if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
t.Fatalf("body %q: %v", rec.Body.String(), err)
}
if rec.Code != http.StatusOK || body["queued"] != float64(1) || body["services"] != float64(1) {
t.Errorf("during a cycle: %d %v, want 200 with the previous cycle's queued 1 and services 1", rec.Code, body)
}
}

// Before the first cycle has finished there is no status yet, and /healthz answers 503.
func TestHealthzIs503BeforeTheFirstCycle(t *testing.T) {
c, _ := alerting(&fakeGH{}, &fakeRW{})
if code, body := health(t, c); code != http.StatusServiceUnavailable || body["services"] != float64(0) {
t.Fatalf("before any cycle: %d %v, want 503", code, body)
}
}
Loading