diff --git a/lib/imagepush/manager.go b/lib/imagepush/manager.go index 5f73f540d..f951945d0 100644 --- a/lib/imagepush/manager.go +++ b/lib/imagepush/manager.go @@ -178,6 +178,15 @@ func (m *manager) CreatePush(ctx context.Context, req PushRequest) (*Push, error queuePos := m.queue.Enqueue(key, func() { m.executePush(context.Background(), &metaCopy, provider) }, m.releaseInflight(key)) + if queuePos < 0 { + // The job never entered the queue, so no completion hook will fire: + // drop the reservation and the record so a retry starts clean. + m.mu.Lock() + delete(m.inflight, key) + m.mu.Unlock() + removePushData(m.paths, meta.ID) + return nil, fmt.Errorf("push queue is full: %w", queue.ErrFull) + } push := meta.toPush() if queuePos > 0 { @@ -443,9 +452,19 @@ func (m *manager) recoverInterruptedPushes() { m.mu.Unlock() metaCopy := *meta - m.queue.Enqueue(key, func() { + if m.queue.Enqueue(key, func() { m.executePush(context.Background(), &metaCopy, m.provider) - }, m.releaseInflight(key)) + }, m.releaseInflight(key)) < 0 { + // Keep recovery bounded: a record that cannot enter the queue is + // failed with a reason instead of sitting queued forever. The + // reservation above must go too: no job will ever run to release it, + // and a stale entry would keep handing this failed record to later + // CreatePush calls for the same digest and target. + m.mu.Lock() + delete(m.inflight, key) + m.mu.Unlock() + m.failRecovered(meta, "push queue is full during recovery; resubmit the push") + } } } diff --git a/lib/imagepush/recovery_queue_bound_test.go b/lib/imagepush/recovery_queue_bound_test.go new file mode 100644 index 000000000..83d86e6a7 --- /dev/null +++ b/lib/imagepush/recovery_queue_bound_test.go @@ -0,0 +1,95 @@ +package imagepush + +import ( + "context" + "fmt" + "sync" + "testing" + "time" + + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/queue" + "github.com/kernel/hypeman/lib/registrypush" + "github.com/stretchr/testify/require" +) + +// fullQueue returns a queue whose single worker is blocked and whose pending +// slots are all taken, so the next Enqueue is refused. drain releases the +// blocker so the queue empties. +func fullQueue(t *testing.T) (q *queue.Queue, drain func()) { + t.Helper() + + q = queue.New(1) + release := make(chan struct{}) + var once sync.Once + drain = func() { + once.Do(func() { close(release) }) + require.Eventually(t, func() bool { return q.QueueLength() == 0 }, 10*time.Second, 10*time.Millisecond) + } + t.Cleanup(drain) + + require.Equal(t, 0, q.Enqueue("blocker", func() { <-release }, nil)) + for i := 0; i < queue.MaxPending; i++ { + require.Greater(t, q.Enqueue(fmt.Sprintf("filler-%d", i), func() {}, nil), 0) + } + return q, drain +} + +func TestRecoveryRefusedByFullQueueLeavesNoInflightEntry(t *testing.T) { + p, digest := cacheFixture(t) + host := openRegistry(t) + target := host + "/export/recovered:v1" + + meta := &pushMetadata{ + ID: "recovered-push", + Status: StatusPushing, + Image: "myapp:v1", + Digest: digest, + Target: target, + Insecure: true, + CreatedAt: time.Now(), + } + writePushes(t, p, meta) + + q, drain := fullQueue(t) + mgr := &manager{ + paths: p, + resolver: &fakeResolver{images: map[string]*images.Image{ + "myapp:v1": readyImage("myapp:v1", digest), + }}, + provider: ®istrypush.KeychainProvider{}, + queue: q, + pushTimeout: DefaultPushTimeout, + inflight: make(map[string]inflightPush), + subscribers: make(map[string][]chan StatusEvent), + } + mgr.recoverInterruptedPushes() + + // The refused record is terminal so it cannot sit queued forever. + failed, err := readMetadata(p, "recovered-push") + require.NoError(t, err) + require.Equal(t, StatusFailed, failed.Status) + require.NotNil(t, failed.Error) + require.Contains(t, *failed.Error, "queue is full") + + // No reservation and no queued job survived the refusal. + key := pushKey(digest, target, true) + mgr.mu.Lock() + _, inflight := mgr.inflight[key] + mgr.mu.Unlock() + require.False(t, inflight, "the refused recovery must drop its inflight entry") + require.Equal(t, 1+queue.MaxPending, q.QueueLength(), "the refused job must not be queued") + + // Once the queue drains, a CreatePush for the same digest and target runs + // as a fresh job instead of handing back the failed record. + drain() + push, err := mgr.CreatePush(context.Background(), PushRequest{ + Image: "myapp:v1", Target: target, Insecure: true, + }) + require.NoError(t, err) + require.NotEmpty(t, push.ID) + require.NotEqual(t, "recovered-push", push.ID, "a retry must not dedup onto the failed record") + + mustPushed(t, mgr, push.ID) + waitNoInflight(t, mgr) +} diff --git a/lib/images/credentials_test.go b/lib/images/credentials_test.go index 7f680b980..ab0c14706 100644 --- a/lib/images/credentials_test.go +++ b/lib/images/credentials_test.go @@ -89,7 +89,7 @@ func TestInflightPullRejectsDifferentCredentials(t *testing.T) { } const digest = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" credentials := &authn.AuthConfig{Username: "AWS", Password: "token-a"} - inflight := m.registerInflightPull(digest, credentials) + inflight, _ := m.registerInflightPull(digest, credentials) t.Cleanup(m.releaseInflightPull(digest, inflight)) assert.True(t, m.inflightCredentialsMatch(digest, &authn.AuthConfig{Username: "AWS", Password: "token-a"})) @@ -103,7 +103,7 @@ func TestBorrowedCredentialsExpireWhileQueued(t *testing.T) { borrowedCredentialsTimeout: time.Millisecond, } const digest = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" - inflight := m.registerInflightPull(digest, &authn.AuthConfig{Username: "AWS", Password: "secret"}) + inflight, _ := m.registerInflightPull(digest, &authn.AuthConfig{Username: "AWS", Password: "secret"}) defer m.releaseInflightPull(digest, inflight)() require.Eventually(t, func() bool { @@ -121,8 +121,8 @@ func TestBorrowedCredentialsExpireWhileQueued(t *testing.T) { func TestBorrowedAuthRejectsReplacedInflightPull(t *testing.T) { m := &manager{inflightPulls: make(map[string]*inflightImagePull)} const digest = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" - first := m.registerInflightPull(digest, &authn.AuthConfig{Username: "first"}) - second := m.registerInflightPull(digest, &authn.AuthConfig{Username: "second"}) + first, _ := m.registerInflightPull(digest, &authn.AuthConfig{Username: "first"}) + second, _ := m.registerInflightPull(digest, &authn.AuthConfig{Username: "second"}) defer m.releaseInflightPull(digest, second)() credentials, _, expired, stale := m.borrowedAuth(digest, first) diff --git a/lib/images/manager.go b/lib/images/manager.go index 8b1f733c3..fcd68d5b7 100644 --- a/lib/images/manager.go +++ b/lib/images/manager.go @@ -316,11 +316,14 @@ func (m *manager) inflightCredentialsMatch(digest string, credentials *authn.Aut return existingFingerprint == credentialFingerprint(credentials) } -func (m *manager) registerInflightPull(digest string, credentials *authn.AuthConfig) *inflightImagePull { +// registerInflightPull publishes a new handle for digest. It returns the +// handle it replaced, if any, so a refused successor can put that one back. +func (m *manager) registerInflightPull(digest string, credentials *authn.AuthConfig) (current, previous *inflightImagePull) { if m.inflightPulls == nil { m.inflightPulls = make(map[string]*inflightImagePull) } - if previous := m.inflightPulls[digest]; previous != nil && previous.timer != nil { + previous = m.inflightPulls[digest] + if previous != nil && previous.timer != nil { previous.timer.Stop() } inflight := &inflightImagePull{ @@ -328,36 +331,71 @@ func (m *manager) registerInflightPull(digest string, credentials *authn.AuthCon credentials: credentials, } m.inflightPulls[digest] = inflight - if credentials == nil { - return inflight + if credentials != nil { + if m.borrowedCredentialsTimeout <= 0 { + m.borrowedCredentialsTimeout = DefaultBorrowedCredentialsTimeout + } + inflight.credentialsExpireAt = time.Now().Add(m.borrowedCredentialsTimeout) + m.armInflightExpiryLocked(digest, inflight) } - if m.borrowedCredentialsTimeout <= 0 { - m.borrowedCredentialsTimeout = DefaultBorrowedCredentialsTimeout + return inflight, previous +} + +// armInflightExpiryLocked starts the credentials expiry timer from the deadline +// the handle already carries, so a restored handle keeps its original bound. +// createMu must be held. +func (m *manager) armInflightExpiryLocked(digest string, inflight *inflightImagePull) { + if inflight.credentials == nil { + return + } + remaining := time.Until(inflight.credentialsExpireAt) + if remaining <= 0 { + // Expired credentials must not survive as current: drop them so the + // pull fails with ErrBorrowedCredentialsExpired instead of running + // unauthenticated. + inflight.credentials = nil + return } - inflight.credentialsExpireAt = time.Now().Add(m.borrowedCredentialsTimeout) - inflight.timer = time.AfterFunc(m.borrowedCredentialsTimeout, func() { + inflight.timer = time.AfterFunc(remaining, func() { m.createMu.Lock() defer m.createMu.Unlock() if m.inflightPulls[digest] == inflight { inflight.credentials = nil } }) - return inflight +} + +// restoreInflightPullLocked makes previous current again after its successor +// was refused, so the pull already running for digest keeps its credentials. +// It re-arms the previous handle's expiry bound. createMu must be held. +func (m *manager) restoreInflightPullLocked(digest string, previous *inflightImagePull) { + if previous == nil { + delete(m.inflightPulls, digest) + return + } + m.inflightPulls[digest] = previous + m.armInflightExpiryLocked(digest, previous) } func (m *manager) releaseInflightPull(digest string, inflight *inflightImagePull) func() { return func() { m.createMu.Lock() defer m.createMu.Unlock() - if m.inflightPulls[digest] != inflight { - return - } - delete(m.inflightPulls, digest) - if inflight.timer != nil { - inflight.timer.Stop() - } - inflight.credentials = nil + m.releaseInflightPullLocked(digest, inflight) + } +} + +// releaseInflightPullLocked drops inflight when it is still the current handle. +// createMu must be held. +func (m *manager) releaseInflightPullLocked(digest string, inflight *inflightImagePull) { + if m.inflightPulls[digest] != inflight { + return + } + delete(m.inflightPulls, digest) + if inflight.timer != nil { + inflight.timer.Stop() } + inflight.credentials = nil } func (m *manager) borrowedAuth(digest string, expected *inflightImagePull) (*authn.AuthConfig, time.Time, bool, bool) { @@ -400,7 +438,7 @@ func (m *manager) createAndQueueImage(ref *ResolvedRef, req CreateImageRequest, // Keep borrowed credentials outside the queued closure so their lifetime is // bounded even when this job waits behind another pull. - inflight := m.registerInflightPull(ref.Digest(), req.Credentials) + inflight, previousInflight := m.registerInflightPull(ref.Digest(), req.Credentials) buildID := meta.BuildID queuePos := m.queue.EnqueueSuccessor(ref.Digest(), func() { credentials, deadline, expired, stale := m.borrowedAuth(ref.Digest(), inflight) @@ -419,6 +457,16 @@ func (m *manager) createAndQueueImage(ref *ResolvedRef, req CreateImageRequest, } m.buildImage(ctx, ref, credentials, buildID) }, m.releaseInflightPull(ref.Digest(), inflight)) + if queuePos < 0 { + // Rejected before entering the queue: drop the reserved handle, put back + // the pull that was already running for this digest, then fail the + // record so it is not left pending forever. createMu is held here (both + // callers hold it), so the locked variants are used. + m.releaseInflightPullLocked(ref.Digest(), inflight) + m.restoreInflightPullLocked(ref.Digest(), previousInflight) + m.updateStatusByDigestLocked(ref, StatusFailed, queue.ErrFull, buildID) + return nil, queue.ErrFull + } img := meta.toImageFor(ref.String()) if queuePos > 0 { @@ -597,7 +645,12 @@ func (m *manager) recordImageBuildPhase(ctx context.Context, digest, phase strin func (m *manager) updateStatusByDigest(ref *ResolvedRef, status string, err error, buildID string) { m.createMu.Lock() defer m.createMu.Unlock() + m.updateStatusByDigestLocked(ref, status, err, buildID) +} +// updateStatusByDigestLocked applies a status change unless the record moved on +// to another build. createMu must be held. +func (m *manager) updateStatusByDigestLocked(ref *ResolvedRef, status string, err error, buildID string) { layout := resolveImageLayout(m.paths, ref.Repository(), ref.DigestHex()) meta, readErr := readMetadataAt(layout) if readErr != nil || meta.BuildID != buildID { @@ -669,9 +722,13 @@ func (m *manager) recoverInterruptedBuild(meta *imageMetadata) bool { return true } buildID := meta.BuildID - m.queue.Enqueue(meta.Digest, func() { + if m.queue.Enqueue(meta.Digest, func() { m.buildImage(context.Background(), ref, nil, buildID) - }, nil) + }, nil) < 0 { + // A rejected recovery would stay pending with no runner; fail it so + // the record reaches a terminal state a caller can see. + m.updateStatusByDigest(ref, StatusFailed, queue.ErrFull, buildID) + } return true } diff --git a/lib/images/queue_bound_test.go b/lib/images/queue_bound_test.go new file mode 100644 index 000000000..c23e55142 --- /dev/null +++ b/lib/images/queue_bound_test.go @@ -0,0 +1,132 @@ +package images + +import ( + "context" + "fmt" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/registry" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/kernel/hypeman/lib/paths" + "github.com/kernel/hypeman/lib/queue" + "github.com/stretchr/testify/require" +) + +// fillQueue replaces m's queue with one whose single worker is blocked and +// whose pending slots are all taken, so the next Enqueue reports a full queue. +func fillQueue(t *testing.T, m *manager) { + t.Helper() + + q := queue.New(1) + release := make(chan struct{}) + t.Cleanup(func() { close(release) }) + + require.Equal(t, 0, q.Enqueue("blocker", func() { <-release }, nil)) + for i := 0; i < queue.MaxPending; i++ { + require.Greater(t, q.Enqueue(fmt.Sprintf("filler-%d", i), func() {}, nil), 0) + } + m.queue = q +} + +func TestCreateImageRefusedByFullQueueDoesNotBlock(t *testing.T) { + reg := registry.New() + server := httptest.NewServer(reg) + defer server.Close() + host := strings.TrimPrefix(server.URL, "http://") + + var refs []string + for _, path := range []string{"/library/full-a:latest", "/library/full-b:latest"} { + ref := host + path + parsed, err := name.ParseReference(ref, name.Insecure) + require.NoError(t, err) + img, err := random.Image(256, 1) + require.NoError(t, err) + require.NoError(t, remote.Write(parsed, img)) + refs = append(refs, ref) + } + + mgr, err := NewManager(paths.New(t.TempDir()), 1, nil) + require.NoError(t, err) + m := mgr.(*manager) + fillQueue(t, m) + + ctx := context.Background() + callCreate := func(ref string) error { + t.Helper() + done := make(chan error, 1) + go func() { + _, err := mgr.CreateImage(ctx, CreateImageRequest{Name: ref}) + done <- err + }() + select { + case err := <-done: + return err + case <-time.After(10 * time.Second): + t.Fatalf("CreateImage(%s) blocked while the queue was full", ref) + return nil + } + } + + for _, ref := range refs { + require.ErrorIs(t, callCreate(ref), queue.ErrFull, "create of %s must surface the refusal", ref) + } + + // The manager must still be usable: another create is refused promptly + // instead of deadlocking, and a delete completes. + require.ErrorIs(t, callCreate(refs[0]), queue.ErrFull) + + done := make(chan error, 1) + go func() { done <- mgr.DeleteImage(ctx, refs[0]) }() + select { + case err := <-done: + require.NoError(t, err) + case <-time.After(10 * time.Second): + t.Fatal("DeleteImage blocked after a refused create") + } +} + +func TestRefusedSuccessorKeepsPreviousInflightPull(t *testing.T) { + mgr, err := NewManager(paths.New(t.TempDir()), 1, nil) + require.NoError(t, err) + m := mgr.(*manager) + fillQueue(t, m) + m.borrowedCredentialsTimeout = time.Second + + normalized, err := ParseNormalizedRef("registry.example/library/successor:latest") + require.NoError(t, err) + const digest = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ref := NewResolvedRef(normalized, digest) + credentials := &authn.AuthConfig{Username: "borrower", Password: "secret"} + + m.createMu.Lock() + previous, _ := m.registerInflightPull(ref.Digest(), credentials) + _, err = m.createAndQueueImage(ref, CreateImageRequest{Name: normalized.String(), Credentials: credentials}, hostPlatform()) + m.createMu.Unlock() + require.ErrorIs(t, err, queue.ErrFull) + + // The pull that was already running keeps its handle, so it still builds. + gotCredentials, _, expired, stale := m.borrowedAuth(ref.Digest(), previous) + require.False(t, stale, "the running pull must stay current after a refused successor") + require.False(t, expired) + require.NotNil(t, gotCredentials) + + // Its expiry bound is still enforced on the restored handle. + require.Eventually(t, func() bool { + _, _, expired, stale := m.borrowedAuth(ref.Digest(), previous) + return expired && !stale + }, 5*time.Second, 10*time.Millisecond) + + // The refused successor's record is terminal instead of pending with no + // runner. + meta, err := readMetadata(m.paths, ref.Repository(), ref.DigestHex()) + require.NoError(t, err) + require.Equal(t, StatusFailed, meta.Status) + require.NotNil(t, meta.Error) + require.Contains(t, *meta.Error, queue.ErrFull.Error()) +} diff --git a/lib/queue/queue.go b/lib/queue/queue.go index 6e6c55c1b..be48c8acf 100644 --- a/lib/queue/queue.go +++ b/lib/queue/queue.go @@ -3,7 +3,19 @@ // already active or pending returns its position instead of re-enqueueing. package queue -import "sync" +import ( + "errors" + "sync" +) + +// ErrFull is wrapped by callers when Enqueue reports a full queue, so an +// unbounded pile of pending jobs cannot be grown by request input. +var ErrFull = errors.New("queue pending limit reached") + +// MaxPending bounds the pending queue. Enqueue returns -1 once it is reached; +// callers must treat that as a rejected job (fail it or tell the caller to +// retry) instead of assuming it will start later. +const MaxPending = 512 type job struct { key string @@ -16,6 +28,7 @@ type Queue struct { maxConcurrent int active map[string]bool pending []job + pendingIndex map[string]int // key -> index in pending, for O(1) dedup mu sync.Mutex } @@ -28,16 +41,18 @@ func New(maxConcurrent int) *Queue { maxConcurrent: maxConcurrent, active: make(map[string]bool), pending: make([]job, 0), + pendingIndex: make(map[string]int), } } // Enqueue adds a job keyed by key. Returns the queue position: 0 if it -// started immediately, >0 if queued behind other jobs. If the key is already -// active or pending, Enqueue dedups and returns its current position without -// re-enqueueing. A completion hook may be provided; when non-nil it runs -// after the key leaves the active set (so a caller's bookkeeping for the key -// is torn down only once the queue is done with it), but only when this call -// actually started the job — a dedup'd enqueue does not run it. +// started immediately, >0 if queued behind other jobs, and -1 if the pending +// queue is full (the job was not queued and will not run). If the key is +// already active or pending, Enqueue dedups and returns its current position +// without re-enqueueing. A completion hook may be provided; when non-nil it +// runs after the key leaves the active set (so a caller's bookkeeping for the +// key is torn down only once the queue is done with it), but only when this +// call actually started the job — a dedup'd enqueue does not run it. func (q *Queue) Enqueue(key string, startFn func(), done func()) int { return q.enqueue(key, startFn, done, false, false) } @@ -68,13 +83,11 @@ func (q *Queue) enqueue(key string, startFn func(), done func(), allowActive, re startFn() } - for i, j := range q.pending { - if j.key == key { - if replacePending { - q.pending[i] = job{key: key, startFn: wrappedFn} - } - return i + 1 + if i, ok := q.pendingIndex[key]; ok { + if replacePending { + q.pending[i] = job{key: key, startFn: wrappedFn} } + return i + 1 } if len(q.active) < q.maxConcurrent && !q.active[key] { @@ -88,7 +101,12 @@ func (q *Queue) enqueue(key string, startFn func(), done func(), allowActive, re return 0 } + if len(q.pending) >= MaxPending { + return -1 + } + q.pending = append(q.pending, job{key: key, startFn: wrappedFn}) + q.pendingIndex[key] = len(q.pending) - 1 return len(q.pending) } @@ -106,6 +124,10 @@ func (q *Queue) complete(key string) { continue } q.pending = append(q.pending[:i], q.pending[i+1:]...) + delete(q.pendingIndex, pending.key) + for j := i; j < len(q.pending); j++ { + q.pendingIndex[q.pending[j].key] = j + } q.active[pending.key] = true next = &pending break @@ -124,11 +146,9 @@ func (q *Queue) GetPosition(key string) *int { q.mu.Lock() defer q.mu.Unlock() - for i, j := range q.pending { - if j.key == key { - pos := i + 1 - return &pos - } + if i, ok := q.pendingIndex[key]; ok { + pos := i + 1 + return &pos } return nil } diff --git a/lib/queue/queue_test.go b/lib/queue/queue_test.go index be12f68c6..bf0c50c1a 100644 --- a/lib/queue/queue_test.go +++ b/lib/queue/queue_test.go @@ -1,6 +1,7 @@ package queue import ( + "fmt" "sync" "sync/atomic" "testing" @@ -242,3 +243,33 @@ func TestDoneRunsAfterKeyReleased(t *testing.T) { time.Sleep(10 * time.Millisecond) } } + +func TestEnqueueRejectsAtPendingLimit(t *testing.T) { + q := New(1) + block := make(chan struct{}) + started := make(chan struct{}) + q.Enqueue("active", func() { close(started); <-block }, nil) + <-started + + for i := 0; i < MaxPending; i++ { + if pos := q.Enqueue(fmt.Sprintf("p-%d", i), func() {}, nil); pos != i+1 { + t.Fatalf("enqueue %d: pos = %d, want %d", i, pos, i+1) + } + } + if pos := q.Enqueue("overflow", func() {}, nil); pos != -1 { + t.Fatalf("overflow Enqueue = %d, want -1", pos) + } + if pos := q.Enqueue("p-0", func() {}, nil); pos != 1 { + t.Fatalf("dedup at full queue = %d, want 1", pos) + } + + close(block) + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + if q.Enqueue("after", func() {}, nil) >= 0 { + return + } + time.Sleep(time.Millisecond) + } + t.Fatal("queue did not accept a job after the active one completed") +}