From 18af2f930b51e5612dd720a4bdbda26110e73432 Mon Sep 17 00:00:00 2001 From: walsykerem Date: Sat, 21 Nov 2026 20:45:20 +0300 Subject: [PATCH 1/2] fix(queue): bound pending depth and make dedup constant time --- lib/imagepush/manager.go | 17 ++++++++++-- lib/images/manager.go | 15 +++++++++-- lib/queue/queue.go | 56 +++++++++++++++++++++++++++------------- lib/queue/queue_test.go | 31 ++++++++++++++++++++++ 4 files changed, 97 insertions(+), 22 deletions(-) diff --git a/lib/imagepush/manager.go b/lib/imagepush/manager.go index 5f73f540d..fdcc9fe83 100644 --- a/lib/imagepush/manager.go +++ b/lib/imagepush/manager.go @@ -178,6 +178,15 @@ func (m *manager) CreatePush(ctx context.Context, req PushRequest) (*Push, error queuePos := m.queue.Enqueue(key, func() { m.executePush(context.Background(), &metaCopy, provider) }, m.releaseInflight(key)) + if queuePos < 0 { + // The job never entered the queue, so no completion hook will fire: + // drop the reservation and the record so a retry starts clean. + m.mu.Lock() + delete(m.inflight, key) + m.mu.Unlock() + removePushData(m.paths, meta.ID) + return nil, fmt.Errorf("push queue is full: %w", queue.ErrFull) + } push := meta.toPush() if queuePos > 0 { @@ -443,9 +452,13 @@ func (m *manager) recoverInterruptedPushes() { m.mu.Unlock() metaCopy := *meta - m.queue.Enqueue(key, func() { + if m.queue.Enqueue(key, func() { m.executePush(context.Background(), &metaCopy, m.provider) - }, m.releaseInflight(key)) + }, m.releaseInflight(key)) < 0 { + // Keep recovery bounded: a record that cannot enter the queue is + // failed with a reason instead of sitting queued forever. + m.failRecovered(meta, "push queue is full during recovery; resubmit the push") + } } } diff --git a/lib/images/manager.go b/lib/images/manager.go index 8b1f733c3..cb93c937c 100644 --- a/lib/images/manager.go +++ b/lib/images/manager.go @@ -419,6 +419,13 @@ func (m *manager) createAndQueueImage(ref *ResolvedRef, req CreateImageRequest, } m.buildImage(ctx, ref, credentials, buildID) }, m.releaseInflightPull(ref.Digest(), inflight)) + if queuePos < 0 { + // Rejected before entering the queue: release the borrowed credentials + // and fail the record so it is not left pending forever. + m.releaseInflightPull(ref.Digest(), inflight)() + m.updateStatusByDigest(ref, StatusFailed, queue.ErrFull, buildID) + return nil, queue.ErrFull + } img := meta.toImageFor(ref.String()) if queuePos > 0 { @@ -669,9 +676,13 @@ func (m *manager) recoverInterruptedBuild(meta *imageMetadata) bool { return true } buildID := meta.BuildID - m.queue.Enqueue(meta.Digest, func() { + if m.queue.Enqueue(meta.Digest, func() { m.buildImage(context.Background(), ref, nil, buildID) - }, nil) + }, nil) < 0 { + // A rejected recovery would stay pending with no runner; fail it so + // the record reaches a terminal state a caller can see. + m.updateStatusByDigest(ref, StatusFailed, queue.ErrFull, buildID) + } return true } diff --git a/lib/queue/queue.go b/lib/queue/queue.go index 6e6c55c1b..be48c8acf 100644 --- a/lib/queue/queue.go +++ b/lib/queue/queue.go @@ -3,7 +3,19 @@ // already active or pending returns its position instead of re-enqueueing. package queue -import "sync" +import ( + "errors" + "sync" +) + +// ErrFull is wrapped by callers when Enqueue reports a full queue, so an +// unbounded pile of pending jobs cannot be grown by request input. +var ErrFull = errors.New("queue pending limit reached") + +// MaxPending bounds the pending queue. Enqueue returns -1 once it is reached; +// callers must treat that as a rejected job (fail it or tell the caller to +// retry) instead of assuming it will start later. +const MaxPending = 512 type job struct { key string @@ -16,6 +28,7 @@ type Queue struct { maxConcurrent int active map[string]bool pending []job + pendingIndex map[string]int // key -> index in pending, for O(1) dedup mu sync.Mutex } @@ -28,16 +41,18 @@ func New(maxConcurrent int) *Queue { maxConcurrent: maxConcurrent, active: make(map[string]bool), pending: make([]job, 0), + pendingIndex: make(map[string]int), } } // Enqueue adds a job keyed by key. Returns the queue position: 0 if it -// started immediately, >0 if queued behind other jobs. If the key is already -// active or pending, Enqueue dedups and returns its current position without -// re-enqueueing. A completion hook may be provided; when non-nil it runs -// after the key leaves the active set (so a caller's bookkeeping for the key -// is torn down only once the queue is done with it), but only when this call -// actually started the job — a dedup'd enqueue does not run it. +// started immediately, >0 if queued behind other jobs, and -1 if the pending +// queue is full (the job was not queued and will not run). If the key is +// already active or pending, Enqueue dedups and returns its current position +// without re-enqueueing. A completion hook may be provided; when non-nil it +// runs after the key leaves the active set (so a caller's bookkeeping for the +// key is torn down only once the queue is done with it), but only when this +// call actually started the job — a dedup'd enqueue does not run it. func (q *Queue) Enqueue(key string, startFn func(), done func()) int { return q.enqueue(key, startFn, done, false, false) } @@ -68,13 +83,11 @@ func (q *Queue) enqueue(key string, startFn func(), done func(), allowActive, re startFn() } - for i, j := range q.pending { - if j.key == key { - if replacePending { - q.pending[i] = job{key: key, startFn: wrappedFn} - } - return i + 1 + if i, ok := q.pendingIndex[key]; ok { + if replacePending { + q.pending[i] = job{key: key, startFn: wrappedFn} } + return i + 1 } if len(q.active) < q.maxConcurrent && !q.active[key] { @@ -88,7 +101,12 @@ func (q *Queue) enqueue(key string, startFn func(), done func(), allowActive, re return 0 } + if len(q.pending) >= MaxPending { + return -1 + } + q.pending = append(q.pending, job{key: key, startFn: wrappedFn}) + q.pendingIndex[key] = len(q.pending) - 1 return len(q.pending) } @@ -106,6 +124,10 @@ func (q *Queue) complete(key string) { continue } q.pending = append(q.pending[:i], q.pending[i+1:]...) + delete(q.pendingIndex, pending.key) + for j := i; j < len(q.pending); j++ { + q.pendingIndex[q.pending[j].key] = j + } q.active[pending.key] = true next = &pending break @@ -124,11 +146,9 @@ func (q *Queue) GetPosition(key string) *int { q.mu.Lock() defer q.mu.Unlock() - for i, j := range q.pending { - if j.key == key { - pos := i + 1 - return &pos - } + if i, ok := q.pendingIndex[key]; ok { + pos := i + 1 + return &pos } return nil } diff --git a/lib/queue/queue_test.go b/lib/queue/queue_test.go index be12f68c6..bf0c50c1a 100644 --- a/lib/queue/queue_test.go +++ b/lib/queue/queue_test.go @@ -1,6 +1,7 @@ package queue import ( + "fmt" "sync" "sync/atomic" "testing" @@ -242,3 +243,33 @@ func TestDoneRunsAfterKeyReleased(t *testing.T) { time.Sleep(10 * time.Millisecond) } } + +func TestEnqueueRejectsAtPendingLimit(t *testing.T) { + q := New(1) + block := make(chan struct{}) + started := make(chan struct{}) + q.Enqueue("active", func() { close(started); <-block }, nil) + <-started + + for i := 0; i < MaxPending; i++ { + if pos := q.Enqueue(fmt.Sprintf("p-%d", i), func() {}, nil); pos != i+1 { + t.Fatalf("enqueue %d: pos = %d, want %d", i, pos, i+1) + } + } + if pos := q.Enqueue("overflow", func() {}, nil); pos != -1 { + t.Fatalf("overflow Enqueue = %d, want -1", pos) + } + if pos := q.Enqueue("p-0", func() {}, nil); pos != 1 { + t.Fatalf("dedup at full queue = %d, want 1", pos) + } + + close(block) + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + if q.Enqueue("after", func() {}, nil) >= 0 { + return + } + time.Sleep(time.Millisecond) + } + t.Fatal("queue did not accept a job after the active one completed") +} From a93ff2791dff42e0bf1fe29e9f1cd4e82ce54347 Mon Sep 17 00:00:00 2001 From: walsykerem Date: Sat, 21 Nov 2026 22:15:57 +0300 Subject: [PATCH 2/2] fix(images,imagepush): survive a full pending queue A full pending queue used to leave the image and push managers in a bad state. Refusing a pull deadlocked the image manager, a refused recovery leaked a push reservation, and a refused successor left the running pull without a handle to build with. - images: split releaseInflightPull and updateStatusByDigest into ...Locked bodies plus locking wrappers, and use the locked variants in the rejection path where createMu is already held. The record still ends failed with the queue error, credentials and timer cleared. - images: registerInflightPull now returns the handle it replaced, and a refused successor restores it with its credential expiry re-armed, so the pull already running keeps going and no expired handle stays current. - imagepush: a recovery refused by the full queue drops its inflight entry before failing the record, so a later push for the same digest and target can run and garbage collection stops treating it as live. Tests fill the queue through lib/queue directly and cover the refused create, the refused successor and the refused recovery. --- lib/imagepush/manager.go | 8 +- lib/imagepush/recovery_queue_bound_test.go | 95 +++++++++++++++ lib/images/credentials_test.go | 8 +- lib/images/manager.go | 90 ++++++++++---- lib/images/queue_bound_test.go | 132 +++++++++++++++++++++ 5 files changed, 306 insertions(+), 27 deletions(-) create mode 100644 lib/imagepush/recovery_queue_bound_test.go create mode 100644 lib/images/queue_bound_test.go diff --git a/lib/imagepush/manager.go b/lib/imagepush/manager.go index fdcc9fe83..f951945d0 100644 --- a/lib/imagepush/manager.go +++ b/lib/imagepush/manager.go @@ -456,7 +456,13 @@ func (m *manager) recoverInterruptedPushes() { m.executePush(context.Background(), &metaCopy, m.provider) }, m.releaseInflight(key)) < 0 { // Keep recovery bounded: a record that cannot enter the queue is - // failed with a reason instead of sitting queued forever. + // failed with a reason instead of sitting queued forever. The + // reservation above must go too: no job will ever run to release it, + // and a stale entry would keep handing this failed record to later + // CreatePush calls for the same digest and target. + m.mu.Lock() + delete(m.inflight, key) + m.mu.Unlock() m.failRecovered(meta, "push queue is full during recovery; resubmit the push") } } diff --git a/lib/imagepush/recovery_queue_bound_test.go b/lib/imagepush/recovery_queue_bound_test.go new file mode 100644 index 000000000..83d86e6a7 --- /dev/null +++ b/lib/imagepush/recovery_queue_bound_test.go @@ -0,0 +1,95 @@ +package imagepush + +import ( + "context" + "fmt" + "sync" + "testing" + "time" + + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/queue" + "github.com/kernel/hypeman/lib/registrypush" + "github.com/stretchr/testify/require" +) + +// fullQueue returns a queue whose single worker is blocked and whose pending +// slots are all taken, so the next Enqueue is refused. drain releases the +// blocker so the queue empties. +func fullQueue(t *testing.T) (q *queue.Queue, drain func()) { + t.Helper() + + q = queue.New(1) + release := make(chan struct{}) + var once sync.Once + drain = func() { + once.Do(func() { close(release) }) + require.Eventually(t, func() bool { return q.QueueLength() == 0 }, 10*time.Second, 10*time.Millisecond) + } + t.Cleanup(drain) + + require.Equal(t, 0, q.Enqueue("blocker", func() { <-release }, nil)) + for i := 0; i < queue.MaxPending; i++ { + require.Greater(t, q.Enqueue(fmt.Sprintf("filler-%d", i), func() {}, nil), 0) + } + return q, drain +} + +func TestRecoveryRefusedByFullQueueLeavesNoInflightEntry(t *testing.T) { + p, digest := cacheFixture(t) + host := openRegistry(t) + target := host + "/export/recovered:v1" + + meta := &pushMetadata{ + ID: "recovered-push", + Status: StatusPushing, + Image: "myapp:v1", + Digest: digest, + Target: target, + Insecure: true, + CreatedAt: time.Now(), + } + writePushes(t, p, meta) + + q, drain := fullQueue(t) + mgr := &manager{ + paths: p, + resolver: &fakeResolver{images: map[string]*images.Image{ + "myapp:v1": readyImage("myapp:v1", digest), + }}, + provider: ®istrypush.KeychainProvider{}, + queue: q, + pushTimeout: DefaultPushTimeout, + inflight: make(map[string]inflightPush), + subscribers: make(map[string][]chan StatusEvent), + } + mgr.recoverInterruptedPushes() + + // The refused record is terminal so it cannot sit queued forever. + failed, err := readMetadata(p, "recovered-push") + require.NoError(t, err) + require.Equal(t, StatusFailed, failed.Status) + require.NotNil(t, failed.Error) + require.Contains(t, *failed.Error, "queue is full") + + // No reservation and no queued job survived the refusal. + key := pushKey(digest, target, true) + mgr.mu.Lock() + _, inflight := mgr.inflight[key] + mgr.mu.Unlock() + require.False(t, inflight, "the refused recovery must drop its inflight entry") + require.Equal(t, 1+queue.MaxPending, q.QueueLength(), "the refused job must not be queued") + + // Once the queue drains, a CreatePush for the same digest and target runs + // as a fresh job instead of handing back the failed record. + drain() + push, err := mgr.CreatePush(context.Background(), PushRequest{ + Image: "myapp:v1", Target: target, Insecure: true, + }) + require.NoError(t, err) + require.NotEmpty(t, push.ID) + require.NotEqual(t, "recovered-push", push.ID, "a retry must not dedup onto the failed record") + + mustPushed(t, mgr, push.ID) + waitNoInflight(t, mgr) +} diff --git a/lib/images/credentials_test.go b/lib/images/credentials_test.go index 7f680b980..ab0c14706 100644 --- a/lib/images/credentials_test.go +++ b/lib/images/credentials_test.go @@ -89,7 +89,7 @@ func TestInflightPullRejectsDifferentCredentials(t *testing.T) { } const digest = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" credentials := &authn.AuthConfig{Username: "AWS", Password: "token-a"} - inflight := m.registerInflightPull(digest, credentials) + inflight, _ := m.registerInflightPull(digest, credentials) t.Cleanup(m.releaseInflightPull(digest, inflight)) assert.True(t, m.inflightCredentialsMatch(digest, &authn.AuthConfig{Username: "AWS", Password: "token-a"})) @@ -103,7 +103,7 @@ func TestBorrowedCredentialsExpireWhileQueued(t *testing.T) { borrowedCredentialsTimeout: time.Millisecond, } const digest = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" - inflight := m.registerInflightPull(digest, &authn.AuthConfig{Username: "AWS", Password: "secret"}) + inflight, _ := m.registerInflightPull(digest, &authn.AuthConfig{Username: "AWS", Password: "secret"}) defer m.releaseInflightPull(digest, inflight)() require.Eventually(t, func() bool { @@ -121,8 +121,8 @@ func TestBorrowedCredentialsExpireWhileQueued(t *testing.T) { func TestBorrowedAuthRejectsReplacedInflightPull(t *testing.T) { m := &manager{inflightPulls: make(map[string]*inflightImagePull)} const digest = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" - first := m.registerInflightPull(digest, &authn.AuthConfig{Username: "first"}) - second := m.registerInflightPull(digest, &authn.AuthConfig{Username: "second"}) + first, _ := m.registerInflightPull(digest, &authn.AuthConfig{Username: "first"}) + second, _ := m.registerInflightPull(digest, &authn.AuthConfig{Username: "second"}) defer m.releaseInflightPull(digest, second)() credentials, _, expired, stale := m.borrowedAuth(digest, first) diff --git a/lib/images/manager.go b/lib/images/manager.go index cb93c937c..fcd68d5b7 100644 --- a/lib/images/manager.go +++ b/lib/images/manager.go @@ -316,11 +316,14 @@ func (m *manager) inflightCredentialsMatch(digest string, credentials *authn.Aut return existingFingerprint == credentialFingerprint(credentials) } -func (m *manager) registerInflightPull(digest string, credentials *authn.AuthConfig) *inflightImagePull { +// registerInflightPull publishes a new handle for digest. It returns the +// handle it replaced, if any, so a refused successor can put that one back. +func (m *manager) registerInflightPull(digest string, credentials *authn.AuthConfig) (current, previous *inflightImagePull) { if m.inflightPulls == nil { m.inflightPulls = make(map[string]*inflightImagePull) } - if previous := m.inflightPulls[digest]; previous != nil && previous.timer != nil { + previous = m.inflightPulls[digest] + if previous != nil && previous.timer != nil { previous.timer.Stop() } inflight := &inflightImagePull{ @@ -328,36 +331,71 @@ func (m *manager) registerInflightPull(digest string, credentials *authn.AuthCon credentials: credentials, } m.inflightPulls[digest] = inflight - if credentials == nil { - return inflight + if credentials != nil { + if m.borrowedCredentialsTimeout <= 0 { + m.borrowedCredentialsTimeout = DefaultBorrowedCredentialsTimeout + } + inflight.credentialsExpireAt = time.Now().Add(m.borrowedCredentialsTimeout) + m.armInflightExpiryLocked(digest, inflight) + } + return inflight, previous +} + +// armInflightExpiryLocked starts the credentials expiry timer from the deadline +// the handle already carries, so a restored handle keeps its original bound. +// createMu must be held. +func (m *manager) armInflightExpiryLocked(digest string, inflight *inflightImagePull) { + if inflight.credentials == nil { + return } - if m.borrowedCredentialsTimeout <= 0 { - m.borrowedCredentialsTimeout = DefaultBorrowedCredentialsTimeout + remaining := time.Until(inflight.credentialsExpireAt) + if remaining <= 0 { + // Expired credentials must not survive as current: drop them so the + // pull fails with ErrBorrowedCredentialsExpired instead of running + // unauthenticated. + inflight.credentials = nil + return } - inflight.credentialsExpireAt = time.Now().Add(m.borrowedCredentialsTimeout) - inflight.timer = time.AfterFunc(m.borrowedCredentialsTimeout, func() { + inflight.timer = time.AfterFunc(remaining, func() { m.createMu.Lock() defer m.createMu.Unlock() if m.inflightPulls[digest] == inflight { inflight.credentials = nil } }) - return inflight +} + +// restoreInflightPullLocked makes previous current again after its successor +// was refused, so the pull already running for digest keeps its credentials. +// It re-arms the previous handle's expiry bound. createMu must be held. +func (m *manager) restoreInflightPullLocked(digest string, previous *inflightImagePull) { + if previous == nil { + delete(m.inflightPulls, digest) + return + } + m.inflightPulls[digest] = previous + m.armInflightExpiryLocked(digest, previous) } func (m *manager) releaseInflightPull(digest string, inflight *inflightImagePull) func() { return func() { m.createMu.Lock() defer m.createMu.Unlock() - if m.inflightPulls[digest] != inflight { - return - } - delete(m.inflightPulls, digest) - if inflight.timer != nil { - inflight.timer.Stop() - } - inflight.credentials = nil + m.releaseInflightPullLocked(digest, inflight) + } +} + +// releaseInflightPullLocked drops inflight when it is still the current handle. +// createMu must be held. +func (m *manager) releaseInflightPullLocked(digest string, inflight *inflightImagePull) { + if m.inflightPulls[digest] != inflight { + return } + delete(m.inflightPulls, digest) + if inflight.timer != nil { + inflight.timer.Stop() + } + inflight.credentials = nil } func (m *manager) borrowedAuth(digest string, expected *inflightImagePull) (*authn.AuthConfig, time.Time, bool, bool) { @@ -400,7 +438,7 @@ func (m *manager) createAndQueueImage(ref *ResolvedRef, req CreateImageRequest, // Keep borrowed credentials outside the queued closure so their lifetime is // bounded even when this job waits behind another pull. - inflight := m.registerInflightPull(ref.Digest(), req.Credentials) + inflight, previousInflight := m.registerInflightPull(ref.Digest(), req.Credentials) buildID := meta.BuildID queuePos := m.queue.EnqueueSuccessor(ref.Digest(), func() { credentials, deadline, expired, stale := m.borrowedAuth(ref.Digest(), inflight) @@ -420,10 +458,13 @@ func (m *manager) createAndQueueImage(ref *ResolvedRef, req CreateImageRequest, m.buildImage(ctx, ref, credentials, buildID) }, m.releaseInflightPull(ref.Digest(), inflight)) if queuePos < 0 { - // Rejected before entering the queue: release the borrowed credentials - // and fail the record so it is not left pending forever. - m.releaseInflightPull(ref.Digest(), inflight)() - m.updateStatusByDigest(ref, StatusFailed, queue.ErrFull, buildID) + // Rejected before entering the queue: drop the reserved handle, put back + // the pull that was already running for this digest, then fail the + // record so it is not left pending forever. createMu is held here (both + // callers hold it), so the locked variants are used. + m.releaseInflightPullLocked(ref.Digest(), inflight) + m.restoreInflightPullLocked(ref.Digest(), previousInflight) + m.updateStatusByDigestLocked(ref, StatusFailed, queue.ErrFull, buildID) return nil, queue.ErrFull } @@ -604,7 +645,12 @@ func (m *manager) recordImageBuildPhase(ctx context.Context, digest, phase strin func (m *manager) updateStatusByDigest(ref *ResolvedRef, status string, err error, buildID string) { m.createMu.Lock() defer m.createMu.Unlock() + m.updateStatusByDigestLocked(ref, status, err, buildID) +} +// updateStatusByDigestLocked applies a status change unless the record moved on +// to another build. createMu must be held. +func (m *manager) updateStatusByDigestLocked(ref *ResolvedRef, status string, err error, buildID string) { layout := resolveImageLayout(m.paths, ref.Repository(), ref.DigestHex()) meta, readErr := readMetadataAt(layout) if readErr != nil || meta.BuildID != buildID { diff --git a/lib/images/queue_bound_test.go b/lib/images/queue_bound_test.go new file mode 100644 index 000000000..c23e55142 --- /dev/null +++ b/lib/images/queue_bound_test.go @@ -0,0 +1,132 @@ +package images + +import ( + "context" + "fmt" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/registry" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/kernel/hypeman/lib/paths" + "github.com/kernel/hypeman/lib/queue" + "github.com/stretchr/testify/require" +) + +// fillQueue replaces m's queue with one whose single worker is blocked and +// whose pending slots are all taken, so the next Enqueue reports a full queue. +func fillQueue(t *testing.T, m *manager) { + t.Helper() + + q := queue.New(1) + release := make(chan struct{}) + t.Cleanup(func() { close(release) }) + + require.Equal(t, 0, q.Enqueue("blocker", func() { <-release }, nil)) + for i := 0; i < queue.MaxPending; i++ { + require.Greater(t, q.Enqueue(fmt.Sprintf("filler-%d", i), func() {}, nil), 0) + } + m.queue = q +} + +func TestCreateImageRefusedByFullQueueDoesNotBlock(t *testing.T) { + reg := registry.New() + server := httptest.NewServer(reg) + defer server.Close() + host := strings.TrimPrefix(server.URL, "http://") + + var refs []string + for _, path := range []string{"/library/full-a:latest", "/library/full-b:latest"} { + ref := host + path + parsed, err := name.ParseReference(ref, name.Insecure) + require.NoError(t, err) + img, err := random.Image(256, 1) + require.NoError(t, err) + require.NoError(t, remote.Write(parsed, img)) + refs = append(refs, ref) + } + + mgr, err := NewManager(paths.New(t.TempDir()), 1, nil) + require.NoError(t, err) + m := mgr.(*manager) + fillQueue(t, m) + + ctx := context.Background() + callCreate := func(ref string) error { + t.Helper() + done := make(chan error, 1) + go func() { + _, err := mgr.CreateImage(ctx, CreateImageRequest{Name: ref}) + done <- err + }() + select { + case err := <-done: + return err + case <-time.After(10 * time.Second): + t.Fatalf("CreateImage(%s) blocked while the queue was full", ref) + return nil + } + } + + for _, ref := range refs { + require.ErrorIs(t, callCreate(ref), queue.ErrFull, "create of %s must surface the refusal", ref) + } + + // The manager must still be usable: another create is refused promptly + // instead of deadlocking, and a delete completes. + require.ErrorIs(t, callCreate(refs[0]), queue.ErrFull) + + done := make(chan error, 1) + go func() { done <- mgr.DeleteImage(ctx, refs[0]) }() + select { + case err := <-done: + require.NoError(t, err) + case <-time.After(10 * time.Second): + t.Fatal("DeleteImage blocked after a refused create") + } +} + +func TestRefusedSuccessorKeepsPreviousInflightPull(t *testing.T) { + mgr, err := NewManager(paths.New(t.TempDir()), 1, nil) + require.NoError(t, err) + m := mgr.(*manager) + fillQueue(t, m) + m.borrowedCredentialsTimeout = time.Second + + normalized, err := ParseNormalizedRef("registry.example/library/successor:latest") + require.NoError(t, err) + const digest = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ref := NewResolvedRef(normalized, digest) + credentials := &authn.AuthConfig{Username: "borrower", Password: "secret"} + + m.createMu.Lock() + previous, _ := m.registerInflightPull(ref.Digest(), credentials) + _, err = m.createAndQueueImage(ref, CreateImageRequest{Name: normalized.String(), Credentials: credentials}, hostPlatform()) + m.createMu.Unlock() + require.ErrorIs(t, err, queue.ErrFull) + + // The pull that was already running keeps its handle, so it still builds. + gotCredentials, _, expired, stale := m.borrowedAuth(ref.Digest(), previous) + require.False(t, stale, "the running pull must stay current after a refused successor") + require.False(t, expired) + require.NotNil(t, gotCredentials) + + // Its expiry bound is still enforced on the restored handle. + require.Eventually(t, func() bool { + _, _, expired, stale := m.borrowedAuth(ref.Digest(), previous) + return expired && !stale + }, 5*time.Second, 10*time.Millisecond) + + // The refused successor's record is terminal instead of pending with no + // runner. + meta, err := readMetadata(m.paths, ref.Repository(), ref.DigestHex()) + require.NoError(t, err) + require.Equal(t, StatusFailed, meta.Status) + require.NotNil(t, meta.Error) + require.Contains(t, *meta.Error, queue.ErrFull.Error()) +}