Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 21 additions & 2 deletions lib/imagepush/manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,15 @@ func (m *manager) CreatePush(ctx context.Context, req PushRequest) (*Push, error
queuePos := m.queue.Enqueue(key, func() {
m.executePush(context.Background(), &metaCopy, provider)
}, m.releaseInflight(key))
if queuePos < 0 {
// The job never entered the queue, so no completion hook will fire:
// drop the reservation and the record so a retry starts clean.
m.mu.Lock()
delete(m.inflight, key)
m.mu.Unlock()
removePushData(m.paths, meta.ID)
return nil, fmt.Errorf("push queue is full: %w", queue.ErrFull)
}

push := meta.toPush()
if queuePos > 0 {
Expand Down Expand Up @@ -443,9 +452,19 @@ func (m *manager) recoverInterruptedPushes() {
m.mu.Unlock()

metaCopy := *meta
m.queue.Enqueue(key, func() {
if m.queue.Enqueue(key, func() {
m.executePush(context.Background(), &metaCopy, m.provider)
}, m.releaseInflight(key))
}, m.releaseInflight(key)) < 0 {
// Keep recovery bounded: a record that cannot enter the queue is
// failed with a reason instead of sitting queued forever. The
// reservation above must go too: no job will ever run to release it,
// and a stale entry would keep handing this failed record to later
// CreatePush calls for the same digest and target.
m.mu.Lock()
delete(m.inflight, key)
m.mu.Unlock()
m.failRecovered(meta, "push queue is full during recovery; resubmit the push")
}
Comment thread
cursor[bot] marked this conversation as resolved.
Resolved
}
}

Expand Down
95 changes: 95 additions & 0 deletions lib/imagepush/recovery_queue_bound_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
package imagepush

import (
"context"
"fmt"
"sync"
"testing"
"time"

"github.com/kernel/hypeman/lib/images"
"github.com/kernel/hypeman/lib/queue"
"github.com/kernel/hypeman/lib/registrypush"
"github.com/stretchr/testify/require"
)

// fullQueue returns a queue whose single worker is blocked and whose pending
// slots are all taken, so the next Enqueue is refused. drain releases the
// blocker so the queue empties.
func fullQueue(t *testing.T) (q *queue.Queue, drain func()) {
t.Helper()

q = queue.New(1)
release := make(chan struct{})
var once sync.Once
drain = func() {
once.Do(func() { close(release) })
require.Eventually(t, func() bool { return q.QueueLength() == 0 }, 10*time.Second, 10*time.Millisecond)
}
t.Cleanup(drain)

require.Equal(t, 0, q.Enqueue("blocker", func() { <-release }, nil))
for i := 0; i < queue.MaxPending; i++ {
require.Greater(t, q.Enqueue(fmt.Sprintf("filler-%d", i), func() {}, nil), 0)
}
return q, drain
}

func TestRecoveryRefusedByFullQueueLeavesNoInflightEntry(t *testing.T) {
p, digest := cacheFixture(t)
host := openRegistry(t)
target := host + "/export/recovered:v1"

meta := &pushMetadata{
ID: "recovered-push",
Status: StatusPushing,
Image: "myapp:v1",
Digest: digest,
Target: target,
Insecure: true,
CreatedAt: time.Now(),
}
writePushes(t, p, meta)

q, drain := fullQueue(t)
mgr := &manager{
paths: p,
resolver: &fakeResolver{images: map[string]*images.Image{
"myapp:v1": readyImage("myapp:v1", digest),
}},
provider: &registrypush.KeychainProvider{},
queue: q,
pushTimeout: DefaultPushTimeout,
inflight: make(map[string]inflightPush),
subscribers: make(map[string][]chan StatusEvent),
}
mgr.recoverInterruptedPushes()

// The refused record is terminal so it cannot sit queued forever.
failed, err := readMetadata(p, "recovered-push")
require.NoError(t, err)
require.Equal(t, StatusFailed, failed.Status)
require.NotNil(t, failed.Error)
require.Contains(t, *failed.Error, "queue is full")

// No reservation and no queued job survived the refusal.
key := pushKey(digest, target, true)
mgr.mu.Lock()
_, inflight := mgr.inflight[key]
mgr.mu.Unlock()
require.False(t, inflight, "the refused recovery must drop its inflight entry")
require.Equal(t, 1+queue.MaxPending, q.QueueLength(), "the refused job must not be queued")

// Once the queue drains, a CreatePush for the same digest and target runs
// as a fresh job instead of handing back the failed record.
drain()
push, err := mgr.CreatePush(context.Background(), PushRequest{
Image: "myapp:v1", Target: target, Insecure: true,
})
require.NoError(t, err)
require.NotEmpty(t, push.ID)
require.NotEqual(t, "recovered-push", push.ID, "a retry must not dedup onto the failed record")

mustPushed(t, mgr, push.ID)
waitNoInflight(t, mgr)
}
8 changes: 4 additions & 4 deletions lib/images/credentials_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ func TestInflightPullRejectsDifferentCredentials(t *testing.T) {
}
const digest = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
credentials := &authn.AuthConfig{Username: "AWS", Password: "token-a"}
inflight := m.registerInflightPull(digest, credentials)
inflight, _ := m.registerInflightPull(digest, credentials)
t.Cleanup(m.releaseInflightPull(digest, inflight))

assert.True(t, m.inflightCredentialsMatch(digest, &authn.AuthConfig{Username: "AWS", Password: "token-a"}))
Expand All @@ -103,7 +103,7 @@ func TestBorrowedCredentialsExpireWhileQueued(t *testing.T) {
borrowedCredentialsTimeout: time.Millisecond,
}
const digest = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
inflight := m.registerInflightPull(digest, &authn.AuthConfig{Username: "AWS", Password: "secret"})
inflight, _ := m.registerInflightPull(digest, &authn.AuthConfig{Username: "AWS", Password: "secret"})
defer m.releaseInflightPull(digest, inflight)()

require.Eventually(t, func() bool {
Expand All @@ -121,8 +121,8 @@ func TestBorrowedCredentialsExpireWhileQueued(t *testing.T) {
func TestBorrowedAuthRejectsReplacedInflightPull(t *testing.T) {
m := &manager{inflightPulls: make(map[string]*inflightImagePull)}
const digest = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
first := m.registerInflightPull(digest, &authn.AuthConfig{Username: "first"})
second := m.registerInflightPull(digest, &authn.AuthConfig{Username: "second"})
first, _ := m.registerInflightPull(digest, &authn.AuthConfig{Username: "first"})
second, _ := m.registerInflightPull(digest, &authn.AuthConfig{Username: "second"})
defer m.releaseInflightPull(digest, second)()

credentials, _, expired, stale := m.borrowedAuth(digest, first)
Expand Down
97 changes: 77 additions & 20 deletions lib/images/manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -316,48 +316,86 @@ func (m *manager) inflightCredentialsMatch(digest string, credentials *authn.Aut
return existingFingerprint == credentialFingerprint(credentials)
}

func (m *manager) registerInflightPull(digest string, credentials *authn.AuthConfig) *inflightImagePull {
// registerInflightPull publishes a new handle for digest. It returns the
// handle it replaced, if any, so a refused successor can put that one back.
func (m *manager) registerInflightPull(digest string, credentials *authn.AuthConfig) (current, previous *inflightImagePull) {
if m.inflightPulls == nil {
m.inflightPulls = make(map[string]*inflightImagePull)
}
if previous := m.inflightPulls[digest]; previous != nil && previous.timer != nil {
previous = m.inflightPulls[digest]
if previous != nil && previous.timer != nil {
previous.timer.Stop()
}
inflight := &inflightImagePull{
fingerprint: credentialFingerprint(credentials),
credentials: credentials,
}
m.inflightPulls[digest] = inflight
if credentials == nil {
return inflight
if credentials != nil {
if m.borrowedCredentialsTimeout <= 0 {
m.borrowedCredentialsTimeout = DefaultBorrowedCredentialsTimeout
}
inflight.credentialsExpireAt = time.Now().Add(m.borrowedCredentialsTimeout)
m.armInflightExpiryLocked(digest, inflight)
}
if m.borrowedCredentialsTimeout <= 0 {
m.borrowedCredentialsTimeout = DefaultBorrowedCredentialsTimeout
return inflight, previous
}

// armInflightExpiryLocked starts the credentials expiry timer from the deadline
// the handle already carries, so a restored handle keeps its original bound.
// createMu must be held.
func (m *manager) armInflightExpiryLocked(digest string, inflight *inflightImagePull) {
if inflight.credentials == nil {
return
}
remaining := time.Until(inflight.credentialsExpireAt)
if remaining <= 0 {
// Expired credentials must not survive as current: drop them so the
// pull fails with ErrBorrowedCredentialsExpired instead of running
// unauthenticated.
inflight.credentials = nil
return
}
inflight.credentialsExpireAt = time.Now().Add(m.borrowedCredentialsTimeout)
inflight.timer = time.AfterFunc(m.borrowedCredentialsTimeout, func() {
inflight.timer = time.AfterFunc(remaining, func() {
m.createMu.Lock()
defer m.createMu.Unlock()
if m.inflightPulls[digest] == inflight {
inflight.credentials = nil
}
})
return inflight
}

// restoreInflightPullLocked makes previous current again after its successor
// was refused, so the pull already running for digest keeps its credentials.
// It re-arms the previous handle's expiry bound. createMu must be held.
func (m *manager) restoreInflightPullLocked(digest string, previous *inflightImagePull) {
if previous == nil {
delete(m.inflightPulls, digest)
return
}
m.inflightPulls[digest] = previous
m.armInflightExpiryLocked(digest, previous)
}

func (m *manager) releaseInflightPull(digest string, inflight *inflightImagePull) func() {
return func() {
m.createMu.Lock()
defer m.createMu.Unlock()
if m.inflightPulls[digest] != inflight {
return
}
delete(m.inflightPulls, digest)
if inflight.timer != nil {
inflight.timer.Stop()
}
inflight.credentials = nil
m.releaseInflightPullLocked(digest, inflight)
}
}

// releaseInflightPullLocked drops inflight when it is still the current handle.
// createMu must be held.
func (m *manager) releaseInflightPullLocked(digest string, inflight *inflightImagePull) {
if m.inflightPulls[digest] != inflight {
return
}
delete(m.inflightPulls, digest)
if inflight.timer != nil {
inflight.timer.Stop()
}
inflight.credentials = nil
}

func (m *manager) borrowedAuth(digest string, expected *inflightImagePull) (*authn.AuthConfig, time.Time, bool, bool) {
Expand Down Expand Up @@ -400,7 +438,7 @@ func (m *manager) createAndQueueImage(ref *ResolvedRef, req CreateImageRequest,

// Keep borrowed credentials outside the queued closure so their lifetime is
// bounded even when this job waits behind another pull.
inflight := m.registerInflightPull(ref.Digest(), req.Credentials)
inflight, previousInflight := m.registerInflightPull(ref.Digest(), req.Credentials)
buildID := meta.BuildID
queuePos := m.queue.EnqueueSuccessor(ref.Digest(), func() {
credentials, deadline, expired, stale := m.borrowedAuth(ref.Digest(), inflight)
Expand All @@ -419,6 +457,16 @@ func (m *manager) createAndQueueImage(ref *ResolvedRef, req CreateImageRequest,
}
m.buildImage(ctx, ref, credentials, buildID)
}, m.releaseInflightPull(ref.Digest(), inflight))
if queuePos < 0 {
// Rejected before entering the queue: drop the reserved handle, put back
// the pull that was already running for this digest, then fail the
// record so it is not left pending forever. createMu is held here (both
// callers hold it), so the locked variants are used.
m.releaseInflightPullLocked(ref.Digest(), inflight)
m.restoreInflightPullLocked(ref.Digest(), previousInflight)
m.updateStatusByDigestLocked(ref, StatusFailed, queue.ErrFull, buildID)
return nil, queue.ErrFull
}
Comment thread
cursor[bot] marked this conversation as resolved.
Resolved
Comment thread
cursor[bot] marked this conversation as resolved.
Resolved

img := meta.toImageFor(ref.String())
if queuePos > 0 {
Expand Down Expand Up @@ -597,7 +645,12 @@ func (m *manager) recordImageBuildPhase(ctx context.Context, digest, phase strin
func (m *manager) updateStatusByDigest(ref *ResolvedRef, status string, err error, buildID string) {
m.createMu.Lock()
defer m.createMu.Unlock()
m.updateStatusByDigestLocked(ref, status, err, buildID)
}

// updateStatusByDigestLocked applies a status change unless the record moved on
// to another build. createMu must be held.
func (m *manager) updateStatusByDigestLocked(ref *ResolvedRef, status string, err error, buildID string) {
layout := resolveImageLayout(m.paths, ref.Repository(), ref.DigestHex())
meta, readErr := readMetadataAt(layout)
if readErr != nil || meta.BuildID != buildID {
Expand Down Expand Up @@ -669,9 +722,13 @@ func (m *manager) recoverInterruptedBuild(meta *imageMetadata) bool {
return true
}
buildID := meta.BuildID
m.queue.Enqueue(meta.Digest, func() {
if m.queue.Enqueue(meta.Digest, func() {
m.buildImage(context.Background(), ref, nil, buildID)
}, nil)
}, nil) < 0 {
// A rejected recovery would stay pending with no runner; fail it so
// the record reaches a terminal state a caller can see.
m.updateStatusByDigest(ref, StatusFailed, queue.ErrFull, buildID)
}
return true
}

Expand Down
Loading