WIP: Sketch PutBlob... digest choice in c/image/copy + transports

This is:
- UNUSABLE (no external API)
- INCOMPLETE (manifest digests, TryReusingBlob, ...)
- BROKEN (various code assumes that digest returned from upload matches input)
- INCONSISTENT (PutBlobOptions.Digests is only implemented in one transport,
  and silently ignored elsewhere)

Signed-off-by: Miloslav Trmač <mitr@redhat.com>

Author: mtrmac

image/copy/blob.go

@@ -101,6 +101,11 @@ func (ic *imageCopier) copyBlobFromStream(ctx context.Context, srcReader io.Read
 		Cache:      ic.c.blobInfoCache,
 		IsConfig:   isConfig,
 		EmptyLayer: emptyLayer,
+		Digests:    ic.c.options.digestOptions,
+		// CannotChangeDigestReason requires stream.info.Digest to always be set, and it is:
+		// If ic.cannotModifyManifestReason, stream.info was not modified since its initialization at the top of this
+		// function, and the caller is required to provide a digest.
+		CannotChangeDigestReason: ic.cannotModifyManifestReason,
 	}
 	if !isConfig {
 		options.LayerIndex = &layerIndex

image/directory/directory_dest.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/opencontainers/go-digest"
 	"github.com/sirupsen/logrus"
+	"go.podman.io/image/v5/internal/digests"
 	"go.podman.io/image/v5/internal/imagedestination/impl"
 	"go.podman.io/image/v5/internal/imagedestination/stubs"
 	"go.podman.io/image/v5/internal/private"
@@ -150,7 +151,11 @@ func (d *dirImageDestination) PutBlobWithOptions(ctx context.Context, stream io.
 		}
 	}()
 
-	digester, stream := putblobdigest.DigestIfUnknown(stream, inputInfo)
+	algorithm, err := options.Digests.Choose(digests.Situation{Preexisting: inputInfo.Digest, CannotChangeAlgorithmReason: options.CannotChangeDigestReason})
+	if err != nil {
+		return private.UploadedBlob{}, err
+	}
+	digester, stream := putblobdigest.DigestIfAlgorithmUnknown(stream, inputInfo, algorithm)
 
 	// TODO: This can take quite some time, and should ideally be cancellable using ctx.Done().
 	size, err := io.Copy(blobFile, stream)

image/docker/docker_image_dest.go

@@ -23,6 +23,7 @@ import (
 	"github.com/sirupsen/logrus"
 	"go.podman.io/image/v5/docker/reference"
 	"go.podman.io/image/v5/internal/blobinfocache"
+	"go.podman.io/image/v5/internal/digests"
 	"go.podman.io/image/v5/internal/imagedestination/impl"
 	"go.podman.io/image/v5/internal/imagedestination/stubs"
 	"go.podman.io/image/v5/internal/iolimits"
@@ -705,6 +706,11 @@ func (d *dockerImageDestination) putSignaturesToSigstoreAttachments(ctx context.
 		return errors.New("writing sigstore attachments is disabled by configuration")
 	}
 
+	digestOptions, err := digests.CanonicalDefault().WithDefault(digest.Canonical) // FIXME: This is bad and redundant, but we ultimately want the choice to be provided by the caller; and this way it shows up on audit searches for digest.Canonical.
+	if err != nil {
+		return err
+	}
+
 	ociManifest, err := d.c.getSigstoreAttachmentManifest(ctx, d.ref, manifestDigest)
 	if err != nil {
 		return err
@@ -760,6 +766,7 @@ func (d *dockerImageDestination) putSignaturesToSigstoreAttachments(ctx context.
 			IsConfig:   false,
 			EmptyLayer: false,
 			LayerIndex: nil,
+			Digests:    digestOptions,
 		})
 		if err != nil {
 			return err
@@ -781,6 +788,7 @@ func (d *dockerImageDestination) putSignaturesToSigstoreAttachments(ctx context.
 		IsConfig:   true,
 		EmptyLayer: false,
 		LayerIndex: nil,
+		Digests:    digestOptions,
 	})
 	if err != nil {
 		return err

image/internal/imagedestination/impl/compat.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/opencontainers/go-digest"
 	"go.podman.io/image/v5/internal/blobinfocache"
+	"go.podman.io/image/v5/internal/digests"
 	"go.podman.io/image/v5/internal/private"
 	"go.podman.io/image/v5/internal/signature"
 	"go.podman.io/image/v5/types"
@@ -46,6 +47,8 @@ func (c *Compat) PutBlob(ctx context.Context, stream io.Reader, inputInfo types.
 	res, err := c.dest.PutBlobWithOptions(ctx, stream, inputInfo, private.PutBlobOptions{
 		Cache:    blobinfocache.FromBlobInfoCache(cache),
 		IsConfig: isConfig,
+
+		Digests: digests.CanonicalDefault(),
 	})
 	if err != nil {
 		return types.BlobInfo{}, err

image/internal/imagedestination/wrapper.go

@@ -5,6 +5,7 @@ import (
 	"io"
 
 	"github.com/opencontainers/go-digest"
+	"go.podman.io/image/v5/internal/digests"
 	"go.podman.io/image/v5/internal/imagedestination/stubs"
 	"go.podman.io/image/v5/internal/private"
 	"go.podman.io/image/v5/internal/signature"
@@ -53,6 +54,15 @@ func (w *wrapped) PutBlobWithOptions(ctx context.Context, stream io.Reader, inpu
 	if err != nil {
 		return private.UploadedBlob{}, err
 	}
+	// Check that the returned digest is compatible with options.Digests. If it isn’t, there’s nothing we can do, but at least the callers of PutBlobWithOptions
+	// won’t need to double-check.
+	if _, err := options.Digests.Choose(digests.Situation{
+		Preexisting:                 res.Digest,
+		CannotChangeAlgorithmReason: "external transport API does not allow choosing a digest algorithm",
+	}); err != nil {
+		return private.UploadedBlob{}, err
+	}
+
 	return private.UploadedBlob{
 		Digest: res.Digest,
 		Size:   res.Size,

image/internal/private/private.go

@@ -9,6 +9,7 @@ import (
 	imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
 	"go.podman.io/image/v5/docker/reference"
 	"go.podman.io/image/v5/internal/blobinfocache"
+	"go.podman.io/image/v5/internal/digests"
 	"go.podman.io/image/v5/internal/signature"
 	compression "go.podman.io/image/v5/pkg/compression/types"
 	"go.podman.io/image/v5/types"
@@ -111,8 +112,10 @@ type PutBlobOptions struct {
 	// if they use internal/imagedestination/impl.Compat;
 	// in that case, they will all be consistently zero-valued.
 
-	EmptyLayer bool // True if the blob is an "empty"/"throwaway" layer, and may not necessarily be physically represented.
-	LayerIndex *int // If the blob is a layer, a zero-based index of the layer within the image; nil otherwise.
+	EmptyLayer               bool            // True if the blob is an "empty"/"throwaway" layer, and may not necessarily be physically represented.
+	LayerIndex               *int            // If the blob is a layer, a zero-based index of the layer within the image; nil otherwise.
+	Digests                  digests.Options // Unlike other private fields, this is always initialized, and mandatory requests are enforced by the compatibility wrapper.
+	CannotChangeDigestReason string          // The reason why PutBlobWithOptions is provided with a digest and the destination must use precisely that one (in particular, use that algorithm).
 }
 
 // PutBlobPartialOptions are used in PutBlobPartial.

image/pkg/blobcache/dest.go

@@ -12,6 +12,7 @@ import (
 	digest "github.com/opencontainers/go-digest"
 	imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/sirupsen/logrus"
+	"go.podman.io/image/v5/internal/digests"
 	"go.podman.io/image/v5/internal/imagedestination"
 	"go.podman.io/image/v5/internal/imagedestination/impl"
 	"go.podman.io/image/v5/internal/private"
@@ -268,6 +269,10 @@ func (d *blobCacheDestination) TryReusingBlobWithOptions(ctx context.Context, in
 		return present, reusedInfo, err
 	}
 
+	digestOptions, err := digests.CanonicalDefault().WithDefault(digest.Canonical) // FIXME: This is bad and redundant, but we ultimately want the choice to be provided by the caller; and this way it shows up on audit searches for digest.Canonical.
+	if err != nil {
+		return false, private.ReusedBlob{}, err
+	}
 	blobPath, _, isConfig, err := d.reference.findBlob(info)
 	if err != nil {
 		return false, private.ReusedBlob{}, err
@@ -281,6 +286,7 @@ func (d *blobCacheDestination) TryReusingBlobWithOptions(ctx context.Context, in
 				IsConfig:   isConfig,
 				EmptyLayer: options.EmptyLayer,
 				LayerIndex: options.LayerIndex,
+				Digests:    digestOptions,
 			})
 			if err != nil {
 				return false, private.ReusedBlob{}, err