feat(snapshot s3): fingerprint buckets from metadata

cmd/kosli/root.go

@@ -119,6 +119,8 @@ Paths the list already matches stay excluded whatever is later added there, so k
 	// the server is the authority on which types are actually accepted
 	validEnvTypesList = "K8S, ECS, S3, lambda, server, docker, azure-apps, cloud-run, logical"
 
+	validS3FingerprintSources = "content, metadata"
+
 	// single source of truth for the service account privilege list shown in
 	// flag help texts; the server is the authority on which privileges are
 	// actually accepted
@@ -267,8 +269,9 @@ Paths the list already matches stay excluded whatever is later added there, so k
 	awsSecretKeyFlag                = "The AWS secret access key."
 	awsRegionFlag                   = "The AWS region."
 	bucketNameFlag                  = "The name of the S3 bucket."
-	downloadConcurrencyFlag         = "[optional] The number of S3 objects to download at the same time when fingerprinting the bucket. Each object in flight may hold up to 40 MB of download buffers in memory, on top of the disk the --download-budget allows."
-	downloadBudgetFlag              = "[optional] The maximum total size of the S3 objects downloading at the same time, which caps the temporary disk the snapshot uses. A bare number is megabytes; add K, M, G or T (optionally followed by B) to choose the unit, e.g. 512M or 8G. An object larger than the budget still downloads, on its own. Objects are downloaded to the OS temporary directory."
+	downloadConcurrencyFlag         = "[optional] The number of S3 objects to fetch at the same time when fingerprinting the bucket. When downloading, the default, each object in flight may hold up to 40 MB of download buffers in memory, on top of the disk the --download-budget allows. With --fingerprint-source metadata it bounds the checksum reads instead, which hold neither, and defaults to 32."
+	downloadBudgetFlag              = "[optional] The maximum total size of the S3 objects downloading at the same time, which caps the temporary disk the snapshot uses. A bare number is megabytes; add K, M, G or T (optionally followed by B) to choose the unit, e.g. 512M or 8G. An object larger than the budget still downloads, on its own. Objects are downloaded to the OS temporary directory. Has no effect with --fingerprint-source metadata, which reads stored checksums and uses no temporary disk."
+	s3FingerprintSourceFlag         = "[defaulted] Where each object's SHA256 comes from when fingerprinting the bucket. Valid sources are: [" + validS3FingerprintSources + "]. 'content' downloads every contributing object and hashes it. 'metadata' reads the SHA256 checksum S3 stores for each object instead, which skips the download but requires every contributing object to have been uploaded with a full-object SHA256 checksum. Both produce the same fingerprint and need the same permissions."
 	bucketPathsFlag                 = "[optional] The comma separated list of file and/or directory paths in the S3 bucket to include when fingerprinting. Paths match by literal prefix. Cannot be used together with --exclude or --exclude-regex."
 	bucketPathsRegexFlag            = "[optional] The comma separated list of Go regular expressions matched against object keys in the S3 bucket to include when fingerprinting. Cannot be used together with --exclude or --exclude-regex."
 	excludeBucketPathsFlag          = "[optional] The comma separated list of file and/or directory paths in the S3 bucket to exclude when fingerprinting. Paths match by literal prefix. Cannot be used together with --include or --include-regex."

cmd/kosli/snapshotS3.go

@@ -16,9 +16,14 @@ const snapshotS3ShortDesc = `Report a snapshot of the content of an AWS S3 bucke
 const snapshotS3LongDesc = snapshotS3ShortDesc + awsAuthDesc + `
 You can report the entire bucket content, or filter some of the content using ^--include^ / ^--exclude^ (literal prefix match) or ^--include-regex^ / ^--exclude-regex^ (Go regular expressions matched against the full object key).
 In all cases, the content is reported as one artifact. If you wish to report separate files/dirs within the same bucket as separate artifacts, you need to run the command twice.
-Object keys are never used as local file names: each object is downloaded to a temporary file, hashed and removed, and the fingerprint is computed from the keys and the content digests, so any key S3 accepts can be fingerprinted on any operating system.
+Object keys are never used as local file names: by default each object is downloaded to a temporary file, hashed and removed, and the fingerprint is computed from the keys and the content digests, so any key S3 accepts can be fingerprinted on any operating system.
 Keys that cannot form a directory tree are rejected and fail the snapshot, naming every key involved: a key containing a ^..^ segment, two keys that resolve to the same path (such as ^a//b^ and ^a/b^), or an object whose key is also a prefix of other objects (such as ^a^ beside ^a/b^). A legitimate key of that shape can be left out with ^--exclude-regex^ (anchor and escape it, since the pattern is a regular expression matched against the whole key); when ^--include^ or ^--include-regex^ is set, exclude filters are ignored, so narrow the include filter instead.
 
+By default each object's SHA256 comes from downloading the object and hashing it. ^--fingerprint-source metadata^ reads the SHA256 checksum S3 stores for the object instead, which skips the download, the temporary disk and the hashing. Everything else, like the keys, the ^.kosli_ignore^ rules, the way digests combine into the fingerprint, is the same in both modes, so the fingerprint is identical and a snapshot matches the artifact you attested either way. Two conditions apply:
+- Every contributing object must carry a full-object SHA256 checksum. S3 only stores one when the upload asked for it, for example ^aws s3api put-object --checksum-algorithm SHA256^. Objects without one fail the snapshot, all named in one run.
+- A multipart upload gets a composite SHA256, which hashes the checksums of the parts rather than the object content, so it cannot serve as the object's fingerprint. Such an object can be collapsed into a single part in place with ^aws s3api copy-object --checksum-algorithm SHA256 --copy-source yourBucket/yourKey --bucket yourBucket --key yourKey^.
+A root ^.kosli_ignore^ is still downloaded in this mode, because its rules decide which objects contribute; the objects it excludes are never fetched and need no checksum. Reading a checksum does not need fewer permissions than downloading: AWS requires ^s3:GetObject^ for both, and an SSE-KMS encrypted object additionally needs ^kms:GenerateDataKey^ and ^kms:Decrypt^ either way.
+
 ` + kosliIgnoreDescNoExclude
 
 const snapshotS3Example = `
@@ -69,14 +74,28 @@ kosli snapshot s3 yourEnvironmentName \
 	--exclude-regex '.*\.png$' \
 	--api-token yourAPIToken \
 	--org yourOrgName
+
+# report contents of an AWS S3 bucket without downloading the objects,
+# using the SHA256 checksums S3 stores for them:
+kosli snapshot s3 yourEnvironmentName \
+	--bucket yourBucketName \
+	--fingerprint-source metadata \
+	--api-token yourAPIToken \
+	--org yourOrgName
 `
 
+const (
+	fingerprintSourceContent  = "content"
+	fingerprintSourceMetadata = "metadata"
+)
+
 type snapshotS3Options struct {
 	bucket              string
 	includePaths        []string
 	includeRegex        []string
 	excludePaths        []string
 	excludeRegex        []string
+	fingerprintSource   string
 	downloadConcurrency int
 	downloadBudget      string
 	downloadLimits      aws.DownloadLimits
@@ -112,7 +131,14 @@ func newSnapshotS3Cmd(out io.Writer) *cobra.Command {
 				}
 			}
 
-			return o.resolveDownloadLimits()
+			if o.fingerprintSource != fingerprintSourceContent && o.fingerprintSource != fingerprintSourceMetadata {
+				return ErrorBeforePrintingUsage(cmd, fmt.Sprintf(
+					"%s is not a valid fingerprint source. Valid sources are: [%s]",
+					o.fingerprintSource, validS3FingerprintSources))
+			}
+
+			// Changed covers env vars and config too: bindFlags applies them with Flags().Set.
+			return o.resolveDownloadLimits(cmd.Flags().Changed("download-concurrency"))
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
 			return o.run(args)
@@ -124,6 +150,7 @@ func newSnapshotS3Cmd(out io.Writer) *cobra.Command {
 	cmd.Flags().StringSliceVar(&o.includeRegex, "include-regex", []string{}, bucketPathsRegexFlag)
 	cmd.Flags().StringSliceVarP(&o.excludePaths, "exclude", "x", []string{}, excludeBucketPathsFlag)
 	cmd.Flags().StringSliceVar(&o.excludeRegex, "exclude-regex", []string{}, excludeBucketPathsRegexFlag)
+	cmd.Flags().StringVar(&o.fingerprintSource, "fingerprint-source", fingerprintSourceContent, s3FingerprintSourceFlag)
 	cmd.Flags().IntVar(&o.downloadConcurrency, "download-concurrency", aws.DefaultDownloadLimits.Concurrency, downloadConcurrencyFlag)
 	cmd.Flags().StringVar(&o.downloadBudget, "download-budget", defaultDownloadBudget, downloadBudgetFlag)
 	addAWSAuthFlags(cmd, o.awsStaticCreds)
@@ -149,7 +176,12 @@ func (o *snapshotS3Options) run(args []string) error {
 		return err
 	}
 
-	s3Data, err := o.awsStaticCreds.GetS3Data(o.bucket, o.includePaths, o.includeRegex, o.excludePaths, o.excludeRegex, o.downloadLimits, logger)
+	harvest := o.awsStaticCreds.GetS3Data
+	if o.fingerprintSource == fingerprintSourceMetadata {
+		harvest = o.awsStaticCreds.GetS3DataFromMetadata
+	}
+
+	s3Data, err := harvest(o.bucket, o.includePaths, o.includeRegex, o.excludePaths, o.excludeRegex, o.downloadLimits, logger)
 	if err != nil {
 		return err
 	}
@@ -175,14 +207,18 @@ func (o *snapshotS3Options) run(args []string) error {
 // spells it; a test keeps the two equal.
 const defaultDownloadBudget = "512M"
 
-func (o *snapshotS3Options) resolveDownloadLimits() error {
+func (o *snapshotS3Options) resolveDownloadLimits(concurrencySet bool) error {
 	if o.downloadConcurrency < 1 {
 		return fmt.Errorf("--download-concurrency must be at least 1, got %d", o.downloadConcurrency)
 	}
 	budget, err := parseByteSize(o.downloadBudget)
 	if err != nil {
 		return fmt.Errorf("invalid --download-budget: %w", err)
 	}
-	o.downloadLimits = aws.DownloadLimits{Concurrency: o.downloadConcurrency, BytesInFlight: budget}
+	concurrency := o.downloadConcurrency
+	if o.fingerprintSource == fingerprintSourceMetadata && !concurrencySet {
+		concurrency = aws.DefaultMetadataConcurrency
+	}
+	o.downloadLimits = aws.DownloadLimits{Concurrency: concurrency, BytesInFlight: budget}
 	return nil
 }

cmd/kosli/snapshotS3Limits_test.go

@@ -0,0 +1,46 @@
+package main
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/kosli-dev/cli/internal/aws"
+	"github.com/stretchr/testify/require"
+)
+
+func TestResolveDownloadLimitsPicksTheConcurrencyForTheSource(t *testing.T) {
+	for _, tc := range []struct {
+		name           string
+		source         string
+		concurrency    int
+		concurrencySet bool
+		want           int
+	}{
+		{name: "content mode takes the download default", source: fingerprintSourceContent,
+			concurrency: aws.DefaultDownloadLimits.Concurrency, want: aws.DefaultDownloadLimits.Concurrency},
+		{name: "metadata mode takes its own default", source: fingerprintSourceMetadata,
+			concurrency: aws.DefaultDownloadLimits.Concurrency, want: aws.DefaultMetadataConcurrency},
+		{name: "an explicit value wins in metadata mode", source: fingerprintSourceMetadata,
+			concurrency: 4, concurrencySet: true, want: 4},
+		{name: "an explicit value that equals the download default still wins", source: fingerprintSourceMetadata,
+			concurrency: aws.DefaultDownloadLimits.Concurrency, concurrencySet: true, want: aws.DefaultDownloadLimits.Concurrency},
+		{name: "an explicit value wins in content mode", source: fingerprintSourceContent,
+			concurrency: 4, concurrencySet: true, want: 4},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			o := &snapshotS3Options{fingerprintSource: tc.source, downloadConcurrency: tc.concurrency, downloadBudget: defaultDownloadBudget}
+			require.NoError(t, o.resolveDownloadLimits(tc.concurrencySet))
+			require.Equal(t, tc.want, o.downloadLimits.Concurrency)
+		})
+	}
+}
+
+func TestDefaultMetadataConcurrencyIsWiderThanTheDownloadDefault(t *testing.T) {
+	require.Greater(t, aws.DefaultMetadataConcurrency, aws.DefaultDownloadLimits.Concurrency,
+		"a source that holds no buffers should not be throttled below the download default")
+}
+
+// cobra shows only the flag's own default, so the help text spells out the metadata one.
+func TestDownloadConcurrencyHelpStatesTheMetadataDefault(t *testing.T) {
+	require.Contains(t, downloadConcurrencyFlag, fmt.Sprintf("defaults to %d", aws.DefaultMetadataConcurrency))
+}

cmd/kosli/snapshotS3_test.go

@@ -1,9 +1,12 @@
 package main
 
 import (
+	"crypto/sha256"
+	"encoding/base64"
 	"fmt"
 	"testing"
 
+	s3Types "github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/kosli-dev/cli/internal/aws"
 	"github.com/stretchr/testify/suite"
 )
@@ -33,12 +36,18 @@ func (suite *SnapshotS3TestSuite) SetupTest() {
 	// Inject a fake S3 client so tests run without AWS credentials.
 	// The fake is seeded with the objects the test cases filter on.
 	bucketName := suite.bucketName
+	objects := map[string][]byte{
+		"README.md":                  []byte("# kosli cli public\n"),
+		"dummy/dummy_2/template.yml": []byte("key: value\n"),
+	}
+	// Only README.md has a stored checksum, so the metadata cases can cover both outcomes.
+	readmeSum := sha256.Sum256(objects["README.md"])
 	aws.NewS3ClientFunc = func(_ *aws.AWSStaticCreds) (aws.S3API, error) {
 		return &aws.FakeS3Client{
-			Bucket: bucketName,
-			Objects: map[string][]byte{
-				"README.md":                  []byte("# kosli cli public\n"),
-				"dummy/dummy_2/template.yml": []byte("key: value\n"),
+			Bucket:  bucketName,
+			Objects: objects,
+			Checksums: map[string]aws.FakeS3Checksum{
+				"README.md": {SHA256: base64.StdEncoding.EncodeToString(readmeSum[:]), Type: s3Types.ChecksumTypeFullObject},
 			},
 		}, nil
 	}
@@ -141,6 +150,28 @@ func (suite *SnapshotS3TestSuite) TestSnapshotS3Cmd() {
 			cmd:       fmt.Sprintf(`snapshot s3 %s %s --bucket %s --download-budget 0`, suite.envName, suite.defaultKosliArguments, suite.bucketName),
 			golden:    "Error: invalid --download-budget: size \"0\" must be at least 1 byte\n",
 		},
+		{
+			name:   "--fingerprint-source metadata fingerprints from the stored checksum",
+			cmd:    fmt.Sprintf(`snapshot s3 %s %s --bucket %s --include README.md --fingerprint-source metadata`, suite.envName, suite.defaultKosliArguments, suite.bucketName),
+			golden: "bucket kosli-cli-public was reported to environment snapshot-s3-env\n",
+		},
+		{
+			name:   "--fingerprint-source content is the default behaviour",
+			cmd:    fmt.Sprintf(`snapshot s3 %s %s --bucket %s --fingerprint-source content`, suite.envName, suite.defaultKosliArguments, suite.bucketName),
+			golden: "bucket kosli-cli-public was reported to environment snapshot-s3-env\n",
+		},
+		{
+			wantError: true,
+			name:      "--fingerprint-source rejects an unknown value",
+			cmd:       fmt.Sprintf(`snapshot s3 %s %s --bucket %s --fingerprint-source etag`, suite.envName, suite.defaultKosliArguments, suite.bucketName),
+			golden:    "Error: etag is not a valid fingerprint source. Valid sources are: [content, metadata]\nUsage: kosli snapshot s3 ENVIRONMENT-NAME [flags]\n",
+		},
+		{
+			wantError: true,
+			name:      "--fingerprint-source metadata fails on an object with no stored checksum",
+			cmd:       fmt.Sprintf(`snapshot s3 %s %s --bucket %s --include dummy --fingerprint-source metadata`, suite.envName, suite.defaultKosliArguments, suite.bucketName),
+			golden:    "Error: object key [dummy/dummy_2/template.yml] has no SHA256 checksum, so its fingerprint cannot be read from S3 metadata. Upload it with one: aws s3api put-object --bucket kosli-cli-public --key dummy/dummy_2/template.yml --body <file> --checksum-algorithm SHA256; or fingerprint by downloading the objects instead\n",
+		},
 	}
 
 	for _, t := range tests {

cmd/kosli/testdata/empty-flag-audit-coverage.json

@@ -891,6 +891,7 @@
   "dry-run": "bool",
   "exclude": "stringSlice",
   "exclude-regex": "stringSlice",
+  "fingerprint-source": "string",
   "include": "stringSlice",
   "include-regex": "stringSlice"
  },