diff --git a/AUTHORS b/AUTHORS index ea69b2987..b94947cf1 100644 --- a/AUTHORS +++ b/AUTHORS @@ -84,3 +84,4 @@ List of contributors, in chronological order: * Zhang Xiao (https://github.com/xzhang1) * Tom Nguyen (https://github.com/lecafard) * Philip Cramer (https://github.com/PhilipCramer) +* mmxxixx (https://github.com/mmxxixx) diff --git a/context/context.go b/context/context.go index a1a522380..1e582603b 100644 --- a/context/context.go +++ b/context/context.go @@ -400,6 +400,24 @@ func (context *AptlyContext) PackagePool() aptly.PackagePool { if err != nil { Fatal(err) } + } else if storageConfig.S3 != nil { + var err error + context.packagePool, err = s3.NewPackagePool( + storageConfig.S3.AccessKeyID, + storageConfig.S3.SecretAccessKey, + storageConfig.S3.SessionToken, + storageConfig.S3.Bucket, + storageConfig.S3.Prefix, + storageConfig.S3.ACL, + storageConfig.S3.StorageClass, + storageConfig.S3.EncryptionMethod, + storageConfig.S3.Region, + storageConfig.S3.Endpoint, + storageConfig.S3.ForceVirtualHostedStyle, + storageConfig.S3.Debug) + if err != nil { + Fatal(err) + } } else { poolRoot := context.config().PackagePoolStorage.Local.Path if poolRoot == "" { diff --git a/debian/aptly.conf b/debian/aptly.conf index 043b46ebe..9366dd5d6 100644 --- a/debian/aptly.conf +++ b/debian/aptly.conf @@ -402,6 +402,7 @@ azure_publish_endpoints: # Type must be one of: # * local # * azure +# * s3 packagepool_storage: # Local Pool type: local @@ -425,3 +426,21 @@ packagepool_storage: # # defaults to "https://.blob.core.windows.net" # endpoint: "" + # # S3 Object Storage Pool + # type: s3 + # # S3 bucket to store the pool in + # bucket: pool1 + # # Prefix (optional) + # # Store the pool under specified prefix in the bucket, defaults to no prefix (bucket root) + # prefix: "" + # # Region (i.e. us-east-1) + # region: "" + # # Credentials (optional) + # # empty credentials use the standard AWS credential chain (env vars, shared credentials file, IAM role) + # access_key_id: "" + # secret_access_key: "" + # session_token: "" + # # Endpoint URL (optional) + # # for S3-compatible object stores (MinIO, etc.), defaults to AWS S3 + # endpoint: "" + diff --git a/man/aptly.1 b/man/aptly.1 index 67351d7a7..aad5324a6 100644 --- a/man/aptly.1 +++ b/man/aptly.1 @@ -439,6 +439,7 @@ The legacy json configuration is still supported (and also supports comments): // Type must be one of: // * local // * azure + // * s3 "packagePoolStorage": { // Local Pool "type": "local", @@ -463,6 +464,28 @@ The legacy json configuration is still supported (and also supports comments): // // See: Azure documentation https://docs\.microsoft\.com/en\-us/azure/storage/common/storage\-configure\-connection\-string // // defaults to "https://\.blob\.core\.windows\.net" // "endpoint": "" + + // // S3 Object Storage Pool + // "type": "s3", + // // S3 bucket to store the pool in + // "bucket": "pool1", + + // // Prefix (optional) + // // Store the pool under specified prefix in the bucket, defaults to no prefix (bucket root) + // "prefix": "", + + // // Region (i\.e\. us\-east\-1) + // "region": "", + + // // Credentials (optional) + // // empty credentials use the standard AWS credential chain (env vars, shared credentials file, IAM role) + // "awsAccessKeyID": "", + // "awsSecretAccessKey": "", + // "awsSessionToken": "", + + // // Endpoint URL (optional) + // // for S3\-compatible object stores (MinIO, etc\.), defaults to AWS S3 + // "endpoint": "" } // End of config diff --git a/man/aptly.1.ronn.tmpl b/man/aptly.1.ronn.tmpl index 2119e40a5..d7cf71416 100644 --- a/man/aptly.1.ronn.tmpl +++ b/man/aptly.1.ronn.tmpl @@ -451,6 +451,7 @@ The legacy json configuration is still supported (and also supports comments): // Type must be one of: // * local // * azure + // * s3 "packagePoolStorage": { // Local Pool "type": "local", @@ -475,6 +476,28 @@ The legacy json configuration is still supported (and also supports comments): // // See: Azure documentation https://docs.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string // // defaults to "https://.blob.core.windows.net" // "endpoint": "" + + // // S3 Object Storage Pool + // "type": "s3", + // // S3 bucket to store the pool in + // "bucket": "pool1", + + // // Prefix (optional) + // // Store the pool under specified prefix in the bucket, defaults to no prefix (bucket root) + // "prefix": "", + + // // Region (i.e. us-east-1) + // "region": "", + + // // Credentials (optional) + // // empty credentials use the standard AWS credential chain (env vars, shared credentials file, IAM role) + // "awsAccessKeyID": "", + // "awsSecretAccessKey": "", + // "awsSessionToken": "", + + // // Endpoint URL (optional) + // // for S3-compatible object stores (MinIO, etc.), defaults to AWS S3 + // "endpoint": "" } // End of config diff --git a/s3/package_pool.go b/s3/package_pool.go new file mode 100644 index 000000000..9160d719a --- /dev/null +++ b/s3/package_pool.go @@ -0,0 +1,389 @@ +package s3 + +import ( + "context" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + + "github.com/aptly-dev/aptly/aptly" + "github.com/aptly-dev/aptly/utils" + "github.com/aws/aws-sdk-go-v2/aws" + signer "github.com/aws/aws-sdk-go-v2/aws/signer/v4" + awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" + + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + smithy "github.com/aws/smithy-go" + "github.com/pkg/errors" +) + +type PackagePool struct { + s3 *s3.Client + bucket string + prefix string + acl types.ObjectCannedACL + storageClass types.StorageClass + encryptionMethod types.ServerSideEncryption +} + +// Check interface +var ( + _ aptly.PackagePool = (*PackagePool)(nil) +) + +func NewPackagePool(accessKey, secretKey, sessionToken, bucket, prefix, defaultACL, + storageClass, encryptionMethod, region, endpoint string, forceVirtualHostedStyle, debug bool) (*PackagePool, error) { + + opts := []func(*config.LoadOptions) error{config.WithRegion(region)} + if accessKey != "" { + opts = append(opts, config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accessKey, secretKey, sessionToken))) + } + + if debug { + opts = append(opts, config.WithLogger(&logger{})) + } + + cfg, err := config.LoadDefaultConfig(context.Background(), opts...) + if err != nil { + return nil, errors.Wrap(err, "failed to load AWS configuration") + } + + var acl types.ObjectCannedACL + if defaultACL == "" || defaultACL == "private" { + acl = types.ObjectCannedACLPrivate + } else if defaultACL == "public-read" { + acl = types.ObjectCannedACLPublicRead + } else if defaultACL == "none" { + acl = "" + } + + if storageClass == string(types.StorageClassStandard) { + storageClass = "" + } + + var baseEndpoint *string + if endpoint != "" { + baseEndpoint = aws.String(endpoint) + } + + client := s3.NewFromConfig(cfg, func(o *s3.Options) { + o.UsePathStyle = !forceVirtualHostedStyle + o.HTTPSignerV4 = signer.NewSigner() + o.BaseEndpoint = baseEndpoint + }) + + return &PackagePool{ + s3: client, + bucket: bucket, + prefix: prefix, + acl: acl, + storageClass: types.StorageClass(storageClass), + encryptionMethod: types.ServerSideEncryption(encryptionMethod), + }, nil +} + +// String returns the storage as string +func (pool *PackagePool) String() string { + return fmt.Sprintf("S3: %s/%s", pool.bucket, pool.prefix) +} + +func (pool *PackagePool) buildPoolPath(filename string, checksums *utils.ChecksumInfo) string { + hash := checksums.SHA256 + // Use the same path as the file pool, for compat reasons. + return filepath.Join(hash[0:2], hash[2:4], hash[4:32]+"_"+filename) +} + +func (pool *PackagePool) ensureChecksums(poolPath string, checksumStorage aptly.ChecksumStorage) (*utils.ChecksumInfo, error) { + targetChecksums, err := checksumStorage.Get(poolPath) + if err != nil { + return nil, err + } + + if targetChecksums == nil { + key := pool.poolKey(poolPath) + + // we don't have checksums stored yet for this file + output, err := pool.s3.GetObject(context.Background(), &s3.GetObjectInput{ + Key: &key, + Bucket: &pool.bucket, + }) + if err != nil { + if isNotFound(err) { + return nil, nil + } + return nil, errors.Wrapf(err, "error downloading object at %s from %s", poolPath, pool) + } + defer func() { _ = output.Body.Close() }() + + targetChecksums = &utils.ChecksumInfo{} + *targetChecksums, err = utils.ChecksumsForReader(output.Body) + if err != nil { + return nil, errors.Wrapf(err, "error checksumming blob at %s from %s", poolPath, pool) + } + + err = checksumStorage.Update(poolPath, targetChecksums) + if err != nil { + return nil, errors.Wrap(err, "error updating checksumStorage") + } + } + + return targetChecksums, nil +} + +// FilepathList returns file paths of all the files in the pool +func (pool *PackagePool) FilepathList(progress aptly.Progress) ([]string, error) { + if progress != nil { + progress.InitBar(0, false, aptly.BarGeneralBuildFileList) + defer progress.ShutdownBar() + } + + paths := make([]string, 0, 1024) + prefix := pool.prefix + if prefix != "" { + prefix += "/" + } + + maxKeys := int32(1000) + params := &s3.ListObjectsV2Input{ + Bucket: &pool.bucket, + Prefix: &prefix, + MaxKeys: &maxKeys, + } + + p := s3.NewListObjectsV2Paginator(pool.s3, params) + for i := 1; p.HasMorePages(); i++ { + page, err := p.NextPage(context.TODO()) + if err != nil { + return nil, errors.Wrapf(err, "error listing pool %s (page %d)", pool, i) + } + + for _, key := range page.Contents { + if prefix == "" { + paths = append(paths, *key.Key) + } else { + paths = append(paths, (*key.Key)[len(prefix):]) + } + } + } + + return paths, nil +} + +func (pool *PackagePool) LegacyPath(_ string, _ *utils.ChecksumInfo) (string, error) { + return "", errors.New("S3 package pool does not support legacy paths") +} + +func (pool *PackagePool) Size(path string) (int64, error) { + headObject, found, err := pool.headObject(path) + if err != nil { + return 0, errors.Wrapf(err, "error getting headObject for %s", path) + } + if !found { + return 0, fmt.Errorf("%s not found", path) + } + + return *headObject.ContentLength, nil +} + +func (pool *PackagePool) Open(path string) (aptly.ReadSeekerCloser, error) { + key := pool.poolKey(path) + output, err := pool.s3.GetObject(context.TODO(), &s3.GetObjectInput{ + Bucket: &pool.bucket, + Key: &key, + }) + if err != nil { + return nil, errors.Wrapf(err, "error downloading object %s from %s", path, pool) + } + defer func() { _ = output.Body.Close() }() + + temp, err := os.CreateTemp("", "s3-pool-download") + if err != nil { + return nil, errors.Wrapf(err, "error creating tempfile for %s", path) + } + defer func() { _ = os.Remove(temp.Name()) }() + + _, err = io.Copy(temp, output.Body) + if err != nil { + _ = temp.Close() + return nil, errors.Wrapf(err, "error downloading object %s from %s", path, pool) + } + + _, err = temp.Seek(0, io.SeekStart) + if err != nil { + _ = temp.Close() + return nil, errors.Wrapf(err, "error seeking tempfile for %s", path) + } + + return temp, nil +} + +func (pool *PackagePool) Remove(path string) (int64, error) { + headObject, found, err := pool.headObject(path) + if err != nil { + return 0, errors.Wrapf(err, "error getting headObject for %s", path) + } + if !found { + return 0, fmt.Errorf("%s not found in %s", path, pool) + } + key := pool.poolKey(path) + _, err = pool.s3.DeleteObject(context.Background(), &s3.DeleteObjectInput{ + Bucket: &pool.bucket, + Key: &key, + }) + if err != nil { + return 0, errors.Wrapf(err, "error deleting %s from %s", path, pool) + } + + return *headObject.ContentLength, nil +} + +func (pool *PackagePool) Import(srcPath, basename string, checksums *utils.ChecksumInfo, _ bool, checksumStorage aptly.ChecksumStorage) (string, error) { + if checksums.MD5 == "" || checksums.SHA256 == "" || checksums.SHA512 == "" { + // need to update checksums, MD5 and SHA256 should be always defined + var err error + *checksums, err = utils.ChecksumsForFile(srcPath) + if err != nil { + return "", err + } + } + + path := pool.buildPoolPath(basename, checksums) + targetChecksums, err := pool.ensureChecksums(path, checksumStorage) + if err != nil { + return "", err + } else if targetChecksums != nil { + // target already exists + *checksums = *targetChecksums + return path, nil + } + + source, err := os.Open(srcPath) + if err != nil { + return "", err + } + defer func() { _ = source.Close() }() + + err = pool.putFile(path, source) + if err != nil { + return "", errors.Wrapf(err, "error uploading %s to %s", srcPath, pool) + } + + if !checksums.Complete() { + // need full checksums here + *checksums, err = utils.ChecksumsForFile(srcPath) + if err != nil { + return "", err + } + } + + err = checksumStorage.Update(path, checksums) + if err != nil { + return "", err + } + + return path, nil +} + +func (pool *PackagePool) Verify(poolPath, basename string, checksums *utils.ChecksumInfo, checksumStorage aptly.ChecksumStorage) (string, bool, error) { + if poolPath == "" { + if checksums.SHA256 != "" { + poolPath = pool.buildPoolPath(basename, checksums) + } else { + // No checksums or pool path, so no idea what file to look for. + return "", false, nil + } + } + + output, found, err := pool.headObject(poolPath) + if err != nil { + return "", false, errors.Wrapf(err, "error examining %s from %s", poolPath, pool) + } + if !found || *output.ContentLength != checksums.Size { + return "", false, nil + } + + targetChecksums, err := pool.ensureChecksums(poolPath, checksumStorage) + if err != nil { + return "", false, err + } else if targetChecksums == nil { + return "", false, nil + } + + if checksums.MD5 != "" && targetChecksums.MD5 != checksums.MD5 || + checksums.SHA256 != "" && targetChecksums.SHA256 != checksums.SHA256 { + // wrong file? + return "", false, nil + } + + // fill back checksums + *checksums = *targetChecksums + return poolPath, true, nil +} + +func (pool *PackagePool) putFile(path string, source io.ReadSeeker) error { + key := pool.poolKey(path) + params := &s3.PutObjectInput{ + Bucket: &pool.bucket, + Key: &key, + Body: source, + ACL: pool.acl, + } + + if pool.storageClass != "" { + params.StorageClass = pool.storageClass + } + if pool.encryptionMethod != "" { + params.ServerSideEncryption = pool.encryptionMethod + } + + _, err := pool.s3.PutObject(context.Background(), params) + return err +} + +// headObject returns object metadata, with found=false (and no error) when the +// object does not exist +func (pool *PackagePool) headObject(path string) (*s3.HeadObjectOutput, bool, error) { + key := pool.poolKey(path) + output, err := pool.s3.HeadObject(context.Background(), &s3.HeadObjectInput{ + Bucket: &pool.bucket, + Key: &key, + }) + + if err != nil { + if isNotFound(err) { + return nil, false, nil + } + return nil, false, err + } + + return output, true, nil +} + +func (pool *PackagePool) poolKey(path string) string { + return filepath.Join(pool.prefix, path) +} + +func isNotFound(err error) bool { + var ae smithy.APIError + if errors.As(err, &ae) { + switch ae.ErrorCode() { + case "NoSuchKey", "NotFound": + return true + case "NoSuchBucket", "AccessDenied": + // also 404/403 — but these are real failures, not a missing object + return false + } + } + + var re *awshttp.ResponseError + if errors.As(err, &re) { + return re.HTTPStatusCode() == http.StatusNotFound + } + + return false +} diff --git a/s3/package_pool_test.go b/s3/package_pool_test.go new file mode 100644 index 000000000..243094870 --- /dev/null +++ b/s3/package_pool_test.go @@ -0,0 +1,280 @@ +package s3 + +import ( + "context" + "io" + "path/filepath" + "runtime" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + + "github.com/aptly-dev/aptly/aptly" + "github.com/aptly-dev/aptly/files" + "github.com/aptly-dev/aptly/utils" + + . "gopkg.in/check.v1" +) + +type PackagePoolSuite struct { + srv *Server + pool, prefixedPool *PackagePool + debFile string + cs aptly.ChecksumStorage +} + +var _ = Suite(&PackagePoolSuite{}) + +func (s *PackagePoolSuite) SetUpTest(c *C) { + var err error + s.srv, err = NewServer(&Config{}) + c.Assert(err, IsNil) + c.Assert(s.srv, NotNil) + + // accessKey, secretKey, sessionToken, bucket, prefix, defaultACL, + // storageClass, encryptionMethod, region, endpoint, forceVirtualHostedStyle, debug + s.pool, err = NewPackagePool("aa", "bb", "", "pool-test", "", "", "", "", "test-1", s.srv.URL(), false, false) + c.Assert(err, IsNil) + + s.prefixedPool, err = NewPackagePool("aa", "bb", "", "pool-test", "lala", "", "", "", "test-1", s.srv.URL(), false, false) + c.Assert(err, IsNil) + + _, err = s.pool.s3.CreateBucket(context.TODO(), &s3.CreateBucketInput{ + Bucket: aws.String("pool-test"), + CreateBucketConfiguration: &types.CreateBucketConfiguration{ + LocationConstraint: "test-1", + }}) + c.Assert(err, IsNil) + + _, _File, _, _ := runtime.Caller(0) + s.debFile = filepath.Join(filepath.Dir(_File), "../system/files/libboost-program-options-dev_1.49.0.1_i386.deb") + s.cs = files.NewMockChecksumStorage() +} + +func (s *PackagePoolSuite) TearDownTest(c *C) { + s.srv.Quit() +} + +func (s *PackagePoolSuite) TestFilepathList(c *C) { + list, err := s.pool.FilepathList(nil) + c.Check(err, IsNil) + c.Check(list, DeepEquals, []string{}) + + _, _ = s.pool.Import(s.debFile, "a.deb", &utils.ChecksumInfo{}, false, s.cs) + _, _ = s.pool.Import(s.debFile, "b.deb", &utils.ChecksumInfo{}, false, s.cs) + + list, err = s.pool.FilepathList(nil) + c.Check(err, IsNil) + c.Check(list, DeepEquals, []string{ + "c7/6b/4bd12fd92e4dfe1b55b18a67a669_a.deb", + "c7/6b/4bd12fd92e4dfe1b55b18a67a669_b.deb", + }) +} + +func (s *PackagePoolSuite) TestRemove(c *C) { + _, _ = s.pool.Import(s.debFile, "a.deb", &utils.ChecksumInfo{}, false, s.cs) + _, _ = s.pool.Import(s.debFile, "b.deb", &utils.ChecksumInfo{}, false, s.cs) + + size, err := s.pool.Remove("c7/6b/4bd12fd92e4dfe1b55b18a67a669_a.deb") + c.Check(err, IsNil) + c.Check(size, Equals, int64(2738)) + + _, err = s.pool.Remove("c7/6b/4bd12fd92e4dfe1b55b18a67a669_a.deb") + c.Check(err, ErrorMatches, "(.|\n)*not found(.|\n)*") + + list, err := s.pool.FilepathList(nil) + c.Check(err, IsNil) + c.Check(list, DeepEquals, []string{"c7/6b/4bd12fd92e4dfe1b55b18a67a669_b.deb"}) +} + +func (s *PackagePoolSuite) TestImportOk(c *C) { + var checksum utils.ChecksumInfo + path, err := s.pool.Import(s.debFile, filepath.Base(s.debFile), &checksum, false, s.cs) + c.Check(err, IsNil) + c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_libboost-program-options-dev_1.49.0.1_i386.deb") + // SHA256 should be automatically calculated + c.Check(checksum.SHA256, Equals, "c76b4bd12fd92e4dfe1b55b18a67a669d92f62985d6a96c8a21d96120982cf12") + // checksum storage is filled with new checksum + c.Check(s.cs.(*files.MockChecksumStorage).Store[path].SHA256, Equals, "c76b4bd12fd92e4dfe1b55b18a67a669d92f62985d6a96c8a21d96120982cf12") + + size, err := s.pool.Size(path) + c.Assert(err, IsNil) + c.Check(size, Equals, int64(2738)) + + // import as different name + checksum = utils.ChecksumInfo{} + path, err = s.pool.Import(s.debFile, "some.deb", &checksum, false, s.cs) + c.Check(err, IsNil) + c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_some.deb") + c.Check(s.cs.(*files.MockChecksumStorage).Store[path].SHA256, Equals, "c76b4bd12fd92e4dfe1b55b18a67a669d92f62985d6a96c8a21d96120982cf12") + + // double import, should be ok (dedup: no re-upload) + checksum = utils.ChecksumInfo{} + path, err = s.pool.Import(s.debFile, filepath.Base(s.debFile), &checksum, false, s.cs) + c.Check(err, IsNil) + c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_libboost-program-options-dev_1.49.0.1_i386.deb") + // checksum is filled back from checksum storage (cache level 1) + c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c") + + // clear checksum storage, then double-import again: checksums must be + // recomputed by downloading the object back from S3 (cache level 2) + delete(s.cs.(*files.MockChecksumStorage).Store, path) + checksum = utils.ChecksumInfo{} + path, err = s.pool.Import(s.debFile, filepath.Base(s.debFile), &checksum, false, s.cs) + c.Check(err, IsNil) + c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_libboost-program-options-dev_1.49.0.1_i386.deb") + c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c") + + // import under a new name, with only the path-relevant checksum filled in. + // SHA1 is missing here, so Import must top the checksums back up from the + // source file before writing them to the checksum storage. + checksum = utils.ChecksumInfo{SHA256: checksum.SHA256} + path, err = s.pool.Import(s.debFile, "other.deb", &checksum, false, s.cs) + c.Check(err, IsNil) + c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_other.deb") + c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c") +} + +func (s *PackagePoolSuite) TestVerify(c *C) { + // pool is empty: a missing object is NOT an error, just not-found + ppath, exists, err := s.pool.Verify("", filepath.Base(s.debFile), &utils.ChecksumInfo{}, s.cs) + c.Check(ppath, Equals, "") + c.Check(err, IsNil) + c.Check(exists, Equals, false) + + // import file + checksum := utils.ChecksumInfo{} + path, err := s.pool.Import(s.debFile, filepath.Base(s.debFile), &checksum, false, s.cs) + c.Check(err, IsNil) + c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_libboost-program-options-dev_1.49.0.1_i386.deb") + + // check existence, path derived from checksums + ppath, exists, err = s.pool.Verify("", filepath.Base(s.debFile), &checksum, s.cs) + c.Check(ppath, Equals, path) + c.Check(err, IsNil) + c.Check(exists, Equals, true) + c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c") + + // check existence with an explicit pool path + checksum = utils.ChecksumInfo{Size: checksum.Size} + ppath, exists, err = s.pool.Verify(path, filepath.Base(s.debFile), &checksum, s.cs) + c.Check(ppath, Equals, path) + c.Check(err, IsNil) + c.Check(exists, Equals, true) + c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c") + + // checksums missing that aren't needed to find the path: filled back from storage + checksum.SHA512 = "" + ppath, exists, err = s.pool.Verify("", filepath.Base(s.debFile), &checksum, s.cs) + c.Check(ppath, Equals, path) + c.Check(err, IsNil) + c.Check(exists, Equals, true) + c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c") + + // wrong checksum, but correct path and size: the object is there but it is + // the wrong file, so not-found (and NOT an error) + ppath, exists, err = s.pool.Verify(path, filepath.Base(s.debFile), &utils.ChecksumInfo{ + SHA256: "abc", + Size: checksum.Size, + }, s.cs) + c.Check(ppath, Equals, "") + c.Check(err, IsNil) + c.Check(exists, Equals, false) + + // nothing in checksum storage: recomputed by downloading from S3 + delete(s.cs.(*files.MockChecksumStorage).Store, path) + checksum.SHA512 = "" + ppath, exists, err = s.pool.Verify("", filepath.Base(s.debFile), &checksum, s.cs) + c.Check(ppath, Equals, path) + c.Check(err, IsNil) + c.Check(exists, Equals, true) + c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c") + + // wrong size: not-found + checksum = utils.ChecksumInfo{Size: 13455} + ppath, exists, err = s.pool.Verify(path, filepath.Base(s.debFile), &checksum, s.cs) + c.Check(ppath, Equals, "") + c.Check(err, IsNil) + c.Check(exists, Equals, false) + + // empty checksum info and no pool path: nothing to look for + ppath, exists, err = s.pool.Verify("", filepath.Base(s.debFile), &utils.ChecksumInfo{}, s.cs) + c.Check(ppath, Equals, "") + c.Check(err, IsNil) + c.Check(exists, Equals, false) + + // object genuinely not in the pool at all: a 404 must surface as + // not-found, never as an error (deliberate deviation from azure.PackagePool) + ppath, exists, err = s.pool.Verify("", "missing.deb", &utils.ChecksumInfo{ + SHA256: "0000000000000000000000000000000000000000000000000000000000000000", + Size: 2738, + }, files.NewMockChecksumStorage()) + c.Check(ppath, Equals, "") + c.Check(err, IsNil) + c.Check(exists, Equals, false) +} + +func (s *PackagePoolSuite) TestImportNotExist(c *C) { + _, err := s.pool.Import("no-such-file", "a.deb", &utils.ChecksumInfo{}, false, s.cs) + c.Check(err, ErrorMatches, ".*no such file or directory") +} + +func (s *PackagePoolSuite) TestSize(c *C) { + path, err := s.pool.Import(s.debFile, filepath.Base(s.debFile), &utils.ChecksumInfo{}, false, s.cs) + c.Check(err, IsNil) + + size, err := s.pool.Size(path) + c.Assert(err, IsNil) + c.Check(size, Equals, int64(2738)) + + _, err = s.pool.Size("do/es/ntexist") + c.Check(err, ErrorMatches, "(.|\n)*not found(.|\n)*") +} + +func (s *PackagePoolSuite) TestOpen(c *C) { + path, err := s.pool.Import(s.debFile, filepath.Base(s.debFile), &utils.ChecksumInfo{}, false, s.cs) + c.Check(err, IsNil) + + f, err := s.pool.Open(path) + c.Assert(err, IsNil) + contents, err := io.ReadAll(f) + c.Assert(err, IsNil) + c.Check(len(contents), Equals, 2738) + c.Check(f.Close(), IsNil) + + _, err = s.pool.Open("do/es/ntexist") + c.Check(err, ErrorMatches, "(.|\n)*error downloading(.|\n)*") +} + +func (s *PackagePoolSuite) TestLegacyPath(c *C) { + _, err := s.pool.LegacyPath("a.deb", &utils.ChecksumInfo{MD5: "abcdef00"}) + c.Check(err, NotNil) +} + +func (s *PackagePoolSuite) TestPrefixedPool(c *C) { + path, err := s.prefixedPool.Import(s.debFile, "a.deb", &utils.ChecksumInfo{}, false, s.cs) + c.Check(err, IsNil) + + // the returned pool path is prefix-relative + c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_a.deb") + + list, err := s.prefixedPool.FilepathList(nil) + c.Check(err, IsNil) + c.Check(list, DeepEquals, []string{"c7/6b/4bd12fd92e4dfe1b55b18a67a669_a.deb"}) + + size, err := s.prefixedPool.Size(path) + c.Assert(err, IsNil) + c.Check(size, Equals, int64(2738)) + + resp, err := s.pool.s3.GetObject(context.TODO(), &s3.GetObjectInput{ + Bucket: aws.String("pool-test"), + Key: aws.String("lala/c7/6b/4bd12fd92e4dfe1b55b18a67a669_a.deb"), + }) + c.Assert(err, IsNil) + _ = resp.Body.Close() + + removed, err := s.prefixedPool.Remove(path) + c.Check(err, IsNil) + c.Check(removed, Equals, int64(2738)) +} diff --git a/system/s3_lib.py b/system/s3_lib.py index 6e622b2d7..b7c79168b 100644 --- a/system/s3_lib.py +++ b/system/s3_lib.py @@ -22,6 +22,7 @@ class S3Test(BaseTest): """ s3Overrides = {} + use_s3_pool = False def fixture_available(self): return super(S3Test, self).fixture_available() and \ @@ -42,6 +43,15 @@ def prepare(self): self.configOverride["S3PublishEndpoints"]["test1"].update(**self.s3Overrides) + if self.use_s3_pool: + self.configOverride["packagePoolStorage"] = { + "type": "s3", + "region": "us-east-1", + "bucket": self.bucket_name, + "awsAccessKeyID": os.environ["AWS_ACCESS_KEY_ID"], + "awsSecretAccessKey": os.environ["AWS_SECRET_ACCESS_KEY"], + } + super(S3Test, self).prepare() def shutdown(self): @@ -77,6 +87,10 @@ def check_exists(self, path): if not self.check_path(path): raise Exception("path %s doesn't exist" % (path, )) + def check_exists_s3_only(self, path): + self.check_exists(path) + BaseTest.check_not_exists(self, path) + def check_not_exists(self, path): if self.check_path(path): raise Exception("path %s exists" % (path, )) diff --git a/system/t02_config/CreateConfigTest_gold b/system/t02_config/CreateConfigTest_gold index d8e39c9dc..46aea6fba 100644 --- a/system/t02_config/CreateConfigTest_gold +++ b/system/t02_config/CreateConfigTest_gold @@ -402,6 +402,7 @@ azure_publish_endpoints: # Type must be one of: # * local # * azure +# * s3 packagepool_storage: # Local Pool type: local @@ -425,5 +426,23 @@ packagepool_storage: # # defaults to "https://.blob.core.windows.net" # endpoint: "" + # # S3 Object Storage Pool + # type: s3 + # # S3 bucket to store the pool in + # bucket: pool1 + # # Prefix (optional) + # # Store the pool under specified prefix in the bucket, defaults to no prefix (bucket root) + # prefix: "" + # # Region (i.e. us-east-1) + # region: "" + # # Credentials (optional) + # # empty credentials use the standard AWS credential chain (env vars, shared credentials file, IAM role) + # access_key_id: "" + # secret_access_key: "" + # session_token: "" + # # Endpoint URL (optional) + # # for S3-compatible object stores (MinIO, etc.), defaults to AWS S3 + # endpoint: "" + diff --git a/system/t09_repo/S3RepoTest_gold b/system/t09_repo/S3RepoTest_gold new file mode 100644 index 000000000..7ddaaa9f2 --- /dev/null +++ b/system/t09_repo/S3RepoTest_gold @@ -0,0 +1,5 @@ +Loading packages... +[+] libboost-program-options-dev_1.49.0.1_i386 added +[+] libboost-program-options-dev_1.62.0.1_i386 added +[+] pyspi_0.6.1-1.4_source added +[+] pyspi_0.6.1-1.3_source added diff --git a/system/t09_repo/S3RepoTest_repo_show b/system/t09_repo/S3RepoTest_repo_show new file mode 100644 index 000000000..17dbc6eff --- /dev/null +++ b/system/t09_repo/S3RepoTest_repo_show @@ -0,0 +1,10 @@ +Name: repo +Comment: Repo +Default Distribution: squeeze +Default Component: main +Number of packages: 4 +Packages: + libboost-program-options-dev_1.62.0.1_i386 + libboost-program-options-dev_1.49.0.1_i386 + pyspi_0.6.1-1.4_source + pyspi_0.6.1-1.3_source diff --git a/system/t09_repo/s3.py b/system/t09_repo/s3.py new file mode 100644 index 000000000..e937225d7 --- /dev/null +++ b/system/t09_repo/s3.py @@ -0,0 +1,35 @@ +from s3_lib import S3Test + + +class S3RepoTest(S3Test): + """ + S3: add directory to repo with S3 package pool + """ + + fixtureCmds = [ + 'aptly repo create -comment=Repo -distribution=squeeze repo', + ] + runCmd = 'aptly repo add repo ${files}' + + use_s3_pool = True + + def check(self): + self.check_output() + self.check_cmd_output('aptly repo show -with-packages repo', 'repo_show') + + # check pool + self.check_exists_s3_only( + 'c7/6b/4bd12fd92e4dfe1b55b18a67a669_libboost-program-options-dev_1.49.0.1_i386.deb' + ) + self.check_exists_s3_only( + '2e/77/0b28df948f3197ed0b679bdea99f_pyspi_0.6.1-1.3.diff.gz' + ) + self.check_exists_s3_only( + 'd4/94/aaf526f1ec6b02f14c2f81e060a5_pyspi_0.6.1-1.3.dsc' + ) + self.check_exists_s3_only( + '64/06/9ee828c50b1c597d10a3fefbba27_pyspi_0.6.1.orig.tar.gz' + ) + self.check_exists_s3_only( + '28/9d/3aefa970876e9c43686ce2b02f47_pyspi-0.6.1-1.3.stripped.dsc' + ) diff --git a/utils/config.go b/utils/config.go index 1c148e4f1..d3656fef9 100644 --- a/utils/config.go +++ b/utils/config.go @@ -83,10 +83,12 @@ type LocalPoolStorage struct { type PackagePoolStorage struct { Local *LocalPoolStorage Azure *AzureEndpoint + S3 *S3PublishRoot } var AZURE = "azure" var LOCAL = "local" +var S3 = "s3" func (pool *PackagePoolStorage) UnmarshalJSON(data []byte) error { var discriminator struct { @@ -101,6 +103,9 @@ func (pool *PackagePoolStorage) UnmarshalJSON(data []byte) error { case AZURE: pool.Azure = &AzureEndpoint{} return json.Unmarshal(data, &pool.Azure) + case S3: + pool.S3 = &S3PublishRoot{} + return json.Unmarshal(data, &pool.S3) case LOCAL, "": pool.Local = &LocalPoolStorage{} return json.Unmarshal(data, &pool.Local) @@ -121,6 +126,9 @@ func (pool *PackagePoolStorage) UnmarshalYAML(unmarshal func(interface{}) error) case AZURE: pool.Azure = &AzureEndpoint{} return unmarshal(&pool.Azure) + case S3: + pool.S3 = &S3PublishRoot{} + return unmarshal(&pool.S3) case LOCAL, "": pool.Local = &LocalPoolStorage{} return unmarshal(&pool.Local) @@ -130,39 +138,53 @@ func (pool *PackagePoolStorage) UnmarshalYAML(unmarshal func(interface{}) error) } func (pool *PackagePoolStorage) MarshalJSON() ([]byte, error) { - var wrapper struct { - Type string `json:"type,omitempty"` - *LocalPoolStorage - *AzureEndpoint - } + switch { + case pool.S3 != nil: + return json.Marshal(struct { + Type string `json:"type"` + *S3PublishRoot + }{S3, pool.S3}) + + case pool.Azure != nil: + return json.Marshal(struct { + Type string `json:"type"` + *AzureEndpoint + }{AZURE, pool.Azure}) + + case pool.Local != nil && pool.Local.Path != "": + return json.Marshal(struct { + Type string `json:"type"` + *LocalPoolStorage + }{LOCAL, pool.Local}) - if pool.Azure != nil { - wrapper.Type = "azure" - wrapper.AzureEndpoint = pool.Azure - } else if pool.Local.Path != "" { - wrapper.Type = "local" - wrapper.LocalPoolStorage = pool.Local + default: + return json.Marshal(struct{}{}) } - - return json.Marshal(wrapper) } func (pool PackagePoolStorage) MarshalYAML() (interface{}, error) { - var wrapper struct { - Type string `yaml:"type,omitempty"` - *LocalPoolStorage `yaml:",inline"` - *AzureEndpoint `yaml:",inline"` - } + switch { + case pool.S3 != nil: + return struct { + Type string `yaml:"type"` + *S3PublishRoot `yaml:",inline"` + }{S3, pool.S3}, nil + + case pool.Azure != nil: + return struct { + Type string `yaml:"type"` + *AzureEndpoint `yaml:",inline"` + }{AZURE, pool.Azure}, nil + + case pool.Local != nil && pool.Local.Path != "": + return struct { + Type string `yaml:"type"` + *LocalPoolStorage `yaml:",inline"` + }{LOCAL, pool.Local}, nil - if pool.Azure != nil { - wrapper.Type = "azure" - wrapper.AzureEndpoint = pool.Azure - } else if pool.Local.Path != "" { - wrapper.Type = "local" - wrapper.LocalPoolStorage = pool.Local + default: + return struct{}{}, nil } - - return wrapper, nil } // FileSystemPublishRoot describes single filesystem publishing entry point diff --git a/utils/config_test.go b/utils/config_test.go index abc59e492..d44d3348d 100644 --- a/utils/config_test.go +++ b/utils/config_test.go @@ -1,10 +1,12 @@ package utils import ( + "encoding/json" "os" "path/filepath" . "gopkg.in/check.v1" + yaml "gopkg.in/yaml.v3" ) type ConfigSuite struct { @@ -433,3 +435,75 @@ packagepool_storage: const configFileYAMLError = `packagepool_storage: type: invalid ` + +func (s *ConfigSuite) TestPackagePoolStorageS3JSON(c *C) { + data := []byte(`{"type": "s3", "region": "us-east-1", "bucket": "aptly-pool", "prefix": "pool", "endpoint": "http://minio:9000"}`) + + var pool PackagePoolStorage + err := json.Unmarshal(data, &pool) + c.Assert(err, IsNil) + c.Assert(pool.S3, NotNil) + // the s3 discriminator must not leak into the other backends + c.Check(pool.Local, IsNil) + c.Check(pool.Azure, IsNil) + c.Check(pool.S3.Region, Equals, "us-east-1") + c.Check(pool.S3.Bucket, Equals, "aptly-pool") + c.Check(pool.S3.Prefix, Equals, "pool") + c.Check(pool.S3.Endpoint, Equals, "http://minio:9000") + + out, err := json.Marshal(&pool) + c.Assert(err, IsNil) + c.Check(string(out), Matches, `.*"type":"s3".*`) + // prefix and endpoint collide with AzureEndpoint's tags. Embedding both in + // one wrapper makes encoding/json drop them silently (no error), so assert + // they survive the round trip. + c.Check(string(out), Matches, `.*"prefix":"pool".*`) + c.Check(string(out), Matches, `.*"endpoint":"http://minio:9000".*`) + + var pool2 PackagePoolStorage + err = json.Unmarshal(out, &pool2) + c.Assert(err, IsNil) + c.Assert(pool2.S3, NotNil) + c.Check(*pool2.S3, DeepEquals, *pool.S3) +} + +func (s *ConfigSuite) TestPackagePoolStorageS3YAML(c *C) { + data := []byte("type: s3\nregion: us-east-1\nbucket: aptly-pool\nprefix: pool\nendpoint: http://minio:9000\n") + + var pool PackagePoolStorage + err := yaml.Unmarshal(data, &pool) + c.Assert(err, IsNil) + c.Assert(pool.S3, NotNil) + c.Check(pool.Local, IsNil) + c.Check(pool.Azure, IsNil) + c.Check(pool.S3.Region, Equals, "us-east-1") + c.Check(pool.S3.Bucket, Equals, "aptly-pool") + c.Check(pool.S3.Prefix, Equals, "pool") + c.Check(pool.S3.Endpoint, Equals, "http://minio:9000") + + // a duplicate `prefix` inline tag makes yaml.v3 panic rather than drop it, + // so this also guards the shared-wrapper regression + out, err := yaml.Marshal(pool) + c.Assert(err, IsNil) + c.Check(string(out), Matches, "(?s).*type: s3.*") + c.Check(string(out), Matches, "(?s).*prefix: pool.*") + c.Check(string(out), Matches, "(?s).*endpoint: http://minio:9000.*") + + var pool2 PackagePoolStorage + err = yaml.Unmarshal(out, &pool2) + c.Assert(err, IsNil) + c.Assert(pool2.S3, NotNil) + c.Check(*pool2.S3, DeepEquals, *pool.S3) +} + +func (s *ConfigSuite) TestPackagePoolStorageMarshalNil(c *C) { + // an all-nil pool storage must not panic on the Local dereference + var pool PackagePoolStorage + + out, err := json.Marshal(&pool) + c.Assert(err, IsNil) + c.Check(string(out), Equals, "{}") + + _, err = yaml.Marshal(pool) + c.Assert(err, IsNil) +}