add s3 remote package sources (#487)

* add s3 remote package sources

* cleanup fetch_s3_package

* add check for last modified date on s3

* rearrange imports for style (placing standard imports first)

* add datetutil for utc conversion; cleanup doc warnings

* add additional debug statements

Author: troyready

docs/config.rst

@@ -117,7 +117,7 @@ See the AWS documentation for `AWS CloudFormation Service Roles`_.
 
 Remote Packages
 ---------------
-The **package_sources** top level keyword can be used to define remote git
+The **package_sources** top level keyword can be used to define remote
 sources for blueprints (e.g., retrieving ``stacker_blueprints`` on github at
 tag ``v1.0.2``).
 
@@ -136,13 +136,30 @@ The only required key for a git repository config is ``uri``, but ``branch``,
         - uri: git@github.com:contoso/foo.git
           commit: 12345678
 
-Use the ``paths`` option when subdirectories of the repo should be added to
-Stacker's ``sys.path``.
-
 If no specific commit or tag is specified for a repo, the remote repository
 will be checked for newer commits on every execution of Stacker.
 
-Cloned repositories will be cached between builds; the cache location defaults
+For ``.tar.gz`` & ``zip`` archives on s3, specify a ``bucket`` & ``key``::
+
+    package_sources:
+      s3:
+        - bucket: mystackers3bucket
+          key: archives/blueprints-v1.zip
+          paths:
+            - stacker_blueprints
+        - bucket: anothers3bucket
+          key: public/public-blueprints-v2.tar.gz
+          requester_pays: true
+        - bucket: yetanothers3bucket
+          key: sallys-blueprints-v1.tar.gz
+          # use_latest defaults to true - will update local copy if the
+          # last modified date on S3 changes
+          use_latest: false
+
+Use the ``paths`` option when subdirectories of the repo/archive should be
+added to Stacker's ``sys.path``.
+
+Cloned repos/archives will be cached between builds; the cache location defaults
 to ~/.stacker but can be manually specified via the **stacker_cache_dir** top
 level keyword.
 

setup.py

@@ -13,7 +13,8 @@
     "colorama~=0.3.7",
     "formic~=0.9b",
     "gitpython~=2.0",
-    "schematics~=2.0.1"
+    "schematics~=2.0.1",
+    "python-dateutil~=2.0"
 ]
 
 tests_require = [

stacker/config/__init__.py

@@ -242,9 +242,21 @@ class GitPackageSource(Model):
     configs = ListType(StringType, serialize_when_none=False)
 
 
+class S3PackageSource(Model):
+    bucket = StringType(required=True)
+
+    key = StringType(required=True)
+
+    use_latest = BooleanType(serialize_when_none=False)
+
+    requester_pays = BooleanType(serialize_when_none=False)
+
+
 class PackageSources(Model):
     git = ListType(ModelType(GitPackageSource))
 
+    s3 = ListType(ModelType(S3PackageSource))
+
 
 class Hook(Model):
     path = StringType(required=True)

stacker/tests/test_config.py

@@ -198,6 +198,15 @@ def test_parse(self):
             args:
               domain: mydomain.com
         package_sources:
+          s3:
+            - bucket: acmecorpbucket
+              key: public/acmecorp-blueprints-v1.zip
+            - bucket: examplecorpbucket
+              key: public/examplecorp-blueprints-v2.tar.gz
+              requester_pays: true
+            - bucket: anotherexamplebucket
+              key: example-blueprints-v3.tar.gz
+              use_latest: false
           git:
             - uri: git@github.com:acmecorp/stacker_blueprints.git
             - uri: git@github.com:remind101/stacker_blueprints.git
@@ -250,6 +259,15 @@ def test_parse(self):
             args:
               domain: mydomain.com
         package_sources:
+          s3:
+            - bucket: acmecorpbucket
+              key: public/acmecorp-blueprints-v1.zip
+            - bucket: examplecorpbucket
+              key: public/examplecorp-blueprints-v2.tar.gz
+              requester_pays: true
+            - bucket: anotherexamplebucket
+              key: example-blueprints-v3.tar.gz
+              use_latest: false
           git:
             - uri: git@github.com:acmecorp/stacker_blueprints.git
             - uri: git@github.com:remind101/stacker_blueprints.git
@@ -292,6 +310,25 @@ def test_parse(self):
                 self.assertEqual(
                     hooks[0].args, {"domain": "mydomain.com"})
 
+            self.assertEqual(
+                config.package_sources.s3[0].bucket,
+                "acmecorpbucket")
+            self.assertEqual(
+                config.package_sources.s3[0].key,
+                "public/acmecorp-blueprints-v1.zip")
+            self.assertEqual(
+                config.package_sources.s3[1].bucket,
+                "examplecorpbucket")
+            self.assertEqual(
+                config.package_sources.s3[1].key,
+                "public/examplecorp-blueprints-v2.tar.gz")
+            self.assertEqual(
+                config.package_sources.s3[1].requester_pays,
+                True)
+            self.assertEqual(
+                config.package_sources.s3[2].use_latest,
+                False)
+
             self.assertEqual(
                 config.package_sources.git[0].uri,
                 "git@github.com:acmecorp/stacker_blueprints.git")

stacker/tests/test_util.py

@@ -20,6 +20,10 @@
     get_client_region,
     get_s3_endpoint,
     s3_bucket_location_constraint,
+    Extractor,
+    TarExtractor,
+    TarGzipExtractor,
+    ZipExtractor,
     SourceProcessor
 )
 
@@ -141,6 +145,15 @@ def test_s3_bucket_location_constraint(self):
                 result
             )
 
+    def test_extractors(self):
+        self.assertEqual(Extractor('test.zip').archive, 'test.zip')
+        self.assertEqual(TarExtractor().extension(), '.tar')
+        self.assertEqual(TarGzipExtractor().extension(), '.tar.gz')
+        self.assertEqual(ZipExtractor().extension(), '.zip')
+        for i in [TarExtractor(), ZipExtractor(), ZipExtractor()]:
+            i.set_archive('/tmp/foo')
+            self.assertEqual(i.archive.endswith(i.extension()), True)
+
     def test_SourceProcessor_helpers(self):
         with mock.patch.object(SourceProcessor,
                                'create_cache_directories',
@@ -151,6 +164,10 @@ def test_SourceProcessor_helpers(self):
                 sp.sanitize_git_path('git@github.com:foo/bar.git'),
                 'git_github.com_foo_bar'
             )
+            self.assertEqual(
+                sp.sanitize_uri_path('http://example.com/foo/bar.gz@1'),
+                'http___example.com_foo_bar.gz_1'
+            )
             self.assertEqual(
                 sp.sanitize_git_path('git@github.com:foo/bar.git', 'v1'),
                 'git_github.com_foo_bar-v1'