Ensure s3 bucket issue 495 - Bucket Tags

docs/config.rst

@@ -90,6 +90,32 @@ See the `CloudFormation Limits Reference`_.
 
 .. _`CloudFormation Limits Reference`: http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/cloudformation-limits.html
 
+
+S3 Bucket tags
+--------------
+
+Various resources in AWS support arbitrary key-value pair tags. You can set
+the `bucket_tags` Top Level Keyword to populate tags on all S3 buckets Staker
+attempts to create for CloudFormation template uploads, inclduing the S3 bucket
+created by the aws_lambda pre-hook.
+
+If bucket_tags is not set in your Configuration, stacker will fallback to the
+method used to determine tags in your config by the `tags` top level keyword.
+The `bucket_tags` keyword takes precedence over `tags` when applying. Example::
+
+  bucket_tags:
+    "hello": world
+    "my_tag:with_colons_in_key": ${dynamic_tag_value_from_my_env}
+    simple_tag: simple value
+
+If you prefer to have no tags applied to your stacks (versus the default tags
+that stacker applies), specify an empty map for the top-level keyword::
+
+  bucket_tags: {}
+
+S3 Bucket Tags updates get applied on every stacker run
+
+
 Module Paths
 ------------
 When setting the ``classpath`` for blueprints/hooks, it is sometimes desirable to
@@ -137,7 +163,7 @@ The only required key for a git repository config is ``uri``, but ``branch``,
           commit: 12345678
 
 If no specific commit or tag is specified for a repo, the remote repository
-will be checked for newer commits on every execution of Stacker.
+will be checked for newer commits on every execution of stacker.
 
 For ``.tar.gz`` & ``zip`` archives on s3, specify a ``bucket`` & ``key``::
 
@@ -157,7 +183,7 @@ For ``.tar.gz`` & ``zip`` archives on s3, specify a ``bucket`` & ``key``::
           use_latest: false
 
 Use the ``paths`` option when subdirectories of the repo/archive should be
-added to Stacker's ``sys.path``.
+added to stacker's ``sys.path``.
 
 Cloned repos/archives will be cached between builds; the cache location defaults
 to ~/.stacker but can be manually specified via the **stacker_cache_dir** top
@@ -236,23 +262,32 @@ the build action::
 Tags
 ----
 
-CloudFormation supports arbitrary key-value pair tags. All stack-level, including automatically created tags, are
-propagated to resources that AWS CloudFormation supports. See `AWS CloudFormation Resource Tags Type`_ for more details.
-If no tags are specified, the `stacker_namespace` tag is applied to your stack with the value of `namespace` as the
-tag value.
+Various resources in AWS support arbitrary key-value pair tags. You can set
+the `tags` Top Level Keyword to populate tags on all Resources that stacker
+attempts to create via CloudFormation. All CloudFormation stack-level resources,
+including automatically created tags, are propagated to resources that AWS
+CloudFormation supports. See `AWS CloudFormation Resource Tags Type`_ for
+more details.
 
-If you prefer to apply a custom set of tags, specify the top-level keyword `tags` as a map. Example::
+If no tags are specified, the `stacker_namespace` tag is applied to your stack
+with the value of `namespace` as the tag value.
+
+If you prefer to apply a custom set of tags, specify the top-level keyword
+`tags` as a map. Example::
 
   tags:
     "hello": world
     "my_tag:with_colons_in_key": ${dynamic_tag_value_from_my_env}
     simple_tag: simple value
 
+
 If you prefer to have no tags applied to your stacks (versus the default tags that stacker applies), specify an empty
 map for the top-level keyword::
 
   tags: {}
 
+Tags updates get applied on every stacker run
+
 .. _`AWS CloudFormation Resource Tags Type`: http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-resource-tags.html
 
 Mappings

stacker/actions/base.py

@@ -86,7 +86,8 @@ def bucket_region(self):
 
     def ensure_cfn_bucket(self):
         """The CloudFormation bucket where templates will be stored."""
-        ensure_s3_bucket(self.s3_conn, self.bucket_name, self.bucket_region)
+        ensure_s3_bucket(self.s3_conn, self.bucket_name, self.bucket_region,
+                         self.context)
 
     def stack_template_url(self, blueprint):
         return stack_template_url(

stacker/config/__init__.py

@@ -351,6 +351,8 @@ class Config(Model):
 
     tags = DictType(StringType, serialize_when_none=False)
 
+    bucket_tags = DictType(StringType, serialize_when_none=False)
+
     mappings = DictType(
         DictType(DictType(StringType)), serialize_when_none=False)
 

stacker/context.py

@@ -102,6 +102,19 @@ def tags(self):
             return {"stacker_namespace": self.namespace}
         return {}
 
+    @property
+    def s3_bucket_tags(self):
+        s3_bucket_tags = self.config.bucket_tags
+        if s3_bucket_tags is not None:
+            return s3_bucket_tags
+        else:
+            s3_bucket_tags = self.config.tags
+            if s3_bucket_tags is not None:
+                return s3_bucket_tags
+            if self.namespace:
+                return {"stacker_namespace": self.namespace}
+            return {}
+
     @property
     def _base_fqn(self):
         return self.namespace.replace(".", "-").lower()

stacker/hooks/aws_lambda.py

@@ -490,7 +490,7 @@ def create_template(self):
     session = get_session(bucket_region)
     s3_client = session.client('s3')
 
-    ensure_s3_bucket(s3_client, bucket_name, bucket_region)
+    ensure_s3_bucket(s3_client, bucket_name, bucket_region, context)
 
     prefix = kwargs.get('prefix', '')
 

stacker/tests/actions/test_base.py

@@ -42,6 +42,17 @@ def test_ensure_cfn_bucket_exists(self):
                 "Bucket": ANY,
             }
         )
+        stubber.add_response(
+            "put_bucket_tagging",
+            service_response={},
+            expected_params={
+                "Bucket": ANY,
+                "Tagging": {
+                    "TagSet": [
+                        {"Key": "stacker_namespace",
+                         "Value": u"mynamespace"}]}
+            }
+        )
         with stubber:
             action.ensure_cfn_bucket()
 
@@ -65,6 +76,17 @@ def test_ensure_cfn_bucket_doesnt_exist_us_east(self):
                 "Bucket": ANY,
             }
         )
+        stubber.add_response(
+            "put_bucket_tagging",
+            service_response={},
+            expected_params={
+                "Bucket": ANY,
+                "Tagging": {
+                    "TagSet": [
+                        {"Key": "stacker_namespace",
+                         "Value": u"mynamespace"}]}
+            }
+        )
         with stubber:
             action.ensure_cfn_bucket()
 
@@ -91,6 +113,17 @@ def test_ensure_cfn_bucket_doesnt_exist_us_west(self):
                 }
             }
         )
+        stubber.add_response(
+            "put_bucket_tagging",
+            service_response={},
+            expected_params={
+                "Bucket": ANY,
+                "Tagging": {
+                    "TagSet": [
+                        {"Key": "stacker_namespace",
+                         "Value": u"mynamespace"}]}
+            }
+        )
         with stubber:
             action.ensure_cfn_bucket()
 

stacker/util.py

@@ -17,9 +17,12 @@
 
 import botocore.client
 import botocore.exceptions
+
 import dateutil
-import yaml
+
 from git import Repo
+
+import yaml
 from yaml.constructor import ConstructorError
 from yaml.nodes import MappingNode
 
@@ -525,7 +528,7 @@ def s3_bucket_location_constraint(region):
     return region
 
 
-def ensure_s3_bucket(s3_client, bucket_name, bucket_region):
+def ensure_s3_bucket(s3_client, bucket_name, bucket_region, context):
     """Ensure an s3 bucket exists, if it does not then create it.
 
     Args:
@@ -534,8 +537,13 @@ def ensure_s3_bucket(s3_client, bucket_name, bucket_region):
         bucket_name (str): The bucket being checked/created.
         bucket_region (str, optional): The region to create the bucket in. If
             not provided, will be determined by s3_client's region.
+        context (:class:`stacker.context.Context`): The stacker context, used
+            set the S3 bucket tags from the stacker config
+
     """
+    tagset = _s3_bucket_tags(context)
     try:
+        logger.debug("Checking that bucket '%s' exists.", bucket_name)
         s3_client.head_bucket(Bucket=bucket_name)
     except botocore.exceptions.ClientError as e:
         if e.response['Error']['Message'] == "Not Found":
@@ -548,6 +556,7 @@ def ensure_s3_bucket(s3_client, bucket_name, bucket_region):
                 create_args["CreateBucketConfiguration"] = {
                     "LocationConstraint": location_constraint
                 }
+            # pulling tags from s3_bucket_tags function
             s3_client.create_bucket(**create_args)
         elif e.response['Error']['Message'] == "Forbidden":
             logger.exception("Access denied for bucket %s.  Did " +
@@ -559,6 +568,29 @@ def ensure_s3_bucket(s3_client, bucket_name, bucket_region):
                              bucket_name, e.response)
             raise
 
+    logger.debug(
+        "Setting tags on bucket '%s': %s", bucket_name, context.s3_bucket_tags
+    )
+
+    # setting tags on every run - must have permission to perform
+    # the s3:PutBucketTagging action
+    s3_client.put_bucket_tagging(Bucket=bucket_name,
+                                 Tagging={'TagSet': tagset})
+
+
+def _s3_bucket_tags(context):
+    """Returns the tags to be applied for a S3 bucket.
+
+    Args:
+        context (:class:`stacker.context.Context`): The stacker context, used
+            set the S3 bucket tags from the stacker config
+
+    Returns:
+        List of dictionaries containing tags to apply to that bucket.
+    """
+    return [
+        {'Key': t[0], 'Value': t[1]} for t in context.s3_bucket_tags.items()]
+
 
 class Extractor(object):
     """Base class for extractors."""
@@ -629,8 +661,9 @@ def extension():
         return '.zip'
 
 
-class SourceProcessor(object):
-    """Makes remote python package sources available in current environment."""
+class SourceProcessor():
+    """Makes remote python package sources available in the running python
+       environment."""
 
     ISO8601_FORMAT = '%Y%m%dT%H%M%SZ'