From 8130fa6e02a3935afedce7f13307f96dc896756c Mon Sep 17 00:00:00 2001 From: Daniel Miranda Date: Sun, 1 Jul 2018 18:03:23 -0300 Subject: [PATCH 01/10] config: add simplified ExternalStack model To import outputs from stacks that are not managed directly by a Stacker config and require custom settings (such as a profile or region), it was previously possible to just omit both a `class_path` and `template_path`. Make that pattern "official" by introducing an `external` option, and a simplified `ExternalStack` model that omits all the stack keys that do not make sense if we're never going to create/update the stack. Note: the upgrade to schematics 2.1.0 is required due to a bug [1] where polymorphic models were not properly validated. [1]: https://github.com/schematics/schematics/issues/440 --- setup.py | 2 +- stacker/config/__init__.py | 40 +++++++++++++++++++----------- stacker/tests/test_config.py | 48 ++++++++++++++++++++++++++++++++++-- 3 files changed, 72 insertions(+), 18 deletions(-) diff --git a/setup.py b/setup.py index 8aae81862..296bc1df2 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ "awacs>=0.6.0", "gitpython>=2.0,<3.0", "jinja2>=2.7,<3.0", - "schematics>=2.0.1,<2.1.0", + "schematics>=2.1,<3.0", "formic2", "python-dateutil>=2.0,<3.0", ] diff --git a/stacker/config/__init__.py b/stacker/config/__init__.py index 5fdde4162..1fee277c6 100644 --- a/stacker/config/__init__.py +++ b/stacker/config/__init__.py @@ -5,11 +5,11 @@ standard_library.install_aliases() from builtins import str import copy -import sys import logging +import sys -from string import Template from io import StringIO +from string import Template from schematics import Model from schematics.exceptions import ValidationError @@ -19,12 +19,13 @@ ) from schematics.types import ( - ModelType, - ListType, - StringType, + BaseType, BooleanType, DictType, - BaseType + ListType, + ModelType, + PolyModelType, + StringType ) import yaml @@ -225,12 +226,6 @@ def process_remote_sources(raw_config, environment=None): return raw_config -def not_empty_list(value): - if not value or len(value) < 1: - raise ValidationError("Should have more than one element.") - return value - - class AnyType(BaseType): pass @@ -299,7 +294,7 @@ class Target(Model): required_by = ListType(StringType, serialize_when_none=False) -class Stack(Model): +class BaseStack(Model): name = StringType(required=True) stack_name = StringType(serialize_when_none=False) @@ -308,6 +303,21 @@ class Stack(Model): profile = StringType(serialize_when_none=False) + external = BooleanType(default=False) + + +class ExternalStack(BaseStack): + @classmethod + def _claim_polymorphic(cls, value): + if value.get("external", False) is True: + return True + + def __init__(self, *args, **kwargs): + super(ExternalStack, self).__init__(*args, **kwargs) + self.external = True + + +class Stack(BaseStack): class_path = StringType(serialize_when_none=False) template_path = StringType(serialize_when_none=False) @@ -368,7 +378,6 @@ def validate_parameters(self, data, value): "dthedocs.io/en/latest/config.html#variables for " "additional information." % stack_name) - return value class Config(Model): @@ -431,7 +440,8 @@ class Config(Model): ModelType(Target), serialize_when_none=False) stacks = ListType( - ModelType(Stack), default=[]) + PolyModelType([ExternalStack, Stack]), + default=[], validators=[]) log_formats = DictType(StringType, serialize_when_none=False) diff --git a/stacker/tests/test_config.py b/stacker/tests/test_config.py index 87876c0d2..3f582824d 100644 --- a/stacker/tests/test_config.py +++ b/stacker/tests/test_config.py @@ -13,7 +13,7 @@ dump, process_remote_sources ) -from stacker.config import Config, Stack +from stacker.config import Config, ExternalStack, Stack from stacker.environment import parse_environment from stacker import exceptions from stacker.lookups.registry import LOOKUP_HANDLERS @@ -190,6 +190,44 @@ def test_parse_with_deprecated_parameters(self): " your config. See https://stacker.readthedocs.io/en/latest/c" "onfig.html#variables for additional information.") + def test_parse_external(self): + config = parse(""" + namespace: prod + stacks: + - name: vpc + stack_name: cool-vpc + class_path: blueprints.VPC + parameters: + Foo: bar + - name: external-vpc + stack_name: other-cool-vpc + external: yes + """) + + local_stack, external_stack = config.stacks + self.assertIsInstance(local_stack, Stack) + self.assertEquals(local_stack.name, 'vpc') + self.assertEquals(local_stack.stack_name, 'cool-vpc') + self.assertIsInstance(external_stack, ExternalStack) + self.assertEquals(external_stack.name, 'external-vpc') + self.assertEquals(external_stack.stack_name, 'other-cool-vpc') + + def test_parse_external_invalid(self): + config = parse(""" + namespace: prod + stacks: + - name: vpc + class_path: blueprints.VPC + parameters: + Foo: bar + - name: external-vpc + stack_name: some-other-vpc + external: yes + """) + + with self.assertRaises(exceptions.InvalidConfig): + config.validate() + def test_config_build(self): vpc = Stack({"name": "vpc", "class_path": "blueprints.VPC"}) config = Config({"namespace": "prod", "stacks": [vpc]}) @@ -428,22 +466,28 @@ def test_dump_complex(self): Stack({ "name": "bastion", "class_path": "blueprints.Bastion", - "requires": ["vpc"]})]}) + "requires": ["vpc"]}), + ExternalStack({ + "name": "external"})]}) self.assertEqual(dump(config), b"""namespace: prod stacks: - class_path: blueprints.VPC enabled: true + external: false locked: false name: vpc protected: false - class_path: blueprints.Bastion enabled: true + external: false locked: false name: bastion protected: false requires: - vpc +- external: true + name: external """) def test_load_register_custom_lookups(self): From 17ff01b13a444f0b90225809e715aaea701edd61 Mon Sep 17 00:00:00 2001 From: Daniel Miranda Date: Sun, 1 Jul 2018 18:10:05 -0300 Subject: [PATCH 02/10] Properly handle external stacks in actions Don't ever attempt to update them, but always submit them so outputs can be grabbed. --- stacker/actions/build.py | 9 ++++++ stacker/context.py | 28 +++++++++------- stacker/stack.py | 50 +++++++++++++++++++++++++++++ stacker/tests/actions/test_build.py | 47 +++++++++++++-------------- 4 files changed, 99 insertions(+), 35 deletions(-) diff --git a/stacker/actions/build.py b/stacker/actions/build.py index bd2b91714..77732ba4b 100644 --- a/stacker/actions/build.py +++ b/stacker/actions/build.py @@ -48,6 +48,10 @@ def should_update(stack): bool: If the stack should be updated, return True. """ + + if not stack.blueprint: + return False + if stack.locked: if not stack.force: logger.debug("Stack %s locked and not in --force list. " @@ -69,6 +73,11 @@ def should_submit(stack): bool: If the stack should be submitted, return True. """ + + # Submit stack without a blueprint just to grab outputs + if not stack.blueprint: + return True + if stack.enabled: return True diff --git a/stacker/context.py b/stacker/context.py index 0eac9236f..eb9b57686 100644 --- a/stacker/context.py +++ b/stacker/context.py @@ -5,8 +5,8 @@ import collections import logging -from stacker.config import Config -from .stack import Stack +from stacker.config import Config, ExternalStack as ExternalStackModel +from .stack import ExternalStack, Stack from .target import Target logger = logging.getLogger(__name__) @@ -153,15 +153,21 @@ def get_stacks(self): stacks = [] definitions = self._get_stack_definitions() for stack_def in definitions: - stack = Stack( - definition=stack_def, - context=self, - mappings=self.mappings, - force=stack_def.name in self.force_stacks, - locked=stack_def.locked, - enabled=stack_def.enabled, - protected=stack_def.protected, - ) + if isinstance(stack_def, ExternalStackModel): + stack = ExternalStack( + definition=stack_def, + context=self + ) + else: + stack = Stack( + definition=stack_def, + context=self, + mappings=self.mappings, + force=stack_def.name in self.force_stacks, + locked=stack_def.locked, + enabled=stack_def.enabled, + protected=stack_def.protected, + ) stacks.append(stack) self._stacks = stacks return self._stacks diff --git a/stacker/stack.py b/stacker/stack.py index aa5ab81b4..88cad1280 100644 --- a/stacker/stack.py +++ b/stacker/stack.py @@ -195,3 +195,53 @@ def resolve(self, context, provider): def set_outputs(self, outputs): self.outputs = outputs + + +class ExternalStack(Stack): + """Represents gathered information about an existing external stack + + Args: + definition (:class:`stacker.config.Stack`): A stack definition. + context (:class:`stacker.context.Context`): Current context for + building the stack. + + """ + + def __init__(self, definition, context): + self.name = definition.name + self.fqn = definition.stack_name + self.region = definition.region + self.profile = definition.profile + self.definition = definition + self.context = context + self.outputs = None + + @property + def requires(self): + return set() + + @property + def stack_policy(self): + return None + + @property + def blueprint(self): + return None + + @property + def tags(self): + return dict() + + @property + def parameter_values(self): + return dict() + + @property + def required_parameter_definitions(self): + return dict() + + def resolve(self, context, provider): + pass + + def set_outputs(self, outputs): + self.outputs = outputs diff --git a/stacker/tests/actions/test_build.py b/stacker/tests/actions/test_build.py index 018101401..4e1be9a41 100644 --- a/stacker/tests/actions/test_build.py +++ b/stacker/tests/actions/test_build.py @@ -157,28 +157,28 @@ def test_execute_plan_when_outline_not_specified(self): self.assertEqual(mock_generate_plan().execute.call_count, 1) def test_should_update(self): - test_scenario = namedtuple("test_scenario", - ["locked", "force", "result"]) test_scenarios = ( - test_scenario(locked=False, force=False, result=True), - test_scenario(locked=False, force=True, result=True), - test_scenario(locked=True, force=False, result=False), - test_scenario(locked=True, force=True, result=True) + dict(blueprint=None, locked=False, force=False, result=False), + dict(blueprint="BLUEPRINT", locked=False, force=False, + result=True), + dict(blueprint="BLUEPRINT", locked=False, force=True, result=True), + dict(blueprint="BLUEPRINT", locked=True, force=False, + result=False), + dict(blueprint="BLUEPRINT", locked=True, force=True, result=True) ) - mock_stack = mock.MagicMock(["locked", "force", "name"]) - mock_stack.name = "test-stack" for t in test_scenarios: - mock_stack.locked = t.locked - mock_stack.force = t.force - self.assertEqual(build.should_update(mock_stack), t.result) + mock_stack = mock.MagicMock( + ["blueprint", "locked", "force", "name"], + name='test-stack', **t) + self.assertEqual(build.should_update(mock_stack), t['result']) def test_should_ensure_cfn_bucket(self): test_scenarios = [ - {"outline": False, "dump": False, "result": True}, - {"outline": True, "dump": False, "result": False}, - {"outline": False, "dump": True, "result": False}, - {"outline": True, "dump": True, "result": False}, - {"outline": True, "dump": "DUMP", "result": False} + dict(outline=False, dump=False, result=True), + dict(outline=True, dump=False, result=False), + dict(outline=False, dump=True, result=False), + dict(outline=True, dump=True, result=False), + dict(outline=True, dump="DUMP", result=False) ] for scenario in test_scenarios: @@ -193,18 +193,17 @@ def test_should_ensure_cfn_bucket(self): raise def test_should_submit(self): - test_scenario = namedtuple("test_scenario", - ["enabled", "result"]) test_scenarios = ( - test_scenario(enabled=False, result=False), - test_scenario(enabled=True, result=True), + dict(blueprint=None, enabled=False, result=True), + dict(blueprint="BLUEPRINT", enabled=False, result=False), + dict(blueprint="BLUEPRINT", enabled=True, result=True), ) - mock_stack = mock.MagicMock(["enabled", "name"]) - mock_stack.name = "test-stack" for t in test_scenarios: - mock_stack.enabled = t.enabled - self.assertEqual(build.should_submit(mock_stack), t.result) + mock_stack = mock.MagicMock( + ["blueprint", "enabled", "name"], + name='test-stack', **t) + self.assertEqual(build.should_submit(mock_stack), t['result']) class TestLaunchStack(TestBuildAction): From f6148857870cd15d1a8a6f2673a4061195d6737e Mon Sep 17 00:00:00 2001 From: Daniel Miranda Date: Sun, 1 Jul 2018 19:59:38 -0300 Subject: [PATCH 03/10] config: add `fqn` option to external stacks --- stacker/config/__init__.py | 8 ++++++++ stacker/stack.py | 5 +++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/stacker/config/__init__.py b/stacker/config/__init__.py index 1fee277c6..63d9f2b44 100644 --- a/stacker/config/__init__.py +++ b/stacker/config/__init__.py @@ -307,6 +307,8 @@ class BaseStack(Model): class ExternalStack(BaseStack): + fqn = StringType(serialize_when_none=False) + @classmethod def _claim_polymorphic(cls, value): if value.get("external", False) is True: @@ -316,6 +318,12 @@ def __init__(self, *args, **kwargs): super(ExternalStack, self).__init__(*args, **kwargs) self.external = True + def validate_fqn(self, data, value): + if (not value and not data["stack_name"]) or \ + (value and data["stack_name"]): + raise ValidationError("Exactly one of `fqn` and `stack_name` must " + "be provided for external stacks") + class Stack(BaseStack): class_path = StringType(serialize_when_none=False) diff --git a/stacker/stack.py b/stacker/stack.py index 88cad1280..aba91ec3d 100644 --- a/stacker/stack.py +++ b/stacker/stack.py @@ -201,7 +201,7 @@ class ExternalStack(Stack): """Represents gathered information about an existing external stack Args: - definition (:class:`stacker.config.Stack`): A stack definition. + definition (:class:`stacker.config.ExternalStack`): A stack definition. context (:class:`stacker.context.Context`): Current context for building the stack. @@ -209,7 +209,8 @@ class ExternalStack(Stack): def __init__(self, definition, context): self.name = definition.name - self.fqn = definition.stack_name + stack_name = definition.stack_name or self.name + self.fqn = definition.fqn or context.get_fqn(stack_name) self.region = definition.region self.profile = definition.profile self.definition = definition From fc868ad1aaf84a3fa9e18fc43d75fb3f0d917243 Mon Sep 17 00:00:00 2001 From: Daniel Miranda Date: Sun, 1 Jul 2018 20:00:02 -0300 Subject: [PATCH 04/10] tests: add functional tests for external stacks --- stacker/tests/actions/test_build.py | 3 +- stacker/tests/fixtures/mock_blueprints.py | 2 + .../34_stacker_build-external-stacks.bats | 74 +++++++++++++++++++ 3 files changed, 77 insertions(+), 2 deletions(-) create mode 100644 tests/test_suite/34_stacker_build-external-stacks.bats diff --git a/stacker/tests/actions/test_build.py b/stacker/tests/actions/test_build.py index 4e1be9a41..d21950def 100644 --- a/stacker/tests/actions/test_build.py +++ b/stacker/tests/actions/test_build.py @@ -3,13 +3,11 @@ from __future__ import absolute_import from builtins import str import unittest -from collections import namedtuple import mock from stacker import exceptions from stacker.actions import build -from stacker.session_cache import get_session from stacker.actions.build import ( _resolve_parameters, _handle_missing_parameters, @@ -20,6 +18,7 @@ from stacker.exceptions import StackDidNotChange, StackDoesNotExist from stacker.providers.base import BaseProvider from stacker.providers.aws.default import Provider +from stacker.session_cache import get_session from stacker.status import ( NotSubmittedStatus, COMPLETE, diff --git a/stacker/tests/fixtures/mock_blueprints.py b/stacker/tests/fixtures/mock_blueprints.py index adfbd0f4c..d8a7484b9 100644 --- a/stacker/tests/fixtures/mock_blueprints.py +++ b/stacker/tests/fixtures/mock_blueprints.py @@ -181,8 +181,10 @@ class Dummy(Blueprint): } def create_template(self): + input = self.get_variables()["StringVariable"] self.template.add_resource(WaitConditionHandle("Dummy")) self.template.add_output(Output("DummyId", Value="dummy-1234")) + self.template.add_output(Output("StringOutput", Value=input)) self.template.add_output(Output("Region", Value=Ref("AWS::Region"))) diff --git a/tests/test_suite/34_stacker_build-external-stacks.bats b/tests/test_suite/34_stacker_build-external-stacks.bats new file mode 100644 index 000000000..215a3f605 --- /dev/null +++ b/tests/test_suite/34_stacker_build-external-stacks.bats @@ -0,0 +1,74 @@ +#!/usr/bin/env bats + +load ../test_helper + +@test "stacker build - external stack" { + needs_aws + + config1() { + cat <- + \${output vpc/west::Region} + \${output vpc/east::Region} +EOF + } + + teardown() { + stacker destroy --force <(config2) + stacker destroy --force <(config1) + } + + # Create the new stacks. + stacker build <(config1) + assert "$status" -eq 0 + assert_has_line "Using default AWS provider mode" + assert_has_line "vpc/west: submitted (creating new stack)" + assert_has_line "vpc/west: complete (creating new stack)" + assert_has_line "vpc/east: submitted (creating new stack)" + assert_has_line "vpc/east: complete (creating new stack)" + + stacker build <(config2) + assert "$status" -eq 0 + assert_has_line "Using default AWS provider mode" + assert_has_line "vpc/combo: submitted (creating new stack)" + assert_has_line "vpc/combo: complete (creating new stack)" + + stacker info <(config2) + assert_has_line "StringOutput: us-west-1 us-east-1" + +} From d0ee28e3fb988bbc702f5e15f6d6d05103801a88 Mon Sep 17 00:00:00 2001 From: Daniel Miranda Date: Mon, 2 Jul 2018 00:15:23 -0300 Subject: [PATCH 05/10] Move should_update and should_submit methods to Stack subclasses This makes handling the difference between managed and external stacks much easier. Also clean up the build tests a little bit by creating a TestStack class instead of using Mocks. --- stacker/actions/build.py | 51 ++------------------------------- stacker/actions/diff.py | 4 +-- stacker/stack.py | 56 ++++++++++++++++++++++++++++++++----- stacker/tests/factories.py | 11 +++++++- stacker/tests/test_stack.py | 44 +++++++++++++++++++++++++++-- 5 files changed, 105 insertions(+), 61 deletions(-) diff --git a/stacker/actions/build.py b/stacker/actions/build.py index 77732ba4b..41f54cf19 100644 --- a/stacker/actions/build.py +++ b/stacker/actions/build.py @@ -38,53 +38,6 @@ def build_stack_tags(stack): return [{'Key': t[0], 'Value': t[1]} for t in stack.tags.items()] -def should_update(stack): - """Tests whether a stack should be submitted for updates to CF. - - Args: - stack (:class:`stacker.stack.Stack`): The stack object to check. - - Returns: - bool: If the stack should be updated, return True. - - """ - - if not stack.blueprint: - return False - - if stack.locked: - if not stack.force: - logger.debug("Stack %s locked and not in --force list. " - "Refusing to update.", stack.name) - return False - else: - logger.debug("Stack %s locked, but is in --force " - "list.", stack.name) - return True - - -def should_submit(stack): - """Tests whether a stack should be submitted to CF for update/create - - Args: - stack (:class:`stacker.stack.Stack`): The stack object to check. - - Returns: - bool: If the stack should be submitted, return True. - - """ - - # Submit stack without a blueprint just to grab outputs - if not stack.blueprint: - return True - - if stack.enabled: - return True - - logger.debug("Stack %s is not enabled. Skipping.", stack.name) - return False - - def should_ensure_cfn_bucket(outline, dump): """Test whether access to the cloudformation template bucket is required @@ -271,7 +224,7 @@ def _launch_stack(self, stack, **kwargs): if self.cancel.wait(wait_time): return INTERRUPTED - if not should_submit(stack): + if not stack.should_submit(): return NotSubmittedStatus() provider = self.build_provider(stack) @@ -281,7 +234,7 @@ def _launch_stack(self, stack, **kwargs): except StackDoesNotExist: provider_stack = None - if provider_stack and not should_update(stack): + if provider_stack and not stack.should_update(): stack.set_outputs( self.provider.get_output_dict(provider_stack)) return NotUpdatedStatus() diff --git a/stacker/actions/diff.py b/stacker/actions/diff.py index 97801ae7d..c1a5755e5 100644 --- a/stacker/actions/diff.py +++ b/stacker/actions/diff.py @@ -222,10 +222,10 @@ def _diff_stack(self, stack, **kwargs): if self.cancel.wait(0): return INTERRUPTED - if not build.should_submit(stack): + if not stack.should_submit(): return NotSubmittedStatus() - if not build.should_update(stack): + if not stack.should_update(): return NotUpdatedStatus() provider = self.build_provider(stack) diff --git a/stacker/stack.py b/stacker/stack.py index aba91ec3d..9ea979cd0 100644 --- a/stacker/stack.py +++ b/stacker/stack.py @@ -3,6 +3,7 @@ from __future__ import absolute_import from builtins import object import copy +import logging from . import util from .variables import ( @@ -12,6 +13,8 @@ from .blueprints.raw import RawTemplateBlueprint +logger = logging.getLogger(__name__) + def _gather_variables(stack_def): """Merges context provided & stack defined variables. @@ -28,13 +31,15 @@ def _gather_variables(stack_def): - variable defined within the stack definition Args: - stack_def (dict): The stack definition being worked on. + stack_def (:class:`stacker.config.Stack`): The stack definition being + worked on. Returns: - dict: Contains key/value pairs of the collected variables. + :obj:`list` of :class:`stacker.variables.Variable`: Contains key/value + pairs of the collected variables. Raises: - AttributeError: Raised when the stack definitition contains an invalid + AttributeError: Raised when the stack definition contains an invalid attribute. Currently only when using old parameters, rather than variables. """ @@ -43,18 +48,15 @@ def _gather_variables(stack_def): class Stack(object): - """Represents gathered information about a stack to be built/updated. Args: definition (:class:`stacker.config.Stack`): A stack definition. context (:class:`stacker.context.Context`): Current context for building the stack. - mappings (dict, optional): Cloudformation mappings passed to the + mappings (dict, optional): CloudFormation mappings passed to the blueprint. - locked (bool, optional): Whether or not the stack is locked. force (bool, optional): Whether to force updates on this stack. - enabled (bool, optional): Whether this stack is enabled """ @@ -196,6 +198,38 @@ def resolve(self, context, provider): def set_outputs(self, outputs): self.outputs = outputs + def should_submit(self): + """Tests whether this stack should be submitted to CF + + Returns: + bool: If the stack should be submitted, return True. + + """ + + if self.enabled: + return True + + logger.debug("Stack %s is not enabled. Skipping.", self.name) + return False + + def should_update(self): + """Tests whether this stack should be submitted for updates to CF. + + Returns: + bool: If the stack should be updated, return True. + + """ + + if self.locked: + if not self.force: + logger.debug("Stack %s locked and not in --force list. " + "Refusing to update.", self.name) + return False + else: + logger.debug("Stack %s locked, but is in --force " + "list.", self.name) + return True + class ExternalStack(Stack): """Represents gathered information about an existing external stack @@ -246,3 +280,11 @@ def resolve(self, context, provider): def set_outputs(self, outputs): self.outputs = outputs + + def should_submit(self): + # Always submit this stack to load outputs + return True + + def should_update(self): + # Never update an external stack + return False diff --git a/stacker/tests/factories.py b/stacker/tests/factories.py index f930c5177..494f9e06f 100644 --- a/stacker/tests/factories.py +++ b/stacker/tests/factories.py @@ -5,7 +5,7 @@ from mock import MagicMock from stacker.context import Context -from stacker.config import Config, Stack +from stacker.config import Config, Stack, ExternalStack from stacker.lookups import Lookup @@ -53,6 +53,15 @@ def generate_definition(base_name, stack_id, **overrides): return Stack(definition) +def generate_external_definition(base_name, stack_id, **overrides): + definition = { + "name": "%s.%d" % (base_name, stack_id), + "external": True, + } + definition.update(overrides) + return ExternalStack(definition) + + def mock_lookup(lookup_input, lookup_type, raw=None): if raw is None: raw = "%s %s" % (lookup_type, lookup_input) diff --git a/stacker/tests/test_stack.py b/stacker/tests/test_stack.py index c1bba0156..e4df97bc8 100644 --- a/stacker/tests/test_stack.py +++ b/stacker/tests/test_stack.py @@ -7,8 +7,8 @@ from stacker.lookups import register_lookup_handler from stacker.context import Context from stacker.config import Config -from stacker.stack import Stack -from .factories import generate_definition +from stacker.stack import Stack, ExternalStack +from .factories import generate_definition, generate_external_definition class TestStack(unittest.TestCase): @@ -133,6 +133,46 @@ def test_stack_tags_extra(self): stack = Stack(definition=definition, context=self.context) self.assertEquals(stack.tags, {"environment": "prod", "app": "graph"}) + def test_stack_should_update(self): + test_scenarios = [ + dict(locked=False, force=False, result=True), + dict(locked=False, force=True, result=True), + dict(locked=True, force=False, result=False), + dict(locked=True, force=True, result=True) + ] + + for t in test_scenarios: + definition = generate_definition( + base_name="vpc", + stack_id=1, + locked=t['locked']) + stack = Stack(definition=definition, context=self.context, + force=t['force']) + self.assertEqual(stack.should_update(), t['result']) + + def test_stack_should_submit(self): + for enabled in (True, False): + definition = generate_definition( + base_name="vpc", + stack_id=1, + enabled=enabled) + stack = Stack(definition=definition, context=self.context) + self.assertEqual(stack.should_submit(), enabled) + + def test_external_stack_should_update(self): + definition = generate_external_definition( + base_name="vpc", + stack_id=1) + stack = ExternalStack(definition=definition, context=self.context) + self.assertEqual(stack.should_update(), False) + + def test_external_stack_should_submit(self): + definition = generate_external_definition( + base_name="vpc", + stack_id=1) + stack = ExternalStack(definition=definition, context=self.context) + self.assertEqual(stack.should_submit(), True) + if __name__ == '__main__': unittest.main() From 19a038689ee68f2a307cd64b4ef003fbd916deff Mon Sep 17 00:00:00 2001 From: Daniel Miranda Date: Mon, 2 Jul 2018 00:15:55 -0300 Subject: [PATCH 06/10] stack: remove redundant parameters from constructor --- stacker/context.py | 5 +---- stacker/stack.py | 9 ++++----- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/stacker/context.py b/stacker/context.py index eb9b57686..36c414c9c 100644 --- a/stacker/context.py +++ b/stacker/context.py @@ -163,10 +163,7 @@ def get_stacks(self): definition=stack_def, context=self, mappings=self.mappings, - force=stack_def.name in self.force_stacks, - locked=stack_def.locked, - enabled=stack_def.enabled, - protected=stack_def.protected, + force=stack_def.name in self.force_stacks ) stacks.append(stack) self._stacks = stacks diff --git a/stacker/stack.py b/stacker/stack.py index 9ea979cd0..c4a5f76fc 100644 --- a/stacker/stack.py +++ b/stacker/stack.py @@ -60,20 +60,19 @@ class Stack(object): """ - def __init__(self, definition, context, variables=None, mappings=None, - locked=False, force=False, enabled=True, protected=False): + def __init__(self, definition, context, mappings=None, force=False): self.logging = True self.name = definition.name self.fqn = context.get_fqn(definition.stack_name or self.name) self.region = definition.region self.profile = definition.profile + self.locked = definition.locked + self.enabled = definition.enabled + self.protected = definition.protected self.definition = definition self.variables = _gather_variables(definition) self.mappings = mappings - self.locked = locked self.force = force - self.enabled = enabled - self.protected = protected self.context = context self.outputs = None self.in_progress_behavior = definition.in_progress_behavior From 9a7e88927d89e37ad9096cc0fde4f5d5aef79670 Mon Sep 17 00:00:00 2001 From: Daniel Miranda Date: Mon, 2 Jul 2018 00:16:15 -0300 Subject: [PATCH 07/10] session_cache: clarify arg types --- stacker/session_cache.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stacker/session_cache.py b/stacker/session_cache.py index 9442d4726..b7e76d578 100644 --- a/stacker/session_cache.py +++ b/stacker/session_cache.py @@ -21,8 +21,8 @@ def get_session(region, profile=None): """Creates a boto3 session with a cache Args: - region (str): The region for the session - profile (str): The profile for the session + region (str, optional): The region for the session + profile (str, optional): The profile for the session Returns: :class:`boto3.session.Session`: A boto3 session with From 69787169c6df944c6e65fd3a873acaa94e5afd83 Mon Sep 17 00:00:00 2001 From: Daniel Miranda Date: Wed, 4 Jul 2018 15:18:17 -0300 Subject: [PATCH 08/10] config: validate FQN correctly in ExternalStack --- stacker/config/__init__.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/stacker/config/__init__.py b/stacker/config/__init__.py index 63d9f2b44..d0c9b8df0 100644 --- a/stacker/config/__init__.py +++ b/stacker/config/__init__.py @@ -319,9 +319,8 @@ def __init__(self, *args, **kwargs): self.external = True def validate_fqn(self, data, value): - if (not value and not data["stack_name"]) or \ - (value and data["stack_name"]): - raise ValidationError("Exactly one of `fqn` and `stack_name` must " + if value and data["stack_name"]: + raise ValidationError("At most one of `fqn` and `stack_name` must " "be provided for external stacks") From 8e36bf6d7dcad22a001d5fab49c6222d1f845a60 Mon Sep 17 00:00:00 2001 From: Daniel Miranda Date: Sun, 14 Oct 2018 16:52:29 -0300 Subject: [PATCH 09/10] Update Build Action tests for external stack changes --- stacker/plan.py | 2 +- stacker/tests/actions/test_build.py | 65 +++++++++++------------------ 2 files changed, 25 insertions(+), 42 deletions(-) diff --git a/stacker/plan.py b/stacker/plan.py index 24b415e04..80f394384 100644 --- a/stacker/plan.py +++ b/stacker/plan.py @@ -34,7 +34,7 @@ def log_step(step): - msg = "%s: %s" % (step, step.status.name) + msg = "%s: %s" % (step.name, step.status.name) if step.status.reason: msg += " (%s)" % (step.status.reason) color_code = COLOR_CODES.get(step.status.code, 37) diff --git a/stacker/tests/actions/test_build.py b/stacker/tests/actions/test_build.py index d21950def..7e1b7ba17 100644 --- a/stacker/tests/actions/test_build.py +++ b/stacker/tests/actions/test_build.py @@ -13,12 +13,14 @@ _handle_missing_parameters, UsePreviousParameterValue, ) +from stacker.blueprints.base import Blueprint from stacker.blueprints.variables.types import CFNString from stacker.context import Context, Config from stacker.exceptions import StackDidNotChange, StackDoesNotExist from stacker.providers.base import BaseProvider from stacker.providers.aws.default import Provider from stacker.session_cache import get_session +from stacker.stack import Stack from stacker.status import ( NotSubmittedStatus, COMPLETE, @@ -28,7 +30,11 @@ FAILED ) -from ..factories import MockThreadingEvent, MockProviderBuilder +from ..factories import ( + MockThreadingEvent, + MockProviderBuilder, + generate_definition +) def mock_stack_parameters(parameters): @@ -155,22 +161,6 @@ def test_execute_plan_when_outline_not_specified(self): build_action.run(outline=False) self.assertEqual(mock_generate_plan().execute.call_count, 1) - def test_should_update(self): - test_scenarios = ( - dict(blueprint=None, locked=False, force=False, result=False), - dict(blueprint="BLUEPRINT", locked=False, force=False, - result=True), - dict(blueprint="BLUEPRINT", locked=False, force=True, result=True), - dict(blueprint="BLUEPRINT", locked=True, force=False, - result=False), - dict(blueprint="BLUEPRINT", locked=True, force=True, result=True) - ) - for t in test_scenarios: - mock_stack = mock.MagicMock( - ["blueprint", "locked", "force", "name"], - name='test-stack', **t) - self.assertEqual(build.should_update(mock_stack), t['result']) - def test_should_ensure_cfn_bucket(self): test_scenarios = [ dict(outline=False, dump=False, result=True), @@ -191,21 +181,18 @@ def test_should_ensure_cfn_bucket(self): e.args += ("scenario", str(scenario)) raise - def test_should_submit(self): - test_scenarios = ( - dict(blueprint=None, enabled=False, result=True), - dict(blueprint="BLUEPRINT", enabled=False, result=False), - dict(blueprint="BLUEPRINT", enabled=True, result=True), - ) - for t in test_scenarios: - mock_stack = mock.MagicMock( - ["blueprint", "enabled", "name"], - name='test-stack', **t) - self.assertEqual(build.should_submit(mock_stack), t['result']) +class TestLaunchStack(TestBuildAction): + def _get_stack(self): + stack = Stack(definition=generate_definition("vpc", 1), + context=self.context,) + blueprint_mock = mock.patch.object(type(stack), 'blueprint', + spec=Blueprint, rendered='{}') + self.addCleanup(blueprint_mock.stop) + blueprint_mock.start() + return stack -class TestLaunchStack(TestBuildAction): def setUp(self): self.context = self._get_context() self.session = get_session(region=None) @@ -215,13 +202,7 @@ def setUp(self): self.build_action = build.Action(self.context, provider_builder=provider_builder, cancel=MockThreadingEvent()) - - self.stack = mock.MagicMock() - self.stack.region = None - self.stack.name = 'vpc' - self.stack.fqn = 'vpc' - self.stack.blueprint.rendered = '{}' - self.stack.locked = False + self.stack = self._get_stack() self.stack_status = None plan = self.build_action._generate_plan() @@ -233,14 +214,16 @@ def patch_object(*args, **kwargs): self.addCleanup(m.stop) m.start() - def get_stack(name, *args, **kwargs): - if name != self.stack.name or not self.stack_status: - raise StackDoesNotExist(name) + def get_stack(fqn, *args, **kwargs): + if fqn != self.stack.fqn or not self.stack_status: + raise StackDoesNotExist(fqn) + tags = [{'Key': key, 'Value': value} + for (key, value) in self.stack.tags.items()] return {'StackName': self.stack.name, 'StackStatus': self.stack_status, - 'Outputs': [], - 'Tags': []} + 'Outputs': {}, + 'Tags': tags} def get_events(name, *args, **kwargs): return [{'ResourceStatus': 'ROLLBACK_IN_PROGRESS', From 2ce72e2bbd323bf1d468cd23cf79a2f6e82a090f Mon Sep 17 00:00:00 2001 From: Daniel Miranda Date: Sun, 10 Mar 2019 04:40:50 -0300 Subject: [PATCH 10/10] docs: document external stacks --- docs/config.rst | 117 ++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 93 insertions(+), 24 deletions(-) diff --git a/docs/config.rst b/docs/config.rst index a6804926b..93c32d683 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -334,13 +334,49 @@ stacks that will be deployed in the environment. The top level keyword *stacks* is populated with a list of dictionaries, each representing a single stack to be built. -A stack has the following keys: +Stacks are managed by Stacker by default. They will be created and updated as +needed using the template file or blueprint class specified. Read-only +external stacks can also be defined if complex cross-references are needed +involving multiple accounts or regions. + +The following options are available for all stacks: **name:** The logical name for this stack, which can be used in conjuction with the ``output`` lookup. The value here must be unique within the config. If no ``stack_name`` is provided, the value here will be used for the name of the CloudFormation stack. +**stack_name:** + (optional) If provided, this will be used as the name of the CloudFormation + stack. Unlike ``name``, the value doesn't need to be unique within the config, + since you could have multiple stacks with the same name, but in different + regions or accounts. (note: the namespace from the environment will be + prepended to this) +**region**: + (optional): If provided, specifies the name of the region that the + CloudFormation stack should reside in. If not provided, the default region + will be used (``AWS_DEFAULT_REGION``, ``~/.aws/config`` or the ``--region`` + flag). If both ``region`` and ``profile`` are specified, the value here takes + precedence over the value in the profile. +**profile**: + (optional): If provided, specifies the name of a AWS profile to use when + performing AWS API calls for this stack. This can be used to provision stacks + in multiple accounts or regions. +**external**: + (optional): If set to true, this stack is considered read-only, will not be + modified by Stacker, and most of the options related to stack deployment + should be omitted. + +The following options are available for external stacks: + +**fqn**: + (optional): Fully-qualified physical name of the stack to be loaded from + CloudFormation. Use instead of ``stack_name`` if the stack lies in a + different namespace, as this value *does not* get the namespace applied to + it. + +The following options are available for managed stacks: + **class_path:** The python class path to the Blueprint to be used. Specify this or ``template_path`` for the stack. @@ -350,7 +386,6 @@ A stack has the following keys: working directory (e.g. templates stored alongside the Config), or relative to a directory in the python ``sys.path`` (i.e. for loading templates retrieved via ``packages_sources``). - **description:** A short description to apply to the stack. This overwrites any description provided in the Blueprint. See: http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/template-description-structure.html @@ -363,8 +398,10 @@ A stack has the following keys: updated unless the stack is passed to stacker via the *--force* flag. This is useful for *risky* stacks that you don't want to take the risk of allowing CloudFormation to update, but still want to make - sure get launched when the environment is first created. When ``locked``, - it's not necessary to specify a ``class_path`` or ``template_path``. + sure get launched when the environment is first created. + Note: when ``locked``, it's not necessary to specify a ``class_path`` + or ``template_path``, but this functionality is deprecated in favour of + ``external``. **enabled:** (optional) If set to false, the stack is disabled, and will not be built or updated. This can allow you to disable stacks in different @@ -383,22 +420,6 @@ A stack has the following keys: **tags:** (optional) a dictionary of CloudFormation tags to apply to this stack. This will be combined with the global tags, but these tags will take precendence. -**stack_name:** - (optional) If provided, this will be used as the name of the CloudFormation - stack. Unlike ``name``, the value doesn't need to be unique within the config, - since you could have multiple stacks with the same name, but in different - regions or accounts. (note: the namespace from the environment will be - prepended to this) -**region**: - (optional): If provided, specifies the name of the region that the - CloudFormation stack should reside in. If not provided, the default region - will be used (``AWS_DEFAULT_REGION``, ``~/.aws/config`` or the ``--region`` - flag). If both ``region`` and ``profile`` are specified, the value here takes - precedence over the value in the profile. -**profile**: - (optional): If provided, specifies the name of a AWS profile to use when - performing AWS API calls for this stack. This can be used to provision stacks - in multiple accounts or regions. **stack_policy_path**: (optional): If provided, specifies the path to a JSON formatted stack policy that will be applied when the CloudFormation stack is created and updated. @@ -411,13 +432,20 @@ A stack has the following keys: option to `wait` and stacker will wait for the previous update to complete before attempting to update the stack. -Stacks Example -~~~~~~~~~~~~~~ +Examples +~~~~~~~~ + +VPC + Instances +::::::::::::::: -Here's an example from stacker_blueprints_, used to create a VPC:: +Here's an example from stacker_blueprints_, used to create a VPC and and two EC2 +Instances:: + + namespace: example stacks: - - name: vpc-example + - name: vpc + stack_name: test-vpc class_path: stacker_blueprints.vpc.VPC locked: false enabled: true @@ -438,6 +466,47 @@ Here's an example from stacker_blueprints_, used to create a VPC:: - 10.128.20.0/22 CidrBlock: 10.128.0.0/16 + - name: instances + stack_name: + class_path: stacker_blueprints.ec2.Instances + enabled: true + variables: + SmallInstance: + InstanceType: t2.small + ImageId: &amazon_linux_ami "${ami owners:amazon name_regex:amzn-ami-hvm-2018.03.*-x86_64-gp2}" + AvailabilityZone: ${output vpc::AvailabilityZone0} + SubnetId: ${output vpc::PublicSubnet0} + LargeInstance: + InstanceType: m5.xlarge + ImageId: *amazon_linux_ami + AvailabilityZone: ${output vpc::AvailabilityZone1} + SubnetId: ${output vpc::PublicSubnet1} + + +Referencing External Stacks +::::::::::::::::::::::::::: + +This example creates a security group in VPC from the previous example by +importing it as an external stack with a custom profile:: + + namespace: other-example + stacks: + - name: vpc + fqn: example-test-vpc + profile: custom-profile + external: yes + + - name: sg + class_path: stacker_blueprints.ec2.SecurityGroups + variables: + SecurityGroups: + VpcId: ${output vpc::VpcId} + SecurityGroupIngress: + - CidrIp: 0.0.0.0/0 + FromPort: 22 + ToPort: 22 + IpProtocol: tcp + Targets -------