[WIP/RFC] Integrate hooks into stack execution graph

setup.py

@@ -27,7 +27,7 @@
     "mock~=2.0",
     "moto~=1.3.7",
     "testfixtures~=4.10.0",
-    "flake8-future-import",
+    "flake8-future-import"
 ]
 
 scripts = [

stacker/actions/base.py

@@ -8,20 +8,15 @@
 import threading
 
 from ..dag import walk, ThreadedWalker, UnlimitedSemaphore
-from ..plan import Step, build_plan, build_graph
+from ..plan import Graph, Plan, Step
+from ..target import Target
 
 import botocore.exceptions
 from stacker.session_cache import get_session
 from stacker.exceptions import PlanFailed
+from stacker.status import COMPLETE
+from stacker.util import ensure_s3_bucket, get_s3_endpoint
 
-from ..status import (
-    COMPLETE
-)
-
-from stacker.util import (
-    ensure_s3_bucket,
-    get_s3_endpoint,
-)
 
 logger = logging.getLogger(__name__)
 
@@ -61,42 +56,6 @@ def build_walker(concurrency):
     return ThreadedWalker(semaphore).walk
 
 
-def plan(description, stack_action, context,
-         tail=None, reverse=False):
-    """A simple helper that builds a graph based plan from a set of stacks.
-
-    Args:
-        description (str): a description of the plan.
-        action (func): a function to call for each stack.
-        context (:class:`stacker.context.Context`): a
-            :class:`stacker.context.Context` to build the plan from.
-        tail (func): an optional function to call to tail the stack progress.
-        reverse (bool): if True, execute the graph in reverse (useful for
-            destroy actions).
-
-    Returns:
-        :class:`plan.Plan`: The resulting plan object
-    """
-
-    def target_fn(*args, **kwargs):
-        return COMPLETE
-
-    steps = [
-        Step(stack, fn=stack_action, watch_func=tail)
-        for stack in context.get_stacks()]
-
-    steps += [
-        Step(target, fn=target_fn) for target in context.get_targets()]
-
-    graph = build_graph(steps)
-
-    return build_plan(
-        description=description,
-        graph=graph,
-        targets=context.stack_names,
-        reverse=reverse)
-
-
 def stack_template_key_name(blueprint):
     """Given a blueprint, produce an appropriate key name.
 
@@ -156,6 +115,119 @@ def __init__(self, context, provider_builder=None, cancel=None):
             self.bucket_region = provider_builder.region
         self.s3_conn = get_session(self.bucket_region).client('s3')
 
+    def plan(self, description, action_name, action, context, tail=None,
+             reverse=False, run_hooks=True):
+        """A helper that builds a graph based plan from a set of stacks.
+
+        Args:
+            description (str): a description of the plan.
+            action_name (str): name of the action being run. Used to generate
+                target names and filter out which hooks to run.
+            action (func): a function to call for each stack.
+            context (stacker.context.Context): a context to build the plan
+                from.
+            tail (func): an optional function to call to tail the stack
+                progress.
+            reverse (bool): whether to flip the direction of dependencies.
+                Use it when planning an action for destroying resources,
+                which usually must happen in the reverse order of creation.
+                Note: this does not change the order of execution of pre/post
+                action hooks,  as the build and destroy hooks are currently
+                configured in separate.
+            run_hooks (bool): whether to run hooks configured for this action
+
+        Returns: stacker.plan.Plan: the resulting plan for this action
+        """
+
+        def target_fn(*args, **kwargs):
+            return COMPLETE
+
+        def hook_fn(hook, *args, **kwargs):
+            return hook.run_step(provider_builder=self.provider_builder,
+                                 context=self.context)
+
+        pre_hooks_target = Target(
+            name="pre_{}_hooks".format(action_name))
+        pre_action_target = Target(
+            name="pre_{}".format(action_name),
+            requires=[pre_hooks_target.name])
+        action_target = Target(
+            name=action_name,
+            requires=[pre_action_target.name])
+        post_action_target = Target(
+            name="post_{}".format(action_name),
+            requires=[action_target.name])
+        post_hooks_target = Target(
+            name="post_{}_hooks".format(action_name),
+            requires=[post_action_target.name])
+
+        def steps():
+            yield Step.from_target(pre_hooks_target, fn=target_fn)
+            yield Step.from_target(pre_action_target, fn=target_fn)
+            yield Step.from_target(action_target, fn=target_fn)
+            yield Step.from_target(post_action_target, fn=target_fn)
+            yield Step.from_target(post_hooks_target, fn=target_fn)
+
+            if run_hooks:
+                # Since we need to maintain compatibility with legacy hooks,
+                # we separate them completely from the new hooks.
+                # The legacy hooks will run in two separate phases, completely
+                # isolated from regular stacks and targets, and any of the new
+                # hooks.
+                # Hence, all legacy pre-hooks will finish before any of the
+                # new hooks, and all legacy post-hooks will only start after
+                # the new hooks.
+
+                hooks = self.context.get_hooks_for_action(action_name)
+                logger.debug("Found hooks for action {}: {}".format(
+                    action_name, hooks))
+
+                for hook in hooks.pre:
+                    yield Step.from_hook(
+                        hook, fn=hook_fn,
+                        required_by=[pre_hooks_target.name])
+
+                for hook in hooks.custom:
+                    step = Step.from_hook(
+                        hook, fn=hook_fn)
+                    if reverse:
+                        step.reverse_requirements()
+
+                    step.requires.add(pre_action_target.name)
+                    step.required_by.add(post_action_target.name)
+                    yield step
+
+                for hook in hooks.post:
+                    yield Step.from_hook(
+                        hook, fn=hook_fn,
+                        requires=[post_hooks_target.name])
+
+            for target in context.get_targets():
+                step = Step.from_target(target, fn=target_fn)
+                if reverse:
+                    step.reverse_requirements()
+
+                yield step
+
+            for stack in context.get_stacks():
+                step = Step.from_stack(stack, fn=action, watch_func=tail)
+                if reverse:
+                    step.reverse_requirements()
+
+                # Contain stack execution in the boundaries of the pre_action
+                # and post_action targets.
+                step.requires.add(pre_action_target.name)
+                step.required_by.add(action_target.name)
+
+                yield step
+
+        graph = Graph.from_steps(list(steps()))
+
+        return Plan.from_graph(
+            description=description,
+            graph=graph,
+            targets=context.stack_names)
+
     def ensure_cfn_bucket(self):
         """The CloudFormation bucket where templates will be stored."""
         if self.bucket_name:

stacker/actions/build.py

@@ -3,11 +3,10 @@
 from __future__ import absolute_import
 import logging
 
-from .base import BaseAction, plan, build_walker
+from .base import BaseAction, build_walker
 from .base import STACK_POLL_TIME
 
 from ..providers.base import Template
-from .. import util
 from ..exceptions import (
     MissingParameterException,
     StackDidNotChange,
@@ -181,29 +180,6 @@ def _handle_missing_parameters(parameter_values, all_params, required_params,
     return list(parameter_values.items())
 
 
-def handle_hooks(stage, hooks, provider, context, dump, outline):
-    """Handle pre/post hooks.
-
-    Args:
-        stage (str): The name of the hook stage - pre_build/post_build.
-        hooks (list): A list of dictionaries containing the hooks to execute.
-        provider (:class:`stacker.provider.base.BaseProvider`): The provider
-            the current stack is using.
-        context (:class:`stacker.context.Context`): The current stacker
-            context.
-        dump (bool): Whether running with dump set or not.
-        outline (bool): Whether running with outline set or not.
-
-    """
-    if not outline and not dump and hooks:
-        util.handle_hooks(
-            stage=stage,
-            hooks=hooks,
-            provider=provider,
-            context=context
-        )
-
-
 class Action(BaseAction):
     """Responsible for building & coordinating CloudFormation stacks.
 
@@ -273,8 +249,6 @@ def _launch_stack(self, stack, **kwargs):
             provider_stack = None
 
         if provider_stack and not should_update(stack):
-            stack.set_outputs(
-                self.provider.get_output_dict(provider_stack))
             return NotUpdatedStatus()
 
         recreate = False
@@ -316,8 +290,6 @@ def _launch_stack(self, stack, **kwargs):
                 return FailedStatus(reason)
 
             elif provider.is_stack_completed(provider_stack):
-                stack.set_outputs(
-                    provider.get_output_dict(provider_stack))
                 return CompleteStatus(old_status.reason)
             else:
                 return old_status
@@ -366,10 +338,8 @@ def _launch_stack(self, stack, **kwargs):
             else:
                 return SubmittedStatus("destroying stack for re-creation")
         except CancelExecution:
-            stack.set_outputs(provider.get_output_dict(provider_stack))
             return SkippedStatus(reason="canceled execution")
         except StackDidNotChange:
-            stack.set_outputs(provider.get_output_dict(provider_stack))
             return DidNotChangeStatus()
 
     def _template(self, blueprint):
@@ -391,26 +361,19 @@ def _stack_policy(self, stack):
         if stack.stack_policy:
             return Template(body=stack.stack_policy)
 
-    def _generate_plan(self, tail=False):
-        return plan(
+    def _generate_plan(self, tail=False, outline=False, dump=False):
+        return self.plan(
             description="Create/Update stacks",
-            stack_action=self._launch_stack,
+            action_name="build",
+            action=self._launch_stack,
             tail=self._tail_stack if tail else None,
-            context=self.context)
+            context=self.context,
+            run_hooks=not outline and not dump)
 
     def pre_run(self, outline=False, dump=False, *args, **kwargs):
         """Any steps that need to be taken prior to running the action."""
         if should_ensure_cfn_bucket(outline, dump):
             self.ensure_cfn_bucket()
-        hooks = self.context.config.pre_build
-        handle_hooks(
-            "pre_build",
-            hooks,
-            self.provider,
-            self.context,
-            dump,
-            outline
-        )
 
     def run(self, concurrency=0, outline=False,
             tail=False, dump=False, *args, **kwargs):
@@ -419,7 +382,7 @@ def run(self, concurrency=0, outline=False,
         This is the main entry point for the Builder.
 
         """
-        plan = self._generate_plan(tail=tail)
+        plan = self._generate_plan(tail=tail, outline=outline, dump=dump)
         if not plan.keys():
             logger.warn('WARNING: No stacks detected (error in config?)')
         if not outline and not dump:
@@ -433,15 +396,3 @@ def run(self, concurrency=0, outline=False,
             if dump:
                 plan.dump(directory=dump, context=self.context,
                           provider=self.provider)
-
-    def post_run(self, outline=False, dump=False, *args, **kwargs):
-        """Any steps that need to be taken after running the action."""
-        hooks = self.context.config.post_build
-        handle_hooks(
-            "post_build",
-            hooks,
-            self.provider,
-            self.context,
-            dump,
-            outline
-        )

stacker/actions/destroy.py

@@ -3,10 +3,9 @@
 from __future__ import absolute_import
 import logging
 
-from .base import BaseAction, plan, build_walker
+from .base import BaseAction, build_walker
 from .base import STACK_POLL_TIME
 from ..exceptions import StackDoesNotExist
-from .. import util
 from ..status import (
     CompleteStatus,
     SubmittedStatus,
@@ -37,12 +36,14 @@ class Action(BaseAction):
     """
 
     def _generate_plan(self, tail=False):
-        return plan(
+        return self.plan(
             description="Destroy stacks",
-            stack_action=self._destroy_stack,
+            action_name='destroy',
+            action=self._destroy_stack,
             tail=self._tail_stack if tail else None,
             context=self.context,
-            reverse=True)
+            reverse=True,
+            run_hooks=True)
 
     def _destroy_stack(self, stack, **kwargs):
         old_status = kwargs.get("status")
@@ -78,16 +79,6 @@ def _destroy_stack(self, stack, **kwargs):
             provider.destroy_stack(provider_stack)
         return DestroyingStatus
 
-    def pre_run(self, outline=False, *args, **kwargs):
-        """Any steps that need to be taken prior to running the action."""
-        pre_destroy = self.context.config.pre_destroy
-        if not outline and pre_destroy:
-            util.handle_hooks(
-                stage="pre_destroy",
-                hooks=pre_destroy,
-                provider=self.provider,
-                context=self.context)
-
     def run(self, force, concurrency=0, tail=False, *args, **kwargs):
         plan = self._generate_plan(tail=tail)
         if not plan.keys():
@@ -101,13 +92,3 @@ def run(self, force, concurrency=0, tail=False, *args, **kwargs):
         else:
             plan.outline(message="To execute this plan, run with \"--force\" "
                                  "flag.")
-
-    def post_run(self, outline=False, *args, **kwargs):
-        """Any steps that need to be taken after running the action."""
-        post_destroy = self.context.config.post_destroy
-        if not outline and post_destroy:
-            util.handle_hooks(
-                stage="post_destroy",
-                hooks=post_destroy,
-                provider=self.provider,
-                context=self.context)