Added source to ACR workflow instructions (#103)

This PR focuses around ironing out the flow of running `spk project init` or
`spk project init -m`, creating pipelines via the generated
`azure-pipelines.yaml` files, and ensuring the pipeline can create appropriately
named/tagged docker images in ACR.

In order to allow for flexibility in how SRE's choose to deploy the pipelines,
this PR does not automate the actual deployment of the pipeline. Instead, `info`
level instructions have been added specifying how to run the
`spk service create-pipeline` to deploy the `azure-pipelines.yaml` generated; If
the the user has configured the spk config correctly, the outputted command is
copy/paste ready to deploy the generated pipelines yaml:

If ran on a mono-repo:
`spk service create-pipeline --project-dir packages my-service` will be logged.

If ran on a standard repo: `spk service create-pipeline my-service` will be
logged.

The `azure-pipelines.yaml` expects 4 environment variables to be present in
order to login to `az` cli and build and push an image to ACR:

- `ACR_NAME` - name of the target ACR
- `SP_APP_ID` - service principal ID
- `SP_PASS` - service principal password
- `SP_TENANT` - service principal tenant

These requirements are also `info` logged to the end user upon
`azure-pipelines.yaml` generation.

Previously `spk service create-pipeline` only worked for mono-repositories and
the code has been refactored such that the existence of the `--packages-dir`
options implies whether or not the repository is a mono-repository or a standard
one.

closes microsoft/bedrock#646

Author: evanlouie

docs/service-management.md

@@ -33,8 +33,9 @@ Global options:
 Add a new service into this initialized spk project repository
 
 ```
-Usage:
-spk service create|c [options] <service-name>
+Usage: service create|c [options] <service-name>
+
+Add a new service into this initialized spk project repository
 
 Options:
   -c, --helm-chart-chart <helm-chart>            bedrock helm chart name. --helm-chart-* and --helm-config-* are exclusive; you may only use one. (default: "")
@@ -61,7 +62,7 @@ Configure Azure DevOps for a bedrock managed service.
 ```
 Usage: service create-pipeline|p [options] <service-name>
 
-Configure Azure DevOps for a bedrock managed service
+Configure Azure DevOps for a bedrock managed service.
 
 Options:
   -n, --pipeline-name <pipeline-name>                  Name of the pipeline to be created
@@ -70,7 +71,7 @@ Options:
   -r, --repo-name <repo-name>                          Repository Name in Azure DevOps
   -u, --repo-url <repo-url>                            Repository URL
   -d, --devops-project <devops-project>                Azure DevOps Project
-  -l, --packages-dir <packages-dir>                    The monorepository directory containing this service definition. ie. '--packages-dir packages' if my-service is located under ./packages/my-service.
+  -l, --packages-dir <packages-dir>                    The mono-repository directory containing this service definition. ie. '--packages-dir packages' if my-service is located under ./packages/my-service. Omitting this option implies this is a not a mono-repository.
   -h, --help                                           output usage information
 ```
 

src/commands/hld/pipeline.ts

@@ -63,7 +63,7 @@ export const installHldToManifestPipelineDecorator = (
         );
       } catch (err) {
         logger.error(
-          `Error occured installing pipeline for HLD to Manifest pipeline`
+          `Error occurred installing pipeline for HLD to Manifest pipeline`
         );
         logger.error(err);
         process.exit(1);

src/commands/service/create.ts

@@ -1,8 +1,6 @@
 import commander from "commander";
 import path from "path";
 import shelljs from "shelljs";
-import { logger } from "../../logger";
-
 import {
   addNewServiceToBedrockFile,
   addNewServiceToMaintainersFile,
@@ -11,6 +9,7 @@ import {
   generateStarterAzurePipelinesYaml
 } from "../../lib/fileutils";
 import { checkoutCommitPushCreatePRLink } from "../../lib/gitutils";
+import { logger } from "../../logger";
 import { IHelmConfig, IUser } from "../../types";
 
 /**

src/commands/service/pipeline.ts

@@ -1,5 +1,6 @@
 import { IBuildApi } from "azure-devops-node-api/BuildApi";
-import commander = require("commander");
+import { BuildDefinition } from "azure-devops-node-api/interfaces/BuildInterfaces";
+import commander from "commander";
 import path from "path";
 import { Config } from "../../config";
 import {
@@ -15,15 +16,13 @@ import {
 } from "../../lib/pipelines/pipelines";
 import { logger } from "../../logger";
 
-import { BuildDefinition } from "azure-devops-node-api/interfaces/BuildInterfaces";
-
 export const createPipelineCommandDecorator = (
   command: commander.Command
 ): void => {
   command
     .command("create-pipeline <service-name>")
     .alias("p")
-    .description("Configure Azure DevOps for a bedrock managed service")
+    .description("Configure Azure DevOps for a bedrock managed service.")
     .option(
       "-n, --pipeline-name <pipeline-name>",
       "Name of the pipeline to be created"
@@ -38,7 +37,7 @@ export const createPipelineCommandDecorator = (
     .option("-d, --devops-project <devops-project>", "Azure DevOps Project")
     .option(
       "-l, --packages-dir <packages-dir>",
-      "The monorepository directory containing this service definition. ie. '--packages-dir packages' if my-service is located under ./packages/my-service."
+      "The mono-repository directory containing this service definition. ie. '--packages-dir packages' if my-service is located under ./packages/my-service. Omitting this option implies this is a not a mono-repository."
     )
     .action(async (serviceName, opts) => {
       const gitOriginUrl = await getOriginUrl();
@@ -49,7 +48,7 @@ export const createPipelineCommandDecorator = (
         personalAccessToken = azure_devops && azure_devops.access_token,
         devopsProject = azure_devops && azure_devops.project,
         pipelineName = serviceName + "-pipeline",
-        packagesDir = "./",
+        packagesDir, // allow to be undefined in the case of a mono-repo
         repoName = getRepositoryName(gitOriginUrl),
         repoUrl = getRepositoryUrl(gitOriginUrl)
       } = opts;
@@ -98,12 +97,6 @@ export const createPipelineCommandDecorator = (
             `--devops-project must be of type 'string', ${typeof devopsProject} given.`
           );
         }
-
-        if (typeof packagesDir !== "string") {
-          throw new Error(
-            `--packages-dir must be of type 'string', ${typeof packagesDir} given.`
-          );
-        }
       } catch (err) {
         logger.error(`Error occurred validating inputs for ${serviceName}`);
         logger.error(err);
@@ -123,7 +116,7 @@ export const createPipelineCommandDecorator = (
           process.exit
         );
       } catch (err) {
-        logger.error(`Error occured installing pipeline for ${serviceName}`);
+        logger.error(`Error occurred installing pipeline for ${serviceName}`);
         logger.error(err);
         process.exit(1);
       }
@@ -132,27 +125,30 @@ export const createPipelineCommandDecorator = (
 
 /**
  * Install a pipeline for the service in an azure devops org.
- * @param serviceName
+ *
+ * @param serviceName Name of the service this pipeline belongs to; this is only used when `packagesDir` is defined as a means to locate the azure-pipelines.yaml file
  * @param orgName
  * @param personalAccessToken
  * @param pipelineName
- * @param repoName
- * @param repoUrl
+ * @param repositoryName
+ * @param repositoryUrl
  * @param project
+ * @param packagesDir The directory containing the services for a mono-repo. If undefined; implies that we are operating on a standard service repository
+ * @param exitFn
  */
 export const installPipeline = async (
   serviceName: string,
   orgName: string,
   personalAccessToken: string,
   pipelineName: string,
-  repoName: string,
-  repoUrl: string,
+  repositoryName: string,
+  repositoryUrl: string,
   project: string,
-  packagesDir: string,
+  packagesDir: string | undefined,
   exitFn: (status: number) => void
 ) => {
-  let devopsClient;
-  let builtDefinition;
+  let devopsClient: IBuildApi | undefined;
+  let builtDefinition: BuildDefinition | undefined;
 
   try {
     devopsClient = await getBuildApiClient(orgName, personalAccessToken);
@@ -166,16 +162,23 @@ export const installPipeline = async (
     branchFilters: ["master"],
     maximumConcurrentBuilds: 1,
     /* tslint:disable-next-line object-literal-shorthand */
-    pipelineName: pipelineName,
-    repositoryName: repoName,
-    repositoryUrl: repoUrl,
+    pipelineName,
+    repositoryName,
+    repositoryUrl,
     yamlFileBranch: "master",
-    yamlFilePath: path.join(packagesDir, serviceName, "azure-pipelines.yaml") // This may not work if we're using a non-mono repository and azure-pipelines.yaml is in the root directory.
+    yamlFilePath: packagesDir // if a packages dir is supplied, its a mono-repo
+      ? path.join(packagesDir, serviceName, "azure-pipelines.yaml") // if a packages dir is supplied, its a mono-repo; concat <packages-dir>/<service-name>
+      : "azure-pipelines.yaml" // if no packages dir, its a standard repo; so the azure-pipelines.yaml is in the root
   });
 
   try {
+    logger.debug(
+      `Creating pipeline for project '${project}' with definition '${JSON.stringify(
+        definition
+      )}'`
+    );
     builtDefinition = await createPipelineForDefinition(
-      devopsClient as IBuildApi,
+      devopsClient,
       project,
       definition
     );
@@ -184,16 +187,18 @@ export const installPipeline = async (
     logger.error(err);
     return exitFn(1);
   }
+  if (typeof builtDefinition.id === "undefined") {
+    const builtDefnString = JSON.stringify(builtDefinition);
+    throw Error(
+      `Invalid BuildDefinition created, parameter 'id' is missing from ${builtDefnString}`
+    );
+  }
 
   logger.info(`Created pipeline for ${pipelineName}`);
-  logger.info(`Pipeline ID: ${(builtDefinition as BuildDefinition).id}`);
+  logger.info(`Pipeline ID: ${builtDefinition.id}`);
 
   try {
-    await queueBuild(
-      devopsClient as IBuildApi,
-      project,
-      (builtDefinition as BuildDefinition).id as number
-    );
+    await queueBuild(devopsClient, project, builtDefinition.id);
   } catch (err) {
     logger.error(`Error occurred when queueing build for ${pipelineName}`);
     logger.error(err);

src/commands/variable-group/create.ts

@@ -1,6 +1,7 @@
 import { VariableGroup } from "azure-devops-node-api/interfaces/ReleaseInterfaces";
 import commander from "commander";
 import fs from "fs";
+import path from "path";
 import { readYaml } from "../../config";
 import { IAzureDevOpsOpts } from "../../lib/git";
 import {
@@ -102,11 +103,13 @@ export const create = async (
   filepath: string,
   accessOpts: IAzureDevOpsOpts
 ) => {
-  logger.info("Creating variale group");
+  logger.info(
+    `Creating Variable Group from group definition '${path.resolve(filepath)}'`
+  );
   try {
     fs.statSync(filepath);
     const data = readYaml<IVariableGroupData>(filepath);
-    logger.debug(`Varible Group Yaml data: ${JSON.stringify(data)}`);
+    logger.debug(`Variable Group Yaml data: ${JSON.stringify(data)}`);
 
     // validate variable group type
 
@@ -118,7 +121,7 @@ export const create = async (
       variableGroup = await addVariableGroup(data, accessOpts);
     } else {
       throw new Error(
-        `Varible Group type "${data.type}" is not supported. Only "Vsts" and "AzureKeyVault" are valid types and case sensitive.`
+        `Variable Group type "${data.type}" is not supported. Only "Vsts" and "AzureKeyVault" are valid types and case sensitive.`
       );
     }
   } catch (err) {

src/lib/fileutils.ts

@@ -89,12 +89,14 @@ export const starterAzurePipelines = async (opts: {
   vmImage?: string;
   branches?: string[];
   variableGroups?: string[];
+  variables?: Array<{ name: string; value: string }>;
 }): Promise<IAzurePipelinesYaml> => {
   const {
     relProjectPaths = ["."],
     vmImage = "ubuntu-latest",
     branches = ["master"],
-    variableGroups = []
+    variableGroups = [],
+    variables = []
   } = opts;
 
   // Ensure any blank paths are turned into "./"
@@ -109,20 +111,11 @@ export const starterAzurePipelines = async (opts: {
       branches: { include: branches },
       paths: { include: cleanedPaths }
     },
-    variables: [...variableGroups.map(group => ({ group }))],
+    variables: [...variableGroups.map(group => ({ group })), ...variables],
     pool: {
       vmImage
     },
     steps: [
-      {
-        script: generateYamlScript([
-          `printenv | sort`,
-          `pwd`,
-          `ls -la`,
-          `echo "The name of this service is: $(BUILD.BUILDNUMBER)"`
-        ]),
-        displayName: "Run a multi-line script"
-      },
       {
         script: generateYamlScript([
           `echo "az login --service-principal --username $(SP_APP_ID) --password $(SP_PASS) --tenant $(SP_TENANT)"`,
@@ -131,23 +124,56 @@ export const starterAzurePipelines = async (opts: {
         displayName: "Azure Login"
       },
       ...cleanedPaths.map(projectPath => {
+        const projectPathParts = projectPath
+          .split(path.sep)
+          .filter(p => p !== "");
+        // If a the projectPath contains more than 1 segment (a service in a
+        // mono-repo), use the last part as the project name as it will the
+        // folder containing the Dockerfile. Otherwise, its a standard service
+        // and does not need a a project name
+        const projectName =
+          projectPathParts.length > 1
+            ? "-" + projectPathParts.slice(-1)[0]
+            : "";
         return {
           script: generateYamlScript([
             `cd ${projectPath} # Need to make sure Build.DefinitionName matches directory. It's case sensitive`,
             `echo "az acr build -r $(ACR_NAME) --image $(Build.DefinitionName):$(build.SourceBranchName)-$(build.BuildId) ."`,
-            `az acr build -r $(ACR_NAME) --image $(Build.DefinitionName):$(build.SourceBranchName)-$(build.BuildId) .`
+            `az acr build -r $(ACR_NAME) --image $(Build.DefinitionName)${projectName}:$(build.SourceBranchName)-$(build.BuildId) .`
           ]),
           displayName: "ACR Build and Publish"
         };
-      }),
-      {
-        script: generateYamlScript([`echo Hello, world!`]),
-        displayName: "Run a one-line script"
-      }
+      })
     ]
   };
   // tslint:enable: object-literal-sort-keys
 
+  const requiredPipelineVariables = [
+    `'ACR_NAME' (name of your ACR)`,
+    `'SP_APP_ID' (service principal ID with access to your ACR)`,
+    `'SP_PASS' (service principal secret)`,
+    `'SP_TENANT' (service principal tenant)`
+  ].join(", ");
+
+  for (const relPath of cleanedPaths) {
+    const relPathParts = relPath.split(path.sep).filter(p => p !== "");
+    const packagesDir =
+      relPathParts.length > 1 ? relPathParts.slice(-2)[0] : undefined;
+    const packagesOption = packagesDir ? `--packages-dir ${packagesDir} ` : "";
+    const serviceName =
+      relPathParts.length > 1
+        ? relPathParts.slice(-2)[1]
+        : process
+            .cwd()
+            .split(path.sep)
+            .slice(-1)[0];
+    const spkServiceCreatePipelineCmd =
+      "spk service create-pipeline " + packagesOption + serviceName;
+    logger.info(
+      `Generated azure-pipelines.yaml for service in path '${relPath}'. Commit and push this file to master before attempting to deploy via the command '${spkServiceCreatePipelineCmd}'; before running the pipeline ensure the following environment variables are available to your pipeline: ${requiredPipelineVariables}`
+    );
+  }
+
   return starter;
 };
 

src/lib/pipelines/pipelines.ts

@@ -212,10 +212,23 @@ export const createPipelineForDefinition = async (
   logger.info("Creating pipeline for definition");
 
   try {
-    return await buildApi.createDefinition(definition, azdoProject);
+    logger.debug(
+      `Creating BuildDefinition based on ${JSON.stringify(definition)}`
+    );
+    const createdDefn = await buildApi.createDefinition(
+      definition,
+      azdoProject
+    );
+    // type definition for createDefinition is wrong. It will resolve a `null` if an error occurs in azdo
+    if (!createdDefn) {
+      throw Error(
+        `Error creating BuildDefinition; buildApi.createDefinition() returned an invalid value of ${createdDefn}`
+      );
+    }
+    return createdDefn;
   } catch (e) {
     logger.error(e);
-    throw new Error("Error creating definition");
+    throw Error("Error creating definition");
   }
 };
 
@@ -231,16 +244,16 @@ export const queueBuild = async (
   azdoProject: string,
   definitionId: number
 ): Promise<Build> => {
-  const buildReference = {
+  const buildReference: Build = {
     definition: {
       id: definitionId
     }
-  } as Build;
+  };
 
   try {
     return await buildApi.queueBuild(buildReference, azdoProject);
   } catch (e) {
     logger.error(e);
-    throw new Error("Error queueing build");
+    throw Error("Error queueing build");
   }
 };