spk infra generate (iteration 2) (#132)

* Inheritance functionality 
* Fixed regex bug
* Removed `fs` operations from unit tests
* Modularized generate.ts
* Copy exclude `terraform.tfvars`

Author: yradsmikham

docs/cloud-infra-management.md

@@ -141,7 +141,7 @@ Generates a deployment folder of an infrastructure scaffolded project containing
 a `definition.json` that contains a `source`, `template` and `version` to obtain
 and complete the terraform template files.
 
-It will do the following (**In Progress**):
+It will do the following:
 
 - Check if a provided project folder contains a `definition.json`
 - Check if the terraform template `source` provided has a valid remote
@@ -156,6 +156,82 @@ It will do the following (**In Progress**):
 - Create a `spk.tfvars` in the generated directory based on the variables
   provided in `definition.json`
 
+```
+Usage:
+spk infra generate|g [options]
+
+Generate scaffold for terraform cluster deployment.
+
+Options:
+  -p, --project <path to project folder to generate>   Location of the definition.json file that will be generated
+  -h, --help                                           output usage information
+```
+
+### generate example
+
+Assuming you have the following setup:
+
+```
+discovery-service
+    |- definition.json
+    |- east/
+        |- definition.json
+    |- central/
+        |- definition.json
+```
+
+When executing the following command **in the `discovery-service` directory**:
+
+```
+spk infra generate --project east
+```
+
+The following hiearchy of directories will be generated _alongside_ the targeted
+directory. In addition, the appropriate versioned Terraform templates will be
+copied over to the leaf directory with a `spk.tfvars`, which contains the
+variables accumulated from parent **and** leaf definition.json files.
+
+```
+discovery-service
+    |- definition.json
+    |- east/
+        |- definition.json
+    |- central/
+        |- definition.json
+discovery-service-generated
+    |- east
+        |- main.tf
+        |- variables.tf
+        |- spk.tfvars
+```
+
+You can also have a "single-tree" generation by executing `spk infra generate`
+at the level above the targeted directory. For example, if you had the following
+tree structure:
+
+```
+discovery-service
+    |- east/
+        |- definition.json
+    |- central/
+        |- definition.json
+```
+
+and wanted to create _just_ an `east-generated` directory, you could run
+`spk infra generate -p east`, and this will result in the following:
+
+```
+discovery-service
+    |- east/
+        |- definition.json
+    |- east-generated/
+        |- main.tf
+        |- variables.tf
+        |- spk.tfvars
+    |- central/
+        |- definition.json
+```
+
 ### Authentication
 
 Spk currently supports the use of Personal Access Tokens to authenticate with
@@ -168,3 +244,4 @@ build scaffolded definitions using a private AzDO repo, do one of the following:
 - **Using arguments** - Pass in your formatted source url for your private AzDO
   repo with the PAT and arbitrary username specified. Example
   `spk infra scaffold --name discovery-service --source https://spk:{my_PAT_Token}@dev.azure.com/microsoft/spk/_git/infra_repo --version v0.0.1 --template cluster/environments/azure-single-keyvault`
+

docs/infra/spk-terragrunt-day-2-scenarios.md

@@ -379,4 +379,4 @@ Additional Thoughts:
 
 ## Holder for comparison between Terragrunt & Bespoke
 
-![](../images/Capture.PNG)
+![](../images/Capture.png)

src/commands/infra/generate.test.ts

@@ -8,6 +8,7 @@ import {
 import {
   generateSpkTfvars,
   parseDefinitionJson,
+  readDefinitionJson,
   validateDefinition,
   validateRemoteSource,
   validateTemplateSource
@@ -83,15 +84,8 @@ describe("Validate template path from a definition.json", () => {
 describe("Validate spk.tfvars file", () => {
   test("Validating that a spk.tfvars is generated and has appropriate format", async () => {
     const mockProjectPath = "src/commands/infra/mocks";
-    const generateTfvars = await generateSpkTfvars(
-      mockProjectPath,
-      mockProjectPath
-    );
-    const data = fs.readFileSync(
-      path.join(mockProjectPath, "spk.tfvars"),
-      "utf-8"
-    );
-    expect(data).toContain('gitops_poll_interval = "5m"');
-    fs.unlinkSync(path.join(mockProjectPath, "spk.tfvars"));
+    const definitionJSON = await readDefinitionJson(mockProjectPath);
+    const spkTfvarsObject = await generateSpkTfvars(definitionJSON);
+    expect(spkTfvarsObject).toContain('gitops_poll_interval = "5m"');
   });
 });

src/commands/infra/generate.ts

@@ -39,10 +39,7 @@ export const generateCommandDecorator = (command: commander.Command): void => {
         await validateDefinition(opts.project);
         const jsonSource = await validateTemplateSource(opts.project);
         await validateRemoteSource(jsonSource);
-        const generatedDir = await createGenerated(opts.project);
-        const templatePath = await parseDefinitionJson(opts.project);
-        await copyTfTemplate(templatePath, generatedDir);
-        await generateSpkTfvars(opts.project, generatedDir);
+        await generateConfig(opts.project);
       } catch (err) {
         logger.error(
           "Error occurred while generating project deployment files"
@@ -175,14 +172,82 @@ export const validateRemoteSource = async (
   return true;
 };
 
+/**
+ * Creates "generated" directory if it does not already exists
+ *
+ * @param projectPath Path to the definition.json file
+ */
+export const generateConfig = async (projectPath: string): Promise<void> => {
+  try {
+    // First, search for definition.json in current working directory
+    const templatePath = await parseDefinitionJson(projectPath);
+    const cwdPath = process.cwd();
+    if (fs.existsSync(path.join(cwdPath, "definition.json"))) {
+      // If there exists a definition.json, then read file
+      logger.info(`A definition.json was found in the parent directory.`);
+      const parentDefinitionJSON = await readDefinitionJson(cwdPath);
+      const leafDefinitionJSON = await readDefinitionJson(projectPath);
+      /* Iterate through parent and leaf JSON objects to find matches
+        If there is a match, then replace parent key-value
+        If there is no match between the parent and leaf,
+        then append leaf key-value parent key-value JSON */
+      for (const parentKey in parentDefinitionJSON.variables) {
+        if (parentKey) {
+          for (const leafKey in leafDefinitionJSON.variables) {
+            if (parentKey === leafKey) {
+              let parentVal = parentDefinitionJSON.variables[parentKey];
+              parentVal = leafDefinitionJSON.variables[leafKey];
+            } else {
+              // Append to parent variables block
+              const leafVal = leafDefinitionJSON.variables[leafKey];
+              parentDefinitionJSON.variables[leafKey] = leafVal;
+            }
+          }
+        }
+      }
+      // Create a generated parent directory
+      const parentDirectory = await createGenerated(cwdPath + "-generated");
+      // Then, create generated child directory
+      const childDirectory = await createGenerated(
+        path.join(parentDirectory, projectPath)
+      );
+      // Generate Terraform files in generated directory
+      const spkTfvarsObject = await generateSpkTfvars(
+        parentDefinitionJSON.variables
+      );
+      await checkSpkTfvars(childDirectory);
+      await writeToSpkTfvarsFile(spkTfvarsObject, childDirectory);
+      // const templatePath = await parseDefinitionJson(projectPath);
+      await copyTfTemplate(templatePath, childDirectory);
+    } else {
+      // If there is not a definition.json in current working directory,
+      // then proceed with reading definition.json in project path
+      // await createGenerated(projectPath)
+      // logger.info(`A definition.json was not found in the parent directory.`)
+      const definitionJSON = await readDefinitionJson(projectPath);
+      // Create a generated directory
+      const generatedDirectory = await createGenerated(
+        projectPath + "-generated"
+      );
+      // Generate Terraform files in generated directory
+      const spkTfvarsObject = await generateSpkTfvars(definitionJSON.variables);
+      await checkSpkTfvars(generatedDirectory);
+      await writeToSpkTfvarsFile(spkTfvarsObject, generatedDirectory);
+      await copyTfTemplate(templatePath, generatedDirectory);
+    }
+  } catch (err) {
+    return err;
+  }
+};
+
 /**
  * Creates "generated" directory if it does not already exists
  *
  * @param projectPath Path to the definition.json file
  */
 export const createGenerated = async (projectPath: string): Promise<string> => {
   try {
-    const newGeneratedPath = projectPath + "-generated";
+    const newGeneratedPath = projectPath;
     mkdirp.sync(newGeneratedPath);
     logger.info(`Created generated directory: ${newGeneratedPath}`);
     return newGeneratedPath;
@@ -210,6 +275,22 @@ export const parseDefinitionJson = async (projectPath: string) => {
   return templatePath;
 };
 
+/**
+ * Checks if an spk.tfvars
+ *
+ * @param projectPath Path to the spk.tfvars file
+ */
+export const checkSpkTfvars = async (generatedPath: string): Promise<void> => {
+  try {
+    // Remove existing spk.tfvars if it already exists
+    if (fs.existsSync(path.join(generatedPath, "spk.tfvars"))) {
+      fs.unlinkSync(path.join(generatedPath, "spk.tfvars"));
+    }
+  } catch (err) {
+    return err;
+  }
+};
+
 /**
  *
  * Takes in the "variables" block from definition.json file and returns
@@ -224,41 +305,49 @@ export const parseDefinitionJson = async (projectPath: string) => {
  * key = "value"
  *
  */
-export const generateSpkTfvars = async (
-  projectPath: string,
-  generatedPath: string
-) => {
+export const generateSpkTfvars = async (definitionJSON: string[]) => {
   try {
-    // Remove existing spk.tfvars if it already exists
-    if (fs.existsSync(path.join(generatedPath, "spk.tfvars"))) {
-      fs.unlinkSync(path.join(generatedPath, "spk.tfvars"));
-    }
-    // Parse definition.json and extract "variables"
-    const definitionJSON = await readDefinitionJson(projectPath);
-    const variables = definitionJSON.variables;
+    const tfVars: string[] = [];
+    // Parse definition.json "variables" block
+    const variables = definitionJSON;
     // Restructure the format of variables text
     const tfVariables = JSON.stringify(variables)
-      .replace(/\:/g, "=")
       .replace(/\{|\}/g, "")
       .replace(/\,/g, "\n")
       .split("\n");
     // (re)Create spk.tfvars
     tfVariables.forEach(t => {
-      const tfVar = t.split("=");
-      if (tfVar[0].length > 0) {
-        tfVar[0] = tfVar[0].replace(/\"/g, "");
+      const tfVar = t.split(":");
+      const result = tfVar.slice(0, 1);
+      result.push(tfVar.slice(1).join(":"));
+      if (result[0].length > 0) {
+        result[0] = result[0].replace(/\"/g, "");
       }
-      const newTfVar = tfVar.join(" = ");
-      fs.appendFileSync(
-        path.join(generatedPath, "spk.tfvars"),
-        newTfVar + "\n"
-      );
+      const newTfVar = result.join(" = ");
+      tfVars.push(newTfVar);
     });
+    return tfVars;
   } catch (err) {
-    logger.error(err);
+    logger.error(`There was an error with generating the spk tfvars object.`);
+    return err;
   }
 };
 
+/**
+ * Reads in a tfVars object and returns a spk.tfvars file
+ *
+ * @param spkTfVars spk tfvars object in an array
+ * @param generatedPath Path to write the spk.tfvars file to
+ */
+export const writeToSpkTfvarsFile = async (
+  spkTfVars: string[],
+  generatedPath: string
+) => {
+  spkTfVars.forEach(tfvar => {
+    fs.appendFileSync(path.join(generatedPath, "spk.tfvars"), tfvar + "\n");
+  });
+};
+
 /**
  * Reads a definition.json and returns a JSON object
  *

src/commands/infra/scaffold.ts

@@ -135,7 +135,7 @@ export const renameTfvars = async (dir: string): Promise<void> => {
   try {
     const tfFiles = fs.readdirSync(dir);
     tfFiles.forEach(file => {
-      if (file.substr(file.lastIndexOf(".") + 1) === "tfvars") {
+      if (file === "terraform.tfvars") {
         fs.renameSync(path.join(dir, file), path.join(dir, file + ".backup"));
       }
     });
@@ -156,7 +156,11 @@ export const copyTfTemplate = async (
   envName: string
 ): Promise<boolean> => {
   try {
-    await fsextra.copy(templatePath, envName);
+    await fsextra.copy(templatePath, envName, {
+      filter: file => {
+        return !(file.indexOf("terraform.tfvars") > -1);
+      }
+    });
     logger.info(`Terraform template files copied from ${templatePath}`);
   } catch (err) {
     logger.error(