feat(installer): support templated install configs from vault

cli/cmd/config.go

@@ -0,0 +1,26 @@
+// Copyright (c) Codesphere Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+package cmd
+
+import (
+	"github.com/codesphere-cloud/cs-go/pkg/io"
+	"github.com/spf13/cobra"
+)
+
+type ConfigCmd struct {
+	cmd *cobra.Command
+}
+
+func AddConfigCmd(rootCmd *cobra.Command, opts *GlobalOptions) {
+	config := ConfigCmd{
+		cmd: &cobra.Command{
+			Use:   "config",
+			Short: "Work with OMS configuration files",
+			Long:  io.Long(`Work with OMS configuration files.`),
+		},
+	}
+
+	AddConfigTemplateCmd(config.cmd, opts)
+	AddCmd(rootCmd, config.cmd)
+}

cli/cmd/config_template.go

@@ -0,0 +1,87 @@
+// Copyright (c) Codesphere Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+package cmd
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/codesphere-cloud/cs-go/pkg/io"
+	"github.com/codesphere-cloud/oms/internal/configtemplating"
+	"github.com/codesphere-cloud/oms/internal/installer"
+	"github.com/codesphere-cloud/oms/internal/util"
+	"github.com/spf13/cobra"
+)
+
+type ConfigTemplateCmd struct {
+	cmd  *cobra.Command
+	Opts ConfigTemplateOpts
+}
+
+type ConfigTemplateOpts struct {
+	*GlobalOptions
+	Config string
+	Vault  string
+	AgeKey string
+}
+
+func (c *ConfigTemplateCmd) RunE(cmd *cobra.Command, _ []string) error {
+	rendered, err := c.Render()
+	if err != nil {
+		return err
+	}
+
+	if _, err := fmt.Fprint(cmd.OutOrStdout(), string(rendered)); err != nil {
+		return fmt.Errorf("failed to write rendered config: %w", err)
+	}
+
+	return nil
+}
+
+func AddConfigTemplateCmd(parentCmd *cobra.Command, opts *GlobalOptions) {
+	templateCmd := &ConfigTemplateCmd{
+		cmd: &cobra.Command{
+			Use:   "template",
+			Short: "Render a config.yaml template using secrets from a vault file",
+			Long: io.Long(`Render a config.yaml template using secrets from a prod.vault.yaml file.
+
+This command prints the rendered configuration to stdout so templating can be tested without running an installation.`),
+			Example: formatExamples("config template", []io.Example{
+				{
+					Cmd:  "--config config.yaml --vault prod.vault.yaml --age-key age_key.txt",
+					Desc: "Render config.yaml with secrets from prod.vault.yaml",
+				},
+			}),
+			Args: cobra.ExactArgs(0),
+		},
+		Opts: ConfigTemplateOpts{GlobalOptions: opts},
+	}
+
+	templateCmd.cmd.Flags().StringVarP(&templateCmd.Opts.Config, "config", "c", "", "Path to the config.yaml template to render (required)")
+	templateCmd.cmd.Flags().StringVarP(&templateCmd.Opts.Vault, "vault", "v", "", "Path to the SOPS-encrypted prod.vault.yaml file (required)")
+	templateCmd.cmd.Flags().StringVarP(&templateCmd.Opts.AgeKey, "age-key", "k", "", "Path to the age key file used to decrypt the vault (required)")
+
+	util.MarkFlagRequired(templateCmd.cmd, "config")
+	util.MarkFlagRequired(templateCmd.cmd, "vault")
+	util.MarkFlagRequired(templateCmd.cmd, "age-key")
+
+	AddCmd(parentCmd, templateCmd.cmd)
+
+	templateCmd.cmd.RunE = templateCmd.RunE
+}
+
+func (c *ConfigTemplateCmd) Render() ([]byte, error) {
+	data, err := os.ReadFile(c.Opts.Config)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read config file %s: %w", c.Opts.Config, err)
+	}
+
+	store := installer.NewLazyVaultTemplatingSecretStore(c.Opts.Vault, c.Opts.AgeKey)
+	rendered, err := configtemplating.RenderInstallConfigTemplate(data, store)
+	if err != nil {
+		return nil, fmt.Errorf("failed to render config template: %w", err)
+	}
+
+	return rendered, nil
+}

cli/cmd/config_template_test.go

@@ -0,0 +1,114 @@
+// Copyright (c) Codesphere Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+package cmd_test
+
+import (
+	"bytes"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	"github.com/codesphere-cloud/oms/cli/cmd"
+	"github.com/codesphere-cloud/oms/internal/installer"
+	"github.com/codesphere-cloud/oms/internal/installer/files"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("ConfigTemplateCmd", func() {
+	It("renders config templates with secrets from a vault file", func() {
+		if !sopsAndAgeAvailable() {
+			Skip("sops and age-keygen not available")
+		}
+
+		tempDir := GinkgoT().TempDir()
+		configPath := filepath.Join(tempDir, "config.yaml")
+		vaultPath := filepath.Join(tempDir, "prod.vault.yaml")
+		plaintextVaultPath := filepath.Join(tempDir, "prod.vault.plain.yaml")
+		ageKeyPath := filepath.Join(tempDir, "age_key.txt")
+
+		Expect(os.WriteFile(configPath, []byte(`codesphere:
+  override:
+    global:
+      license:
+        key: '{{ secret "codesphereLicenseKey" }}'
+postgres:
+  override:
+    auth:
+      username: '{{ secret "postgresAdmin" "fields.username" }}'
+      password: '{{ secret "postgresAdmin" "fields.password" }}'
+`), 0644)).To(Succeed())
+		Expect(exec.Command("age-keygen", "-o", ageKeyPath).Run()).To(Succeed())
+
+		vault := &files.InstallVault{
+			Secrets: []files.SecretEntry{
+				{
+					Name: "codesphereLicenseKey",
+					File: &files.SecretFile{Content: "license-secret"},
+				},
+				{
+					Name: "postgresAdmin",
+					Fields: &files.SecretFields{
+						Username: "postgres",
+						Password: "admin-secret",
+					},
+				},
+			},
+		}
+		vaultYaml, err := vault.Marshal()
+		Expect(err).NotTo(HaveOccurred())
+		Expect(os.WriteFile(plaintextVaultPath, vaultYaml, 0600)).To(Succeed())
+		recipient, err := exec.Command("age-keygen", "-y", ageKeyPath).Output()
+		Expect(err).NotTo(HaveOccurred())
+		Expect(installer.EncryptFileWithSOPS(plaintextVaultPath, vaultPath, strings.TrimSpace(string(recipient)))).To(Succeed())
+
+		rootCmd := cmd.GetRootCmd()
+		var output bytes.Buffer
+		rootCmd.SetOut(&output)
+		rootCmd.SetErr(&output)
+		rootCmd.SetArgs([]string{
+			"config",
+			"template",
+			"--config",
+			configPath,
+			"--vault",
+			vaultPath,
+			"--age-key",
+			ageKeyPath,
+		})
+
+		err = rootCmd.Execute()
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(output.String()).To(ContainSubstring("key: 'license-secret'"))
+		Expect(output.String()).To(ContainSubstring("username: 'postgres'"))
+		Expect(output.String()).To(ContainSubstring("password: 'admin-secret'"))
+	})
+
+	It("adds the template command with required flags", func() {
+		rootCmd := cmd.GetRootCmd()
+
+		configCmd, _, err := rootCmd.Find([]string{"config", "template"})
+		Expect(err).NotTo(HaveOccurred())
+		Expect(configCmd).NotTo(BeNil())
+		Expect(configCmd.Use).To(Equal("template"))
+		Expect(configCmd.Short).To(Equal("Render a config.yaml template using secrets from a vault file"))
+
+		Expect(configCmd.Flags().Lookup("config")).NotTo(BeNil())
+		Expect(configCmd.Flags().Lookup("vault")).NotTo(BeNil())
+		Expect(configCmd.Flags().Lookup("age-key")).NotTo(BeNil())
+	})
+})
+
+func sopsAndAgeAvailable() bool {
+	if _, err := exec.LookPath("sops"); err != nil {
+		return false
+	}
+	if _, err := exec.LookPath("age-keygen"); err != nil {
+		return false
+	}
+	return true
+}

cli/cmd/init_install_config_test.go

@@ -5,6 +5,9 @@ package cmd
 
 import (
 	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -162,15 +165,34 @@ codesphere:
 
 	Context("valid configuration", func() {
 		It("validates successfully", func() {
+			if !sopsAndAgeAvailableForUpdateInstallConfig() {
+				Skip("sops and age-keygen not available")
+			}
+
 			_, err := configFile.WriteString(validConfig)
 			Expect(err).NotTo(HaveOccurred())
 			err = configFile.Close()
 			Expect(err).NotTo(HaveOccurred())
 
-			_, err = vaultFile.WriteString(validVault)
-			Expect(err).NotTo(HaveOccurred())
 			err = vaultFile.Close()
 			Expect(err).NotTo(HaveOccurred())
+			tempDir := GinkgoT().TempDir()
+			ageKeyPath := filepath.Join(tempDir, "age_key.txt")
+			plaintextVaultPath := filepath.Join(tempDir, "prod.vault.plain.yaml")
+			Expect(os.WriteFile(plaintextVaultPath, []byte(validVault), 0600)).To(Succeed())
+			Expect(exec.Command("age-keygen", "-o", ageKeyPath).Run()).To(Succeed())
+			recipient, err := exec.Command("age-keygen", "-y", ageKeyPath).Output()
+			Expect(err).NotTo(HaveOccurred())
+			Expect(installer.EncryptFileWithSOPS(plaintextVaultPath, vaultFile.Name(), strings.TrimSpace(string(recipient)))).To(Succeed())
+			previousAgeKeyFile, hadPreviousAgeKeyFile := os.LookupEnv("SOPS_AGE_KEY_FILE")
+			Expect(os.Setenv("SOPS_AGE_KEY_FILE", ageKeyPath)).To(Succeed())
+			DeferCleanup(func() {
+				if hadPreviousAgeKeyFile {
+					Expect(os.Setenv("SOPS_AGE_KEY_FILE", previousAgeKeyFile)).To(Succeed())
+					return
+				}
+				Expect(os.Unsetenv("SOPS_AGE_KEY_FILE")).To(Succeed())
+			})
 
 			c := &InitInstallConfigCmd{
 				Opts: &InitInstallConfigOpts{

cli/cmd/install_codesphere.go

@@ -16,8 +16,10 @@ import (
 	"strings"
 
 	"github.com/codesphere-cloud/cs-go/pkg/io"
+	"github.com/codesphere-cloud/oms/internal/configtemplating"
 	"github.com/codesphere-cloud/oms/internal/env"
 	"github.com/codesphere-cloud/oms/internal/installer"
+	"github.com/codesphere-cloud/oms/internal/installer/files"
 	"github.com/codesphere-cloud/oms/internal/system"
 	"github.com/codesphere-cloud/oms/internal/util"
 	"github.com/spf13/cobra"
@@ -35,6 +37,7 @@ type InstallCodesphereOpts struct {
 	Package          string
 	Force            bool
 	Config           string
+	Vault            string
 	PrivKey          string
 	SkipSteps        []string
 	CodesphereOnly   bool
@@ -79,6 +82,7 @@ func AddInstallCodesphereCmd(install *cobra.Command, opts *GlobalOptions) {
 	codesphere.cmd.Flags().StringVarP(&codesphere.Opts.Package, "package", "p", "", "Package file (e.g. codesphere-v1.2.3-installer.tar.gz) to load binaries, installer etc. from")
 	codesphere.cmd.Flags().BoolVarP(&codesphere.Opts.Force, "force", "f", false, "Enforce package extraction")
 	codesphere.cmd.Flags().StringVarP(&codesphere.Opts.Config, "config", "c", "", "Path to the Codesphere Private Cloud configuration file (yaml)")
+	codesphere.cmd.Flags().StringVar(&codesphere.Opts.Vault, "vault", "prod.vault.yaml", "Path to the SOPS-encrypted prod.vault.yaml file used for config templating")
 	codesphere.cmd.Flags().StringVarP(&codesphere.Opts.PrivKey, "priv-key", "k", "", "Path to the private key to encrypt/decrypt secrets")
 	codesphere.cmd.Flags().StringSliceVarP(&codesphere.Opts.SkipSteps, "skip-steps", "s", []string{}, "Steps to be skipped. E.g. copy-dependencies, extract-dependencies, load-container-images, ceph, kubernetes")
 	codesphere.cmd.Flags().BoolVar(&codesphere.Opts.CodesphereOnly, "codesphere-only", false, "Install only Codesphere without dependencies")
@@ -98,10 +102,27 @@ func (c *InstallCodesphereCmd) ExtractAndInstall(pm installer.PackageManager, cm
 		return fmt.Errorf("codesphere installation is only supported on Linux amd64. Current platform: %s/%s", goos, goarch)
 	}
 
+	originalConfig := c.Opts.Config
+	cleanup := func() {}
+	if c.Opts.Vault != "" {
+		store := installer.NewLazyVaultTemplatingSecretStore(c.Opts.Vault, c.Opts.PrivKey)
+		renderedConfig, renderCleanup, err := configtemplating.RenderConfigFileToTempIfNeeded(c.Opts.Config, store)
+		if err != nil {
+			return fmt.Errorf("failed to render config template: %w", err)
+		}
+		cleanup = renderCleanup
+		c.Opts.Config = renderedConfig
+	}
+	defer cleanup()
+	defer func() {
+		c.Opts.Config = originalConfig
+	}()
+
 	config, err := cm.ParseConfigYaml(c.Opts.Config)
 	if err != nil {
 		return fmt.Errorf("failed to extract config.yaml: %w", err)
 	}
+	c.warnIfVaultDirDiffersFromSecretsDir(config)
 
 	err = pm.Extract(c.Opts.Force)
 	if err != nil {
@@ -246,6 +267,28 @@ func (c *InstallCodesphereCmd) ExtractAndInstall(pm installer.PackageManager, cm
 	return nil
 }
 
+func (c *InstallCodesphereCmd) warnIfVaultDirDiffersFromSecretsDir(config files.RootConfig) {
+	if c.Opts.Vault == "" || config.Secrets.BaseDir == "" {
+		return
+	}
+
+	vaultDir, err := filepath.Abs(filepath.Dir(c.Opts.Vault))
+	if err != nil {
+		log.Printf("Warning: failed to resolve vault directory for %s: %v", c.Opts.Vault, err)
+		return
+	}
+
+	secretsDir, err := filepath.Abs(config.Secrets.BaseDir)
+	if err != nil {
+		log.Printf("Warning: failed to resolve configured secrets baseDir %s: %v", config.Secrets.BaseDir, err)
+		return
+	}
+
+	if vaultDir != secretsDir {
+		log.Printf("Warning: config secrets.baseDir (%s) does not match the directory of --vault (%s)", secretsDir, vaultDir)
+	}
+}
+
 func (c *InstallCodesphereCmd) ListPackageContents(pm installer.PackageManager) ([]string, error) {
 	packageDir := pm.GetWorkDir()
 	if !pm.FileIO().Exists(packageDir) {

cli/cmd/install_codesphere_test.go

@@ -510,6 +510,10 @@ var _ = Describe("AddInstallCodesphereCmd", func() {
 		Expect(privKeyFlag).NotTo(BeNil())
 		Expect(privKeyFlag.Shorthand).To(Equal("k"))
 
+		vaultFlag := codesphereCmd.Flags().Lookup("vault")
+		Expect(vaultFlag).NotTo(BeNil())
+		Expect(vaultFlag.DefValue).To(Equal("prod.vault.yaml"))
+
 		skipStepFlag := codesphereCmd.Flags().Lookup("skip-steps")
 		Expect(skipStepFlag).NotTo(BeNil())
 		Expect(skipStepFlag.Shorthand).To(Equal("s"))

cli/cmd/root.go

@@ -74,6 +74,7 @@ func GetRootCmd() *cobra.Command {
 	AddDownloadCmd(rootCmd, opts)
 	AddInstallCmd(rootCmd, opts)
 	AddInitCmd(rootCmd, opts)
+	AddConfigCmd(rootCmd, opts)
 	AddBuildCmd(rootCmd, opts)
 	AddLicensesCmd(rootCmd)