diff --git a/.eirctl/.gitkeep b/.eirctl/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..992fdca --- /dev/null +++ b/.gitattributes @@ -0,0 +1,5 @@ +* text eol=lf +*.jpg binary +*.jpeg binary +*.png binary +*.gif binary diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ebccfcd..86aa17f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -14,9 +14,9 @@ permissions: jobs: set-version: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 container: - image: mcr.microsoft.com/dotnet/sdk:6.0 + image: mcr.microsoft.com/dotnet/sdk:10.0 outputs: semVer: ${{ steps.gitversion.outputs.semVer }} steps: @@ -29,10 +29,12 @@ jobs: git config --global --add safe.directory "$GITHUB_WORKSPACE" git config user.email ${{ github.actor }}-ci@gha.org git config user.name ${{ github.actor }} + - name: Install GitVersion uses: gittools/actions/gitversion/setup@v3.0.0 with: versionSpec: "5.x" + - name: Set SemVer Version uses: gittools/actions/gitversion/execute@v3.0.0 id: gitversion @@ -43,7 +45,7 @@ jobs: echo "VERSION -> $GITVERSION_SEMVER" test: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 needs: set-version env: SEMVER: ${{ needs.set-version.outputs.semVer }} @@ -52,17 +54,22 @@ jobs: steps: - uses: actions/checkout@v4 with: - fetch-depth: 1 + fetch-depth: 0 - name: Install Eirctl uses: ensono/actions/eirctl-setup@v0.3.1 with: - version: 0.9.3 + version: 0.11.2 isPrerelease: false + - name: Run Commit Lint + run: | + eirctl commit-lint + - name: Run Lint run: | eirctl run pipeline lints + - name: Run Tests run: | eirctl run pipeline gha:unit:test diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3ddc903..94b479c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -14,10 +14,10 @@ permissions: jobs: set-version: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 if: ${{ github.event.workflow_run.head_branch == 'master' && github.event.workflow_run.conclusion == 'success' }} container: - image: mcr.microsoft.com/dotnet/sdk:6.0 + image: mcr.microsoft.com/dotnet/sdk:10.0 outputs: semVer: ${{ steps.gitversion.outputs.semVer }} steps: @@ -39,7 +39,7 @@ jobs: id: gitversion release: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 needs: set-version env: SEMVER: ${{ needs.set-version.outputs.semVer }} @@ -50,7 +50,7 @@ jobs: - name: Install Eirctl uses: ensono/actions/eirctl-setup@v0.3.1 - with: + with: version: 0.7.6 isPrerelease: false diff --git a/.github/workflows/release_container.yml b/.github/workflows/release_container.yml index 9a1c45d..d422123 100644 --- a/.github/workflows/release_container.yml +++ b/.github/workflows/release_container.yml @@ -5,12 +5,12 @@ on: workflows: ['Lint and Test'] types: - completed - branches: + branches: - main permissions: contents: write - packages: write + packages: write jobs: set-version-tag: @@ -33,7 +33,7 @@ jobs: id: gitversion build-and-push: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 needs: set-version-tag env: SEMVER: ${{ needs.set-version-tag.outputs.semVer }} diff --git a/.gitignore b/.gitignore index 573d3c6..a4b8090 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,8 @@ dist # local testers and local/ + +# Eirctl +eirctl.env +.eirctl/* +!.eirctl/.gitkeep diff --git a/GitVersion.yaml b/GitVersion.yaml new file mode 100644 index 0000000..d246bab --- /dev/null +++ b/GitVersion.yaml @@ -0,0 +1,9 @@ +# Conventional Commits versioning rules: +# Major: any type with ! (e.g. feat!:, fix(scope)!:) or BREAKING CHANGE in commit footer +# Minor: feat: or feat(scope): +# Patch: all other types (fix:, refactor:, debug:, chore:, etc.) +mode: Mainline +commit-message-incrementing: Enabled +major-version-bump-message: '^(\w+)(\(\w+\))?!:|BREAKING CHANGE' +minor-version-bump-message: '^feat(\(\w+\))?:' +patch-version-bump-message: '^(\w+)(\(\w+\))?:' diff --git a/cmd/configmanager/configmanager.go b/cmd/configmanager/configmanager.go index 01af646..466a18a 100644 --- a/cmd/configmanager/configmanager.go +++ b/cmd/configmanager/configmanager.go @@ -20,6 +20,7 @@ var ( type rootCmdFlags struct { verbose bool + strict bool tokenSeparator string keySeparator string enableEnvSubst bool @@ -38,14 +39,16 @@ func NewRootCmd(logger log.ILogger) *Root { //channelOut, channelErr io.Writer Short: fmt.Sprintf("%s CLI for retrieving and inserting config or secret variables", config.SELF_NAME), Long: fmt.Sprintf(`%s CLI for retrieving config or secret variables. Using a specific tokens as an array item`, config.SELF_NAME), - SilenceUsage: true, - Version: fmt.Sprintf("%s-%s", Version, Revision), + SilenceUsage: true, + SilenceErrors: true, + Version: fmt.Sprintf("%s-%s", Version, Revision), }, logger: logger, rootFlags: &rootCmdFlags{}, } rc.Cmd.PersistentFlags().BoolVarP(&rc.rootFlags.verbose, "verbose", "v", false, "Verbosity level") + rc.Cmd.PersistentFlags().BoolVar(&rc.rootFlags.strict, "strict", false, "Exit with a non-zero exit code if any provider fails") rc.Cmd.PersistentFlags().StringVarP(&rc.rootFlags.tokenSeparator, "token-separator", "s", "#", "Separator to use to mark concrete store and the key within it") rc.Cmd.PersistentFlags().StringVarP(&rc.rootFlags.keySeparator, "key-separator", "k", "|", "Separator to use to mark a key look up in a map. e.g. AWSSECRETS#/token/map|key1") rc.Cmd.PersistentFlags().BoolVarP(&rc.rootFlags.enableEnvSubst, "enable-envsubst", "e", false, "Enable envsubst on input. This will fail on any unset or empty variables") @@ -72,7 +75,7 @@ func cmdutilsInit(rootCmd *Root, cmd *cobra.Command, path string) (*cmdutils.Cmd } cm := configmanager.New(cmd.Context()) - cm.Config.WithTokenSeparator(rootCmd.rootFlags.tokenSeparator).WithOutputPath(path).WithKeySeparator(rootCmd.rootFlags.keySeparator).WithEnvSubst(rootCmd.rootFlags.enableEnvSubst) + cm.Config.WithTokenSeparator(rootCmd.rootFlags.tokenSeparator).WithOutputPath(path).WithKeySeparator(rootCmd.rootFlags.keySeparator).WithEnvSubst(rootCmd.rootFlags.enableEnvSubst).WithStrict(rootCmd.rootFlags.strict) gnrtr := generator.NewGenerator(cmd.Context(), func(gv *generator.GenVars) { if rootCmd.rootFlags.verbose { rootCmd.logger.SetLevel(log.DebugLvl) diff --git a/cmd/configmanager/fromfileinput.go b/cmd/configmanager/fromfileinput.go index 4b8e915..252daa9 100644 --- a/cmd/configmanager/fromfileinput.go +++ b/cmd/configmanager/fromfileinput.go @@ -2,6 +2,7 @@ package cmd import ( "fmt" + "os" "github.com/DevLabFoundry/configmanager/v2/internal/cmdutils" "github.com/spf13/cobra" @@ -30,9 +31,16 @@ func newFromStrCmd(rootCmd *Root) { if err != nil { return err } - defer outputWriter.Close() - return cu.GenerateStrOut(inputReader, f.input == f.path) + err = cu.GenerateStrOut(inputReader, f.input == f.path) + outputWriter.Close() + if err != nil { + if rootCmd.rootFlags.strict && f.path != "stdout" { + os.Remove(f.path) + } + return err + } + return nil }, PreRunE: func(cmd *cobra.Command, args []string) error { if len(f.input) < 1 { diff --git a/cmd/main.go b/cmd/main.go index ee4999a..3f9802d 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -2,16 +2,20 @@ package main import ( "context" + "fmt" "os" cfgmgr "github.com/DevLabFoundry/configmanager/v2/cmd/configmanager" "github.com/DevLabFoundry/configmanager/v2/internal/log" ) +const redTerminal = "\x1b[31m%s\x1b[0m" + func main() { logger := log.New(os.Stderr) cmd := cfgmgr.NewRootCmd(logger) if err := cmd.Execute(context.Background()); err != nil { + fmt.Fprintf(os.Stderr, redTerminal+"\n", err) os.Exit(1) } } diff --git a/committed.toml b/committed.toml new file mode 100644 index 0000000..41478de --- /dev/null +++ b/committed.toml @@ -0,0 +1,10 @@ +# committed.toml +# Enforces Conventional Commits format: type(scope)?!?: subject +# Versioning impact (configured in GitVersion.yaml): +# Major: any type with ! suffix (e.g. feat!:, fix(scope)!:) or BREAKING CHANGE in footer +# Minor: feat: or feat(scope): +# Patch: all other types (fix:, refactor:, debug:, chore:, etc.) + +style = "conventional" +subject_capitalized = false +imperative_subject = false diff --git a/eirctl.yaml b/eirctl.yaml index 5d75335..f6ed115 100644 --- a/eirctl.yaml +++ b/eirctl.yaml @@ -11,9 +11,18 @@ contexts: container: name: mirror.gcr.io/bash:5.0.18-alpine3.22 + committed: + container: + name: alpine:3 + shell: /bin/sh + shell_args: ["-c"] + pipelines: + commit-lint: + - task: lint:commits + gha:unit:test: - - pipeline: test:unit + - pipeline: test:unit env: ROOT_PKG_NAME: github.com/DevLabFoundry - task: sonar:coverage:prep @@ -30,6 +39,26 @@ pipelines: depends_on: clean tasks: + lint:commits: + context: committed + description: Lint commit messages against Conventional Commits using committed + command: + - | + apk add --no-cache git + git config --global safe.directory '*' + if [ ! -f /eirctl/.eirctl/committed ]; then + echo "Downloading committed binary..." + wget -qO- https://github.com/crate-ci/committed/releases/download/v1.1.11/committed-v1.1.11-x86_64-unknown-linux-musl.tar.gz \ + | tar xz -C /tmp + install -m 755 /tmp/committed /eirctl/.eirctl/committed + fi + if [ -n "$GITHUB_BASE_REF" ]; then + RANGE="HEAD^1..HEAD^2" + else + RANGE="origin/main..HEAD" + fi + /eirctl/.eirctl/committed $RANGE --no-merge-commit + show:coverage: description: Opens the current coverage viewer for the the configmanager utility. command: go tool cover -html=.coverage/out @@ -39,7 +68,6 @@ tasks: Opens a webview with godoc running Already filters the packages to this one and enables internal/private package documentation - # go install golang.org/x/tools/cmd/godoc@latest command: | open http://localhost:6060/pkg/github.com/DevLabFoundry/configmanager/v2/?m=all godoc -notes "BUG|TODO" -play -http=:6060 @@ -90,7 +118,7 @@ tasks: sonar:coverage:prep: context: bash command: - - | + - | sed -i 's|github.com/DevLabFoundry/configmanager/v2/||g' .coverage/out echo "Coverage file first 20 lines after conversion:" head -20 .coverage/out diff --git a/internal/cmdutils/cmdutils.go b/internal/cmdutils/cmdutils.go index f1f71dc..718908f 100644 --- a/internal/cmdutils/cmdutils.go +++ b/internal/cmdutils/cmdutils.go @@ -48,7 +48,8 @@ func (c *CmdUtils) generateFromToken(tokens []string) error { pm, err := c.configManager.Retrieve(tokens) if err != nil { // return full error to terminal if no tokens were parsed - if len(pm) < 1 { + // or if strict mode is enabled + if len(pm) < 1 || c.configManager.GeneratorConfig().Strict() { return err } c.logger.Error("%v", err) diff --git a/internal/config/config.go b/internal/config/config.go index 50aecce..c4e37c3 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -57,6 +57,7 @@ type GenVarsConfig struct { tokenSeparator string keySeparator string enableEnvSubst bool + strict bool // parseAdditionalVars func(token string) TokenConfigVars } @@ -94,6 +95,12 @@ func (c *GenVarsConfig) WithEnvSubst(enabled bool) *GenVarsConfig { return c } +// WithStrict sets strict mode - errors from any provider will be returned +func (c *GenVarsConfig) WithStrict(strict bool) *GenVarsConfig { + c.strict = strict + return c +} + // OutputPath returns the outpath set in the config func (c *GenVarsConfig) OutputPath() string { return c.outpath @@ -114,6 +121,11 @@ func (c *GenVarsConfig) EnvSubstEnabled() bool { return c.enableEnvSubst } +// Strict returns whether strict mode is enabled +func (c *GenVarsConfig) Strict() bool { + return c.strict +} + // Config returns the derefed value func (c *GenVarsConfig) Config() GenVarsConfig { cc := *c diff --git a/internal/strategy/strategy_test.go b/internal/strategy/strategy_test.go index 93c4c6c..a24a71a 100644 --- a/internal/strategy/strategy_test.go +++ b/internal/strategy/strategy_test.go @@ -232,7 +232,7 @@ func Test_SelectImpl_With(t *testing.T) { }, "success GCPSECRETS": { func() func() { - cf, _ := os.CreateTemp(".", "*") + cf, _ := os.CreateTemp(os.TempDir(), "gcp-creds*") cf.Write(TEST_GCP_CREDS) os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", cf.Name()) return func() { diff --git a/pkg/generator/generator.go b/pkg/generator/generator.go index 4d8bd69..5039415 100644 --- a/pkg/generator/generator.go +++ b/pkg/generator/generator.go @@ -3,9 +3,11 @@ package generator import ( "context" "encoding/json" + "errors" "fmt" "io" "strconv" + "strings" "sync" "github.com/DevLabFoundry/configmanager/v2/internal/config" @@ -114,13 +116,15 @@ func (tms *tokenMapSafe) getTokenMap() ParsedMap { return tms.tokenMap } -func (tms *tokenMapSafe) addKeyVal(key *config.ParsedTokenConfig, val string) { +func (tms *tokenMapSafe) addKeyVal(key *config.ParsedTokenConfig, val string) error { tms.mu.Lock() defer tms.mu.Unlock() // NOTE: still use the metadata in the key // there could be different versions / labels for the same token and hence different values // However the JSONpath look up - tms.tokenMap[key.String()] = keySeparatorLookup(key, val) + v, err := keySeparatorLookup(key, val) + tms.tokenMap[key.String()] = v + return err } type rawTokenMap struct { @@ -199,17 +203,15 @@ func (c *GenVars) generate(rawMap *rawTokenMap) error { // Fan-in: receive results with pure select received := 0 + var errs []error for received < tokenCount { select { case cr := <-outCh: if cr == nil { continue // defensive (shouldn't happen) } - c.Logger.Debug("cro: %+v", cr) - if cr.Err != nil { - c.Logger.Debug("cr.err %v, for token: %s", cr.Err, cr.Key()) - } else { - c.rawMap.addKeyVal(cr.Key(), cr.Value()) + if err := c.handleTokenResponse(cr); err != nil { + errs = append(errs, err) } received++ case <-c.ctx.Done(): @@ -217,9 +219,44 @@ func (c *GenVars) generate(rawMap *rawTokenMap) error { return c.ctx.Err() // propagate context error (cancel/timeout) } } + if c.config.Strict() && len(errs) > 0 { + return fmt.Errorf("%d token(s) failed:\n %s", len(errs), joinErrors(errs)) + } return nil } +func (c *GenVars) handleTokenResponse(cr *strategy.TokenResponse) error { + c.Logger.Debug("cro: %+v", cr) + if cr.Err != nil { + c.Logger.Debug("cr.err %v, for token: %s", cr.Err, cr.Key()) + return fmt.Errorf("%s: %s", cr.Key(), rootCause(cr.Err)) + } + if err := c.rawMap.addKeyVal(cr.Key(), cr.Value()); err != nil { + return fmt.Errorf("%s: %s", cr.Key(), err) + } + return nil +} + +// rootCause unwraps an error chain and returns the innermost error message. +func rootCause(err error) string { + for { + inner := errors.Unwrap(err) + if inner == nil { + return err.Error() + } + err = inner + } +} + +// joinErrors formats errors as an indented list. +func joinErrors(errs []error) string { + msgs := make([]string, len(errs)) + for i, e := range errs { + msgs[i] = e.Error() + } + return strings.Join(msgs, "\n ") +} + // IsParsed will try to parse the return found string into // map[string]string // If found it will convert that to a map with all keys uppercased @@ -233,18 +270,18 @@ func IsParsed(v any, trm ParsedMap) bool { // keySeparatorLookup checks if the key contains // keySeparator character // If it does contain one then it tries to parse -func keySeparatorLookup(key *config.ParsedTokenConfig, val string) string { +func keySeparatorLookup(key *config.ParsedTokenConfig, val string) (string, error) { // key has separator k := key.LookupKeys() if k == "" { // c.logger.Info("no keyseparator found") - return val + return val, nil } keys, err := ajson.JSONPath([]byte(val), fmt.Sprintf("$..%s", k)) if err != nil { // c.logger.Debug("unable to parse as json object %v", err.Error()) - return val + return val, nil } if len(keys) == 1 { @@ -253,14 +290,13 @@ func keySeparatorLookup(key *config.ParsedTokenConfig, val string) string { str, err := strconv.Unquote(fmt.Sprintf("%v", v)) if err != nil { // c.logger.Debug("unable to unquote value: %v returning as is", v) - return fmt.Sprintf("%v", v) + return fmt.Sprintf("%v", v), nil } - return str + return str, nil } - return fmt.Sprintf("%v", v) + return fmt.Sprintf("%v", v), nil } - // c.logger.Info("no value found in json using path expression") - return "" + return "", fmt.Errorf("key %q not found in JSON value", k) } diff --git a/pkg/generator/generator_test.go b/pkg/generator/generator_test.go index bfb91b9..87aa92f 100644 --- a/pkg/generator/generator_test.go +++ b/pkg/generator/generator_test.go @@ -546,3 +546,62 @@ func Test_IsParsed(t *testing.T) { // }) // } // } + +func Test_Generate_StrictMode(t *testing.T) { + t.Run("strict mode returns error on provider failure", func(t *testing.T) { + var custFunc = func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { + m := &mockGenerate{"UNKNOWN://mountPath/token", "", fmt.Errorf("provider unavailable")} + return m, nil + } + + conf := config.NewConfig() + conf.WithStrict(true) + g := generator.NewGenerator(context.TODO(), func(gv *generator.GenVars) { + gv.Logger = log.New(&bytes.Buffer{}) + }) + g.WithConfig(conf) + g.WithStrategyMap(strategy.StrategyFuncMap{config.UnknownPrefix: custFunc}) + _, err := g.Generate([]string{"UNKNOWN://mountPath/token"}) + + if err == nil { + t.Fatal("expected error in strict mode but got nil") + } + }) + + t.Run("strict mode returns error on missing JSON key", func(t *testing.T) { + var custFunc = func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { + m := &mockGenerate{"token", `{"foo":"bar"}`, nil} + return m, nil + } + + conf := config.NewConfig() + conf.WithStrict(true) + g := generator.NewGenerator(context.TODO(), func(gv *generator.GenVars) { + gv.Logger = log.New(&bytes.Buffer{}) + }) + g.WithConfig(conf) + g.WithStrategyMap(strategy.StrategyFuncMap{config.UnknownPrefix: custFunc}) + _, err := g.Generate([]string{"UNKNOWN://mountPath/token|nonexistent"}) + + if err == nil { + t.Fatal("expected error in strict mode for missing JSON key but got nil") + } + }) + + t.Run("non-strict mode does not return error on provider failure", func(t *testing.T) { + var custFunc = func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { + m := &mockGenerate{"UNKNOWN://mountPath/token", "", fmt.Errorf("provider unavailable")} + return m, nil + } + + g := generator.NewGenerator(context.TODO(), func(gv *generator.GenVars) { + gv.Logger = log.New(&bytes.Buffer{}) + }) + g.WithStrategyMap(strategy.StrategyFuncMap{config.UnknownPrefix: custFunc}) + _, err := g.Generate([]string{"UNKNOWN://mountPath/token"}) + + if err != nil { + t.Fatalf("expected no error in non-strict mode but got: %v", err) + } + }) +}