feat(secret): secret reference

Author: vm-001

config.yml

@@ -180,3 +180,34 @@ tracing:
                                                 # Example of grpc:
                                                 #   protocol: grpc
                                                 #   endpoint: localhost:4317
+
+
+#------------------------------------------------------------------------------
+# Secret (remote secret)
+# reference: {secret://<provider>/<name>[?<properties>][#/jsonpointer]}
+# Example: {secret://aws/path/to/secret#/password?ttl=1}
+#------------------------------------------------------------------------------
+secret:
+  providers:                                    # Specifies enabled providers.
+    - '@default'
+    - '@all'
+  ttl: 10s
+  aws:                                          # AWS SecretManager provider
+    region: us-west-1
+    url: http://localhost:4566 # aws_url?
+
+  vault:                                        # HashiCorp Vault provider
+    address: http://localhost:8200
+    mount_path: secret
+    namespace:
+    auth_method: approle
+    authn:
+      token:
+        token:
+      approle:
+        role_id:
+        secret_id:
+        response_wrapping: false
+      kubernetes:
+        role:
+        token_path:

config/config.go

@@ -1,12 +1,17 @@
 package config
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"slices"
 
 	"github.com/creasty/defaults"
 	"github.com/webhookx-io/webhookx/pkg/envconfig"
+	"github.com/webhookx-io/webhookx/pkg/secret"
+	"github.com/webhookx-io/webhookx/pkg/secret/reference"
+	"github.com/webhookx-io/webhookx/utils"
+	"go.uber.org/zap"
 	"gopkg.in/yaml.v3"
 )
 
@@ -37,6 +42,7 @@ type Config struct {
 	Tracing          TracingConfig   `yaml:"tracing" json:"tracing" envconfig:"TRACING"`
 	Role             Role            `yaml:"role" json:"role" envconfig:"ROLE" default:"standalone"`
 	AnonymousReports bool            `yaml:"anonymous_reports" json:"anonymous_reports" envconfig:"ANONYMOUS_REPORTS" default:"true"`
+	Secret           SecretConfig    `yaml:"secret" json:"secret" envconfig:"SECRET"`
 }
 
 func (cfg Config) String() string {
@@ -81,6 +87,9 @@ func (cfg Config) Validate() error {
 	if !slices.Contains([]Role{RoleStandalone, RoleCP, RoleDPWorker, RoleDPProxy}, cfg.Role) {
 		return fmt.Errorf("invalid role: '%s'", cfg.Role)
 	}
+	if err := cfg.Secret.Validate(); err != nil {
+		return err
+	}
 
 	return nil
 }
@@ -89,27 +98,126 @@ type Options struct {
 	YAML []byte
 }
 
+func resolveReferences(n *yaml.Node, manager *secret.Manager) error {
+	switch n.Kind {
+	case yaml.ScalarNode:
+		if reference.IsReference(n.Value) {
+			ref, err := reference.Parse(n.Value)
+			if err != nil {
+				return err
+			}
+			val, err := manager.ResolveReference(context.TODO(), ref)
+			if err != nil {
+				return err
+			}
+			n.Value = val
+		}
+	case yaml.MappingNode:
+		for i := 0; i < len(n.Content); i += 2 {
+			if err := resolveReferences(n.Content[i+1], manager); err != nil {
+				return err
+			}
+		}
+	case yaml.AliasNode:
+		if n.Alias != nil {
+			if err := resolveReferences(n.Alias, manager); err != nil {
+				return err
+			}
+		}
+	default:
+		for _, c := range n.Content {
+			if err := resolveReferences(c, manager); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
 func New(opts *Options) (*Config, error) {
 	var cfg Config
 	err := defaults.Set(&cfg)
 	if err != nil {
 		return nil, err
 	}
 
-	if opts != nil {
-		if len(opts.YAML) > 0 {
-			if err := yaml.Unmarshal(opts.YAML, &cfg); err != nil {
-				return nil, err
+	if opts == nil {
+		opts = &Options{}
+	}
+
+	var doc *yaml.Node
+	if len(opts.YAML) > 0 {
+		doc = new(yaml.Node)
+		if err := yaml.Unmarshal(opts.YAML, doc); err != nil {
+			return nil, err
+		}
+	}
+
+	// todo: only if secret feature is allowed
+	if doc != nil {
+		// fixme
+		if len(doc.Content) > 0 {
+			if node := utils.FindYaml(doc.Content[0], "secret"); node != nil {
+				if err := node.Decode(&cfg.Secret); err != nil {
+					return nil, err
+				}
 			}
 		}
 	}
+	if err := envconfig.Process("WEBHOOKX_SECRET", &cfg.Secret); err != nil {
+		return nil, err
+	}
 
-	err = envconfig.Process("WEBHOOKX", &cfg)
-	if err != nil {
+	var reader = envconfig.EnvironmentReader
+
+	if err := cfg.Secret.Validate(); err != nil {
 		return nil, err
 	}
 
-	return &cfg, nil
+	if cfg.Secret.Enabled() {
+		providers := make(map[string]map[string]interface{})
+		for _, p := range cfg.Secret.Providers {
+			name := string(p)
+			providers[name] = cfg.Secret.GetProviderConfiguration(name)
+		}
+		manager := secret.NewManager(zap.S(), providers)
+		reader = func(key string) (string, bool, error) {
+			value, ok, _ := envconfig.EnvironmentReader.Read(key)
+			if ok && reference.IsReference(value) {
+				ref, err := reference.Parse(value)
+				if err != nil {
+					return "", false, err
+				}
+				resolved, err := manager.ResolveReference(context.TODO(), ref)
+				if err != nil {
+					return "", false, err
+				}
+				value = resolved
+			}
+			return value, ok, nil
+		}
+
+		if doc != nil {
+			if len(doc.Content) > 0 {
+				if err := resolveReferences(doc.Content[0], manager); err != nil {
+					return nil, err
+				}
+			}
+			resolvedYaml := utils.Must(yaml.Marshal(doc))
+			fmt.Println(string(resolvedYaml))
+			doc = new(yaml.Node)
+			_ = yaml.Unmarshal(resolvedYaml, doc)
+		}
+	}
+
+	if doc != nil {
+		if err := doc.Decode(&cfg); err != nil {
+			return nil, err
+		}
+	}
+
+	err = envconfig.ProcessWithReader("WEBHOOKX", &cfg, reader)
+	return &cfg, err
 }
 
 func (cfg *Config) OverrideByRole(role Role) {

config/config_test.go

@@ -481,6 +481,47 @@ func TestWorkerProxyConfig(t *testing.T) {
 	}
 }
 
+func TestSecretConfig(t *testing.T) {
+	tests := []struct {
+		desc        string
+		cfg         SecretConfig
+		validateErr error
+	}{
+		{
+			desc: "sanity",
+			cfg: SecretConfig{
+				Vault: VaultProviderConfig{
+					AuthMethod: "token",
+				},
+			},
+			validateErr: nil,
+		},
+		{
+			desc: "invalid provider",
+			cfg: SecretConfig{
+				Providers: []Provider{"aws", "vault", "unknown"},
+				Vault: VaultProviderConfig{
+					AuthMethod: "token",
+				},
+			},
+			validateErr: errors.New("invalid provider: unknown"),
+		},
+		{
+			desc: "invalid vault.auth_method",
+			cfg: SecretConfig{
+				Vault: VaultProviderConfig{
+					AuthMethod: "unknown",
+				},
+			},
+			validateErr: errors.New("invalid auth_method: unknown"),
+		},
+	}
+	for _, test := range tests {
+		actual := test.cfg.Validate()
+		assert.Equal(t, test.validateErr, actual, "expected %v got %v", test.validateErr, actual)
+	}
+}
+
 func TestConfig(t *testing.T) {
 	cfg, err := New(nil)
 	assert.Nil(t, err)

config/secret.go

@@ -0,0 +1,76 @@
+package config
+
+import (
+	"fmt"
+	"slices"
+
+	"github.com/webhookx-io/webhookx/pkg/secret/provider/vault"
+	"github.com/webhookx-io/webhookx/utils"
+)
+
+type Provider string
+
+const (
+	ProviderAWS   Provider = "aws"
+	ProviderVault Provider = "vault"
+)
+
+type SecretConfig struct {
+	Providers []Provider          `json:"providers" yaml:"providers" default:"[\"aws\", \"vault\"]"`
+	Aws       AwsProviderConfig   `json:"aws" yaml:"aws"`
+	Vault     VaultProviderConfig `json:"vault" yaml:"vault"`
+}
+
+func (cfg *SecretConfig) Validate() error {
+	for _, name := range cfg.Providers {
+		if !slices.Contains([]Provider{ProviderAWS, ProviderVault}, name) {
+			return fmt.Errorf("invalid provider: %s", name)
+		}
+	}
+	if err := cfg.Aws.Validate(); err != nil {
+		return err
+	}
+	if err := cfg.Vault.Validate(); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (cfg *SecretConfig) Enabled() bool {
+	return len(cfg.Providers) > 0
+}
+
+func (cfg *SecretConfig) GetProviderConfiguration(name string) map[string]interface{} {
+	switch Provider(name) {
+	case ProviderAWS:
+		return utils.Must(utils.StructToMap(cfg.Aws))
+	case ProviderVault:
+		return utils.Must(utils.StructToMap(cfg.Vault))
+	default:
+		return nil
+	}
+}
+
+type AwsProviderConfig struct {
+	Region string `json:"region" yaml:"region"`
+	URL    string `json:"url" yaml:"url"`
+}
+
+func (cfg *AwsProviderConfig) Validate() error {
+	return nil
+}
+
+type VaultProviderConfig struct {
+	Address    string      `json:"address" yaml:"address" default:"http://localhost:8200"`
+	MountPath  string      `json:"mount_path" yaml:"mount_path" default:"secret" split_words:"true"`
+	Namespace  string      `json:"namespace" yaml:"namespace"`
+	AuthMethod string      `json:"auth_method" yaml:"auth_method" default:"token" split_words:"true"`
+	AuthN      vault.AuthN `json:"authn" yaml:"authn"`
+}
+
+func (cfg *VaultProviderConfig) Validate() error {
+	if !slices.Contains([]string{"token", "approle", "kubernetes"}, cfg.AuthMethod) {
+		return fmt.Errorf("invalid auth_method: %s", cfg.AuthMethod)
+	}
+	return nil
+}