diff --git a/admin/admin.go b/admin/admin.go index a951f842..05eb81b9 100644 --- a/admin/admin.go +++ b/admin/admin.go @@ -7,18 +7,18 @@ import ( "os" "time" - "github.com/webhookx-io/webhookx/config" + "github.com/webhookx-io/webhookx/config/modules" "go.uber.org/zap" ) // Admin is an HTTP Server type Admin struct { - cfg *config.AdminConfig + cfg *modules.AdminConfig s *http.Server log *zap.SugaredLogger } -func NewAdmin(cfg config.AdminConfig, handler http.Handler) *Admin { +func NewAdmin(cfg modules.AdminConfig, handler http.Handler) *Admin { s := &http.Server{ Handler: handler, Addr: cfg.Listen, diff --git a/app/app.go b/app/app.go index b0950b6b..0cf9f051 100644 --- a/app/app.go +++ b/app/app.go @@ -93,7 +93,6 @@ func New(cfg *config.Config) (*Application, error) { func (app *Application) initialize() error { cfg := app.cfg - cfg.OverrideByRole(cfg.Role) log, err := log.NewZapLogger(&cfg.Log) if err != nil { diff --git a/cmd/db.go b/cmd/db.go index 243dc3b5..709a14e6 100644 --- a/cmd/db.go +++ b/cmd/db.go @@ -1,10 +1,12 @@ package cmd import ( + "context" "errors" "github.com/golang-migrate/migrate/v4" "github.com/spf13/cobra" + "github.com/webhookx-io/webhookx/config" "github.com/webhookx-io/webhookx/db" "github.com/webhookx-io/webhookx/db/migrator" ) @@ -25,6 +27,7 @@ func newDatabaseResetCmd() *cobra.Command { return errors.New("canceled") } } + cfg := cmd.Context().Value("config").(*config.Config) db, err := db.NewSqlDB(cfg.Database) if err != nil { return err @@ -47,11 +50,18 @@ func newDatabaseResetCmd() *cobra.Command { } func newDatabaseCmd() *cobra.Command { - database := &cobra.Command{ Use: "db", Short: "Database commands", Long: ``, + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + cfg, err := initConfig(configurationFile) + if err != nil { + return err + } + cmd.SetContext(context.WithValue(cmd.Context(), "config", cfg)) + return nil + }, } database.PersistentFlags().StringVarP(&configurationFile, "config", "", "", "The configuration filename") @@ -62,6 +72,7 @@ func newDatabaseCmd() *cobra.Command { Short: "Print the migration status", Long: ``, RunE: func(cmd *cobra.Command, args []string) error { + cfg := cmd.Context().Value("config").(*config.Config) db, err := db.NewSqlDB(cfg.Database) if err != nil { return err @@ -81,6 +92,7 @@ func newDatabaseCmd() *cobra.Command { Short: "Run any new migrations", Long: ``, RunE: func(cmd *cobra.Command, args []string) error { + cfg := cmd.Context().Value("config").(*config.Config) db, err := db.NewSqlDB(cfg.Database) if err != nil { return err diff --git a/cmd/root.go b/cmd/root.go index 481aaed7..c6b1c140 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -3,6 +3,7 @@ package cmd import ( "os" + "github.com/pkg/errors" "github.com/spf13/cobra" "github.com/webhookx-io/webhookx/config" ) @@ -14,24 +15,20 @@ var ( var ( configurationFile string verbose bool - cfg *config.Config ) -func initConfig() { - var err error - - var options config.Options - if configurationFile != "" { - buf, err := os.ReadFile(configurationFile) - cobra.CheckErr(err) - options.YAML = buf +func initConfig(filename string) (*config.Config, error) { + cfg := config.New() + err := config.Load(filename, cfg) + if err != nil { + return nil, errors.Wrap(err, "could not load configuration") } - cfg, err = config.New(&options) - cobra.CheckErr(err) - err = cfg.Validate() - cobra.CheckErr(err) + if err != nil { + return nil, errors.Wrap(err, "invalid configuration") + } + return cfg, nil } func NewRootCmd() *cobra.Command { @@ -41,7 +38,6 @@ func NewRootCmd() *cobra.Command { Long: ``, SilenceUsage: true, } - cobra.OnInitialize(initConfig) cmd.SetOut(os.Stdout) cmd.PersistentFlags().BoolVarP(&verbose, "verbose", "", false, "Verbose logging.") diff --git a/cmd/start.go b/cmd/start.go index 6b90ef64..368fe245 100644 --- a/cmd/start.go +++ b/cmd/start.go @@ -16,6 +16,11 @@ func newStartCmd() *cobra.Command { Short: "Start server", Long: ``, RunE: func(cmd *cobra.Command, args []string) error { + cfg, err := initConfig(configurationFile) + if err != nil { + return err + } + app, err := app.New(cfg) if err != nil { return err diff --git a/config.yml b/config.yml index 786b83a5..bfe0f98d 100644 --- a/config.yml +++ b/config.yml @@ -180,3 +180,51 @@ tracing: # Example of grpc: # protocol: grpc # endpoint: localhost:4317 + + +#------------------------------------------------------------------------------ +# Secret Reference (External Secret Reference) +# Secret Reference allows fetching values from external secret providers. +# +# Syntax: +# {secret:///[.][?]} +# +# Components: +# - The provider name (e.g.`aws`, `vault`). +# - The secret name. +# - A optional JSON Path to extract value from a JSON. +# JSON Path is a series of keys separated by a `.` character. +# Examples: `database.username`, `credentials.1.username`. +# - The optional parameters. +# +# Examples: +# {secret://aws/path/to/mysecret} +# {secret://aws/path/to/mysecret.password} +#------------------------------------------------------------------------------ +secret: + providers: # Specifies enabled providers. + - '@default' # Supported values: + # - `@default`: an alias of built-in providers. + # - `aws` + # - `vault` + + aws: # AWS SecretsManager provider + region: # AWS region. + url: # Optional custom endpoint. + # If unset, uses AWS default endpoint resolution. + + vault: # HashiCorp Vault provider (KV v2) + address: http://127.0.0.1:8200 # Vault server address. + mount_path: secret # The mount path for KV secrets engine. + namespace: # Vault namespace (for Vault Enterprise). + auth_method: token # Authentication method. Supported values: `token`, `approle`, `kubernetes`. + authn: # Authentication configuration. + token: + token: # The token used to making requests to Vault. + approle: + role_id: # RoleID used for login. + secret_id: # SecretID used for login. + response_wrapping: false # Whether to use response-wrapping. Defaults to false. + kubernetes: + role: # Vault role bound to the Kubernetes service account. + token_path: # Path to JWT token file. diff --git a/config/config.go b/config/config.go index 2240976c..a490c2e2 100644 --- a/config/config.go +++ b/config/config.go @@ -6,8 +6,8 @@ import ( "slices" "github.com/creasty/defaults" - "github.com/webhookx-io/webhookx/pkg/envconfig" - "gopkg.in/yaml.v3" + "github.com/webhookx-io/webhookx/config/core" + "github.com/webhookx-io/webhookx/config/modules" ) var ( @@ -24,19 +24,54 @@ const ( RoleDPProxy Role = "dp_proxy" ) +var _ core.Config = &Config{} + type Config struct { - Log LogConfig `yaml:"log" json:"log" envconfig:"LOG"` - AccessLog AccessLogConfig `yaml:"access_log" json:"access_log" envconfig:"ACCESS_LOG"` - Database DatabaseConfig `yaml:"database" json:"database" envconfig:"DATABASE"` - Redis RedisConfig `yaml:"redis" json:"redis" envconfig:"REDIS"` - Admin AdminConfig `yaml:"admin" json:"admin" envconfig:"ADMIN"` - Status StatusConfig `yaml:"status" json:"status" envconfig:"STATUS"` - Proxy ProxyConfig `yaml:"proxy" json:"proxy" envconfig:"PROXY"` - Worker WorkerConfig `yaml:"worker" json:"worker" envconfig:"WORKER"` - Metrics MetricsConfig `yaml:"metrics" json:"metrics" envconfig:"METRICS"` - Tracing TracingConfig `yaml:"tracing" json:"tracing" envconfig:"TRACING"` - Role Role `yaml:"role" json:"role" envconfig:"ROLE" default:"standalone"` - AnonymousReports bool `yaml:"anonymous_reports" json:"anonymous_reports" envconfig:"ANONYMOUS_REPORTS" default:"true"` + core.BaseConfig + Log modules.LogConfig `yaml:"log" json:"log" envconfig:"LOG"` + AccessLog modules.AccessLogConfig `yaml:"access_log" json:"access_log" envconfig:"ACCESS_LOG"` + Database modules.DatabaseConfig `yaml:"database" json:"database" envconfig:"DATABASE"` + Redis modules.RedisConfig `yaml:"redis" json:"redis" envconfig:"REDIS"` + Admin modules.AdminConfig `yaml:"admin" json:"admin" envconfig:"ADMIN"` + Status modules.StatusConfig `yaml:"status" json:"status" envconfig:"STATUS"` + Proxy modules.ProxyConfig `yaml:"proxy" json:"proxy" envconfig:"PROXY"` + Worker modules.WorkerConfig `yaml:"worker" json:"worker" envconfig:"WORKER"` + Metrics modules.MetricsConfig `yaml:"metrics" json:"metrics" envconfig:"METRICS"` + Tracing modules.TracingConfig `yaml:"tracing" json:"tracing" envconfig:"TRACING"` + Role Role `yaml:"role" json:"role" envconfig:"ROLE" default:"standalone"` + AnonymousReports bool `yaml:"anonymous_reports" json:"anonymous_reports" envconfig:"ANONYMOUS_REPORTS" default:"true"` + Secret modules.SecretConfig `yaml:"secret" json:"secret" envconfig:"SECRET"` +} + +func (c *Config) GetSecret() *modules.SecretConfig { + return &c.Secret +} + +func (cfg *Config) Sanitize() { + //TODO implement me + panic("implement me") +} + +func (cfg *Config) PostProcess() error { + switch cfg.Role { + case RoleCP: + if cfg.Admin.Listen == "" { + cfg.Admin.Listen = "127.0.0.1:9601" + } + cfg.Proxy.Listen = "" + cfg.Worker.Enabled = false + case RoleDPProxy: + if cfg.Proxy.Listen == "" { + cfg.Proxy.Listen = "0.0.0.0:9600" + } + cfg.Admin.Listen = "" + cfg.Worker.Enabled = false + case RoleDPWorker: + cfg.Admin.Listen = "" + cfg.Proxy.Listen = "" + cfg.Worker.Enabled = true + } + return nil } func (cfg Config) String() string { @@ -81,54 +116,17 @@ func (cfg Config) Validate() error { if !slices.Contains([]Role{RoleStandalone, RoleCP, RoleDPWorker, RoleDPProxy}, cfg.Role) { return fmt.Errorf("invalid role: '%s'", cfg.Role) } + if err := cfg.Secret.Validate(); err != nil { + return err + } return nil } -type Options struct { - YAML []byte -} - -func New(opts *Options) (*Config, error) { +func New() *Config { var cfg Config - err := defaults.Set(&cfg) - if err != nil { - return nil, err - } - - if opts != nil { - if len(opts.YAML) > 0 { - if err := yaml.Unmarshal(opts.YAML, &cfg); err != nil { - return nil, err - } - } - } - - err = envconfig.Process("WEBHOOKX", &cfg) - if err != nil { - return nil, err - } - - return &cfg, nil -} - -func (cfg *Config) OverrideByRole(role Role) { - switch role { - case RoleCP: - if cfg.Admin.Listen == "" { - cfg.Admin.Listen = "127.0.0.1:9601" - } - cfg.Proxy.Listen = "" - cfg.Worker.Enabled = false - case RoleDPProxy: - if cfg.Proxy.Listen == "" { - cfg.Proxy.Listen = "0.0.0.0:9600" - } - cfg.Admin.Listen = "" - cfg.Worker.Enabled = false - case RoleDPWorker: - cfg.Admin.Listen = "" - cfg.Proxy.Listen = "" - cfg.Worker.Enabled = true + if err := defaults.Set(&cfg); err != nil { + panic(err) } + return &cfg } diff --git a/config/config_test.go b/config/config_test.go index 09ad507d..5ed1799c 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -3,21 +3,21 @@ package config import ( "encoding/json" "errors" - "os" "testing" "github.com/stretchr/testify/assert" + "github.com/webhookx-io/webhookx/config/modules" ) func TestRedisConfig(t *testing.T) { tests := []struct { desc string - cfg RedisConfig + cfg modules.RedisConfig expectedValidateErr error }{ { desc: "sanity", - cfg: RedisConfig{ + cfg: modules.RedisConfig{ Host: "127.0.0.1", Port: 6379, Password: "", @@ -26,7 +26,7 @@ func TestRedisConfig(t *testing.T) { }, { desc: "invalid port", - cfg: RedisConfig{ + cfg: modules.RedisConfig{ Host: "127.0.0.1", Port: 65536, Password: "", @@ -43,36 +43,36 @@ func TestRedisConfig(t *testing.T) { func TestLogConfig(t *testing.T) { tests := []struct { desc string - cfg LogConfig + cfg modules.LogConfig expectedValidateErr error }{ { desc: "sanity", - cfg: LogConfig{ - Level: LogLevelInfo, - Format: LogFormatText, + cfg: modules.LogConfig{ + Level: modules.LogLevelInfo, + Format: modules.LogFormatText, }, expectedValidateErr: nil, }, { desc: "invalid level", - cfg: LogConfig{ + cfg: modules.LogConfig{ Level: "", - Format: LogFormatText, + Format: modules.LogFormatText, }, expectedValidateErr: errors.New("invalid level: "), }, { desc: "invalid level: x", - cfg: LogConfig{ + cfg: modules.LogConfig{ Level: "x", - Format: LogFormatText, + Format: modules.LogFormatText, }, expectedValidateErr: errors.New("invalid level: x"), }, { desc: "invalid format", - cfg: LogConfig{ + cfg: modules.LogConfig{ Level: "info", Format: "", }, @@ -80,7 +80,7 @@ func TestLogConfig(t *testing.T) { }, { desc: "invalid format: x", - cfg: LogConfig{ + cfg: modules.LogConfig{ Level: "info", Format: "x", }, @@ -96,13 +96,13 @@ func TestLogConfig(t *testing.T) { func TestProxyConfig(t *testing.T) { tests := []struct { desc string - cfg ProxyConfig + cfg modules.ProxyConfig expectedValidateErr error }{ { desc: "sanity", - cfg: ProxyConfig{ - Queue: Queue{ + cfg: modules.ProxyConfig{ + Queue: modules.Queue{ Type: "redis", }, }, @@ -110,9 +110,9 @@ func TestProxyConfig(t *testing.T) { }, { desc: "max_request_body_size cannot be negative value", - cfg: ProxyConfig{ + cfg: modules.ProxyConfig{ MaxRequestBodySize: -1, - Queue: Queue{ + Queue: modules.Queue{ Type: "redis", }, }, @@ -120,9 +120,9 @@ func TestProxyConfig(t *testing.T) { }, { desc: "timeout_read cannot be negative value", - cfg: ProxyConfig{ + cfg: modules.ProxyConfig{ TimeoutRead: -1, - Queue: Queue{ + Queue: modules.Queue{ Type: "redis", }, }, @@ -130,9 +130,9 @@ func TestProxyConfig(t *testing.T) { }, { desc: "timeout_write cannot be negative value", - cfg: ProxyConfig{ + cfg: modules.ProxyConfig{ TimeoutWrite: -1, - Queue: Queue{ + Queue: modules.Queue{ Type: "redis", }, }, @@ -140,8 +140,8 @@ func TestProxyConfig(t *testing.T) { }, { desc: "invalid type: unknown", - cfg: ProxyConfig{ - Queue: Queue{ + cfg: modules.ProxyConfig{ + Queue: modules.Queue{ Type: "unknown", }, }, @@ -149,10 +149,10 @@ func TestProxyConfig(t *testing.T) { }, { desc: "invalid queue", - cfg: ProxyConfig{ - Queue: Queue{ + cfg: modules.ProxyConfig{ + Queue: modules.Queue{ Type: "redis", - Redis: RedisConfig{ + Redis: modules.RedisConfig{ Port: 65536, }, }, @@ -169,16 +169,16 @@ func TestProxyConfig(t *testing.T) { func TestMetricsConfig(t *testing.T) { tests := []struct { desc string - cfg MetricsConfig + cfg modules.MetricsConfig expectedValidateErr error }{ { desc: "sanity", - cfg: MetricsConfig{ + cfg: modules.MetricsConfig{ Attributes: nil, Exports: nil, PushInterval: 1, - Opentelemetry: OpentelemetryMetrics{ + Opentelemetry: modules.OpentelemetryMetrics{ Protocol: "http/protobuf", }, }, @@ -186,11 +186,11 @@ func TestMetricsConfig(t *testing.T) { }, { desc: "invalid export", - cfg: MetricsConfig{ + cfg: modules.MetricsConfig{ Attributes: nil, - Exports: []Export{"unknown"}, + Exports: []modules.Export{"unknown"}, PushInterval: 1, - Opentelemetry: OpentelemetryMetrics{ + Opentelemetry: modules.OpentelemetryMetrics{ Protocol: "http/protobuf", }, }, @@ -198,11 +198,11 @@ func TestMetricsConfig(t *testing.T) { }, { desc: "invalid protocol", - cfg: MetricsConfig{ + cfg: modules.MetricsConfig{ Attributes: nil, Exports: nil, PushInterval: 1, - Opentelemetry: OpentelemetryMetrics{ + Opentelemetry: modules.OpentelemetryMetrics{ Protocol: "unknown", }, }, @@ -210,11 +210,11 @@ func TestMetricsConfig(t *testing.T) { }, { desc: "invalid PushInterval", - cfg: MetricsConfig{ + cfg: modules.MetricsConfig{ Attributes: nil, Exports: nil, PushInterval: 61, - Opentelemetry: OpentelemetryMetrics{ + Opentelemetry: modules.OpentelemetryMetrics{ Protocol: "http/protobuf", }, }, @@ -231,15 +231,15 @@ func TestMetricsConfig(t *testing.T) { func TestTracingConfig(t *testing.T) { tests := []struct { desc string - cfg TracingConfig + cfg modules.TracingConfig expectedValidateErr error }{ { desc: "sanity", - cfg: TracingConfig{ + cfg: modules.TracingConfig{ Enabled: true, SamplingRate: 0, - Opentelemetry: OpentelemetryTracing{ + Opentelemetry: modules.OpentelemetryTracing{ Protocol: "http/protobuf", Endpoint: "http://localhost:4318/v1/traces", }, @@ -248,10 +248,10 @@ func TestTracingConfig(t *testing.T) { }, { desc: "invalid sampling rate", - cfg: TracingConfig{ + cfg: modules.TracingConfig{ Enabled: true, SamplingRate: 1.1, - Opentelemetry: OpentelemetryTracing{ + Opentelemetry: modules.OpentelemetryTracing{ Protocol: "http/protobuf", Endpoint: "http://localhost:4318/v1/traces", }, @@ -260,8 +260,8 @@ func TestTracingConfig(t *testing.T) { }, { desc: "invalid protocol", - cfg: TracingConfig{ - Opentelemetry: OpentelemetryTracing{ + cfg: modules.TracingConfig{ + Opentelemetry: modules.OpentelemetryTracing{ Protocol: "unknown", }, }, @@ -277,12 +277,12 @@ func TestTracingConfig(t *testing.T) { func TestAccessLogConfig(t *testing.T) { tests := []struct { desc string - cfg AccessLogConfig + cfg modules.AccessLogConfig expectedValidateErr error }{ { desc: "sanity", - cfg: AccessLogConfig{ + cfg: modules.AccessLogConfig{ File: "/dev/stdout", Format: "text", }, @@ -290,7 +290,7 @@ func TestAccessLogConfig(t *testing.T) { }, { desc: "invalid format", - cfg: AccessLogConfig{ + cfg: modules.AccessLogConfig{ File: "/dev/stdout", Format: "", }, @@ -298,7 +298,7 @@ func TestAccessLogConfig(t *testing.T) { }, { desc: "invalid format: x", - cfg: AccessLogConfig{ + cfg: modules.AccessLogConfig{ File: "/dev/stdout", Format: "x", }, @@ -314,12 +314,12 @@ func TestAccessLogConfig(t *testing.T) { func TestStatusConfig(t *testing.T) { tests := []struct { desc string - cfg StatusConfig + cfg modules.StatusConfig expectedValidateErr error }{ { desc: "sanity", - cfg: StatusConfig{ + cfg: modules.StatusConfig{ Listen: "", DebugEndpoints: false, }, @@ -327,7 +327,7 @@ func TestStatusConfig(t *testing.T) { }, { desc: "invalid listen", - cfg: StatusConfig{ + cfg: modules.StatusConfig{ Listen: "invalid", DebugEndpoints: true, }, @@ -341,8 +341,7 @@ func TestStatusConfig(t *testing.T) { } func TestRole(t *testing.T) { - cfg, err := New(nil) - assert.Nil(t, err) + cfg := New() cfg.Role = "standalone" assert.Nil(t, cfg.Validate()) @@ -363,39 +362,39 @@ func TestRole(t *testing.T) { func TestWorkerConfig(t *testing.T) { tests := []struct { desc string - cfg WorkerConfig + cfg modules.WorkerConfig validateErr error }{ { desc: "sanity", - cfg: WorkerConfig{ + cfg: modules.WorkerConfig{ Enabled: false, - Deliverer: WorkerDeliverer{ + Deliverer: modules.WorkerDeliverer{ Timeout: 0, - ACL: ACLConfig{ + ACL: modules.ACLConfig{ Deny: []string{"@default", "0.0.0.0", "0.0.0.0/32", "*.example.com", "foo.example.com", "::1/128"}, }, }, - Pool: Pool{}, + Pool: modules.Pool{}, }, validateErr: nil, }, { desc: "invalid deliverer configuration: negative timeout", - cfg: WorkerConfig{ - Deliverer: WorkerDeliverer{ + cfg: modules.WorkerConfig{ + Deliverer: modules.WorkerDeliverer{ Timeout: -1, - ACL: ACLConfig{}, + ACL: modules.ACLConfig{}, }, }, validateErr: errors.New("deliverer.timeout cannot be negative"), }, { desc: "invalid deliverer configuration: invalid acl configuration 1", - cfg: WorkerConfig{ - Deliverer: WorkerDeliverer{ + cfg: modules.WorkerConfig{ + Deliverer: modules.WorkerDeliverer{ Timeout: 0, - ACL: ACLConfig{ + ACL: modules.ACLConfig{ Deny: []string{"default"}, }, }, @@ -404,10 +403,10 @@ func TestWorkerConfig(t *testing.T) { }, { desc: "invalid deliverer configuration: invalid acl configuration 2", - cfg: WorkerConfig{ - Deliverer: WorkerDeliverer{ + cfg: modules.WorkerConfig{ + Deliverer: modules.WorkerDeliverer{ Timeout: 0, - ACL: ACLConfig{ + ACL: modules.ACLConfig{ Deny: []string{"*"}, }, }, @@ -416,10 +415,10 @@ func TestWorkerConfig(t *testing.T) { }, { desc: "invalid deliverer configuration: unicode hostname", - cfg: WorkerConfig{ - Deliverer: WorkerDeliverer{ + cfg: modules.WorkerConfig{ + Deliverer: modules.WorkerDeliverer{ Timeout: 0, - ACL: ACLConfig{ + ACL: modules.ACLConfig{ Deny: []string{"ั‚ะตัั‚.example.com"}, }, }, @@ -436,40 +435,40 @@ func TestWorkerConfig(t *testing.T) { func TestWorkerProxyConfig(t *testing.T) { tests := []struct { desc string - cfg WorkerDeliverer + cfg modules.WorkerDeliverer validateErr error }{ { desc: "sanity", - cfg: WorkerDeliverer{ + cfg: modules.WorkerDeliverer{ Proxy: "http://example.com:8080", }, validateErr: nil, }, { desc: "invalid proxy url: missing schema", - cfg: WorkerDeliverer{ + cfg: modules.WorkerDeliverer{ Proxy: "example.com", }, validateErr: errors.New("invalid proxy url: 'example.com'"), }, { desc: "invalid proxy url: invalid schema ", - cfg: WorkerDeliverer{ + cfg: modules.WorkerDeliverer{ Proxy: "ftp://example.com", }, validateErr: errors.New("proxy schema must be http or https"), }, { desc: "invalid proxy url: missing host ", - cfg: WorkerDeliverer{ + cfg: modules.WorkerDeliverer{ Proxy: "http://", }, validateErr: errors.New("invalid proxy url: 'http://'"), }, { desc: "invalid proxy url: missing host ", - cfg: WorkerDeliverer{ + cfg: modules.WorkerDeliverer{ Proxy: "http ://", }, validateErr: errors.New("invalid proxy url: parse \"http ://\": first path segment in URL cannot contain colon"), @@ -481,13 +480,53 @@ func TestWorkerProxyConfig(t *testing.T) { } } +func TestSecretConfig(t *testing.T) { + tests := []struct { + desc string + cfg modules.SecretConfig + validateErr error + }{ + { + desc: "sanity", + cfg: modules.SecretConfig{ + Vault: modules.VaultProviderConfig{ + AuthMethod: "token", + }, + }, + validateErr: nil, + }, + { + desc: "invalid provider", + cfg: modules.SecretConfig{ + Providers: []modules.Provider{"aws", "vault", "unknown"}, + Vault: modules.VaultProviderConfig{ + AuthMethod: "token", + }, + }, + validateErr: errors.New("invalid provider: unknown"), + }, + { + desc: "invalid vault.auth_method", + cfg: modules.SecretConfig{ + Vault: modules.VaultProviderConfig{ + AuthMethod: "unknown", + }, + }, + validateErr: errors.New("invalid auth_method: unknown"), + }, + } + for _, test := range tests { + actual := test.cfg.Validate() + assert.Equal(t, test.validateErr, actual, "expected %v got %v", test.validateErr, actual) + } +} + func TestConfig(t *testing.T) { - cfg, err := New(nil) - assert.Nil(t, err) + cfg := New() assert.Nil(t, cfg.Validate()) str := cfg.String() cfg2 := &Config{} - err = json.Unmarshal([]byte(str), cfg2) + err := json.Unmarshal([]byte(str), cfg2) // restore password cfg2.Database.Password = cfg.Database.Password cfg2.Redis.Password = cfg.Redis.Password @@ -497,9 +536,8 @@ func TestConfig(t *testing.T) { } func TestInitWithFile(t *testing.T) { - b, err := os.ReadFile("./testdata/config-empty.yml") - assert.NoError(t, err) - cfg, err := New(&Options{YAML: b}) + cfg := New() + err := Load("./testdata/config-empty.yml", cfg) assert.Nil(t, err) assert.Nil(t, cfg.Validate()) } diff --git a/config/core/config.go b/config/core/config.go new file mode 100644 index 00000000..5dd496fc --- /dev/null +++ b/config/core/config.go @@ -0,0 +1,17 @@ +package core + +const SanitizedValue = "*****" + +type Config interface { + Sanitize() + Validate() error + PostProcess() error +} + +var _ Config = BaseConfig{} + +type BaseConfig struct{} + +func (c BaseConfig) Sanitize() {} +func (c BaseConfig) PostProcess() error { return nil } +func (c BaseConfig) Validate() error { return nil } diff --git a/config/types.go b/config/core/types.go similarity index 94% rename from config/types.go rename to config/core/types.go index 002868d9..c9077137 100644 --- a/config/types.go +++ b/config/core/types.go @@ -1,4 +1,4 @@ -package config +package core import "encoding/json" diff --git a/config/database.go b/config/database.go deleted file mode 100644 index 5c8d9e2c..00000000 --- a/config/database.go +++ /dev/null @@ -1,37 +0,0 @@ -package config - -import ( - "fmt" -) - -type DatabaseConfig struct { - Host string `yaml:"host" json:"host" default:"localhost"` - Port uint32 `yaml:"port" json:"port" default:"5432"` - Username string `yaml:"username" json:"username" default:"webhookx"` - Password Password `yaml:"password" json:"password" default:""` - Database string `yaml:"database" json:"database" default:"webhookx"` - Parameters string `yaml:"parameters" json:"parameters" default:"application_name=webhookx&sslmode=disable&connect_timeout=10"` - MaxPoolSize uint32 `yaml:"max_pool_size" json:"max_pool_size" default:"40" envconfig:"MAX_POOL_SIZE"` - MaxLifetime uint32 `yaml:"max_life_time" json:"max_life_time" default:"1800" envconfig:"MAX_LIFETIME"` -} - -func (cfg DatabaseConfig) GetDSN() string { - dsn := fmt.Sprintf("postgres://%s:%s@%s:%d/%s", - cfg.Username, - cfg.Password, - cfg.Host, - cfg.Port, - cfg.Database, - ) - if len(cfg.Parameters) > 0 { - dsn = fmt.Sprintf("%s?%s", dsn, cfg.Parameters) - } - return dsn -} - -func (cfg DatabaseConfig) Validate() error { - if cfg.Port > 65535 { - return fmt.Errorf("port must be in the range [0, 65535]") - } - return nil -} diff --git a/config/loader.go b/config/loader.go new file mode 100644 index 00000000..79ee52b2 --- /dev/null +++ b/config/loader.go @@ -0,0 +1,72 @@ +package config + +import ( + "github.com/webhookx-io/webhookx/config/core" + "github.com/webhookx-io/webhookx/config/modules" + "github.com/webhookx-io/webhookx/config/providers" + "github.com/webhookx-io/webhookx/pkg/secret" + "go.uber.org/zap" +) + +type Loader struct { + cfg core.Config + filename string + fileContent []byte +} + +func NewLoader(cfg core.Config) *Loader { + return &Loader{cfg: cfg} +} + +func (l *Loader) WithFilename(filename string) *Loader { + l.filename = filename + return l +} + +func (l *Loader) WithFileContent(content []byte) *Loader { + l.fileContent = content + return l +} + +func (l *Loader) Load() error { + var secretCfg = l.cfg.(interface{ GetSecret() *modules.SecretConfig }).GetSecret() + + if err := providers.NewYAMLProvider(l.filename, l.fileContent).WithKey("secret").Load(secretCfg); err != nil { + return err + } + if err := providers.NewEnvProvider("WEBHOOKX_SECRET").Load(secretCfg); err != nil { + return err + } + + var manager *secret.Manager + if secretCfg.Enabled() { + providers := make(map[string]map[string]interface{}) + for _, p := range secretCfg.GetProviders() { + name := string(p) + providers[name] = secretCfg.GetProviderConfiguration(name) + } + var err error + manager, err = secret.NewManager(zap.S(), providers) + if err != nil { + return err + } + } + + list := make([]providers.ConfigProvider, 0) + list = append(list, providers.NewYAMLProvider(l.filename, l.fileContent).WithManager(manager)) + list = append(list, providers.NewEnvProvider("WEBHOOKX").WithManager(manager)) + + for _, provider := range list { + err := provider.Load(l.cfg) + if err != nil { + return err + } + } + + return l.cfg.PostProcess() + // todo l.cfg.Validate() ? +} + +func Load(filename string, cfg core.Config) error { + return NewLoader(cfg).WithFilename(filename).Load() +} diff --git a/config/access_log.go b/config/modules/access_log.go similarity index 86% rename from config/access_log.go rename to config/modules/access_log.go index 5bbe9553..255e166d 100644 --- a/config/access_log.go +++ b/config/modules/access_log.go @@ -1,11 +1,14 @@ -package config +package modules import ( "fmt" "slices" + + "github.com/webhookx-io/webhookx/config/core" ) type AccessLogConfig struct { + core.BaseConfig Enabled bool `yaml:"enabled" json:"enabled" default:"true"` Format LogFormat `yaml:"format" json:"format" default:"text"` Colored bool `yaml:"colored" json:"colored" default:"true"` diff --git a/config/admin.go b/config/modules/admin.go similarity index 87% rename from config/admin.go rename to config/modules/admin.go index 2b591649..425803a7 100644 --- a/config/admin.go +++ b/config/modules/admin.go @@ -1,6 +1,9 @@ -package config +package modules + +import "github.com/webhookx-io/webhookx/config/core" type AdminConfig struct { + core.BaseConfig Listen string `yaml:"listen" json:"listen" default:"127.0.0.1:9601"` DebugEndpoints bool `yaml:"debug_endpoints" json:"debug_endpoints" envconfig:"DEBUG_ENDPOINTS"` TLS TLS `yaml:"tls" json:"tls"` diff --git a/config/modules/database.go b/config/modules/database.go new file mode 100644 index 00000000..79f1f9d8 --- /dev/null +++ b/config/modules/database.go @@ -0,0 +1,40 @@ +package modules + +import ( + "fmt" + + "github.com/webhookx-io/webhookx/config/core" +) + +type DatabaseConfig struct { + core.BaseConfig + Host string `yaml:"host" json:"host" default:"localhost"` + Port uint32 `yaml:"port" json:"port" default:"5432"` + Username string `yaml:"username" json:"username" default:"webhookx"` + Password core.Password `yaml:"password" json:"password" default:""` + Database string `yaml:"database" json:"database" default:"webhookx"` + Parameters string `yaml:"parameters" json:"parameters" default:"application_name=webhookx&sslmode=disable&connect_timeout=10"` + MaxPoolSize uint32 `yaml:"max_pool_size" json:"max_pool_size" default:"40" envconfig:"MAX_POOL_SIZE"` + MaxLifetime uint32 `yaml:"max_life_time" json:"max_life_time" default:"1800" envconfig:"MAX_LIFETIME"` +} + +func (cfg DatabaseConfig) GetDSN() string { + dsn := fmt.Sprintf("postgres://%s:%s@%s:%d/%s", + cfg.Username, + cfg.Password, + cfg.Host, + cfg.Port, + cfg.Database, + ) + if len(cfg.Parameters) > 0 { + dsn = fmt.Sprintf("%s?%s", dsn, cfg.Parameters) + } + return dsn +} + +func (cfg DatabaseConfig) Validate() error { + if cfg.Port > 65535 { + return fmt.Errorf("port must be in the range [0, 65535]") + } + return nil +} diff --git a/config/log.go b/config/modules/log.go similarity index 91% rename from config/log.go rename to config/modules/log.go index 291738f1..46aee4b5 100644 --- a/config/log.go +++ b/config/modules/log.go @@ -1,8 +1,10 @@ -package config +package modules import ( "fmt" "slices" + + "github.com/webhookx-io/webhookx/config/core" ) type LogLevel string @@ -22,6 +24,7 @@ const ( ) type LogConfig struct { + core.BaseConfig Level LogLevel `yaml:"level" json:"level" default:"info"` Format LogFormat `yaml:"format" json:"format" default:"text"` Colored bool `yaml:"colored" json:"colored" default:"true"` diff --git a/config/metrics.go b/config/modules/metrics.go similarity index 89% rename from config/metrics.go rename to config/modules/metrics.go index 7e8bc313..14b04d99 100644 --- a/config/metrics.go +++ b/config/modules/metrics.go @@ -1,12 +1,15 @@ -package config +package modules import ( "fmt" "slices" + + "github.com/webhookx-io/webhookx/config/core" ) type MetricsConfig struct { - Attributes Map `yaml:"attributes" json:"attributes"` + core.BaseConfig + Attributes core.Map `yaml:"attributes" json:"attributes"` Exports []Export `yaml:"exports" json:"exports"` PushInterval uint32 `yaml:"push_interval" json:"push_interval" default:"10" envconfig:"PUSH_INTERVAL"` Opentelemetry OpentelemetryMetrics `yaml:"opentelemetry" json:"opentelemetry"` diff --git a/config/opentelemetry.go b/config/modules/opentelemetry.go similarity index 88% rename from config/opentelemetry.go rename to config/modules/opentelemetry.go index 909b3285..6876559a 100644 --- a/config/opentelemetry.go +++ b/config/modules/opentelemetry.go @@ -1,4 +1,4 @@ -package config +package modules type OtlpProtocol string diff --git a/config/proxy.go b/config/modules/proxy.go similarity index 96% rename from config/proxy.go rename to config/modules/proxy.go index f4d6d9fa..38af9edc 100644 --- a/config/proxy.go +++ b/config/modules/proxy.go @@ -1,9 +1,11 @@ -package config +package modules import ( "errors" "fmt" "slices" + + "github.com/webhookx-io/webhookx/config/core" ) type ProxyResponse struct { @@ -37,6 +39,7 @@ func (cfg Queue) Validate() error { } type ProxyConfig struct { + core.BaseConfig Listen string `yaml:"listen" json:"listen" default:"0.0.0.0:9600"` TLS TLS `yaml:"tls" json:"tls"` TimeoutRead int64 `yaml:"timeout_read" json:"timeout_read" default:"10" envconfig:"TIMEOUT_READ"` diff --git a/config/redis.go b/config/modules/redis.go similarity index 59% rename from config/redis.go rename to config/modules/redis.go index 92fa52ca..380e1db6 100644 --- a/config/redis.go +++ b/config/modules/redis.go @@ -1,18 +1,20 @@ -package config +package modules import ( "fmt" "github.com/redis/go-redis/v9" "github.com/redis/go-redis/v9/maintnotifications" + "github.com/webhookx-io/webhookx/config/core" ) type RedisConfig struct { - Host string `yaml:"host" json:"host" default:"localhost"` - Port uint32 `yaml:"port" json:"port" default:"6379"` - Password Password `yaml:"password" json:"password" default:""` - Database uint32 `yaml:"database" json:"database" default:"0"` - MaxPoolSize uint32 `yaml:"max_pool_size" json:"max_pool_size" default:"0"` + core.BaseConfig + Host string `yaml:"host" json:"host" default:"localhost"` + Port uint32 `yaml:"port" json:"port" default:"6379"` + Password core.Password `yaml:"password" json:"password" default:""` + Database uint32 `yaml:"database" json:"database" default:"0"` + MaxPoolSize uint32 `yaml:"max_pool_size" json:"max_pool_size" default:"0"` } func (cfg RedisConfig) GetClient() *redis.Client { diff --git a/config/modules/secret.go b/config/modules/secret.go new file mode 100644 index 00000000..74b08a6a --- /dev/null +++ b/config/modules/secret.go @@ -0,0 +1,90 @@ +package modules + +import ( + "fmt" + "slices" + + "github.com/webhookx-io/webhookx/config/core" + "github.com/webhookx-io/webhookx/pkg/secret/provider/vault" + "github.com/webhookx-io/webhookx/utils" +) + +type Provider string + +const ( + ProviderAWS Provider = "aws" + ProviderVault Provider = "vault" +) + +type SecretConfig struct { + core.BaseConfig + Providers []Provider `json:"providers" yaml:"providers" default:"[\"@default\"]"` + Aws AwsProviderConfig `json:"aws" yaml:"aws"` + Vault VaultProviderConfig `json:"vault" yaml:"vault"` +} + +func (cfg *SecretConfig) Validate() error { + for _, name := range cfg.Providers { + if !slices.Contains([]Provider{"@default", ProviderAWS, ProviderVault}, name) { + return fmt.Errorf("invalid provider: %s", name) + } + } + if err := cfg.Aws.Validate(); err != nil { + return err + } + if err := cfg.Vault.Validate(); err != nil { + return err + } + return nil +} + +func (cfg *SecretConfig) Enabled() bool { + return len(cfg.Providers) > 0 +} + +func (cfg *SecretConfig) GetProviders() []Provider { + names := make([]Provider, 0) + for _, p := range cfg.Providers { + if p == "@default" { + names = append(names, ProviderAWS, ProviderVault) + } else { + names = append(names, p) + } + } + return names +} + +func (cfg *SecretConfig) GetProviderConfiguration(name string) map[string]interface{} { + switch Provider(name) { + case ProviderAWS: + return utils.Must(utils.StructToMap(cfg.Aws)) + case ProviderVault: + return utils.Must(utils.StructToMap(cfg.Vault)) + default: + return nil + } +} + +type AwsProviderConfig struct { + Region string `json:"region" yaml:"region"` + URL string `json:"url" yaml:"url"` +} + +func (cfg *AwsProviderConfig) Validate() error { + return nil +} + +type VaultProviderConfig struct { + Address string `json:"address" yaml:"address" default:"http://localhost:8200"` + MountPath string `json:"mount_path" yaml:"mount_path" default:"secret" split_words:"true"` + Namespace string `json:"namespace" yaml:"namespace"` + AuthMethod string `json:"auth_method" yaml:"auth_method" default:"token" split_words:"true"` + AuthN vault.AuthN `json:"authn" yaml:"authn"` +} + +func (cfg *VaultProviderConfig) Validate() error { + if !slices.Contains([]string{"token", "approle", "kubernetes"}, cfg.AuthMethod) { + return fmt.Errorf("invalid auth_method: %s", cfg.AuthMethod) + } + return nil +} diff --git a/config/status.go b/config/modules/status.go similarity index 87% rename from config/status.go rename to config/modules/status.go index edf25b47..f442d418 100644 --- a/config/status.go +++ b/config/modules/status.go @@ -1,11 +1,14 @@ -package config +package modules import ( "fmt" "net" + + "github.com/webhookx-io/webhookx/config/core" ) type StatusConfig struct { + core.BaseConfig Listen string `yaml:"listen" json:"listen" default:"127.0.0.1:9602"` DebugEndpoints bool `yaml:"debug_endpoints" json:"debug_endpoints" default:"true" envconfig:"DEBUG_ENDPOINTS"` } diff --git a/config/tracing.go b/config/modules/tracing.go similarity index 88% rename from config/tracing.go rename to config/modules/tracing.go index 3e4c31c2..727ba29d 100644 --- a/config/tracing.go +++ b/config/modules/tracing.go @@ -1,14 +1,17 @@ -package config +package modules import ( "errors" "fmt" "slices" + + "github.com/webhookx-io/webhookx/config/core" ) type TracingConfig struct { + core.BaseConfig Enabled bool `yaml:"enabled" json:"enabled" default:"false"` - Attributes Map `yaml:"attributes" json:"attributes"` + Attributes core.Map `yaml:"attributes" json:"attributes"` Opentelemetry OpentelemetryTracing `yaml:"opentelemetry" json:"opentelemetry"` SamplingRate float64 `yaml:"sampling_rate" json:"sampling_rate" default:"1.0" envconfig:"SAMPLING_RATE"` } diff --git a/config/worker.go b/config/modules/worker.go similarity index 96% rename from config/worker.go rename to config/modules/worker.go index 659aa650..96e77585 100644 --- a/config/worker.go +++ b/config/modules/worker.go @@ -1,4 +1,4 @@ -package config +package modules import ( "fmt" @@ -6,6 +6,8 @@ import ( "net/url" "regexp" "slices" + + "github.com/webhookx-io/webhookx/config/core" ) type WorkerDeliverer struct { @@ -47,6 +49,7 @@ type Pool struct { } type WorkerConfig struct { + core.BaseConfig Enabled bool `yaml:"enabled" json:"enabled" default:"true"` Deliverer WorkerDeliverer `yaml:"deliverer" json:"deliverer"` Pool Pool `yaml:"pool" json:"pool"` diff --git a/config/providers/env.go b/config/providers/env.go new file mode 100644 index 00000000..97fb2872 --- /dev/null +++ b/config/providers/env.go @@ -0,0 +1,46 @@ +package providers + +import ( + "context" + + "github.com/webhookx-io/webhookx/pkg/envconfig" + "github.com/webhookx-io/webhookx/pkg/secret" + "github.com/webhookx-io/webhookx/pkg/secret/reference" +) + +type EnvProvider struct { + prefix string + manager *secret.Manager +} + +func (p *EnvProvider) WithManager(manager *secret.Manager) *EnvProvider { + p.manager = manager + return p +} + +func (p *EnvProvider) Load(cfg any) error { + var reader = envconfig.EnvironmentReader + if p.manager != nil { + reader = func(key string) (string, bool, error) { + value, ok, _ := envconfig.EnvironmentReader.Read(key) + if ok && reference.IsReference(value) { + ref, err := reference.Parse(value) + if err != nil { + return "", false, err + } + resolved, err := p.manager.ResolveReference(context.TODO(), ref) + if err != nil { + return "", false, err + } + value = resolved + } + return value, ok, nil + } + } + + return envconfig.ProcessWithReader(p.prefix, cfg, reader) +} + +func NewEnvProvider(prefix string) *EnvProvider { + return &EnvProvider{prefix: prefix} +} diff --git a/config/providers/provider.go b/config/providers/provider.go new file mode 100644 index 00000000..bc7c93c0 --- /dev/null +++ b/config/providers/provider.go @@ -0,0 +1,5 @@ +package providers + +type ConfigProvider interface { + Load(cfg any) error +} diff --git a/config/providers/yaml.go b/config/providers/yaml.go new file mode 100644 index 00000000..de4b98e2 --- /dev/null +++ b/config/providers/yaml.go @@ -0,0 +1,116 @@ +package providers + +import ( + "context" + "os" + + "github.com/webhookx-io/webhookx/pkg/secret" + "github.com/webhookx-io/webhookx/pkg/secret/reference" + "gopkg.in/yaml.v3" +) + +type YAMLProvider struct { + filename string + content []byte + key string + manager *secret.Manager +} + +func NewYAMLProvider(filename string, content []byte) *YAMLProvider { + return &YAMLProvider{ + filename: filename, + content: content, + } +} + +func (p *YAMLProvider) WithManager(manager *secret.Manager) *YAMLProvider { + p.manager = manager + return p +} + +func (p *YAMLProvider) WithKey(key string) *YAMLProvider { + p.key = key + return p +} + +func resolveReference(n *yaml.Node, manager *secret.Manager) error { + switch n.Kind { + case yaml.ScalarNode: + if reference.IsReference(n.Value) { + ref, err := reference.Parse(n.Value) + if err != nil { + return err + } + val, err := manager.ResolveReference(context.TODO(), ref) + if err != nil { + return err + } + n.Value = val + } + case yaml.MappingNode: + for i := 0; i < len(n.Content); i += 2 { + if err := resolveReference(n.Content[i+1], manager); err != nil { + return err + } + } + case yaml.AliasNode: + if n.Alias != nil { + if err := resolveReference(n.Alias, manager); err != nil { + return err + } + } + default: + for _, c := range n.Content { + if err := resolveReference(c, manager); err != nil { + return err + } + } + } + return nil +} + +func (p *YAMLProvider) Load(cfg any) error { + if p.filename == "" && p.content == nil { + return nil + } + + if p.filename != "" { + b, err := os.ReadFile(p.filename) + if err != nil { + return err + } + p.content = b + } + + var doc yaml.Node + if err := yaml.Unmarshal(p.content, &doc); err != nil { + return err + } + + if p.key != "" && len(doc.Content) > 0 { + if node := findYaml(doc.Content[0], p.key); node != nil { + doc = *node + } + } + + if p.manager != nil { + if err := resolveReference(&doc, p.manager); err != nil { + return err + } + } + + return doc.Decode(cfg) +} + +func findYaml(n *yaml.Node, key string) *yaml.Node { + if n.Kind != yaml.MappingNode { + return nil + } + for i := 0; i < len(n.Content); i += 2 { + k := n.Content[i] + if k.Value == key { + return n.Content[i+1] + } + } + return nil +} diff --git a/db/db.go b/db/db.go index a9fed6e2..1029f208 100644 --- a/db/db.go +++ b/db/db.go @@ -9,7 +9,7 @@ import ( _ "github.com/jackc/pgx/v5/stdlib" "github.com/jmoiron/sqlx" "github.com/pkg/errors" - "github.com/webhookx-io/webhookx/config" + "github.com/webhookx-io/webhookx/config/modules" "github.com/webhookx-io/webhookx/db/dao" "github.com/webhookx-io/webhookx/db/transaction" "github.com/webhookx-io/webhookx/eventbus" @@ -37,7 +37,7 @@ type DB struct { PluginsWS dao.PluginDAO } -func NewSqlDB(cfg config.DatabaseConfig) (*sql.DB, error) { +func NewSqlDB(cfg modules.DatabaseConfig) (*sql.DB, error) { db, err := sql.Open("pgx", cfg.GetDSN()) if err != nil { return nil, err diff --git a/go.mod b/go.mod index db616eab..4ef8a86a 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,10 @@ go 1.25.4 require ( github.com/Masterminds/squirrel v1.5.4 github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef + github.com/aws/aws-sdk-go-v2 v1.40.0 + github.com/aws/aws-sdk-go-v2/config v1.32.0 + github.com/aws/aws-sdk-go-v2/credentials v1.19.0 + github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.40.1 github.com/creasty/defaults v1.8.0 github.com/dop251/goja v0.0.0-20251103141225-af2ceb9156d7 github.com/elazarl/goproxy v1.7.2 @@ -17,9 +21,13 @@ require ( github.com/golang-migrate/migrate/v4 v4.19.0 github.com/gorilla/mux v1.8.1 github.com/hashicorp/golang-lru/v2 v2.0.7 + github.com/hashicorp/vault/api v1.22.0 + github.com/hashicorp/vault/api/auth/approle v0.11.0 + github.com/hashicorp/vault/api/auth/kubernetes v0.10.0 github.com/jackc/pgx/v5 v5.7.6 github.com/jmoiron/sqlx v1.4.0 github.com/lib/pq v1.10.9 + github.com/mitchellh/mapstructure v1.5.0 github.com/onsi/ginkgo/v2 v2.27.2 github.com/onsi/gomega v1.38.2 github.com/pkg/errors v0.9.1 @@ -30,6 +38,7 @@ require ( github.com/spf13/cobra v1.10.1 github.com/stretchr/testify v1.11.1 github.com/tetratelabs/wazero v1.10.0 + github.com/tidwall/gjson v1.18.0 github.com/vmihailenco/msgpack/v5 v5.4.1 go.opentelemetry.io/contrib/propagators/autoprop v0.63.0 go.opentelemetry.io/otel v1.38.0 @@ -52,23 +61,46 @@ require github.com/felixge/httpsnoop v1.0.4 // indirect require ( github.com/Masterminds/semver/v3 v3.4.0 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.14 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.14 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.14 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.14 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.0.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.4 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.8 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.41.1 // indirect + github.com/aws/smithy-go v1.23.2 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/dlclark/regexp2 v1.11.4 // indirect + github.com/go-jose/go-jose/v4 v4.1.2 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect github.com/go-openapi/swag v0.23.0 // indirect github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-retryablehttp v0.7.8 // indirect + github.com/hashicorp/go-rootcerts v1.0.2 // indirect + github.com/hashicorp/go-secure-stdlib/parseutil v0.2.0 // indirect + github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect + github.com/hashicorp/go-sockaddr v1.0.7 // indirect + github.com/hashicorp/hcl v1.0.1-vault-7 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect - github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.19 // indirect - github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/mattn/go-colorable v0.1.14 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect github.com/oasdiff/yaml v0.0.0-20250309154309-f31be36b4037 // indirect github.com/oasdiff/yaml3 v0.0.0-20250309153720-d2182401db90 // indirect github.com/perimeterx/marshmallow v1.1.5 // indirect + github.com/ryanuber/go-glob v1.0.0 // indirect + github.com/tidwall/match v1.2.0 // indirect + github.com/tidwall/pretty v1.2.1 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect github.com/woodsbury/decimal128 v1.3.0 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect @@ -78,6 +110,7 @@ require ( go.opentelemetry.io/contrib/propagators/ot v1.38.0 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/mod v0.27.0 // indirect + golang.org/x/time v0.12.0 // indirect ) require ( diff --git a/go.sum b/go.sum index a26907f3..fcc0f95c 100644 --- a/go.sum +++ b/go.sum @@ -12,10 +12,42 @@ github.com/VividCortex/gohistogram v1.0.0 h1:6+hBz+qvs0JOrrNhhmR7lFxo5sINxBCGXrd github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef h1:2JGTg6JapxP9/R33ZaagQtAM4EkkSYnIAlOG5EI8gkM= github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef/go.mod h1:JS7hed4L1fj0hXcyEejnW57/7LCetXggd+vwrRnYeII= +github.com/aws/aws-sdk-go-v2 v1.40.0 h1:/WMUA0kjhZExjOQN2z3oLALDREea1A7TobfuiBrKlwc= +github.com/aws/aws-sdk-go-v2 v1.40.0/go.mod h1:c9pm7VwuW0UPxAEYGyTmyurVcNrbF6Rt/wixFqDhcjE= +github.com/aws/aws-sdk-go-v2/config v1.32.0 h1:T5WWJYnam9SzBLbsVYDu2HscLDe+GU1AUJtfcDAc/vA= +github.com/aws/aws-sdk-go-v2/config v1.32.0/go.mod h1:pSRm/+D3TxBixGMXlgtX4+MPO9VNtEEtiFmNpxksoxw= +github.com/aws/aws-sdk-go-v2/credentials v1.19.0 h1:7zm+ez+qEqLaNsCSRaistkvJRJv8sByDOVuCnyHbP7M= +github.com/aws/aws-sdk-go-v2/credentials v1.19.0/go.mod h1:pHKPblrT7hqFGkNLxqoS3FlGoPrQg4hMIa+4asZzBfs= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.14 h1:WZVR5DbDgxzA0BJeudId89Kmgy6DIU4ORpxwsVHz0qA= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.14/go.mod h1:Dadl9QO0kHgbrH1GRqGiZdYtW5w+IXXaBNCHTIaheM4= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.14 h1:PZHqQACxYb8mYgms4RZbhZG0a7dPW06xOjmaH0EJC/I= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.14/go.mod h1:VymhrMJUWs69D8u0/lZ7jSB6WgaG/NqHi3gX0aYf6U0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.14 h1:bOS19y6zlJwagBfHxs0ESzr1XCOU2KXJCWcq3E2vfjY= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.14/go.mod h1:1ipeGBMAxZ0xcTm6y6paC2C/J6f6OO7LBODV9afuAyM= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3 h1:x2Ibm/Af8Fi+BH+Hsn9TXGdT+hKbDd5XOTZxTMxDk7o= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3/go.mod h1:IW1jwyrQgMdhisceG8fQLmQIydcT/jWY21rFhzgaKwo= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.14 h1:FIouAnCE46kyYqyhs0XEBDFFSREtdnr8HQuLPQPLCrY= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.14/go.mod h1:UTwDc5COa5+guonQU8qBikJo1ZJ4ln2r1MkF7Dqag1E= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.40.1 h1:w6a0H79HrHf3lr+zrw+pSzR5B+caiQFAKiNHlrUcnoc= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.40.1/go.mod h1:c6Vg0BRiU7v0MVhHupw90RyL120QBwAMLbDCzptGeMk= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.1 h1:BDgIUYGEo5TkayOWv/oBLPphWwNm/A91AebUjAu5L5g= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.1/go.mod h1:iS6EPmNeqCsGo+xQmXv0jIMjyYtQfnwg36zl2FwEouk= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.4 h1:U//SlnkE1wOQiIImxzdY5PXat4Wq+8rlfVEw4Y7J8as= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.4/go.mod h1:av+ArJpoYf3pgyrj6tcehSFW+y9/QvAY8kMooR9bZCw= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.8 h1:MvlNs/f+9eM0mOjD9JzBUbf5jghyTk3p+O9yHMXX94Y= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.8/go.mod h1:/j67Z5XBVDx8nZVp9EuFM9/BS5dvBznbqILGuu73hug= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.1 h1:GdGmKtG+/Krag7VfyOXV17xjTCz0i9NT+JnqLTOI5nA= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.1/go.mod h1:6TxbXoDSgBQ225Qd8Q+MbxUxUh6TtNKwbRt/EPS9xso= +github.com/aws/smithy-go v1.23.2 h1:Crv0eatJUQhaManss33hS5r40CG3ZFH+21XSkqMrIUM= +github.com/aws/smithy-go v1.23.2/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= @@ -50,6 +82,8 @@ github.com/dop251/goja v0.0.0-20251103141225-af2ceb9156d7 h1:jxmXU5V9tXxJnydU5v/ github.com/dop251/goja v0.0.0-20251103141225-af2ceb9156d7/go.mod h1:MxLav0peU43GgvwVgNbLAj1s/bSGboKkhuULvq/7hx4= github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o= github.com/elazarl/goproxy v1.7.2/go.mod h1:82vkLNir0ALaW14Rc399OTTjyNREgmdL2cVoIbS6XaE= +github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= +github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/gabriel-vasile/mimetype v1.4.10 h1:zyueNbySn/z8mJZHLt6IPw0KoZsiQNszIpU+bX4+ZK0= @@ -62,6 +96,8 @@ github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZ github.com/gkampitakis/go-diff v1.3.2/go.mod h1:LLgOrpqleQe26cte8s36HTWcTmMEur6OPYerdAAS9tk= github.com/gkampitakis/go-snaps v0.5.15 h1:amyJrvM1D33cPHwVrjo9jQxX8g/7E2wYdZ+01KS3zGE= github.com/gkampitakis/go-snaps v0.5.15/go.mod h1:HNpx/9GoKisdhw9AFOBT1N7DBs9DiHo/hGheFGBZ+mc= +github.com/go-jose/go-jose/v4 v4.1.2 h1:TK/7NqRQZfgAh+Td8AlsrvtPoUyiHh0LqVvokh+1vHI= +github.com/go-jose/go-jose/v4 v4.1.2/go.mod h1:22cg9HWM1pOlnRiY+9cQYJ9XHmya1bYW8OeDM6Ku6Oo= github.com/go-kit/kit v0.13.0 h1:OoneCcHKHQ03LfBpoQCUfCluwd2Vt3ohz+kvbJneZAU= github.com/go-kit/kit v0.13.0/go.mod h1:phqEHMMUbyrCFCTgH48JueqrM3md2HcAZ8N3XE4FKDg= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -99,8 +135,8 @@ github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpv github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= -github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= -github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= +github.com/go-test/deep v1.1.1 h1:0r/53hagsehfO4bzD2Pgr/+RgHqhmf+k1Bpse2cTu1U= +github.com/go-test/deep v1.1.1/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw= github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -125,10 +161,32 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2/go.mod h1:pkJQ2tZHJ0aFOVEEot6o github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= +github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/go-retryablehttp v0.7.8 h1:ylXZWnqa7Lhqpk0L1P1LzDtGcCR0rPVUrx/c8Unxc48= +github.com/hashicorp/go-retryablehttp v0.7.8/go.mod h1:rjiScheydd+CxvumBsIrFKlx3iS0jrZ7LvzFGFmuKbw= +github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc= +github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= +github.com/hashicorp/go-secure-stdlib/parseutil v0.2.0 h1:U+kC2dOhMFQctRfhK0gRctKAPTloZdMU5ZJxaesJ/VM= +github.com/hashicorp/go-secure-stdlib/parseutil v0.2.0/go.mod h1:Ll013mhdmsVDuoIXVfBtvgGJsXDYkTw1kooNcoCXuE0= +github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 h1:kes8mmyCpxJsI7FTwtzRqEy9CdjCtrXrXGuOpxEA7Ts= +github.com/hashicorp/go-secure-stdlib/strutil v0.1.2/go.mod h1:Gou2R9+il93BqX25LAKCLuM+y9U2T4hlwvT1yprcna4= +github.com/hashicorp/go-sockaddr v1.0.7 h1:G+pTkSO01HpR5qCxg7lxfsFEZaG+C0VssTy/9dbT+Fw= +github.com/hashicorp/go-sockaddr v1.0.7/go.mod h1:FZQbEYa1pxkQ7WLpyXJ6cbjpT8q0YgQaK/JakXqGyWw= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hashicorp/hcl v1.0.1-vault-7 h1:ag5OxFVy3QYTFTJODRzTKVZ6xvdfLLCA1cy/Y6xGI0I= +github.com/hashicorp/hcl v1.0.1-vault-7/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= +github.com/hashicorp/vault/api v1.22.0 h1:+HYFquE35/B74fHoIeXlZIP2YADVboaPjaSicHEZiH0= +github.com/hashicorp/vault/api v1.22.0/go.mod h1:IUZA2cDvr4Ok3+NtK2Oq/r+lJeXkeCrHRmqdyWfpmGM= +github.com/hashicorp/vault/api/auth/approle v0.11.0 h1:ViUvgqoSTqHkMi1L1Rr/LnQ+PWiRaGUBGvx4UPfmKOw= +github.com/hashicorp/vault/api/auth/approle v0.11.0/go.mod h1:v8ZqBRw+GP264ikIw2sEBKF0VT72MEhLWnZqWt3xEG8= +github.com/hashicorp/vault/api/auth/kubernetes v0.10.0 h1:5rqWmUFxnu3S7XYq9dafURwBgabYDFzo2Wv+AMopPHs= +github.com/hashicorp/vault/api/auth/kubernetes v0.10.0/go.mod h1:cZZmhF6xboMDmDbMY52oj2DKW6gS0cQ9g0pJ5XIXQ5U= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= @@ -161,15 +219,19 @@ github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= -github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= +github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE= github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= @@ -211,6 +273,8 @@ github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY= github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk= +github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/segmentio/ksuid v1.0.4 h1:sBo2BdShXjmcugAMwjugoGUdUV0pcxY5mW4xKRn3v4c= @@ -231,8 +295,10 @@ github.com/tetratelabs/wazero v1.10.0 h1:CXP3zneLDl6J4Zy8N/J+d5JsWKfrjE6GtvVK1fp github.com/tetratelabs/wazero v1.10.0/go.mod h1:DRm5twOQ5Gr1AoEdSi0CLjDQF1J9ZAuyqFIjl1KKfQU= github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/match v1.2.0 h1:0pt8FlkOwjN2fPt4bIl4BoNxb98gGHN2ObFEDkrfZnM= +github.com/tidwall/match v1.2.0/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= @@ -306,8 +372,8 @@ golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= -golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= -golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= +golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg= golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= diff --git a/pkg/envconfig/env_os.go b/pkg/envconfig/env_os.go deleted file mode 100644 index 6b85e3f5..00000000 --- a/pkg/envconfig/env_os.go +++ /dev/null @@ -1,8 +0,0 @@ -//go:build appengine || go1.5 -// +build appengine go1.5 - -package envconfig - -import "os" - -var lookupEnv = os.LookupEnv diff --git a/pkg/envconfig/env_syscall.go b/pkg/envconfig/env_syscall.go deleted file mode 100644 index 221ff7fa..00000000 --- a/pkg/envconfig/env_syscall.go +++ /dev/null @@ -1,8 +0,0 @@ -//go:build !appengine && !go1.5 -// +build !appengine,!go1.5 - -package envconfig - -import "syscall" - -var lookupEnv = syscall.Getenv diff --git a/pkg/envconfig/envconfig.go b/pkg/envconfig/envconfig.go index 5af9fa2d..ab90123a 100644 --- a/pkg/envconfig/envconfig.go +++ b/pkg/envconfig/envconfig.go @@ -180,30 +180,46 @@ func CheckDisallowed(prefix string, spec interface{}) error { return nil } +type Reader interface { + Read(string) (string, bool, error) +} + +type ReaderFunc func(string) (string, bool, error) + +func (f ReaderFunc) Read(key string) (string, bool, error) { + return f(key) +} + +var ( + EnvironmentReader = ReaderFunc(func(key string) (string, bool, error) { + v, ok := os.LookupEnv(key) + return v, ok, nil + }) +) + // Process populates the specified struct based on environment variables func Process(prefix string, spec interface{}) error { + return ProcessWithReader(prefix, spec, EnvironmentReader) +} + +func ProcessWithReader(prefix string, spec interface{}, reader Reader) error { infos, err := gatherInfo(prefix, spec) for _, info := range infos { - - // `os.Getenv` cannot differentiate between an explicitly set empty value - // and an unset value. `os.LookupEnv` is preferred to `syscall.Getenv`, - // but it is only available in go1.5 or newer. We're using Go build tags - // here to use os.LookupEnv for >=go1.5 - value, ok := lookupEnv(info.Key) - if !ok && info.Alt != "" { - value, ok = lookupEnv(info.Alt) + value, ok, err := reader.Read(info.Key) + if err != nil { + return err // todo warp error? } - //patch: does not handle 'default' tag - def := "" + //patch: do not handle 'default' tag + //def := "" //def := info.Tags.Get("default") //if def != "" && !ok { // value = def //} req := info.Tags.Get("required") - if !ok && def == "" { + if !ok { if isTrue(req) { key := info.Key if info.Alt != "" { diff --git a/pkg/log/zap.go b/pkg/log/zap.go index 6b228be7..bc342311 100644 --- a/pkg/log/zap.go +++ b/pkg/log/zap.go @@ -4,13 +4,13 @@ import ( "fmt" "time" - "github.com/webhookx-io/webhookx/config" + "github.com/webhookx-io/webhookx/config/modules" "github.com/webhookx-io/webhookx/utils" "go.uber.org/zap" "go.uber.org/zap/zapcore" ) -func NewZapLogger(cfg *config.LogConfig) (*zap.SugaredLogger, error) { +func NewZapLogger(cfg *modules.LogConfig) (*zap.SugaredLogger, error) { level, err := zapcore.ParseLevel(string(cfg.Level)) if err != nil { return nil, err @@ -35,7 +35,7 @@ func NewZapLogger(cfg *config.LogConfig) (*zap.SugaredLogger, error) { zapConfig.EncoderConfig.EncodeTime = func(t time.Time, enc zapcore.PrimitiveArrayEncoder) { enc.AppendString(t.Format("2006/01/02 15:04:05.000")) } - if cfg.Format == config.LogFormatText { + if cfg.Format == modules.LogFormatText { zapConfig.EncoderConfig.EncodeName = func(loggerName string, enc zapcore.PrimitiveArrayEncoder) { enc.AppendString(fmt.Sprintf("%-8s", "["+loggerName+"]")) } diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 896034d8..63eafce5 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -6,7 +6,7 @@ import ( "time" "github.com/go-kit/kit/metrics" - "github.com/webhookx-io/webhookx/config" + "github.com/webhookx-io/webhookx/config/modules" "github.com/webhookx-io/webhookx/pkg/schedule" "go.uber.org/zap" ) @@ -55,7 +55,7 @@ func (m *Metrics) Stop() error { return nil } -func New(cfg config.MetricsConfig) (*Metrics, error) { +func New(cfg modules.MetricsConfig) (*Metrics, error) { ctx, cancel := context.WithCancel(context.Background()) m := &Metrics{ ctx: ctx, diff --git a/pkg/metrics/opentelemetry.go b/pkg/metrics/opentelemetry.go index a7bb30c6..19cf8254 100644 --- a/pkg/metrics/opentelemetry.go +++ b/pkg/metrics/opentelemetry.go @@ -6,6 +6,7 @@ import ( "time" "github.com/webhookx-io/webhookx/config" + "github.com/webhookx-io/webhookx/config/modules" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc" @@ -34,13 +35,13 @@ func newGRPCExporter(endpoint string) (metric.Exporter, error) { return otlpmetricgrpc.New(context.Background(), opts...) } -func SetupOpentelemetry(attributes map[string]string, cfg config.OpentelemetryMetrics, metrics *Metrics) error { +func SetupOpentelemetry(attributes map[string]string, cfg modules.OpentelemetryMetrics, metrics *Metrics) error { var err error var exporter metric.Exporter switch cfg.Protocol { - case config.OtlpProtocolHTTP: + case modules.OtlpProtocolHTTP: exporter, err = newHTTPExporter(cfg.Endpoint) - case config.OtlpProtocolGRPC: + case modules.OtlpProtocolGRPC: exporter, err = newGRPCExporter(cfg.Endpoint) } if err != nil { diff --git a/pkg/secret/manager.go b/pkg/secret/manager.go new file mode 100644 index 00000000..76a49ed3 --- /dev/null +++ b/pkg/secret/manager.go @@ -0,0 +1,99 @@ +package secret + +import ( + "context" + "errors" + "fmt" + + "github.com/tidwall/gjson" + "github.com/webhookx-io/webhookx/pkg/secret/provider" + "github.com/webhookx-io/webhookx/pkg/secret/provider/aws" + "github.com/webhookx-io/webhookx/pkg/secret/provider/vault" + "github.com/webhookx-io/webhookx/pkg/secret/reference" + "go.uber.org/zap" +) + +var ( + // TODO: optimize message + ErrUnsupportedProvider = errors.New("unsupported provider") + ErrInvalidJson = errors.New("value is not a valid json") + ErrJsonPropertyNotFound = errors.New("json property not found") +) + +type ProviderType string + +const ( + AwsProviderType ProviderType = "aws" + VaultProviderType ProviderType = "vault" +) + +type Manager struct { + log *zap.SugaredLogger + providers map[string]provider.Provider +} + +func NewManager(log *zap.SugaredLogger, providers map[string]map[string]interface{}) (*Manager, error) { + manager := &Manager{ + log: log, + providers: make(map[string]provider.Provider), + } + for k, v := range providers { + err := manager.registerProvider(k, v) + if err != nil { + return nil, err + } + } + return manager, nil +} + +func (p *Manager) registerProvider(name string, cfg map[string]interface{}) error { + switch ProviderType(name) { + case AwsProviderType: + provider, err := aws.NewProvider(cfg) + if err != nil { + return err + } + p.providers[name] = provider + case VaultProviderType: + provider, err := vault.NewProvider(cfg) + if err != nil { + return err + } + p.providers[name] = provider + default: + return errors.New("unknown provider " + name) + } + return nil +} + +func (p *Manager) getProvider(name string) provider.Provider { + return p.providers[name] +} + +// ResolveReference returns resolved value of a reference +func (p *Manager) ResolveReference(ctx context.Context, ref *reference.Reference) (string, error) { + provider := p.getProvider(ref.Provider) + if provider == nil { + return "", fmt.Errorf("%w: %s", ErrUnsupportedProvider, ref.Reference) + } + + value, err := provider.GetValue(ctx, ref.Name, ref.Properties) + if err != nil { + return "", fmt.Errorf("failed to resolve reference value '%s': %s", ref.Reference, err) + } + + if ref.JsonPointer != "" { + if !gjson.Valid(value) { + return "", ErrInvalidJson + } + result := gjson.Get(value, ref.JsonPointer) + if !result.Exists() { + return "", fmt.Errorf("%w: %s", ErrJsonPropertyNotFound, ref.JsonPointer) + } + value = result.String() + } + + //fmt.Printf("resolved %s to '%s'\n", ref.Reference, value) + + return value, nil +} diff --git a/pkg/secret/provider/aws/aws.go b/pkg/secret/provider/aws/aws.go new file mode 100644 index 00000000..55f2c305 --- /dev/null +++ b/pkg/secret/provider/aws/aws.go @@ -0,0 +1,54 @@ +package aws + +import ( + "context" + "errors" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/secretsmanager" + "github.com/aws/aws-sdk-go-v2/service/secretsmanager/types" + "github.com/webhookx-io/webhookx/pkg/secret/provider" +) + +var ( + ErrSecretNotFound = errors.New("secret not found") +) + +type AwsProvider struct { + cfg interface{} + client *secretsmanager.Client +} + +func NewProvider(cfg map[string]interface{}) (provider.Provider, error) { + opts := make([]func(*config.LoadOptions) error, 0) + if region := cfg["region"].(string); region != "" { + opts = append(opts, config.WithRegion(region)) + } + if url := cfg["url"].(string); url != "" { + opts = append(opts, config.WithBaseEndpoint(url)) + } + awsconfig, err := config.LoadDefaultConfig(context.TODO(), opts...) + if err != nil { + return nil, err + } + + p := &AwsProvider{ + cfg: cfg, + } + p.client = secretsmanager.NewFromConfig(awsconfig, func(options *secretsmanager.Options) {}) + + return p, nil +} + +func (p *AwsProvider) GetValue(ctx context.Context, key string, properties map[string]string) (string, error) { + result, err := p.client.GetSecretValue(ctx, &secretsmanager.GetSecretValueInput{SecretId: aws.String(key)}) + if err != nil { + var awsErr *types.ResourceNotFoundException + if errors.As(err, &awsErr) { + return "", ErrSecretNotFound + } + return "", err + } + return *result.SecretString, nil +} diff --git a/pkg/secret/provider/provider.go b/pkg/secret/provider/provider.go new file mode 100644 index 00000000..6143576f --- /dev/null +++ b/pkg/secret/provider/provider.go @@ -0,0 +1,9 @@ +package provider + +import ( + "context" +) + +type Provider interface { + GetValue(ctx context.Context, key string, properties map[string]string) (string, error) +} diff --git a/pkg/secret/provider/vault/authn.go b/pkg/secret/provider/vault/authn.go new file mode 100644 index 00000000..86de91e6 --- /dev/null +++ b/pkg/secret/provider/vault/authn.go @@ -0,0 +1,22 @@ +package vault + +type AuthN struct { + Token TokenAuth `json:"token" yaml:"token"` + AppRole AppRoleAuth `json:"approle" yaml:"approle"` + Kubernetes KubernetesAuth `json:"kubernetes" yaml:"kubernetes"` +} + +type TokenAuth struct { + Token string `json:"token" yaml:"token"` +} + +type AppRoleAuth struct { + RoleID string `json:"role_id" yaml:"role_id" split_words:"true"` + SecretID string `json:"secret_id" yaml:"secret_id" split_words:"true"` + ResponseWrapping bool `json:"response_wrapping" yaml:"response_wrapping" split_words:"true"` +} + +type KubernetesAuth struct { + Role string `json:"role" yaml:"role"` + TokenPath string `json:"token_path" yaml:"token_path" split_words:"true"` +} diff --git a/pkg/secret/provider/vault/vault.go b/pkg/secret/provider/vault/vault.go new file mode 100644 index 00000000..f46503b1 --- /dev/null +++ b/pkg/secret/provider/vault/vault.go @@ -0,0 +1,125 @@ +package vault + +import ( + "context" + "encoding/json" + "errors" + + "github.com/hashicorp/vault/api" + "github.com/hashicorp/vault/api/auth/approle" + "github.com/hashicorp/vault/api/auth/kubernetes" + "github.com/mitchellh/mapstructure" + "github.com/webhookx-io/webhookx/pkg/secret/provider" +) + +var ( + ErrSecretNotFound = errors.New("secret not found") + ErrSecretNoData = errors.New("secret no data") +) + +type VaultProvider struct { + mountPath string + client *api.Client + cfg interface{} +} + +func NewProvider(cfg map[string]interface{}) (provider.Provider, error) { + config := api.DefaultConfig() + config.Address = cfg["address"].(string) + client, err := api.NewClient(config) + if err != nil { + return nil, err + } + + method := cfg["auth_method"].(string) + authn := cfg["authn"].(map[string]interface{}) + if err := setupClientAuth(client, method, authn); err != nil { + return nil, err + } + + p := &VaultProvider{ + mountPath: cfg["mount_path"].(string), + client: client, + cfg: cfg, + } + return p, nil +} + +func setupClientAuth(client *api.Client, method string, cfg map[string]interface{}) error { + var auth AuthN + decoder, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ + TagName: "json", + Result: &auth, + }) + if err != nil { + return err + } + if err := decoder.Decode(cfg); err != nil { + return err + } + switch method { + case "token": + client.SetToken(auth.Token.Token) + case "approle": + opts := make([]approle.LoginOption, 0) + if auth.AppRole.ResponseWrapping { + opts = append(opts, approle.WithWrappingToken()) + } + appRoleAuth, err := approle.NewAppRoleAuth( + auth.AppRole.RoleID, + &approle.SecretID{FromString: auth.AppRole.SecretID}, + opts..., + ) + if err != nil { + return err + } + _, err = client.Auth().Login(context.TODO(), appRoleAuth) + if err != nil { + return err + } + case "kubernetes": + opts := make([]kubernetes.LoginOption, 0) + if auth.Kubernetes.TokenPath != "" { + opts = append(opts, kubernetes.WithServiceAccountTokenPath(auth.Kubernetes.TokenPath)) + } + auth, err := kubernetes.NewKubernetesAuth( + auth.Kubernetes.Role, + opts..., + ) + if err != nil { + return err + } + _, err = client.Auth().Login(context.TODO(), auth) + if err != nil { + return err + } + } + + return nil +} + +func (p *VaultProvider) GetValue(ctx context.Context, key string, properties map[string]string) (string, error) { + secret, err := p.client.KVv2(p.mountPath).Get(ctx, key) + if err != nil { + if errors.Is(err, api.ErrSecretNotFound) { + return "", ErrSecretNotFound + } + //var vaultErr *api.ResponseError + //if errors.As(err, &vaultErr) { + // return "", fmt.Errorf("request failed %d %v", vaultErr.StatusCode, vaultErr.Errors) + //} + return "", err + } + + if secret == nil { + return "", ErrSecretNotFound + } + if secret.Data == nil { + return "", ErrSecretNoData + } + value, err := json.Marshal(secret.Data) + if err != nil { + return "", err + } + return string(value), nil +} diff --git a/pkg/secret/reference/reference.go b/pkg/secret/reference/reference.go new file mode 100644 index 00000000..2d460001 --- /dev/null +++ b/pkg/secret/reference/reference.go @@ -0,0 +1,75 @@ +package reference + +import ( + "errors" + "fmt" + "net/url" + "strings" +) + +var ( + ErrReferenceInvalid = errors.New("invalid reference") +) + +// Reference represents the definition of a reference. +// Syntax: {secret:///[.][?]} +type Reference struct { + Reference string + Provider string + Name string + JsonPointer string + Properties map[string]string +} + +//func (r *Reference) String() string { +// values := url.Values{} +// for k, v := range r.Properties { +// values.Set(k, v) +// } +// name := r.Name +// if r.JsonPointer != "" { +// name = name + "." + r.JsonPointer +// } +// return fmt.Sprintf("{secret://%s/%s?%s}", r.Provider, name, values.Encode()) +//} + +func Parse(reference string) (*Reference, error) { + s := strings.TrimPrefix(reference, "{") + s = strings.TrimSuffix(s, "}") + u, err := url.Parse(s) + if err != nil { + return nil, fmt.Errorf("%w: %s", ErrReferenceInvalid, err) + } + if u.Scheme != "secret" { + return nil, fmt.Errorf("%w: %q", ErrReferenceInvalid, "invalid reference scheme") + } + if u.Host == "" { + return nil, fmt.Errorf("%w: %q", ErrReferenceInvalid, "invalid reference provider") + } + if u.Path == "" || u.Path == "/" { + return nil, fmt.Errorf("%w: %q", ErrReferenceInvalid, "invalid reference name") + } + values, err := url.ParseQuery(u.RawQuery) + if err != nil { + return nil, fmt.Errorf("%w: %q", ErrReferenceInvalid, "invalid reference properties") + } + + ref := &Reference{ + Reference: reference, + Name: strings.TrimPrefix(u.Path, "/"), + Provider: u.Host, + Properties: make(map[string]string), + } + for k := range values { + ref.Properties[k] = values.Get(k) + } + if parts := strings.SplitN(ref.Name, ".", 2); len(parts) == 2 { + ref.Name = parts[0] + ref.JsonPointer = parts[1] + } + return ref, err +} + +func IsReference(s string) bool { + return strings.HasPrefix(s, "{secret://") && strings.HasSuffix(s, "}") +} diff --git a/pkg/secret/reference/reference_test.go b/pkg/secret/reference/reference_test.go new file mode 100644 index 00000000..4ce10904 --- /dev/null +++ b/pkg/secret/reference/reference_test.go @@ -0,0 +1,104 @@ +package reference + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestParse(t *testing.T) { + tests := []struct { + scenario string + reference string + expect Reference + }{ + { + reference: "{secret://aws/path/to/value?k1=v1&k2=v2}", + expect: Reference{ + Provider: "aws", + Name: "path/to/value", + Properties: map[string]string{ + "k1": "v1", + "k2": "v2", + }, + }, + }, + { + reference: "{secret://aws//mysecret}", + expect: Reference{ + Provider: "aws", + Name: "/mysecret", + Properties: map[string]string{}, + }, + }, + { + reference: "{secret://aws/json.}", + expect: Reference{ + Provider: "aws", + Name: "json", + JsonPointer: "", + Properties: map[string]string{}, + }, + }, + { + reference: "{secret://aws/path/to/json.credentials.0.password?k1=v1&k2=v2}", + expect: Reference{ + Provider: "aws", + Name: "path/to/json", + JsonPointer: "credentials.0.password", + Properties: map[string]string{ + "k1": "v1", + "k2": "v2", + }, + }, + }, + } + for _, test := range tests { + ref, err := Parse(test.reference) + assert.NoError(t, err) + test.expect.Reference = test.reference + assert.EqualValues(t, test.expect, *ref) + } +} + +func TestParseError(t *testing.T) { + tests := []struct { + scenario string + reference string + expect Reference + expectErr error + }{ + { + scenario: "", + reference: "{:}", + expectErr: fmt.Errorf("invalid reference: parse \":\": missing protocol scheme"), + }, + { + scenario: "", + reference: "{}", + expectErr: fmt.Errorf(`invalid reference: "invalid reference scheme"`), + }, + { + scenario: "", + reference: "{secret:///name}", + expectErr: fmt.Errorf(`invalid reference: "invalid reference provider"`), + }, + { + scenario: "", + reference: "{secret://aws/}", + expectErr: fmt.Errorf(`invalid reference: "invalid reference name"`), + }, + { + scenario: "", + reference: "{secret://aws/name?key;}", + expectErr: fmt.Errorf(`invalid reference: "invalid reference properties"`), + }, + } + + for _, test := range tests { + _, err := Parse(test.reference) + assert.EqualError(t, err, test.expectErr.Error()) + } + +} diff --git a/pkg/tracing/opentelemetry.go b/pkg/tracing/opentelemetry.go index b2c4fc4b..871dcb37 100644 --- a/pkg/tracing/opentelemetry.go +++ b/pkg/tracing/opentelemetry.go @@ -7,6 +7,7 @@ import ( "net/url" "github.com/webhookx-io/webhookx/config" + "github.com/webhookx-io/webhookx/config/modules" "go.opentelemetry.io/contrib/propagators/autoprop" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" @@ -22,13 +23,14 @@ import ( const instrumentationName = "github.com/webhookx-io/webhookx" -func SetupOTEL(o *config.TracingConfig) (trace.TracerProvider, error) { +func SetupOTEL(o *modules.TracingConfig) (trace.TracerProvider, error) { var err error var exporter *otlptrace.Exporter - if o.Opentelemetry.Protocol == config.OtlpProtocolHTTP { + switch o.Opentelemetry.Protocol { + case modules.OtlpProtocolHTTP: exporter, err = setupHTTPExporter(o.Opentelemetry) - } else if o.Opentelemetry.Protocol == config.OtlpProtocolGRPC { + case modules.OtlpProtocolGRPC: exporter, err = setupGRPCExporter(o.Opentelemetry) } @@ -65,7 +67,7 @@ func SetupOTEL(o *config.TracingConfig) (trace.TracerProvider, error) { return tracerProvider, err } -func setupHTTPExporter(c config.OpentelemetryTracing) (*otlptrace.Exporter, error) { +func setupHTTPExporter(c modules.OpentelemetryTracing) (*otlptrace.Exporter, error) { endpoint, err := url.Parse(c.Endpoint) if err != nil { return nil, fmt.Errorf("invalid collector endpoint %q: %w", c.Endpoint, err) @@ -87,7 +89,7 @@ func setupHTTPExporter(c config.OpentelemetryTracing) (*otlptrace.Exporter, erro return otlptrace.New(context.Background(), otlptracehttp.NewClient(opts...)) } -func setupGRPCExporter(c config.OpentelemetryTracing) (*otlptrace.Exporter, error) { +func setupGRPCExporter(c modules.OpentelemetryTracing) (*otlptrace.Exporter, error) { host, port, err := net.SplitHostPort(c.Endpoint) if err != nil { return nil, fmt.Errorf("invalid collector endpoint %q: %w", c.Endpoint, err) diff --git a/pkg/tracing/tracing.go b/pkg/tracing/tracing.go index 0a565326..5050a0e4 100644 --- a/pkg/tracing/tracing.go +++ b/pkg/tracing/tracing.go @@ -4,14 +4,14 @@ import ( "context" "time" - "github.com/webhookx-io/webhookx/config" + "github.com/webhookx-io/webhookx/config/modules" "go.opentelemetry.io/otel" sdktrace "go.opentelemetry.io/otel/sdk/trace" "go.opentelemetry.io/otel/trace" "go.opentelemetry.io/otel/trace/noop" ) -func New(conf *config.TracingConfig) (*Tracer, error) { +func New(conf *modules.TracingConfig) (*Tracer, error) { if !conf.Enabled { otel.SetTracerProvider(noop.NewTracerProvider()) return nil, nil diff --git a/proxy/gateway.go b/proxy/gateway.go index 9943e3a1..527ab9cf 100644 --- a/proxy/gateway.go +++ b/proxy/gateway.go @@ -15,7 +15,7 @@ import ( "time" "github.com/gorilla/mux" - "github.com/webhookx-io/webhookx/config" + "github.com/webhookx-io/webhookx/config/modules" "github.com/webhookx-io/webhookx/constants" "github.com/webhookx-io/webhookx/db" "github.com/webhookx-io/webhookx/db/entities" @@ -59,7 +59,7 @@ type Gateway struct { ctx context.Context cancel context.CancelFunc - cfg *config.ProxyConfig + cfg *modules.ProxyConfig log *zap.SugaredLogger s *http.Server @@ -82,7 +82,7 @@ type Gateway struct { } type Options struct { - Cfg *config.ProxyConfig + Cfg *modules.ProxyConfig Middlewares []mux.MiddlewareFunc DB *db.DB Dispatcher *dispatcher.Dispatcher diff --git a/status/status.go b/status/status.go index 89301623..fd3f0579 100644 --- a/status/status.go +++ b/status/status.go @@ -8,6 +8,7 @@ import ( "time" "github.com/webhookx-io/webhookx/config" + "github.com/webhookx-io/webhookx/config/modules" "github.com/webhookx-io/webhookx/pkg/accesslog" "github.com/webhookx-io/webhookx/pkg/tracing" "github.com/webhookx-io/webhookx/status/health" @@ -16,7 +17,7 @@ import ( type Status struct { api *API - cfg *config.StatusConfig + cfg *modules.StatusConfig s *http.Server log *zap.SugaredLogger } @@ -27,7 +28,7 @@ type Options struct { Indicators []*health.Indicator } -func NewStatus(cfg config.StatusConfig, tracer *tracing.Tracer, opts Options) *Status { +func NewStatus(cfg modules.StatusConfig, tracer *tracing.Tracer, opts Options) *Status { api := &API{ debugEndpoints: cfg.DebugEndpoints, tracer: tracer, diff --git a/test/anonymous/anonymous_test.go b/test/anonymous/anonymous_test.go index 1a25663a..b3fd0733 100644 --- a/test/anonymous/anonymous_test.go +++ b/test/anonymous/anonymous_test.go @@ -2,6 +2,7 @@ package admin import ( "testing" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -28,9 +29,11 @@ var _ = Describe("anonymous reports", Ordered, func() { }) It("should display log when anonymous_reports is disabled", func() { - matched, err := helper.FileHasLine(helper.LogFile, "^.*anonymous reports is disabled$") - assert.Nil(GinkgoT(), err) - assert.Equal(GinkgoT(), true, matched) + assert.Eventually(GinkgoT(), func() bool { + matched, err := helper.FileHasLine(helper.LogFile, "^.*anonymous reports is disabled$") + assert.Nil(GinkgoT(), err) + return matched + }, time.Second, time.Millisecond*100) }) }) diff --git a/test/cache/cache_test.go b/test/cache/cache_test.go index 95ce3d48..4f7720dd 100644 --- a/test/cache/cache_test.go +++ b/test/cache/cache_test.go @@ -17,7 +17,8 @@ var _ = Describe("cache", Ordered, func() { var redisCache cache.Cache BeforeAll(func() { - cfg, err := config.New(nil) + cfg := config.New() + err := config.Load("", cfg) assert.NoError(GinkgoT(), err) redisCache = cache.NewRedisCache(cfg.Redis.GetClient()) }) diff --git a/test/cmd/start_test.go b/test/cmd/start_test.go new file mode 100644 index 00000000..67fa71a8 --- /dev/null +++ b/test/cmd/start_test.go @@ -0,0 +1,30 @@ +package cmd + +import ( + . "github.com/onsi/ginkgo/v2" + "github.com/stretchr/testify/assert" + "github.com/webhookx-io/webhookx/test" + "github.com/webhookx-io/webhookx/test/helper" +) + +var _ = Describe("start", Ordered, func() { + Context("errors", func() { + It("should return error when configuration file is invalid", func() { + output, err := helper.ExecAppCommand("start", "--config", "config.yml") + assert.NotNil(GinkgoT(), err) + assert.Equal(GinkgoT(), "Error: could not load configuration: open config.yml: no such file or directory\n", output) + }) + + It("should return error when configuration file is invalid", func() { + output, err := helper.ExecAppCommand("start", "--config", test.FilePath("fixtures/malformed-config.yml")) + assert.NotNil(GinkgoT(), err) + assert.Equal(GinkgoT(), "Error: could not load configuration: yaml: unmarshal errors:\n line 1: cannot unmarshal !!str `๐Ÿ‘ป` into modules.SecretConfig\n", output) + }) + + It("should return error when configuration is invalid", func() { + output, err := helper.ExecAppCommand("start", "--config", test.FilePath("fixtures/invalid-config.yml")) + assert.NotNil(GinkgoT(), err) + assert.Equal(GinkgoT(), "Error: invalid configuration: port must be in the range [0, 65535]\n", output) + }) + }) +}) diff --git a/test/docker-compose.yml b/test/docker-compose.yml index e7ee67c6..08d6cb1f 100644 --- a/test/docker-compose.yml +++ b/test/docker-compose.yml @@ -11,25 +11,25 @@ services: timeout: 5s retries: 3 ports: - - 5432:5432 + - "5432:5432" redis: image: redis:6.2 command: "--appendonly yes --appendfsync everysec" ports: - - 6379:6379 + - "6379:6379" httpbin: image: kennethreitz/httpbin ports: - - 9999:80 + - "9999:80" httpsbin: image: nginx:latest depends_on: - httpbin ports: - - 9443:443 + - "9443:443" volumes: - ./nginx.conf:/etc/nginx/nginx.conf:ro - ./server.crt:/etc/nginx/certs/server.crt:ro @@ -41,5 +41,45 @@ services: - ./otel-collector-config.yml:/etc/otelcol-contrib/config.yaml - ./output/otel:/tmp/otel ports: - - 4317:4317 - - 4318:4318 + - "4317:4317" + - "4318:4318" + + localstack: + container_name: "localstack" + image: localstack/localstack + ports: + - "4566:4566" + environment: + DEBUG: "1" + + vault: + image: hashicorp/vault:1.21 + container_name: vault + cap_add: + - IPC_LOCK + network_mode: host + environment: + VAULT_LOG_LEVEL: debug + VAULT_DEV_ROOT_TOKEN_ID: root + VAULT_DEV_LISTEN_ADDRESS: "0.0.0.0:8200" + ports: + - "8200:8200" + healthcheck: + test: [ "CMD-SHELL", "VAULT_ADDR=http://127.0.0.1:8200 vault status" ] + interval: 3s + timeout: 5s + retries: 3 + + vault-init: + image: hashicorp/vault:1.21 + container_name: vault-init + network_mode: host + environment: + VAULT_ADDR: 'http://localhost:8200' + VAULT_TOKEN: 'root' + volumes: + - ./init-vault.sh:/init-vault.sh + entrypoint: /bin/sh -c "/init-vault.sh" + depends_on: + vault: + condition: service_healthy diff --git a/test/fixtures/invalid-config.yml b/test/fixtures/invalid-config.yml new file mode 100644 index 00000000..aff9d2db --- /dev/null +++ b/test/fixtures/invalid-config.yml @@ -0,0 +1,2 @@ +database: + port: 65536 diff --git a/test/fixtures/malformed-config.yml b/test/fixtures/malformed-config.yml new file mode 100644 index 00000000..db52c3b5 --- /dev/null +++ b/test/fixtures/malformed-config.yml @@ -0,0 +1 @@ +๐Ÿ‘ป diff --git a/test/helper/helper.go b/test/helper/helper.go index 06e402d7..d2f072c9 100644 --- a/test/helper/helper.go +++ b/test/helper/helper.go @@ -16,7 +16,12 @@ import ( "regexp" "time" + "github.com/aws/aws-sdk-go-v2/aws" + awsconfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/secretsmanager" "github.com/go-resty/resty/v2" + vault "github.com/hashicorp/vault/api" "github.com/redis/go-redis/v9" uuid "github.com/satori/go.uuid" "github.com/webhookx-io/webhookx/app" @@ -62,6 +67,9 @@ var ( // SetEnvs sets envs and returns a function to restore envs func SetEnvs(defaults map[string]string, sets map[string]string) func() { + if defaults == nil { + defaults = map[string]string{} + } envs := maps.Clone(defaults) maps.Copy(envs, sets) originals := make(map[string]*string) @@ -88,7 +96,9 @@ func SetEnvs(defaults map[string]string, sets map[string]string) func() { func NewConfig(envs map[string]string) (*config.Config, error) { cancel := SetEnvs(Environments, envs) defer cancel() - return config.New(nil) + cfg := config.New() + err := config.Load("", cfg) + return cfg, err } // Start starts application with given environment variables @@ -101,7 +111,8 @@ func Start(envs map[string]string) (application *app.Application, err error) { } }() - cfg, err := config.New(nil) + cfg := config.New() + err = config.Load("", cfg) if err != nil { return } @@ -444,3 +455,31 @@ func WaitForServer(urlstring string, timeout time.Duration) error { } return fmt.Errorf("server at %s not ready after %v", u.Host, timeout) } + +func VaultClient() *vault.Client { + cfg := vault.DefaultConfig() + cfg.Address = "http://127.0.0.1:8200" + client, err := vault.NewClient(cfg) + if err != nil { + panic(err) + } + client.SetToken("root") + return client +} + +func SecretManangerClient() *secretsmanager.Client { + cfg, err := awsconfig.LoadDefaultConfig(context.TODO(), + awsconfig.WithBaseEndpoint("http://localhost:4566"), + awsconfig.WithRegion("us-east-1"), + awsconfig.WithCredentialsProvider(aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider( + "test", + "test", + "", + ))), + ) + if err != nil { + panic(err) + } + client := secretsmanager.NewFromConfig(cfg, func(options *secretsmanager.Options) {}) + return client +} diff --git a/test/init-vault.sh b/test/init-vault.sh new file mode 100755 index 00000000..d40b3b8d --- /dev/null +++ b/test/init-vault.sh @@ -0,0 +1,28 @@ +#!/bin/sh + +echo "setting approle auth" +vault auth enable approle +vault policy write webhookx-read - <