From c2f547dbc71c92d2f76b496b2d1065ac2b6533b9 Mon Sep 17 00:00:00 2001 From: Jan Hacker Date: Mon, 9 Mar 2026 11:22:38 +0100 Subject: [PATCH] initial migration of bosh-s3cli patch proposal: support config from env --- main.go | 13 ++++++++----- s3/client/sdk.go | 3 +++ s3/config/config.go | 42 ++++++++++++++++++++++++++++++++++++++++++ storage/factory.go | 7 ++++++- 4 files changed, 59 insertions(+), 6 deletions(-) diff --git a/main.go b/main.go index 3069907..65424fb 100644 --- a/main.go +++ b/main.go @@ -99,12 +99,15 @@ func main() { // configure storage-cli config common.InitConfig(parseLogLevel(*logLevel)) - // check client config file exists - configFile, err := os.Open(*configPath) - if err != nil { - fatalLog("", err) + // try reading config file. if not provided, env will be tried as source. + var configFile *os.File + if *configPath != "" { + configFile, err := os.Open(*configPath) + if err != nil { + fatalLog("", err) + } + defer configFile.Close() //nolint:errcheck } - defer configFile.Close() //nolint:errcheck // create client client, err := storage.NewStorageClient(*storageType, configFile) diff --git a/s3/client/sdk.go b/s3/client/sdk.go index b29ccc5..c7b58f1 100644 --- a/s3/client/sdk.go +++ b/s3/client/sdk.go @@ -94,6 +94,9 @@ func NewAwsS3ClientWithApiOptions( } o.BaseEndpoint = aws.String(endpoint) } + if c.Debug { + o.ClientLogMode = aws.LogResponse | aws.LogRequest + } // Apply custom middlewares if provided o.APIOptions = append(o.APIOptions, apiOptions...) }) diff --git a/s3/config/config.go b/s3/config/config.go index b174f4d..0d4d4da 100644 --- a/s3/config/config.go +++ b/s3/config/config.go @@ -1,10 +1,13 @@ package config import ( + "bytes" "encoding/json" "errors" "fmt" "io" + "os" + "strconv" "strings" ) @@ -27,6 +30,7 @@ type S3Cli struct { HostStyle bool `json:"host_style"` SwiftAuthAccount string `json:"swift_auth_account"` SwiftTempURLKey string `json:"swift_temp_url_key"` + Debug bool `json:"debug"` RequestChecksumCalculationEnabled bool ResponseChecksumCalculationEnabled bool UploaderRequestChecksumCalculationEnabled bool @@ -74,6 +78,44 @@ func newStaticCredentialsPresentError(desiredSource string) error { return errorStaticCredentialsPresent{credentialsSource: desiredSource} } +// NewReader provides an io.Reader on given configFile, using environment as fall-back. +func NewReader(configFile *os.File) (io.Reader, error) { + if configFile != nil { + return configFile, nil + } + // Try reading config from env if no config file handle was provided + port := 443 + if altPort, isset := os.LookupEnv("S3_PORT"); isset { + var err error + port, err = strconv.Atoi(altPort) + if err != nil { + return nil, err + } + } + c := S3Cli{ + AccessKeyID: os.Getenv("S3_ACCESS_KEY_ID"), + BucketName: os.Getenv("S3_BUCKET_NAME"), + // Fixate CredentialSource to StaticCredentialsSource, making S3_ACCESS_KEY_ID & S3_SECRET_ACCESS_KEY required + CredentialsSource: StaticCredentialsSource, + Host: os.Getenv("S3_HOST"), + Port: port, + Region: os.Getenv("S3_REGION"), + SecretAccessKey: os.Getenv("S3_SECRET_ACCESS_KEY"), + SSLVerifyPeer: !(os.Getenv("S3_INSECURE_SSL") != ""), + // Use SSL/TLS/https:// unless S3_DISABLE_SSL is set + UseSSL: !(os.Getenv("S3_DISABLE_SSL") != ""), + // Use PathStyle=true by default (endpoint/bucket instead of DNS bucket.endpoint), if S3_USE_HOSTSTYLE unset. See client/sdk.go + HostStyle: os.Getenv("S3_USE_HOSTSTYLE") != "", + // Enable HTTP(S) request debugging if S3_DEBUG has non-empty value + Debug: os.Getenv("S3_DEBUG") != "", + } + json, err := json.Marshal(c) + if err != nil { + return nil, err + } + return bytes.NewReader(json), nil +} + // NewFromReader returns a new s3cli configuration struct from the contents of reader. // reader.Read() is expected to return valid JSON func NewFromReader(reader io.Reader) (S3Cli, error) { diff --git a/storage/factory.go b/storage/factory.go index a64634d..26c1ba7 100644 --- a/storage/factory.go +++ b/storage/factory.go @@ -72,7 +72,12 @@ var newGcsClient = func(configFile *os.File) (Storager, error) { } var newS3Client = func(configFile *os.File) (Storager, error) { - s3Config, err := s3config.NewFromReader(configFile) + configReader, err := s3config.NewReader(configFile) + if err != nil { + return nil, err + } + + s3Config, err := s3config.NewFromReader(configReader) if err != nil { return nil, err }