From 448b5236fb017583ece3cf748966453c435b6542 Mon Sep 17 00:00:00 2001 From: Jochen Ehret Date: Fri, 14 Aug 2026 13:08:09 +0200 Subject: [PATCH 1/3] Configurable http client timeout for Azure Blob Storage --- azurebs/README.md | 8 +++- azurebs/client/storage_client.go | 79 +++++++++++++++++++++++++------- azurebs/config/config.go | 39 ++++++++++++++-- azurebs/config/config_test.go | 41 +++++++++++++++++ 4 files changed, 144 insertions(+), 23 deletions(-) diff --git a/azurebs/README.md b/azurebs/README.md index 500662a..76994de 100644 --- a/azurebs/README.md +++ b/azurebs/README.md @@ -15,10 +15,14 @@ The Azure client requires a JSON configuration file with the following structure "account_name": " (required)", "account_key": " (required)", "container_name": " (required)", - "environment": " (optional, default: 'AzureCloud')" + "environment": " (optional, default: 'AzureCloud')", + "put_timeout_in_seconds": " (optional, e.g. '30', default: no timeout)", + "http_request_timeout": " (optional, Go duration e.g. '30s', default: no timeout)" } ``` +`put_timeout_in_seconds` sets a context-level timeout for upload operations. `http_request_timeout` sets a per-request HTTP client timeout that applies to all operations (upload, download, delete, list, etc.). + **Usage examples:** ``` bash # Upload a blob @@ -66,7 +70,7 @@ go test $(go list ./azurebs/... | grep -v integration) 1. Export the following variables into your environment. ```bash - export ACCOUNT_NAME= + export ACCOUNT_NAME= export ACCOUNT_KEY= export CONTAINER_NAME= ``` diff --git a/azurebs/client/storage_client.go b/azurebs/client/storage_client.go index 0590df0..fc5ed8f 100644 --- a/azurebs/client/storage_client.go +++ b/azurebs/client/storage_client.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "log/slog" + "net/http" "os" "strconv" "strings" @@ -104,9 +105,45 @@ func createContext(dsc DefaultStorageClient) (context.Context, context.CancelFun } type DefaultStorageClient struct { - credential *azblob.SharedKeyCredential - serviceURL string - storageConfig config.AZStorageConfig + credential *azblob.SharedKeyCredential + serviceURL string + storageConfig config.AZStorageConfig + httpRequestTimeout time.Duration +} + +// clientOptions returns azblob.ClientOptions with a timeout-configured http.Client, +// or nil when no http_request_timeout is set (use SDK defaults). +func (dsc DefaultStorageClient) blockblobClientOptions() *blockblob.ClientOptions { + if dsc.httpRequestTimeout == 0 { + return nil + } + return &blockblob.ClientOptions{ + ClientOptions: azcore.ClientOptions{ + Transport: &http.Client{Timeout: dsc.httpRequestTimeout}, + }, + } +} + +func (dsc DefaultStorageClient) blobClientOptions() *azBlob.ClientOptions { + if dsc.httpRequestTimeout == 0 { + return nil + } + return &azBlob.ClientOptions{ + ClientOptions: azcore.ClientOptions{ + Transport: &http.Client{Timeout: dsc.httpRequestTimeout}, + }, + } +} + +func (dsc DefaultStorageClient) containerClientOptions() *azContainer.ClientOptions { + if dsc.httpRequestTimeout == 0 { + return nil + } + return &azContainer.ClientOptions{ + ClientOptions: azcore.ClientOptions{ + Transport: &http.Client{Timeout: dsc.httpRequestTimeout}, + }, + } } func NewStorageClient(storageConfig config.AZStorageConfig) (StorageClient, error) { @@ -115,9 +152,19 @@ func NewStorageClient(storageConfig config.AZStorageConfig) (StorageClient, erro return nil, err } + httpRequestTimeout, err := storageConfig.HTTPRequestTimeoutValue() + if err != nil { + return nil, err + } + serviceURL := fmt.Sprintf("https://%s.%s/%s", storageConfig.AccountName, storageConfig.StorageEndpoint(), storageConfig.ContainerName) - return DefaultStorageClient{credential: credential, serviceURL: serviceURL, storageConfig: storageConfig}, nil + return DefaultStorageClient{ + credential: credential, + serviceURL: serviceURL, + storageConfig: storageConfig, + httpRequestTimeout: httpRequestTimeout, + }, nil } func (dsc DefaultStorageClient) Upload( @@ -138,7 +185,7 @@ func (dsc DefaultStorageClient) Upload( } defer cancel() - client, err := blockblob.NewClientWithSharedKeyCredential(blobURL, dsc.credential, nil) + client, err := blockblob.NewClientWithSharedKeyCredential(blobURL, dsc.credential, dsc.blockblobClientOptions()) if err != nil { return nil, err } @@ -173,7 +220,7 @@ func (dsc DefaultStorageClient) UploadStream( } defer cancel() - client, err := blockblob.NewClientWithSharedKeyCredential(blobURL, dsc.credential, nil) + client, err := blockblob.NewClientWithSharedKeyCredential(blobURL, dsc.credential, dsc.blockblobClientOptions()) if err != nil { return err } @@ -196,7 +243,7 @@ func (dsc DefaultStorageClient) Download( ) error { blobURL := fmt.Sprintf("%s/%s", dsc.serviceURL, source) slog.Info("Downloading blob from container", "container", dsc.storageConfig.ContainerName, "blob", source, "local_file", dest.Name()) - client, err := blockblob.NewClientWithSharedKeyCredential(blobURL, dsc.credential, nil) + client, err := blockblob.NewClientWithSharedKeyCredential(blobURL, dsc.credential, dsc.blockblobClientOptions()) if err != nil { return err } @@ -226,7 +273,7 @@ func (dsc DefaultStorageClient) Copy( srcURL := fmt.Sprintf("%s/%s", dsc.serviceURL, srcBlob) destURL := fmt.Sprintf("%s/%s", dsc.serviceURL, destBlob) - destClient, err := blockblob.NewClientWithSharedKeyCredential(destURL, dsc.credential, nil) + destClient, err := blockblob.NewClientWithSharedKeyCredential(destURL, dsc.credential, dsc.blockblobClientOptions()) if err != nil { return fmt.Errorf("failed to create destination client: %w", err) } @@ -268,7 +315,7 @@ func (dsc DefaultStorageClient) Delete( blobURL := fmt.Sprintf("%s/%s", dsc.serviceURL, dest) slog.Info("Deleting blob from container", "container", dsc.storageConfig.ContainerName, "blob", dest, "url", blobURL) - client, err := blockblob.NewClientWithSharedKeyCredential(blobURL, dsc.credential, nil) + client, err := blockblob.NewClientWithSharedKeyCredential(blobURL, dsc.credential, dsc.blockblobClientOptions()) if err != nil { return err } @@ -295,7 +342,7 @@ func (dsc DefaultStorageClient) DeleteRecursive( slog.Info("Deleting all blobs in container", "container", dsc.storageConfig.ContainerName) } - containerClient, err := azContainer.NewClientWithSharedKeyCredential(dsc.serviceURL, dsc.credential, nil) + containerClient, err := azContainer.NewClientWithSharedKeyCredential(dsc.serviceURL, dsc.credential, dsc.containerClientOptions()) if err != nil { return fmt.Errorf("failed to create container client: %w", err) } @@ -315,7 +362,7 @@ func (dsc DefaultStorageClient) DeleteRecursive( for _, blob := range resp.Segment.BlobItems { blobURL := fmt.Sprintf("%s/%s", dsc.serviceURL, *blob.Name) - blobClient, err := blockblob.NewClientWithSharedKeyCredential(blobURL, dsc.credential, nil) + blobClient, err := blockblob.NewClientWithSharedKeyCredential(blobURL, dsc.credential, dsc.blockblobClientOptions()) if err != nil { slog.Error("Failed to create blob client", "blob", *blob.Name, "error", err) continue @@ -338,7 +385,7 @@ func (dsc DefaultStorageClient) Exists( blobURL := fmt.Sprintf("%s/%s", dsc.serviceURL, dest) slog.Info("Checking if blob exists", "container", dsc.storageConfig.ContainerName, "blob", dest, "url", blobURL) - client, err := blockblob.NewClientWithSharedKeyCredential(blobURL, dsc.credential, nil) + client, err := blockblob.NewClientWithSharedKeyCredential(blobURL, dsc.credential, dsc.blockblobClientOptions()) if err != nil { return false, err } @@ -365,7 +412,7 @@ func (dsc DefaultStorageClient) SignedUrl( blobURL := fmt.Sprintf("%s/%s", dsc.serviceURL, dest) slog.Info("Generating SAS URL for blob", "container", dsc.storageConfig.ContainerName, "blob", dest, "request_type", requestType, "expiration", expiration) - client, err := azBlob.NewClientWithSharedKeyCredential(blobURL, dsc.credential, nil) + client, err := azBlob.NewClientWithSharedKeyCredential(blobURL, dsc.credential, dsc.blobClientOptions()) if err != nil { return "", err } @@ -398,7 +445,7 @@ func (dsc DefaultStorageClient) List( slog.Info("Listing blobs in container", "container", dsc.storageConfig.ContainerName) } - client, err := azContainer.NewClientWithSharedKeyCredential(dsc.serviceURL, dsc.credential, nil) + client, err := azContainer.NewClientWithSharedKeyCredential(dsc.serviceURL, dsc.credential, dsc.containerClientOptions()) if err != nil { return nil, fmt.Errorf("failed to create container client: %w", err) } @@ -437,7 +484,7 @@ func (dsc DefaultStorageClient) Properties( blobURL := fmt.Sprintf("%s/%s", dsc.serviceURL, dest) slog.Info("Getting properties for blob", "container", dsc.storageConfig.ContainerName, "blob", dest, "url", blobURL) - client, err := blockblob.NewClientWithSharedKeyCredential(blobURL, dsc.credential, nil) + client, err := blockblob.NewClientWithSharedKeyCredential(blobURL, dsc.credential, dsc.blockblobClientOptions()) if err != nil { return err } @@ -469,7 +516,7 @@ func (dsc DefaultStorageClient) Properties( func (dsc DefaultStorageClient) EnsureContainerExists() error { slog.Info("Ensuring container exists", "container", dsc.storageConfig.ContainerName) - containerClient, err := azContainer.NewClientWithSharedKeyCredential(dsc.serviceURL, dsc.credential, nil) + containerClient, err := azContainer.NewClientWithSharedKeyCredential(dsc.serviceURL, dsc.credential, dsc.containerClientOptions()) if err != nil { return fmt.Errorf("failed to create container client: %w", err) } diff --git a/azurebs/config/config.go b/azurebs/config/config.go index 1407094..bcf1bec 100644 --- a/azurebs/config/config.go +++ b/azurebs/config/config.go @@ -3,7 +3,9 @@ package config import ( "encoding/json" "errors" + "fmt" "io" + "time" "github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud" ) @@ -27,11 +29,34 @@ func init() { } type AZStorageConfig struct { - AccountName string `json:"account_name"` - AccountKey string `json:"account_key"` - ContainerName string `json:"container_name"` - Environment string `json:"environment"` - Timeout string `json:"put_timeout_in_seconds"` + AccountName string `json:"account_name"` + AccountKey string `json:"account_key"` + ContainerName string `json:"container_name"` + Environment string `json:"environment"` + Timeout string `json:"put_timeout_in_seconds"` + HTTPRequestTimeout string `json:"http_request_timeout"` +} + +// ErrNonPositiveHTTPRequestTimeout is returned when http_request_timeout is <= 0. +var ErrNonPositiveHTTPRequestTimeout = errors.New("http_request_timeout must be greater than 0") + +// HTTPRequestTimeoutValue parses HTTPRequestTimeout as a Go duration string. +// Returns 0 (no timeout) if the field is empty. +func (c *AZStorageConfig) HTTPRequestTimeoutValue() (time.Duration, error) { + if c.HTTPRequestTimeout == "" { + return 0, nil + } + + d, err := time.ParseDuration(c.HTTPRequestTimeout) + if err != nil { + return 0, fmt.Errorf("invalid http_request_timeout: %w", err) + } + + if d <= 0 { + return 0, ErrNonPositiveHTTPRequestTimeout + } + + return d, nil } // NewFromReader returns a new azure-storage-cli configuration struct from the contents of reader. @@ -53,6 +78,10 @@ func NewFromReader(reader io.Reader) (AZStorageConfig, error) { return AZStorageConfig{}, err } + if _, err = config.HTTPRequestTimeoutValue(); err != nil { + return AZStorageConfig{}, err + } + return config, nil } diff --git a/azurebs/config/config_test.go b/azurebs/config/config_test.go index 9f54c4f..53d5a8f 100644 --- a/azurebs/config/config_test.go +++ b/azurebs/config/config_test.go @@ -3,6 +3,7 @@ package config_test import ( "bytes" "errors" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -87,6 +88,46 @@ var _ = Describe("Config", func() { }) }) }) + Context("http_request_timeout", func() { + When("not set", func() { + It("returns 0 duration", func() { + configJson := []byte(`{"account_name": "a", "account_key": "b", "container_name": "c"}`) + cfg, err := config.NewFromReader(bytes.NewReader(configJson)) + Expect(err).ToNot(HaveOccurred()) + d, err := cfg.HTTPRequestTimeoutValue() + Expect(err).ToNot(HaveOccurred()) + Expect(d).To(Equal(time.Duration(0))) + }) + }) + + When("set to a valid duration", func() { + It("returns the parsed duration", func() { + configJson := []byte(`{"account_name": "a", "account_key": "b", "container_name": "c", "http_request_timeout": "30s"}`) + cfg, err := config.NewFromReader(bytes.NewReader(configJson)) + Expect(err).ToNot(HaveOccurred()) + d, err := cfg.HTTPRequestTimeoutValue() + Expect(err).ToNot(HaveOccurred()) + Expect(d).To(Equal(30 * time.Second)) + }) + }) + + When("set to an invalid duration string", func() { + It("returns an error", func() { + configJson := []byte(`{"account_name": "a", "account_key": "b", "container_name": "c", "http_request_timeout": "not-a-duration"}`) + _, err := config.NewFromReader(bytes.NewReader(configJson)) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("invalid http_request_timeout")) + }) + }) + + When("set to a non-positive duration", func() { + It("returns an error", func() { + configJson := []byte(`{"account_name": "a", "account_key": "b", "container_name": "c", "http_request_timeout": "-5s"}`) + _, err := config.NewFromReader(bytes.NewReader(configJson)) + Expect(err).To(MatchError(config.ErrNonPositiveHTTPRequestTimeout)) + }) + }) + }) }) type explodingReader struct{} From e7c8109ef1368788ef3e857a7ea36ed47ef49202 Mon Sep 17 00:00:00 2001 From: Jochen Ehret Date: Fri, 14 Aug 2026 14:47:55 +0200 Subject: [PATCH 2/3] Integration test for http_request_timeout parameter (azurebs) --- azurebs/integration/general_azure_test.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/azurebs/integration/general_azure_test.go b/azurebs/integration/general_azure_test.go index bedc8a5..b01e90b 100644 --- a/azurebs/integration/general_azure_test.go +++ b/azurebs/integration/general_azure_test.go @@ -75,6 +75,14 @@ var _ = Describe("General testing for all Azure regions", func() { func(cfg *config.AZStorageConfig) { integration.AssertLifecycleWorks(cliPath, cfg) }, configurations, ) + DescribeTable("Blobstore lifecycle works with http_request_timeout set", + func(cfg *config.AZStorageConfig) { + cfgCopy := *cfg + cfgCopy.HTTPRequestTimeout = "30s" + integration.AssertLifecycleWorks(cliPath, &cfgCopy) + }, + configurations, + ) DescribeTable("Invoking `get` on a non-existent-key fails", func(cfg *config.AZStorageConfig) { integration.AssertGetNonexistentFails(cliPath, cfg) }, configurations, From c27e2cbaca014a543d1097c14e6e85a6ff3b5189 Mon Sep 17 00:00:00 2001 From: Jochen Ehret Date: Fri, 14 Aug 2026 15:40:24 +0200 Subject: [PATCH 3/3] chore: retrigger CI checks