impl(storage): clean up remaining client_options code

google/cloud/storage/client.cc

@@ -13,19 +13,35 @@
 // limitations under the License.
 
 #include "google/cloud/storage/client.h"
+#include "google/cloud/storage/idempotency_policy.h"
 #include "google/cloud/storage/internal/base64.h"
 #include "google/cloud/storage/internal/connection_factory.h"
+#include "google/cloud/storage/internal/unified_rest_credentials.h"
+#include "google/cloud/storage/oauth2/credentials.h"
+#include "google/cloud/storage/oauth2/google_credentials.h"
 #include "google/cloud/storage/oauth2/service_account_credentials.h"
+#include "google/cloud/storage/options.h"
 #include "google/cloud/internal/absl_str_cat_quiet.h"
+#include "google/cloud/internal/absl_str_join_quiet.h"
 #include "google/cloud/internal/curl_handle.h"
 #include "google/cloud/internal/curl_options.h"
 #include "google/cloud/internal/filesystem.h"
+#include "google/cloud/internal/getenv.h"
 #include "google/cloud/internal/make_status.h"
 #include "google/cloud/internal/opentelemetry.h"
+#include "google/cloud/internal/populate_common_options.h"
+#include "google/cloud/internal/rest_options.h"
+#include "google/cloud/internal/rest_response.h"
+#include "google/cloud/internal/service_endpoint.h"
 #include "google/cloud/log.h"
+#include "google/cloud/opentelemetry_options.h"
+#include "google/cloud/universe_domain_options.h"
 #include "absl/strings/str_split.h"
+#include <cstdlib>
 #include <fstream>
 #include <memory>
+#include <set>
+#include <sstream>
 #include <string>
 #include <thread>
 #include <utility>
@@ -387,8 +403,219 @@ std::string Client::EndpointAuthority() const {
   return std::string(endpoint_authority);
 }
 
+// This magic number was obtained by experimentation summarized in #2657
+#ifndef GOOGLE_CLOUD_CPP_STORAGE_DEFAULT_UPLOAD_BUFFER_SIZE
+#define GOOGLE_CLOUD_CPP_STORAGE_DEFAULT_UPLOAD_BUFFER_SIZE (8 * 1024 * 1024)
+#endif  // GOOGLE_CLOUD_CPP_STORAGE_DEFAULT_UPLOAD_BUFFER_SIZE
+
+// This magic number was obtained by experimentation summarized in #2657
+#ifndef GOOGLE_CLOUD_CPP_STORAGE_DEFAULT_DOWNLOAD_BUFFER_SIZE
+#define GOOGLE_CLOUD_CPP_STORAGE_DEFAULT_DOWNLOAD_BUFFER_SIZE \
+  (3 * 1024 * 1024 / 2)
+#endif  // GOOGLE_CLOUD_CPP_STORAGE_DEFAULT_DOWNLOAD_BUFFER_SIZE
+
+// This is a result of experiments performed in #2657.
+#ifndef GOOGLE_CLOUD_CPP_STORAGE_DEFAULT_MAXIMUM_SIMPLE_UPLOAD_SIZE
+#define GOOGLE_CLOUD_CPP_STORAGE_DEFAULT_MAXIMUM_SIMPLE_UPLOAD_SIZE \
+  (20 * 1024 * 1024L)
+#endif  // GOOGLE_CLOUD_CPP_STORAGE_DEFAULT_MAXIMUM_SIMPLE_UPLOAD_SIZE
+
+#ifndef GOOGLE_CLOUD_CPP_STORAGE_DEFAULT_DOWNLOAD_STALL_TIMEOUT
+#define GOOGLE_CLOUD_CPP_STORAGE_DEFAULT_DOWNLOAD_STALL_TIMEOUT 120
+#endif  // GOOGLE_CLOUD_CPP_STORAGE_DEFAULT_DOWNLOAD_STALL_TIMEOUT
+
+// Define the defaults using a pre-processor macro, this allows the application
+// developers to change the defaults for their application by compiling with
+// different values.
+#ifndef STORAGE_CLIENT_DEFAULT_MAXIMUM_RETRY_PERIOD
+#define STORAGE_CLIENT_DEFAULT_MAXIMUM_RETRY_PERIOD std::chrono::minutes(15)
+#endif  // STORAGE_CLIENT_DEFAULT_MAXIMUM_RETRY_PERIOD
+
+#ifndef STORAGE_CLIENT_DEFAULT_INITIAL_BACKOFF_DELAY
+#define STORAGE_CLIENT_DEFAULT_INITIAL_BACKOFF_DELAY std::chrono::seconds(1)
+#endif  // STORAGE_CLIENT_DEFAULT_INITIAL_BACKOFF_DELAY
+
+#ifndef STORAGE_CLIENT_DEFAULT_MAXIMUM_BACKOFF_DELAY
+#define STORAGE_CLIENT_DEFAULT_MAXIMUM_BACKOFF_DELAY std::chrono::minutes(5)
+#endif  // STORAGE_CLIENT_DEFAULT_MAXIMUM_BACKOFF_DELAY
+
+#ifndef STORAGE_CLIENT_DEFAULT_BACKOFF_SCALING
+#define STORAGE_CLIENT_DEFAULT_BACKOFF_SCALING 2.0
+#endif  //  STORAGE_CLIENT_DEFAULT_BACKOFF_SCALING
+
+namespace {
+
+using ::google::cloud::internal::GetEnv;
+
+absl::optional<std::string> GetEmulator() {
+  auto emulator = GetEnv("CLOUD_STORAGE_EMULATOR_ENDPOINT");
+  if (emulator) return emulator;
+  return GetEnv("CLOUD_STORAGE_TESTBENCH_ENDPOINT");
+}
+
+std::size_t DefaultConnectionPoolSize() {
+  std::size_t nthreads = std::thread::hardware_concurrency();
+  if (nthreads == 0) {
+    return 4;
+  }
+  return 4 * nthreads;
+}
+
+}  // namespace
+
 namespace internal {
 
+Options ApplyPolicy(Options opts, RetryPolicy const& p) {
+  opts.set<RetryPolicyOption>(p.clone());
+  return opts;
+}
+
+Options ApplyPolicy(Options opts, BackoffPolicy const& p) {
+  opts.set<BackoffPolicyOption>(p.clone());
+  return opts;
+}
+
+Options ApplyPolicy(Options opts, IdempotencyPolicy const& p) {
+  opts.set<IdempotencyPolicyOption>(p.clone());
+  return opts;
+}
+
+Options DefaultOptions(std::shared_ptr<oauth2::Credentials> credentials,
+                       Options opts) {
+  auto ud = GetEnv("GOOGLE_CLOUD_UNIVERSE_DOMAIN");
+  if (ud && !ud->empty()) {
+    opts.set<google::cloud::internal::UniverseDomainOption>(*std::move(ud));
+  }
+  auto gcs_ep = google::cloud::internal::UniverseDomainEndpoint(
+      "https://storage.googleapis.com", opts);
+  auto iam_ep = absl::StrCat(google::cloud::internal::UniverseDomainEndpoint(
+                                 "https://iamcredentials.googleapis.com", opts),
+                             "/v1");
+  auto o =
+      Options{}
+          .set<Oauth2CredentialsOption>(std::move(credentials))
+          .set<RestEndpointOption>(std::move(gcs_ep))
+          .set<IamEndpointOption>(std::move(iam_ep))
+          .set<TargetApiVersionOption>("v1")
+          .set<ConnectionPoolSizeOption>(DefaultConnectionPoolSize())
+          .set<DownloadBufferSizeOption>(
+              GOOGLE_CLOUD_CPP_STORAGE_DEFAULT_DOWNLOAD_BUFFER_SIZE)
+          .set<UploadBufferSizeOption>(
+              GOOGLE_CLOUD_CPP_STORAGE_DEFAULT_UPLOAD_BUFFER_SIZE)
+          .set<MaximumSimpleUploadSizeOption>(
+              GOOGLE_CLOUD_CPP_STORAGE_DEFAULT_MAXIMUM_SIMPLE_UPLOAD_SIZE)
+          .set<EnableCurlSslLockingOption>(true)
+          .set<EnableCurlSigpipeHandlerOption>(true)
+          .set<MaximumCurlSocketRecvSizeOption>(0)
+          .set<MaximumCurlSocketSendSizeOption>(0)
+          .set<TransferStallTimeoutOption>(std::chrono::seconds(
+              GOOGLE_CLOUD_CPP_STORAGE_DEFAULT_DOWNLOAD_STALL_TIMEOUT))
+          .set<TransferStallMinimumRateOption>(1)
+          .set<DownloadStallMinimumRateOption>(1)
+          .set<RetryPolicyOption>(
+              LimitedTimeRetryPolicy(
+                  STORAGE_CLIENT_DEFAULT_MAXIMUM_RETRY_PERIOD)
+                  .clone())
+          .set<BackoffPolicyOption>(
+              ExponentialBackoffPolicy(
+                  STORAGE_CLIENT_DEFAULT_INITIAL_BACKOFF_DELAY,
+                  STORAGE_CLIENT_DEFAULT_MAXIMUM_BACKOFF_DELAY,
+                  STORAGE_CLIENT_DEFAULT_BACKOFF_SCALING)
+                  .clone())
+          .set<IdempotencyPolicyOption>(AlwaysRetryIdempotencyPolicy().clone());
+
+  o = google::cloud::internal::MergeOptions(std::move(opts), std::move(o));
+  // If the application did not set `DownloadStallTimeoutOption` then use the
+  // same value as `TransferStallTimeoutOption` (which could be the default
+  // value). Some applications need tighter timeouts for downloads, but longer
+  // timeouts for other transfers.
+  if (!o.has<DownloadStallTimeoutOption>()) {
+    o.set<DownloadStallTimeoutOption>(o.get<TransferStallTimeoutOption>());
+  }
+
+  auto emulator = GetEmulator();
+  if (emulator.has_value()) {
+    o.set<RestEndpointOption>(*emulator).set<IamEndpointOption>(*emulator +
+                                                                "/iamapi");
+  }
+
+  auto logging = GetEnv("CLOUD_STORAGE_ENABLE_TRACING");
+  if (logging) {
+    for (auto c : absl::StrSplit(*logging, ',')) {
+      GCP_LOG(INFO) << "Enabling logging for " << c;
+      o.lookup<LoggingComponentsOption>().insert(std::string(c));
+    }
+  }
+
+  auto tracing = GetEnv("GOOGLE_CLOUD_CPP_OPENTELEMETRY_TRACING");
+  if (tracing && !tracing->empty()) {
+    o.set<OpenTelemetryTracingOption>(true);
+  }
+
+  auto project_id = GetEnv("GOOGLE_CLOUD_PROJECT");
+  if (project_id.has_value()) {
+    o.set<ProjectIdOption>(std::move(*project_id));
+  }
+
+  // Always apply the RestClient defaults, even if it is not in use. Now that we
+  // use the low-level initialization code in
+  // google/cloud/internal/curl_wrappers.cc, these are always needed.
+  namespace rest = ::google::cloud::rest_internal;
+  auto rest_defaults = Options{}
+                           .set<rest::DownloadStallTimeoutOption>(
+                               o.get<DownloadStallTimeoutOption>())
+                           .set<rest::DownloadStallMinimumRateOption>(
+                               o.get<DownloadStallMinimumRateOption>())
+                           .set<rest::TransferStallTimeoutOption>(
+                               o.get<TransferStallTimeoutOption>())
+                           .set<rest::TransferStallMinimumRateOption>(
+                               o.get<TransferStallMinimumRateOption>())
+                           .set<rest::MaximumCurlSocketRecvSizeOption>(
+                               o.get<MaximumCurlSocketRecvSizeOption>())
+                           .set<rest::MaximumCurlSocketSendSizeOption>(
+                               o.get<MaximumCurlSocketSendSizeOption>())
+                           .set<rest::ConnectionPoolSizeOption>(
+                               o.get<ConnectionPoolSizeOption>())
+                           .set<rest::EnableCurlSslLockingOption>(
+                               o.get<EnableCurlSslLockingOption>())
+                           .set<rest::EnableCurlSigpipeHandlerOption>(
+                               o.get<EnableCurlSigpipeHandlerOption>());
+
+  // These two are not always present, but if they are, and only if they are, we
+  // need to map their value to the corresponding option in `rest_internal::`.
+  if (o.has<storage_experimental::HttpVersionOption>()) {
+    rest_defaults.set<rest::HttpVersionOption>(
+        o.get<storage_experimental::HttpVersionOption>());
+  }
+  if (o.has<internal::CAPathOption>()) {
+    rest_defaults.set<rest::CAPathOption>(o.get<internal::CAPathOption>());
+  }
+
+  return google::cloud::internal::MergeOptions(std::move(o),
+                                               std::move(rest_defaults));
+}
+
+Options DefaultOptionsWithCredentials(Options opts) {
+  if (opts.has<Oauth2CredentialsOption>()) {
+    auto credentials = opts.get<Oauth2CredentialsOption>();
+    return internal::DefaultOptions(std::move(credentials), std::move(opts));
+  }
+  if (opts.has<UnifiedCredentialsOption>()) {
+    auto credentials =
+        internal::MapCredentials(*opts.get<UnifiedCredentialsOption>());
+    return internal::DefaultOptions(std::move(credentials), std::move(opts));
+  }
+  if (GetEmulator().has_value()) {
+    return internal::DefaultOptions(
+        internal::MapCredentials(*google::cloud::MakeInsecureCredentials()),
+        std::move(opts));
+  }
+  auto credentials =
+      internal::MapCredentials(*google::cloud::MakeGoogleDefaultCredentials(
+          google::cloud::internal::MakeAuthOptions(opts)));
+  return internal::DefaultOptions(std::move(credentials), std::move(opts));
+}
+
 Client ClientImplDetails::CreateWithDecorations(
     Options const& opts, std::shared_ptr<StorageConnection> connection) {
   return Client(

google/cloud/storage/client.h

@@ -15,7 +15,6 @@
 #ifndef GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_STORAGE_CLIENT_H
 #define GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_STORAGE_CLIENT_H
 
-#include "google/cloud/storage/client_options.h"
 #include "google/cloud/storage/hmac_key_metadata.h"
 #include "google/cloud/storage/internal/policy_document_request.h"
 #include "google/cloud/storage/internal/request_project_id.h"
@@ -55,6 +54,23 @@ namespace internal {
 class NonResumableParallelUploadState;
 class ResumableParallelUploadState;
 struct ClientImplDetails;
+
+Options ApplyPolicy(Options opts, RetryPolicy const& p);
+Options ApplyPolicy(Options opts, BackoffPolicy const& p);
+Options ApplyPolicy(Options opts, IdempotencyPolicy const& p);
+
+inline Options ApplyPolicies(Options opts) { return opts; }
+
+template <typename P, typename... Policies>
+Options ApplyPolicies(Options opts, P&& head, Policies&&... tail) {
+  opts = ApplyPolicy(std::move(opts), std::forward<P>(head));
+  return ApplyPolicies(std::move(opts), std::forward<Policies>(tail)...);
+}
+
+Options DefaultOptions(std::shared_ptr<oauth2::Credentials> credentials,
+                       Options opts);
+Options DefaultOptionsWithCredentials(Options opts);
+
 }  // namespace internal
 /**
  * The Google Cloud Storage (GCS) Client.