diff --git a/packages/google-cloud-storage/google/cloud/storage/asyncio/async_grpc_client.py b/packages/google-cloud-storage/google/cloud/storage/asyncio/async_grpc_client.py index e113943857a8..12a3d8ff722f 100644 --- a/packages/google-cloud-storage/google/cloud/storage/asyncio/async_grpc_client.py +++ b/packages/google-cloud-storage/google/cloud/storage/asyncio/async_grpc_client.py @@ -26,9 +26,8 @@ from google.cloud.storage._helpers import create_trace_span_helper from google.cloud.storage.asyncio import _utils from google.cloud.storage.grpc_client import ( - _DEFAULT_HOST, - _resolve_direct_path_interconnect, - _rewrite_host_for_interconnect, + _resolve_grpc_config, + _resolve_universe_domain, ) @@ -118,6 +117,7 @@ def __init__( raise ValueError( "Either client_options or `client_option.api_endpoint` is None. Please provide api_endpoint when `AnonymousCredentials` is used " ) + self._universe_domain = _resolve_universe_domain(client_options) self._grpc_client = self._create_anonymous_client( client_options, credentials ) @@ -132,18 +132,20 @@ def __init__( if agent_version not in client_info.user_agent: client_info.user_agent += f" {agent_version} " - use_dp_interconnect = _resolve_direct_path_interconnect( - attempt_direct_path_xds_over_interconnect - ) - + self._universe_domain = _resolve_universe_domain(client_options) self._grpc_client = self._create_async_grpc_client( credentials=credentials, client_info=client_info, client_options=client_options, attempt_direct_path=attempt_direct_path, - attempt_direct_path_xds_over_interconnect=use_dp_interconnect, + attempt_direct_path_xds_over_interconnect=attempt_direct_path_xds_over_interconnect, ) + @property + def universe_domain(self) -> str: + """The universe domain used by the client instance.""" + return self._universe_domain + @property def metrics_enabled(self) -> bool: """(Experimental) Returns ``True`` if OpenTelemetry metrics recording is active for this client.""" @@ -177,40 +179,34 @@ def _create_async_grpc_client( attempt_direct_path=True, attempt_direct_path_xds_over_interconnect=False, ): + ( + self._universe_domain, + host, + quota_project_id, + attempt_direct_path, + use_dp_interconnect, + ) = _resolve_grpc_config( + client_options, + attempt_direct_path, + attempt_direct_path_xds_over_interconnect, + ) transport_cls = storage_v2.StorageAsyncClient.get_transport_class( "grpc_asyncio" ) primary_user_agent = client_info.to_user_agent() - - host = _DEFAULT_HOST - quota_project_id = None - if isinstance(client_options, dict): - host = client_options.get("api_endpoint") or _DEFAULT_HOST - quota_project_id = client_options.get("quota_project_id") - elif client_options: - host = getattr(client_options, "api_endpoint", None) or _DEFAULT_HOST - quota_project_id = getattr(client_options, "quota_project_id", None) - - if attempt_direct_path_xds_over_interconnect: - host = _rewrite_host_for_interconnect(host) - channel = transport_cls.create_channel( - host=host, - quota_project_id=quota_project_id, - attempt_direct_path=True, - attempt_direct_path_xds_over_interconnect=True, - credentials=credentials, - options=(("grpc.primary_user_agent", primary_user_agent),), - ) - else: - channel = transport_cls.create_channel( - host=host, - quota_project_id=quota_project_id, - attempt_direct_path=attempt_direct_path, - credentials=credentials, - options=(("grpc.primary_user_agent", primary_user_agent),), - ) - transport = transport_cls(channel=channel) + channel_kwargs = { + "host": host, + "quota_project_id": quota_project_id, + "attempt_direct_path": attempt_direct_path, + "credentials": credentials, + "options": (("grpc.primary_user_agent", primary_user_agent),), + } + if use_dp_interconnect: + channel_kwargs["attempt_direct_path_xds_over_interconnect"] = True + + channel = transport_cls.create_channel(**channel_kwargs) + transport = transport_cls(host=host, channel=channel) return storage_v2.StorageAsyncClient( transport=transport, diff --git a/packages/google-cloud-storage/google/cloud/storage/grpc_client.py b/packages/google-cloud-storage/google/cloud/storage/grpc_client.py index 79bcb4f0d8b1..d5c0e302415b 100644 --- a/packages/google-cloud-storage/google/cloud/storage/grpc_client.py +++ b/packages/google-cloud-storage/google/cloud/storage/grpc_client.py @@ -16,12 +16,15 @@ import os +from google.api_core import universe from google.cloud.client import ClientWithProject from google.cloud import _storage_v2 as storage_v2 +from google.cloud._storage_v2._compat import get_universe_domain from google.cloud.storage import _opentelemetry_metrics _marker = object() +_DEFAULT_UNIVERSE_DOMAIN = universe.DEFAULT_UNIVERSE _DEFAULT_HOST = "storage.googleapis.com" _DEFAULT_HOST_DIRECT_PATH = "storage-direct.googleapis.com" _DIRECT_PATH_INTERCONNECT_ENV = "GOOGLE_CLOUD_ENABLE_DIRECT_PATH_XDS_OVER_INTERCONNECT" @@ -60,6 +63,54 @@ def _rewrite_host_for_interconnect( return host +def _get_option(client_options, name): + """Reads an option from a dict, a ``ClientOptions`` instance, or ``None``.""" + if isinstance(client_options, dict): + return client_options.get(name) + return getattr(client_options, name, None) + + +def _resolve_universe_domain(client_options) -> str: + """Resolves the universe domain with the helper the generated client uses.""" + return get_universe_domain( + _get_option(client_options, "universe_domain"), + os.getenv("GOOGLE_CLOUD_UNIVERSE_DOMAIN"), + default_universe=_DEFAULT_UNIVERSE_DOMAIN, + ) + + +def _resolve_grpc_config( + client_options, + attempt_direct_path, + attempt_direct_path_xds_over_interconnect: bool, +): + """Resolves universe domain, target host, quota project, and DirectPath flags. + + DirectPath (including DirectPath over Interconnect) is only attempted in the + default universe. + """ + universe_domain = _resolve_universe_domain(client_options) + use_dp_interconnect = _resolve_direct_path_interconnect( + attempt_direct_path_xds_over_interconnect + ) + if universe_domain.lower() != _DEFAULT_UNIVERSE_DOMAIN.lower(): + attempt_direct_path = False + use_dp_interconnect = False + + host = _get_option(client_options, "api_endpoint") or f"storage.{universe_domain}" + if use_dp_interconnect: + host = _rewrite_host_for_interconnect(host) + attempt_direct_path = True + + return ( + universe_domain, + host, + _get_option(client_options, "quota_project_id"), + attempt_direct_path, + use_dp_interconnect, + ) + + class GrpcClient(ClientWithProject): """A client for interacting with Google Cloud Storage using the gRPC API. @@ -151,18 +202,20 @@ def __init__( elif api_key: client_options.api_key = api_key - use_dp_interconnect = _resolve_direct_path_interconnect( - attempt_direct_path_xds_over_interconnect - ) - + self._universe_domain = _resolve_universe_domain(client_options) self._grpc_client = self._create_gapic_client( credentials=credentials, client_info=client_info, client_options=client_options, attempt_direct_path=attempt_direct_path, - attempt_direct_path_xds_over_interconnect=use_dp_interconnect, + attempt_direct_path_xds_over_interconnect=attempt_direct_path_xds_over_interconnect, ) + @property + def universe_domain(self) -> str: + """The universe domain used by the client instance.""" + return self._universe_domain + @property def metrics_enabled(self) -> bool: """(Experimental) Returns ``True`` if OpenTelemetry metrics recording is active for this client.""" @@ -182,36 +235,33 @@ def _create_gapic_client( attempt_direct_path_xds_over_interconnect=False, ): """Creates and configures the low-level GAPIC `storage_v2` client.""" - transport_cls = storage_v2.StorageClient.get_transport_class("grpc") - - if attempt_direct_path_xds_over_interconnect: - host = _DEFAULT_HOST - quota_project_id = None - if isinstance(client_options, dict): - host = client_options.get("api_endpoint") or _DEFAULT_HOST - quota_project_id = client_options.get("quota_project_id") - elif client_options is not None: - host = getattr(client_options, "api_endpoint", None) or _DEFAULT_HOST - quota_project_id = getattr(client_options, "quota_project_id", None) - host = _rewrite_host_for_interconnect(host) - channel_kwargs = { - "host": host, - "credentials": credentials, - "attempt_direct_path": True, - "attempt_direct_path_xds_over_interconnect": True, - } - if quota_project_id is not None: - channel_kwargs["quota_project_id"] = quota_project_id - channel = transport_cls.create_channel(**channel_kwargs) - else: - channel = transport_cls.create_channel( - attempt_direct_path=attempt_direct_path - ) + ( + self._universe_domain, + host, + quota_project_id, + attempt_direct_path, + use_dp_interconnect, + ) = _resolve_grpc_config( + client_options, + attempt_direct_path, + attempt_direct_path_xds_over_interconnect, + ) - transport = transport_cls(credentials=credentials, channel=channel) + transport_cls = storage_v2.StorageClient.get_transport_class("grpc") + channel_kwargs = { + "host": host, + "credentials": credentials, + "attempt_direct_path": attempt_direct_path, + } + if use_dp_interconnect: + channel_kwargs["attempt_direct_path_xds_over_interconnect"] = True + if quota_project_id is not None: + channel_kwargs["quota_project_id"] = quota_project_id + + channel = transport_cls.create_channel(**channel_kwargs) + transport = transport_cls(host=host, channel=channel) return storage_v2.StorageClient( - credentials=credentials, transport=transport, client_info=client_info, client_options=client_options, diff --git a/packages/google-cloud-storage/tests/unit/asyncio/test_async_grpc_client.py b/packages/google-cloud-storage/tests/unit/asyncio/test_async_grpc_client.py index 0b96a92eb055..242c26243af7 100644 --- a/packages/google-cloud-storage/tests/unit/asyncio/test_async_grpc_client.py +++ b/packages/google-cloud-storage/tests/unit/asyncio/test_async_grpc_client.py @@ -24,6 +24,15 @@ from google.cloud.storage.asyncio import async_grpc_client +@pytest.fixture(autouse=True) +def _hermetic_env(monkeypatch): + """Keeps the runner's environment from changing host or DirectPath resolution.""" + monkeypatch.delenv("GOOGLE_CLOUD_UNIVERSE_DOMAIN", raising=False) + monkeypatch.delenv( + "GOOGLE_CLOUD_ENABLE_DIRECT_PATH_XDS_OVER_INTERCONNECT", raising=False + ) + + def _make_credentials(spec=None): if spec is None: return mock.Mock(spec=auth_credentials.Credentials) @@ -60,7 +69,9 @@ def test_constructor_default_options(self, mock_async_storage_client): options=expected_options, ) mock_channel = mock_transport_cls.create_channel.return_value - mock_transport_cls.assert_called_once_with(channel=mock_channel) + mock_transport_cls.assert_called_once_with( + host="storage.googleapis.com", channel=mock_channel + ) mock_transport = mock_transport_cls.return_value assert kwargs["transport"] is mock_transport assert kwargs["client_options"] is None @@ -147,7 +158,9 @@ def test_constructor_disables_directpath(self, mock_async_storage_client): options=expected_options, ) mock_channel = mock_transport_cls.create_channel.return_value - mock_transport_cls.assert_called_once_with(channel=mock_channel) + mock_transport_cls.assert_called_once_with( + host="storage.googleapis.com", channel=mock_channel + ) @mock.patch("google.cloud._storage_v2.StorageAsyncClient") def test_metrics_properties(self, mock_async_storage_client): @@ -291,7 +304,9 @@ def test_grpc_client_property(self, mock_grpc_gapic_client): credentials=mock_creds, options=expected_options, ) - mock_transport_cls.assert_called_once_with(channel=channel_sentinel) + mock_transport_cls.assert_called_once_with( + host="storage.googleapis.com", channel=channel_sentinel + ) mock_grpc_gapic_client.assert_called_once_with( transport=mock_transport_instance, client_info=client_info, @@ -648,3 +663,124 @@ def test_async_grpc_client_interconnect_env_override( ): with pytest.raises(ValueError): async_grpc_client.AsyncGrpcClient(credentials=mock_creds) + + @pytest.mark.parametrize( + "env,opts,expected_host", + [ + ( + {"GOOGLE_CLOUD_ENABLE_DIRECT_PATH_XDS_OVER_INTERCONNECT": "true"}, + {"universe_domain": "example.com"}, + "storage.example.com", + ), + ( + {"GOOGLE_CLOUD_UNIVERSE_DOMAIN": "example.com"}, + None, + "storage.example.com", + ), + ( + {}, + client_options.ClientOptions( + universe_domain="example.com", api_endpoint="custom.example.com:443" + ), + "custom.example.com:443", + ), + ], + ) + @mock.patch("google.cloud._storage_v2.StorageAsyncClient") + def test_universe_domain_routing_and_direct_path_disabled( + self, mock_async_storage_client, env, opts, expected_host + ): + mock_transport_cls = mock.MagicMock() + mock_async_storage_client.get_transport_class.return_value = mock_transport_cls + mock_creds = _make_credentials() + mock_creds.universe_domain = "example.com" + + with mock.patch.dict("os.environ", env): + client = async_grpc_client.AsyncGrpcClient( + credentials=mock_creds, + client_options=opts, + attempt_direct_path=True, + attempt_direct_path_xds_over_interconnect=True, + ) + + assert client.universe_domain == "example.com" + primary_ua = mock_async_storage_client.call_args.kwargs[ + "client_info" + ].to_user_agent() + mock_transport_cls.create_channel.assert_called_once_with( + host=expected_host, + quota_project_id=None, + attempt_direct_path=False, + credentials=mock_creds, + options=(("grpc.primary_user_agent", primary_ua),), + ) + mock_transport_cls.assert_called_once_with( + host=expected_host, + channel=mock_transport_cls.create_channel.return_value, + ) + + @mock.patch("google.cloud._storage_v2.StorageAsyncClient") + def test_empty_universe_domain_error(self, mock_async_storage_client): + mock_transport_cls = mock.MagicMock() + mock_async_storage_client.get_transport_class.return_value = mock_transport_cls + + with pytest.raises( + ValueError, match="Universe Domain cannot be an empty string" + ): + async_grpc_client.AsyncGrpcClient( + credentials=_make_credentials(), + client_options=client_options.ClientOptions(universe_domain=""), + ) + mock_transport_cls.create_channel.assert_not_called() + + @pytest.mark.parametrize( + "env,universe_opt,expected", + [ + ({}, "example.com", "example.com"), + ({"GOOGLE_CLOUD_UNIVERSE_DOMAIN": "example.com"}, None, "example.com"), + ({}, None, "googleapis.com"), + ], + ) + @mock.patch("google.cloud._storage_v2.StorageAsyncClient") + @mock.patch( + "google.cloud.storage.asyncio.async_grpc_client.grpc.aio.insecure_channel" + ) + def test_anonymous_credentials_universe_domain( + self, + mock_insecure_channel, + mock_async_storage_client, + env, + universe_opt, + expected, + ): + with mock.patch.dict("os.environ", env): + client = async_grpc_client.AsyncGrpcClient( + client_options=client_options.ClientOptions( + api_endpoint="localhost:8888", universe_domain=universe_opt + ), + credentials=AnonymousCredentials(), + ) + + assert client.universe_domain == expected + mock_insecure_channel.assert_called_once_with("localhost:8888") + + @mock.patch("google.api_core.grpc_helpers_async.create_channel") + def test_real_gapic_client_integration_non_default_universe( + self, mock_create_channel + ): + import grpc.aio + + mock_channel = mock.MagicMock(spec=grpc.aio.Channel) + mock_channel._unary_unary_interceptors = [] + mock_create_channel.return_value = mock_channel + mock_creds = _make_credentials() + mock_creds.universe_domain = "example.com" + client = async_grpc_client.AsyncGrpcClient( + credentials=mock_creds, + client_options=client_options.ClientOptions(universe_domain="example.com"), + ) + assert client.universe_domain == "example.com" + assert client.grpc_client.universe_domain == "example.com" + assert client.grpc_client.api_endpoint == "storage.example.com:443" + assert mock_create_channel.call_args.args[0] == "storage.example.com" + assert mock_create_channel.call_args.kwargs["attempt_direct_path"] is False diff --git a/packages/google-cloud-storage/tests/unit/test_grpc_client.py b/packages/google-cloud-storage/tests/unit/test_grpc_client.py index 5e3789549696..bb8203df7810 100644 --- a/packages/google-cloud-storage/tests/unit/test_grpc_client.py +++ b/packages/google-cloud-storage/tests/unit/test_grpc_client.py @@ -15,12 +15,22 @@ import unittest from unittest import mock +import pytest from google.api_core import client_options as client_options_lib from google.auth import credentials as auth_credentials from google.cloud.storage import grpc_client +@pytest.fixture(autouse=True) +def _hermetic_env(monkeypatch): + """Keeps the runner's environment from changing host or DirectPath resolution.""" + monkeypatch.delenv("GOOGLE_CLOUD_UNIVERSE_DOMAIN", raising=False) + monkeypatch.delenv( + "GOOGLE_CLOUD_ENABLE_DIRECT_PATH_XDS_OVER_INTERCONNECT", raising=False + ) + + def _make_credentials(spec=None): if spec is None: return mock.Mock(spec=auth_credentials.Credentials) @@ -54,16 +64,17 @@ def test_constructor_defaults_and_options( project="test-project", credentials=mock_creds ) - # 2. Assert DirectPath is ON by default. + # 2. Assert DirectPath is ON by default and endpoint/credentials are forwarded. mock_storage_client.get_transport_class.assert_called_once_with("grpc") mock_transport_cls.create_channel.assert_called_once_with( - attempt_direct_path=True + host="test.endpoint", + credentials=mock_creds, + attempt_direct_path=True, ) # 3. Assert the GAPIC client was created with the correct options. mock_transport = mock_transport_cls.return_value mock_storage_client.assert_called_once_with( - credentials=mock_creds, transport=mock_transport, client_info=mock_client_info, client_options=client_options_dict, @@ -90,7 +101,9 @@ def test_constructor_disables_direct_path( ) mock_transport_cls.create_channel.assert_called_once_with( - attempt_direct_path=False + host="storage.googleapis.com", + credentials=mock_creds, + attempt_direct_path=False, ) @mock.patch("google.cloud.storage.grpc_client.ClientWithProject") @@ -115,7 +128,6 @@ def test_constructor_initialize_with_api_key( # that contains the api_key. mock_transport = mock_transport_cls.return_value mock_storage_client.assert_called_once_with( - credentials=mock_creds, transport=mock_transport, client_info=None, client_options={"api_key": "test-api-key"}, @@ -159,7 +171,6 @@ def test_constructor_with_api_key_and_client_options( ) mock_storage_client.assert_called_once_with( - credentials=mock_creds, transport=mock_transport, client_info=None, client_options=client_options_obj, @@ -192,7 +203,6 @@ def test_constructor_with_api_key_and_dict_options( "api_key": "new-test-key", } mock_storage_client.assert_called_once_with( - credentials=mock_creds, transport=mock_transport, client_info=None, client_options=expected_options, @@ -314,7 +324,9 @@ def test_grpc_client_attempt_direct_path_xds_over_interconnect_env_override( ) mock_transport_cls.create_channel.assert_called_once_with( - attempt_direct_path=True + host="storage.googleapis.com", + credentials=mock_creds, + attempt_direct_path=True, ) with mock.patch.dict( @@ -327,6 +339,106 @@ def test_grpc_client_attempt_direct_path_xds_over_interconnect_env_override( credentials=mock_creds, ) + @mock.patch("google.cloud._storage_v2.StorageClient") + def test_universe_domain_routing_and_direct_path_disabled( + self, mock_storage_client + ): + other_creds = _make_credentials() + other_creds.universe_domain = "example.com" + anon_creds = auth_credentials.AnonymousCredentials() + cases = [ + ( + {"GOOGLE_CLOUD_ENABLE_DIRECT_PATH_XDS_OVER_INTERCONNECT": "true"}, + other_creds, + {"universe_domain": "example.com", "quota_project_id": "qp"}, + "storage.example.com", + {"quota_project_id": "qp"}, + ), + ( + {"GOOGLE_CLOUD_UNIVERSE_DOMAIN": "example.com"}, + other_creds, + None, + "storage.example.com", + {}, + ), + ( + {}, + other_creds, + client_options_lib.ClientOptions( + universe_domain="example.com", api_endpoint="custom.example.com:443" + ), + "custom.example.com:443", + {}, + ), + ( + {}, + anon_creds, + {"universe_domain": "example.com", "api_endpoint": "localhost:8888"}, + "localhost:8888", + {}, + ), + ] + for env, creds, opts, expected_host, extra_kw in cases: + mock_transport_cls = mock.MagicMock() + mock_storage_client.get_transport_class.return_value = mock_transport_cls + with ( + self.subTest(expected_host=expected_host, opts=opts), + mock.patch.dict("os.environ", env, clear=False), + ): + client = grpc_client.GrpcClient( + project="test-project", + credentials=creds, + client_options=opts, + attempt_direct_path=True, + attempt_direct_path_xds_over_interconnect=True, + ) + self.assertEqual(client.universe_domain, "example.com") + mock_transport_cls.create_channel.assert_called_once_with( + host=expected_host, + credentials=creds, + attempt_direct_path=False, + **extra_kw, + ) + mock_transport_cls.assert_called_once_with( + host=expected_host, + channel=mock_transport_cls.create_channel.return_value, + ) + + @mock.patch("google.cloud._storage_v2.StorageClient") + def test_empty_universe_domain_error(self, mock_storage_client): + mock_transport_cls = mock.MagicMock() + mock_storage_client.get_transport_class.return_value = mock_transport_cls + + with self.assertRaisesRegex( + ValueError, "Universe Domain cannot be an empty string" + ): + grpc_client.GrpcClient( + project="test-project", + credentials=_make_credentials(), + client_options={"universe_domain": ""}, + ) + mock_transport_cls.create_channel.assert_not_called() + + @mock.patch("google.api_core.grpc_helpers.create_channel") + def test_real_gapic_client_integration_non_default_universe( + self, mock_create_channel + ): + import grpc + + mock_create_channel.return_value = mock.MagicMock(spec=grpc.Channel) + mock_creds = _make_credentials() + mock_creds.universe_domain = "example.com" + client = grpc_client.GrpcClient( + project="test-project", + credentials=mock_creds, + client_options={"universe_domain": "example.com"}, + ) + self.assertEqual(client.universe_domain, "example.com") + self.assertEqual(client.grpc_client.universe_domain, "example.com") + self.assertEqual(client.grpc_client.api_endpoint, "storage.example.com:443") + self.assertEqual(mock_create_channel.call_args.args[0], "storage.example.com") + self.assertFalse(mock_create_channel.call_args.kwargs["attempt_direct_path"]) + def test_rewrite_host_for_interconnect_delimiters(self): cases = [ ("storage.googleapis.com", "storage-direct.googleapis.com"),