From b9d4a049c3b7701af706fdfe5b6206f4f2ba7629 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Thu, 11 Sep 2025 12:03:25 -0400 Subject: [PATCH 01/25] chore: removes old proof of concept --- .../services/centralized_service/__init__.py | 18 - .../services/centralized_service/_helpers.py | 24 -- .../services/centralized_service/client.py | 249 ------------- .../bigquery_v2/test_centralized_service.py | 332 ------------------ 4 files changed, 623 deletions(-) delete mode 100644 google/cloud/bigquery_v2/services/centralized_service/__init__.py delete mode 100644 google/cloud/bigquery_v2/services/centralized_service/_helpers.py delete mode 100644 google/cloud/bigquery_v2/services/centralized_service/client.py delete mode 100644 tests/unit/gapic/bigquery_v2/test_centralized_service.py diff --git a/google/cloud/bigquery_v2/services/centralized_service/__init__.py b/google/cloud/bigquery_v2/services/centralized_service/__init__.py deleted file mode 100644 index 03ddf6619..000000000 --- a/google/cloud/bigquery_v2/services/centralized_service/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from .client import BigQueryClient - -__all__ = "BigQueryClient" diff --git a/google/cloud/bigquery_v2/services/centralized_service/_helpers.py b/google/cloud/bigquery_v2/services/centralized_service/_helpers.py deleted file mode 100644 index 53b585b18..000000000 --- a/google/cloud/bigquery_v2/services/centralized_service/_helpers.py +++ /dev/null @@ -1,24 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -def _drop_self_key(kwargs): - "Drops 'self' key from a given kwargs dict." - - if not isinstance(kwargs, dict): - raise TypeError("kwargs must be a dict.") - kwargs.pop("self", None) # Essentially a no-op if 'self' key does not exist - return kwargs diff --git a/google/cloud/bigquery_v2/services/centralized_service/client.py b/google/cloud/bigquery_v2/services/centralized_service/client.py deleted file mode 100644 index 3b53fb53b..000000000 --- a/google/cloud/bigquery_v2/services/centralized_service/client.py +++ /dev/null @@ -1,249 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import os -from typing import ( - Optional, - Sequence, - Tuple, - Union, -) - -from google.cloud.bigquery_v2.services.centralized_service import _helpers - -# Import service client modules -from google.cloud.bigquery_v2.services import ( - dataset_service, - job_service, - model_service, -) - -# Import types modules (to access *Requests classes) -from google.cloud.bigquery_v2.types import ( - dataset, - job, - model, -) - -from google.api_core import client_options as client_options_lib -from google.api_core import gapic_v1 -from google.api_core import retry as retries -from google.auth import credentials as auth_credentials - -# Create a type alias -try: - OptionalRetry = Union[retries.Retry, gapic_v1.method._MethodDefault, None] -except AttributeError: # pragma: NO COVER - OptionalRetry = Union[retries.Retry, object, None] # type: ignore - -# TODO: This line is here to simplify prototyping, etc. -PROJECT_ID = os.environ.get("GOOGLE_CLOUD_PROJECT") - -DEFAULT_RETRY: OptionalRetry = gapic_v1.method.DEFAULT -DEFAULT_TIMEOUT: Union[float, object] = gapic_v1.method.DEFAULT -DEFAULT_METADATA: Sequence[Tuple[str, Union[str, bytes]]] = () - - -# Create Centralized Client -class BigQueryClient: - def __init__( - self, - *, - credentials: Optional[auth_credentials.Credentials] = None, - client_options: Optional[Union[client_options_lib.ClientOptions, dict]] = None, - ): - self._clients = {} - self._credentials = credentials - self._client_options = client_options - - @property - def dataset_service_client(self): - if "dataset" not in self._clients: - from google.cloud.bigquery_v2.services import dataset_service - - self._clients["dataset"] = dataset_service.DatasetServiceClient( - credentials=self._credentials, client_options=self._client_options - ) - return self._clients["dataset"] - - @dataset_service_client.setter - def dataset_service_client(self, value): - # Check for the methods the centralized client exposes (to allow duck-typing) - required_methods = [ - "get_dataset", - "insert_dataset", - "patch_dataset", - "update_dataset", - "delete_dataset", - "list_datasets", - "undelete_dataset", - ] - for method in required_methods: - if not hasattr(value, method) or not callable(getattr(value, method)): - raise AttributeError( - f"Object assigned to dataset_service_client is missing a callable '{method}' method." - ) - self._clients["dataset"] = value - - @property - def job_service_client(self): - if "job" not in self._clients: - from google.cloud.bigquery_v2.services import job_service - - self._clients["job"] = job_service.JobServiceClient( - credentials=self._credentials, client_options=self._client_options - ) - return self._clients["job"] - - @job_service_client.setter - def job_service_client(self, value): - required_methods = [ - "get_job", - "insert_job", - "cancel_job", - "delete_job", - "list_jobs", - ] - for method in required_methods: - if not hasattr(value, method) or not callable(getattr(value, method)): - raise AttributeError( - f"Object assigned to job_service_client is missing a callable '{method}' method." - ) - self._clients["job"] = value - - @property - def model_service_client(self): - if "model" not in self._clients: - from google.cloud.bigquery_v2.services import model_service - - self._clients["model"] = model_service.ModelServiceClient( - credentials=self._credentials, client_options=self._client_options - ) - return self._clients["model"] - - @model_service_client.setter - def model_service_client(self, value): - required_methods = [ - "get_model", - "delete_model", - "patch_model", - "list_models", - ] - for method in required_methods: - if not hasattr(value, method) or not callable(getattr(value, method)): - raise AttributeError( - f"Object assigned to model_service_client is missing a callable '{method}' method." - ) - self._clients["model"] = value - - def get_dataset( - self, - request: Optional[Union[dataset.GetDatasetRequest, dict]] = None, - *, - retry: OptionalRetry = DEFAULT_RETRY, - timeout: Union[float, object] = DEFAULT_TIMEOUT, - metadata: Sequence[Tuple[str, Union[str, bytes]]] = DEFAULT_METADATA, - ): - """ - TODO: Docstring is purposefully blank. microgenerator will add automatically. - """ - kwargs = _helpers._drop_self_key(locals()) - return self.dataset_service_client.get_dataset(**kwargs) - - def list_datasets( - self, - request: Optional[Union[dataset.ListDatasetsRequest, dict]] = None, - *, - retry: OptionalRetry = DEFAULT_RETRY, - timeout: Union[float, object] = DEFAULT_TIMEOUT, - metadata: Sequence[Tuple[str, Union[str, bytes]]] = DEFAULT_METADATA, - ): - """ - TODO: Docstring is purposefully blank. microgenerator will add automatically. - """ - kwargs = _helpers._drop_self_key(locals()) - return self.dataset_service_client.list_datasets(**kwargs) - - def list_jobs( - self, - request: Optional[Union[job.ListJobsRequest, dict]] = None, - *, - retry: OptionalRetry = DEFAULT_RETRY, - timeout: Union[float, object] = DEFAULT_TIMEOUT, - metadata: Sequence[Tuple[str, Union[str, bytes]]] = DEFAULT_METADATA, - ): - """ - TODO: Docstring is purposefully blank. microgenerator will add automatically. - """ - kwargs = _helpers._drop_self_key(locals()) - return self.job_service_client.list_jobs(**kwargs) - - def get_model( - self, - request: Optional[Union[model.GetModelRequest, dict]] = None, - *, - retry: OptionalRetry = DEFAULT_RETRY, - timeout: Union[float, object] = DEFAULT_TIMEOUT, - metadata: Sequence[Tuple[str, Union[str, bytes]]] = DEFAULT_METADATA, - ): - """ - TODO: Docstring is purposefully blank. microgenerator will add automatically. - """ - kwargs = _helpers._drop_self_key(locals()) - return self.model_service_client.get_model(**kwargs) - - def delete_model( - self, - request: Optional[Union[model.DeleteModelRequest, dict]] = None, - *, - retry: OptionalRetry = DEFAULT_RETRY, - timeout: Union[float, object] = DEFAULT_TIMEOUT, - metadata: Sequence[Tuple[str, Union[str, bytes]]] = DEFAULT_METADATA, - ): - """ - TODO: Docstring is purposefully blank. microgenerator will add automatically. - """ - kwargs = _helpers._drop_self_key(locals()) - # The underlying GAPIC client returns None on success. - return self.model_service_client.delete_model(**kwargs) - - def patch_model( - self, - request: Optional[Union[model.PatchModelRequest, dict]] = None, - *, - retry: OptionalRetry = DEFAULT_RETRY, - timeout: Union[float, object] = DEFAULT_TIMEOUT, - metadata: Sequence[Tuple[str, Union[str, bytes]]] = DEFAULT_METADATA, - ): - """ - TODO: Docstring is purposefully blank. microgenerator will add automatically. - """ - kwargs = _helpers._drop_self_key(locals()) - return self.model_service_client.patch_model(**kwargs) - - def list_models( - self, - request: Optional[Union[model.ListModelsRequest, dict]] = None, - *, - retry: OptionalRetry = DEFAULT_RETRY, - timeout: Union[float, object] = DEFAULT_TIMEOUT, - metadata: Sequence[Tuple[str, Union[str, bytes]]] = DEFAULT_METADATA, - ): - """ - TODO: Docstring is purposefully blank. microgenerator will add automatically. - """ - kwargs = _helpers._drop_self_key(locals()) - return self.model_service_client.list_models(**kwargs) diff --git a/tests/unit/gapic/bigquery_v2/test_centralized_service.py b/tests/unit/gapic/bigquery_v2/test_centralized_service.py deleted file mode 100644 index 9160dc7b1..000000000 --- a/tests/unit/gapic/bigquery_v2/test_centralized_service.py +++ /dev/null @@ -1,332 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import pytest -from typing import ( - Optional, - Sequence, - Tuple, - Union, -) -from unittest import mock - -from google.api_core import client_options as client_options_lib -from google.api_core import gapic_v1 -from google.api_core import retry as retries -from google.auth import credentials as auth_credentials - -# --- IMPORT SERVICECLIENT MODULES --- -from google.cloud.bigquery_v2.services import ( - centralized_service, - dataset_service, - job_service, - model_service, -) - -# --- IMPORT TYPES MODULES (to access *Requests classes) --- -from google.cloud.bigquery_v2.types import ( - dataset, - job, - model, -) - -# --- TYPE ALIASES --- -try: - OptionalRetry = Union[retries.Retry, gapic_v1.method._MethodDefault, None] -except AttributeError: # pragma: NO COVER - OptionalRetry = Union[retries.Retry, object, None] # type: ignore - -AnyRequest = Union[ - dataset.GetDatasetRequest, - model.GetModelRequest, - model.DeleteModelRequest, - model.PatchModelRequest, - job.ListJobsRequest, - model.ListModelsRequest, -] - -# --- CONSTANTS --- -PROJECT_ID = "test-project" -DATASET_ID = "test_dataset" -JOB_ID = "test_job" -MODEL_ID = "test_model" -DEFAULT_ETAG = "test_etag" - -DEFAULT_RETRY: OptionalRetry = gapic_v1.method.DEFAULT -DEFAULT_TIMEOUT: Union[float, object] = gapic_v1.method.DEFAULT -DEFAULT_METADATA: Sequence[Tuple[str, Union[str, bytes]]] = () - -# --- HELPERS --- -def assert_client_called_once_with( - mock_method: mock.Mock, - request: AnyRequest, - retry: OptionalRetry = DEFAULT_RETRY, - timeout: Union[float, object] = DEFAULT_TIMEOUT, - metadata: Sequence[Tuple[str, Union[str, bytes]]] = DEFAULT_METADATA, -): - """Helper to assert a client method was called with default args.""" - mock_method.assert_called_once_with( - request=request, - retry=retry, - timeout=timeout, - metadata=metadata, - ) - - -# --- FIXTURES --- -@pytest.fixture -def mock_dataset_service_client(): - """Mocks the DatasetServiceClient.""" - with mock.patch( - "google.cloud.bigquery_v2.services.dataset_service.DatasetServiceClient", - autospec=True, - ) as mock_client: - yield mock_client - - -@pytest.fixture -def mock_job_service_client(): - """Mocks the JobServiceClient.""" - with mock.patch( - "google.cloud.bigquery_v2.services.job_service.JobServiceClient", - autospec=True, - ) as mock_client: - yield mock_client - - -@pytest.fixture -def mock_model_service_client(): - """Mocks the ModelServiceClient.""" - with mock.patch( - "google.cloud.bigquery_v2.services.model_service.ModelServiceClient", - autospec=True, - ) as mock_client: - yield mock_client - - -# TODO: figure out a solution for this... is there an easier way to feed in clients? -# TODO: is there an easier way to make mock_x_service_clients? -@pytest.fixture -def bq_client( - mock_dataset_service_client, mock_job_service_client, mock_model_service_client -): - """Provides a BigQueryClient with mocked underlying services.""" - client = centralized_service.BigQueryClient() - client.dataset_service_client = mock_dataset_service_client - client.job_service_client = mock_job_service_client - client.model_service_client = mock_model_service_client - ... - return client - - -# --- TEST CLASSES --- - -from google.api_core import client_options as client_options_lib - -# from google.api_core.client_options import ClientOptions -from google.auth import credentials as auth_credentials - -# from google.auth.credentials import Credentials - - -class TestCentralizedClientInitialization: - @pytest.mark.parametrize( - "credentials, client_options", - [ - (None, None), - (mock.MagicMock(spec=auth_credentials.Credentials), None), - ( - None, - client_options_lib.ClientOptions(api_endpoint="test.googleapis.com"), - ), - ( - mock.MagicMock(spec=auth_credentials.Credentials), - client_options_lib.ClientOptions(api_endpoint="test.googleapis.com"), - ), - ], - ) - def test_client_initialization_arguments( - self, - credentials, - client_options, - mock_dataset_service_client, - mock_job_service_client, - mock_model_service_client, - ): - # Act - client = centralized_service.BigQueryClient( - credentials=credentials, client_options=client_options - ) - - # Assert - # The BigQueryClient should have been initialized. Accessing the - # service client properties should instantiate them with the correct arguments. - - # Access the property to trigger instantiation - _ = client.dataset_service_client - mock_dataset_service_client.assert_called_once_with( - credentials=credentials, client_options=client_options - ) - - _ = client.job_service_client - mock_job_service_client.assert_called_once_with( - credentials=credentials, client_options=client_options - ) - - _ = client.model_service_client - mock_model_service_client.assert_called_once_with( - credentials=credentials, client_options=client_options - ) - - -class TestCentralizedClientDatasetService: - def test_get_dataset(self, bq_client, mock_dataset_service_client): - # Arrange - expected_dataset = dataset.Dataset( - kind="bigquery#dataset", id=f"{PROJECT_ID}:{DATASET_ID}" - ) - mock_dataset_service_client.get_dataset.return_value = expected_dataset - get_dataset_request = dataset.GetDatasetRequest( - project_id=PROJECT_ID, dataset_id=DATASET_ID - ) - - # Act - dataset_response = bq_client.get_dataset(request=get_dataset_request) - - # Assert - assert dataset_response == expected_dataset - assert_client_called_once_with( - mock_dataset_service_client.get_dataset, get_dataset_request - ) - - -class TestCentralizedClientJobService: - def test_list_jobs(self, bq_client, mock_job_service_client): - # Arrange - expected_jobs = [job.Job(kind="bigquery#job", id=f"{PROJECT_ID}:{JOB_ID}")] - mock_job_service_client.list_jobs.return_value = expected_jobs - list_jobs_request = job.ListJobsRequest(project_id=PROJECT_ID) - - # Act - jobs_response = bq_client.list_jobs(request=list_jobs_request) - - # Assert - assert jobs_response == expected_jobs - assert_client_called_once_with( - mock_job_service_client.list_jobs, list_jobs_request - ) - - -class TestCentralizedClientModelService: - def test_get_model(self, bq_client, mock_model_service_client): - # Arrange - expected_model = model.Model( - etag=DEFAULT_ETAG, - model_reference={ - "project_id": PROJECT_ID, - "dataset_id": DATASET_ID, - "model_id": MODEL_ID, - }, - ) - mock_model_service_client.get_model.return_value = expected_model - get_model_request = model.GetModelRequest( - project_id=PROJECT_ID, dataset_id=DATASET_ID, model_id=MODEL_ID - ) - - # Act - model_response = bq_client.get_model(request=get_model_request) - - # Assert - assert model_response == expected_model - assert_client_called_once_with( - mock_model_service_client.get_model, get_model_request - ) - - def test_delete_model(self, bq_client, mock_model_service_client): - # Arrange - # The underlying service call returns nothing on success. - mock_model_service_client.delete_model.return_value = None - delete_model_request = model.DeleteModelRequest( - project_id=PROJECT_ID, dataset_id=DATASET_ID, model_id=MODEL_ID - ) - - # Act - # The wrapper method should also return nothing. - result = bq_client.delete_model(request=delete_model_request) - - # Assert - # 1. Assert the return value is None. This fails if the method doesn't exist. - assert result is None - # 2. Assert the underlying service was called correctly. - assert_client_called_once_with( - mock_model_service_client.delete_model, - delete_model_request, - ) - - def test_patch_model(self, bq_client, mock_model_service_client): - # Arrange - expected_model = model.Model( - etag="new_etag", - model_reference={ - "project_id": PROJECT_ID, - "dataset_id": DATASET_ID, - "model_id": MODEL_ID, - }, - description="A newly patched description.", - ) - mock_model_service_client.patch_model.return_value = expected_model - - model_patch = model.Model(description="A newly patched description.") - patch_model_request = model.PatchModelRequest( - project_id=PROJECT_ID, - dataset_id=DATASET_ID, - model_id=MODEL_ID, - model=model_patch, - ) - - # Act - patched_model = bq_client.patch_model(request=patch_model_request) - - # Assert - assert patched_model == expected_model - assert_client_called_once_with( - mock_model_service_client.patch_model, patch_model_request - ) - - def test_list_models(self, bq_client, mock_model_service_client): - # Arrange - expected_models = [ - model.Model( - etag=DEFAULT_ETAG, - model_reference={ - "project_id": PROJECT_ID, - "dataset_id": DATASET_ID, - "model_id": MODEL_ID, - }, - ) - ] - mock_model_service_client.list_models.return_value = expected_models - list_models_request = model.ListModelsRequest( - project_id=PROJECT_ID, dataset_id=DATASET_ID - ) - # Act - models_response = bq_client.list_models(request=list_models_request) - - # Assert - assert models_response == expected_models - assert_client_called_once_with( - mock_model_service_client.list_models, list_models_request - ) From 5b4d538a1053c5381c9853e9337a58f1ecc69e56 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Thu, 11 Sep 2025 12:10:11 -0400 Subject: [PATCH 02/25] removes old __init__.py --- google/cloud/bigquery_v2/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/google/cloud/bigquery_v2/__init__.py b/google/cloud/bigquery_v2/__init__.py index 30b6a63d6..83c82e729 100644 --- a/google/cloud/bigquery_v2/__init__.py +++ b/google/cloud/bigquery_v2/__init__.py @@ -24,7 +24,6 @@ from .services.routine_service import RoutineServiceClient from .services.row_access_policy_service import RowAccessPolicyServiceClient from .services.table_service import TableServiceClient -from .services.centralized_service import BigQueryClient from .types.biglake_config import BigLakeConfiguration from .types.clustering import Clustering @@ -215,7 +214,6 @@ "BiEngineReason", "BiEngineStatistics", "BigLakeConfiguration", - "BigQueryClient", "BigtableColumn", "BigtableColumnFamily", "BigtableOptions", From 132c571224b93f0220088ad3f2d641490775c694 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Thu, 11 Sep 2025 12:28:50 -0400 Subject: [PATCH 03/25] Adds two utility files to handle basic tasks --- scripts/microgenerator/name_utils.py | 73 ++++++++++++++++ scripts/microgenerator/utils.py | 120 +++++++++++++++++++++++++++ 2 files changed, 193 insertions(+) create mode 100644 scripts/microgenerator/name_utils.py create mode 100644 scripts/microgenerator/utils.py diff --git a/scripts/microgenerator/name_utils.py b/scripts/microgenerator/name_utils.py new file mode 100644 index 000000000..129050c37 --- /dev/null +++ b/scripts/microgenerator/name_utils.py @@ -0,0 +1,73 @@ +# -*- coding: utf-8 -*- +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A utility module for handling name transformations.""" + +import re +from typing import Dict + + +def to_snake_case(name: str) -> str: + """Converts a PascalCase name to snake_case.""" + return re.sub(r"(? Dict[str, str]: + """ + Generates various name formats for a service based on its client class name. + + Args: + class_name: The PascalCase name of the service client class + (e.g., 'DatasetServiceClient'). + + Returns: + A dictionary containing different name variations. + """ + snake_case_name = to_snake_case(class_name) + module_name = snake_case_name.replace("_client", "") + service_name = module_name.replace("_service", "") + + return { + "service_name": service_name, + "service_module_name": module_name, + "service_client_class": class_name, + "property_name": snake_case_name, # Direct use of snake_case_name + } + + +def method_to_request_class_name(method_name: str) -> str: + """ + Converts a snake_case method name to a PascalCase Request class name. + + This follows the convention where a method like `get_dataset` corresponds + to a `GetDatasetRequest` class. + + Args: + method_name: The snake_case name of the API method. + + Returns: + The inferred PascalCase name for the corresponding request class. + + Example: + >>> method_to_request_class_name('get_dataset') + 'GetDatasetRequest' + >>> method_to_request_class_name('list_jobs') + 'ListJobsRequest' + """ + # e.g., "get_dataset" -> ["get", "dataset"] + parts = method_name.split("_") + # e.g., ["get", "dataset"] -> "GetDataset" + pascal_case_base = "".join(part.capitalize() for part in parts) + return f"{pascal_case_base}Request" diff --git a/scripts/microgenerator/utils.py b/scripts/microgenerator/utils.py new file mode 100644 index 000000000..c81387d57 --- /dev/null +++ b/scripts/microgenerator/utils.py @@ -0,0 +1,120 @@ +# -*- coding: utf-8 -*- +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Utility functions for the microgenerator.""" + +import os +import sys +import yaml +import jinja2 +from typing import Dict, Any, Iterator, Callable + + +def _load_resource( + loader_func: Callable, + path: str, + not_found_exc: type, + parse_exc: type, + resource_type_name: str, +) -> Any: + """ + Generic resource loader with common error handling. + + Args: + loader_func: A callable that performs the loading and returns the resource. + It should raise appropriate exceptions on failure. + path: The path/name of the resource for use in error messages. + not_found_exc: The exception type to catch for a missing resource. + parse_exc: The exception type to catch for a malformed resource. + resource_type_name: A human-readable name for the resource type. + """ + try: + return loader_func() + except not_found_exc: + print(f"Error: {resource_type_name} '{path}' not found.", file=sys.stderr) + sys.exit(1) + except parse_exc as e: + print( + f"Error: Could not load {resource_type_name.lower()} from '{path}': {e}", + file=sys.stderr, + ) + sys.exit(1) + + +def load_template(template_path: str) -> jinja2.Template: + """ + Loads a Jinja2 template from a given file path. + """ + template_dir = os.path.dirname(template_path) + template_name = os.path.basename(template_path) + + def _loader() -> jinja2.Template: + env = jinja2.Environment( + loader=jinja2.FileSystemLoader(template_dir or "."), + trim_blocks=True, + lstrip_blocks=True, + ) + return env.get_template(template_name) + + return _load_resource( + loader_func=_loader, + path=template_path, + not_found_exc=jinja2.exceptions.TemplateNotFound, + parse_exc=jinja2.exceptions.TemplateError, + resource_type_name="Template file", + ) + + +def load_config(config_path: str) -> Dict[str, Any]: + """Loads the generator's configuration from a YAML file.""" + + def _loader() -> Dict[str, Any]: + with open(config_path, "r", encoding="utf-8") as f: + return yaml.safe_load(f) + + return _load_resource( + loader_func=_loader, + path=config_path, + not_found_exc=FileNotFoundError, + parse_exc=yaml.YAMLError, + resource_type_name="Configuration file", + ) + + +def walk_codebase(path: str) -> Iterator[str]: + """Yields all .py file paths in a directory.""" + for root, _, files in os.walk(path): + for file in files: + if file.endswith(".py"): + yield os.path.join(root, file) + + +def write_code_to_file(output_path: str, content: str): + """Ensures the output directory exists and writes content to the file.""" + output_dir = os.path.dirname(output_path) + + # An empty output_dir means the file is in the current directory. + if output_dir: + print(f" Ensuring output directory exists: {os.path.abspath(output_dir)}") + os.makedirs(output_dir, exist_ok=True) + if not os.path.isdir(output_dir): + print(f" Error: Output directory was not created.", file=sys.stderr) + sys.exit(1) + + print(f" Writing generated code to: {os.path.abspath(output_path)}") + with open(output_path, "w", encoding="utf-8") as f: + f.write(content) + print(f"Successfully generated {output_path}") From 90b224eef468a856c49edff308d65f6c2cd5a997 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Thu, 11 Sep 2025 12:54:36 -0400 Subject: [PATCH 04/25] Adds a configuration file for the microgenerator --- scripts/microgenerator/config.yaml | 75 ++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 scripts/microgenerator/config.yaml diff --git a/scripts/microgenerator/config.yaml b/scripts/microgenerator/config.yaml new file mode 100644 index 000000000..a9c608f29 --- /dev/null +++ b/scripts/microgenerator/config.yaml @@ -0,0 +1,75 @@ +# config.yaml + +# The name of the service, used for variable names and comments. +service_name: "bigquery" + +# A list of paths to the source code files to be parsed. +# Globs are supported. +source_files: + services: + - "google/cloud/bigquery_v2/services/dataset_service/client.py" + - "google/cloud/bigquery_v2/services/job_service/client.py" + - "google/cloud/bigquery_v2/services/model_service/client.py" + - "google/cloud/bigquery_v2/services/project_service/client.py" + - "google/cloud/bigquery_v2/services/routine_service/client.py" + - "google/cloud/bigquery_v2/services/row_access_policy_service/client.py" + - "google/cloud/bigquery_v2/services/table_service/client.py" + types: + - "google/cloud/bigquery_v2/types/dataset.py" + - "google/cloud/bigquery_v2/types/job.py" + - "google/cloud/bigquery_v2/types/model.py" + - "google/cloud/bigquery_v2/types/project.py" + - "google/cloud/bigquery_v2/types/routine.py" + - "google/cloud/bigquery_v2/types/row_access_policy.py" + - "google/cloud/bigquery_v2/types/table.py" + + +# Filtering rules for classes and methods. +filter: + classes: + # Only include classes with these suffixes. + include_suffixes: + - "ServiceClient" + - "Request" + # Exclude classes with these suffixes. + exclude_suffixes: + - "BigQueryClient" + methods: + # Include methods with these prefixes. + include_prefixes: + - "batch_delete_" + - "cancel_" + - "create_" + - "delete_" + - "get_" + - "insert_" + - "list_" + - "patch_" + - "undelete_" + - "update_" + # Exclude methods with these prefixes. + exclude_prefixes: + - "get_mtls_endpoint_and_cert_source" + overrides: + patch_table: + request_class_name: "UpdateOrPatchTableRequest" + patch_dataset: + request_class_name: "UpdateOrPatchDatasetRequest" + +# A list of templates to render and their corresponding output files. +# A list of templates to render and their corresponding output files. +templates: + - template: "templates/client.py.j2" + output: "google/cloud/bigquery_v2/services/centralized_service/client.py" + - template: "templates/_helpers.py.j2" + output: "google/cloud/bigquery_v2/services/centralized_service/_helpers.py" + - template: "templates/__init__.py.j2" + output: "google/cloud/bigquery_v2/services/centralized_service/__init__.py" + +post_processing_templates: + - template: "templates/post-processing/init.py.j2" + target_file: "google/cloud/bigquery_v2/__init__.py" + add_imports: + - "from .services.centralized_service import BigQueryClient" + add_to_all: + - "BigQueryClient" \ No newline at end of file From e071eabdc8c33c680755c5463fc2d5e9ee78adf6 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Thu, 11 Sep 2025 12:58:05 -0400 Subject: [PATCH 05/25] Removes unused comment --- scripts/microgenerator/config.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/microgenerator/config.yaml b/scripts/microgenerator/config.yaml index a9c608f29..57330af31 100644 --- a/scripts/microgenerator/config.yaml +++ b/scripts/microgenerator/config.yaml @@ -56,7 +56,6 @@ filter: patch_dataset: request_class_name: "UpdateOrPatchDatasetRequest" -# A list of templates to render and their corresponding output files. # A list of templates to render and their corresponding output files. templates: - template: "templates/client.py.j2" From dc72a9858a2573e57503012967c0cfdad0cdb101 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Thu, 11 Sep 2025 14:47:05 -0400 Subject: [PATCH 06/25] chore: adds noxfile.py for the microgenerator --- scripts/microgenerator/noxfile.py | 382 ++++++++++++++++++++++++++++++ 1 file changed, 382 insertions(+) create mode 100644 scripts/microgenerator/noxfile.py diff --git a/scripts/microgenerator/noxfile.py b/scripts/microgenerator/noxfile.py new file mode 100644 index 000000000..be3870ba4 --- /dev/null +++ b/scripts/microgenerator/noxfile.py @@ -0,0 +1,382 @@ +# Copyright 2016 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import + +from functools import wraps +import pathlib +import os +import shutil +import nox +import time + + +MYPY_VERSION = "mypy==1.6.1" +PYTYPE_VERSION = "pytype==2024.9.13" +BLACK_VERSION = "black==23.7.0" +BLACK_PATHS = (".",) + +DEFAULT_PYTHON_VERSION = "3.9" +SYSTEM_TEST_PYTHON_VERSIONS = ["3.9", "3.11", "3.12", "3.13"] +UNIT_TEST_PYTHON_VERSIONS = ["3.9", "3.11", "3.12", "3.13"] +CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute() + + +def _calculate_duration(func): + """This decorator prints the execution time for the decorated function.""" + + @wraps(func) + def wrapper(*args, **kwargs): + start = time.monotonic() + result = func(*args, **kwargs) + end = time.monotonic() + total_seconds = round(end - start) + hours = total_seconds // 3600 # Integer division to get hours + remaining_seconds = total_seconds % 3600 # Modulo to find remaining seconds + minutes = remaining_seconds // 60 + seconds = remaining_seconds % 60 + human_time = f"{hours:}:{minutes:0>2}:{seconds:0>2}" + print(f"Session ran in {total_seconds} seconds ({human_time})") + return result + + return wrapper + + +# 'docfx' is excluded since it only needs to run in 'docs-presubmit' +nox.options.sessions = [ + "unit", + "system", + "cover", + "lint", + "lint_setup_py", + "blacken", + "mypy", + "pytype", + "docs", +] + + +def default(session, install_extras=True): + """Default unit test session. + + This is intended to be run **without** an interpreter set, so + that the current ``python`` (on the ``PATH``) or the version of + Python corresponding to the ``nox`` binary the ``PATH`` can + run the tests. + """ + + constraints_path = str( + CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" + ) + + # Install all test dependencies, then install local packages in-place. + session.install( + "pytest", + "google-cloud-testutils", + "pytest-cov", + "pytest-xdist", + "freezegun", + "-c", + constraints_path, + ) + # We have logic in the magics.py file that checks for whether 'bigquery_magics' + # is imported OR not. If yes, we use a context object from that library. + # If no, we use our own context object from magics.py. In order to exercise + # that logic (and the associated tests) we avoid installing the [ipython] extra + # which has a downstream effect of then avoiding installing bigquery_magics. + if install_extras and session.python == UNIT_TEST_PYTHON_VERSIONS[0]: + install_target = ".[bqstorage,pandas,ipywidgets,geopandas,matplotlib,tqdm,opentelemetry,bigquery_v2]" + elif install_extras: # run against all other UNIT_TEST_PYTHON_VERSIONS + install_target = ".[all]" + else: + install_target = "." + session.install("-e", install_target, "-c", constraints_path) + + # Test with some broken "extras" in case the user didn't install the extra + # directly. For example, pandas-gbq is recommended for pandas features, but + # we want to test that we fallback to the previous behavior. For context, + # see internal document go/pandas-gbq-and-bigframes-redundancy. + if session.python == UNIT_TEST_PYTHON_VERSIONS[0]: + session.run("python", "-m", "pip", "uninstall", "pandas-gbq", "-y") + + session.run("python", "-m", "pip", "freeze") + + # Run py.test against the unit tests. + session.run( + "py.test", + "-n=8", + "--quiet", + "-W default::PendingDeprecationWarning", + "--cov=google/cloud/bigquery", + "--cov=tests/unit", + "--cov-append", + "--cov-config=.coveragerc", + "--cov-report=", + "--cov-fail-under=0", + os.path.join("tests", "unit"), + *session.posargs, + ) + + +@nox.session(python=UNIT_TEST_PYTHON_VERSIONS) +@_calculate_duration +def unit(session): + """Run the unit test suite.""" + + default(session) + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +@_calculate_duration +def mypy(session): + """Run type checks with mypy.""" + + session.install("-e", ".[all]") + session.install(MYPY_VERSION) + + # Just install the dependencies' type info directly, since "mypy --install-types" + # might require an additional pass. + session.install( + "types-protobuf", + "types-python-dateutil", + "types-requests", + "types-setuptools", + ) + session.run("python", "-m", "pip", "freeze") + session.run("mypy", "-p", "google", "--show-traceback") + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +@_calculate_duration +def pytype(session): + """Run type checks with pytype.""" + # An indirect dependecy attrs==21.1.0 breaks the check, and installing a less + # recent version avoids the error until a possibly better fix is found. + # https://github.com/googleapis/python-bigquery/issues/655 + + session.install("attrs==20.3.0") + session.install("-e", ".[all]") + session.install(PYTYPE_VERSION) + session.run("python", "-m", "pip", "freeze") + # See https://github.com/google/pytype/issues/464 + session.run("pytype", "-P", ".", "google/cloud/bigquery") + + +@nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS) +@_calculate_duration +def system(session): + """Run the system test suite.""" + + constraints_path = str( + CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" + ) + + # Sanity check: Only run system tests if the environment variable is set. + if not os.environ.get("GOOGLE_APPLICATION_CREDENTIALS", ""): + session.skip("Credentials must be set via environment variable.") + + # Use pre-release gRPC for system tests. + # Exclude version 1.49.0rc1 which has a known issue. + # See https://github.com/grpc/grpc/pull/30642 + session.install("--pre", "grpcio!=1.49.0rc1", "-c", constraints_path) + + # Install all test dependencies, then install local packages in place. + session.install( + "pytest", + "psutil", + "pytest-xdist", + "google-cloud-testutils", + "-c", + constraints_path, + ) + if os.environ.get("GOOGLE_API_USE_CLIENT_CERTIFICATE", "") == "true": + # mTLS test requires pyopenssl and latest google-cloud-storage + session.install("google-cloud-storage", "pyopenssl") + else: + session.install("google-cloud-storage", "-c", constraints_path) + + # Data Catalog needed for the column ACL test with a real Policy Tag. + session.install("google-cloud-datacatalog", "-c", constraints_path) + + # Resource Manager needed for test with a real Resource Tag. + session.install("google-cloud-resource-manager", "-c", constraints_path) + + if session.python in ["3.11", "3.12"]: + extras = "[bqstorage,ipywidgets,pandas,tqdm,opentelemetry]" + else: + extras = "[all]" + session.install("-e", f".{extras}", "-c", constraints_path) + + # Test with some broken "extras" in case the user didn't install the extra + # directly. For example, pandas-gbq is recommended for pandas features, but + # we want to test that we fallback to the previous behavior. For context, + # see internal document go/pandas-gbq-and-bigframes-redundancy. + if session.python == SYSTEM_TEST_PYTHON_VERSIONS[0]: + session.run("python", "-m", "pip", "uninstall", "pandas-gbq", "-y") + + # print versions of all dependencies + session.run("python", "-m", "pip", "freeze") + + # Run py.test against the system tests. + session.run( + "py.test", + "-n=auto", + "--quiet", + "-W default::PendingDeprecationWarning", + os.path.join("tests", "system"), + *session.posargs, + ) + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +@_calculate_duration +def cover(session): + """Run the final coverage report. + + This outputs the coverage report aggregating coverage from the unit + test runs (not system test runs), and then erases coverage data. + """ + + session.install("coverage", "pytest-cov") + session.run("python", "-m", "pip", "freeze") + session.run("coverage", "report", "--show-missing", "--fail-under=100") + session.run("coverage", "erase") + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +@_calculate_duration +def lint(session): + """Run linters. + + Returns a failure if the linters find linting errors or sufficiently + serious code quality issues. + """ + + session.install("flake8", BLACK_VERSION) + session.install("-e", ".") + session.run("python", "-m", "pip", "freeze") + session.run("flake8", os.path.join("google", "cloud", "bigquery")) + session.run("flake8", "tests") + session.run("flake8", os.path.join("docs", "samples")) + session.run("flake8", os.path.join("docs", "snippets.py")) + session.run("flake8", "benchmark") + session.run("black", "--check", *BLACK_PATHS) + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +@_calculate_duration +def lint_setup_py(session): + """Verify that setup.py is valid (including RST check).""" + + session.install("docutils", "Pygments") + session.run("python", "-m", "pip", "freeze") + session.run("python", "setup.py", "check", "--restructuredtext", "--strict") + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +@_calculate_duration +def blacken(session): + """Run black. + Format code to uniform standard. + """ + + session.install(BLACK_VERSION) + session.run("python", "-m", "pip", "freeze") + session.run("black", *BLACK_PATHS) + + +@nox.session(python="3.10") +@_calculate_duration +def docs(session): + """Build the docs.""" + + session.install( + # We need to pin to specific versions of the `sphinxcontrib-*` packages + # which still support sphinx 4.x. + # See https://github.com/googleapis/sphinx-docfx-yaml/issues/344 + # and https://github.com/googleapis/sphinx-docfx-yaml/issues/345. + "sphinxcontrib-applehelp==1.0.4", + "sphinxcontrib-devhelp==1.0.2", + "sphinxcontrib-htmlhelp==2.0.1", + "sphinxcontrib-qthelp==1.0.3", + "sphinxcontrib-serializinghtml==1.1.5", + "sphinx==4.5.0", + "alabaster", + "recommonmark", + ) + session.install("google-cloud-storage") + session.install("-e", ".[all]") + + shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) + session.run("python", "-m", "pip", "freeze") + session.run( + "sphinx-build", + "-W", # warnings as errors + "-T", # show full traceback on exception + "-N", # no colors + "-b", + "html", + "-d", + os.path.join("docs", "_build", "doctrees", ""), + os.path.join("docs", ""), + os.path.join("docs", "_build", "html", ""), + ) + + +@nox.session(python="3.10") +@_calculate_duration +def docfx(session): + """Build the docfx yaml files for this library.""" + + session.install("-e", ".") + session.install( + # We need to pin to specific versions of the `sphinxcontrib-*` packages + # which still support sphinx 4.x. + # See https://github.com/googleapis/sphinx-docfx-yaml/issues/344 + # and https://github.com/googleapis/sphinx-docfx-yaml/issues/345. + "sphinxcontrib-applehelp==1.0.4", + "sphinxcontrib-devhelp==1.0.2", + "sphinxcontrib-htmlhelp==2.0.1", + "sphinxcontrib-qthelp==1.0.3", + "sphinxcontrib-serializinghtml==1.1.5", + "gcp-sphinx-docfx-yaml", + "alabaster", + "recommonmark", + ) + + shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) + session.run("python", "-m", "pip", "freeze") + session.run( + "sphinx-build", + "-T", # show full traceback on exception + "-N", # no colors + "-D", + ( + "extensions=sphinx.ext.autodoc," + "sphinx.ext.autosummary," + "docfx_yaml.extension," + "sphinx.ext.intersphinx," + "sphinx.ext.coverage," + "sphinx.ext.napoleon," + "sphinx.ext.todo," + "sphinx.ext.viewcode," + "recommonmark" + ), + "-b", + "html", + "-d", + os.path.join("docs", "_build", "doctrees", ""), + os.path.join("docs", ""), + os.path.join("docs", "_build", "html", ""), + ) From 7318f0b2d20ae7afd257af252b4f171230310a6d Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Fri, 12 Sep 2025 05:19:36 -0400 Subject: [PATCH 07/25] feat: microgen - adds two init file templates --- .../microgenerator/templates/__init__.py.j2 | 19 ++++++++++++ .../templates/post-processing/init.py.j2 | 29 +++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 scripts/microgenerator/templates/__init__.py.j2 create mode 100644 scripts/microgenerator/templates/post-processing/init.py.j2 diff --git a/scripts/microgenerator/templates/__init__.py.j2 b/scripts/microgenerator/templates/__init__.py.j2 new file mode 100644 index 000000000..6eebab277 --- /dev/null +++ b/scripts/microgenerator/templates/__init__.py.j2 @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from .client import BigQueryClient + +__all__ = ("BigQueryClient",) diff --git a/scripts/microgenerator/templates/post-processing/init.py.j2 b/scripts/microgenerator/templates/post-processing/init.py.j2 new file mode 100644 index 000000000..09b992865 --- /dev/null +++ b/scripts/microgenerator/templates/post-processing/init.py.j2 @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from google.cloud.bigquery_v2 import gapic_version as package_version + +__version__ = package_version.__version__ + +{% for import in imports %} +{{ import }} +{%- endfor %} + +__all__ = ( +{%- for item in all_list %} + "{{ item }}", +{%- endfor %} +) From 07910c5c759c3cb96fbd006205240d13a065b38d Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Fri, 12 Sep 2025 05:34:00 -0400 Subject: [PATCH 08/25] feat: adds _helpers.py.js template --- .../microgenerator/templates/_helpers.py.j2 | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 scripts/microgenerator/templates/_helpers.py.j2 diff --git a/scripts/microgenerator/templates/_helpers.py.j2 b/scripts/microgenerator/templates/_helpers.py.j2 new file mode 100644 index 000000000..4798a4272 --- /dev/null +++ b/scripts/microgenerator/templates/_helpers.py.j2 @@ -0,0 +1,54 @@ +from typing import Any, Dict, List, Optional, Type + + +def _create_request( + request_class: Type, + path_identifier: str, + expected_args: List[str], + default_project_id: Optional[str] = None, +) -> Any: + """ + Constructs a *Request object from a class, path_identifier, and expected args. + + Args: + request_class: The class of the request object to create (e.g., GetDatasetRequest). + path_identifier: The dot-separated string of resource IDs. + expected_args: An ordered list of the argument names the request object + expects (e.g., ['project_id', 'dataset_id', 'table_id']). + default_project_id: The default project ID to use if needed. + + Returns: + An instantiated request object. + """ + # Start of inlined parse_path_to_request_inputs + segments = path_identifier.split(".") + num_segments = len(segments) + num_expected = len(expected_args) + project_id_is_expected = "project_id" in expected_args + + # Validate the number of segments. + if not ( + num_segments == num_expected + or (project_id_is_expected and num_segments == num_expected - 1) + ): + raise ValueError( + f"Invalid path identifier '{path_identifier}'. Expected " + f"{num_expected} parts (or {num_expected - 1} if project_id is " + f"omitted), but got {num_segments}." + ) + + # If project_id is implicitly expected, use the default. + if project_id_is_expected and num_segments == num_expected - 1: + if not default_project_id: + raise ValueError( + f"Missing project_id in path '{path_identifier}' and no " + "default_project_id was provided." + ) + # Prepend the default project_id to the segments. + segments.insert(0, default_project_id) + + request_inputs = dict(zip(expected_args, segments)) + # End of inlined parse_path_to_request_inputs + + # Instantiate the request object. + return request_class(**request_inputs) From dc54c9991dde11ec192e5a6ebc7c35bb605c080b Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Fri, 12 Sep 2025 06:12:24 -0400 Subject: [PATCH 09/25] Updates with two usage examples --- .../microgenerator/templates/_helpers.py.j2 | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/scripts/microgenerator/templates/_helpers.py.j2 b/scripts/microgenerator/templates/_helpers.py.j2 index 4798a4272..49efbf842 100644 --- a/scripts/microgenerator/templates/_helpers.py.j2 +++ b/scripts/microgenerator/templates/_helpers.py.j2 @@ -19,6 +19,31 @@ def _create_request( Returns: An instantiated request object. + + Examples: + >>> # Example with project_id provided in path_identifier + >>> request = _create_request( + ... request_class=GetDatasetRequest, + ... path_identifier="my-project.my-dataset", + ... expected_args=["project_id", "dataset_id"] + ... ) + >>> request.project_id + 'my-project' + >>> request.dataset_id + 'my-dataset' + + >>> # Example with project_id omitted from path_identifier, using default_project_id + >>> request = _create_request( + ... request_class=GetDatasetRequest, + ... path_identifier="my-dataset", + ... expected_args=["project_id", "dataset_id"], + ... default_project_id="my-default-project" + ... ) + >>> request.project_id + 'my-default-project' + >>> request.dataset_id + 'my-dataset' + """ # Start of inlined parse_path_to_request_inputs segments = path_identifier.split(".") From 28de5f81980483173df799f0f4b114f1b110827a Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Fri, 12 Sep 2025 06:44:20 -0400 Subject: [PATCH 10/25] feat: adds two partial templates for creating method signatures --- .../partials/_method_with_request_builder.j2 | 43 +++++++++++++++++++ .../partials/_simple_passthrough_method.j2 | 18 ++++++++ 2 files changed, 61 insertions(+) create mode 100644 scripts/microgenerator/templates/partials/_method_with_request_builder.j2 create mode 100644 scripts/microgenerator/templates/partials/_simple_passthrough_method.j2 diff --git a/scripts/microgenerator/templates/partials/_method_with_request_builder.j2 b/scripts/microgenerator/templates/partials/_method_with_request_builder.j2 new file mode 100644 index 000000000..887aceff5 --- /dev/null +++ b/scripts/microgenerator/templates/partials/_method_with_request_builder.j2 @@ -0,0 +1,43 @@ +def {{ method.name }}( + self, + {{ method.request_id_args[-1] }}: Optional[str] = None, + *, + request: Optional[{{ '.'.join(method.request_class_full_name.split('.')[-2:]) }}] = None, + retry: OptionalRetry = DEFAULT_RETRY, + timeout: Union[float, object] = DEFAULT_TIMEOUT, + metadata: Sequence[Tuple[str, Union[str, bytes]]] = DEFAULT_METADATA, +) -> "{{ method.return_type }}": + """ + TODO: Docstring is purposefully blank. microgenerator will add automatically. + """ + if request and {{ method.request_id_args[-1] }} is not None: + raise ValueError("Cannot provide both request and {{ method.request_id_args[-1] }}.") + + if request: + final_request = request + else: + path_identifier = {{ method.request_id_args[-1] }} + if path_identifier is None: + if len({{ method.request_id_args }}) == 1: + path_identifier = self.project + else: + raise ValueError("Either request or {{ method.request_id_args[-1] }} must be provided.") + + if path_identifier is None: + raise ValueError("Could not determine a path identifier.") + + request_class = {{ '.'.join(method.request_class_full_name.split('.')[-2:]) }} + + final_request = _helpers._create_request( + request_class=request_class, + path_identifier=path_identifier, + expected_args={{ method.request_id_args }}, + default_project_id=self.project, + ) + + return self.{{ class_to_instance_map[method.class_name] }}.{{ method.name }}( + request=final_request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) \ No newline at end of file diff --git a/scripts/microgenerator/templates/partials/_simple_passthrough_method.j2 b/scripts/microgenerator/templates/partials/_simple_passthrough_method.j2 new file mode 100644 index 000000000..39d28b7a3 --- /dev/null +++ b/scripts/microgenerator/templates/partials/_simple_passthrough_method.j2 @@ -0,0 +1,18 @@ +def {{ method.name }}( + self, + *, + request: Optional[dict] = None, + retry: OptionalRetry = DEFAULT_RETRY, + timeout: Union[float, object] = DEFAULT_TIMEOUT, + metadata: Sequence[Tuple[str, Union[str, bytes]]] = DEFAULT_METADATA, +) -> "{{ method.return_type }}": + """ + TODO: Docstring is purposefully blank. microgenerator will add automatically. + """ + + return self.{{ class_to_instance_map[method.class_name] }}.{{ method.name }}( + request=request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) \ No newline at end of file From c4577545cdccfd20cc9f7e19d370d39eba0b3a46 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Mon, 15 Sep 2025 09:04:58 -0400 Subject: [PATCH 11/25] feat: Add microgenerator __init__.py Migrates the empty __init__.py file to the microgenerator package. --- scripts/microgenerator/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 scripts/microgenerator/__init__.py diff --git a/scripts/microgenerator/__init__.py b/scripts/microgenerator/__init__.py new file mode 100644 index 000000000..e69de29bb From 595e59f48cb8d1823c6676bacb590b0aaf5d89b3 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Mon, 15 Sep 2025 09:30:57 -0400 Subject: [PATCH 12/25] feat: Add AST analysis utilities Introduces the CodeAnalyzer class and helper functions for parsing Python code using the ast module. This provides the foundation for understanding service client structures. --- scripts/microgenerator/generate.py | 354 +++++++++++++++++++++++++++++ 1 file changed, 354 insertions(+) create mode 100644 scripts/microgenerator/generate.py diff --git a/scripts/microgenerator/generate.py b/scripts/microgenerator/generate.py new file mode 100644 index 000000000..7aa3d9668 --- /dev/null +++ b/scripts/microgenerator/generate.py @@ -0,0 +1,354 @@ +# -*- coding: utf-8 -*- +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" +A dual-purpose module for Python code analysis and BigQuery client generation. + +When run as a script, it generates the BigQueryClient source code. +When imported, it provides utility functions for parsing and exploring +any Python codebase using the `ast` module. +""" + +import ast +import os +from collections import defaultdict +from typing import List, Dict, Any, Iterator + +from . import utils + +# ============================================================================= +# Section 1: Generic AST Analysis Utilities +# ============================================================================= + + +class CodeAnalyzer(ast.NodeVisitor): + """ + A node visitor to traverse an AST and extract structured information + about classes, methods, and their arguments. + """ + + def __init__(self): + self.structure: List[Dict[str, Any]] = [] + self.imports: set[str] = set() + self.types: set[str] = set() + self._current_class_info: Dict[str, Any] | None = None + self._is_in_method: bool = False + + def _get_type_str(self, node: ast.AST | None) -> str | None: + """Recursively reconstructs a type annotation string from an AST node.""" + if node is None: + return None + # Handles simple names like 'str', 'int', 'HttpRequest' + if isinstance(node, ast.Name): + return node.id + # Handles dotted names like 'service.GetDatasetRequest' + if isinstance(node, ast.Attribute): + # Attempt to reconstruct the full dotted path + parts = [] + curr = node + while isinstance(curr, ast.Attribute): + parts.append(curr.attr) + curr = curr.value + if isinstance(curr, ast.Name): + parts.append(curr.id) + return ".".join(reversed(parts)) + # Handles subscripted types like 'list[str]', 'Optional[...]' + if isinstance(node, ast.Subscript): + value_str = self._get_type_str(node.value) + slice_str = self._get_type_str(node.slice) + return f"{value_str}[{slice_str}]" + # Handles tuples inside subscripts, e.g., 'dict[str, int]' + if isinstance(node, ast.Tuple): + return ", ".join( + [s for s in (self._get_type_str(e) for e in node.elts) if s] + ) + # Handles forward references as strings, e.g., '"Dataset"' + if isinstance(node, ast.Constant): + return repr(node.value) + return None # Fallback for unhandled types + + def _collect_types_from_node(self, node: ast.AST | None) -> None: + """Recursively traverses an annotation node to find and collect all type names.""" + if node is None: + return + + if isinstance(node, ast.Name): + self.types.add(node.id) + elif isinstance(node, ast.Attribute): + type_str = self._get_type_str(node) + if type_str: + self.types.add(type_str) + elif isinstance(node, ast.Subscript): + self._collect_types_from_node(node.value) + self._collect_types_from_node(node.slice) + elif isinstance(node, (ast.Tuple, ast.List)): + for elt in node.elts: + self._collect_types_from_node(elt) + elif isinstance(node, ast.Constant) and isinstance(node.value, str): + self.types.add(node.value) + elif isinstance(node, ast.BinOp) and isinstance( + node.op, ast.BitOr + ): # For | union type + self._collect_types_from_node(node.left) + self._collect_types_from_node(node.right) + + def visit_Import(self, node: ast.Import) -> None: + """Catches 'import X' and 'import X as Y' statements.""" + for alias in node.names: + if alias.asname: + self.imports.add(f"import {alias.name} as {alias.asname}") + else: + self.imports.add(f"import {alias.name}") + self.generic_visit(node) + + def visit_ImportFrom(self, node: ast.ImportFrom) -> None: + """Catches 'from X import Y' statements.""" + module = node.module or "" + if not module: + module = "." * node.level + else: + module = "." * node.level + module + + names = [] + for alias in node.names: + if alias.asname: + names.append(f"{alias.name} as {alias.asname}") + else: + names.append(alias.name) + + if names: + self.imports.add(f"from {module} import {', '.join(names)}") + self.generic_visit(node) + + def visit_ClassDef(self, node: ast.ClassDef) -> None: + """Visits a class definition node.""" + class_info = { + "class_name": node.name, + "methods": [], + "attributes": [], + } + + # Extract class-level attributes (for proto.Message classes) + for item in node.body: + if isinstance(item, ast.AnnAssign) and isinstance(item.target, ast.Name): + attr_name = item.target.id + type_str = self._get_type_str(item.annotation) + class_info["attributes"].append({"name": attr_name, "type": type_str}) + + self.structure.append(class_info) + self._current_class_info = class_info + self.generic_visit(node) + self._current_class_info = None + + def visit_FunctionDef(self, node: ast.FunctionDef) -> None: + """Visits a function/method definition node.""" + if self._current_class_info: # This is a method + args_info = [] + + # Get default values + defaults = [self._get_type_str(d) for d in node.args.defaults] + num_defaults = len(defaults) + num_args = len(node.args.args) + + for i, arg in enumerate(node.args.args): + arg_data = {"name": arg.arg, "type": self._get_type_str(arg.annotation)} + + # Match defaults to arguments from the end + default_index = i - (num_args - num_defaults) + if default_index >= 0: + arg_data["default"] = defaults[default_index] + + args_info.append(arg_data) + self._collect_types_from_node(arg.annotation) + + # Collect return type + return_type = self._get_type_str(node.returns) + self._collect_types_from_node(node.returns) + + method_info = { + "method_name": node.name, + "args": args_info, + "return_type": return_type, + } + self._current_class_info["methods"].append(method_info) + + # Visit nodes inside the method to find instance attributes. + self._is_in_method = True + self.generic_visit(node) + self._is_in_method = False + + def _add_attribute(self, attr_name: str, attr_type: str | None = None): + """Adds a unique attribute to the current class context.""" + if self._current_class_info: + # Create a list of attribute names for easy lookup + attr_names = [ + attr.get("name") for attr in self._current_class_info["attributes"] + ] + if attr_name not in attr_names: + self._current_class_info["attributes"].append( + {"name": attr_name, "type": attr_type} + ) + + def visit_Assign(self, node: ast.Assign) -> None: + """Handles attribute assignments: `x = ...` and `self.x = ...`.""" + if self._current_class_info: + for target in node.targets: + # Instance attribute: self.x = ... + if ( + isinstance(target, ast.Attribute) + and isinstance(target.value, ast.Name) + and target.value.id == "self" + ): + self._add_attribute(target.attr) + # Class attribute: x = ... (only if not inside a method) + elif isinstance(target, ast.Name) and not self._is_in_method: + self._add_attribute(target.id) + self.generic_visit(node) + + def visit_AnnAssign(self, node: ast.AnnAssign) -> None: + """Handles annotated assignments: `x: int = ...` and `self.x: int = ...`.""" + if self._current_class_info: + target = node.target + # Instance attribute: self.x: int = ... + if ( + isinstance(target, ast.Attribute) + and isinstance(target.value, ast.Name) + and target.value.id == "self" + ): + self._add_attribute(target.attr, self._get_type_str(node.annotation)) + # Class attribute: x: int = ... + # We identify it as a class attribute if the assignment happens + # directly within the class body, not inside a method. + elif isinstance(target, ast.Name) and not self._is_in_method: + self._add_attribute(target.id, self._get_type_str(node.annotation)) + self.generic_visit(node) + + +def parse_code(code: str) -> tuple[List[Dict[str, Any]], set[str], set[str]]: + """ + Parses a string of Python code into a structured list of classes, a set of imports, + and a set of all type annotations found. + + Args: + code: A string containing Python code. + + Returns: + A tuple containing: + - A list of dictionaries, where each dictionary represents a class. + - A set of strings, where each string is an import statement. + - A set of strings, where each string is a type annotation. + """ + tree = ast.parse(code) + analyzer = CodeAnalyzer() + analyzer.visit(tree) + return analyzer.structure, analyzer.imports, analyzer.types + + +def parse_file(file_path: str) -> tuple[List[Dict[str, Any]], set[str], set[str]]: + """ + Parses a Python file into a structured list of classes, a set of imports, + and a set of all type annotations found. + + Args: + file_path: The absolute path to the Python file. + + Returns: + A tuple containing the class structure, a set of import statements, + and a set of type annotations. + """ + with open(file_path, "r", encoding="utf-8") as source: + code = source.read() + return parse_code(code) + + +def list_code_objects( + path: str, + show_methods: bool = False, + show_attributes: bool = False, + show_arguments: bool = False, +) -> Any: + """ + Lists classes and optionally their methods, attributes, and arguments + from a given Python file or directory. + + This function consolidates the functionality of the various `list_*` functions. + + Args: + path (str): The absolute path to a Python file or directory. + show_methods (bool): Whether to include methods in the output. + show_attributes (bool): Whether to include attributes in the output. + show_arguments (bool): If True, includes method arguments. Implies show_methods. + + Returns: + - If `show_methods` and `show_attributes` are both False, returns a + sorted `List[str]` of class names (mimicking `list_classes`). + - Otherwise, returns a `Dict[str, Dict[str, Any]]` containing the + requested details about each class. + """ + # If show_arguments is True, we must show methods. + if show_arguments: + show_methods = True + + results = defaultdict(dict) + all_class_keys = [] + + def process_structure( + structure: List[Dict[str, Any]], file_name: str | None = None + ): + """Populates the results dictionary from the parsed AST structure.""" + for class_info in structure: + key = class_info["class_name"] + if file_name: + key = f"{key} (in {file_name})" + + all_class_keys.append(key) + + # Skip filling details if not needed for the dictionary. + if not show_methods and not show_attributes: + continue + + if show_attributes: + results[key]["attributes"] = sorted(class_info["attributes"]) + + if show_methods: + if show_arguments: + method_details = {} + # Sort methods by name for consistent output + for method in sorted( + class_info["methods"], key=lambda m: m["method_name"] + ): + method_details[method["method_name"]] = method["args"] + results[key]["methods"] = method_details + else: + results[key]["methods"] = sorted( + [m["method_name"] for m in class_info["methods"]] + ) + + # Determine if the path is a file or directory and process accordingly + if os.path.isfile(path) and path.endswith(".py"): + structure, _, _ = parse_file(path) + process_structure(structure) + elif os.path.isdir(path): + # This assumes `utils.walk_codebase` is defined elsewhere. + for file_path in utils.walk_codebase(path): + structure, _, _ = parse_file(file_path) + process_structure(structure, file_name=os.path.basename(file_path)) + + # Return the data in the desired format based on the flags + if not show_methods and not show_attributes: + return sorted(all_class_keys) + else: + return dict(results) From 44a077707642aa61e6378c0cf7897eb216a19430 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Mon, 15 Sep 2025 11:12:57 -0400 Subject: [PATCH 13/25] feat: Add source file analysis capabilities Implements functions to analyze Python source files, including: - Filtering classes and methods based on configuration. - Building a schema of request classes and their arguments. - Processing service client files to extract relevant information. --- scripts/microgenerator/generate.py | 140 ++++++++++++++++++++++++++++- 1 file changed, 139 insertions(+), 1 deletion(-) diff --git a/scripts/microgenerator/generate.py b/scripts/microgenerator/generate.py index 7aa3d9668..2004c0211 100644 --- a/scripts/microgenerator/generate.py +++ b/scripts/microgenerator/generate.py @@ -24,6 +24,9 @@ import ast import os +import glob +import logging +import re from collections import defaultdict from typing import List, Dict, Any, Iterator @@ -65,7 +68,7 @@ def _get_type_str(self, node: ast.AST | None) -> str | None: if isinstance(curr, ast.Name): parts.append(curr.id) return ".".join(reversed(parts)) - # Handles subscripted types like 'list[str]', 'Optional[...]' + # Handles subscripted types like 'list[str]', 'Optional[...]' if isinstance(node, ast.Subscript): value_str = self._get_type_str(node.value) slice_str = self._get_type_str(node.slice) @@ -352,3 +355,138 @@ def process_structure( return sorted(all_class_keys) else: return dict(results) + + +# ============================================================================= +# Section 2: Source file data gathering +# ============================================================================= + + +def _should_include_class(class_name: str, class_filters: Dict[str, Any]) -> bool: + """Checks if a class should be included based on filter criteria.""" + if class_filters.get("include_suffixes"): + if not class_name.endswith(tuple(class_filters["include_suffixes"])): + return False + if class_filters.get("exclude_suffixes"): + if class_name.endswith(tuple(class_filters["exclude_suffixes"])): + return False + return True + + +def _should_include_method(method_name: str, method_filters: Dict[str, Any]) -> bool: + """Checks if a method should be included based on filter criteria.""" + if method_filters.get("include_prefixes"): + if not any( + method_name.startswith(p) for p in method_filters["include_prefixes"] + ): + return False + if method_filters.get("exclude_prefixes"): + if any(method_name.startswith(p) for p in method_filters["exclude_prefixes"]): + return False + return True + + +def _build_request_arg_schema( + source_files: List[str], project_root: str +) -> Dict[str, List[str]]: + """Parses type files to build a schema of request classes and their _id arguments.""" + request_arg_schema: Dict[str, List[str]] = {} + for file_path in source_files: + if "/types/" not in file_path: + continue + + # Correctly determine the module name from the file path + relative_path = os.path.relpath(file_path, project_root) + module_name = os.path.splitext(relative_path)[0].replace(os.path.sep, ".") + + try: + structure, _, _ = parse_file(file_path) + if not structure: + continue + + for class_info in structure: + class_name = class_info.get("class_name", "Unknown") + if class_name.endswith("Request"): + full_class_name = f"{module_name}.{class_name}" + id_args = [ + attr["name"] + for attr in class_info.get("attributes", []) + if attr.get("name", "").endswith("_id") + ] + if id_args: + request_arg_schema[full_class_name] = id_args + except Exception as e: + logging.warning(f"Failed to parse {file_path}: {e}") + return request_arg_schema + + +def _process_service_clients( + source_files: List[str], class_filters: Dict, method_filters: Dict +) -> tuple[defaultdict, set, set]: + """Parses service client files to extract class and method information.""" + parsed_data = defaultdict(dict) + all_imports: set[str] = set() + all_types: set[str] = set() + + for file_path in source_files: + if "/services/" not in file_path: + continue + + structure, imports, types = parse_file(file_path) + all_imports.update(imports) + all_types.update(types) + + for class_info in structure: + class_name = class_info["class_name"] + if not _should_include_class(class_name, class_filters): + continue + + parsed_data[class_name] # Ensure class is in dict + + for method in class_info["methods"]: + method_name = method["method_name"] + if not _should_include_method(method_name, method_filters): + continue + parsed_data[class_name][method_name] = method + return parsed_data, all_imports, all_types + + +def analyze_source_files( + config: Dict[str, Any], +) -> tuple[Dict[str, Any], set[str], set[str], Dict[str, List[str]]]: + """ + Analyzes source files per the configuration to extract class and method info, + as well as information on imports and typehints. + + Args: + config: The generator's configuration dictionary. + + Returns: + A tuple containing: + - A dictionary containing the data needed for template rendering. + - A set of all import statements required by the parsed methods. + - A set of all type annotations found in the parsed methods. + - A dictionary mapping request class names to their `_id` arguments. + """ + project_root = config["project_root"] + source_patterns_dict = config.get("source_files", {}) + filter_rules = config.get("filter", {}) + class_filters = filter_rules.get("classes", {}) + method_filters = filter_rules.get("methods", {}) + + source_files = [] + for group in source_patterns_dict.values(): + for pattern in group: + # Make the pattern absolute + absolute_pattern = os.path.join(project_root, pattern) + source_files.extend(glob.glob(absolute_pattern, recursive=True)) + + # PASS 1: Build the request argument schema from the types files. + request_arg_schema = _build_request_arg_schema(source_files, project_root) + + # PASS 2: Process the service client files. + parsed_data, all_imports, all_types = _process_service_clients( + source_files, class_filters, method_filters + ) + + return parsed_data, all_imports, all_types, request_arg_schema From 3e9ade669b6347528bebe58c3ef09e6b358dbe94 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Mon, 15 Sep 2025 12:33:51 -0400 Subject: [PATCH 14/25] feat: adds code generation logic --- scripts/microgenerator/generate.py | 121 +++++++++++++++++++++++++++++ 1 file changed, 121 insertions(+) diff --git a/scripts/microgenerator/generate.py b/scripts/microgenerator/generate.py index 2004c0211..4de740532 100644 --- a/scripts/microgenerator/generate.py +++ b/scripts/microgenerator/generate.py @@ -30,6 +30,7 @@ from collections import defaultdict from typing import List, Dict, Any, Iterator +from . import name_utils from . import utils # ============================================================================= @@ -490,3 +491,123 @@ def analyze_source_files( ) return parsed_data, all_imports, all_types, request_arg_schema + + +# ============================================================================= +# Section 3: Code Generation +# ============================================================================= + + +def _generate_import_statement( + context: List[Dict[str, Any]], key: str, path: str +) -> str: + """Generates a formatted import statement from a list of context dictionaries. + + Args: + context: A list of dictionaries containing the data. + key: The key to extract from each dictionary in the context. + path: The base import path (e.g., "google.cloud.bigquery_v2.services"). + + Returns: + A formatted, multi-line import statement string. + """ + names = sorted(list(set([item[key] for item in context]))) + names_str = ",\n ".join(names) + return f"from {path} import (\n {names_str}\n)" + + +def generate_code(config: Dict[str, Any], analysis_results: tuple) -> None: + """ + Generates source code files using Jinja2 templates. + """ + data, all_imports, all_types, request_arg_schema = analysis_results + project_root = config["project_root"] + config_dir = config["config_dir"] + + templates_config = config.get("templates", []) + for item in templates_config: + template_path = os.path.join(config_dir, item["template"]) + output_path = os.path.join(project_root, item["output"]) + + template = utils.load_template(template_path) + methods_context = [] + for class_name, methods in data.items(): + for method_name, method_info in methods.items(): + context = { + "name": method_name, + "class_name": class_name, + "return_type": method_info["return_type"], + } + + # Infer the request class and find its schema. + inferred_request_name = name_utils.method_to_request_class_name( + method_name + ) + + # Check for a request class name override in the config. + method_overrides = ( + config.get("filter", {}).get("methods", {}).get("overrides", {}) + ) + if method_name in method_overrides: + inferred_request_name = method_overrides[method_name].get( + "request_class_name", inferred_request_name + ) + + fq_request_name = "" + for key in request_arg_schema.keys(): + if key.endswith(f".{inferred_request_name}"): + fq_request_name = key + break + + # If found, augment the method context. + if fq_request_name: + context["request_class_full_name"] = fq_request_name + context["request_id_args"] = request_arg_schema[fq_request_name] + + methods_context.append(context) + + # Prepare imports for the template + services_context = [] + client_class_names = sorted( + list(set([m["class_name"] for m in methods_context])) + ) + + for class_name in client_class_names: + service_name_cluster = name_utils.generate_service_names(class_name) + services_context.append(service_name_cluster) + + # Also need to update methods_context to include the service_name and module_name + # so the template knows which client to use for each method. + class_to_service_map = {s["service_client_class"]: s for s in services_context} + for method in methods_context: + service_info = class_to_service_map.get(method["class_name"]) + if service_info: + method["service_name"] = service_info["service_name"] + method["service_module_name"] = service_info["service_module_name"] + + # Prepare new imports + service_imports = [ + _generate_import_statement( + services_context, + "service_module_name", + "google.cloud.bigquery_v2.services", + ) + ] + + # Prepare type imports + type_imports = [ + _generate_import_statement( + services_context, "service_name", "google.cloud.bigquery_v2.types" + ) + ] + + final_code = template.render( + service_name=config.get("service_name"), + methods=methods_context, + services=services_context, + service_imports=service_imports, + type_imports=type_imports, + request_arg_schema=request_arg_schema, + ) + + utils.write_code_to_file(output_path, final_code) From 485b9d4f0648feca1ad6abf4044803aa4b7ff7f1 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Mon, 15 Sep 2025 12:48:53 -0400 Subject: [PATCH 15/25] removes extraneous content --- scripts/microgenerator/generate.py | 119 ----------------------------- 1 file changed, 119 deletions(-) diff --git a/scripts/microgenerator/generate.py b/scripts/microgenerator/generate.py index 4de740532..53b18a2f3 100644 --- a/scripts/microgenerator/generate.py +++ b/scripts/microgenerator/generate.py @@ -492,122 +492,3 @@ def analyze_source_files( return parsed_data, all_imports, all_types, request_arg_schema - -# ============================================================================= -# Section 3: Code Generation -# ============================================================================= - - -def _generate_import_statement( - context: List[Dict[str, Any]], key: str, path: str -) -> str: - """Generates a formatted import statement from a list of context dictionaries. - - Args: - context: A list of dictionaries containing the data. - key: The key to extract from each dictionary in the context. - path: The base import path (e.g., "google.cloud.bigquery_v2.services"). - - Returns: - A formatted, multi-line import statement string. - """ - names = sorted(list(set([item[key] for item in context]))) - names_str = ",\n ".join(names) - return f"from {path} import (\n {names_str}\n)" - - -def generate_code(config: Dict[str, Any], analysis_results: tuple) -> None: - """ - Generates source code files using Jinja2 templates. - """ - data, all_imports, all_types, request_arg_schema = analysis_results - project_root = config["project_root"] - config_dir = config["config_dir"] - - templates_config = config.get("templates", []) - for item in templates_config: - template_path = os.path.join(config_dir, item["template"]) - output_path = os.path.join(project_root, item["output"]) - - template = utils.load_template(template_path) - methods_context = [] - for class_name, methods in data.items(): - for method_name, method_info in methods.items(): - context = { - "name": method_name, - "class_name": class_name, - "return_type": method_info["return_type"], - } - - # Infer the request class and find its schema. - inferred_request_name = name_utils.method_to_request_class_name( - method_name - ) - - # Check for a request class name override in the config. - method_overrides = ( - config.get("filter", {}).get("methods", {}).get("overrides", {}) - ) - if method_name in method_overrides: - inferred_request_name = method_overrides[method_name].get( - "request_class_name", inferred_request_name - ) - - fq_request_name = "" - for key in request_arg_schema.keys(): - if key.endswith(f".{inferred_request_name}"): - fq_request_name = key - break - - # If found, augment the method context. - if fq_request_name: - context["request_class_full_name"] = fq_request_name - context["request_id_args"] = request_arg_schema[fq_request_name] - - methods_context.append(context) - - # Prepare imports for the template - services_context = [] - client_class_names = sorted( - list(set([m["class_name"] for m in methods_context])) - ) - - for class_name in client_class_names: - service_name_cluster = name_utils.generate_service_names(class_name) - services_context.append(service_name_cluster) - - # Also need to update methods_context to include the service_name and module_name - # so the template knows which client to use for each method. - class_to_service_map = {s["service_client_class"]: s for s in services_context} - for method in methods_context: - service_info = class_to_service_map.get(method["class_name"]) - if service_info: - method["service_name"] = service_info["service_name"] - method["service_module_name"] = service_info["service_module_name"] - - # Prepare new imports - service_imports = [ - _generate_import_statement( - services_context, - "service_module_name", - "google.cloud.bigquery_v2.services", - ) - ] - - # Prepare type imports - type_imports = [ - _generate_import_statement( - services_context, "service_name", "google.cloud.bigquery_v2.types" - ) - ] - - final_code = template.render( - service_name=config.get("service_name"), - methods=methods_context, - services=services_context, - service_imports=service_imports, - type_imports=type_imports, - request_arg_schema=request_arg_schema, - ) - - utils.write_code_to_file(output_path, final_code) From a4276fec57f63d09fbc0622ebfd220c4cdff3669 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Mon, 15 Sep 2025 13:58:40 -0400 Subject: [PATCH 16/25] feat: microgen - adds code generation logic --- scripts/microgenerator/generate.py | 118 +++++++++++++++++++++++++++++ 1 file changed, 118 insertions(+) diff --git a/scripts/microgenerator/generate.py b/scripts/microgenerator/generate.py index 53b18a2f3..1d1e44642 100644 --- a/scripts/microgenerator/generate.py +++ b/scripts/microgenerator/generate.py @@ -492,3 +492,121 @@ def analyze_source_files( return parsed_data, all_imports, all_types, request_arg_schema + +# ============================================================================= +# Section 3: Code Generation +# ============================================================================= + +def _generate_import_statement( + context: List[Dict[str, Any]], key: str, path: str +) -> str: + """Generates a formatted import statement from a list of context dictionaries. + + Args: + context: A list of dictionaries containing the data. + key: The key to extract from each dictionary in the context. + path: The base import path (e.g., "google.cloud.bigquery_v2.services"). + + Returns: + A formatted, multi-line import statement string. + """ + names = sorted(list(set([item[key] for item in context]))) + names_str = ",\n ".join(names) + return f"from {path} import (\n {names_str}\n)" + + +def generate_code(config: Dict[str, Any], analysis_results: tuple) -> None: + """ + Generates source code files using Jinja2 templates. + """ + data, all_imports, all_types, request_arg_schema = analysis_results + project_root = config["project_root"] + config_dir = config["config_dir"] + + templates_config = config.get("templates", []) + for item in templates_config: + template_path = os.path.join(config_dir, item["template"]) + output_path = os.path.join(project_root, item["output"]) + + template = utils.load_template(template_path) + methods_context = [] + for class_name, methods in data.items(): + for method_name, method_info in methods.items(): + context = { + "name": method_name, + "class_name": class_name, + "return_type": method_info["return_type"], + } + + # Infer the request class and find its schema. + inferred_request_name = name_utils.method_to_request_class_name( + method_name + ) + + # Check for a request class name override in the config. + method_overrides = ( + config.get("filter", {}).get("methods", {}).get("overrides", {}) + ) + if method_name in method_overrides: + inferred_request_name = method_overrides[method_name].get( + "request_class_name", inferred_request_name + ) + + fq_request_name = "" + for key in request_arg_schema.keys(): + if key.endswith(f".{inferred_request_name}"): + fq_request_name = key + break + + # If found, augment the method context. + if fq_request_name: + context["request_class_full_name"] = fq_request_name + context["request_id_args"] = request_arg_schema[fq_request_name] + + methods_context.append(context) + + # Prepare imports for the template + services_context = [] + client_class_names = sorted( + list(set([m["class_name"] for m in methods_context])) + ) + + for class_name in client_class_names: + service_name_cluster = name_utils.generate_service_names(class_name) + services_context.append(service_name_cluster) + + # Also need to update methods_context to include the service_name and module_name + # so the template knows which client to use for each method. + class_to_service_map = {s["service_client_class"]: s for s in services_context} + for method in methods_context: + service_info = class_to_service_map.get(method["class_name"]) + if service_info: + method["service_name"] = service_info["service_name"] + method["service_module_name"] = service_info["service_module_name"] + + # Prepare new imports + service_imports = [ + _generate_import_statement( + services_context, + "service_module_name", + "google.cloud.bigquery_v2.services", + ) + ] + + # Prepare type imports + type_imports = [ + _generate_import_statement( + services_context, "service_name", "google.cloud.bigquery_v2.types" + ) + ] + + final_code = template.render( + service_name=config.get("service_name"), + methods=methods_context, + services=services_context, + service_imports=service_imports, + type_imports=type_imports, + request_arg_schema=request_arg_schema, + ) + + utils.write_code_to_file(output_path, final_code) From 1d0d036231aec09af0f701f94d1ea8664c25ccad Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Mon, 15 Sep 2025 14:17:57 -0400 Subject: [PATCH 17/25] feat: microgen - adds main execution and post-processing logic --- scripts/microgenerator/generate.py | 134 +++++++++++++++++++++++++++++ 1 file changed, 134 insertions(+) diff --git a/scripts/microgenerator/generate.py b/scripts/microgenerator/generate.py index 1d1e44642..28c322aee 100644 --- a/scripts/microgenerator/generate.py +++ b/scripts/microgenerator/generate.py @@ -24,6 +24,7 @@ import ast import os +import argparse import glob import logging import re @@ -497,6 +498,7 @@ def analyze_source_files( # Section 3: Code Generation # ============================================================================= + def _generate_import_statement( context: List[Dict[str, Any]], key: str, path: str ) -> str: @@ -610,3 +612,135 @@ def generate_code(config: Dict[str, Any], analysis_results: tuple) -> None: ) utils.write_code_to_file(output_path, final_code) + + +# ============================================================================= +# Section 4: Main Execution +# ============================================================================= + + +def setup_config_and_paths(config_path: str) -> Dict[str, Any]: + """Loads the configuration and sets up necessary paths. + + Args: + config_path: The path to the YAML configuration file. + + Returns: + A dictionary containing the loaded configuration and paths. + """ + + def find_project_root(start_path: str, markers: list[str]) -> str | None: + """Finds the project root by searching upwards for a marker.""" + current_path = os.path.abspath(start_path) + while True: + for marker in markers: + if os.path.exists(os.path.join(current_path, marker)): + return current_path + parent_path = os.path.dirname(current_path) + if parent_path == current_path: # Filesystem root + return None + current_path = parent_path + + # Load configuration from the YAML file. + config = utils.load_config(config_path) + + # Determine the project root. + script_dir = os.path.dirname(os.path.abspath(__file__)) + project_root = find_project_root(script_dir, ["setup.py"]) + if not project_root: + project_root = os.getcwd() # Fallback to current directory + + # Set paths in the config dictionary. + config["project_root"] = project_root + config["config_dir"] = os.path.dirname(os.path.abspath(config_path)) + + return config + + +def _execute_post_processing(config: Dict[str, Any]): + """ + Executes post-processing steps, such as patching existing files. + """ + project_root = config["project_root"] + post_processing_jobs = config.get("post_processing_templates", []) + + for job in post_processing_jobs: + template_path = os.path.join(config["config_dir"], job["template"]) + target_file_path = os.path.join(project_root, job["target_file"]) + + if not os.path.exists(target_file_path): + logging.warning( + f"Target file {target_file_path} not found, skipping post-processing job." + ) + continue + + # Read the target file + with open(target_file_path, "r") as f: + lines = f.readlines() + + # --- Extract existing imports and __all__ members --- + imports = [] + all_list = [] + all_start_index = -1 + all_end_index = -1 + + for i, line in enumerate(lines): + if line.strip().startswith("from ."): + imports.append(line.strip()) + if line.strip() == "__all__ = (": + all_start_index = i + if all_start_index != -1 and line.strip() == ")": + all_end_index = i + + if all_start_index != -1 and all_end_index != -1: + for i in range(all_start_index + 1, all_end_index): + member = lines[i].strip().replace('"', "").replace(",", "") + if member: + all_list.append(member) + + # --- Add new items and sort --- + for new_import in job.get("add_imports", []): + if new_import not in imports: + imports.append(new_import) + imports.sort() + imports = [f"{imp}\n" for imp in imports] # re-add newlines + + for new_member in job.get("add_to_all", []): + if new_member not in all_list: + all_list.append(new_member) + all_list.sort() + + # --- Render the new file content --- + template = utils.load_template(template_path) + new_content = template.render( + imports=imports, + all_list=all_list, + ) + + # --- Overwrite the target file --- + with open(target_file_path, "w") as f: + f.write(new_content) + + logging.info(f"Successfully post-processed and overwrote {target_file_path}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="A generic Python code generator for clients." + ) + parser.add_argument("config", help="Path to the YAML configuration file.") + args = parser.parse_args() + + # Load config and set up paths. + config = setup_config_and_paths(args.config) + + # Analyze the source code. + analysis_results = analyze_source_files(config) + + # Generate the new client code. + generate_code(config, analysis_results) + + # Run post-processing steps. + _execute_post_processing(config) + + # TODO: Ensure blacken gets called on the generated source files as a final step. From eff7223ae86cb84cbab206e8e326492703d3c442 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Mon, 15 Sep 2025 14:28:10 -0400 Subject: [PATCH 18/25] minor tweak to markers --- scripts/microgenerator/generate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/microgenerator/generate.py b/scripts/microgenerator/generate.py index 28c322aee..f2421918d 100644 --- a/scripts/microgenerator/generate.py +++ b/scripts/microgenerator/generate.py @@ -646,7 +646,7 @@ def find_project_root(start_path: str, markers: list[str]) -> str | None: # Determine the project root. script_dir = os.path.dirname(os.path.abspath(__file__)) - project_root = find_project_root(script_dir, ["setup.py"]) + project_root = find_project_root(script_dir, ["setup.py", ".git"]) if not project_root: project_root = os.getcwd() # Fallback to current directory From 0734bf8e604653b617d5c6dc07a072b1be38df70 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Tue, 16 Sep 2025 08:05:34 -0400 Subject: [PATCH 19/25] feat: Add testing directory\n\nAdds the scripts/microgenerator/testing directory. --- scripts/microgenerator/testing/constraints-3.13.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 scripts/microgenerator/testing/constraints-3.13.txt diff --git a/scripts/microgenerator/testing/constraints-3.13.txt b/scripts/microgenerator/testing/constraints-3.13.txt new file mode 100644 index 000000000..56a6051ca --- /dev/null +++ b/scripts/microgenerator/testing/constraints-3.13.txt @@ -0,0 +1 @@ +1 \ No newline at end of file From 510a87b0d94d768720304c9e2bb9287107fb3c98 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Tue, 16 Sep 2025 08:38:58 -0400 Subject: [PATCH 20/25] feat: Enhance to_snake_case to handle acronyms\n\nImproves the to_snake_case function in name_utils.py to correctly convert PascalCase names containing acronyms to snake_case. --- scripts/microgenerator/name_utils.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/scripts/microgenerator/name_utils.py b/scripts/microgenerator/name_utils.py index 129050c37..61eb82eba 100644 --- a/scripts/microgenerator/name_utils.py +++ b/scripts/microgenerator/name_utils.py @@ -20,8 +20,14 @@ def to_snake_case(name: str) -> str: - """Converts a PascalCase name to snake_case.""" - return re.sub(r"(? Dict[str, str]: @@ -60,14 +66,20 @@ def method_to_request_class_name(method_name: str) -> str: Returns: The inferred PascalCase name for the corresponding request class. + Raises: + ValueError: If method_name is empty. + Example: >>> method_to_request_class_name('get_dataset') 'GetDatasetRequest' >>> method_to_request_class_name('list_jobs') 'ListJobsRequest' """ + if not method_name: + raise ValueError("method_name cannot be empty") + # e.g., "get_dataset" -> ["get", "dataset"] parts = method_name.split("_") # e.g., ["get", "dataset"] -> "GetDataset" pascal_case_base = "".join(part.capitalize() for part in parts) - return f"{pascal_case_base}Request" + return f"{pascal_case_base}Request" \ No newline at end of file From a3117d894e3069eca671ef8508497beeeaf9af3e Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Tue, 16 Sep 2025 08:55:18 -0400 Subject: [PATCH 21/25] feat: Add client.py.j2 template\n\nAdds the main Jinja2 template for generating the BigQueryClient class. --- scripts/microgenerator/templates/client.py.j2 | 119 ++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 scripts/microgenerator/templates/client.py.j2 diff --git a/scripts/microgenerator/templates/client.py.j2 b/scripts/microgenerator/templates/client.py.j2 new file mode 100644 index 000000000..b1262c14d --- /dev/null +++ b/scripts/microgenerator/templates/client.py.j2 @@ -0,0 +1,119 @@ +# TODO: Add a header if needed. + +# ======== 🦕 HERE THERE BE DINOSAURS 🦖 ========= +# This content is subject to significant change. Not for review yet. +# Included as a proof of concept for context or testing ONLY. +# ================================================ + +# Imports +import os + +from typing import ( + Any, + Dict, + List, + Optional, + Sequence, + Tuple, + Union, +) + +# Service Client Imports +{% for imp in service_imports %}{{ imp }}{% endfor %} + +# Helper Imports +from . import _helpers + +# Type Imports +{% for imp in type_imports %}{{ imp }}{% endfor %} + +from google.api_core import client_options as client_options_lib +from google.api_core import gapic_v1 +from google.api_core import retry as retries +from google.auth import credentials as auth_credentials +import google.auth + +# Create type aliases +try: + OptionalRetry = Union[retries.Retry, gapic_v1.method._MethodDefault, None] +except AttributeError: # pragma: NO COVER + OptionalRetry = Union[retries.Retry, object, None] # type: ignore + +DEFAULT_RETRY: OptionalRetry = gapic_v1.method.DEFAULT +DEFAULT_TIMEOUT: Union[float, object] = gapic_v1.method.DEFAULT +DEFAULT_METADATA: Sequence[Tuple[str, Union[str, bytes]]] = () + +{#- Create a mapping from the ServiceClient class name to its property name on the main client. + e.g., {'DatasetServiceClient': 'dataset_service_client'} +-#} +{% set class_to_instance_map = {} %} +{% for service in services %} + {% set _ = class_to_instance_map.update({service.service_client_class: service.property_name}) %} +{% endfor %} + + +class BigQueryClient: + def __init__( + self, + project: Optional[str] = None, + credentials: Optional[auth_credentials.Credentials] = None, + client_options: Optional[client_options_lib.ClientOptions] = None, + ): + if credentials is None: + credentials, project_id = google.auth.default() + else: + project_id = None # project_id is not available from non-default credentials + + if project is None: + project = project_id + + self.project = project + self._credentials = credentials + self._client_options = client_options + self._clients: Dict[str, Any] = {} + + # --- *METHOD SECTION --- + +{% for method in methods %} +{% if method.request_id_args is not none and method.request_id_args|length > 0 %} +{% filter indent(4, True) %} + {% include 'partials/_method_with_request_builder.j2' %} +{% endfilter %} +{% else %} +{% filter indent(4, True) %} + {% include 'partials/_simple_passthrough_method.j2' %} +{% endfilter %} +{% endif %} + + +{% endfor %} + +{#- *ServiceClient Properties Section: methods to get/set service clients -#} + # --- *SERVICECLIENT PROPERTIES --- +{% for service in services %} + @property + def {{ service.property_name }}(self): + if "{{ service.service_name }}" not in self._clients: + self._clients["{{ service.service_name }}"] = {{ service.service_module_name }}.{{ service.service_client_class }}( + credentials=self._credentials, client_options=self._client_options + ) + return self._clients["{{ service.service_name }}"] + + @{{ service.property_name }}.setter + def {{ service.property_name }}(self, value): + if not isinstance(value, {{ service.service_module_name }}.{{ service.service_client_class }}): + raise TypeError( + "Expected an instance of {{ service.service_client_class }}." + ) + self._clients["{{ service.service_name }}"] = value + +{% endfor %} + +{#- Helper Section: methods included from partial template -#} + {#- include "partials/_client_helpers.j2" #} + + +# ======== 🦕 HERE THERE WERE DINOSAURS 🦖 ========= +# The above content is subject to significant change. Not for review yet. +# Included as a proof of concept for context or testing ONLY. +# ================================================ \ No newline at end of file From ae7d3e1f0fbcd6d2deef61a12ba0edb0a98b392f Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Tue, 16 Sep 2025 09:11:02 -0400 Subject: [PATCH 22/25] feat: Add _client_helpers.j2 partial template\n\nAdds a Jinja2 partial template containing helper macros for the client generation. --- .../templates/partials/_client_helpers.j2 | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 scripts/microgenerator/templates/partials/_client_helpers.j2 diff --git a/scripts/microgenerator/templates/partials/_client_helpers.j2 b/scripts/microgenerator/templates/partials/_client_helpers.j2 new file mode 100644 index 000000000..a6e6343b5 --- /dev/null +++ b/scripts/microgenerator/templates/partials/_client_helpers.j2 @@ -0,0 +1,49 @@ + {# + This is a partial template file intended to be included in other templates. + It contains helper methods for the BigQueryClient class. + #} + + # --- HELPER METHODS --- + def _parse_dataset_path(self, dataset_path: str) -> Tuple[Optional[str], str]: + """ + Helper to parse project_id and/or dataset_id from a string identifier. + + Args: + dataset_path: A string in the format 'project_id.dataset_id' or + 'dataset_id'. + + Returns: + A tuple of (project_id, dataset_id). + """ + if "." in dataset_path: + # Use rsplit to handle legacy paths like `google.com:my-project.my_dataset`. + project_id, dataset_id = dataset_path.rsplit(".", 1) + return project_id, dataset_id + return self.project, dataset_path + + def _parse_dataset_id_to_dict(self, dataset_id: "DatasetIdentifier") -> dict: + """ + Helper to create a dictionary from a project_id and dataset_id to pass + internally between helper functions. + + Args: + dataset_id: A string or DatasetReference. + + Returns: + A dict of {"project_id": project_id, "dataset_id": dataset_id_str }. + """ + if isinstance(dataset_id, str): + project_id, dataset_id_str = self._parse_dataset_path(dataset_id) + return {"project_id": project_id, "dataset_id": dataset_id_str} + elif isinstance(dataset_id, dataset_reference.DatasetReference): + return { + "project_id": dataset_id.project_id, + "dataset_id": dataset_id.dataset_id, + } + else: + raise TypeError(f"Invalid type for dataset_id: {type(dataset_id)}") + + def _parse_project_id_to_dict(self, project_id: Optional[str] = None) -> dict: + """Helper to create a request dictionary from a project_id.""" + final_project_id = project_id or self.project + return {"project_id": final_project_id} \ No newline at end of file From 913f52108e3dc93bb8aa1c30afebf04a383db6cb Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Mon, 6 Oct 2025 06:37:31 -0400 Subject: [PATCH 23/25] Update scripts/microgenerator/testing/constraints-3.13.txt --- scripts/microgenerator/testing/constraints-3.13.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/microgenerator/testing/constraints-3.13.txt b/scripts/microgenerator/testing/constraints-3.13.txt index 56a6051ca..e69de29bb 100644 --- a/scripts/microgenerator/testing/constraints-3.13.txt +++ b/scripts/microgenerator/testing/constraints-3.13.txt @@ -1 +0,0 @@ -1 \ No newline at end of file From 3bf9f165fd6dbb5a23374d5f4074f38634a9ccab Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Mon, 6 Oct 2025 06:37:38 -0400 Subject: [PATCH 24/25] Update scripts/microgenerator/generate.py --- scripts/microgenerator/generate.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/microgenerator/generate.py b/scripts/microgenerator/generate.py index 253cd4b94..53e24aa95 100644 --- a/scripts/microgenerator/generate.py +++ b/scripts/microgenerator/generate.py @@ -123,7 +123,6 @@ def visit_Import(self, node: ast.Import) -> None: def visit_ImportFrom(self, node: ast.ImportFrom) -> None: """Catches 'from X import Y' statements.""" - if self._depth == 0: # Only top-level imports module = node.module or "" if not module: From 898eab95648c162389c0b890bed55e61c52bafe3 Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Mon, 6 Oct 2025 06:37:52 -0400 Subject: [PATCH 25/25] Update scripts/microgenerator/generate.py --- scripts/microgenerator/generate.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/microgenerator/generate.py b/scripts/microgenerator/generate.py index 53e24aa95..2fc356464 100644 --- a/scripts/microgenerator/generate.py +++ b/scripts/microgenerator/generate.py @@ -158,7 +158,6 @@ def visit_ClassDef(self, node: ast.ClassDef) -> None: self.structure.append(class_info) self._current_class_info = class_info - self._depth += 1 self.generic_visit(node) self._depth -= 1