From 9d7e1ba32ef8f7559188476c1b085ca36e661988 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Tue, 27 Jan 2026 12:59:53 +0800 Subject: [PATCH 01/18] add multi thread --- .../pygen/codegen/serializers/__init__.py | 90 +++++++++++++------ 1 file changed, 61 insertions(+), 29 deletions(-) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py index e7654299885..c20494a5fe0 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py @@ -5,6 +5,7 @@ # -------------------------------------------------------------------------- import logging import json +import concurrent.futures from collections import namedtuple import re from typing import Any, Optional, Union @@ -538,6 +539,10 @@ def _generated_tests_samples_folder(self, folder_name: str) -> Path: def _serialize_and_write_sample(self, env: Environment): out_path = self._generated_tests_samples_folder("generated_samples") + sample_additional_folder = self.sample_additional_folder + + # Collect all sample work items + sample_tasks = [] for client in self.code_model.clients: for op_group in client.operation_groups: for operation in op_group.operations: @@ -545,24 +550,36 @@ def _serialize_and_write_sample(self, env: Environment): if not samples or operation.name.startswith("_"): continue for value in samples.values(): - file = value.get("x-ms-original-file", "sample.json") - file_name = to_snake_case(extract_sample_name(file)) + ".py" - try: - self.write_file( - out_path / self.sample_additional_folder / _sample_output_path(file) / file_name, - SampleSerializer( - code_model=self.code_model, - env=env, - operation_group=op_group, - operation=operation, - sample=value, - file_name=file_name, - ).serialize(), - ) - except Exception as e: # pylint: disable=broad-except - # sample generation shall not block code generation, so just log error - log_error = f"error happens in sample {file}: {e}" - _LOGGER.error(log_error) + sample_tasks.append((op_group, operation, value)) + + def generate_single_sample(task): + op_group, operation, sample_value = task + file = sample_value.get("x-ms-original-file", "sample.json") + file_name = to_snake_case(extract_sample_name(file)) + ".py" + try: + content = SampleSerializer( + code_model=self.code_model, + env=env, + operation_group=op_group, + operation=operation, + sample=sample_value, + file_name=file_name, + ).serialize() + output_path = out_path / sample_additional_folder / _sample_output_path(file) / file_name + return (output_path, content, None) + except Exception as e: # pylint: disable=broad-except + return (None, None, f"error happens in sample {file}: {e}") + + # Process samples in parallel using ThreadPoolExecutor + with concurrent.futures.ThreadPoolExecutor() as executor: + results = list(executor.map(generate_single_sample, sample_tasks)) + + # Write files and log errors + for output_path, content, error in results: + if error: + _LOGGER.error(error) + else: + self.write_file(output_path, content) def _serialize_and_write_test(self, env: Environment): self.code_model.for_test = True @@ -578,18 +595,33 @@ def _serialize_and_write_test(self, env: Environment): general_serializer.serialize_testpreparer(), ) + # Collect all test generation tasks + test_tasks = [] for client in self.code_model.clients: for og in client.operation_groups: - test_serializer = TestSerializer(self.code_model, env, client=client, operation_group=og) for async_mode in (True, False): - try: - test_serializer.async_mode = async_mode - self.write_file( - out_path / f"{to_snake_case(test_serializer.test_class_name)}.py", - test_serializer.serialize_test(), - ) - except Exception as e: # pylint: disable=broad-except - # test generation shall not block code generation, so just log error - log_error = f"error happens in test generation for operation group {og.class_name}: {e}" - _LOGGER.error(log_error) + test_tasks.append((client, og, async_mode)) + + def generate_single_test(task): + client, og, async_mode = task + try: + test_serializer = TestSerializer(self.code_model, env, client=client, operation_group=og) + test_serializer.async_mode = async_mode + content = test_serializer.serialize_test() + output_path = out_path / f"{to_snake_case(test_serializer.test_class_name)}.py" + return (output_path, content, None) + except Exception as e: # pylint: disable=broad-except + return (None, None, f"error happens in test generation for operation group {og.class_name}: {e}") + + # Process tests in parallel using ThreadPoolExecutor + with concurrent.futures.ThreadPoolExecutor() as executor: + results = list(executor.map(generate_single_test, test_tasks)) + + # Write files and log errors + for output_path, content, error in results: + if error: + _LOGGER.error(error) + else: + self.write_file(output_path, content) + self.code_model.for_test = False From aa825eb072485326f0197dc992203c924c2d6ec4 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Tue, 27 Jan 2026 05:25:18 +0000 Subject: [PATCH 02/18] use fake value if required parameter missing in sample --- .../codegen/serializers/sample_serializer.py | 11 ++++++--- .../codegen/serializers/test_serializer.py | 24 ++++--------------- .../pygen/codegen/serializers/utils.py | 17 +++++++++++++ 3 files changed, 29 insertions(+), 23 deletions(-) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/sample_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/sample_serializer.py index 3fd57800168..03d3fd6770f 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/sample_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/sample_serializer.py @@ -20,6 +20,7 @@ BodyParameter, FileImport, ) +from .utils import json_dumps_template, get_sub_type, get_model_type _LOGGER = logging.getLogger(__name__) @@ -102,14 +103,18 @@ def _operation_params(self) -> dict[str, Any]: for p in (self.operation.parameters.positional + self.operation.parameters.keyword_only) if not p.client_default_value ] - failure_info = "fail to find required param named {}" operation_params = {} for param in params: if not param.optional: param_value = self.sample_params.get(param.wire_name) if not param_value: - raise Exception(failure_info.format(param.client_name)) # pylint: disable=broad-exception-raised - operation_params[param.client_name] = self.handle_param(param, param_value) + model_type = get_model_type(param.type) + param_type = get_sub_type(model_type) if model_type else param.type + operation_params[param.client_name] = json_dumps_template( + param_type.get_json_template_representation() + ) + else: + operation_params[param.client_name] = self.handle_param(param, param_value) return operation_params def _operation_group_name(self) -> str: diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/test_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/test_serializer.py index 73baf5dd8d2..7c3d882d1df 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/test_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/test_serializer.py @@ -3,7 +3,7 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- -from typing import Any, Optional +from typing import Any from jinja2 import Environment from .import_serializer import FileImportSerializer @@ -14,12 +14,9 @@ OperationGroup, Client, OperationType, - ModelType, - BaseType, - CombinedType, FileImport, ) -from .utils import json_dumps_template +from .utils import json_dumps_template, get_sub_type, get_model_type def is_lro(operation_type: str) -> bool: @@ -226,25 +223,12 @@ def breadth_search_operation_group(self) -> list[list[OperationGroup]]: queue.extend([current + [og] for og in current[-1].operation_groups]) return result - def get_sub_type(self, param_type: ModelType) -> ModelType: - if param_type.discriminated_subtypes: - for item in param_type.discriminated_subtypes.values(): - return self.get_sub_type(item) - return param_type - - def get_model_type(self, param_type: BaseType) -> Optional[ModelType]: - if isinstance(param_type, ModelType): - return param_type - if isinstance(param_type, CombinedType): - return param_type.target_model_subtype((ModelType,)) - return None - def get_operation_params(self, operation: OperationType) -> dict[str, Any]: operation_params = {} required_params = [p for p in operation.parameters.method if not p.optional] for param in required_params: - model_type = self.get_model_type(param.type) - param_type = self.get_sub_type(model_type) if model_type else param.type + model_type = get_model_type(param.type) + param_type = get_sub_type(model_type) if model_type else param.type operation_params[param.client_name] = json_dumps_template(param_type.get_json_template_representation()) return operation_params diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/utils.py b/packages/http-client-python/generator/pygen/codegen/serializers/utils.py index 9ea6c85c77b..dc4b7d44ee3 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/utils.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/utils.py @@ -7,6 +7,23 @@ from typing import Optional, Any from pathlib import Path +from ..models import ModelType, BaseType, CombinedType + + +def get_sub_type(param_type: ModelType) -> ModelType: + if param_type.discriminated_subtypes: + for item in param_type.discriminated_subtypes.values(): + return get_sub_type(item) + return param_type + + +def get_model_type(param_type: BaseType) -> Optional[ModelType]: + if isinstance(param_type, ModelType): + return param_type + if isinstance(param_type, CombinedType): + return param_type.target_model_subtype((ModelType,)) + return None + def method_signature_and_response_type_annotation_template( *, From aa496b0d8e1c56d17345c00369932801329c97fb Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Tue, 27 Jan 2026 05:32:04 +0000 Subject: [PATCH 03/18] optimize util --- .../codegen/serializers/sample_serializer.py | 8 ++---- .../codegen/serializers/test_serializer.py | 6 ++--- .../pygen/codegen/serializers/utils.py | 26 +++++++++++++------ 3 files changed, 22 insertions(+), 18 deletions(-) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/sample_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/sample_serializer.py index 03d3fd6770f..c460594a6c6 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/sample_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/sample_serializer.py @@ -20,7 +20,7 @@ BodyParameter, FileImport, ) -from .utils import json_dumps_template, get_sub_type, get_model_type +from .utils import create_fake_value _LOGGER = logging.getLogger(__name__) @@ -108,11 +108,7 @@ def _operation_params(self) -> dict[str, Any]: if not param.optional: param_value = self.sample_params.get(param.wire_name) if not param_value: - model_type = get_model_type(param.type) - param_type = get_sub_type(model_type) if model_type else param.type - operation_params[param.client_name] = json_dumps_template( - param_type.get_json_template_representation() - ) + operation_params[param.client_name] = create_fake_value(param.type) else: operation_params[param.client_name] = self.handle_param(param, param_value) return operation_params diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/test_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/test_serializer.py index 7c3d882d1df..d61813c0a45 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/test_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/test_serializer.py @@ -16,7 +16,7 @@ OperationType, FileImport, ) -from .utils import json_dumps_template, get_sub_type, get_model_type +from .utils import create_fake_value def is_lro(operation_type: str) -> bool: @@ -227,9 +227,7 @@ def get_operation_params(self, operation: OperationType) -> dict[str, Any]: operation_params = {} required_params = [p for p in operation.parameters.method if not p.optional] for param in required_params: - model_type = get_model_type(param.type) - param_type = get_sub_type(model_type) if model_type else param.type - operation_params[param.client_name] = json_dumps_template(param_type.get_json_template_representation()) + operation_params[param.client_name] = create_fake_value(param.type) return operation_params def get_test(self) -> Test: diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/utils.py b/packages/http-client-python/generator/pygen/codegen/serializers/utils.py index dc4b7d44ee3..ca361ddc813 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/utils.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/utils.py @@ -17,14 +17,6 @@ def get_sub_type(param_type: ModelType) -> ModelType: return param_type -def get_model_type(param_type: BaseType) -> Optional[ModelType]: - if isinstance(param_type, ModelType): - return param_type - if isinstance(param_type, CombinedType): - return param_type.target_model_subtype((ModelType,)) - return None - - def method_signature_and_response_type_annotation_template( *, method_signature: str, @@ -69,3 +61,21 @@ def _improve_json_string(template_representation: str) -> Any: def json_dumps_template(template_representation: Any) -> Any: # only for template use, since it wraps everything in strings return _improve_json_string(json.dumps(template_representation, indent=4)) + + +def create_fake_value(param_type: BaseType) -> Any: + """Create a fake value for a parameter type by getting its JSON template representation. + + This function generates a fake value suitable for samples and tests. + + :param param_type: The parameter type to create a fake value for. + :return: A string representation of the fake value. + """ + if isinstance(param_type, ModelType): + model_type = param_type + elif isinstance(param_type, CombinedType): + model_type = param_type.target_model_subtype((ModelType,)) + else: + model_type = None + resolved_type = get_sub_type(model_type) if model_type else param_type + return json_dumps_template(resolved_type.get_json_template_representation()) From 0abee3e94bd5ca5506e09c8daf23f8ba3a4a3c0f Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Tue, 27 Jan 2026 06:52:34 +0000 Subject: [PATCH 04/18] update --- .../pygen/codegen/serializers/__init__.py | 15 +++++++-------- .../codegen/serializers/sample_serializer.py | 9 ++------- 2 files changed, 9 insertions(+), 15 deletions(-) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py index c20494a5fe0..bba36216b08 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py @@ -602,26 +602,25 @@ def _serialize_and_write_test(self, env: Environment): for async_mode in (True, False): test_tasks.append((client, og, async_mode)) - def generate_single_test(task): + def generate_and_write_single_test(task): client, og, async_mode = task try: test_serializer = TestSerializer(self.code_model, env, client=client, operation_group=og) test_serializer.async_mode = async_mode content = test_serializer.serialize_test() output_path = out_path / f"{to_snake_case(test_serializer.test_class_name)}.py" - return (output_path, content, None) + self.write_file(output_path, content) + return None except Exception as e: # pylint: disable=broad-except - return (None, None, f"error happens in test generation for operation group {og.class_name}: {e}") + return f"error happens in test generation for operation group {og.class_name}: {e}" # Process tests in parallel using ThreadPoolExecutor with concurrent.futures.ThreadPoolExecutor() as executor: - results = list(executor.map(generate_single_test, test_tasks)) + results = list(executor.map(generate_and_write_single_test, test_tasks)) - # Write files and log errors - for output_path, content, error in results: + # Log errors + for error in results: if error: _LOGGER.error(error) - else: - self.write_file(output_path, content) self.code_model.for_test = False diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/sample_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/sample_serializer.py index c460594a6c6..d6a9e6de4a3 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/sample_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/sample_serializer.py @@ -98,14 +98,9 @@ def handle_param(param: Union[Parameter, BodyParameter], param_value: Any) -> st # prepare operation parameters def _operation_params(self) -> dict[str, Any]: - params = [ - p - for p in (self.operation.parameters.positional + self.operation.parameters.keyword_only) - if not p.client_default_value - ] operation_params = {} - for param in params: - if not param.optional: + for param in (self.operation.parameters.positional + self.operation.parameters.keyword_only): + if not param.optional and not param.client_default_value: param_value = self.sample_params.get(param.wire_name) if not param_value: operation_params[param.client_name] = create_fake_value(param.type) From 7b85fe9f3c8c3c1166280c3a2a3f80cf35370a87 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Tue, 27 Jan 2026 07:30:53 +0000 Subject: [PATCH 05/18] update --- .../pygen/codegen/serializers/__init__.py | 49 +++++++++---------- 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py index bba36216b08..57ce0e58a20 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py @@ -584,43 +584,42 @@ def generate_single_sample(task): def _serialize_and_write_test(self, env: Environment): self.code_model.for_test = True out_path = self._generated_tests_samples_folder("generated_tests") + + # Phase 1: Generate all content (CPU-bound) + files_to_write: list[tuple[Path, str]] = [] + general_serializer = TestGeneralSerializer(code_model=self.code_model, env=env) - self.write_file(out_path / "conftest.py", general_serializer.serialize_conftest()) + files_to_write.append((out_path / "conftest.py", general_serializer.serialize_conftest())) + if not self.code_model.options["azure-arm"]: for async_mode in (True, False): async_suffix = "_async" if async_mode else "" general_serializer.async_mode = async_mode - self.write_file( + files_to_write.append(( out_path / f"testpreparer{async_suffix}.py", general_serializer.serialize_testpreparer(), - ) + )) - # Collect all test generation tasks - test_tasks = [] + # Generate test files - reuse serializer per operation group, toggle async_mode for client in self.code_model.clients: for og in client.operation_groups: - for async_mode in (True, False): - test_tasks.append((client, og, async_mode)) - - def generate_and_write_single_test(task): - client, og, async_mode = task - try: + # Create serializer once per operation group test_serializer = TestSerializer(self.code_model, env, client=client, operation_group=og) - test_serializer.async_mode = async_mode - content = test_serializer.serialize_test() - output_path = out_path / f"{to_snake_case(test_serializer.test_class_name)}.py" - self.write_file(output_path, content) - return None - except Exception as e: # pylint: disable=broad-except - return f"error happens in test generation for operation group {og.class_name}: {e}" + for async_mode in (True, False): + try: + test_serializer.async_mode = async_mode + content = test_serializer.serialize_test() + output_path = out_path / f"{to_snake_case(test_serializer.test_class_name)}.py" + files_to_write.append((output_path, content)) + except Exception as e: # pylint: disable=broad-except + _LOGGER.error(f"error happens in test generation for operation group {og.class_name}: {e}") + + # Phase 2: Write all files (I/O-bound, threading helps here) + def write_single_file(item: tuple[Path, str]) -> None: + path, content = item + self.write_file(path, content) - # Process tests in parallel using ThreadPoolExecutor with concurrent.futures.ThreadPoolExecutor() as executor: - results = list(executor.map(generate_and_write_single_test, test_tasks)) - - # Log errors - for error in results: - if error: - _LOGGER.error(error) + executor.map(write_single_file, files_to_write) self.code_model.for_test = False From 5c2d19bcbd2b893a869cb9adf7a3dddc397977eb Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Wed, 28 Jan 2026 06:00:15 +0000 Subject: [PATCH 06/18] update --- .../pygen/codegen/serializers/test_serializer.py | 3 ++- .../pygen/codegen/templates/test.py.jinja2 | 16 ++++++++-------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/test_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/test_serializer.py index d61813c0a45..27a84c943d8 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/test_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/test_serializer.py @@ -265,6 +265,7 @@ def test_class_name(self) -> str: def serialize_test(self) -> str: return self.env.get_template("test.py.jinja2").render( imports=self.import_test, - code_model=self.code_model, + is_azure_arm=self.code_model.options["azure-arm"], + license_header=self.code_model.license_header, test=self.get_test(), ) diff --git a/packages/http-client-python/generator/pygen/codegen/templates/test.py.jinja2 b/packages/http-client-python/generator/pygen/codegen/templates/test.py.jinja2 index 40b9e06a600..15a9148d716 100644 --- a/packages/http-client-python/generator/pygen/codegen/templates/test.py.jinja2 +++ b/packages/http-client-python/generator/pygen/codegen/templates/test.py.jinja2 @@ -1,21 +1,21 @@ {% set prefix_lower = test.prefix|lower %} -{% set client_var = "self.client" if code_model.options["azure-arm"] else "client" %} +{% set client_var = "self.client" if is_azure_arm else "client" %} {% set async = "async " if test.async_mode else "" %} {% set async_suffix = "_async" if test.async_mode else "" %} # coding=utf-8 -{% if code_model.license_header %} -{{ code_model.license_header }} +{% if license_header %} +{{ license_header }} {% endif %} import pytest {{ imports }} -{% if code_model.options["azure-arm"] %} +{% if is_azure_arm %} AZURE_LOCATION = "eastus" {% endif %} @pytest.mark.skip("you may need to update the auto-generated test case before run it") class {{ test.test_class_name }}({{ test.base_test_class_name }}): -{% if code_model.options["azure-arm"] %} +{% if is_azure_arm %} def setup_method(self, method): {% if test.async_mode %} self.client = self.create_mgmt_client({{ test.client_name }}, is_async=True) @@ -24,13 +24,13 @@ class {{ test.test_class_name }}({{ test.base_test_class_name }}): {% endif %} {% endif %} {% for testcase in test.testcases %} - {% if code_model.options["azure-arm"] %} + {% if is_azure_arm %} @{{ test.preparer_name }}(location=AZURE_LOCATION) {% else %} @{{ test.preparer_name }}() {% endif %} @recorded_by_proxy{{ async_suffix }} - {% if code_model.options["azure-arm"] %} + {% if is_azure_arm %} {{ async }}def test_{{ testcase.name }}(self, resource_group): {% else %} {{ async }}def test_{{ testcase.name }}(self, {{ prefix_lower }}_endpoint): @@ -38,7 +38,7 @@ class {{ test.test_class_name }}({{ test.base_test_class_name }}): {% endif %} {{testcase.response }}{{ client_var }}{{ testcase.operation_group_prefix }}.{{ testcase.operation.name }}( {% for key, value in testcase.params.items() %} - {% if code_model.options["azure-arm"] and key == "resource_group_name" %} + {% if is_azure_arm and key == "resource_group_name" %} {{ key }}=resource_group.name, {% else %} {{ key }}={{ value|indent(12) }}, From 937f38971f75cf06286f9323aa3aab34c9fb0a92 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Wed, 28 Jan 2026 06:07:39 +0000 Subject: [PATCH 07/18] update --- .../pygen/codegen/templates/test.py.jinja2 | 29 ++++++++++++------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/packages/http-client-python/generator/pygen/codegen/templates/test.py.jinja2 b/packages/http-client-python/generator/pygen/codegen/templates/test.py.jinja2 index 15a9148d716..e25d6212fb7 100644 --- a/packages/http-client-python/generator/pygen/codegen/templates/test.py.jinja2 +++ b/packages/http-client-python/generator/pygen/codegen/templates/test.py.jinja2 @@ -23,22 +23,14 @@ class {{ test.test_class_name }}({{ test.base_test_class_name }}): self.client = self.create_mgmt_client({{ test.client_name }}) {% endif %} {% endif %} +{% if is_azure_arm %} {% for testcase in test.testcases %} - {% if is_azure_arm %} @{{ test.preparer_name }}(location=AZURE_LOCATION) - {% else %} - @{{ test.preparer_name }}() - {% endif %} @recorded_by_proxy{{ async_suffix }} - {% if is_azure_arm %} {{ async }}def test_{{ testcase.name }}(self, resource_group): - {% else %} - {{ async }}def test_{{ testcase.name }}(self, {{ prefix_lower }}_endpoint): - {{ client_var }} = self.{{ test.create_client_name }}(endpoint={{ prefix_lower }}_endpoint) - {% endif %} {{testcase.response }}{{ client_var }}{{ testcase.operation_group_prefix }}.{{ testcase.operation.name }}( {% for key, value in testcase.params.items() %} - {% if is_azure_arm and key == "resource_group_name" %} + {% if key == "resource_group_name" %} {{ key }}=resource_group.name, {% else %} {{ key }}={{ value|indent(12) }}, @@ -50,3 +42,20 @@ class {{ test.test_class_name }}({{ test.base_test_class_name }}): # ... {% endfor %} +{% else %} +{% for testcase in test.testcases %} + @{{ test.preparer_name }}() + @recorded_by_proxy{{ async_suffix }} + {{ async }}def test_{{ testcase.name }}(self, {{ prefix_lower }}_endpoint): + {{ client_var }} = self.{{ test.create_client_name }}(endpoint={{ prefix_lower }}_endpoint) + {{testcase.response }}{{ client_var }}{{ testcase.operation_group_prefix }}.{{ testcase.operation.name }}( + {% for key, value in testcase.params.items() %} + {{ key }}={{ value|indent(12) }}, + {% endfor %} + ){{ testcase.operation_suffix }} + {{ testcase.extra_operation }} + # please add some check logic here by yourself + # ... + +{% endfor %} +{% endif %} From ea7fe89df63110e27c2cc4e3053efa6ec0412fd9 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Wed, 28 Jan 2026 06:20:45 +0000 Subject: [PATCH 08/18] update --- .../pygen/codegen/serializers/test_serializer.py | 14 +++++++++++++- .../pygen/codegen/templates/test.py.jinja2 | 7 +------ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/test_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/test_serializer.py index 27a84c943d8..4deecd83a95 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/test_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/test_serializer.py @@ -70,11 +70,22 @@ def __init__( operation: OperationType, *, async_mode: bool = False, + is_azure_arm: bool = False, ) -> None: self.operation_groups = operation_groups - self.params = params + self._params = params self.operation = operation self.async_mode = async_mode + self.is_azure_arm = is_azure_arm + + @property + def params(self) -> dict[str, Any]: + if self.is_azure_arm: + return { + k: ("resource_group.name" if k == "resource_group_name" else v) + for k, v in self._params.items() + } + return self._params @property def name(self) -> str: @@ -242,6 +253,7 @@ def get_test(self) -> Test: params=operation_params, operation=operation, async_mode=self.async_mode, + is_azure_arm=self.code_model.options["azure-arm"], ) testcases.append(testcase) if not testcases: diff --git a/packages/http-client-python/generator/pygen/codegen/templates/test.py.jinja2 b/packages/http-client-python/generator/pygen/codegen/templates/test.py.jinja2 index e25d6212fb7..7b96fc9472a 100644 --- a/packages/http-client-python/generator/pygen/codegen/templates/test.py.jinja2 +++ b/packages/http-client-python/generator/pygen/codegen/templates/test.py.jinja2 @@ -22,19 +22,14 @@ class {{ test.test_class_name }}({{ test.base_test_class_name }}): {% else %} self.client = self.create_mgmt_client({{ test.client_name }}) {% endif %} -{% endif %} -{% if is_azure_arm %} + {% for testcase in test.testcases %} @{{ test.preparer_name }}(location=AZURE_LOCATION) @recorded_by_proxy{{ async_suffix }} {{ async }}def test_{{ testcase.name }}(self, resource_group): {{testcase.response }}{{ client_var }}{{ testcase.operation_group_prefix }}.{{ testcase.operation.name }}( {% for key, value in testcase.params.items() %} - {% if key == "resource_group_name" %} - {{ key }}=resource_group.name, - {% else %} {{ key }}={{ value|indent(12) }}, - {% endif %} {% endfor %} ){{ testcase.operation_suffix }} {{ testcase.extra_operation }} From d55fb73365fe3caaf91647384cffe07fe83838f3 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Wed, 28 Jan 2026 06:24:44 +0000 Subject: [PATCH 09/18] update --- .../codegen/serializers/test_serializer.py | 122 +++++++----------- 1 file changed, 50 insertions(+), 72 deletions(-) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/test_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/test_serializer.py index 4deecd83a95..74744d62618 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/test_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/test_serializer.py @@ -36,30 +36,15 @@ def __init__(self, code_model: CodeModel, client_name: str, *, async_mode: bool self.code_model = code_model self.client_name = client_name self.async_mode = async_mode - - @property - def async_suffix_capt(self) -> str: - return "Async" if self.async_mode else "" - - @property - def create_client_name(self) -> str: - return "create_async_client" if self.async_mode else "create_client" - - @property - def prefix(self) -> str: - return self.client_name.replace("Client", "") - - @property - def preparer_name(self) -> str: - if self.code_model.options["azure-arm"]: - return "RandomNameResourceGroupPreparer" - return self.prefix + "Preparer" - - @property - def base_test_class_name(self) -> str: - if self.code_model.options["azure-arm"]: - return "AzureMgmtRecordedTestCase" - return f"{self.client_name}TestBase{self.async_suffix_capt}" + # Pre-compute values for render speed optimization + self.async_suffix_capt = "Async" if async_mode else "" + self.create_client_name = "create_async_client" if async_mode else "create_client" + self.prefix = client_name.replace("Client", "") + is_azure_arm = code_model.options["azure-arm"] + self.preparer_name = "RandomNameResourceGroupPreparer" if is_azure_arm else self.prefix + "Preparer" + self.base_test_class_name = ( + "AzureMgmtRecordedTestCase" if is_azure_arm else f"{client_name}TestBase{self.async_suffix_capt}" + ) class TestCase: @@ -73,58 +58,51 @@ def __init__( is_azure_arm: bool = False, ) -> None: self.operation_groups = operation_groups - self._params = params self.operation = operation self.async_mode = async_mode self.is_azure_arm = is_azure_arm - - @property - def params(self) -> dict[str, Any]: - if self.is_azure_arm: - return { - k: ("resource_group.name" if k == "resource_group_name" else v) - for k, v in self._params.items() + # Pre-compute params + if is_azure_arm: + self.params = { + k: ("resource_group.name" if k == "resource_group_name" else v) for k, v in params.items() } - return self._params - - @property - def name(self) -> str: - if self.operation_groups[-1].is_mixin: - return self.operation.name - return "_".join([og.property_name for og in self.operation_groups] + [self.operation.name]) - - @property - def operation_group_prefix(self) -> str: - if self.operation_groups[-1].is_mixin: - return "" - return "." + ".".join([og.property_name for og in self.operation_groups]) - - @property - def response(self) -> str: - if self.async_mode: - if is_lro(self.operation.operation_type): - return "response = await (await " - if is_common_operation(self.operation.operation_type): - return "response = await " - return "response = " - - @property - def lro_comment(self) -> str: - return " # call '.result()' to poll until service return final result" - - @property - def operation_suffix(self) -> str: - if is_lro(self.operation.operation_type): - extra = ")" if self.async_mode else "" - return f"{extra}.result(){self.lro_comment}" - return "" - - @property - def extra_operation(self) -> str: - if is_paging(self.operation.operation_type): - async_str = "async " if self.async_mode else "" - return f"result = [r {async_str}for r in response]" - return "" + else: + self.params = params + # Pre-compute name + if operation_groups[-1].is_mixin: + self.name = operation.name + else: + self.name = "_".join([og.property_name for og in operation_groups] + [operation.name]) + # Pre-compute operation_group_prefix + if operation_groups[-1].is_mixin: + self.operation_group_prefix = "" + else: + self.operation_group_prefix = "." + ".".join([og.property_name for og in operation_groups]) + # Pre-compute response + operation_type = operation.operation_type + if async_mode: + if is_lro(operation_type): + self.response = "response = await (await " + elif is_common_operation(operation_type): + self.response = "response = await " + else: + self.response = "response = " + else: + self.response = "response = " + # Pre-compute lro_comment + self.lro_comment = " # call '.result()' to poll until service return final result" + # Pre-compute operation_suffix + if is_lro(operation_type): + extra = ")" if async_mode else "" + self.operation_suffix = f"{extra}.result(){self.lro_comment}" + else: + self.operation_suffix = "" + # Pre-compute extra_operation + if is_paging(operation_type): + async_str = "async " if async_mode else "" + self.extra_operation = f"result = [r {async_str}for r in response]" + else: + self.extra_operation = "" class Test(TestName): From a4c3643e45362e1b6a5575448b3672ebec7c8f12 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Wed, 28 Jan 2026 06:59:10 +0000 Subject: [PATCH 10/18] update --- .../generator/pygen/codegen/templates/test.py.jinja2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/http-client-python/generator/pygen/codegen/templates/test.py.jinja2 b/packages/http-client-python/generator/pygen/codegen/templates/test.py.jinja2 index 7b96fc9472a..e3f4fc4df37 100644 --- a/packages/http-client-python/generator/pygen/codegen/templates/test.py.jinja2 +++ b/packages/http-client-python/generator/pygen/codegen/templates/test.py.jinja2 @@ -29,7 +29,7 @@ class {{ test.test_class_name }}({{ test.base_test_class_name }}): {{ async }}def test_{{ testcase.name }}(self, resource_group): {{testcase.response }}{{ client_var }}{{ testcase.operation_group_prefix }}.{{ testcase.operation.name }}( {% for key, value in testcase.params.items() %} - {{ key }}={{ value|indent(12) }}, + {{ key }}={{ value }}, {% endfor %} ){{ testcase.operation_suffix }} {{ testcase.extra_operation }} @@ -45,7 +45,7 @@ class {{ test.test_class_name }}({{ test.base_test_class_name }}): {{ client_var }} = self.{{ test.create_client_name }}(endpoint={{ prefix_lower }}_endpoint) {{testcase.response }}{{ client_var }}{{ testcase.operation_group_prefix }}.{{ testcase.operation.name }}( {% for key, value in testcase.params.items() %} - {{ key }}={{ value|indent(12) }}, + {{ key }}={{ value }}, {% endfor %} ){{ testcase.operation_suffix }} {{ testcase.extra_operation }} From cbe9d4f6742bce4d23470ba19cd1685cb7e06595 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Wed, 28 Jan 2026 09:22:26 +0000 Subject: [PATCH 11/18] update --- .../generator/pygen/codegen/serializers/__init__.py | 6 ++++++ .../pygen/codegen/serializers/test_serializer.py | 12 ++++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py index 57ce0e58a20..75f7017645a 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py @@ -601,6 +601,8 @@ def _serialize_and_write_test(self, env: Environment): )) # Generate test files - reuse serializer per operation group, toggle async_mode + # Cache import_test per (client.name, async_mode) since it's expensive to compute + import_test_cache: dict[tuple[str, bool], str] = {} for client in self.code_model.clients: for og in client.operation_groups: # Create serializer once per operation group @@ -608,6 +610,10 @@ def _serialize_and_write_test(self, env: Environment): for async_mode in (True, False): try: test_serializer.async_mode = async_mode + cache_key = (client.name, async_mode) + if cache_key not in import_test_cache: + import_test_cache[cache_key] = test_serializer.get_import_test() + test_serializer.import_test = import_test_cache[cache_key] content = test_serializer.serialize_test() output_path = out_path / f"{to_snake_case(test_serializer.test_class_name)}.py" files_to_write.append((output_path, content)) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/test_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/test_serializer.py index 74744d62618..de6a5b8137a 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/test_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/test_serializer.py @@ -175,9 +175,17 @@ def __init__( super().__init__(code_model, env, async_mode=async_mode) self.client = client self.operation_group = operation_group + self._import_test: str = "" @property - def import_test(self) -> FileImportSerializer: + def import_test(self) -> str: + return self._import_test + + @import_test.setter + def import_test(self, value: str) -> None: + self._import_test = value + + def get_import_test(self) -> str: imports = self.init_file_import() test_name = TestName(self.code_model, self.client.name, async_mode=self.async_mode) async_suffix = "_async" if self.async_mode else "" @@ -198,7 +206,7 @@ def import_test(self) -> FileImportSerializer: ) if self.code_model.options["azure-arm"]: self.add_import_client(imports) - return FileImportSerializer(imports, self.async_mode) + return str(FileImportSerializer(imports, self.async_mode)) @property def breadth_search_operation_group(self) -> list[list[OperationGroup]]: From ca05fe9d9eed8021ddfaa176ac8ce8d581df34de Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Thu, 29 Jan 2026 06:22:04 +0000 Subject: [PATCH 12/18] update --- .../pygen/codegen/serializers/__init__.py | 84 ++++++++++--------- .../codegen/serializers/sample_serializer.py | 24 ++++-- .../codegen/serializers/test_serializer.py | 4 +- .../pygen/codegen/serializers/utils.py | 12 ++- 4 files changed, 74 insertions(+), 50 deletions(-) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py index 75f7017645a..412894809f1 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py @@ -35,11 +35,7 @@ from .test_serializer import TestSerializer, TestGeneralSerializer from .types_serializer import TypesSerializer from ...utils import to_snake_case, VALID_PACKAGE_MODE -from .utils import ( - extract_sample_name, - get_namespace_from_package_name, - get_namespace_config, -) +from .utils import extract_sample_name, get_namespace_from_package_name, get_namespace_config, hash_file_import _LOGGER = logging.getLogger(__name__) @@ -541,45 +537,49 @@ def _serialize_and_write_sample(self, env: Environment): out_path = self._generated_tests_samples_folder("generated_samples") sample_additional_folder = self.sample_additional_folder - # Collect all sample work items - sample_tasks = [] + # Phase 1: Generate all content (CPU-bound) + files_to_write: list[tuple[Path, str]] = [] + # Cache import_test per (client_namespace, imports_hash_string) since it's expensive to compute + import_sample_cache: dict[tuple[str, str], str] = {} + for client in self.code_model.clients: for op_group in client.operation_groups: for operation in op_group.operations: samples = operation.yaml_data.get("samples") if not samples or operation.name.startswith("_"): continue - for value in samples.values(): - sample_tasks.append((op_group, operation, value)) - - def generate_single_sample(task): - op_group, operation, sample_value = task - file = sample_value.get("x-ms-original-file", "sample.json") - file_name = to_snake_case(extract_sample_name(file)) + ".py" - try: - content = SampleSerializer( - code_model=self.code_model, - env=env, - operation_group=op_group, - operation=operation, - sample=sample_value, - file_name=file_name, - ).serialize() - output_path = out_path / sample_additional_folder / _sample_output_path(file) / file_name - return (output_path, content, None) - except Exception as e: # pylint: disable=broad-except - return (None, None, f"error happens in sample {file}: {e}") - - # Process samples in parallel using ThreadPoolExecutor - with concurrent.futures.ThreadPoolExecutor() as executor: - results = list(executor.map(generate_single_sample, sample_tasks)) + for sample_value in samples.values(): + file = sample_value.get("x-ms-original-file", "sample.json") + file_name = to_snake_case(extract_sample_name(file)) + ".py" + try: + sample_ser = SampleSerializer( + code_model=self.code_model, + env=env, + operation_group=op_group, + operation=operation, + sample=sample_value, + file_name=file_name, + ) + file_import = sample_ser.get_file_import() + imports_hash_string = hash_file_import(file_import) + cache_key = (op_group.client.client_namespace, imports_hash_string) + if cache_key not in import_sample_cache: + import_sample_cache[cache_key] = sample_ser.get_imports_from_file_import(file_import) + sample_ser.imports = import_sample_cache[cache_key] + + content = sample_ser.serialize() + output_path = out_path / sample_additional_folder / _sample_output_path(file) / file_name + files_to_write.append((output_path, content)) + except Exception as e: # pylint: disable=broad-except + _LOGGER.error(f"error happens in sample {file}: {e}") - # Write files and log errors - for output_path, content, error in results: - if error: - _LOGGER.error(error) - else: - self.write_file(output_path, content) + # Phase 2: Write all files (I/O-bound, threading helps here) + def write_single_file(item: tuple[Path, str]) -> None: + path, content = item + self.write_file(path, content) + + with concurrent.futures.ThreadPoolExecutor() as executor: + executor.map(write_single_file, files_to_write) def _serialize_and_write_test(self, env: Environment): self.code_model.for_test = True @@ -595,10 +595,12 @@ def _serialize_and_write_test(self, env: Environment): for async_mode in (True, False): async_suffix = "_async" if async_mode else "" general_serializer.async_mode = async_mode - files_to_write.append(( - out_path / f"testpreparer{async_suffix}.py", - general_serializer.serialize_testpreparer(), - )) + files_to_write.append( + ( + out_path / f"testpreparer{async_suffix}.py", + general_serializer.serialize_testpreparer(), + ) + ) # Generate test files - reuse serializer per operation group, toggle async_mode # Cache import_test per (client.name, async_mode) since it's expensive to compute diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/sample_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/sample_serializer.py index d6a9e6de4a3..50102a750f5 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/sample_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/sample_serializer.py @@ -4,7 +4,7 @@ # license information. # -------------------------------------------------------------------------- import logging -from typing import Any, Union +from typing import Any, Optional, Union from jinja2 import Environment from ..models.operation import OperationBase @@ -41,8 +41,17 @@ def __init__( self.sample = sample self.file_name = file_name self.sample_params = sample.get("parameters", {}) + self._imports: str = "" - def _imports(self) -> FileImportSerializer: + @property + def imports(self) -> str: + return self._imports + + @imports.setter + def imports(self, value: str) -> None: + self._imports = value + + def get_file_import(self) -> FileImport: imports = FileImport(self.code_model) client = self.operation_group.client namespace = client.client_namespace @@ -60,7 +69,12 @@ def _imports(self) -> FileImportSerializer: for param in self.operation.parameters.positional + self.operation.parameters.keyword_only: if param.client_default_value is None and not param.optional and param.wire_name in self.sample_params: imports.merge(param.type.imports_for_sample()) - return FileImportSerializer(imports, True) + + return imports + + @staticmethod + def get_imports_from_file_import(file_import: FileImport) -> str: + return str(FileImportSerializer(file_import, True)) def _client_params(self) -> dict[str, Any]: # client params @@ -99,7 +113,7 @@ def handle_param(param: Union[Parameter, BodyParameter], param_value: Any) -> st # prepare operation parameters def _operation_params(self) -> dict[str, Any]: operation_params = {} - for param in (self.operation.parameters.positional + self.operation.parameters.keyword_only): + for param in self.operation.parameters.positional + self.operation.parameters.keyword_only: if not param.optional and not param.client_default_value: param_value = self.sample_params.get(param.wire_name) if not param_value: @@ -150,7 +164,7 @@ def serialize(self) -> str: operation_params=self._operation_params(), operation_group_name=self._operation_group_name(), operation_name=self._operation_name(), - imports=self._imports(), + imports=self.imports, client_params=self._client_params(), origin_file=self._origin_file(), return_var=return_var, diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/test_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/test_serializer.py index de6a5b8137a..410b45cfecb 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/test_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/test_serializer.py @@ -63,9 +63,7 @@ def __init__( self.is_azure_arm = is_azure_arm # Pre-compute params if is_azure_arm: - self.params = { - k: ("resource_group.name" if k == "resource_group_name" else v) for k, v in params.items() - } + self.params = {k: ("resource_group.name" if k == "resource_group_name" else v) for k, v in params.items()} else: self.params = params # Pre-compute name diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/utils.py b/packages/http-client-python/generator/pygen/codegen/serializers/utils.py index ca361ddc813..3dcb83105bb 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/utils.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/utils.py @@ -7,7 +7,7 @@ from typing import Optional, Any from pathlib import Path -from ..models import ModelType, BaseType, CombinedType +from ..models import ModelType, BaseType, CombinedType, FileImport def get_sub_type(param_type: ModelType) -> ModelType: @@ -79,3 +79,13 @@ def create_fake_value(param_type: BaseType) -> Any: model_type = None resolved_type = get_sub_type(model_type) if model_type else param_type return json_dumps_template(resolved_type.get_json_template_representation()) + + +def hash_file_import(file_import: FileImport) -> str: + """Generate a hash for a FileImport object based on its imports. + + :param file_import: The FileImport object to generate a hash for. + :return: A string representing the hash of the FileImport object. + """ + + return "".join(sorted(list(set([str(hash(i)) for i in file_import.imports])))) From 10c1a8b0829c00f9b002679c9fbb89ee515f2757 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Thu, 29 Jan 2026 06:46:55 +0000 Subject: [PATCH 13/18] add changelog --- ...ample-test-generation-optimization-2026-0-29-6-46-35.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .chronus/changes/python-sample-test-generation-optimization-2026-0-29-6-46-35.md diff --git a/.chronus/changes/python-sample-test-generation-optimization-2026-0-29-6-46-35.md b/.chronus/changes/python-sample-test-generation-optimization-2026-0-29-6-46-35.md new file mode 100644 index 00000000000..490c7cbe940 --- /dev/null +++ b/.chronus/changes/python-sample-test-generation-optimization-2026-0-29-6-46-35.md @@ -0,0 +1,7 @@ +--- +changeKind: internal +packages: + - "@typespec/http-client-python" +--- + +Optimize sdk generation performance \ No newline at end of file From f280bbc158007ed2944cb0472442404f7fda0569 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Thu, 29 Jan 2026 06:50:03 +0000 Subject: [PATCH 14/18] update --- .../generator/pygen/codegen/serializers/sample_serializer.py | 2 +- .../generator/pygen/codegen/serializers/utils.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/sample_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/sample_serializer.py index 50102a750f5..733209b8396 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/sample_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/sample_serializer.py @@ -4,7 +4,7 @@ # license information. # -------------------------------------------------------------------------- import logging -from typing import Any, Optional, Union +from typing import Any, Union from jinja2 import Environment from ..models.operation import OperationBase diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/utils.py b/packages/http-client-python/generator/pygen/codegen/serializers/utils.py index 3dcb83105bb..8d37679ec56 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/utils.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/utils.py @@ -71,12 +71,12 @@ def create_fake_value(param_type: BaseType) -> Any: :param param_type: The parameter type to create a fake value for. :return: A string representation of the fake value. """ + + model_type: Optional[ModelType] = None if isinstance(param_type, ModelType): model_type = param_type elif isinstance(param_type, CombinedType): model_type = param_type.target_model_subtype((ModelType,)) - else: - model_type = None resolved_type = get_sub_type(model_type) if model_type else param_type return json_dumps_template(resolved_type.get_json_template_representation()) From 0fa93608653032ca1e10052de200951d01798bd5 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Thu, 29 Jan 2026 12:55:05 +0000 Subject: [PATCH 15/18] fix lint --- .../pygen/codegen/serializers/__init__.py | 79 ++++++++++++------- .../pygen/codegen/serializers/utils.py | 4 +- 2 files changed, 53 insertions(+), 30 deletions(-) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py index 412894809f1..15f4338cc41 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py @@ -533,6 +533,43 @@ def sample_additional_folder(self) -> Path: def _generated_tests_samples_folder(self, folder_name: str) -> Path: return self._root_of_sdk / folder_name + def _process_operation_samples( + self, + samples: dict, + env: Environment, + op_group, + operation, + import_sample_cache: dict[tuple[str, str], str], + out_path: Path, + sample_additional_folder: Path, + files_to_write: list[tuple[Path, str]], + ) -> None: + """Process samples for a single operation.""" + for sample_value in samples.values(): + file = sample_value.get("x-ms-original-file", "sample.json") + file_name = to_snake_case(extract_sample_name(file)) + ".py" + try: + sample_ser = SampleSerializer( + code_model=self.code_model, + env=env, + operation_group=op_group, + operation=operation, + sample=sample_value, + file_name=file_name, + ) + file_import = sample_ser.get_file_import() + imports_hash_string = hash_file_import(file_import) + cache_key = (op_group.client.client_namespace, imports_hash_string) + if cache_key not in import_sample_cache: + import_sample_cache[cache_key] = sample_ser.get_imports_from_file_import(file_import) + sample_ser.imports = import_sample_cache[cache_key] + + content = sample_ser.serialize() + output_path = out_path / sample_additional_folder / _sample_output_path(file) / file_name + files_to_write.append((output_path, content)) + except Exception as e: # pylint: disable=broad-except + _LOGGER.error("error happens in sample %s: %s", file, e) + def _serialize_and_write_sample(self, env: Environment): out_path = self._generated_tests_samples_folder("generated_samples") sample_additional_folder = self.sample_additional_folder @@ -548,30 +585,16 @@ def _serialize_and_write_sample(self, env: Environment): samples = operation.yaml_data.get("samples") if not samples or operation.name.startswith("_"): continue - for sample_value in samples.values(): - file = sample_value.get("x-ms-original-file", "sample.json") - file_name = to_snake_case(extract_sample_name(file)) + ".py" - try: - sample_ser = SampleSerializer( - code_model=self.code_model, - env=env, - operation_group=op_group, - operation=operation, - sample=sample_value, - file_name=file_name, - ) - file_import = sample_ser.get_file_import() - imports_hash_string = hash_file_import(file_import) - cache_key = (op_group.client.client_namespace, imports_hash_string) - if cache_key not in import_sample_cache: - import_sample_cache[cache_key] = sample_ser.get_imports_from_file_import(file_import) - sample_ser.imports = import_sample_cache[cache_key] - - content = sample_ser.serialize() - output_path = out_path / sample_additional_folder / _sample_output_path(file) / file_name - files_to_write.append((output_path, content)) - except Exception as e: # pylint: disable=broad-except - _LOGGER.error(f"error happens in sample {file}: {e}") + self._process_operation_samples( + samples, + env, + op_group, + operation, + import_sample_cache, + out_path, + sample_additional_folder, + files_to_write, + ) # Phase 2: Write all files (I/O-bound, threading helps here) def write_single_file(item: tuple[Path, str]) -> None: @@ -609,8 +632,8 @@ def _serialize_and_write_test(self, env: Environment): for og in client.operation_groups: # Create serializer once per operation group test_serializer = TestSerializer(self.code_model, env, client=client, operation_group=og) - for async_mode in (True, False): - try: + try: + for async_mode in (True, False): test_serializer.async_mode = async_mode cache_key = (client.name, async_mode) if cache_key not in import_test_cache: @@ -619,8 +642,8 @@ def _serialize_and_write_test(self, env: Environment): content = test_serializer.serialize_test() output_path = out_path / f"{to_snake_case(test_serializer.test_class_name)}.py" files_to_write.append((output_path, content)) - except Exception as e: # pylint: disable=broad-except - _LOGGER.error(f"error happens in test generation for operation group {og.class_name}: {e}") + except Exception as e: # pylint: disable=broad-except + _LOGGER.error("error happens in test generation for operation group %s: %s", og.class_name, e) # Phase 2: Write all files (I/O-bound, threading helps here) def write_single_file(item: tuple[Path, str]) -> None: diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/utils.py b/packages/http-client-python/generator/pygen/codegen/serializers/utils.py index 8d37679ec56..52ee4d62e57 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/utils.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/utils.py @@ -71,7 +71,7 @@ def create_fake_value(param_type: BaseType) -> Any: :param param_type: The parameter type to create a fake value for. :return: A string representation of the fake value. """ - + model_type: Optional[ModelType] = None if isinstance(param_type, ModelType): model_type = param_type @@ -88,4 +88,4 @@ def hash_file_import(file_import: FileImport) -> str: :return: A string representing the hash of the FileImport object. """ - return "".join(sorted(list(set([str(hash(i)) for i in file_import.imports])))) + return "".join(sorted({str(hash(i)) for i in file_import.imports})) From 4947a96f2b1ea245de3d1d3622502e5ab02bef56 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Thu, 29 Jan 2026 13:49:45 +0000 Subject: [PATCH 16/18] fix ci --- .../pygen/codegen/serializers/__init__.py | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py index 15f4338cc41..f2371188be6 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py @@ -5,7 +5,6 @@ # -------------------------------------------------------------------------- import logging import json -import concurrent.futures from collections import namedtuple import re from typing import Any, Optional, Union @@ -596,14 +595,10 @@ def _serialize_and_write_sample(self, env: Environment): files_to_write, ) - # Phase 2: Write all files (I/O-bound, threading helps here) - def write_single_file(item: tuple[Path, str]) -> None: - path, content = item + # Phase 2: Write all files + for path, content in files_to_write: self.write_file(path, content) - with concurrent.futures.ThreadPoolExecutor() as executor: - executor.map(write_single_file, files_to_write) - def _serialize_and_write_test(self, env: Environment): self.code_model.for_test = True out_path = self._generated_tests_samples_folder("generated_tests") @@ -645,12 +640,8 @@ def _serialize_and_write_test(self, env: Environment): except Exception as e: # pylint: disable=broad-except _LOGGER.error("error happens in test generation for operation group %s: %s", og.class_name, e) - # Phase 2: Write all files (I/O-bound, threading helps here) - def write_single_file(item: tuple[Path, str]) -> None: - path, content = item + # Phase 2: Write all files + for path, content in files_to_write: self.write_file(path, content) - with concurrent.futures.ThreadPoolExecutor() as executor: - executor.map(write_single_file, files_to_write) - self.code_model.for_test = False From 56bbcbdd76aac323e992392122c3fc1e382ca279 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Fri, 30 Jan 2026 02:36:24 +0000 Subject: [PATCH 17/18] review --- .../pygen/codegen/serializers/__init__.py | 29 ++++--------------- 1 file changed, 6 insertions(+), 23 deletions(-) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py index f2371188be6..6a16ed93e48 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py @@ -541,7 +541,6 @@ def _process_operation_samples( import_sample_cache: dict[tuple[str, str], str], out_path: Path, sample_additional_folder: Path, - files_to_write: list[tuple[Path, str]], ) -> None: """Process samples for a single operation.""" for sample_value in samples.values(): @@ -565,7 +564,7 @@ def _process_operation_samples( content = sample_ser.serialize() output_path = out_path / sample_additional_folder / _sample_output_path(file) / file_name - files_to_write.append((output_path, content)) + self.write_file(output_path, content) except Exception as e: # pylint: disable=broad-except _LOGGER.error("error happens in sample %s: %s", file, e) @@ -573,8 +572,6 @@ def _serialize_and_write_sample(self, env: Environment): out_path = self._generated_tests_samples_folder("generated_samples") sample_additional_folder = self.sample_additional_folder - # Phase 1: Generate all content (CPU-bound) - files_to_write: list[tuple[Path, str]] = [] # Cache import_test per (client_namespace, imports_hash_string) since it's expensive to compute import_sample_cache: dict[tuple[str, str], str] = {} @@ -592,32 +589,22 @@ def _serialize_and_write_sample(self, env: Environment): import_sample_cache, out_path, sample_additional_folder, - files_to_write, ) - # Phase 2: Write all files - for path, content in files_to_write: - self.write_file(path, content) - def _serialize_and_write_test(self, env: Environment): self.code_model.for_test = True out_path = self._generated_tests_samples_folder("generated_tests") - # Phase 1: Generate all content (CPU-bound) - files_to_write: list[tuple[Path, str]] = [] - general_serializer = TestGeneralSerializer(code_model=self.code_model, env=env) - files_to_write.append((out_path / "conftest.py", general_serializer.serialize_conftest())) + self.write_file(out_path / "conftest.py", general_serializer.serialize_conftest()) if not self.code_model.options["azure-arm"]: for async_mode in (True, False): async_suffix = "_async" if async_mode else "" general_serializer.async_mode = async_mode - files_to_write.append( - ( - out_path / f"testpreparer{async_suffix}.py", - general_serializer.serialize_testpreparer(), - ) + self.write_file( + out_path / f"testpreparer{async_suffix}.py", + general_serializer.serialize_testpreparer(), ) # Generate test files - reuse serializer per operation group, toggle async_mode @@ -636,12 +623,8 @@ def _serialize_and_write_test(self, env: Environment): test_serializer.import_test = import_test_cache[cache_key] content = test_serializer.serialize_test() output_path = out_path / f"{to_snake_case(test_serializer.test_class_name)}.py" - files_to_write.append((output_path, content)) + self.write_file(output_path, content) except Exception as e: # pylint: disable=broad-except _LOGGER.error("error happens in test generation for operation group %s: %s", og.class_name, e) - # Phase 2: Write all files - for path, content in files_to_write: - self.write_file(path, content) - self.code_model.for_test = False From d2495e43ccb2c8b2e4b8c598e2353883399f940f Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Fri, 30 Jan 2026 04:19:33 +0000 Subject: [PATCH 18/18] fix black exclude pattern --- packages/http-client-python/emitter/src/emitter.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/http-client-python/emitter/src/emitter.ts b/packages/http-client-python/emitter/src/emitter.ts index 9534c1ec99f..2916a55f6d4 100644 --- a/packages/http-client-python/emitter/src/emitter.ts +++ b/packages/http-client-python/emitter/src/emitter.ts @@ -239,7 +239,7 @@ async function onEmitMain(context: EmitContext) { ".mypy_cache", ".pytest_cache", ".vscode", - "_build", + ".*_build/", "/build/", "dist", ".nox",