From 131b8441443ec2560c5b54ff410fd88406bc582a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 29 Apr 2026 10:26:12 +0000 Subject: [PATCH 1/5] Restore client default value materialization in Model.__init__ Agent-Logs-Url: https://github.com/microsoft/typespec/sessions/91e8124d-723d-42e3-a606-d2e9fa8cd443 Co-authored-by: msyyc <70930885+msyyc@users.noreply.github.com> --- ...lient-default-value-serialization-2026-4-29-10-23-0.md | 7 +++++++ .../pygen/codegen/templates/model_base.py.jinja2 | 8 ++++++++ 2 files changed, 15 insertions(+) create mode 100644 .chronus/changes/python-fix-client-default-value-serialization-2026-4-29-10-23-0.md diff --git a/.chronus/changes/python-fix-client-default-value-serialization-2026-4-29-10-23-0.md b/.chronus/changes/python-fix-client-default-value-serialization-2026-4-29-10-23-0.md new file mode 100644 index 00000000000..3465d555a03 --- /dev/null +++ b/.chronus/changes/python-fix-client-default-value-serialization-2026-4-29-10-23-0.md @@ -0,0 +1,7 @@ +--- +changeKind: fix +packages: + - "@typespec/http-client-python" +--- + +Fix serialization regression where `@clientDefaultValue` defaults on model properties were no longer included in the request body. Defaults are again materialized in the model's data dictionary at construction time so they are sent on the wire, while the attribute-access fallback for unset fields is preserved. diff --git a/packages/http-client-python/generator/pygen/codegen/templates/model_base.py.jinja2 b/packages/http-client-python/generator/pygen/codegen/templates/model_base.py.jinja2 index 258b741a8dd..19f8f222acf 100644 --- a/packages/http-client-python/generator/pygen/codegen/templates/model_base.py.jinja2 +++ b/packages/http-client-python/generator/pygen/codegen/templates/model_base.py.jinja2 @@ -642,6 +642,14 @@ class Model(_MyMutableMapping): if v is not None } ) + # Apply client default values for fields the caller didn't set so that + # defaults are part of `_data` and therefore included during serialization. + for rf in self._attr_to_rest_field.values(): + if rf._default is _UNSET: + continue + if rf._rest_name in dict_to_pass: + continue + dict_to_pass[rf._rest_name] = _create_value(rf, rf._default) super().__init__(dict_to_pass) def _init_from_xml(self, element: ET.Element) -> dict[str, typing.Any]: From f56d56ffac0e837d169b1278a1fb2d88c36e4784 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Wed, 6 May 2026 07:22:52 +0000 Subject: [PATCH 2/5] update spector dep --- packages/http-client-python/package-lock.json | 28 +++++++++---------- packages/http-client-python/package.json | 2 +- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/packages/http-client-python/package-lock.json b/packages/http-client-python/package-lock.json index a6129262122..ff3a016fe9e 100644 --- a/packages/http-client-python/package-lock.json +++ b/packages/http-client-python/package-lock.json @@ -17,7 +17,7 @@ "tsx": "^4.21.0" }, "devDependencies": { - "@azure-tools/azure-http-specs": "0.1.0-alpha.39", + "@azure-tools/azure-http-specs": "0.1.0-alpha.40-dev.12", "@azure-tools/typespec-autorest": "~0.67.0", "@azure-tools/typespec-azure-core": "~0.67.0", "@azure-tools/typespec-azure-resource-manager": "~0.67.0", @@ -66,25 +66,25 @@ } }, "node_modules/@azure-tools/azure-http-specs": { - "version": "0.1.0-alpha.39", - "resolved": "https://registry.npmjs.org/@azure-tools/azure-http-specs/-/azure-http-specs-0.1.0-alpha.39.tgz", - "integrity": "sha512-l9d2Y+B7QBi20ocDJEGO7NpvpCePvdw2ALz1RHAPWBOD0tPUBXUQ4WF1zUC199awz8hQysNRM9jm8x+eoUvjEQ==", + "version": "0.1.0-alpha.40-dev.12", + "resolved": "https://registry.npmjs.org/@azure-tools/azure-http-specs/-/azure-http-specs-0.1.0-alpha.40-dev.12.tgz", + "integrity": "sha512-N2d7IVgA5tw4yE2FYWt6vz55TcNcQgdReFBELvbxQhKmsDQZFwc2lh6euVC2P/MpSkeXMXJMnnkuJ1KH/DLxFA==", "dev": true, "license": "MIT", "dependencies": { - "@typespec/spec-api": "^0.1.0-alpha.14", - "@typespec/spector": "^0.1.0-alpha.25" + "@typespec/spec-api": "^0.1.0-alpha.14 || >=0.1.0-alpha.15-dev <0.1.0-alpha.15", + "@typespec/spector": "^0.1.0-alpha.25 || >=0.1.0-alpha.26-dev <0.1.0-alpha.26" }, "engines": { - "node": ">=20.0.0" + "node": ">=22.0.0" }, "peerDependencies": { - "@azure-tools/typespec-azure-core": "^0.67.0", + "@azure-tools/typespec-azure-core": "^0.67.1 || >=0.68.0-dev <0.68.0", "@typespec/compiler": "^1.11.0", "@typespec/http": "^1.11.0", - "@typespec/rest": "^0.81.0", - "@typespec/versioning": "^0.81.0", - "@typespec/xml": "^0.81.0" + "@typespec/rest": "^0.81.0 || >=0.82.0-dev <0.82.0", + "@typespec/versioning": "^0.81.0 || >=0.82.0-dev <0.82.0", + "@typespec/xml": "^0.81.0 || >=0.82.0-dev <0.82.0" } }, "node_modules/@azure-tools/typespec-autorest": { @@ -114,9 +114,9 @@ } }, "node_modules/@azure-tools/typespec-azure-core": { - "version": "0.67.0", - "resolved": "https://registry.npmjs.org/@azure-tools/typespec-azure-core/-/typespec-azure-core-0.67.0.tgz", - "integrity": "sha512-6DO/fOlVihMlPG0oDXrgURf5MNF4iBzPx5SMA5aaFDx/fW6MjiD+TN9Yy9O+l9mVNh1XaEMjhjA8/lmnHZ/U0g==", + "version": "0.67.1", + "resolved": "https://registry.npmjs.org/@azure-tools/typespec-azure-core/-/typespec-azure-core-0.67.1.tgz", + "integrity": "sha512-HBvigwr8Ub7rsg4RDpTO3WTHS+CIqAw32X3RzxsDNb8NfLoSLSZDANz05VPiDTzAXO7eMfEP42RXkKPV0tlZLg==", "dev": true, "license": "MIT", "engines": { diff --git a/packages/http-client-python/package.json b/packages/http-client-python/package.json index 644950c2daf..6c4dfa859d1 100644 --- a/packages/http-client-python/package.json +++ b/packages/http-client-python/package.json @@ -103,7 +103,7 @@ "@azure-tools/typespec-azure-resource-manager": "~0.67.0", "@azure-tools/typespec-azure-rulesets": "~0.67.0", "@azure-tools/typespec-client-generator-core": "~0.67.0", - "@azure-tools/azure-http-specs": "0.1.0-alpha.39", + "@azure-tools/azure-http-specs": "0.1.0-alpha.40-dev.12", "@typespec/compiler": "^1.11.0", "@typespec/http": "^1.11.0", "@typespec/openapi": "^1.11.0", From 899d204893018f077b8fa466eb1c2e0ecba01c23 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Fri, 8 May 2026 15:54:35 +0800 Subject: [PATCH 3/5] extract regenerate-common.ts --- .../eng/scripts/ci/regenerate-common.ts | 746 ++++++++++++++++++ .../eng/scripts/ci/regenerate.ts | 716 +---------------- 2 files changed, 779 insertions(+), 683 deletions(-) create mode 100644 packages/http-client-python/eng/scripts/ci/regenerate-common.ts diff --git a/packages/http-client-python/eng/scripts/ci/regenerate-common.ts b/packages/http-client-python/eng/scripts/ci/regenerate-common.ts new file mode 100644 index 00000000000..58d1a3d8c23 --- /dev/null +++ b/packages/http-client-python/eng/scripts/ci/regenerate-common.ts @@ -0,0 +1,746 @@ +/* eslint-disable no-console */ +/** + * Shared helpers, types, constants, and data tables used by `regenerate.ts`. + * + * This file is meant to be **byte-identical** between this package and the + * upstream `@typespec/http-client-python`. typespec-python syncs it from + * /core/packages/http-client-python/eng/scripts/ci/regenerate-common.ts + * via `pnpm sync`. + * + * Per-repo divergence (paths, emitter name, single-phase vs two-phase + * orchestration, argv/help text) lives in each repo's own `regenerate.ts`, + * which builds a `RegenerateContext` and feeds it into the helpers exported + * from this module. + */ + +import { compile, NodeHost } from "@typespec/compiler"; +import { execSync } from "child_process"; +import { existsSync, rmSync } from "fs"; +import { access, cp, mkdir, mkdtemp, readdir, writeFile } from "fs/promises"; +import { tmpdir } from "os"; +import { dirname, join, relative, resolve } from "path"; +import pc from "picocolors"; + +// ---- Public types ---- + +export interface RegenerateFlags { + flavor: string; + debug: boolean; + name?: string; +} + +export interface CompileTask { + spec: string; + outputDir: string; + options: Record; +} + +// Group of tasks for the same spec that must run sequentially +export interface TaskGroup { + spec: string; + tasks: CompileTask[]; +} + +/** + * Per-repo context injected into the helpers below. Every value is repo + * specific and must be supplied by the caller's `regenerate.ts`. + */ +export interface RegenerateContext { + /** Absolute path to the package root (the dir containing `package.json`). */ + pluginDir: string; + /** Absolute path to the azure-http-specs `specs/` dir. */ + azureHttpSpecs: string; + /** Absolute path to the http-specs `specs/` dir. */ + httpSpecs: string; + /** Absolute path to where generated SDKs should go (e.g. `/generator`). */ + generatedFolder: string; + /** Emitter name to invoke (e.g. `@azure-tools/typespec-python`). */ + emitterName: string; +} + +/** + * Optional knobs for `buildTaskGroups`. Kept here so the call site in each + * repo's `regenerate.ts` can opt into the upstream two-phase pipeline + * (`emitYamlOnly: true`) or the single-phase pipeline (default). + */ +export interface BuildTaskGroupsOptions { + /** If true, ask the emitter to write YAML only and skip Python codegen. */ + emitYamlOnly?: boolean; +} + +// ---- Public constants ---- + +export const SKIP_SPECS: string[] = ["type/file", "service/multiple-services"]; + +export const SpecialFlags: Record> = { + azure: { + "generate-test": true, + "generate-sample": true, + }, +}; + +// ---- Spec-specific emitter option overrides ---- + +export const AZURE_EMITTER_OPTIONS: Record< + string, + Record | Record[] +> = { + "azure/client-generator-core/access": { + namespace: "specs.azure.clientgenerator.core.access", + }, + "azure/client-generator-core/alternate-type": { + namespace: "specs.azure.clientgenerator.core.alternatetype", + }, + "azure/client-generator-core/api-version": { + namespace: "specs.azure.clientgenerator.core.apiversion", + }, + "azure/client-generator-core/client-initialization/default": { + namespace: "specs.azure.clientgenerator.core.clientinitialization.default", + }, + "azure/client-generator-core/client-initialization/individually": { + namespace: "specs.azure.clientgenerator.core.clientinitialization.individually", + }, + "azure/client-generator-core/client-initialization/individuallyParent": { + namespace: "specs.azure.clientgenerator.core.clientinitialization.individuallyparent", + }, + "azure/client-generator-core/client-location": { + namespace: "specs.azure.clientgenerator.core.clientlocation", + }, + "azure/client-generator-core/deserialize-empty-string-as-null": { + namespace: "specs.azure.clientgenerator.core.emptystring", + }, + "azure/client-generator-core/flatten-property": { + namespace: "specs.azure.clientgenerator.core.flattenproperty", + }, + "azure/client-generator-core/usage": { + namespace: "specs.azure.clientgenerator.core.usage", + }, + "azure/client-generator-core/client-doc": { + namespace: "specs.azure.clientgenerator.core.clientdoc", + }, + "azure/client-generator-core/override": { + namespace: "specs.azure.clientgenerator.core.override", + }, + "azure/client-generator-core/hierarchy-building": { + namespace: "specs.azure.clientgenerator.core.hierarchybuilding", + }, + "azure/core/basic": { + namespace: "specs.azure.core.basic", + }, + "azure/core/lro/rpc": { + namespace: "specs.azure.core.lro.rpc", + }, + "azure/core/lro/standard": { + namespace: "specs.azure.core.lro.standard", + }, + "azure/core/model": { + namespace: "specs.azure.core.model", + }, + "azure/core/page": { + namespace: "specs.azure.core.page", + }, + "azure/core/scalar": { + namespace: "specs.azure.core.scalar", + }, + "azure/core/traits": { + namespace: "specs.azure.core.traits", + }, + "azure/encode/duration": { + namespace: "specs.azure.encode.duration", + }, + "azure/example/basic": { + namespace: "specs.azure.example.basic", + }, + "azure/payload/pageable": { + namespace: "specs.azure.payload.pageable", + }, + "azure/versioning/previewVersion": { + namespace: "specs.azure.versioning.previewversion", + }, + "client/structure/default": { + namespace: "client.structure.service", + }, + "client/structure/multi-client": { + "package-name": "client-structure-multiclient", + namespace: "client.structure.multiclient", + }, + "client/structure/renamed-operation": { + "package-name": "client-structure-renamedoperation", + namespace: "client.structure.renamedoperation", + }, + "client/structure/two-operation-group": { + "package-name": "client-structure-twooperationgroup", + namespace: "client.structure.twooperationgroup", + }, + "client/naming": { + namespace: "client.naming.main", + }, + "client/overload": { + namespace: "client.overload", + }, + "encode/duration": { + namespace: "encode.duration", + }, + "encode/numeric": { + namespace: "encode.numeric", + }, + "parameters/basic": { + namespace: "parameters.basic", + }, + "parameters/spread": { + namespace: "parameters.spread", + }, + "payload/content-negotiation": { + namespace: "payload.contentnegotiation", + }, + "payload/multipart": { + namespace: "payload.multipart", + }, + "serialization/encoded-name/json": { + namespace: "serialization.encodedname.json", + }, + "special-words": { + namespace: "specialwords", + }, + "service/multi-service": { + namespace: "service.multiservice", + }, + "client/structure/client-operation-group": { + "package-name": "client-structure-clientoperationgroup", + namespace: "client.structure.clientoperationgroup", + }, +}; + +export const EMITTER_OPTIONS: Record< + string, + Record | Record[] +> = { + "resiliency/srv-driven/old.tsp": { + "package-name": "resiliency-srv-driven1", + namespace: "resiliency.srv.driven1", + "package-mode": "azure-dataplane", + "package-pprint-name": "ResiliencySrvDriven1", + }, + "resiliency/srv-driven": { + "package-name": "resiliency-srv-driven2", + namespace: "resiliency.srv.driven2", + "package-mode": "azure-dataplane", + "package-pprint-name": "ResiliencySrvDriven2", + }, + "authentication/api-key": { + "clear-output-folder": "true", + }, + "authentication/http/custom": { + "package-name": "authentication-http-custom", + namespace: "authentication.http.custom", + "package-pprint-name": "Authentication Http Custom", + }, + "authentication/union": [ + { + "package-name": "authentication-union", + namespace: "authentication.union", + }, + { + "package-name": "setuppy-authentication-union", + namespace: "setuppy.authentication.union", + "keep-setup-py": "true", + }, + ], + "type/array": { + "package-name": "typetest-array", + namespace: "typetest.array", + }, + "type/dictionary": { + "package-name": "typetest-dictionary", + namespace: "typetest.dictionary", + }, + "type/enum/extensible": { + "package-name": "typetest-enum-extensible", + namespace: "typetest.enum.extensible", + }, + "type/enum/fixed": { + "package-name": "typetest-enum-fixed", + namespace: "typetest.enum.fixed", + }, + "type/model/empty": { + "package-name": "typetest-model-empty", + namespace: "typetest.model.empty", + }, + "type/model/inheritance/enum-discriminator": { + "package-name": "typetest-model-enumdiscriminator", + namespace: "typetest.model.enumdiscriminator", + }, + "type/model/inheritance/nested-discriminator": { + "package-name": "typetest-model-nesteddiscriminator", + namespace: "typetest.model.nesteddiscriminator", + }, + "type/model/inheritance/not-discriminated": { + "package-name": "typetest-model-notdiscriminated", + namespace: "typetest.model.notdiscriminated", + }, + "type/model/inheritance/single-discriminator": { + "package-name": "typetest-model-singlediscriminator", + namespace: "typetest.model.singlediscriminator", + }, + "type/model/inheritance/recursive": [ + { + "package-name": "typetest-model-recursive", + namespace: "typetest.model.recursive", + }, + { + "package-name": "generation-subdir", + namespace: "generation.subdir", + "generation-subdir": "_generated", + "generate-test": "false", + "clear-output-folder": "true", + }, + ], + "type/model/usage": { + "package-name": "typetest-model-usage", + namespace: "typetest.model.usage", + }, + "type/model/visibility": [ + { + "package-name": "typetest-model-visibility", + namespace: "typetest.model.visibility", + }, + { + "package-name": "headasbooleantrue", + namespace: "headasbooleantrue", + "head-as-boolean": "true", + }, + { + "package-name": "headasbooleanfalse", + namespace: "headasbooleanfalse", + "head-as-boolean": "false", + }, + ], + "type/property/nullable": { + "package-name": "typetest-property-nullable", + namespace: "typetest.property.nullable", + }, + "type/property/optionality": { + "package-name": "typetest-property-optional", + namespace: "typetest.property.optional", + }, + "type/property/additional-properties": { + "package-name": "typetest-property-additionalproperties", + namespace: "typetest.property.additionalproperties", + }, + "type/scalar": { + "package-name": "typetest-scalar", + namespace: "typetest.scalar", + }, + "type/property/value-types": { + "package-name": "typetest-property-valuetypes", + namespace: "typetest.property.valuetypes", + }, + "type/union": { + "package-name": "typetest-union", + namespace: "typetest.union", + }, + "type/union/discriminated": { + "package-name": "typetest-discriminatedunion", + namespace: "typetest.discriminatedunion", + }, + "type/file": { + "package-name": "typetest-file", + namespace: "typetest.file", + }, + documentation: { + "package-name": "specs-documentation", + namespace: "specs.documentation", + }, + "versioning/added": [ + { + "package-name": "versioning-added", + namespace: "versioning.added", + }, + { + "package-name": "generation-subdir2", + namespace: "generation.subdir2", + "generate-test": "false", + "generation-subdir": "_generated", + }, + ], +}; + +// ---- Public helpers ---- + +export function toPosix(p: string): string { + return p.replace(/\\/g, "/"); +} + +/** + * Whether a spec path belongs to azure-http-specs (vs standard http-specs). + * Uses the `azure-http-specs` substring rather than `azure` to avoid false + * positives when the working-dir path itself contains "azure" (e.g. + * azure-sdk-for-python). + */ +export function isAzureSpec(spec: string): boolean { + return spec.includes("azure-http-specs"); +} + +export function defaultPackageName(spec: string, ctx: RegenerateContext): string { + const specDir = isAzureSpec(spec) ? ctx.azureHttpSpecs : ctx.httpSpecs; + return toPosix(relative(specDir, dirname(spec))) + .replace(/\//g, "-") + .toLowerCase(); +} + +export function getEmitterOptions( + spec: string, + flavor: string, + ctx: RegenerateContext, +): Record[] { + const specDir = isAzureSpec(spec) ? ctx.azureHttpSpecs : ctx.httpSpecs; + const relativeSpec = toPosix(relative(specDir, spec)); + const key = relativeSpec.includes("resiliency/srv-driven/old.tsp") + ? relativeSpec + : dirname(relativeSpec); + const emitterOpts = + EMITTER_OPTIONS[key] || (flavor === "azure" ? AZURE_EMITTER_OPTIONS[key] : [{}]) || [{}]; + return Array.isArray(emitterOpts) ? emitterOpts : [emitterOpts]; +} + +/** + * Walk `baseDir` and collect every TypeSpec entry-point file that the + * regenerator should compile (handles `client.tsp`, `main.tsp`, and the + * special `resiliency/srv-driven/old.tsp` case). + */ +export async function getSubdirectories( + baseDir: string, + flags: RegenerateFlags, +): Promise { + const subdirectories: string[] = []; + + async function searchDir(currentDir: string) { + const items = await readdir(currentDir, { withFileTypes: true }); + + const promisesArray = items.map(async (item) => { + const subDirPath = join(currentDir, item.name); + if (item.isDirectory()) { + const mainTspPath = join(subDirPath, "main.tsp"); + const clientTspPath = join(subDirPath, "client.tsp"); + + const mainTspRelativePath = toPosix(relative(baseDir, mainTspPath)); + + if (SKIP_SPECS.some((skipSpec) => mainTspRelativePath.includes(skipSpec))) return; + + const hasMainTsp = await access(mainTspPath) + .then(() => true) + .catch(() => false); + const hasClientTsp = await access(clientTspPath) + .then(() => true) + .catch(() => false); + + if (mainTspRelativePath.toLowerCase().includes(flags.name || "")) { + if (mainTspRelativePath.includes("resiliency/srv-driven")) { + subdirectories.push(resolve(subDirPath, "old.tsp")); + } + if (hasClientTsp) { + subdirectories.push(resolve(subDirPath, "client.tsp")); + } else if (hasMainTsp) { + subdirectories.push(resolve(subDirPath, "main.tsp")); + } + } + + await searchDir(subDirPath); + } + }); + + await Promise.all(promisesArray); + } + + await searchDir(baseDir); + return subdirectories; +} + +export function buildTaskGroups( + specs: string[], + flags: RegenerateFlags, + ctx: RegenerateContext, + options: BuildTaskGroupsOptions = {}, +): TaskGroup[] { + const groups: TaskGroup[] = []; + + for (const spec of specs) { + const tasks: CompileTask[] = []; + + for (const emitterConfig of getEmitterOptions(spec, flags.flavor, ctx)) { + // Apply flavor defaults first, then per-spec options so they can override + // (e.g. "generate-test": "false") + const opts: Record = {}; + for (const [k, v] of Object.entries(SpecialFlags[flags.flavor] ?? {})) { + opts[k] = v; + } + Object.assign(opts, emitterConfig); + + opts["flavor"] = flags.flavor; + + // Set output directory - tests/generated// structure. + // Always anchored at /../tests/generated regardless of + // pluginDir, so generator-only checkouts work too. + const packageName = (opts["package-name"] as string) || defaultPackageName(spec, ctx); + const outputDir = + (opts["emitter-output-dir"] as string) || + toPosix(`${ctx.generatedFolder}/../tests/generated/${flags.flavor}/${packageName}`); + opts["emitter-output-dir"] = outputDir; + + if (flags.debug) { + opts["debug"] = true; + } + + opts["examples-dir"] = toPosix(join(dirname(spec), "examples")); + + if (options.emitYamlOnly) { + // Emit YAML only - Python processing is batched after all specs compile. + opts["emit-yaml-only"] = true; + } + + tasks.push({ spec, outputDir, options: opts }); + } + + groups.push({ spec, tasks }); + } + + return groups; +} + +export async function compileSpec( + task: CompileTask, + ctx: RegenerateContext, +): Promise<{ success: boolean; error?: string }> { + const { spec, outputDir, options } = task; + + try { + const compilerOptions = { + emit: [ctx.pluginDir], + options: { + [ctx.emitterName]: options, + }, + }; + + const program = await compile(NodeHost, spec, compilerOptions); + + if (program.hasError()) { + const errors = program.diagnostics + .filter((d) => d.severity === "error") + .map((d) => d.message) + .join("\n"); + return { success: false, error: errors }; + } + + return { success: true }; + } catch (err) { + rmSync(outputDir, { recursive: true, force: true }); + return { success: false, error: String(err) }; + } +} + +export function renderProgressBar( + completed: number, + failed: number, + total: number, + width: number = 40, +): string { + const successCount = completed - failed; + const successWidth = Math.round((successCount / total) * width); + const failWidth = Math.round((failed / total) * width); + const emptyWidth = width - successWidth - failWidth; + + const successBar = pc.bgGreen(" ".repeat(successWidth)); + const failBar = failed > 0 ? pc.bgRed(" ".repeat(failWidth)) : ""; + const emptyBar = pc.dim("░".repeat(Math.max(0, emptyWidth))); + + const percent = Math.round((completed / total) * 100); + return `${successBar}${failBar}${emptyBar} ${pc.cyan(`${percent}%`)} (${completed}/${total})`; +} + +export async function runParallel( + groups: TaskGroup[], + maxJobs: number, + ctx: RegenerateContext, +): Promise> { + const results = new Map(); + const executing: Set> = new Set(); + + const totalTasks = groups.reduce((sum, g) => sum + g.tasks.length, 0); + let completed = 0; + let failed = 0; + const failedSpecs: string[] = []; + + const isTTY = process.stdout.isTTY; + + const updateProgress = () => { + if (isTTY) { + process.stdout.write(`\r${renderProgressBar(completed, failed, totalTasks)}`); + } + }; + + updateProgress(); + + for (const group of groups) { + // Each group runs as a unit - tasks within a group run sequentially + // to avoid state pollution. Different groups run in parallel. + const runGroup = async () => { + const specDir = isAzureSpec(group.spec) ? ctx.azureHttpSpecs : ctx.httpSpecs; + const shortName = toPosix(relative(specDir, dirname(group.spec))); + + let groupSuccess = true; + for (const task of group.tasks) { + const packageName = (task.options["package-name"] as string) || shortName; + + const result = await compileSpec(task, ctx); + completed++; + + if (!result.success) { + failed++; + failedSpecs.push(`${packageName}: ${result.error}`); + groupSuccess = false; + } + + updateProgress(); + } + + results.set(group.spec, groupSuccess); + }; + + const p = runGroup().finally(() => executing.delete(p)); + executing.add(p); + + if (executing.size >= maxJobs) { + await Promise.race(executing); + } + } + + await Promise.all(executing); + + if (isTTY) { + process.stdout.write("\r" + " ".repeat(60) + "\r"); + } + + if (failedSpecs.length > 0) { + console.log(pc.red(`\nFailed specs:`)); + for (const spec of failedSpecs) { + console.log(pc.red(` • ${spec}`)); + } + } + + return results; +} + +/** + * Pre-create the marker files that the test harness expects to find before + * regeneration so it can verify they're cleared/preserved correctly. + */ +export async function preprocess(flavor: string, generatedFolder: string): Promise { + if (flavor !== "azure") return; + + const testsGeneratedDir = resolve(generatedFolder, "../tests/generated/azure"); + + const DELETE_CONTENT = "# This file is to be deleted after regeneration"; + const KEEP_CONTENT = "# This file is to be kept after regeneration"; + const DELETE_FILE = "to_be_deleted.py"; + const entries: { folder: string[]; file: string; content: string }[] = [ + { + folder: ["authentication-api-key", "authentication", "apikey", "_operations"], + file: DELETE_FILE, + content: DELETE_CONTENT, + }, + { + folder: ["generation-subdir", "generation", "subdir", "_generated"], + file: DELETE_FILE, + content: DELETE_CONTENT, + }, + { + folder: ["generation-subdir", "generated_tests"], + file: DELETE_FILE, + content: DELETE_CONTENT, + }, + { + folder: ["generation-subdir", "generation", "subdir"], + file: "to_be_kept.py", + content: KEEP_CONTENT, + }, + ]; + + await Promise.all( + entries.map(async ({ folder, file, content }) => { + const targetFolder = join(testsGeneratedDir, ...folder); + await mkdir(targetFolder, { recursive: true }); + await writeFile(join(targetFolder, file), content); + }), + ); +} + +/** + * Resets the `tests/generated/{azure,unbranded}` baseline by sparse-checking-out + * `eng/tools/emitter/gen` from the Azure/azure-sdk-for-python repo, then + * deleting a couple of fully-generated package folders so regeneration has to + * recreate them from scratch (smoke test of full-emit path). + * + * `generatedFolder` is the per-repo `generator/` directory; baseline lands at + * `/../tests/generated`. + */ +export async function prepareBaselineOfGeneratedCode(generatedFolder: string): Promise { + const repoUrl = "https://github.com/Azure/azure-sdk-for-python.git"; + const branch = "main"; + const sourceSubdir = "eng/tools/emitter/gen"; + const testsGeneratedDir = resolve(generatedFolder, "../tests/generated"); + + console.log(pc.cyan(`\n${"=".repeat(60)}`)); + console.log(pc.cyan(`Resetting baseline from ${repoUrl} (${branch}/${sourceSubdir})`)); + console.log(pc.cyan(`${"=".repeat(60)}\n`)); + + // Wipe tests/generated + if (existsSync(testsGeneratedDir)) { + console.log(pc.dim(`Removing ${testsGeneratedDir}`)); + rmSync(testsGeneratedDir, { recursive: true, force: true }); + } + + // Sparse-checkout the baseline folder into a temp directory + const tempDir = await mkdtemp(join(tmpdir(), "azsdk-baseline-")); + try { + console.log(pc.dim(`Cloning into ${tempDir}`)); + const run = (cmd: string) => + execSync(cmd, { cwd: tempDir, stdio: ["ignore", "ignore", "inherit"] }); + + run(`git init`); + run(`git remote add origin ${repoUrl}`); + run(`git config core.sparseCheckout true`); + run(`git sparse-checkout init --cone`); + run(`git sparse-checkout set ${sourceSubdir}`); + run(`git fetch --depth 1 origin ${branch}`); + run(`git checkout FETCH_HEAD`); + + const sourceRoot = join(tempDir, ...sourceSubdir.split("/")); + for (const flavor of ["azure", "unbranded"]) { + const src = join(sourceRoot, flavor); + const dest = join(testsGeneratedDir, flavor); + if (!existsSync(src)) { + console.warn(pc.yellow(`Baseline folder not found: ${src}`)); + continue; + } + console.log(pc.dim(`Copying ${flavor}/ -> ${dest}`)); + await cp(src, dest, { recursive: true }); + } + + console.log(pc.green(`Baseline reset complete.\n`)); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + + // Delete a couple of fully-generated package folders so regeneration has to + // recreate them from scratch (smoke test of full-emit path). + const targetsToDelete = [ + join(testsGeneratedDir, "azure", "authentication-http-custom"), + join(testsGeneratedDir, "unbranded", "encode-array"), + ]; + for (const target of targetsToDelete) { + if (existsSync(target)) { + console.log(pc.dim(`Deleting ${target}`)); + rmSync(target, { recursive: true, force: true }); + } + } +} diff --git a/packages/http-client-python/eng/scripts/ci/regenerate.ts b/packages/http-client-python/eng/scripts/ci/regenerate.ts index 4620fa0f603..e6c8cee1110 100644 --- a/packages/http-client-python/eng/scripts/ci/regenerate.ts +++ b/packages/http-client-python/eng/scripts/ci/regenerate.ts @@ -2,20 +2,34 @@ /** * Regenerates Python SDK code from TypeSpec definitions. * - * Uses in-process TypeSpec compilation to avoid subprocess spawning overhead. - * This is significantly faster than spawning `tsp compile` for each spec. + * Two-phase pipeline: + * 1. TypeSpec compile (in-process, parallel) -> emits per-spec YAML only. + * 2. Single batched Python subprocess reads all YAMLs and writes the + * final `.py` files. Amortizes Python-startup cost across many specs. + * + * Shared helpers/data live in `regenerate-common.ts` (kept identical with the + * `@azure-tools/typespec-python` wrapper copy). */ -import { compile, NodeHost } from "@typespec/compiler"; import { execSync } from "child_process"; -import { existsSync, rmSync } from "fs"; -import { access, cp, mkdir, mkdtemp, readdir, writeFile } from "fs/promises"; -import { platform, tmpdir } from "os"; -import { dirname, join, relative, resolve } from "path"; +import { existsSync } from "fs"; +import { access, readdir } from "fs/promises"; +import { platform } from "os"; +import { dirname, join, resolve } from "path"; import pc from "picocolors"; import { fileURLToPath } from "url"; import { parseArgs } from "util"; +import { + buildTaskGroups, + getSubdirectories, + prepareBaselineOfGeneratedCode, + preprocess, + RegenerateContext, + RegenerateFlags, + runParallel, +} from "./regenerate-common.js"; + // Parse arguments const argv = parseArgs({ args: process.argv.slice(2), @@ -75,450 +89,6 @@ ${pc.bold("Examples:")} process.exit(0); } -// ---- Shared constants ---- - -const SKIP_SPECS: string[] = ["type/file", "service/multiple-services"]; - -const SpecialFlags: Record> = { - azure: { - "generate-test": true, - "generate-sample": true, - }, -}; - -function toPosix(dir: string): string { - return dir.replace(/\\/g, "/"); -} - -interface RegenerateFlags { - flavor: string; - debug: boolean; - name?: string; - pyodide?: boolean; -} - -// ---- Base emitter options ---- - -const AZURE_EMITTER_OPTIONS: Record | Record[]> = { - "azure/client-generator-core/access": { - namespace: "specs.azure.clientgenerator.core.access", - }, - "azure/client-generator-core/alternate-type": { - namespace: "specs.azure.clientgenerator.core.alternatetype", - }, - "azure/client-generator-core/api-version": { - namespace: "specs.azure.clientgenerator.core.apiversion", - }, - "azure/client-generator-core/client-initialization/default": { - namespace: "specs.azure.clientgenerator.core.clientinitialization.default", - }, - "azure/client-generator-core/client-initialization/individually": { - namespace: "specs.azure.clientgenerator.core.clientinitialization.individually", - }, - "azure/client-generator-core/client-initialization/individuallyParent": { - namespace: "specs.azure.clientgenerator.core.clientinitialization.individuallyparent", - }, - "azure/client-generator-core/client-location": { - namespace: "specs.azure.clientgenerator.core.clientlocation", - }, - "azure/client-generator-core/deserialize-empty-string-as-null": { - namespace: "specs.azure.clientgenerator.core.emptystring", - }, - "azure/client-generator-core/flatten-property": { - namespace: "specs.azure.clientgenerator.core.flattenproperty", - }, - "azure/client-generator-core/usage": { - namespace: "specs.azure.clientgenerator.core.usage", - }, - "azure/client-generator-core/client-doc": { - namespace: "specs.azure.clientgenerator.core.clientdoc", - }, - "azure/client-generator-core/override": { - namespace: "specs.azure.clientgenerator.core.override", - }, - "azure/client-generator-core/hierarchy-building": { - namespace: "specs.azure.clientgenerator.core.hierarchybuilding", - }, - "azure/core/basic": { - namespace: "specs.azure.core.basic", - }, - "azure/core/lro/rpc": { - namespace: "specs.azure.core.lro.rpc", - }, - "azure/core/lro/standard": { - namespace: "specs.azure.core.lro.standard", - }, - "azure/core/model": { - namespace: "specs.azure.core.model", - }, - "azure/core/page": { - namespace: "specs.azure.core.page", - }, - "azure/core/scalar": { - namespace: "specs.azure.core.scalar", - }, - "azure/core/traits": { - namespace: "specs.azure.core.traits", - }, - "azure/encode/duration": { - namespace: "specs.azure.encode.duration", - }, - "azure/example/basic": { - namespace: "specs.azure.example.basic", - }, - "azure/payload/pageable": { - namespace: "specs.azure.payload.pageable", - }, - "azure/versioning/previewVersion": { - namespace: "specs.azure.versioning.previewversion", - }, - "client/structure/default": { - namespace: "client.structure.service", - }, - "client/structure/multi-client": { - "package-name": "client-structure-multiclient", - namespace: "client.structure.multiclient", - }, - "client/structure/renamed-operation": { - "package-name": "client-structure-renamedoperation", - namespace: "client.structure.renamedoperation", - }, - "client/structure/two-operation-group": { - "package-name": "client-structure-twooperationgroup", - namespace: "client.structure.twooperationgroup", - }, - "client/naming": { - namespace: "client.naming.main", - }, - "client/overload": { - namespace: "client.overload", - }, - "encode/duration": { - namespace: "encode.duration", - }, - "encode/numeric": { - namespace: "encode.numeric", - }, - "parameters/basic": { - namespace: "parameters.basic", - }, - "parameters/spread": { - namespace: "parameters.spread", - }, - "payload/content-negotiation": { - namespace: "payload.contentnegotiation", - }, - "payload/multipart": { - namespace: "payload.multipart", - }, - "serialization/encoded-name/json": { - namespace: "serialization.encodedname.json", - }, - "special-words": { - namespace: "specialwords", - }, - "service/multi-service": { - namespace: "service.multiservice", - }, - "client/structure/client-operation-group": { - "package-name": "client-structure-clientoperationgroup", - namespace: "client.structure.clientoperationgroup", - }, -}; - -const EMITTER_OPTIONS: Record | Record[]> = { - "resiliency/srv-driven/old.tsp": { - "package-name": "resiliency-srv-driven1", - namespace: "resiliency.srv.driven1", - "package-mode": "azure-dataplane", - "package-pprint-name": "ResiliencySrvDriven1", - }, - "resiliency/srv-driven": { - "package-name": "resiliency-srv-driven2", - namespace: "resiliency.srv.driven2", - "package-mode": "azure-dataplane", - "package-pprint-name": "ResiliencySrvDriven2", - }, - "authentication/api-key": { - "clear-output-folder": "true", - }, - "authentication/http/custom": { - "package-name": "authentication-http-custom", - namespace: "authentication.http.custom", - "package-pprint-name": "Authentication Http Custom", - }, - "authentication/union": [ - { - "package-name": "authentication-union", - namespace: "authentication.union", - }, - { - "package-name": "setuppy-authentication-union", - namespace: "setuppy.authentication.union", - "keep-setup-py": "true", - }, - ], - "type/array": { - "package-name": "typetest-array", - namespace: "typetest.array", - }, - "type/dictionary": { - "package-name": "typetest-dictionary", - namespace: "typetest.dictionary", - }, - "type/enum/extensible": { - "package-name": "typetest-enum-extensible", - namespace: "typetest.enum.extensible", - }, - "type/enum/fixed": { - "package-name": "typetest-enum-fixed", - namespace: "typetest.enum.fixed", - }, - "type/model/empty": { - "package-name": "typetest-model-empty", - namespace: "typetest.model.empty", - }, - "type/model/inheritance/enum-discriminator": { - "package-name": "typetest-model-enumdiscriminator", - namespace: "typetest.model.enumdiscriminator", - }, - "type/model/inheritance/nested-discriminator": { - "package-name": "typetest-model-nesteddiscriminator", - namespace: "typetest.model.nesteddiscriminator", - }, - "type/model/inheritance/not-discriminated": { - "package-name": "typetest-model-notdiscriminated", - namespace: "typetest.model.notdiscriminated", - }, - "type/model/inheritance/single-discriminator": { - "package-name": "typetest-model-singlediscriminator", - namespace: "typetest.model.singlediscriminator", - }, - "type/model/inheritance/recursive": [ - { - "package-name": "typetest-model-recursive", - namespace: "typetest.model.recursive", - }, - { - "package-name": "generation-subdir", - namespace: "generation.subdir", - "generation-subdir": "_generated", - "generate-test": "false", - "clear-output-folder": "true", - }, - ], - "type/model/usage": { - "package-name": "typetest-model-usage", - namespace: "typetest.model.usage", - }, - "type/model/visibility": [ - { - "package-name": "typetest-model-visibility", - namespace: "typetest.model.visibility", - }, - { - "package-name": "headasbooleantrue", - namespace: "headasbooleantrue", - "head-as-boolean": "true", - }, - { - "package-name": "headasbooleanfalse", - namespace: "headasbooleanfalse", - "head-as-boolean": "false", - }, - ], - "type/property/nullable": { - "package-name": "typetest-property-nullable", - namespace: "typetest.property.nullable", - }, - "type/property/optionality": { - "package-name": "typetest-property-optional", - namespace: "typetest.property.optional", - }, - "type/property/additional-properties": { - "package-name": "typetest-property-additionalproperties", - namespace: "typetest.property.additionalproperties", - }, - "type/scalar": { - "package-name": "typetest-scalar", - namespace: "typetest.scalar", - }, - "type/property/value-types": { - "package-name": "typetest-property-valuetypes", - namespace: "typetest.property.valuetypes", - }, - "type/union": { - "package-name": "typetest-union", - namespace: "typetest.union", - }, - "type/union/discriminated": { - "package-name": "typetest-discriminatedunion", - namespace: "typetest.discriminatedunion", - }, - "type/file": { - "package-name": "typetest-file", - namespace: "typetest.file", - }, - documentation: { - "package-name": "specs-documentation", - namespace: "specs.documentation", - }, - "versioning/added": [ - { - "package-name": "versioning-added", - namespace: "versioning.added", - }, - { - "package-name": "generation-subdir2", - namespace: "generation.subdir2", - "generate-test": "false", - "generation-subdir": "_generated", - }, - ], -}; - -// ---- Shared utility functions ---- - -async function getSubdirectories(baseDir: string, flags: RegenerateFlags): Promise { - const subdirectories: string[] = []; - - async function searchDir(currentDir: string) { - const items = await readdir(currentDir, { withFileTypes: true }); - - const promisesArray = items.map(async (item) => { - const subDirPath = join(currentDir, item.name); - if (item.isDirectory()) { - const mainTspPath = join(subDirPath, "main.tsp"); - const clientTspPath = join(subDirPath, "client.tsp"); - - const mainTspRelativePath = toPosix(relative(baseDir, mainTspPath)); - - if (SKIP_SPECS.some((skipSpec) => mainTspRelativePath.includes(skipSpec))) return; - - const hasMainTsp = await access(mainTspPath) - .then(() => true) - .catch(() => false); - const hasClientTsp = await access(clientTspPath) - .then(() => true) - .catch(() => false); - - if (mainTspRelativePath.toLowerCase().includes(flags.name || "")) { - if (mainTspRelativePath.includes("resiliency/srv-driven")) { - subdirectories.push(resolve(subDirPath, "old.tsp")); - } - if (hasClientTsp) { - subdirectories.push(resolve(subDirPath, "client.tsp")); - } else if (hasMainTsp) { - subdirectories.push(resolve(subDirPath, "main.tsp")); - } - } - - await searchDir(subDirPath); - } - }); - - await Promise.all(promisesArray); - } - - await searchDir(baseDir); - return subdirectories; -} - -/** - * Resets the local `tests/generated` folder to the baseline checked into - * Azure/azure-sdk-for-python at `eng/tools/emitter/gen`. This ensures - * regeneration starts from a clean, known-good baseline which also contains - * necessary customized code. - */ -async function resetBaselineFromSdkRepo(generatedFolder: string): Promise { - const repoUrl = "https://github.com/Azure/azure-sdk-for-python.git"; - const branch = "main"; - const sourceSubdir = "eng/tools/emitter/gen"; - const testsGeneratedDir = resolve(generatedFolder, "../tests/generated"); - - console.log(pc.cyan(`\n${"=".repeat(60)}`)); - console.log(pc.cyan(`Resetting baseline from ${repoUrl} (${branch}/${sourceSubdir})`)); - console.log(pc.cyan(`${"=".repeat(60)}\n`)); - - // Wipe tests/generated - if (existsSync(testsGeneratedDir)) { - console.log(pc.dim(`Removing ${testsGeneratedDir}`)); - rmSync(testsGeneratedDir, { recursive: true, force: true }); - } - await mkdir(testsGeneratedDir, { recursive: true }); - - // Sparse-checkout the baseline folder into a temp directory - const tempDir = await mkdtemp(join(tmpdir(), "azsdk-baseline-")); - try { - console.log(pc.dim(`Cloning into ${tempDir}`)); - const run = (cmd: string) => - execSync(cmd, { cwd: tempDir, stdio: ["ignore", "ignore", "inherit"] }); - - run(`git init`); - run(`git remote add origin ${repoUrl}`); - run(`git config core.sparseCheckout true`); - run(`git sparse-checkout init --cone`); - run(`git sparse-checkout set ${sourceSubdir}`); - run(`git fetch --depth 1 origin ${branch}`); - run(`git checkout FETCH_HEAD`); - - const sourceRoot = join(tempDir, ...sourceSubdir.split("/")); - for (const flavor of ["azure", "unbranded"]) { - const src = join(sourceRoot, flavor); - const dest = join(testsGeneratedDir, flavor); - if (!existsSync(src)) { - console.warn(pc.yellow(`Baseline folder not found: ${src}`)); - continue; - } - console.log(pc.dim(`Copying ${flavor}/ -> ${dest}`)); - await cp(src, dest, { recursive: true }); - } - - console.log(pc.green(`Baseline reset complete.\n`)); - } finally { - rmSync(tempDir, { recursive: true, force: true }); - } -} - -async function preprocess(flavor: string, generatedFolder: string): Promise { - if (flavor === "azure") { - const testsGeneratedDir = resolve(generatedFolder, "../tests/generated/azure"); - - const DELETE_CONTENT = "# This file is to be deleted after regeneration"; - const DELETE_FILE = "to_be_deleted.py"; - const entries: { folder: string[]; file: string; content: string }[] = [ - { - folder: ["authentication-api-key", "authentication", "apikey", "_operations"], - file: DELETE_FILE, - content: DELETE_CONTENT, - }, - { - folder: ["generation-subdir", "generation", "subdir", "_generated"], - file: DELETE_FILE, - content: DELETE_CONTENT, - }, - { - folder: ["generation-subdir", "generated_tests"], - file: DELETE_FILE, - content: DELETE_CONTENT, - }, - { - folder: ["generation-subdir", "generation", "subdir"], - file: "to_be_kept.py", - content: "# This file is to be kept after regeneration", - }, - ]; - - await Promise.all( - entries.map(async ({ folder, file, content }) => { - const targetFolder = join(testsGeneratedDir, ...folder); - await mkdir(targetFolder, { recursive: true }); - await writeFile(join(targetFolder, file), content); - }), - ); - } -} - // Get paths const SCRIPT_DIR = dirname(fileURLToPath(import.meta.url)); const PLUGIN_DIR = argv.values.pluginDir @@ -531,211 +101,13 @@ const GENERATED_FOLDER = argv.values.generatedFolder : resolve(PLUGIN_DIR, "generator"); const EMITTER_NAME = argv.values.emitterName || "@typespec/http-client-python"; -interface CompileTask { - spec: string; - outputDir: string; - options: Record; -} - -// Group of tasks for the same spec that must run sequentially -interface TaskGroup { - spec: string; - tasks: CompileTask[]; -} - -// Check whether a spec path belongs to azure-http-specs (vs standard http-specs). -// Using "azure-http-specs" instead of "azure" to avoid false positives when the -// working directory path contains "azure" (e.g. azure-sdk-for-python). -function isAzureSpec(spec: string): boolean { - return spec.includes("azure-http-specs"); -} - -function defaultPackageName(spec: string): string { - const specDir = isAzureSpec(spec) ? AZURE_HTTP_SPECS : HTTP_SPECS; - return toPosix(relative(specDir, dirname(spec))) - .replace(/\//g, "-") - .toLowerCase(); -} - -function getEmitterOptions(spec: string, flavor: string): Record[] { - const specDir = isAzureSpec(spec) ? AZURE_HTTP_SPECS : HTTP_SPECS; - const relativeSpec = toPosix(relative(specDir, spec)); - const key = relativeSpec.includes("resiliency/srv-driven/old.tsp") - ? relativeSpec - : dirname(relativeSpec); - const emitterOpts = EMITTER_OPTIONS[key] || - (flavor === "azure" ? AZURE_EMITTER_OPTIONS[key] : [{}]) || [{}]; - return Array.isArray(emitterOpts) ? emitterOpts : [emitterOpts]; -} - -function buildTaskGroups(specs: string[], flags: RegenerateFlags): TaskGroup[] { - const groups: TaskGroup[] = []; - - for (const spec of specs) { - const tasks: CompileTask[] = []; - - for (const emitterConfig of getEmitterOptions(spec, flags.flavor)) { - // Apply flavor defaults first, then per-spec options so they can override (e.g., "generate-test": "false") - const options: Record = {}; - for (const [k, v] of Object.entries(SpecialFlags[flags.flavor] ?? {})) { - options[k] = v; - } - Object.assign(options, emitterConfig); - - // Add flavor - options["flavor"] = flags.flavor; - - // Set output directory - use tests/generated// structure - const packageName = (options["package-name"] as string) || defaultPackageName(spec); - const outputDir = - (options["emitter-output-dir"] as string) || - toPosix(`${GENERATED_FOLDER}/../tests/generated/${flags.flavor}/${packageName}`); - options["emitter-output-dir"] = outputDir; - - // Debug mode - if (flags.debug) { - options["debug"] = true; - } - - // Examples directory - options["examples-dir"] = toPosix(join(dirname(spec), "examples")); - - // Emit YAML only - Python processing is batched after all specs compile - options["emit-yaml-only"] = true; - - tasks.push({ spec, outputDir, options }); - } - - groups.push({ spec, tasks }); - } - - return groups; -} - -async function compileSpec(task: CompileTask): Promise<{ success: boolean; error?: string }> { - const { spec, outputDir, options } = task; - - try { - // Build compiler options - const compilerOptions = { - emit: [PLUGIN_DIR], - options: { - [EMITTER_NAME]: options, - }, - }; - - // Compile using TypeSpec compiler directly (no subprocess) - const program = await compile(NodeHost, spec, compilerOptions); - - if (program.hasError()) { - const errors = program.diagnostics - .filter((d) => d.severity === "error") - .map((d) => d.message) - .join("\n"); - return { success: false, error: errors }; - } - - return { success: true }; - } catch (err) { - // Clean up on error - rmSync(outputDir, { recursive: true, force: true }); - return { success: false, error: String(err) }; - } -} - -function renderProgressBar( - completed: number, - failed: number, - total: number, - width: number = 40, -): string { - const successCount = completed - failed; - const successWidth = Math.round((successCount / total) * width); - const failWidth = Math.round((failed / total) * width); - const emptyWidth = width - successWidth - failWidth; - - const successBar = pc.bgGreen(" ".repeat(successWidth)); - const failBar = failed > 0 ? pc.bgRed(" ".repeat(failWidth)) : ""; - const emptyBar = pc.dim("░".repeat(Math.max(0, emptyWidth))); - - const percent = Math.round((completed / total) * 100); - return `${successBar}${failBar}${emptyBar} ${pc.cyan(`${percent}%`)} (${completed}/${total})`; -} - -async function runParallel(groups: TaskGroup[], maxJobs: number): Promise> { - const results = new Map(); - const executing: Set> = new Set(); - - // Count total tasks for progress - const totalTasks = groups.reduce((sum, g) => sum + g.tasks.length, 0); - let completed = 0; - let failed = 0; - const failedSpecs: string[] = []; - - // Check if we're in a TTY for progress bar updates - const isTTY = process.stdout.isTTY; - - const updateProgress = () => { - if (isTTY) { - process.stdout.write(`\r${renderProgressBar(completed, failed, totalTasks)}`); - } - }; - - // Initial progress bar - updateProgress(); - - for (const group of groups) { - // Each group runs as a unit - tasks within a group run sequentially - // But different groups can run in parallel - const runGroup = async () => { - const specDir = isAzureSpec(group.spec) ? AZURE_HTTP_SPECS : HTTP_SPECS; - const shortName = toPosix(relative(specDir, dirname(group.spec))); - - // Run all tasks in this group sequentially to avoid state pollution - let groupSuccess = true; - for (const task of group.tasks) { - const packageName = (task.options["package-name"] as string) || shortName; - - const result = await compileSpec(task); - completed++; - - if (!result.success) { - failed++; - failedSpecs.push(`${packageName}: ${result.error}`); - groupSuccess = false; - } - - updateProgress(); - } - - results.set(group.spec, groupSuccess); - }; - - const p = runGroup().finally(() => executing.delete(p)); - executing.add(p); - - if (executing.size >= maxJobs) { - await Promise.race(executing); - } - } - - await Promise.all(executing); - - // Clear progress bar line and print final status - if (isTTY) { - process.stdout.write("\r" + " ".repeat(60) + "\r"); - } - - // Print failures at the end - if (failedSpecs.length > 0) { - console.log(pc.red(`\nFailed specs:`)); - for (const spec of failedSpecs) { - console.log(pc.red(` • ${spec}`)); - } - } - - return results; -} +const ctx: RegenerateContext = { + pluginDir: PLUGIN_DIR, + azureHttpSpecs: AZURE_HTTP_SPECS, + httpSpecs: HTTP_SPECS, + generatedFolder: GENERATED_FOLDER, + emitterName: EMITTER_NAME, +}; async function collectConfigFiles(generatedDir: string, flavor: string): Promise { const flavorDir = join(generatedDir, "..", "tests", "generated", flavor); @@ -812,8 +184,9 @@ async function regenerateFlavor( const standardSpecs = await getSubdirectories(HTTP_SPECS, flags); const allSpecs = [...azureSpecs, ...standardSpecs]; - // Build task groups (tasks for same spec run sequentially to avoid state pollution) - const groups = buildTaskGroups(allSpecs, flags); + // Build task groups (tasks for same spec run sequentially to avoid state pollution). + // emitYamlOnly: true -> phase 1 emits YAML only; phase 2 (runBatchPythonProcessing) writes .py files. + const groups = buildTaskGroups(allSpecs, flags, ctx, { emitYamlOnly: true }); const totalTasks = groups.reduce((sum, g) => sum + g.tasks.length, 0); console.log(pc.cyan(`Found ${allSpecs.length} specs (${totalTasks} total tasks) to compile`)); @@ -821,7 +194,7 @@ async function regenerateFlavor( // Run compilation (emits YAML only) const startTime = performance.now(); - const results = await runParallel(groups, jobs); + const results = await runParallel(groups, jobs, ctx); const compileTime = (performance.now() - startTime) / 1000; // Summary for TypeSpec compilation @@ -861,29 +234,6 @@ async function regenerateFlavor( return pySuccess; } -/** - * Deletes a couple of fully-generated package folders from the baseline so that - * regeneration has to recreate them from scratch. - */ -function deleteSomeGeneratedFiles() { - const testsGeneratedDir = resolve(GENERATED_FOLDER, "../tests/generated"); - const targets = [ - join(testsGeneratedDir, "azure", "authentication-http-custom"), - join(testsGeneratedDir, "unbranded", "encode-array"), - ]; - for (const target of targets) { - if (existsSync(target)) { - console.log(pc.dim(`Deleting ${target}`)); - rmSync(target, { recursive: true, force: true }); - } - } -} - -async function preProcess() { - await resetBaselineFromSdkRepo(GENERATED_FOLDER); - deleteSomeGeneratedFiles(); -} - async function main() { const isWindows = platform() === "win32"; const flavor = argv.values.flavor; @@ -906,7 +256,7 @@ async function main() { const startTime = performance.now(); let success: boolean; - await preProcess(); + await prepareBaselineOfGeneratedCode(GENERATED_FOLDER); if (flavor) { success = await regenerateFlavor(flavor, name, debug, jobs); From 563234317b52eeea972290180f80e6065f3883f7 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Fri, 8 May 2026 15:59:26 +0800 Subject: [PATCH 4/5] format --- .../eng/scripts/ci/regenerate-common.ts | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/packages/http-client-python/eng/scripts/ci/regenerate-common.ts b/packages/http-client-python/eng/scripts/ci/regenerate-common.ts index 58d1a3d8c23..f21c4013616 100644 --- a/packages/http-client-python/eng/scripts/ci/regenerate-common.ts +++ b/packages/http-client-python/eng/scripts/ci/regenerate-common.ts @@ -70,7 +70,11 @@ export interface BuildTaskGroupsOptions { // ---- Public constants ---- -export const SKIP_SPECS: string[] = ["type/file", "service/multiple-services"]; +export const SKIP_SPECS: string[] = [ + "type/file", + "service/multiple-services", + "azure/client-generator-core/response-as-bool", +]; export const SpecialFlags: Record> = { azure: { @@ -211,10 +215,7 @@ export const AZURE_EMITTER_OPTIONS: Record< }, }; -export const EMITTER_OPTIONS: Record< - string, - Record | Record[] -> = { +export const EMITTER_OPTIONS: Record | Record[]> = { "resiliency/srv-driven/old.tsp": { "package-name": "resiliency-srv-driven1", namespace: "resiliency.srv.driven1", @@ -398,8 +399,8 @@ export function getEmitterOptions( const key = relativeSpec.includes("resiliency/srv-driven/old.tsp") ? relativeSpec : dirname(relativeSpec); - const emitterOpts = - EMITTER_OPTIONS[key] || (flavor === "azure" ? AZURE_EMITTER_OPTIONS[key] : [{}]) || [{}]; + const emitterOpts = EMITTER_OPTIONS[key] || + (flavor === "azure" ? AZURE_EMITTER_OPTIONS[key] : [{}]) || [{}]; return Array.isArray(emitterOpts) ? emitterOpts : [emitterOpts]; } From 8f415a7f92fafc2b684ebde5b369331cf12c237e Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Sat, 9 May 2026 06:26:46 +0000 Subject: [PATCH 5/5] optimize --- packages/http-client-python/generator/pygen/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/http-client-python/generator/pygen/__init__.py b/packages/http-client-python/generator/pygen/__init__.py index 91cfab71f69..ce6b4d3c7bb 100644 --- a/packages/http-client-python/generator/pygen/__init__.py +++ b/packages/http-client-python/generator/pygen/__init__.py @@ -236,8 +236,7 @@ def read_file(self, path: Union[str, Path]) -> str: def write_file(self, filename: Union[str, Path], file_content: str) -> None: """Directly writing to disk""" file_folder = Path(filename).parent - if not Path.is_dir(self.output_folder / file_folder): - Path.mkdir(self.output_folder / file_folder, parents=True) + Path.mkdir(self.output_folder / file_folder, parents=True, exist_ok=True) with open(self.output_folder / Path(filename), "w", encoding="utf-8") as fd: fd.write(file_content)