fixup! test(@angular/build): add library builder scaling benchmark suite

Adds a `--builder` axis (library, ng-packagr) so fixtures can be generated in either builder's project shape for A/B benchmarking. Fixtures only; no results recorded.

Author: dherges

scripts/benchmark.mts

@@ -9,7 +9,12 @@
 import fs from 'node:fs';
 import path from 'node:path';
 import { type BenchmarkCliOptions, runI18nBenchmarks } from './benchmarks/i18n/index.mts';
-import { STYLE_FLAVORS, type StyleFlavor } from './benchmarks/library-builder/fixtures.mts';
+import {
+  LIBRARY_BUILDERS,
+  STYLE_FLAVORS,
+  type LibraryBuilder,
+  type StyleFlavor,
+} from './benchmarks/library-builder/fixtures.mts';
 import {
   type LibraryBuilderBenchmarkOptions,
   runLibraryBuilderBenchmarks,
@@ -62,6 +67,7 @@ function parseStringList(raw: string | undefined): string[] | undefined {
 async function runLibraryBuilderSubsystem(
   options: {
     layout?: string;
+    builder?: string;
     count?: string;
     depth?: string;
     'style-flavor'?: string;
@@ -119,6 +125,21 @@ async function runLibraryBuilderSubsystem(
     return 1;
   }
 
+  const builder = parseStringList(options.builder);
+  const validBuilders: readonly string[] = LIBRARY_BUILDERS;
+  if (builder !== undefined && builder.some((b) => !validBuilders.includes(b))) {
+    // eslint-disable-next-line no-console
+    console.error(
+      'Error: --builder must be a comma-separated list of: ' +
+        LIBRARY_BUILDERS.join(', ') +
+        '. Got "' +
+        options.builder +
+        '".',
+    );
+
+    return 1;
+  }
+
   const styleFlavor = parseStringList(options['style-flavor']);
   const validStyleFlavors: readonly string[] = STYLE_FLAVORS;
   if (styleFlavor !== undefined && styleFlavor.some((f) => !validStyleFlavors.includes(f))) {
@@ -136,6 +157,7 @@ async function runLibraryBuilderSubsystem(
 
   const cliOptions: LibraryBuilderBenchmarkOptions = {
     layout: options.layout as LibraryBuilderBenchmarkOptions['layout'],
+    builder: builder as LibraryBuilder[] | undefined,
     count,
     depth,
     styleFlavor: styleFlavor as StyleFlavor[] | undefined,
@@ -165,6 +187,7 @@ export default async function (
     compareBaseline?: string;
     'compare-baseline'?: string;
     layout?: string;
+    builder?: string;
     count?: string;
     depth?: string;
     'style-flavor'?: string;
@@ -186,7 +209,7 @@ Usage:
 
 Subsystems:
   i18n               Run i18n inliner performance benchmarks (default)
-  library-builder    Run @angular/build:library scaling benchmarks
+  library-builder    Run @angular/build:library (and :ng-packagr) scaling benchmarks
 
 Options (i18n):
   --scenario=<name>          Run a specific scenario (e.g. standard-app, enterprise-multilingual)
@@ -201,6 +224,8 @@ Options (i18n):
 
 Options (library-builder):
   --layout=<flat|deep>       Fixture layout (default: flat)
+  --builder=<b,b,...>        Builders to benchmark, one or more of: library, ng-packagr.
+                             Pass both for an A/B run (default: library)
   --count=<n,n,...>          Entry-point count targets, drives both layouts
                              (default: 10,20,50,100,200,300,400,500,1000,2000)
   --depth=<n,n,...>          Explicit tree depths, layout=deep only. Overrides the depths

scripts/benchmarks/library-builder/README.md

@@ -60,6 +60,20 @@ Each entry point consists of a `public-api.ts` and a component file with a bit o
 content derived from the entry's id (e.g. `<h2>Component no. 0042</h2>`), so entries aren't
 byte-identical copies of each other.
 
+### Builder axis (A/B)
+
+`--builder=<list>` selects which Architect builder each fixture targets. The same synthetic sources
+are emitted in either builder's project shape:
+
+| Builder      | Architect builder           | Project shape                                                                                                            |
+| :----------- | :-------------------------- | :----------------------------------------------------------------------------------------------------------------------- |
+| `library`    | `@angular/build:library`    | One root `package.json` with a flat `exports` map; no `ng-package.json`.                                                 |
+| `ng-packagr` | `@angular/build:ng-packagr` | Root `ng-package.json` (`dest`, `lib.entryFile`) plus one `ng-package.json` per secondary entry point; no `exports` map. |
+
+Defaults to `library` only. Pass `--builder=library,ng-packagr` for an A/B run: the builder is the
+innermost loop, so the two builds of an identical fixture size run back to back. `inline-scss`
+sets `inlineStyleLanguage` on the builder for `library` and in `ng-package.json` for `ng-packagr`.
+
 ### Style-flavor matrix
 
 `--style-flavor` is a second, independent sweep axis (crossed with `--layout` and the
@@ -113,6 +127,7 @@ pnpm admin benchmark library-builder [options]
 | Option                     | Description                                                                                                                                               | Default                                  |
 | :------------------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------- | :--------------------------------------- |
 | `--layout=<flat\|deep>`    | Fixture layout                                                                                                                                            | `flat`                                   |
+| `--builder=<b,b,...>`      | Builders to benchmark, one or more of `library`, `ng-packagr`. Pass both for an A/B run                                                                   | `library`                                |
 | `--count=<n,n,...>`        | Entry-point count targets, drives both layouts                                                                                                            | `10,20,50,100,200,300,400,500,1000,2000` |
 | `--depth=<n,n,...>`        | **`layout=deep` only.** Explicit tree depths, overriding the depths otherwise derived from `--count`                                                      | derived from `--count`                   |
 | `--style-flavor=<f,f,...>` | Style-flavor targets (see "Style-flavor matrix" above), one or more of `inline`, `inline-css`, `inline-scss`, `external`, `external-css`, `external-scss` | `inline`                                 |
@@ -133,6 +148,12 @@ Sweep the flat layout at a single size:
 pnpm admin benchmark library-builder --layout=flat --count=500
 ```
 
+A/B run of both builders at a few sizes:
+
+```sh
+pnpm admin benchmark library-builder --builder=library,ng-packagr --layout=flat --count=10,50,200
+```
+
 Sweep the full style-flavor matrix at a single size, both layouts:
 
 ```sh
@@ -219,9 +240,30 @@ Verification performed:
 - Verified `--style-flavor` rejects an unrecognized flavor name with a clear error listing the
   valid set, consistent with existing `--layout`/`--count`/`--depth` validation.
 
+### 2026-10-22: Fixtures and harness prepared for A/B runs (no results yet)
+
+Added the `--builder` axis (see "Builder axis (A/B)" above): the fixture generator can now emit
+`ng-packagr`-shaped projects (per-entry-point `ng-package.json`) next to the native builder's
+`exports`-map projects, and `builder` is part of the JSON/CSV output.
+
+This entry is preparation only -- **no benchmark results are recorded here**. Verification performed:
+
+- `--count=5` with both layouts, all 6 style flavors, both builders, `--iterations=1`: all 24
+  builds completed with `status: "ok"`.
+- Inspected `ng-packagr` output for a `deep`/`external-scss` fixture: every secondary entry point
+  was built and bundled, and the per-entry SCSS variable compiled to a literal color, i.e. the
+  fixtures exercise the same work on both builders.
+- `--builder` rejects an unrecognized name with a clear error listing the valid set.
+- A reduced, throwaway sweep (flat, counts 10/50/200, 2 iterations, once per builder) ran cleanly
+  on both builders. Those numbers are deliberately not recorded: they are a smoke test, not a
+  measurement.
+
+We now have a basis for a first real A/B run. The full sweep takes hours and needs proper
+preparation (quiet machine, fixed environment, enough iterations) before it is worth running.
+
 ## Future Work
 
-- A/B comparison against `ng-packagr` (both are already registered Architect builders — `@angular/build:ng-packagr` and `@angular/build:library` — so this is a same-builder-package comparison, not a separate tool invocation).
+- First real A/B run against `ng-packagr` (`--builder=library,ng-packagr`; the fixture generation and harness support are in place, see 2026-10-22). Needs a quiet machine and the full default sweep.
 - Full style-flavor matrix sweep (default `--count` sweep x 6 flavors x 2 layouts) to quantify the actual performance impact of stylesheet processing and external file reads, now that the matrix is implemented.
 - Dependencies between entry points.
 - Shared stylesheet tokens imported by multiple components.

scripts/benchmarks/library-builder/fixtures.mts

@@ -7,15 +7,23 @@
  */
 
 // Generates synthetic Angular Package Format (APF) library fixtures for benchmarking
-// `@angular/build:library`, and converts them into the builder's project shape (a flat
-// `exports` map in one root package.json, rather than the per-entry-point `ng-package.json`
-// convention used by third-party tools such as ng-packagr).
+// `@angular/build:library` and `@angular/build:ng-packagr`. The same synthetic sources are emitted
+// in either builder's project shape: a flat `exports` map in one root package.json for the native
+// builder, or the per-entry-point `ng-package.json` convention for ng-packagr.
 
 import fs from 'node:fs';
 import path from 'node:path';
 
 export type Layout = 'flat' | 'deep';
 
+/** Which Architect builder the generated project targets. */
+export type LibraryBuilder = 'library' | 'ng-packagr';
+
+/** All library builders, in A/B order (native builder first, legacy builder second). */
+export const LIBRARY_BUILDERS: readonly LibraryBuilder[] = ['library', 'ng-packagr'];
+
+export const DEFAULT_LIBRARY_BUILDER: LibraryBuilder = 'library';
+
 /**
  * Style-flavor matrix: crosses template location (inline vs. external `templateUrl`) with
  * stylesheet presence/language (none, CSS, SCSS). `external` has no stylesheet, matching
@@ -49,6 +57,9 @@ export interface FixtureOptions {
 
   /** Template/stylesheet flavor for every entry point in the fixture. Defaults to `inline`. */
   styleFlavor?: StyleFlavor;
+
+  /** Builder the generated project targets. Defaults to `library`. */
+  builder?: LibraryBuilder;
 }
 
 /** Per-entry CSS/SCSS content, varied by `id` like the template so entries aren't byte-identical. */
@@ -106,9 +117,22 @@ export class ${className} {}
 }
 
 /** Writes one secondary entry point's source files (component + public-api.ts) into `entryDir`. */
-function writeEntryPointSource(entryDir: string, id: string, styleFlavor: StyleFlavor): void {
+function writeEntryPointSource(
+  entryDir: string,
+  id: string,
+  styleFlavor: StyleFlavor,
+  builder: LibraryBuilder,
+): void {
   fs.mkdirSync(entryDir, { recursive: true });
 
+  if (builder === 'ng-packagr') {
+    // ng-packagr discovers secondary entry points by the presence of a per-directory ng-package.json.
+    fs.writeFileSync(
+      path.join(entryDir, 'ng-package.json'),
+      JSON.stringify({ lib: { entryFile: 'public-api.ts' } }, null, 2) + '\n',
+    );
+  }
+
   const className = `RefComponent${id}`;
   const selector = `ref-comp-${id}`;
   const compBase = `comp-${id}`;
@@ -150,13 +174,18 @@ function writeEntryPointSource(entryDir: string, id: string, styleFlavor: StyleF
 }
 
 /** Generates `count` flat secondary entry points as direct siblings, returning their relative dirs. */
-function generateFlatEntries(outDir: string, count: number, styleFlavor: StyleFlavor): string[] {
+function generateFlatEntries(
+  outDir: string,
+  count: number,
+  styleFlavor: StyleFlavor,
+  builder: LibraryBuilder,
+): string[] {
   const width = String(count).length;
   const dirs: string[] = [];
   for (let i = 1; i <= count; i++) {
     const id = String(i).padStart(width, '0');
     const relDir = `comp-${id}`;
-    writeEntryPointSource(path.join(outDir, relDir), id, styleFlavor);
+    writeEntryPointSource(path.join(outDir, relDir), id, styleFlavor, builder);
     dirs.push(relDir);
   }
 
@@ -167,7 +196,12 @@ function generateFlatEntries(outDir: string, count: number, styleFlavor: StyleFl
  * Generates a binary tree `depth` levels deep (`sub-a`/`sub-b` at each level), placing one
  * secondary entry point at each of the 2^depth leaves. Returns their relative dirs.
  */
-function generateDeepEntries(outDir: string, depth: number, styleFlavor: StyleFlavor): string[] {
+function generateDeepEntries(
+  outDir: string,
+  depth: number,
+  styleFlavor: StyleFlavor,
+  builder: LibraryBuilder,
+): string[] {
   const total = 2 ** depth;
   const width = String(total).length;
   const dirs: string[] = [];
@@ -177,7 +211,7 @@ function generateDeepEntries(outDir: string, depth: number, styleFlavor: StyleFl
     if (level === depth) {
       counter++;
       const id = String(counter).padStart(width, '0');
-      writeEntryPointSource(path.join(outDir, currentRelDir), id, styleFlavor);
+      writeEntryPointSource(path.join(outDir, currentRelDir), id, styleFlavor, builder);
       dirs.push(currentRelDir);
 
       return;
@@ -207,6 +241,7 @@ export function generateFixture(outDir: string, options: FixtureOptions): number
   fs.mkdirSync(outDir, { recursive: true });
 
   const styleFlavor = options.styleFlavor ?? DEFAULT_STYLE_FLAVOR;
+  const builder = options.builder ?? DEFAULT_LIBRARY_BUILDER;
 
   fs.writeFileSync(
     path.join(outDir, 'public-api.ts'),
@@ -215,12 +250,16 @@ export function generateFixture(outDir: string, options: FixtureOptions): number
 
   const secondaryDirs =
     options.layout === 'flat'
-      ? generateFlatEntries(outDir, options.count ?? 10, styleFlavor)
-      : generateDeepEntries(outDir, options.depth ?? 2, styleFlavor);
+      ? generateFlatEntries(outDir, options.count ?? 10, styleFlavor, builder)
+      : generateDeepEntries(outDir, options.depth ?? 2, styleFlavor, builder);
 
+  // Only the native builder reads an `exports` map; ng-packagr derives (and writes) it itself from
+  // the per-entry-point ng-package.json files.
   const exportsMap: Record<string, string> = { '.': './public-api.ts' };
-  for (const relDir of secondaryDirs) {
-    exportsMap[`./${entryKey(relDir)}`] = `./${relDir.split(path.sep).join('/')}/public-api.ts`;
+  if (builder === 'library') {
+    for (const relDir of secondaryDirs) {
+      exportsMap[`./${entryKey(relDir)}`] = `./${relDir.split(path.sep).join('/')}/public-api.ts`;
+    }
   }
 
   // Fallback to a hardcoded version if the path lookup fails in the sandbox/environment.
@@ -247,7 +286,7 @@ export function generateFixture(outDir: string, options: FixtureOptions): number
           '@angular/core': peerRange,
           '@angular/common': peerRange,
         },
-        exports: exportsMap,
+        ...(builder === 'library' ? { exports: exportsMap } : {}),
       },
       null,
       2,
@@ -298,15 +337,36 @@ export function generateFixture(outDir: string, options: FixtureOptions): number
     ) + '\n',
   );
 
-  const buildOptions: Record<string, unknown> = {
-    tsConfig: './tsconfig.lib.json',
-    outputPath: 'dist',
-  };
-  if (styleFlavor === 'inline-scss') {
-    // Inline style language is a project-wide builder option, not per-component -- this is the
-    // only flavor that needs it set, since external .scss files are detected by extension and
-    // 'css' (the default) already covers the other flavors.
-    buildOptions['inlineStyleLanguage'] = 'scss';
+  let buildTarget: { builder: string; options: Record<string, unknown> };
+  if (builder === 'library') {
+    const buildOptions: Record<string, unknown> = {
+      tsConfig: './tsconfig.lib.json',
+      outputPath: 'dist',
+    };
+    if (styleFlavor === 'inline-scss') {
+      // Inline style language is a project-wide builder option, not per-component -- this is the
+      // only flavor that needs it set, since external .scss files are detected by extension and
+      // 'css' (the default) already covers the other flavors.
+      buildOptions['inlineStyleLanguage'] = 'scss';
+    }
+    buildTarget = { builder: '@angular/build:library', options: buildOptions };
+  } else {
+    const ngPackage: Record<string, unknown> = {
+      dest: 'dist',
+      lib: { entryFile: 'public-api.ts' },
+    };
+    if (styleFlavor === 'inline-scss') {
+      // For ng-packagr the inline style language is configured in ng-package.json, not the builder.
+      ngPackage['inlineStyleLanguage'] = 'scss';
+    }
+    fs.writeFileSync(
+      path.join(outDir, 'ng-package.json'),
+      JSON.stringify(ngPackage, null, 2) + '\n',
+    );
+    buildTarget = {
+      builder: '@angular/build:ng-packagr',
+      options: { project: './ng-package.json', tsConfig: './tsconfig.lib.json' },
+    };
   }
 
   fs.writeFileSync(
@@ -320,12 +380,7 @@ export function generateFixture(outDir: string, options: FixtureOptions): number
             root: '.',
             projectType: 'library',
             sourceRoot: '.',
-            targets: {
-              build: {
-                builder: '@angular/build:library',
-                options: buildOptions,
-              },
-            },
+            targets: { build: buildTarget },
           },
         },
       },

scripts/benchmarks/library-builder/harness.mts

@@ -6,16 +6,17 @@
  * found in the LICENSE file at https://angular.dev/license
  */
 
-// Runs and measures a single `@angular/build:library` cold build via Architect's standalone CLI.
+// Runs and measures a single `@angular/build:library` or `@angular/build:ng-packagr` cold build via Architect's standalone CLI.
 // `fixtures.mts` owns generating the on-disk project; this module owns spawning the builder
 // against it and reporting what happened. Kept separate so each file has one job.
 
 import { spawnSync } from 'node:child_process';
 import fs from 'node:fs';
 import path from 'node:path';
-import type { Layout, StyleFlavor } from './fixtures.mts';
+import type { Layout, LibraryBuilder, StyleFlavor } from './fixtures.mts';
 
 export interface BuildMeasurement {
+  builder: LibraryBuilder;
   layout: Layout;
   styleFlavor: StyleFlavor;
   entryPoints: number;