From 7b775f83ecba0256d3dcdaa9ff369a9e7020219a Mon Sep 17 00:00:00 2001 From: David Taylor Date: Mon, 12 Jan 2026 18:09:33 +0000 Subject: [PATCH] feat(babel): add parallel processing via worker threads Add a `parallel` option that processes files concurrently using Node.js worker threads. This reduces build times for large projects where Babel transformation is a bottleneck. This is similar to the existing parallel behavior of `@rollup/plugin-terser`, but uses the `workerpool` package instead of a custom implementation. This required some fairly significant refactoring, because we can only pass serializable objects between the main thread and the worker threads. It also required changes to the plugin's own build config, so that we can generate a dedicated worker entrypoint. Validations are added to ensure that unserializable config (e.g. inline babel plugins) cannot be used alongside the new parallel mode. For people using dedicated babel config files, this isn't a problem, because they are loaded directly by babel in the worker thread itself. The worker threads do have a setup cost, so this only makes sense for large projects. In Discourse, enabling this parallel mode cuts our overall vite (rolldown) build time by about 45% (from ~11s to ~6s) on my machine. --- packages/babel/README.md | 11 ++ packages/babel/package.json | 3 +- packages/babel/rollup.config.mjs | 27 +++- packages/babel/src/index.js | 159 ++++++++++++++++++++--- packages/babel/src/preflightCheck.js | 14 +- packages/babel/src/transformCode.js | 31 +++-- packages/babel/src/worker.js | 16 +++ packages/babel/test/as-input-plugin.mjs | 91 +++++++++++++ packages/babel/test/as-output-plugin.mjs | 63 +++++++++ packages/babel/types/index.d.ts | 16 +++ pnpm-lock.yaml | 8 ++ 11 files changed, 400 insertions(+), 39 deletions(-) create mode 100644 packages/babel/src/worker.js diff --git a/packages/babel/README.md b/packages/babel/README.md index 28ae467ad..c5f2662e1 100644 --- a/packages/babel/README.md +++ b/packages/babel/README.md @@ -135,6 +135,17 @@ Default: `false` Before transpiling your input files this plugin also transpile a short piece of code **for each** input file. This is used to validate some misconfiguration errors, but for sufficiently big projects it can slow your build times so if you are confident about your configuration then you might disable those checks with this option. +### `parallel` + +Type: `Boolean | number`
+Default: `false` + +Enable parallel processing of files in worker threads. This has a setup cost, so is best suited for larger projects. Pass an integer to set the number of workers. Set `true` for the default number of workers (based on CPU cores, capped at 4). + +This option is available for both the input plugin (`babel()`) and the output plugin (`getBabelOutputPlugin()`). + +This option cannot be used alongside custom overrides or non-serializable Babel options. + ### External dependencies Ideally, you should only be transforming your source code, rather than running all of your external dependencies through Babel (to ignore external dependencies from being handled by this plugin you might use `exclude: 'node_modules/**'` option). If you have a dependency that exposes untranspiled ES6 source code that doesn't run in your target environment, then you may need to break this rule, but it often causes problems with unusual `.babelrc` files or mismatched versions of Babel. diff --git a/packages/babel/package.json b/packages/babel/package.json index 55bff366f..d209c5056 100644 --- a/packages/babel/package.json +++ b/packages/babel/package.json @@ -67,7 +67,8 @@ }, "dependencies": { "@babel/helper-module-imports": "^7.18.6", - "@rollup/pluginutils": "^5.0.1" + "@rollup/pluginutils": "^5.0.1", + "workerpool": "^9.0.0" }, "devDependencies": { "@babel/core": "^7.19.1", diff --git a/packages/babel/rollup.config.mjs b/packages/babel/rollup.config.mjs index 686d97f22..4124e6fc1 100644 --- a/packages/babel/rollup.config.mjs +++ b/packages/babel/rollup.config.mjs @@ -1,6 +1,6 @@ import { readFileSync } from 'fs'; -import { createConfig } from '../../shared/rollup.config.mjs'; +import { createConfig, emitModulePackageFile } from '../../shared/rollup.config.mjs'; import { babel } from './src/index.js'; @@ -8,7 +8,30 @@ const pkg = JSON.parse(readFileSync(new URL('./package.json', import.meta.url), export default { ...createConfig({ pkg }), - input: './src/index.js', + input: { + index: './src/index.js', + worker: './src/worker.js' + }, + output: [ + { + format: 'cjs', + dir: 'dist/cjs', + exports: 'named', + footer(chunkInfo) { + if (chunkInfo.name === 'index') { + return 'module.exports = Object.assign(exports.default, exports);'; + } + return null; + }, + sourcemap: true + }, + { + format: 'es', + dir: 'dist/es', + plugins: [emitModulePackageFile()], + sourcemap: true + } + ], plugins: [ babel({ presets: [['@babel/preset-env', { targets: { node: 14 } }]], diff --git a/packages/babel/src/index.js b/packages/babel/src/index.js index 322f8ada0..6284e824e 100644 --- a/packages/babel/src/index.js +++ b/packages/babel/src/index.js @@ -1,11 +1,13 @@ +import { cpus } from 'os'; +import { fileURLToPath } from 'url'; + import * as babel from '@babel/core'; import { createFilter } from '@rollup/pluginutils'; +import workerpool from 'workerpool'; import { BUNDLED, HELPERS } from './constants.js'; -import bundledHelpersPlugin from './bundledHelpersPlugin.js'; -import preflightCheck from './preflightCheck.js'; import transformCode from './transformCode.js'; -import { addBabelPlugin, escapeRegExpCharacters, warnOnce } from './utils.js'; +import { escapeRegExpCharacters, warnOnce } from './utils.js'; const unpackOptions = ({ extensions = babel.DEFAULT_EXTENSIONS, @@ -100,6 +102,68 @@ const returnObject = () => { return {}; }; +function findNonSerializableOption(obj) { + const isSerializable = (value) => { + if (value === null) return true; + if (Array.isArray(value)) return value.every(isSerializable); + switch (typeof value) { + case 'undefined': + case 'string': + case 'number': + case 'boolean': + return true; + case 'object': + return Object.values(value).every(isSerializable); + default: + return false; + } + }; + + for (const key of Object.keys(obj)) { + if (!isSerializable(obj[key])) return key; + } + return null; +} + +const WORKER_PATH = fileURLToPath(new URL('./worker.js', import.meta.url)); + +function createParallelWorkerPool(parallel, overrides) { + if (typeof parallel === 'number' && (!Number.isInteger(parallel) || parallel < 1)) { + throw new Error( + 'The "parallel" option must be true or a positive integer specifying the number of workers.' + ); + } + + if (!parallel) return null; + + if (overrides?.config) { + throw new Error('Cannot use "parallel" mode with a custom "config" override.'); + } + if (overrides?.result) { + throw new Error('Cannot use "parallel" mode with a custom "result" override.'); + } + + // Default limits to 4 workers. Benefits diminish after this point, because of the setup cost. + const workerCount = typeof parallel === 'number' ? parallel : Math.min(cpus().length, 4); + return workerpool.pool(WORKER_PATH, { + maxWorkers: workerCount, + workerType: 'thread' + }); +} + +function transformWithWorkerPool(workerPool, context, transformOpts, babelOptions) { + const nonSerializableKey = findNonSerializableOption(babelOptions); + if (nonSerializableKey) { + return Promise.reject( + new Error( + `Cannot use "parallel" mode because the "${nonSerializableKey}" option is not serializable.` + ) + ); + } + + return workerPool.exec('transform', [transformOpts]).catch((err) => context.error(err.message)); +} + function createBabelInputPluginFactory(customCallback = returnObject) { const overrides = customCallback(babel); @@ -116,9 +180,12 @@ function createBabelInputPluginFactory(customCallback = returnObject) { include, filter: customFilter, skipPreflightCheck, + parallel, ...babelOptions } = unpackInputPluginOptions(pluginOptionsWithOverrides); + const workerPool = createParallelWorkerPool(parallel, overrides); + const extensionRegExp = new RegExp( `(${extensions.map(escapeRegExpCharacters).join('|')})(\\?.*)?(#.*)?$` ); @@ -162,23 +229,45 @@ function createBabelInputPluginFactory(customCallback = returnObject) { if (!(await filter(filename, code))) return null; if (filename === HELPERS) return null; - return transformCode( - code, - { ...babelOptions, filename }, - overrides, + const resolvedBabelOptions = { ...babelOptions, filename }; + + if (workerPool) { + return transformWithWorkerPool( + workerPool, + this, + { + inputCode: code, + babelOptions: resolvedBabelOptions, + skipPreflightCheck, + babelHelpers + }, + resolvedBabelOptions + ); + } + + return transformCode({ + inputCode: code, + babelOptions: resolvedBabelOptions, + overrides: { + config: overrides.config?.bind(this), + result: overrides.result?.bind(this) + }, customOptions, - this, - async (transformOptions) => { - if (!skipPreflightCheck) { - await preflightCheck(this, babelHelpers, transformOptions); - } - - return babelHelpers === BUNDLED - ? addBabelPlugin(transformOptions, bundledHelpersPlugin) - : transformOptions; - } - ); + error: this.error.bind(this), + skipPreflightCheck, + babelHelpers + }); + } + }, + + async closeBundle() { + if (!this.meta.watchMode) { + await workerPool?.terminate(); } + }, + + async closeWatcher() { + await workerPool?.terminate(); } }; }; @@ -207,6 +296,8 @@ function createBabelOutputPluginFactory(customCallback = returnObject) { overrides ); + const workerPool = createParallelWorkerPool(pluginOptionsWithOverrides.parallel, overrides); + // cache for chunk name filter (includeChunks/excludeChunks) let chunkNameFilter; @@ -242,6 +333,7 @@ function createBabelOutputPluginFactory(customCallback = returnObject) { externalHelpers, externalHelpersWhitelist, include, + parallel, runtimeHelpers, ...babelOptions } = unpackOutputPluginOptions(pluginOptionsWithOverrides, outputOptions); @@ -257,7 +349,36 @@ function createBabelOutputPluginFactory(customCallback = returnObject) { } } - return transformCode(code, babelOptions, overrides, customOptions, this); + if (workerPool) { + return transformWithWorkerPool( + workerPool, + this, + { + inputCode: code, + babelOptions, + skipPreflightCheck: true + }, + babelOptions + ); + } + + return transformCode({ + inputCode: code, + babelOptions, + overrides: { + config: overrides.config?.bind(this), + result: overrides.result?.bind(this) + }, + customOptions, + error: this.error.bind(this), + skipPreflightCheck: true + }); + }, + + async generateBundle() { + if (!this.meta.watchMode) { + await workerPool?.terminate(); + } } }; }; diff --git a/packages/babel/src/preflightCheck.js b/packages/babel/src/preflightCheck.js index 1ca8c1c1c..ae1e43407 100644 --- a/packages/babel/src/preflightCheck.js +++ b/packages/babel/src/preflightCheck.js @@ -37,27 +37,27 @@ const mismatchError = (actual, expected, filename) => // Revert to /\/helpers\/(esm\/)?inherits/ when Babel 8 gets released, this was fixed in https://github.com/babel/babel/issues/14185 const inheritsHelperRe = /[\\/]+helpers[\\/]+(esm[\\/]+)?inherits/; -export default async function preflightCheck(ctx, babelHelpers, transformOptions) { +export default async function preflightCheck(error, babelHelpers, transformOptions) { const finalOptions = addBabelPlugin(transformOptions, helpersTestTransform); const check = (await babel.transformAsync(PREFLIGHT_INPUT, finalOptions)).code; // Babel sometimes splits ExportDefaultDeclaration into 2 statements, so we also check for ExportNamedDeclaration if (!/export (d|{)/.test(check)) { - ctx.error(MODULE_ERROR); + error(MODULE_ERROR); } if (inheritsHelperRe.test(check)) { if (babelHelpers === RUNTIME) { return; } - ctx.error(mismatchError(RUNTIME, babelHelpers, transformOptions.filename)); + error(mismatchError(RUNTIME, babelHelpers, transformOptions.filename)); } if (check.includes('babelHelpers.inherits')) { if (babelHelpers === EXTERNAL) { return; } - ctx.error(mismatchError(EXTERNAL, babelHelpers, transformOptions.filename)); + error(mismatchError(EXTERNAL, babelHelpers, transformOptions.filename)); } // test unminifiable string content @@ -66,12 +66,12 @@ export default async function preflightCheck(ctx, babelHelpers, transformOptions return; } if (babelHelpers === RUNTIME && !transformOptions.plugins.length) { - ctx.error( + error( `You must use the \`@babel/plugin-transform-runtime\` plugin when \`babelHelpers\` is "${RUNTIME}".\n` ); } - ctx.error(mismatchError(INLINE, babelHelpers, transformOptions.filename)); + error(mismatchError(INLINE, babelHelpers, transformOptions.filename)); } - ctx.error(UNEXPECTED_ERROR); + error(UNEXPECTED_ERROR); } diff --git a/packages/babel/src/transformCode.js b/packages/babel/src/transformCode.js index f677ea09c..3be62e11c 100644 --- a/packages/babel/src/transformCode.js +++ b/packages/babel/src/transformCode.js @@ -1,13 +1,19 @@ import * as babel from '@babel/core'; -export default async function transformCode( +import bundledHelpersPlugin from './bundledHelpersPlugin.js'; +import preflightCheck from './preflightCheck.js'; +import { BUNDLED } from './constants.js'; +import { addBabelPlugin } from './utils.js'; + +export default async function transformCode({ inputCode, babelOptions, overrides, customOptions, - ctx, - finalizeOptions -) { + error, + skipPreflightCheck, + babelHelpers +}) { // loadPartialConfigAsync has become available in @babel/core@7.8.0 const config = await (babel.loadPartialConfigAsync || babel.loadPartialConfig)(babelOptions); @@ -16,18 +22,23 @@ export default async function transformCode( return null; } - let transformOptions = !overrides.config + let transformOptions = !overrides?.config ? config.options - : await overrides.config.call(ctx, config, { + : await overrides.config(config, { code: inputCode, customOptions }); - if (finalizeOptions) { - transformOptions = await finalizeOptions(transformOptions); + if (!skipPreflightCheck) { + await preflightCheck(error, babelHelpers, transformOptions); } - if (!overrides.result) { + transformOptions = + babelHelpers === BUNDLED + ? addBabelPlugin(transformOptions, bundledHelpersPlugin) + : transformOptions; + + if (!overrides?.result) { const { code, map } = await babel.transformAsync(inputCode, transformOptions); return { code, @@ -36,7 +47,7 @@ export default async function transformCode( } const result = await babel.transformAsync(inputCode, transformOptions); - const { code, map } = await overrides.result.call(ctx, result, { + const { code, map } = await overrides.result(result, { code: inputCode, customOptions, config, diff --git a/packages/babel/src/worker.js b/packages/babel/src/worker.js new file mode 100644 index 000000000..ee7a93ea1 --- /dev/null +++ b/packages/babel/src/worker.js @@ -0,0 +1,16 @@ +import workerpool from 'workerpool'; + +import transformCode from './transformCode.js'; + +async function transform(opts) { + return transformCode({ + ...opts, + error: (msg) => { + throw new Error(msg); + } + }); +} + +workerpool.worker({ + transform +}); diff --git a/packages/babel/test/as-input-plugin.mjs b/packages/babel/test/as-input-plugin.mjs index 08bfcfef4..c388a5b2e 100644 --- a/packages/babel/test/as-input-plugin.mjs +++ b/packages/babel/test/as-input-plugin.mjs @@ -594,3 +594,94 @@ test('works as a CJS plugin', async () => { const code = await getCode(bundle); expect(code.includes('const')).toBe(false); }); + +test('works in parallel as a CJS plugin', async () => { + const require = createRequire(import.meta.url); + const babelPluginCjs = require('current-package'); + const bundle = await rollup({ + input: `${FIXTURES}basic/main.js`, + plugins: [ + babelPluginCjs({ + babelHelpers: 'bundled', + presets: ['@babel/env'], + parallel: true + }) + ] + }); + const code = await getCode(bundle); + expect(code.includes('const')).toBe(false); + expect(code.includes('var answer')).toBe(true); +}); + +test('works in parallel', async () => { + const bundle = await rollup({ + input: `${FIXTURES}proposal-decorators/main.js`, + plugins: [babelPlugin({ parallel: true })] + }); + const code = await getCode(bundle); + + expect(code.includes('_createClass')).toBe(true); +}); + +test('works in parallel with specified worker count', async () => { + const code = await generate('basic/main.js', { parallel: 2 }); + expect(code.includes('const')).toBe(false); + expect(code.includes('var answer = 42')).toBe(true); +}); + +test('throws when parallel option is not a positive integer', () => { + expect(() => babelPlugin({ babelHelpers: 'bundled', parallel: 0 })).toThrow( + /must be true or a positive integer/ + ); + expect(() => babelPlugin({ babelHelpers: 'bundled', parallel: -1 })).toThrow( + /must be true or a positive integer/ + ); + expect(() => babelPlugin({ babelHelpers: 'bundled', parallel: 2.5 })).toThrow( + /must be true or a positive integer/ + ); + expect(() => babelPlugin({ babelHelpers: 'bundled', parallel: NaN })).toThrow( + /must be true or a positive integer/ + ); +}); + +test('throws when using parallel with non-serializable babel options', async () => { + await expect(() => + generate('basic/main.js', { + parallel: true, + plugins: [ + // Functions are not serializable + function customPlugin() { + return { visitor: {} }; + } + ] + }) + ).rejects.toThrow(/Cannot use "parallel" mode because the "plugins" option is not serializable/); +}); + +test('throws when using parallel with config override', () => { + const customBabelPlugin = createBabelInputPluginFactory(() => { + return { + config(cfg) { + return cfg.options; + } + }; + }); + + expect(() => customBabelPlugin({ babelHelpers: 'bundled', parallel: true })).toThrow( + /Cannot use "parallel" mode with a custom "config" override/ + ); +}); + +test('throws when using parallel with result override', () => { + const customBabelPlugin = createBabelInputPluginFactory(() => { + return { + result(result) { + return result; + } + }; + }); + + expect(() => customBabelPlugin({ babelHelpers: 'bundled', parallel: true })).toThrow( + /Cannot use "parallel" mode with a custom "result" override/ + ); +}); diff --git a/packages/babel/test/as-output-plugin.mjs b/packages/babel/test/as-output-plugin.mjs index 25b5c977a..e33e33bd1 100644 --- a/packages/babel/test/as-output-plugin.mjs +++ b/packages/babel/test/as-output-plugin.mjs @@ -403,3 +403,66 @@ test('allows excluding manual chunks from output transform via `excludeChunks`', // Debug output intentionally omitted expect(codes.some((c) => c.includes('=> 42'))).toBe(true); }); + +test('works in parallel', async () => { + const bundle = await rollup({ input: `${FIXTURES}basic/main.js` }); + const code = await getCode(bundle, { + format: 'es', + plugins: [ + getBabelOutputPlugin({ + presets: ['@babel/env'], + parallel: true + }) + ] + }); + expect(code.includes('const')).toBe(false); +}); + +test('works in parallel with specified worker count', async () => { + const bundle = await rollup({ input: `${FIXTURES}basic/main.js` }); + const code = await getCode(bundle, { + format: 'es', + plugins: [ + getBabelOutputPlugin({ + presets: ['@babel/env'], + parallel: 2 + }) + ] + }); + expect(code.includes('const')).toBe(false); +}); + +test('throws when output plugin parallel option is not a positive integer', () => { + expect(() => getBabelOutputPlugin({ parallel: 0 })).toThrow(/must be true or a positive integer/); + expect(() => getBabelOutputPlugin({ parallel: -1 })).toThrow( + /must be true or a positive integer/ + ); +}); + +test('throws when using output plugin parallel with config override', () => { + const customBabelPlugin = createBabelOutputPluginFactory(() => { + return { + config(cfg) { + return cfg.options; + } + }; + }); + + expect(() => customBabelPlugin({ parallel: true })).toThrow( + /Cannot use "parallel" mode with a custom "config" override/ + ); +}); + +test('throws when using output plugin parallel with result override', () => { + const customBabelPlugin = createBabelOutputPluginFactory(() => { + return { + result(result) { + return result; + } + }; + }); + + expect(() => customBabelPlugin({ parallel: true })).toThrow( + /Cannot use "parallel" mode with a custom "result" override/ + ); +}); diff --git a/packages/babel/types/index.d.ts b/packages/babel/types/index.d.ts index 47ee8c786..5a2f8a144 100644 --- a/packages/babel/types/index.d.ts +++ b/packages/babel/types/index.d.ts @@ -39,6 +39,14 @@ export interface RollupBabelInputPluginOptions * @default false */ skipPreflightCheck?: boolean; + /** + * Enable parallel processing of files in worker threads. This has a setup cost, so is best suited for larger projects. + * Pass an integer to set the number of workers. Set `true` for the default number of workers (based on CPU cores, capped at 4). + * + * This option cannot be used alongside custom overrides or non-serializable Babel options. + * @default false + */ + parallel?: boolean | number; } export interface RollupBabelOutputPluginOptions @@ -58,6 +66,14 @@ export interface RollupBabelOutputPluginOptions * These patterns are matched against the `chunk.name` value in Rollup's `renderChunk` hook. */ excludeChunks?: FilterPattern; + /** + * Enable parallel processing of chunks in worker threads. This has a setup cost, so is best suited for larger projects. + * Pass an integer to set the number of workers. Set `true` for the default number of workers (based on CPU cores, capped at 4). + * + * This option cannot be used alongside custom overrides or non-serializable Babel options. + * @default false + */ + parallel?: boolean | number; } export type RollupBabelCustomInputPluginOptions = ( diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 56792c42b..d112c59ea 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -131,6 +131,9 @@ importers: '@rollup/pluginutils': specifier: ^5.0.1 version: 5.0.1(rollup@4.0.0-24) + workerpool: + specifier: ^9.0.0 + version: 9.3.4 devDependencies: '@babel/core': specifier: ^7.19.1 @@ -5292,6 +5295,9 @@ packages: resolution: {integrity: sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==} engines: {node: '>=0.10.0'} + workerpool@9.3.4: + resolution: {integrity: sha512-TmPRQYYSAnnDiEB0P/Ytip7bFGvqnSU6I2BcuSw7Hx+JSg/DsUi5ebYfc8GYaSdpuvOcEs6dXxPurOYpe9QFwg==} + wrap-ansi@6.2.0: resolution: {integrity: sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==} engines: {node: '>=8'} @@ -10118,6 +10124,8 @@ snapshots: word-wrap@1.2.5: {} + workerpool@9.3.4: {} + wrap-ansi@6.2.0: dependencies: ansi-styles: 4.3.0