From e77e3de402693a216088cb75cdf6a245cfd7d0f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Ribaudo?= Date: Wed, 29 Dec 2021 16:33:12 +0100 Subject: [PATCH] [babel 8] Move `@babel/register` transform to a separate worker (#14025) --- .../cli/--config-file/in-files/index.js | 4 +- .../fixtures/cli/--env-name/in-files/index.js | 4 +- .../fixtures/cli/--presets/in-files/index.js | 4 +- .../test/fixtures/cli/-b/in-files/index.js | 4 +- packages/babel-register/.npmignore | 4 + .../babel-register/experimental-worker.js | 1 + packages/babel-register/package.json | 4 +- packages/babel-register/src/browser.js | 10 + packages/babel-register/src/browser.ts | 5 - packages/babel-register/src/cache.js | 3 + .../babel-register/src/experimental-worker.js | 26 ++ packages/babel-register/src/hook.js | 85 ++++ packages/babel-register/src/index.js | 18 +- .../src/is-in-register-worker.js | 20 + packages/babel-register/src/node.js | 13 + packages/babel-register/src/node.ts | 176 -------- .../src/{nodeWrapper.ts => nodeWrapper.js} | 3 +- packages/babel-register/src/worker-client.js | 110 +++++ .../babel-register/src/worker/babel-core.js | 20 + .../src/{cache.ts => worker/cache.js} | 42 +- .../src/worker/handle-message.js | 20 + packages/babel-register/src/worker/index.js | 28 ++ .../babel-register/src/worker/transform.js | 128 ++++++ .../test/fixtures/babelrc/.babelrc | 5 +- .../test/fixtures/babelrc/log.js | 1 + .../test/fixtures/internal-modules/index.js | 15 +- .../test/fixtures/internal-modules/plugin.js | 13 + .../test/fixtures/mjs-babelrc/.babelrc.mjs | 3 + .../test/fixtures/mjs-babelrc/es2015.js | 1 + packages/babel-register/test/index.js | 401 +++++++++++------- yarn.lock | 1 + 31 files changed, 791 insertions(+), 381 deletions(-) create mode 100644 packages/babel-register/experimental-worker.js create mode 100644 packages/babel-register/src/browser.js delete mode 100644 packages/babel-register/src/browser.ts create mode 100644 packages/babel-register/src/cache.js create mode 100644 packages/babel-register/src/experimental-worker.js create mode 100644 packages/babel-register/src/hook.js create mode 100644 packages/babel-register/src/is-in-register-worker.js create mode 100644 packages/babel-register/src/node.js delete mode 100644 packages/babel-register/src/node.ts rename packages/babel-register/src/{nodeWrapper.ts => nodeWrapper.js} (95%) create mode 100644 packages/babel-register/src/worker-client.js create mode 100644 packages/babel-register/src/worker/babel-core.js rename packages/babel-register/src/{cache.ts => worker/cache.js} (83%) create mode 100644 packages/babel-register/src/worker/handle-message.js create mode 100644 packages/babel-register/src/worker/index.js create mode 100644 packages/babel-register/src/worker/transform.js create mode 100644 packages/babel-register/test/fixtures/babelrc/log.js create mode 100644 packages/babel-register/test/fixtures/internal-modules/plugin.js create mode 100644 packages/babel-register/test/fixtures/mjs-babelrc/.babelrc.mjs create mode 100644 packages/babel-register/test/fixtures/mjs-babelrc/es2015.js diff --git a/packages/babel-node/test/fixtures/cli/--config-file/in-files/index.js b/packages/babel-node/test/fixtures/cli/--config-file/in-files/index.js index 85ce559e8f..4d4dfd2c0f 100644 --- a/packages/babel-node/test/fixtures/cli/--config-file/in-files/index.js +++ b/packages/babel-node/test/fixtures/cli/--config-file/in-files/index.js @@ -1 +1,3 @@ -console.log("foo"); +// See https://github.com/babel/babel/pull/14025#issuecomment-986296424 +// for the reason behind using setImmediate. +setImmediate(() => console.log("foo")); diff --git a/packages/babel-node/test/fixtures/cli/--env-name/in-files/index.js b/packages/babel-node/test/fixtures/cli/--env-name/in-files/index.js index 85ce559e8f..4d4dfd2c0f 100644 --- a/packages/babel-node/test/fixtures/cli/--env-name/in-files/index.js +++ b/packages/babel-node/test/fixtures/cli/--env-name/in-files/index.js @@ -1 +1,3 @@ -console.log("foo"); +// See https://github.com/babel/babel/pull/14025#issuecomment-986296424 +// for the reason behind using setImmediate. +setImmediate(() => console.log("foo")); diff --git a/packages/babel-node/test/fixtures/cli/--presets/in-files/index.js b/packages/babel-node/test/fixtures/cli/--presets/in-files/index.js index 85ce559e8f..4d4dfd2c0f 100644 --- a/packages/babel-node/test/fixtures/cli/--presets/in-files/index.js +++ b/packages/babel-node/test/fixtures/cli/--presets/in-files/index.js @@ -1 +1,3 @@ -console.log("foo"); +// See https://github.com/babel/babel/pull/14025#issuecomment-986296424 +// for the reason behind using setImmediate. +setImmediate(() => console.log("foo")); diff --git a/packages/babel-node/test/fixtures/cli/-b/in-files/index.js b/packages/babel-node/test/fixtures/cli/-b/in-files/index.js index 85ce559e8f..4d4dfd2c0f 100644 --- a/packages/babel-node/test/fixtures/cli/-b/in-files/index.js +++ b/packages/babel-node/test/fixtures/cli/-b/in-files/index.js @@ -1 +1,3 @@ -console.log("foo"); +// See https://github.com/babel/babel/pull/14025#issuecomment-986296424 +// for the reason behind using setImmediate. +setImmediate(() => console.log("foo")); diff --git a/packages/babel-register/.npmignore b/packages/babel-register/.npmignore index f980694583..a298a1ada2 100644 --- a/packages/babel-register/.npmignore +++ b/packages/babel-register/.npmignore @@ -1,3 +1,7 @@ src test *.log + +experimental-worker.js +lib/experimental-worker.js +lib/is-in-register-worker.js diff --git a/packages/babel-register/experimental-worker.js b/packages/babel-register/experimental-worker.js new file mode 100644 index 0000000000..259a28c5c0 --- /dev/null +++ b/packages/babel-register/experimental-worker.js @@ -0,0 +1 @@ +module.exports = require("./lib/experimental-worker"); diff --git a/packages/babel-register/package.json b/packages/babel-register/package.json index eff87e7ee0..afc892576e 100644 --- a/packages/babel-register/package.json +++ b/packages/babel-register/package.json @@ -12,9 +12,10 @@ "directory": "packages/babel-register" }, "author": "The Babel Team (https://babel.dev/team)", + "type": "commonjs", "main": "./lib/index.js", "browser": { - "./lib/nodeWrapper.js": "./lib/browser.js" + "./lib/index.js": "./lib/browser.js" }, "dependencies": { "clone-deep": "^4.0.1", @@ -28,6 +29,7 @@ }, "devDependencies": { "@babel/core": "workspace:^", + "@babel/plugin-transform-arrow-functions": "workspace:^", "@babel/plugin-transform-modules-commonjs": "workspace:^", "browserify": "^16.5.2" }, diff --git a/packages/babel-register/src/browser.js b/packages/babel-register/src/browser.js new file mode 100644 index 0000000000..79a4a7f22f --- /dev/null +++ b/packages/babel-register/src/browser.js @@ -0,0 +1,10 @@ +// required to safely use babel/register within a browserify codebase + +function register() {} + +module.exports = Object.assign(register, { + default: register, + register, + revert: function revert() {}, + __esModule: true, +}); diff --git a/packages/babel-register/src/browser.ts b/packages/babel-register/src/browser.ts deleted file mode 100644 index 638a9bc2e5..0000000000 --- a/packages/babel-register/src/browser.ts +++ /dev/null @@ -1,5 +0,0 @@ -// required to safely use babel/register within a browserify codebase - -export default function register() {} - -export function revert() {} diff --git a/packages/babel-register/src/cache.js b/packages/babel-register/src/cache.js new file mode 100644 index 0000000000..d4ac88a4ae --- /dev/null +++ b/packages/babel-register/src/cache.js @@ -0,0 +1,3 @@ +// File moved to ./worker/cache.js +// TODO: Remove this backward-compat "proxy file" in Babel 8 +module.exports = require("./worker/cache"); diff --git a/packages/babel-register/src/experimental-worker.js b/packages/babel-register/src/experimental-worker.js new file mode 100644 index 0000000000..4f26ec5d88 --- /dev/null +++ b/packages/babel-register/src/experimental-worker.js @@ -0,0 +1,26 @@ +// TODO: Move this file to index.js in Babel 8 + +"use strict"; + +const [major, minor] = process.versions.node.split(".").map(Number); + +if (major < 12 || (major === 12 && minor < 3)) { + throw new Error( + "@babel/register/experimental-worker requires Node.js >= 12.3.0", + ); +} + +const hook = require("./hook"); +const { WorkerClient } = require("./worker-client"); + +const register = hook.register.bind(null, new WorkerClient()); + +module.exports = Object.assign(register, { + revert: hook.revert, + default: register, + __esModule: true, +}); + +if (!require("./is-in-register-worker").isInRegisterWorker) { + register(); +} diff --git a/packages/babel-register/src/hook.js b/packages/babel-register/src/hook.js new file mode 100644 index 0000000000..843670a7c1 --- /dev/null +++ b/packages/babel-register/src/hook.js @@ -0,0 +1,85 @@ +"use strict"; + +const { addHook } = require("pirates"); +const sourceMapSupport = require("source-map-support"); + +let piratesRevert; +const maps = Object.create(null); + +function installSourceMapSupport() { + installSourceMapSupport = () => {}; // eslint-disable-line no-func-assign + + sourceMapSupport.install({ + handleUncaughtExceptions: false, + environment: "node", + retrieveSourceMap(filename) { + const map = maps?.[filename]; + if (map) { + return { url: null, map: map }; + } else { + return null; + } + }, + }); +} + +if (!process.env.BABEL_8_BREAKING) { + // Babel 7 compiles files in the same thread where it hooks `require()`, + // so we must prevent mixing Babel plugin dependencies with the files + // to be compiled. + // All the `!process.env.BABEL_8_BREAKING` code in this file is for + // this purpose. + + const Module = require("module"); + + let compiling = false; + const internalModuleCache = Module._cache; + + // eslint-disable-next-line no-var + var compileBabel7 = function compileBabel7(client, code, filename) { + if (!client.isLocalClient) return compile(client, code, filename); + + if (compiling) return code; + + const globalModuleCache = Module._cache; + try { + compiling = true; + Module._cache = internalModuleCache; + return compile(client, code, filename); + } finally { + compiling = false; + Module._cache = globalModuleCache; + } + }; +} + +function compile(client, inputCode, filename) { + const result = client.transform(inputCode, filename); + + if (result === null) return inputCode; + + const { code, map } = result; + if (map) { + maps[filename] = map; + installSourceMapSupport(); + } + return code; +} + +exports.register = function register(client, opts = {}) { + if (piratesRevert) piratesRevert(); + + piratesRevert = addHook( + (process.env.BABEL_8_BREAKING ? compile : compileBabel7).bind(null, client), + { + exts: opts.extensions ?? client.getDefaultExtensions(), + ignoreNodeModules: false, + }, + ); + + client.setOptions(opts); +}; + +exports.revert = function revert() { + if (piratesRevert) piratesRevert(); +}; diff --git a/packages/babel-register/src/index.js b/packages/babel-register/src/index.js index aea88df53e..3b827d75af 100644 --- a/packages/babel-register/src/index.js +++ b/packages/babel-register/src/index.js @@ -4,12 +4,16 @@ * from a compiled Babel import. */ -exports = module.exports = function (...args) { - return register(...args); -}; -exports.__esModule = true; +if (process.env.BABEL_8_BREAKING) { + module.exports = require("./experimental-worker"); +} else { + exports = module.exports = function (...args) { + return register(...args); + }; + exports.__esModule = true; -const node = require("./nodeWrapper"); -const register = node.default; + const node = require("./nodeWrapper"); + const register = node.default; -Object.assign(exports, node); + Object.assign(exports, node); +} diff --git a/packages/babel-register/src/is-in-register-worker.js b/packages/babel-register/src/is-in-register-worker.js new file mode 100644 index 0000000000..64fd894448 --- /dev/null +++ b/packages/babel-register/src/is-in-register-worker.js @@ -0,0 +1,20 @@ +"use strict"; + +/** + * Since workers inherit the exec options from the parent thread, we + * must be careful to avoid infite "@babel/register" setup loops. + * + * If @babel/register is imported using the -r/--require flag, the worker + * will have the same flag and we must avoid registering the @babel/register + * hook again. + * + * - markInRegisterWorker() can be used to mark a set of env vars (that will + * be forwarded to a worker) as being in the @babel/register worker. + * - isInRegisterWorker will be true in @babel/register workers. + */ + +const envVarName = "___INTERNAL___IS_INSIDE_BABEL_REGISTER_WORKER___"; +const envVarValue = "yes_I_am"; + +exports.markInRegisterWorker = env => ({ ...env, [envVarName]: envVarValue }); +exports.isInRegisterWorker = process.env[envVarName] === envVarValue; diff --git a/packages/babel-register/src/node.js b/packages/babel-register/src/node.js new file mode 100644 index 0000000000..8457f228f7 --- /dev/null +++ b/packages/babel-register/src/node.js @@ -0,0 +1,13 @@ +// TODO: Remove this file in Babel 8 + +"use strict"; + +const hook = require("./hook"); +const { LocalClient } = require("./worker-client"); + +const register = hook.register.bind(null, new LocalClient()); + +module.exports = Object.assign(register, { + revert: hook.revert, + default: register, +}); diff --git a/packages/babel-register/src/node.ts b/packages/babel-register/src/node.ts deleted file mode 100644 index f7990885f0..0000000000 --- a/packages/babel-register/src/node.ts +++ /dev/null @@ -1,176 +0,0 @@ -import cloneDeep from "clone-deep"; -import sourceMapSupport from "source-map-support"; -import * as registerCache from "./cache"; -import * as babel from "@babel/core"; -import { OptionManager, DEFAULT_EXTENSIONS } from "@babel/core"; -import { addHook } from "pirates"; -import fs from "fs"; -import path from "path"; -import Module from "module"; - -const maps = {}; -let transformOpts: any = {}; -let piratesRevert = null; - -function installSourceMapSupport() { - sourceMapSupport.install({ - handleUncaughtExceptions: false, - environment: "node", - retrieveSourceMap(source) { - const map = maps && maps[source]; - if (map) { - return { - url: null, - map: map, - }; - } else { - return null; - } - }, - }); -} - -let cache; - -function mtime(filename) { - return +fs.statSync(filename).mtime; -} - -function compile(code, filename) { - // merge in base options and resolve all the plugins and presets relative to this file - const opts = new OptionManager().init( - // sourceRoot can be overwritten - { - sourceRoot: path.dirname(filename) + path.sep, - ...cloneDeep(transformOpts), - filename, - }, - ); - - // Bail out ASAP if the file has been ignored. - if (opts === null) return code; - - let cacheKey = `${JSON.stringify(opts)}:${babel.version}`; - - const env = babel.getEnv(""); - - if (env) cacheKey += `:${env}`; - - let cached, fileMtime; - if (cache) { - cached = cache[cacheKey]; - fileMtime = mtime(filename); - } - - if (!cached || cached.mtime !== fileMtime) { - cached = babel.transform(code, { - ...opts, - sourceMaps: opts.sourceMaps === undefined ? "both" : opts.sourceMaps, - ast: false, - }); - - if (cache) { - cache[cacheKey] = cached; - cached.mtime = fileMtime; - registerCache.setDirty(); - } - } - - if (cached.map) { - if (Object.keys(maps).length === 0) { - installSourceMapSupport(); - } - maps[filename] = cached.map; - } - - return cached.code; -} - -let compiling = false; -// @ts-expect-error field is missing in type definitions -const internalModuleCache = Module._cache; - -function compileHook(code, filename) { - if (compiling) return code; - - // @ts-expect-error field is missing in type definitions - const globalModuleCache = Module._cache; - try { - compiling = true; - // @ts-expect-error field is missing in type definitions - Module._cache = internalModuleCache; - return compile(code, filename); - } finally { - compiling = false; - // @ts-expect-error field is missing in type definitions - Module._cache = globalModuleCache; - } -} - -function hookExtensions(exts) { - if (piratesRevert) piratesRevert(); - piratesRevert = addHook(compileHook, { exts, ignoreNodeModules: false }); -} - -export function revert() { - if (piratesRevert) piratesRevert(); -} - -function escapeRegExp(string) { - return string.replace(/[|\\{}()[\]^$+*?.]/g, "\\$&"); -} - -export default function register(opts: any = {}) { - // Clone to avoid mutating the arguments object with the 'delete's below. - opts = { - ...opts, - }; - hookExtensions(opts.extensions || DEFAULT_EXTENSIONS); - - if (opts.cache === false && cache) { - registerCache.clear(); - cache = null; - } else if (opts.cache !== false && !cache) { - registerCache.load(); - cache = registerCache.get(); - } - - delete opts.extensions; - delete opts.cache; - - transformOpts = { - ...opts, - caller: { - name: "@babel/register", - ...(opts.caller || {}), - }, - }; - - let { cwd = "." } = transformOpts; - - // Ensure that the working directory is resolved up front so that - // things don't break if it changes later. - cwd = transformOpts.cwd = path.resolve(cwd); - - if (transformOpts.ignore === undefined && transformOpts.only === undefined) { - transformOpts.only = [ - // Only compile things inside the current working directory. - // $FlowIgnore - new RegExp("^" + escapeRegExp(cwd), "i"), - ]; - transformOpts.ignore = [ - // Ignore any node_modules inside the current working directory. - new RegExp( - "^" + - // $FlowIgnore - escapeRegExp(cwd) + - "(?:" + - path.sep + - ".*)?" + - // $FlowIgnore - escapeRegExp(path.sep + "node_modules" + path.sep), - "i", - ), - ]; - } -} diff --git a/packages/babel-register/src/nodeWrapper.ts b/packages/babel-register/src/nodeWrapper.js similarity index 95% rename from packages/babel-register/src/nodeWrapper.ts rename to packages/babel-register/src/nodeWrapper.js index 27b3cead2f..d2af9acac1 100644 --- a/packages/babel-register/src/nodeWrapper.ts +++ b/packages/babel-register/src/nodeWrapper.js @@ -5,7 +5,8 @@ * and allows register to transform these modules if they are loaded externally. */ -// @ts-ignore todo(flow->ts) convert to esm +// TODO: Remove this file in Babel 8 + const Module = require("module"); const globalModuleCache = Module._cache; diff --git a/packages/babel-register/src/worker-client.js b/packages/babel-register/src/worker-client.js new file mode 100644 index 0000000000..c61e836a19 --- /dev/null +++ b/packages/babel-register/src/worker-client.js @@ -0,0 +1,110 @@ +const path = require("path"); + +const ACTIONS = { + GET_DEFAULT_EXTENSIONS: "GET_DEFAULT_EXTENSIONS", + SET_OPTIONS: "SET_OPTIONS", + TRANSFORM: "TRANSFORM", + TRANSFORM_SYNC: "TRANSFORM_SYNC", +}; + +class Client { + #send; + + constructor(send) { + this.#send = send; + } + + #eCache; + /** @return {string[]} */ + getDefaultExtensions() { + return (this.#eCache ??= this.#send( + ACTIONS.GET_DEFAULT_EXTENSIONS, + undefined, + )); + } + + /** + * @param {object} options + * @return {void} + */ + setOptions(options) { + return this.#send(ACTIONS.SET_OPTIONS, options); + } + + /** + * @param {string} code + * @param {string} filename + * @return {{ code: string, map: object } | null} + */ + transform(code, filename) { + return this.#send(ACTIONS.TRANSFORM, { code, filename }); + } +} + +// We need to run Babel in a worker because require hooks must +// run synchronously, but many steps of Babel's config loading +// (which is done for each file) can be asynchronous +exports.WorkerClient = class WorkerClient extends Client { + // These two require() calls are in deferred so that they are not imported in + // older Node.js versions (which don't support workers). + // TODO: Hoist them in Babel 8. + + /** @type {typeof import("worker_threads")} */ + static get #worker_threads() { + return require("worker_threads"); + } + + static get #markInRegisterWorker() { + return require("./is-in-register-worker").markInRegisterWorker; + } + + #worker = new WorkerClient.#worker_threads.Worker( + path.resolve(__dirname, "./worker/index.js"), + { env: WorkerClient.#markInRegisterWorker(process.env) }, + ); + + #signal = new Int32Array(new SharedArrayBuffer(4)); + + constructor() { + super((action, payload) => { + this.#signal[0] = 0; + const subChannel = new WorkerClient.#worker_threads.MessageChannel(); + + this.#worker.postMessage( + { signal: this.#signal, port: subChannel.port1, action, payload }, + [subChannel.port1], + ); + + Atomics.wait(this.#signal, 0, 0); + const { message } = WorkerClient.#worker_threads.receiveMessageOnPort( + subChannel.port2, + ); + + if (message.error) throw Object.assign(message.error, message.errorData); + else return message.result; + }); + + // The worker will never exit by itself. Prevent it from keeping + // the main process alive. + this.#worker.unref(); + } +}; + +if (!process.env.BABEL_8_BREAKING) { + exports.LocalClient = class LocalClient extends Client { + isLocalClient = true; + + static #handleMessage; + + constructor() { + LocalClient.#handleMessage ??= require("./worker/handle-message"); + + super((action, payload) => { + return LocalClient.#handleMessage( + action === ACTIONS.TRANSFORM ? ACTIONS.TRANSFORM_SYNC : action, + payload, + ); + }); + } + }; +} diff --git a/packages/babel-register/src/worker/babel-core.js b/packages/babel-register/src/worker/babel-core.js new file mode 100644 index 0000000000..ea3bc3c2e7 --- /dev/null +++ b/packages/babel-register/src/worker/babel-core.js @@ -0,0 +1,20 @@ +function initialize(babel) { + exports.init = null; + exports.version = babel.version; + exports.DEFAULT_EXTENSIONS = babel.DEFAULT_EXTENSIONS; + exports.loadOptionsAsync = babel.loadOptionsAsync; + exports.transformAsync = babel.transformAsync; + exports.getEnv = babel.getEnv; + + if (!process.env.BABEL_8_BREAKING) { + exports.loadOptionsSync = babel.loadOptionsSync; + exports.transformSync = babel.transformSync; + } +} + +if (process.env.BABEL_8_BREAKING) { + // @ts-expect-error CJS-ESM interop. + exports.init = import("@babel/core").then(ns => initialize(ns.default)); +} else { + initialize(require("@babel/core")); +} diff --git a/packages/babel-register/src/cache.ts b/packages/babel-register/src/worker/cache.js similarity index 83% rename from packages/babel-register/src/cache.ts rename to packages/babel-register/src/worker/cache.js index 80495ae782..769fddd268 100644 --- a/packages/babel-register/src/cache.ts +++ b/packages/babel-register/src/worker/cache.js @@ -1,8 +1,10 @@ -import path from "path"; -import fs from "fs"; -import os from "os"; -import * as babel from "@babel/core"; -import findCacheDir from "find-cache-dir"; +"use strict"; + +const path = require("path"); +const fs = require("fs"); +const os = require("os"); +const babel = require("@babel/core"); +const findCacheDir = require("find-cache-dir"); const DEFAULT_CACHE_DIR = findCacheDir({ name: "@babel/register" }) || os.homedir() || os.tmpdir(); @@ -10,8 +12,9 @@ const DEFAULT_FILENAME = path.join( DEFAULT_CACHE_DIR, `.babel.${babel.version}.${babel.getEnv()}.json`, ); -const FILENAME: string = process.env.BABEL_CACHE_PATH || DEFAULT_FILENAME; -let data: any = {}; + +const FILENAME = process.env.BABEL_CACHE_PATH || DEFAULT_FILENAME; +let data = {}; let cacheDirty = false; @@ -20,15 +23,16 @@ let cacheDisabled = false; function isCacheDisabled() { return process.env.BABEL_DISABLE_CACHE ?? cacheDisabled; } + +exports.save = save; /** * Write stringified cache to disk. */ - -export function save() { +function save() { if (isCacheDisabled() || !cacheDirty) return; cacheDirty = false; - let serialised: string = "{}"; + let serialised = "{}"; try { serialised = JSON.stringify(data, null, " "); @@ -74,7 +78,7 @@ because it resides in a readonly filesystem. Cache is disabled.`, * Load cache from disk and parse. */ -export function load() { +exports.load = function load() { if (isCacheDisabled()) { data = {}; return; @@ -106,27 +110,25 @@ due to a permission issue. Cache is disabled.`, try { data = JSON.parse(cacheContent); } catch {} -} +}; /** * Retrieve data from cache. */ - -export function get(): any { +exports.get = function get() { return data; -} +}; /** * Set the cache dirty bit. */ -export function setDirty() { +exports.setDirty = function setDirty() { cacheDirty = true; -} +}; /** * Clear the cache object. */ - -export function clear() { +exports.clear = function clear() { data = {}; -} +}; diff --git a/packages/babel-register/src/worker/handle-message.js b/packages/babel-register/src/worker/handle-message.js new file mode 100644 index 0000000000..08ce0fe619 --- /dev/null +++ b/packages/babel-register/src/worker/handle-message.js @@ -0,0 +1,20 @@ +const babel = require("./babel-core"); +const { setOptions, transform, transformSync } = require("./transform"); + +module.exports = function handleMessage(action, payload) { + switch (action) { + case "GET_DEFAULT_EXTENSIONS": + return babel.DEFAULT_EXTENSIONS; + case "SET_OPTIONS": + setOptions(payload); + return; + case "TRANSFORM": + return transform(payload.code, payload.filename); + case "TRANSFORM_SYNC": + if (!process.env.BABEL_8_BREAKING) { + return transformSync(payload.code, payload.filename); + } + } + + throw new Error(`Unknown internal parser worker action: ${action}`); +}; diff --git a/packages/babel-register/src/worker/index.js b/packages/babel-register/src/worker/index.js new file mode 100644 index 0000000000..8af8e65a1f --- /dev/null +++ b/packages/babel-register/src/worker/index.js @@ -0,0 +1,28 @@ +const babel = require("./babel-core"); +const handleMessage = require("./handle-message"); + +const { parentPort } = require("worker_threads"); + +parentPort.addListener("message", async ({ signal, port, action, payload }) => { + let response; + + try { + if (babel.init) await babel.init; + + response = { result: await handleMessage(action, payload) }; + } catch (error) { + response = { error, errorData: { ...error } }; + } + + try { + port.postMessage(response); + } catch { + port.postMessage({ + error: new Error("Cannot serialize worker response"), + }); + } finally { + port.close(); + Atomics.store(signal, 0, 1); + Atomics.notify(signal, 0); + } +}); diff --git a/packages/babel-register/src/worker/transform.js b/packages/babel-register/src/worker/transform.js new file mode 100644 index 0000000000..0a79370b19 --- /dev/null +++ b/packages/babel-register/src/worker/transform.js @@ -0,0 +1,128 @@ +"use strict"; + +const cloneDeep = require("clone-deep"); +const path = require("path"); +const fs = require("fs"); + +const babel = require("./babel-core"); +const registerCache = require("../cache"); + +const nmRE = escapeRegExp(path.sep + "node_modules" + path.sep); + +function escapeRegExp(string) { + return string.replace(/[|\\{}()[\]^$+*?.]/g, "\\$&"); +} + +let cache; +let transformOpts; +exports.setOptions = function (opts) { + if (opts.cache === false && cache) { + registerCache.clear(); + cache = null; + } else if (opts.cache !== false && !cache) { + registerCache.load(); + cache = registerCache.get(); + } + + delete opts.cache; + delete opts.extensions; + + transformOpts = { + ...opts, + caller: { + name: "@babel/register", + ...(opts.caller || {}), + }, + }; + + let { cwd = "." } = transformOpts; + + // Ensure that the working directory is resolved up front so that + // things don't break if it changes later. + cwd = transformOpts.cwd = path.resolve(cwd); + + if (transformOpts.ignore === undefined && transformOpts.only === undefined) { + const cwdRE = escapeRegExp(cwd); + + // Only compile things inside the current working directory. + transformOpts.only = [new RegExp("^" + cwdRE, "i")]; + // Ignore any node_modules inside the current working directory. + transformOpts.ignore = [ + new RegExp(`^${cwdRE}(?:${path.sep}.*)?${nmRE}`, "i"), + ]; + } +}; + +exports.transform = async function (input, filename) { + const opts = await babel.loadOptionsAsync({ + // sourceRoot can be overwritten + sourceRoot: path.dirname(filename) + path.sep, + ...cloneDeep(transformOpts), + filename, + }); + + // Bail out ASAP if the file has been ignored. + if (opts === null) return null; + + const { cached, store } = cacheLookup(opts, filename); + if (cached) return cached; + + const { code, map } = await babel.transformAsync(input, { + ...opts, + sourceMaps: opts.sourceMaps === undefined ? "both" : opts.sourceMaps, + ast: false, + }); + + return store({ code, map }); +}; + +if (!process.env.BABEL_8_BREAKING) { + exports.transformSync = function (input, filename) { + const opts = babel.loadOptionsSync({ + // sourceRoot can be overwritten + sourceRoot: path.dirname(filename) + path.sep, + ...cloneDeep(transformOpts), + filename, + }); + + // Bail out ASAP if the file has been ignored. + if (opts === null) return null; + + const { cached, store } = cacheLookup(opts, filename); + if (cached) return cached; + + const { code, map } = babel.transformSync(input, { + ...opts, + sourceMaps: opts.sourceMaps === undefined ? "both" : opts.sourceMaps, + ast: false, + }); + + return store({ code, map }); + }; +} + +const id = value => value; + +function cacheLookup(opts, filename) { + if (!cache) return { cached: null, store: id }; + + let cacheKey = `${JSON.stringify(opts)}:${babel.version}`; + + const env = babel.getEnv(); + if (env) cacheKey += `:${env}`; + + const cached = cache[cacheKey]; + const fileMtime = +fs.statSync(filename).mtime; + + if (cached && cached.mtime === fileMtime) { + return { cached: cached.value, store: id }; + } + + return { + cached: null, + store(value) { + cache[cacheKey] = { value, mtime: fileMtime }; + return value; + }, + }; +} diff --git a/packages/babel-register/test/fixtures/babelrc/.babelrc b/packages/babel-register/test/fixtures/babelrc/.babelrc index b3eb3ec8ef..3da31e668f 100644 --- a/packages/babel-register/test/fixtures/babelrc/.babelrc +++ b/packages/babel-register/test/fixtures/babelrc/.babelrc @@ -1,5 +1,6 @@ { "plugins": [ - "@babel/transform-modules-commonjs" + "@babel/transform-modules-commonjs", + "@babel/plugin-transform-arrow-functions" ] -} \ No newline at end of file +} diff --git a/packages/babel-register/test/fixtures/babelrc/log.js b/packages/babel-register/test/fixtures/babelrc/log.js new file mode 100644 index 0000000000..ebda740147 --- /dev/null +++ b/packages/babel-register/test/fixtures/babelrc/log.js @@ -0,0 +1 @@ +console.log("It worked!", (() => {}).toString()); diff --git a/packages/babel-register/test/fixtures/internal-modules/index.js b/packages/babel-register/test/fixtures/internal-modules/index.js index 0be69180a9..98c19c3198 100644 --- a/packages/babel-register/test/fixtures/internal-modules/index.js +++ b/packages/babel-register/test/fixtures/internal-modules/index.js @@ -1,23 +1,10 @@ const register = require('../../..'); -// Plugin to add '/* transformed */' comment to start of function bodies -const plugin = () => ( { - visitor: { - Function(path) { - const bodyNode = path.node.body; - (bodyNode.leadingComments || (bodyNode.leadingComments = [])).push( { - type: 'CommentBlock', - value: ' transformed ' - } ); - }, - }, -} ); - register( { ignore: [], babelrc: false, configFile: false, - plugins: [plugin] + plugins: [require.resolve("./plugin")] } ); console.log( diff --git a/packages/babel-register/test/fixtures/internal-modules/plugin.js b/packages/babel-register/test/fixtures/internal-modules/plugin.js new file mode 100644 index 0000000000..3ce3594b74 --- /dev/null +++ b/packages/babel-register/test/fixtures/internal-modules/plugin.js @@ -0,0 +1,13 @@ +// Plugin to add '/* transformed */' comment to start of function bodies + +module.exports = () => ( { + visitor: { + Function(path) { + const bodyNode = path.node.body; + (bodyNode.leadingComments || (bodyNode.leadingComments = [])).push( { + type: 'CommentBlock', + value: ' transformed ' + } ); + }, + }, +} ); diff --git a/packages/babel-register/test/fixtures/mjs-babelrc/.babelrc.mjs b/packages/babel-register/test/fixtures/mjs-babelrc/.babelrc.mjs new file mode 100644 index 0000000000..51009202ac --- /dev/null +++ b/packages/babel-register/test/fixtures/mjs-babelrc/.babelrc.mjs @@ -0,0 +1,3 @@ +export default { + plugins: ["@babel/transform-modules-commonjs"], +}; diff --git a/packages/babel-register/test/fixtures/mjs-babelrc/es2015.js b/packages/babel-register/test/fixtures/mjs-babelrc/es2015.js new file mode 100644 index 0000000000..69571eb70b --- /dev/null +++ b/packages/babel-register/test/fixtures/mjs-babelrc/es2015.js @@ -0,0 +1 @@ +import "assert"; diff --git a/packages/babel-register/test/index.js b/packages/babel-register/test/index.js index c813ddce47..ba4b0250a1 100644 --- a/packages/babel-register/test/index.js +++ b/packages/babel-register/test/index.js @@ -7,10 +7,12 @@ import { fileURLToPath } from "url"; const dirname = path.dirname(fileURLToPath(import.meta.url)); const require = createRequire(import.meta.url); -const registerFile = require.resolve("../lib/index"); const testCacheFilename = path.join(dirname, ".index.babel"); const testFile = require.resolve("./fixtures/babelrc/es2015"); -const testFileContent = fs.readFileSync(testFile); +const testFileLog = require.resolve("./fixtures/babelrc/log"); +const testFileMjs = require.resolve("./fixtures/mjs-babelrc/es2015"); +const testFileContent = fs.readFileSync(testFile, "utf-8"); +const testFileMjsContent = fs.readFileSync(testFileMjs, "utf-8"); const piratesPath = require.resolve("pirates"); const smsPath = require.resolve("source-map-support"); @@ -57,48 +59,13 @@ describe("@babel/register", function () { }, }; - let babelRegister; - - function setupRegister(config = { babelrc: false }) { - process.env.BABEL_CACHE_PATH = testCacheFilename; - config = { - cwd: path.dirname(testFile), - ...config, - }; - - babelRegister = require(registerFile); - babelRegister.default(config); - } - - function revertRegister() { - if (babelRegister) { - babelRegister.revert(); - delete require.cache[registerFile]; - babelRegister = null; - } - cleanCache(); - } - beforeEach(() => { currentHook = null; currentOptions = null; sourceMapSupport = false; }); - afterEach(async () => { - // @babel/register saves the cache on process.nextTick. - // We need to wait for at least one tick so that when jest - // tears down the testing environment @babel/register has - // already finished. - await new Promise(setImmediate); - - revertRegister(); - }); - - afterAll(() => { - resetCache(); - }); - + let originalRequireCacheDescriptor; if (OLD_JEST_MOCKS) { jest.doMock("pirates", () => mocks["pirates"]); jest.doMock("source-map-support", () => mocks["source-map-support"]); @@ -107,7 +74,6 @@ describe("@babel/register", function () { jest.resetModules(); }); } else { - let originalRequireCacheDescriptor; beforeAll(() => { originalRequireCacheDescriptor = Object.getOwnPropertyDescriptor( Module, @@ -115,142 +81,275 @@ describe("@babel/register", function () { ); }); - beforeEach(() => { - const isEmptyObj = obj => - Object.getPrototypeOf(obj) === null && Object.keys(obj).length === 0; - - // This setter intercepts the Module._cache assignment in - // packages/babel-register/src/nodeWrapper.js to install in the - // internal isolated cache. - const emptyInitialCache = {}; - Object.defineProperty(Module, "_cache", { - get: () => emptyInitialCache, - set(value) { - expect(isEmptyObj(value)).toBe(true); - - Object.defineProperty(Module, "_cache", { - value, - enumerable: originalRequireCacheDescriptor.enumerable, - configurable: originalRequireCacheDescriptor.configurable, - writable: originalRequireCacheDescriptor.writable, - }); - value[piratesPath] = { exports: mocks["pirates"] }; - value[smsPath] = { exports: mocks["source-map-support"] }; - }, - enumerable: originalRequireCacheDescriptor.enumerable, - configurable: originalRequireCacheDescriptor.configurable, - }); - }); - afterAll(() => { Object.defineProperty(Module, "_cache", originalRequireCacheDescriptor); }); } - test("registers hook correctly", () => { - setupRegister(); + if (!process.env.BABEL_8_BREAKING) { + describe("babel 7", () => { + if (!OLD_JEST_MOCKS) { + beforeEach(() => { + const isEmptyObj = obj => + Object.getPrototypeOf(obj) === null && + Object.keys(obj).length === 0; - expect(typeof currentHook).toBe("function"); - expect(currentOptions).toEqual(defaultOptions); - }); + // This setter intercepts the Module._cache assignment in + // packages/babel-register/src/nodeWrapper.js to install in the + // internal isolated cache. + const emptyInitialCache = {}; + Object.defineProperty(Module, "_cache", { + get: () => emptyInitialCache, + set(value) { + expect(isEmptyObj(value)).toBe(true); - test("unregisters hook correctly", () => { - setupRegister(); - revertRegister(); + Object.defineProperty(Module, "_cache", { + value, + enumerable: originalRequireCacheDescriptor.enumerable, + configurable: originalRequireCacheDescriptor.configurable, + writable: originalRequireCacheDescriptor.writable, + }); + value[piratesPath] = { exports: mocks["pirates"] }; + value[smsPath] = { exports: mocks["source-map-support"] }; + }, + enumerable: originalRequireCacheDescriptor.enumerable, + configurable: originalRequireCacheDescriptor.configurable, + }); + }); + } - expect(currentHook).toBeNull(); - expect(currentOptions).toBeNull(); - }); - - test("installs source map support by default", () => { - setupRegister(); - - currentHook("const a = 1;", testFile); - - expect(sourceMapSupport).toBe(true); - }); - - test("installs source map support when requested", () => { - setupRegister({ - babelrc: false, - sourceMaps: true, + buildTests(require.resolve("..")); }); + } - currentHook("const a = 1;", testFile); + const nodeGte12 = (fn, ...args) => { + // "minNodeVersion": "8.0.0" <-- For Ctrl+F when dropping node 6-8-10 + const testFn = /v(?:6|8|10)\./.test(process.version) ? fn.skip : fn; + testFn(...args); + }; - expect(sourceMapSupport).toBe(true); - }); + nodeGte12(describe, "worker", () => { + if (!OLD_JEST_MOCKS) { + beforeEach(() => { + Object.defineProperty(Module, "_cache", { + ...originalRequireCacheDescriptor, + value: { + [piratesPath]: { exports: mocks["pirates"] }, + [smsPath]: { exports: mocks["source-map-support"] }, + }, + }); + }); + } - test("does not install source map support if asked not to", () => { - setupRegister({ - babelrc: false, - sourceMaps: false, - }); - - currentHook("const a = 1;", testFile); - - expect(sourceMapSupport).toBe(false); - }); - - it("returns concatenatable sourceRoot and sources", async () => { - // The Source Maps R3 standard https://sourcemaps.info/spec.html states - // that `sourceRoot` is “prepended to the individual entries in the - // ‘source’ field.” If `sources` contains file names, and `sourceRoot` - // is intended to refer to a directory but doesn’t end with a trailing - // slash, any consumers of the source map are in for a bad day. - // - // The underlying problem seems to only get triggered if one file - // requires() another with @babel/register active, and I couldn’t get - // that working inside a test, possibly because of jest’s mocking - // hooks, so we spawn a separate process. - const output = await spawnNodeAsync([ - "-r", - registerFile, - require.resolve("./fixtures/source-map/index"), - ]); - const sourceMap = JSON.parse(output); - expect(sourceMap.map.sourceRoot + sourceMap.map.sources[0]).toBe( - require.resolve("./fixtures/source-map/foo/bar"), + const { setupRegister } = buildTests( + require.resolve("../experimental-worker"), ); + + it("works with mjs config files", () => { + setupRegister({ + babelrc: true, + sourceMaps: false, + cwd: path.dirname(testFileMjs), + }); + + const result = currentHook(testFileMjsContent, testFileMjs); + + expect(result).toBe('"use strict";\n\nrequire("assert");'); + }); }); - test("hook transpiles with config", () => { - setupRegister({ - babelrc: false, - sourceMaps: false, - plugins: ["@babel/transform-modules-commonjs"], + function buildTests(registerFile) { + let babelRegister; + + function setupRegister(config = { babelrc: false }) { + process.env.BABEL_CACHE_PATH = testCacheFilename; + config = { + cwd: path.dirname(testFile), + ...config, + }; + + babelRegister = require(registerFile); + babelRegister.default(config); + } + + function revertRegister() { + if (babelRegister) { + babelRegister.revert(); + delete require.cache[registerFile]; + babelRegister = null; + } + cleanCache(); + } + + afterEach(async () => { + // @babel/register saves the cache on process.nextTick. + // We need to wait for at least one tick so that when jest + // tears down the testing environment @babel/register has + // already finished. + await new Promise(setImmediate); + + revertRegister(); }); - const result = currentHook(testFileContent, testFile); - - expect(result).toBe('"use strict";\n\nrequire("assert");'); - }); - - test("hook transpiles with babelrc", () => { - setupRegister({ - babelrc: true, - sourceMaps: false, + afterAll(() => { + resetCache(); }); - const result = currentHook(testFileContent, testFile); + test("registers hook correctly", () => { + setupRegister(); - expect(result).toBe('"use strict";\n\nrequire("assert");'); - }); + expect(typeof currentHook).toBe("function"); + expect(currentOptions).toEqual(defaultOptions); + }); - test("transforms modules used within register", async () => { - // Need a clean environment without `convert-source-map` - // already in the require cache, so we spawn a separate process + test("unregisters hook correctly", () => { + setupRegister(); + revertRegister(); - const output = await spawnNodeAsync([ - require.resolve("./fixtures/internal-modules/index.js"), - ]); - const { convertSourceMap } = JSON.parse(output); - expect(convertSourceMap).toMatch("/* transformed */"); - }); + expect(currentHook).toBeNull(); + expect(currentOptions).toBeNull(); + }); + + test("installs source map support by default", () => { + setupRegister(); + + currentHook("const a = 1;", testFile); + + expect(sourceMapSupport).toBe(true); + }); + + test("installs source map support when requested", () => { + setupRegister({ + babelrc: false, + sourceMaps: true, + }); + + currentHook("const a = 1;", testFile); + + expect(sourceMapSupport).toBe(true); + }); + + test("does not install source map support if asked not to", () => { + setupRegister({ + babelrc: false, + sourceMaps: false, + }); + + currentHook("const a = 1;", testFile); + + expect(sourceMapSupport).toBe(false); + }); + + describe("node auto-require", () => { + it("works with the -r flag", async () => { + const output = await spawnNodeAsync( + ["-r", registerFile, testFileLog], + path.dirname(testFileLog), + ); + + expect(output.trim()).toMatchInlineSnapshot( + `"It worked! function () {}"`, + ); + }); + + it("works with the --require flag", async () => { + const output = await spawnNodeAsync( + ["--require", registerFile, testFileLog], + path.dirname(testFileLog), + ); + + expect(output.trim()).toMatchInlineSnapshot( + `"It worked! function () {}"`, + ); + }); + + it("works with the -r flag in NODE_OPTIONS", async () => { + const output = await spawnNodeAsync( + [testFileLog], + path.dirname(testFileLog), + { NODE_OPTIONS: `-r ${registerFile}` }, + ); + + expect(output.trim()).toMatchInlineSnapshot( + `"It worked! function () {}"`, + ); + }); + + it("works with the --require flag in NODE_OPTIONS", async () => { + const output = await spawnNodeAsync( + [testFileLog], + path.dirname(testFileLog), + { NODE_OPTIONS: `--require ${registerFile}` }, + ); + + expect(output.trim()).toMatchInlineSnapshot( + `"It worked! function () {}"`, + ); + }); + }); + + it("returns concatenatable sourceRoot and sources", async () => { + // The Source Maps R3 standard https://sourcemaps.info/spec.html states + // that `sourceRoot` is “prepended to the individual entries in the + // ‘source’ field.” If `sources` contains file names, and `sourceRoot` + // is intended to refer to a directory but doesn’t end with a trailing + // slash, any consumers of the source map are in for a bad day. + // + // The underlying problem seems to only get triggered if one file + // requires() another with @babel/register active, and I couldn’t get + // that working inside a test, possibly because of jest’s mocking + // hooks, so we spawn a separate process. + const output = await spawnNodeAsync([ + "-r", + registerFile, + require.resolve("./fixtures/source-map/index"), + ]); + const sourceMap = JSON.parse(output); + expect(sourceMap.map.sourceRoot + sourceMap.map.sources[0]).toBe( + require.resolve("./fixtures/source-map/foo/bar"), + ); + }); + + test("hook transpiles with config", () => { + setupRegister({ + babelrc: false, + sourceMaps: false, + plugins: ["@babel/transform-modules-commonjs"], + }); + + const result = currentHook(testFileContent, testFile); + + expect(result).toBe('"use strict";\n\nrequire("assert");'); + }); + + test("hook transpiles with babelrc", () => { + setupRegister({ + babelrc: true, + sourceMaps: false, + }); + + const result = currentHook(testFileContent, testFile); + + expect(result).toBe('"use strict";\n\nrequire("assert");'); + }); + + test("transforms modules used within register", async () => { + // Need a clean environment without `convert-source-map` + // already in the require cache, so we spawn a separate process + + const output = await spawnNodeAsync([ + require.resolve("./fixtures/internal-modules/index.js"), + ]); + const { convertSourceMap } = JSON.parse(output); + expect(convertSourceMap).toMatch("/* transformed */"); + }); + + return { setupRegister, revertRegister }; + } }); -function spawnNodeAsync(args) { - const spawn = child.spawn(process.execPath, args, { cwd: dirname }); +function spawnNodeAsync(args, cwd = dirname, env) { + const spawn = child.spawn(process.execPath, args, { cwd, env }); let output = ""; let callback; diff --git a/yarn.lock b/yarn.lock index f28c15e3be..d4e9391066 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3494,6 +3494,7 @@ __metadata: resolution: "@babel/register@workspace:packages/babel-register" dependencies: "@babel/core": "workspace:^" + "@babel/plugin-transform-arrow-functions": "workspace:^" "@babel/plugin-transform-modules-commonjs": "workspace:^" browserify: ^16.5.2 clone-deep: ^4.0.1