Allow testing logs with @babel/helper-transform-fixture-test-runner (#10326)

* Add support for logs to transform-fixture-test-runner

* Test some warnings

* Use stderr/stdout, and add a validateLogs test option

* Normalize CWD in logs and rename vars

* Silence preset-env tests

* Restore old methods

packages/babel-helper-fixtures/src/index.js

@@ -120,6 +120,8 @@ export default function get(entryLoc): Array<Suite> {
       const expectLoc =
         findFile(taskDir + "/output", true /* allowJSON */) ||
         taskDir + "/output.js";
+      const stdoutLoc = taskDir + "/stdout.txt";
+      const stderrLoc = taskDir + "/stderr.txt";
 
       const actualLocAlias =
         suiteName + "/" + taskName + "/" + path.basename(actualLoc);
@@ -146,6 +148,9 @@ export default function get(entryLoc): Array<Suite> {
         title: humanize(taskName, true),
         disabled: taskName[0] === ".",
         options: taskOpts,
+        validateLogs: taskOpts.validateLogs,
+        stdout: { loc: stdoutLoc, code: readFile(stdoutLoc) },
+        stderr: { loc: stderrLoc, code: readFile(stderrLoc) },
         exec: {
           loc: execLoc,
           code: readFile(execLoc),
@@ -222,6 +227,16 @@ export default function get(entryLoc): Array<Suite> {
           );
         }
       }
+
+      if (!test.validateLogs && (test.stdout.code || test.stderr.code)) {
+        throw new Error(
+          "stdout.txt and stderr.txt are only allowed when the 'validateLogs' option is enabled: " +
+            (test.stdout.code ? stdoutLoc : stderrLoc),
+        );
+      }
+
+      // Delete to avoid option validation error
+      delete test.options.validateLogs;
     }
   }
 

packages/babel-helper-transform-fixture-test-runner/src/index.js

@@ -131,11 +131,16 @@ function wrapPackagesArray(type, names, optionsDir) {
 }
 
 function run(task) {
-  const actual = task.actual;
+  const {
-  const expected = task.expect;
+    actual,
-  const exec = task.exec;
+    expect: expected,
-  const opts = task.options;
+    exec,
-  const optionsDir = task.optionsDir;
+    options: opts,
+    optionsDir,
+    validateLogs,
+    stdout,
+    stderr,
+  } = task;
 
   function getOpts(self) {
     const newOpts = merge(
@@ -191,19 +196,34 @@ function run(task) {
     }
   }
 
-  let actualCode = actual.code;
+  const inputCode = actual.code;
-  const expectCode = expected.code;
+  const expectedCode = expected.code;
-  if (!execCode || actualCode) {
+  if (!execCode || inputCode) {
-    result = babel.transform(actualCode, getOpts(actual));
+    const actualLogs = { stdout: "", stderr: "" };
-    const expectedCode = result.code.replace(
+    let restoreSpies = null;
-      escapeRegExp(path.resolve(__dirname, "../../../")),
+    if (validateLogs) {
-      "<CWD>",
+      const spy1 = jest.spyOn(console, "log").mockImplementation(msg => {
-    );
+        actualLogs.stdout += `${msg}\n`;
+      });
+      const spy2 = jest.spyOn(console, "warn").mockImplementation(msg => {
+        actualLogs.stderr += `${msg}\n`;
+      });
+      restoreSpies = () => {
+        spy1.mockRestore();
+        spy2.mockRestore();
+      };
+    }
+
+    result = babel.transform(inputCode, getOpts(actual));
+
+    if (restoreSpies) restoreSpies();
+
+    const outputCode = normalizeOutput(result.code);
 
     checkDuplicatedNodes(babel, result.ast);
     if (
       !expected.code &&
-      expectedCode &&
+      outputCode &&
       !opts.throws &&
       fs.statSync(path.dirname(expected.loc)).isDirectory() &&
       !process.env.CI
@@ -214,7 +234,7 @@ function run(task) {
       );
 
       console.log(`New test file created: ${expectedFile}`);
-      fs.writeFileSync(expectedFile, `${expectedCode}\n`);
+      fs.writeFileSync(expectedFile, `${outputCode}\n`);
 
       if (expected.loc !== expectedFile) {
         try {
@@ -222,25 +242,19 @@ function run(task) {
         } catch (e) {}
       }
     } else {
-      actualCode = expectedCode.trim();
+      validateFile(outputCode, expected.loc, expectedCode);
-      try {
-        expect(actualCode).toEqualFile({
-          filename: expected.loc,
-          code: expectCode,
-        });
-      } catch (e) {
-        if (!process.env.OVERWRITE) throw e;
 
-        console.log(`Updated test file: ${expected.loc}`);
+      if (inputCode) {
-        fs.writeFileSync(expected.loc, `${expectedCode}\n`);
-      }
-
-      if (actualCode) {
         expect(expected.loc).toMatch(
           result.sourceType === "module" ? /\.mjs$/ : /\.js$/,
         );
       }
     }
+
+    if (validateLogs) {
+      validateFile(normalizeOutput(actualLogs.stdout), stdout.loc, stdout.code);
+      validateFile(normalizeOutput(actualLogs.stderr), stderr.loc, stderr.code);
+    }
   }
 
   if (task.sourceMap) {
@@ -263,6 +277,26 @@ function run(task) {
   }
 }
 
+function validateFile(actualCode, expectedLoc, expectedCode) {
+  try {
+    expect(actualCode).toEqualFile({
+      filename: expectedLoc,
+      code: expectedCode,
+    });
+  } catch (e) {
+    if (!process.env.OVERWRITE) throw e;
+
+    console.log(`Updated test file: ${expectedLoc}`);
+    fs.writeFileSync(expectedLoc, `${actualCode}\n`);
+  }
+}
+
+function normalizeOutput(code) {
+  return code
+    .trim()
+    .replace(escapeRegExp(path.resolve(__dirname, "../../../")), "<CWD>");
+}
+
 const toEqualFile = () => ({
   compare: (actual, { filename, code }) => {
     const pass = actual === code;

packages/babel-plugin-proposal-dynamic-import/test/fixtures/systemjs/missing-plugin/options.json

@@ -1,4 +1,5 @@
 {
+  "validateLogs": true,
   "plugins": [
     "syntax-dynamic-import",
     "transform-modules-systemjs",

packages/babel-plugin-proposal-dynamic-import/test/fixtures/systemjs/missing-plugin/stderr.txt

@@ -0,0 +1,3 @@
+WARNING: Dynamic import() transformation must be enabled using the
+         @babel/plugin-proposal-dynamic-import plugin. Babel 8 will
+         no longer transform import() without using that plugin.

packages/babel-preset-env/test/fixtures/corejs2/usage-browserslist-config-ignore/options.json

@@ -1,4 +1,5 @@
 {
+  "validateLogs": true,
   "presets": [
     [
       "../../../../lib",

packages/babel-preset-env/test/fixtures/corejs2/usage-browserslist-config-ignore/stdout.txt

@@ -0,0 +1,33 @@
+@babel/preset-env: `DEBUG` option
+
+Using targets:
+{
+  "chrome": "61",
+  "edge": "16",
+  "firefox": "60",
+  "ios": "10.3",
+  "opera": "48",
+  "safari": "10.1"
+}
+
+Using modules transform: false
+
+Using plugins:
+  transform-template-literals { "ios":"10.3", "safari":"10.1" }
+  transform-function-name { "edge":"16" }
+  transform-dotall-regex { "chrome":"61", "edge":"16", "firefox":"60", "ios":"10.3", "opera":"48", "safari":"10.1" }
+  transform-unicode-regex { "ios":"10.3", "safari":"10.1" }
+  transform-parameters { "edge":"16" }
+  transform-block-scoping { "ios":"10.3", "safari":"10.1" }
+  transform-async-to-generator { "ios":"10.3", "safari":"10.1" }
+  proposal-async-generator-functions { "chrome":"61", "edge":"16", "ios":"10.3", "opera":"48", "safari":"10.1" }
+  proposal-object-rest-spread { "edge":"16", "ios":"10.3", "safari":"10.1" }
+  proposal-unicode-property-regex { "chrome":"61", "edge":"16", "firefox":"60", "ios":"10.3", "opera":"48", "safari":"10.1" }
+  proposal-json-strings { "chrome":"61", "edge":"16", "firefox":"60", "ios":"10.3", "opera":"48", "safari":"10.1" }
+  proposal-optional-catch-binding { "chrome":"61", "edge":"16", "ios":"10.3", "opera":"48", "safari":"10.1" }
+  transform-named-capturing-groups-regex { "chrome":"61", "edge":"16", "firefox":"60", "ios":"10.3", "opera":"48", "safari":"10.1" }
+
+Using polyfills with `usage` option:
+
+[<CWD>/packages/babel-preset-env/test/fixtures/corejs2/usage-browserslist-config-ignore/input.mjs] Added following core-js polyfill:
+  web.dom.iterable { "chrome":"61", "edge":"16", "firefox":"60", "ios":"10.3", "opera":"48", "safari":"10.1" }

packages/babel-preset-env/test/fixtures/corejs3/usage-browserslist-config-ignore/options.json

@@ -1,4 +1,5 @@
 {
+  "validateLogs": true,
   "presets": [
     [
       "../../../../lib",

packages/babel-preset-env/test/fixtures/corejs3/usage-browserslist-config-ignore/stdout.txt

@@ -0,0 +1,34 @@
+@babel/preset-env: `DEBUG` option
+
+Using targets:
+{
+  "chrome": "61",
+  "edge": "16",
+  "firefox": "60",
+  "ios": "10.3",
+  "opera": "48",
+  "safari": "10.1"
+}
+
+Using modules transform: false
+
+Using plugins:
+  transform-template-literals { "ios":"10.3", "safari":"10.1" }
+  transform-function-name { "edge":"16" }
+  transform-dotall-regex { "chrome":"61", "edge":"16", "firefox":"60", "ios":"10.3", "opera":"48", "safari":"10.1" }
+  transform-unicode-regex { "ios":"10.3", "safari":"10.1" }
+  transform-parameters { "edge":"16" }
+  transform-block-scoping { "ios":"10.3", "safari":"10.1" }
+  transform-async-to-generator { "ios":"10.3", "safari":"10.1" }
+  proposal-async-generator-functions { "chrome":"61", "edge":"16", "ios":"10.3", "opera":"48", "safari":"10.1" }
+  proposal-object-rest-spread { "edge":"16", "ios":"10.3", "safari":"10.1" }
+  proposal-unicode-property-regex { "chrome":"61", "edge":"16", "firefox":"60", "ios":"10.3", "opera":"48", "safari":"10.1" }
+  proposal-json-strings { "chrome":"61", "edge":"16", "firefox":"60", "ios":"10.3", "opera":"48", "safari":"10.1" }
+  proposal-optional-catch-binding { "chrome":"61", "edge":"16", "ios":"10.3", "opera":"48", "safari":"10.1" }
+  transform-named-capturing-groups-regex { "chrome":"61", "edge":"16", "firefox":"60", "ios":"10.3", "opera":"48", "safari":"10.1" }
+
+Using polyfills with `usage` option:
+
+[<CWD>/packages/babel-preset-env/test/fixtures/corejs3/usage-browserslist-config-ignore/input.mjs] Added following core-js polyfills:
+  es.array.iterator { "chrome":"61", "opera":"48" }
+  web.dom-collections.iterator { "chrome":"61", "edge":"16", "ios":"10.3", "opera":"48", "safari":"10.1" }

packages/babel-preset-env/test/fixtures/dynamic-import/auto-esm-supported-import-supported/options.json

@@ -1,4 +1,5 @@
 {
+  "validateLogs": true,
   "caller": {
     "name": "test-fixture",
     "supportsStaticESM": true,

packages/babel-preset-env/test/fixtures/dynamic-import/auto-esm-supported-import-unsupported/options.json

@@ -1,4 +1,5 @@
 {
+  "validateLogs": true,
   "caller": {
     "name": "test-fixture",
     "supportsStaticESM": true,

packages/babel-preset-env/test/fixtures/dynamic-import/auto-esm-supported-import-unsupported/stderr.txt

@@ -0,0 +1 @@
+Dynamic import can only be supported when transforming ES modules to AMD, CommonJS or SystemJS. Only the parser plugin will be enabled.

packages/babel-preset-env/test/fixtures/dynamic-import/auto-esm-unsupported-import-unsupported/options.json

@@ -1,4 +1,5 @@
 {
+  "validateLogs": true,
   "caller": {
     "name": "test-fixture",
     "supportsStaticESM": false,

packages/babel-preset-env/test/fixtures/dynamic-import/modules-amd/options.json

@@ -1,3 +1,4 @@
 {
+  "validateLogs": true,
   "presets": [["env", { "modules": "amd" }]]
 }

packages/babel-preset-env/test/fixtures/dynamic-import/modules-cjs/options.json

@@ -1,3 +1,4 @@
 {
+  "validateLogs": true,
   "presets": [["env", { "modules": "cjs" }]]
 }

packages/babel-preset-env/test/fixtures/dynamic-import/modules-false/options.json

@@ -1,3 +1,4 @@
 {
+  "validateLogs": true,
   "presets": [["env", { "modules": false }]]
 }

packages/babel-preset-env/test/fixtures/dynamic-import/modules-systemjs/options.json

@@ -1,3 +1,4 @@
 {
+  "validateLogs": true,
   "presets": [["env", { "modules": "systemjs" }]]
 }

packages/babel-preset-env/test/fixtures/dynamic-import/modules-umd/options.json

@@ -1,3 +1,4 @@
 {
+  "validateLogs": true,
   "presets": [["env", { "modules": "umd" }]]
 }

packages/babel-preset-env/test/fixtures/dynamic-import/modules-umd/stderr.txt

@@ -0,0 +1 @@
+Dynamic import can only be supported when transforming ES modules to AMD, CommonJS or SystemJS. Only the parser plugin will be enabled.

packages/babel-preset-env/test/fixtures/preset-options/safari-10_3-block-scoped/options.json

@@ -1,4 +1,5 @@
 {
+  "validateLogs": true,
   "presets": [
     [
       "../../../../lib",

packages/babel-preset-env/test/fixtures/preset-options/safari-10_3-block-scoped/stdout.txt

@@ -0,0 +1,24 @@
+@babel/preset-env: `DEBUG` option
+
+Using targets:
+{
+  "safari": "10"
+}
+
+Using modules transform: auto
+
+Using plugins:
+  transform-template-literals { "safari":"10" }
+  transform-dotall-regex { "safari":"10" }
+  transform-unicode-regex { "safari":"10" }
+  transform-block-scoping { "safari":"10" }
+  transform-exponentiation-operator { "safari":"10" }
+  transform-async-to-generator { "safari":"10" }
+  proposal-async-generator-functions { "safari":"10" }
+  proposal-object-rest-spread { "safari":"10" }
+  proposal-unicode-property-regex { "safari":"10" }
+  proposal-json-strings { "safari":"10" }
+  proposal-optional-catch-binding { "safari":"10" }
+  transform-named-capturing-groups-regex { "safari":"10" }
+
+Using polyfills: No polyfills were added, since the `useBuiltIns` option was not set.