Merge multiple regex transform plugin (#10447)

* feat: implement create-regexp-features-plugin

* fix: test input is not effective

* refactor: leverage create-regexp-features-plugin

* test: add more test cases

* test: update test fixture

* chore: add type annotation to features

* test: add regression test for issue 9892

* add regression test for issue 9199

* address review comments from Nicolò

* address review comments from Brian

* small tweaks

* Enable dotAllFlag when flags includes u

packages/babel-helper-create-regexp-features-plugin/.npmignore

@@ -0,0 +1,3 @@
+src
+test
+*.log

packages/babel-helper-create-regexp-features-plugin/README.md

@@ -0,0 +1,19 @@
+# @babel/helper-create-regexp-features-plugin
+
+> Compile ESNext Regular Expressions to ES5
+
+See our website [@babel/helper-create-regexp-features-plugin](https://babeljs.io/docs/en/next/babel-helper-create-regexp-features-plugin.html) for more information.
+
+## Install
+
+Using npm:
+
+```sh
+npm install --save-dev @babel/helper-create-regexp-features-plugin
+```
+
+or using yarn:
+
+```sh
+yarn add @babel/helper-create-regexp-features-plugin --dev
+```

packages/babel-helper-create-regexp-features-plugin/package.json

@@ -0,0 +1,31 @@
+{
+  "name": "@babel/helper-create-regexp-features-plugin",
+  "version": "7.6.0",
+  "author": "The Babel Team (https://babeljs.io/team)",
+  "license": "MIT",
+  "description": "Compile ESNext Regular Expressions to ES5",
+  "repository": {
+    "type": "git",
+    "repository": "https://github.com/babel/babel",
+    "directory": "packages/babel-helper-create-regexp-features-plugin"
+  },
+  "main": "lib/index.js",
+  "publishConfig": {
+    "access": "public"
+  },
+  "keywords": [
+    "babel",
+    "babel-plugin"
+  ],
+  "dependencies": {
+    "@babel/helper-regex": "^7.4.4",
+    "regexpu-core": "^4.6.0"
+  },
+  "peerDependencies": {
+    "@babel/core": "^7.0.0"
+  },
+  "devDependencies": {
+    "@babel/core": "^7.6.0",
+    "@babel/helper-plugin-test-runner": "^7.0.0"
+  }
+}

packages/babel-helper-create-regexp-features-plugin/src/features.js

@@ -0,0 +1,27 @@
+// @flow
+export const FEATURES = Object.freeze({
+  unicodeFlag: 1 << 0,
+  dotAllFlag: 1 << 1,
+  unicodePropertyEscape: 1 << 2,
+  namedCaptureGroups: 1 << 3,
+});
+
+// We can't use a symbol because this needs to always be the same, even if
+// this package isn't deduped by npm. e.g.
+//  - node_modules/
+//    - @babel/plugin-regexp-features
+//    - @babel/plugin-proposal-unicode-property-regex
+//      - node_modules
+//        - @babel-plugin-regexp-features
+export const featuresKey = "@babel/plugin-regexp-features/featuresKey";
+export const runtimeKey = "@babel/plugin-regexp-features/runtimeKey";
+
+type FeatureType = $Values<typeof FEATURES>;
+
+export function enableFeature(features: number, feature: FeatureType): number {
+  return features | feature;
+}
+
+export function hasFeature(features: number, feature: FeatureType) {
+  return !!(features & feature);
+}

packages/babel-helper-create-regexp-features-plugin/src/index.js

@@ -0,0 +1,96 @@
+import rewritePattern from "regexpu-core";
+import {
+  featuresKey,
+  FEATURES,
+  enableFeature,
+  runtimeKey,
+  hasFeature,
+} from "./features";
+import { generateRegexpuOptions } from "./util";
+
+import pkg from "../package.json";
+import { types as t } from "@babel/core";
+import { pullFlag } from "@babel/helper-regex";
+
+// Note: Versions are represented as an integer. e.g. 7.1.5 is represented
+//       as 70000100005. This method is easier than using a semver-parsing
+//       package, but it breaks if we release x.y.z where x, y or z are
+//       greater than 99_999.
+const version = pkg.version.split(".").reduce((v, x) => v * 1e5 + +x, 0);
+const versionKey = "@babel/plugin-regexp-features/version";
+
+export function createRegExpFeaturePlugin({ name, feature, options = {} }) {
+  return {
+    name,
+    pre() {
+      const { file } = this;
+      const features = file.get(featuresKey) ?? 0;
+      let newFeatures = enableFeature(features, FEATURES[feature]);
+
+      const { useUnicodeFlag, runtime = true } = options;
+      if (useUnicodeFlag === false) {
+        newFeatures = enableFeature(newFeatures, FEATURES.unicodeFlag);
+      }
+      if (newFeatures !== features) {
+        file.set(featuresKey, newFeatures);
+      }
+
+      if (!runtime) {
+        file.set(runtimeKey, false);
+      }
+
+      if (!file.has(versionKey) || file.get(versionKey) < version) {
+        file.set(versionKey, version);
+      }
+    },
+
+    visitor: {
+      RegExpLiteral(path) {
+        const { node } = path;
+        const { file } = this;
+        const features = file.get(featuresKey);
+        const runtime = file.get(runtimeKey) ?? true;
+        const regexpuOptions = generateRegexpuOptions(node, features);
+        if (regexpuOptions === null) {
+          return;
+        }
+        const namedCaptureGroups = {};
+        if (regexpuOptions.namedGroup) {
+          regexpuOptions.onNamedGroup = (name, index) => {
+            namedCaptureGroups[name] = index;
+          };
+        }
+        node.pattern = rewritePattern(node.pattern, node.flags, regexpuOptions);
+
+        if (
+          regexpuOptions.namedGroup &&
+          Object.keys(namedCaptureGroups).length > 0 &&
+          runtime &&
+          !isRegExpTest(path)
+        ) {
+          path.replaceWith(
+            t.callExpression(this.addHelper("wrapRegExp"), [
+              node,
+              t.valueToNode(namedCaptureGroups),
+            ]),
+          );
+        }
+        if (hasFeature(features, FEATURES.unicodeFlag)) {
+          pullFlag(node, "u");
+        }
+        if (hasFeature(features, FEATURES.dotAllFlag)) {
+          pullFlag(node, "s");
+        }
+      },
+    },
+  };
+}
+
+function isRegExpTest(path) {
+  return (
+    path.parentPath.isMemberExpression({
+      object: path.node,
+      computed: false,
+    }) && path.parentPath.get("property").isIdentifier({ name: "test" })
+  );
+}

packages/babel-helper-create-regexp-features-plugin/src/util.js

@@ -0,0 +1,56 @@
+import { FEATURES, hasFeature } from "./features";
+
+export function generateRegexpuOptions(node, features) {
+  let useUnicodeFlag = false,
+    dotAllFlag = false,
+    unicodePropertyEscape = false,
+    namedGroup = false;
+  const { flags, pattern } = node;
+  const flagsIncludesU = flags.includes("u");
+
+  if (flagsIncludesU) {
+    if (!hasFeature(features, FEATURES.unicodeFlag)) {
+      useUnicodeFlag = true;
+    }
+    if (
+      hasFeature(features, FEATURES.unicodePropertyEscape) &&
+      /\\[pP]{/.test(pattern)
+    ) {
+      unicodePropertyEscape = true;
+    }
+  }
+
+  if (hasFeature(features, FEATURES.dotAllFlag) && flags.indexOf("s") >= 0) {
+    dotAllFlag = true;
+  }
+  if (
+    hasFeature(features, FEATURES.namedCaptureGroups) &&
+    /\(\?<(?![=!])/.test(pattern)
+  ) {
+    namedGroup = true;
+  }
+  if (
+    !namedGroup &&
+    !unicodePropertyEscape &&
+    !dotAllFlag &&
+    (!flagsIncludesU || useUnicodeFlag)
+  ) {
+    return null;
+  }
+  // Now we have to feed regexpu-core the regex
+  if (flagsIncludesU && flags.indexOf("s") >= 0) {
+    // When flags includes u, `config.unicode` will be enabled even if `u` is supported natively.
+    // In this case we have to enable dotAllFlag, otherwise `rewritePattern(/./su)` will return
+    // incorrect result
+    // https://github.com/mathiasbynens/regexpu-core/blob/v4.6.0/rewrite-pattern.js#L191
+    dotAllFlag = true;
+  }
+  return {
+    useUnicodeFlag,
+    onNamedGroup: () => {},
+    namedGroup,
+    unicodePropertyEscape,
+    dotAllFlag,
+    lookbehind: true,
+  };
+}

packages/babel-helper-create-regexp-features-plugin/test/fixtures/plugin-proposal-unicode-property-regex/use-unicode-flag-false-not-overwritten/input.mjs

@@ -0,0 +1 @@
+/\p{Script_Extensions=Wancho}/u

packages/babel-helper-create-regexp-features-plugin/test/fixtures/plugin-proposal-unicode-property-regex/use-unicode-flag-false-not-overwritten/options.json

@@ -0,0 +1,6 @@
+{
+  "plugins": [
+    ["proposal-unicode-property-regex", { "useUnicodeFlag": false }, "name 1"],
+    ["proposal-unicode-property-regex", { "useUnicodeFlag": true }, "name 2"]
+  ]
+}

packages/babel-helper-create-regexp-features-plugin/test/fixtures/plugin-proposal-unicode-property-regex/use-unicode-flag-false-not-overwritten/output.mjs

@@ -0,0 +1 @@
+/(?:\uD838[\uDEC0-\uDEF9\uDEFF])/;

packages/babel-helper-create-regexp-features-plugin/test/fixtures/plugin-transform-named-capturing-groups-regex/runtime-false-not-overwritten/input.mjs

@@ -0,0 +1 @@
+/(?<year>\d{4})/

packages/babel-helper-create-regexp-features-plugin/test/fixtures/plugin-transform-named-capturing-groups-regex/runtime-false-not-overwritten/options.json

@@ -0,0 +1,6 @@
+{
+  "plugins": [
+    ["transform-named-capturing-groups-regex", { "runtime": false }, "name 1"],
+    ["transform-named-capturing-groups-regex", { "runtime": true }, "name 2"]
+  ]
+}

packages/babel-helper-create-regexp-features-plugin/test/fixtures/plugin-transform-named-capturing-groups-regex/runtime-false-not-overwritten/output.mjs

@@ -0,0 +1 @@
+/([0-9]{4})/;

packages/babel-helper-create-regexp-features-plugin/test/index.js

@@ -0,0 +1,3 @@
+import runner from "@babel/helper-plugin-test-runner";
+
+runner(__dirname);

packages/babel-plugin-proposal-unicode-property-regex/package.json

@@ -22,9 +22,8 @@
   "repository": "https://github.com/babel/babel/tree/master/packages/babel-plugin-proposal-unicode-property-regex",
   "bugs": "https://github.com/babel/babel/issues",
   "dependencies": {
-    "@babel/helper-plugin-utils": "^7.0.0",
+    "@babel/helper-create-regexp-features-plugin": "^7.6.0",
-    "@babel/helper-regex": "^7.4.4",
+    "@babel/helper-plugin-utils": "^7.0.0"
-    "regexpu-core": "^4.6.0"
   },
   "peerDependencies": {
     "@babel/core": "^7.0.0-0"

packages/babel-plugin-proposal-unicode-property-regex/src/index.js

@@ -1,6 +1,6 @@
+/* eslint-disable @babel/development/plugin-name */
+import { createRegExpFeaturePlugin } from "@babel/helper-create-regexp-features-plugin";
 import { declare } from "@babel/helper-plugin-utils";
-import rewritePattern from "regexpu-core";
-import * as regex from "@babel/helper-regex";
 
 export default declare((api, options) => {
   api.assertVersion(7);
@@ -10,23 +10,9 @@ export default declare((api, options) => {
     throw new Error(".useUnicodeFlag must be a boolean, or undefined");
   }
 
-  return {
+  return createRegExpFeaturePlugin({
     name: "proposal-unicode-property-regex",
-
+    feature: "unicodePropertyEscape",
-    visitor: {
+    options: { useUnicodeFlag },
-      RegExpLiteral(path) {
+  });
-        const node = path.node;
-        if (!regex.is(node, "u")) {
-          return;
-        }
-        node.pattern = rewritePattern(node.pattern, node.flags, {
-          unicodePropertyEscape: true,
-          useUnicodeFlag,
-        });
-        if (!useUnicodeFlag) {
-          regex.pullFlag(node, "u");
-        }
-      },
-    },
-  };
 });

packages/babel-plugin-proposal-unicode-property-regex/test/fixtures/with-unicode-flag/unicode-12/input.js

@@ -1 +1 @@
-var regex = /[\u{1E2C0}-\u{1E2F9}\u{1E2FF}]/u;
+var regex = /[\p{Script_Extensions=Wancho}]/u;

packages/babel-plugin-transform-dotall-regex/package.json

@@ -8,9 +8,6 @@
     "access": "public"
   },
   "main": "lib/index.js",
-  "engines": {
-    "node": ">=4"
-  },
   "keywords": [
     "babel-plugin",
     "regex",
@@ -21,9 +18,8 @@
   "repository": "https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-dotall-regex",
   "bugs": "https://github.com/babel/babel/issues",
   "dependencies": {
-    "@babel/helper-plugin-utils": "^7.0.0",
+    "@babel/helper-create-regexp-features-plugin": "^7.6.0",
-    "@babel/helper-regex": "^7.4.4",
+    "@babel/helper-plugin-utils": "^7.0.0"
-    "regexpu-core": "^4.6.0"
   },
   "peerDependencies": {
     "@babel/core": "^7.0.0-0"

packages/babel-plugin-transform-dotall-regex/src/index.js

@@ -1,25 +1,12 @@
+/* eslint-disable @babel/development/plugin-name */
+import { createRegExpFeaturePlugin } from "@babel/helper-create-regexp-features-plugin";
 import { declare } from "@babel/helper-plugin-utils";
-import rewritePattern from "regexpu-core";
-import * as regex from "@babel/helper-regex";
 
 export default declare(api => {
   api.assertVersion(7);
 
-  return {
+  return createRegExpFeaturePlugin({
     name: "transform-dotall-regex",
-
+    feature: "dotAllFlag",
-    visitor: {
+  });
-      RegExpLiteral(path) {
-        const node = path.node;
-        if (!regex.is(node, "s")) {
-          return;
-        }
-        node.pattern = rewritePattern(node.pattern, node.flags, {
-          dotAllFlag: true,
-          useUnicodeFlag: regex.is(node, "u"),
-        });
-        regex.pullFlag(node, "s");
-      },
-    },
-  };
 });

packages/babel-plugin-transform-dotall-regex/test/fixtures/dotall-regex/with-unicode-property-escape/input.js

@@ -0,0 +1,2 @@
+var a = /\p{Unified_Ideograph}./u;
+var b = /\p{Unified_Ideograph}./su;

packages/babel-plugin-transform-dotall-regex/test/fixtures/dotall-regex/with-unicode-property-escape/options.json

@@ -0,0 +1,3 @@
+{
+  "plugins": ["transform-dotall-regex", "proposal-unicode-property-regex"]
+}

packages/babel-plugin-transform-dotall-regex/test/fixtures/dotall-regex/with-unicode-property-escape/output.js

@@ -0,0 +1,2 @@
+var a = /[\u3400-\u4DB5\u4E00-\u9FEF\uFA0E\uFA0F\uFA11\uFA13\uFA14\uFA1F\uFA21\uFA23\uFA24\uFA27-\uFA29\u{20000}-\u{2A6D6}\u{2A700}-\u{2B734}\u{2B740}-\u{2B81D}\u{2B820}-\u{2CEA1}\u{2CEB0}-\u{2EBE0}][\0-\t\x0B\f\x0E-\u2027\u202A-\u{10FFFF}]/u;
+var b = /[\u3400-\u4DB5\u4E00-\u9FEF\uFA0E\uFA0F\uFA11\uFA13\uFA14\uFA1F\uFA21\uFA23\uFA24\uFA27-\uFA29\u{20000}-\u{2A6D6}\u{2A700}-\u{2B734}\u{2B740}-\u{2B81D}\u{2B820}-\u{2CEA1}\u{2CEB0}-\u{2EBE0}][\0-\u{10FFFF}]/u;

packages/babel-plugin-transform-named-capturing-groups-regex/package.json

@@ -21,7 +21,7 @@
   },
   "bugs": "https://github.com/babel/babel/issues",
   "dependencies": {
-    "regexpu-core": "^4.6.0"
+    "@babel/helper-create-regexp-features-plugin": "^7.6.0"
   },
   "peerDependencies": {
     "@babel/core": "^7.0.0"

packages/babel-plugin-transform-named-capturing-groups-regex/src/index.js

@@ -1,56 +1,15 @@
-import rewritePattern from "regexpu-core";
+/* eslint-disable @babel/development/plugin-name */
+import { createRegExpFeaturePlugin } from "@babel/helper-create-regexp-features-plugin";
 
-export default function({ types: t }, options) {
+export default function(core, options) {
   const { runtime = true } = options;
   if (typeof runtime !== "boolean") {
     throw new Error("The 'runtime' option must be boolean");
   }
 
-  return {
+  return createRegExpFeaturePlugin({
     name: "transform-named-capturing-groups-regex",
-
+    feature: "namedCaptureGroups",
-    visitor: {
+    options: { runtime },
-      RegExpLiteral(path) {
+  });
-        const node = path.node;
-        if (!/\(\?<(?![=!])/.test(node.pattern)) {
-          // Return early if there are no named groups.
-          // The .indexOf check may have false positives (e.g. /\(?</); in
-          // this case we parse the regex and regexp-tree won't transform it.
-          return;
-        }
-
-        const namedCapturingGroups = {};
-        const result = rewritePattern(node.pattern, node.flags, {
-          namedGroup: true,
-          //todo: consider refactor `lookbehind` true as modular plugin
-          lookbehind: true,
-          onNamedGroup(name, index) {
-            namedCapturingGroups[name] = index;
-          },
-        });
-
-        if (Object.keys(namedCapturingGroups).length > 0) {
-          node.pattern = result;
-
-          if (runtime && !isRegExpTest(path)) {
-            path.replaceWith(
-              t.callExpression(this.addHelper("wrapRegExp"), [
-                node,
-                t.valueToNode(namedCapturingGroups),
-              ]),
-            );
-          }
-        }
-      },
-    },
-  };
-}
-
-function isRegExpTest(path) {
-  return (
-    path.parentPath.isMemberExpression({
-      object: path.node,
-      computed: false,
-    }) && path.parentPath.get("property").isIdentifier({ name: "test" })
-  );
 }

packages/babel-plugin-transform-named-capturing-groups-regex/test/fixtures/runtime/issue-10601/exec.js

@@ -0,0 +1,7 @@
+const regex = /<(?<tag>\d)+>.*?<\/\k<tag>>/su;
+
+const result = regex.exec('<0>xxx\nyyy</0>');
+
+expect(result.groups).toEqual({
+  tag: "0"
+});

packages/babel-plugin-transform-named-capturing-groups-regex/test/fixtures/runtime/issue-10601/options.json

@@ -0,0 +1,3 @@
+{
+  "minNodeVersion": "8.0.0"
+}

packages/babel-plugin-transform-named-capturing-groups-regex/test/fixtures/wrapper/looks-like-a-group-unicode/input.js

@@ -0,0 +1 @@
+/no-groups-\(?<looks-like-a-group>looks\)\u{10000}/u;

packages/babel-plugin-transform-named-capturing-groups-regex/test/fixtures/wrapper/looks-like-a-group-unicode/options.json

@@ -0,0 +1,7 @@
+{
+  "plugins": [
+    ["external-helpers", { "helperVersion": "7.1000.0" }],
+    "transform-named-capturing-groups-regex",
+    "transform-unicode-regex"
+  ]
+}

packages/babel-plugin-transform-named-capturing-groups-regex/test/fixtures/wrapper/looks-like-a-group-unicode/output.js

@@ -0,0 +1 @@
+/no\x2Dgroups\x2D\(?<looks\x2Dlike\x2Da\x2Dgroup>looks\)(?:\uD800\uDC00)/;

packages/babel-plugin-transform-named-capturing-groups-regex/test/fixtures/wrapper/looks-like-a-group/output.js

@@ -1 +1 @@
-/no-groups-\(?<looks-like-a-group>looks\)/;
+/no\x2Dgroups\x2D\(?<looks\x2Dlike\x2Da\x2Dgroup>looks\)/;

packages/babel-plugin-transform-unicode-regex/package.json

@@ -12,9 +12,8 @@
     "babel-plugin"
   ],
   "dependencies": {
-    "@babel/helper-plugin-utils": "^7.0.0",
+    "@babel/helper-create-regexp-features-plugin": "^7.6.0",
-    "@babel/helper-regex": "^7.4.4",
+    "@babel/helper-plugin-utils": "^7.0.0"
-    "regexpu-core": "^4.6.0"
   },
   "peerDependencies": {
     "@babel/core": "^7.0.0-0"

packages/babel-plugin-transform-unicode-regex/src/index.js

@@ -1,19 +1,12 @@
+/* eslint-disable @babel/development/plugin-name */
+import { createRegExpFeaturePlugin } from "@babel/helper-create-regexp-features-plugin";
 import { declare } from "@babel/helper-plugin-utils";
-import rewritePattern from "regexpu-core";
-import * as regex from "@babel/helper-regex";
 
 export default declare(api => {
   api.assertVersion(7);
 
-  return {
+  return createRegExpFeaturePlugin({
     name: "transform-unicode-regex",
-
+    feature: "unicodeFlag",
-    visitor: {
+  });
-      RegExpLiteral({ node }) {
-        if (!regex.is(node, "u")) return;
-        node.pattern = rewritePattern(node.pattern, node.flags);
-        regex.pullFlag(node, "u");
-      },
-    },
-  };
 });

packages/babel-preset-env/test/fixtures/plugins-integration/issue-8951/exec.js

@@ -0,0 +1,3 @@
+expect(/.(?<code>\p{ASCII})/su.exec("\nA").groups).toEqual({
+  code: "A"
+});

packages/babel-preset-env/test/fixtures/plugins-integration/issue-8951/options.json

@@ -0,0 +1,4 @@
+{
+  "plugins": [],
+  "presets": ["../../../../lib"]
+}

packages/babel-preset-env/test/fixtures/plugins-integration/issue-9892/exec.js

@@ -0,0 +1 @@
+expect(/[\p{L}\d_]+/u.test('ŁŻŹĆ')).toBe(true);

packages/babel-preset-env/test/fixtures/plugins-integration/issue-9892/options.json

@@ -0,0 +1,4 @@
+{
+  "plugins": [],
+  "presets": ["../../../../lib"]
+}

packages/babel-preset-env/test/fixtures/preset-options/unicode-property-regex-chrome-49/input.js

@@ -0,0 +1 @@
+/\p{Script_Extensions=Greek}/u;

packages/babel-preset-env/test/fixtures/preset-options/unicode-property-regex-chrome-49/options.json

@@ -0,0 +1,10 @@
+{
+  "presets": [
+    [
+      "../../../../lib",
+      {
+        "targets": "chrome 49"
+      }
+    ]
+  ]
+}

packages/babel-preset-env/test/fixtures/preset-options/unicode-property-regex-chrome-49/output.js

@@ -0,0 +1 @@
+/(?:[\u0342\u0345\u0370-\u0373\u0375-\u0377\u037A-\u037D\u037F\u0384\u0386\u0388-\u038A\u038C\u038E-\u03A1\u03A3-\u03E1\u03F0-\u03FF\u1D26-\u1D2A\u1D5D-\u1D61\u1D66-\u1D6A\u1DBF-\u1DC1\u1F00-\u1F15\u1F18-\u1F1D\u1F20-\u1F45\u1F48-\u1F4D\u1F50-\u1F57\u1F59\u1F5B\u1F5D\u1F5F-\u1F7D\u1F80-\u1FB4\u1FB6-\u1FC4\u1FC6-\u1FD3\u1FD6-\u1FDB\u1FDD-\u1FEF\u1FF2-\u1FF4\u1FF6-\u1FFE\u2126\uAB65]|\uD800[\uDD40-\uDD8E\uDDA0]|\uD834[\uDE00-\uDE45])/;