Fix: unicode characters not allowed in regexes (#7179)

Issue #6691

packages/babylon/src/tokenizer/index.js

@@ -827,13 +827,22 @@ export default class Tokenizer extends LocationParser {
     }
     const content = this.input.slice(start, this.state.pos);
     ++this.state.pos;
-    // Need to use `readWord1` because '\uXXXX' sequences are allowed
+
-    // here (don't ask).
+    const validFlags = /^[gmsiyu]$/;
-    const mods = this.readWord1();
+    let mods = "";
-    if (mods) {
+    while (this.state.pos < this.input.length) {
-      const validFlags = /^[gmsiyu]*$/;
+      const char = this.input[this.state.pos];
-      if (!validFlags.test(mods)) {
+      const charCode = this.fullCharCodeAtPos();
-        this.raise(start, "Invalid regular expression flag");
+      if (validFlags.test(char)) {
+        ++this.state.pos;
+        mods += char;
+      } else if (
+        isIdentifierChar(charCode) ||
+        charCode === charCodes.backslash
+      ) {
+        this.raise(this.state.pos, "Invalid regular expression flag");
+      } else {
+        break;
       }
     }
 

packages/babylon/test/fixtures/core/uncategorised/108/options.json

@@ -0,0 +1,3 @@
+{
+  "throws": "Invalid regular expression flag (1:16)"
+}

packages/babylon/test/fixtures/core/uncategorised/108/output.json

@@ -1,104 +0,0 @@
-{
-  "type": "File",
-  "start": 0,
-  "end": 22,
-  "loc": {
-    "start": {
-      "line": 1,
-      "column": 0
-    },
-    "end": {
-      "line": 1,
-      "column": 22
-    }
-  },
-  "program": {
-    "type": "Program",
-    "start": 0,
-    "end": 22,
-    "loc": {
-      "start": {
-        "line": 1,
-        "column": 0
-      },
-      "end": {
-        "line": 1,
-        "column": 22
-      }
-    },
-    "sourceType": "script",
-    "body": [
-      {
-        "type": "VariableDeclaration",
-        "start": 0,
-        "end": 22,
-        "loc": {
-          "start": {
-            "line": 1,
-            "column": 0
-          },
-          "end": {
-            "line": 1,
-            "column": 22
-          }
-        },
-        "declarations": [
-          {
-            "type": "VariableDeclarator",
-            "start": 4,
-            "end": 22,
-            "loc": {
-              "start": {
-                "line": 1,
-                "column": 4
-              },
-              "end": {
-                "line": 1,
-                "column": 22
-              }
-            },
-            "id": {
-              "type": "Identifier",
-              "start": 4,
-              "end": 5,
-              "loc": {
-                "start": {
-                  "line": 1,
-                  "column": 4
-                },
-                "end": {
-                  "line": 1,
-                  "column": 5
-                },
-                "identifierName": "x"
-              },
-              "name": "x"
-            },
-            "init": {
-              "type": "RegExpLiteral",
-              "start": 8,
-              "end": 22,
-              "loc": {
-                "start": {
-                  "line": 1,
-                  "column": 8
-                },
-                "end": {
-                  "line": 1,
-                  "column": 22
-                }
-              },
-              "extra": {
-                "raw": "/[P QR]/\\u0067"
-              },
-              "pattern": "[P QR]",
-              "flags": "g"
-            }
-          }
-        ],
-        "kind": "var"
-      }
-    ],
-    "directives": []
-  }
-}

packages/babylon/test/fixtures/core/uncategorised/366/options.json

@@ -1,3 +1,3 @@
 {
-  "throws": "Bad character escape sequence (1:17)"
+  "throws": "Invalid regular expression flag (1:15)"
-}
+}

scripts/tests/flow/flow_tests_whitelist.txt

@@ -33,7 +33,6 @@ invalid_syntax/migrated_0000.js
 invalid_syntax/migrated_0001.js
 invalid_syntax/migrated_0002.js
 invalid_syntax/migrated_0003.js
-invalid_syntax/migrated_0010.js
 private_class_properties/valid.js
 types/annotations/migrated_0001.js
 types/annotations_in_comments_invalid/migrated_0003.js

scripts/tests/test262/test262_whitelist.txt

@@ -170,8 +170,6 @@ language/import/escaped-from.js(default)
 language/import/escaped-from.js(strict mode)
 language/literals/regexp/early-err-dup-flag.js(default)
 language/literals/regexp/early-err-dup-flag.js(strict mode)
-language/literals/regexp/early-err-flags-unicode-escape.js(default)
-language/literals/regexp/early-err-flags-unicode-escape.js(strict mode)
 language/literals/regexp/early-err-pattern.js(default)
 language/literals/regexp/early-err-pattern.js(strict mode)
 language/literals/regexp/invalid-braced-quantifier-exact.js(default)