cerxes/babel
3
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #6727 from babel/feat-use-charcode-constants

Browse Source 
[Babylon] Use char codes contants
This commit is contained in:
Sven SAULEAU 2017-11-16 10:35:42 +01:00 committed by GitHub
commit bb89364813
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 221 additions and 195 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
packages/babylon/package.json
View File

@ -25,6 +25,8 @@
"devDependencies": { "devDependencies": {
"@babel/helper-fixtures": "7.0.0-beta.32", "@babel/helper-fixtures": "7.0.0-beta.32",
"babel-plugin-transform-for-of-as-array": "1.0.4", "babel-plugin-transform-for-of-as-array": "1.0.4",
"babel-plugin-transform-charcodes": "0.0.10",
"charcodes": "0.0.10",
"rollup": "^0.50.0", "rollup": "^0.50.0",
"rollup-plugin-babel": "^4.0.0-beta.0", "rollup-plugin-babel": "^4.0.0-beta.0",
"rollup-plugin-node-resolve": "^3.0.0", "rollup-plugin-node-resolve": "^3.0.0",

2
packages/babylon/rollup.config.js
View File

@ -24,7 +24,7 @@ export default {
], ],
"@babel/flow", "@babel/flow",
], ],
plugins: ["transform-for-of-as-array"], plugins: ["transform-charcodes", "transform-for-of-as-array"],
}), }),
nodeResolve(), nodeResolve(),
], ],

29
packages/babylon/src/plugins/jsx/index.js
View File

@ -1,5 +1,7 @@
// @flow // @flow
import * as charCodes from "charcodes";
import XHTMLEntities from "./xhtml"; import XHTMLEntities from "./xhtml";
import type Parser from "../../parser"; import type Parser from "../../parser";
import { TokenType, types as tt } from "../../tokenizer/types"; import { TokenType, types as tt } from "../../tokenizer/types";
@ -84,10 +86,10 @@ export default (superClass: Class<Parser>): Class<Parser> =>
const ch = this.input.charCodeAt(this.state.pos); const ch = this.input.charCodeAt(this.state.pos);
switch (ch) { switch (ch) {
case 60: // "<" case charCodes.lessThan:
case 123: // "{" case charCodes.leftCurlyBrace:
if (this.state.pos === this.state.start) { if (this.state.pos === this.state.start) {
if (ch === 60 && this.state.exprAllowed) { if (ch === charCodes.lessThan && this.state.exprAllowed) {
++this.state.pos; ++this.state.pos;
return this.finishToken(tt.jsxTagStart); return this.finishToken(tt.jsxTagStart);
} }
@ -96,7 +98,7 @@ export default (superClass: Class<Parser>): Class<Parser> =>
out += this.input.slice(chunkStart, this.state.pos); out += this.input.slice(chunkStart, this.state.pos);
return this.finishToken(tt.jsxText, out); return this.finishToken(tt.jsxText, out);
case 38: // "&" case charCodes.ampersand:
out += this.input.slice(chunkStart, this.state.pos); out += this.input.slice(chunkStart, this.state.pos);
out += this.jsxReadEntity(); out += this.jsxReadEntity();
chunkStart = this.state.pos; chunkStart = this.state.pos;
@ -118,7 +120,10 @@ export default (superClass: Class<Parser>): Class<Parser> =>
const ch = this.input.charCodeAt(this.state.pos); const ch = this.input.charCodeAt(this.state.pos);
let out; let out;
++this.state.pos; ++this.state.pos;
if (ch === 13 && this.input.charCodeAt(this.state.pos) === 10) { if (
ch === charCodes.carriageReturn &&
this.input.charCodeAt(this.state.pos) === charCodes.lineFeed
) {
++this.state.pos; ++this.state.pos;
out = normalizeCRLF ? "\n" : "\r\n"; out = normalizeCRLF ? "\n" : "\r\n";
} else { } else {
@ -140,8 +145,7 @@ export default (superClass: Class<Parser>): Class<Parser> =>
const ch = this.input.charCodeAt(this.state.pos); const ch = this.input.charCodeAt(this.state.pos);
if (ch === quote) break; if (ch === quote) break;
if (ch === 38) { if (ch === charCodes.ampersand) {
// "&"
out += this.input.slice(chunkStart, this.state.pos); out += this.input.slice(chunkStart, this.state.pos);
out += this.jsxReadEntity(); out += this.jsxReadEntity();
chunkStart = this.state.pos; chunkStart = this.state.pos;
@ -205,7 +209,7 @@ export default (superClass: Class<Parser>): Class<Parser> =>
const start = this.state.pos; const start = this.state.pos;
do { do {
ch = this.input.charCodeAt(++this.state.pos); ch = this.input.charCodeAt(++this.state.pos);
} while (isIdentifierChar(ch) || ch === 45); // "-" } while (isIdentifierChar(ch) || ch === charCodes.dash);
return this.finishToken( return this.finishToken(
tt.jsxName, tt.jsxName,
this.input.slice(start, this.state.pos), this.input.slice(start, this.state.pos),
@ -513,17 +517,20 @@ export default (superClass: Class<Parser>): Class<Parser> =>
return this.jsxReadWord(); return this.jsxReadWord();
} }
if (code === 62) { if (code === charCodes.greaterThan) {
++this.state.pos; ++this.state.pos;
return this.finishToken(tt.jsxTagEnd); return this.finishToken(tt.jsxTagEnd);
} }
if ((code === 34 || code === 39) && context === tc.j_oTag) { if (
(code === charCodes.quotationMark || code === charCodes.apostrophe) &&
context === tc.j_oTag
) {
return this.jsxReadString(code); return this.jsxReadString(code);
} }
} }
if (code === 60 && this.state.exprAllowed) { if (code === charCodes.lessThan && this.state.exprAllowed) {
++this.state.pos; ++this.state.pos;
return this.finishToken(tt.jsxTagStart); return this.finishToken(tt.jsxTagStart);
} }

383
packages/babylon/src/tokenizer/index.js
View File

@ -4,6 +4,7 @@
import type { Options } from "../options"; import type { Options } from "../options";
import type { Position } from "../util/location"; import type { Position } from "../util/location";
import * as charCodes from "charcodes";
import { import {
isIdentifierStart, isIdentifierStart,
isIdentifierChar, isIdentifierChar,
@ -26,63 +27,65 @@ import State from "./state";
const forbiddenNumericSeparatorSiblings = { const forbiddenNumericSeparatorSiblings = {
decBinOct: [ decBinOct: [
46, // . charCodes.dot,
66, // B charCodes.uppercaseB,
69, // E charCodes.uppercaseE,
79, // O charCodes.uppercaseO,
95, // _ (multiple separators are not allowed) charCodes.underscore, // multiple separators are not allowed
98, // b charCodes.lowercaseB,
101, // e charCodes.lowercaseE,
111, // o charCodes.lowercaseO,
], ],
hex: [ hex: [
46, // . charCodes.dot,
88, // X charCodes.uppercaseX,
95, // _ (multiple separators are not allowed) charCodes.underscore, // multiple separators are not allowed
120, // x charCodes.lowercaseX,
], ],
}; };
const allowedNumericSeparatorSiblings = {}; const allowedNumericSeparatorSiblings = {};
allowedNumericSeparatorSiblings.bin = [ allowedNumericSeparatorSiblings.bin = [
// 0 - 1 // 0 - 1
48, charCodes.digit0,
49, charCodes.digit1,
]; ];
allowedNumericSeparatorSiblings.oct = [ allowedNumericSeparatorSiblings.oct = [
// 0 - 7 // 0 - 7
...allowedNumericSeparatorSiblings.bin, ...allowedNumericSeparatorSiblings.bin,
50,
51, charCodes.digit2,
52, charCodes.digit3,
53, charCodes.digit4,
54, charCodes.digit5,
55, charCodes.digit6,
charCodes.digit7,
]; ];
allowedNumericSeparatorSiblings.dec = [ allowedNumericSeparatorSiblings.dec = [
// 0 - 9 // 0 - 9
...allowedNumericSeparatorSiblings.oct, ...allowedNumericSeparatorSiblings.oct,
56,
57, charCodes.digit8,
charCodes.digit9,
]; ];
allowedNumericSeparatorSiblings.hex = [ allowedNumericSeparatorSiblings.hex = [
// 0 - 9, A - F, a - f, // 0 - 9, A - F, a - f,
...allowedNumericSeparatorSiblings.dec, ...allowedNumericSeparatorSiblings.dec,
// A - F
65, charCodes.uppercaseA,
66, charCodes.uppercaseB,
67, charCodes.uppercaseC,
68, charCodes.uppercaseD,
69, charCodes.uppercaseE,
70, charCodes.uppercaseF,
// a - f
97, charCodes.lowercaseA,
98, charCodes.lowercaseB,
99, charCodes.lowercaseC,
100, charCodes.lowercaseD,
101, charCodes.lowercaseE,
102, charCodes.lowercaseF,
]; ];
// Object type used to represent tokens. Note that normally, tokens // Object type used to represent tokens. Note that normally, tokens
@ -230,7 +233,7 @@ export default class Tokenizer extends LocationParser {
readToken(code: number): void { readToken(code: number): void {
// Identifier or keyword. '\uXXXX' sequences are allowed in // Identifier or keyword. '\uXXXX' sequences are allowed in
// identifiers, so '\' also dispatches to that. // identifiers, so '\' also dispatches to that.
if (isIdentifierStart(code) || code === 92 /* '\' */) { if (isIdentifierStart(code) || code === charCodes.backslash) {
this.readWord(); this.readWord();
} else { } else {
this.getTokenFromCode(code); this.getTokenFromCode(code);
@ -301,10 +304,10 @@ export default class Tokenizer extends LocationParser {
let ch = this.input.charCodeAt((this.state.pos += startSkip)); let ch = this.input.charCodeAt((this.state.pos += startSkip));
if (this.state.pos < this.input.length) { if (this.state.pos < this.input.length) {
while ( while (
ch !== 10 && ch !== charCodes.lineFeed &&
ch !== 13 && ch !== charCodes.carriageReturn &&
ch !== 8232 && ch !== charCodes.lineSeparator &&
ch !== 8233 && ch !== charCodes.paragraphSeparator &&
++this.state.pos < this.input.length ++this.state.pos < this.input.length
) { ) {
ch = this.input.charCodeAt(this.state.pos); ch = this.input.charCodeAt(this.state.pos);
@ -328,31 +331,33 @@ export default class Tokenizer extends LocationParser {
loop: while (this.state.pos < this.input.length) { loop: while (this.state.pos < this.input.length) {
const ch = this.input.charCodeAt(this.state.pos); const ch = this.input.charCodeAt(this.state.pos);
switch (ch) { switch (ch) {
case 32: // space case charCodes.space:
case 160: // non-breaking space case charCodes.nonBreakingSpace:
++this.state.pos; ++this.state.pos;
break; break;
case 13: // '\r' carriage return case charCodes.carriageReturn:
if (this.input.charCodeAt(this.state.pos + 1) === 10) { if (
this.input.charCodeAt(this.state.pos + 1) === charCodes.lineFeed
) {
++this.state.pos; ++this.state.pos;
} }
case 10: // '\n' line feed case charCodes.lineFeed:
case 8232: // line separator case charCodes.lineSeparator:
case 8233: // paragraph separator case charCodes.paragraphSeparator:
++this.state.pos; ++this.state.pos;
++this.state.curLine; ++this.state.curLine;
this.state.lineStart = this.state.pos; this.state.lineStart = this.state.pos;
break; break;
case 47: // '/' case charCodes.slash:
switch (this.input.charCodeAt(this.state.pos + 1)) { switch (this.input.charCodeAt(this.state.pos + 1)) {
case 42: // '*' case charCodes.asterisk:
this.skipBlockComment(); this.skipBlockComment();
break; break;
case 47: case charCodes.slash:
this.skipLineComment(2); this.skipLineComment(2);
break; break;
@ -363,8 +368,9 @@ export default class Tokenizer extends LocationParser {
default: default:
if ( if (
(ch > 8 && ch < 14) || (ch > charCodes.backSpace && ch < charCodes.shiftOut) ||
(ch >= 5760 && nonASCIIwhitespace.test(String.fromCharCode(ch))) (ch >= charCodes.oghamSpaceMark &&
nonASCIIwhitespace.test(String.fromCharCode(ch)))
) { ) {
++this.state.pos; ++this.state.pos;
} else { } else {
@ -400,14 +406,13 @@ export default class Tokenizer extends LocationParser {
// //
readToken_dot(): void { readToken_dot(): void {
const next = this.input.charCodeAt(this.state.pos + 1); const next = this.input.charCodeAt(this.state.pos + 1);
if (next >= 48 && next <= 57) { if (next >= charCodes.digit0 && next <= charCodes.digit9) {
this.readNumber(true); this.readNumber(true);
return; return;
} }
const next2 = this.input.charCodeAt(this.state.pos + 2); const next2 = this.input.charCodeAt(this.state.pos + 2);
if (next === 46 && next2 === 46) { if (next === charCodes.dot && next2 === charCodes.dot) {
// 46 = dot '.'
this.state.pos += 3; this.state.pos += 3;
this.finishToken(tt.ellipsis); this.finishToken(tt.ellipsis);
} else { } else {
@ -425,7 +430,7 @@ export default class Tokenizer extends LocationParser {
} }
const next = this.input.charCodeAt(this.state.pos + 1); const next = this.input.charCodeAt(this.state.pos + 1);
if (next === 61) { if (next === charCodes.equalsTo) {
this.finishOp(tt.assign, 2); this.finishOp(tt.assign, 2);
} else { } else {
this.finishOp(tt.slash, 1); this.finishOp(tt.slash, 1);
@ -434,19 +439,19 @@ export default class Tokenizer extends LocationParser {
readToken_mult_modulo(code: number): void { readToken_mult_modulo(code: number): void {
// '%*' // '%*'
let type = code === 42 ? tt.star : tt.modulo; let type = code === charCodes.asterisk ? tt.star : tt.modulo;
let width = 1; let width = 1;
let next = this.input.charCodeAt(this.state.pos + 1); let next = this.input.charCodeAt(this.state.pos + 1);
const exprAllowed = this.state.exprAllowed; const exprAllowed = this.state.exprAllowed;
// Exponentiation operator ** // Exponentiation operator **
if (code === 42 && next === 42) { if (code === charCodes.asterisk && next === charCodes.asterisk) {
width++; width++;
next = this.input.charCodeAt(this.state.pos + 2); next = this.input.charCodeAt(this.state.pos + 2);
type = tt.exponent; type = tt.exponent;
} }
if (next === 61 && !exprAllowed) { if (next === charCodes.equalsTo && !exprAllowed) {
width++; width++;
type = tt.assign; type = tt.assign;
} }
@ -459,34 +464,40 @@ export default class Tokenizer extends LocationParser {
const next = this.input.charCodeAt(this.state.pos + 1); const next = this.input.charCodeAt(this.state.pos + 1);
if (next === code) { if (next === code) {
this.finishOp(code === 124 ? tt.logicalOR : tt.logicalAND, 2); this.finishOp(
code === charCodes.verticalBar ? tt.logicalOR : tt.logicalAND,
2,
);
return; return;
} }
if (code === 124) { if (code === charCodes.verticalBar) {
// '|>' // '|>'
if (next === 62) { if (next === charCodes.greaterThan) {
this.finishOp(tt.pipeline, 2); this.finishOp(tt.pipeline, 2);
return; return;
} else if (next === 125 && this.hasPlugin("flow")) { } else if (next === charCodes.rightCurlyBrace && this.hasPlugin("flow")) {
// '|}' // '|}'
this.finishOp(tt.braceBarR, 2); this.finishOp(tt.braceBarR, 2);
return; return;
} }
} }
if (next === 61) { if (next === charCodes.equalsTo) {
this.finishOp(tt.assign, 2); this.finishOp(tt.assign, 2);
return; return;
} }
this.finishOp(code === 124 ? tt.bitwiseOR : tt.bitwiseAND, 1); this.finishOp(
code === charCodes.verticalBar ? tt.bitwiseOR : tt.bitwiseAND,
1,
);
} }
readToken_caret(): void { readToken_caret(): void {
// '^' // '^'
const next = this.input.charCodeAt(this.state.pos + 1); const next = this.input.charCodeAt(this.state.pos + 1);
if (next === 61) { if (next === charCodes.equalsTo) {
this.finishOp(tt.assign, 2); this.finishOp(tt.assign, 2);
} else { } else {
this.finishOp(tt.bitwiseXOR, 1); this.finishOp(tt.bitwiseXOR, 1);
@ -499,9 +510,9 @@ export default class Tokenizer extends LocationParser {
if (next === code) { if (next === code) {
if ( if (
next === 45 && next === charCodes.dash &&
!this.inModule && !this.inModule &&
this.input.charCodeAt(this.state.pos + 2) === 62 && this.input.charCodeAt(this.state.pos + 2) === charCodes.greaterThan &&
lineBreak.test(this.input.slice(this.state.lastTokEnd, this.state.pos)) lineBreak.test(this.input.slice(this.state.lastTokEnd, this.state.pos))
) { ) {
// A `-->` line comment // A `-->` line comment
@ -514,7 +525,7 @@ export default class Tokenizer extends LocationParser {
return; return;
} }
if (next === 61) { if (next === charCodes.equalsTo) {
this.finishOp(tt.assign, 2); this.finishOp(tt.assign, 2);
} else { } else {
this.finishOp(tt.plusMin, 1); this.finishOp(tt.plusMin, 1);
@ -528,8 +539,11 @@ export default class Tokenizer extends LocationParser {
if (next === code) { if (next === code) {
size = size =
code === 62 && this.input.charCodeAt(this.state.pos + 2) === 62 ? 3 : 2; code === charCodes.greaterThan &&
if (this.input.charCodeAt(this.state.pos + size) === 61) { this.input.charCodeAt(this.state.pos + 2) === charCodes.greaterThan
? 3
: 2;
if (this.input.charCodeAt(this.state.pos + size) === charCodes.equalsTo) {
this.finishOp(tt.assign, size + 1); this.finishOp(tt.assign, size + 1);
return; return;
} }
@ -538,11 +552,11 @@ export default class Tokenizer extends LocationParser {
} }
if ( if (
next === 33 && next === charCodes.exclamationMark &&
code === 60 && code === charCodes.lessThan &&
!this.inModule && !this.inModule &&
this.input.charCodeAt(this.state.pos + 2) === 45 && this.input.charCodeAt(this.state.pos + 2) === charCodes.dash &&
this.input.charCodeAt(this.state.pos + 3) === 45 this.input.charCodeAt(this.state.pos + 3) === charCodes.dash
) { ) {
// `<!--`, an XML-style comment that should be interpreted as a line comment // `<!--`, an XML-style comment that should be interpreted as a line comment
this.skipLineComment(4); this.skipLineComment(4);
@ -551,7 +565,7 @@ export default class Tokenizer extends LocationParser {
return; return;
} }
if (next === 61) { if (next === charCodes.equalsTo) {
// <= | >= // <= | >=
size = 2; size = 2;
} }
@ -562,30 +576,35 @@ export default class Tokenizer extends LocationParser {
readToken_eq_excl(code: number): void { readToken_eq_excl(code: number): void {
// '=!' // '=!'
const next = this.input.charCodeAt(this.state.pos + 1); const next = this.input.charCodeAt(this.state.pos + 1);
if (next === 61) { if (next === charCodes.equalsTo) {
this.finishOp( this.finishOp(
tt.equality, tt.equality,
this.input.charCodeAt(this.state.pos + 2) === 61 ? 3 : 2, this.input.charCodeAt(this.state.pos + 2) === charCodes.equalsTo
? 3
: 2,
); );
return; return;
} }
if (code === 61 && next === 62) { if (code === charCodes.equalsTo && next === charCodes.greaterThan) {
// '=>' // '=>'
this.state.pos += 2; this.state.pos += 2;
this.finishToken(tt.arrow); this.finishToken(tt.arrow);
return; return;
} }
this.finishOp(code === 61 ? tt.eq : tt.bang, 1); this.finishOp(code === charCodes.equalsTo ? tt.eq : tt.bang, 1);
} }
readToken_question(): void { readToken_question(): void {
// '?' // '?'
const next = this.input.charCodeAt(this.state.pos + 1); const next = this.input.charCodeAt(this.state.pos + 1);
const next2 = this.input.charCodeAt(this.state.pos + 2); const next2 = this.input.charCodeAt(this.state.pos + 2);
if (next === 63) { if (next === charCodes.questionMark) {
// '??' // '??'
this.finishOp(tt.nullishCoalescing, 2); this.finishOp(tt.nullishCoalescing, 2);
} else if (next === 46 && !(next2 >= 48 && next2 <= 57)) { } else if (
next === charCodes.dot &&
!(next2 >= charCodes.digit0 && next2 <= charCodes.digit9)
) {
// '.' not followed by a number // '.' not followed by a number
this.state.pos += 2; this.state.pos += 2;
this.finishToken(tt.questionDot); this.finishToken(tt.questionDot);
@ -597,7 +616,7 @@ export default class Tokenizer extends LocationParser {
getTokenFromCode(code: number): void { getTokenFromCode(code: number): void {
switch (code) { switch (code) {
case 35: // '#' case charCodes.numberSign:
if ( if (
(this.hasPlugin("classPrivateProperties") || (this.hasPlugin("classPrivateProperties") ||
this.hasPlugin("classPrivateMethods")) && this.hasPlugin("classPrivateMethods")) &&
@ -616,40 +635,40 @@ export default class Tokenizer extends LocationParser {
// The interpretation of a dot depends on whether it is followed // The interpretation of a dot depends on whether it is followed
// by a digit or another two dots. // by a digit or another two dots.
case 46: // '.' case charCodes.dot:
this.readToken_dot(); this.readToken_dot();
return; return;
// Punctuation tokens. // Punctuation tokens.
case 40: case charCodes.leftParenthesis:
++this.state.pos; ++this.state.pos;
this.finishToken(tt.parenL); this.finishToken(tt.parenL);
return; return;
case 41: case charCodes.rightParenthesis:
++this.state.pos; ++this.state.pos;
this.finishToken(tt.parenR); this.finishToken(tt.parenR);
return; return;
case 59: case charCodes.semicolon:
++this.state.pos; ++this.state.pos;
this.finishToken(tt.semi); this.finishToken(tt.semi);
return; return;
case 44: case charCodes.comma:
++this.state.pos; ++this.state.pos;
this.finishToken(tt.comma); this.finishToken(tt.comma);
return; return;
case 91: case charCodes.leftSquareBracket:
++this.state.pos; ++this.state.pos;
this.finishToken(tt.bracketL); this.finishToken(tt.bracketL);
return; return;
case 93: case charCodes.rightSquareBracket:
++this.state.pos; ++this.state.pos;
this.finishToken(tt.bracketR); this.finishToken(tt.bracketR);
return; return;
case 123: case charCodes.leftCurlyBrace:
if ( if (
this.hasPlugin("flow") && this.hasPlugin("flow") &&
this.input.charCodeAt(this.state.pos + 1) === 124 this.input.charCodeAt(this.state.pos + 1) === charCodes.verticalBar
) { ) {
this.finishOp(tt.braceBarL, 2); this.finishOp(tt.braceBarL, 2);
} else { } else {
@ -658,15 +677,15 @@ export default class Tokenizer extends LocationParser {
} }
return; return;
case 125: case charCodes.rightCurlyBrace:
++this.state.pos; ++this.state.pos;
this.finishToken(tt.braceR); this.finishToken(tt.braceR);
return; return;
case 58: case charCodes.colon:
if ( if (
this.hasPlugin("functionBind") && this.hasPlugin("functionBind") &&
this.input.charCodeAt(this.state.pos + 1) === 58 this.input.charCodeAt(this.state.pos + 1) === charCodes.colon
) { ) {
this.finishOp(tt.doubleColon, 2); this.finishOp(tt.doubleColon, 2);
} else { } else {
@ -675,97 +694,96 @@ export default class Tokenizer extends LocationParser {
} }
return; return;
case 63: case charCodes.questionMark:
this.readToken_question(); this.readToken_question();
return; return;
case 64: case charCodes.atSign:
++this.state.pos; ++this.state.pos;
this.finishToken(tt.at); this.finishToken(tt.at);
return; return;
case 96: // '`' case charCodes.graveAccent:
++this.state.pos; ++this.state.pos;
this.finishToken(tt.backQuote); this.finishToken(tt.backQuote);
return; return;
case 48: { case charCodes.digit0: {
// '0'
const next = this.input.charCodeAt(this.state.pos + 1); const next = this.input.charCodeAt(this.state.pos + 1);
// '0x', '0X' - hex number // '0x', '0X' - hex number
if (next === 120 || next === 88) { if (next === charCodes.lowercaseX || next === charCodes.uppercaseX) {
this.readRadixNumber(16); this.readRadixNumber(16);
return; return;
} }
// '0o', '0O' - octal number // '0o', '0O' - octal number
if (next === 111 || next === 79) { if (next === charCodes.lowercaseO || next === charCodes.uppercaseO) {
this.readRadixNumber(8); this.readRadixNumber(8);
return; return;
} }
// '0b', '0B' - binary number // '0b', '0B' - binary number
if (next === 98 || next === 66) { if (next === charCodes.lowercaseB || next === charCodes.uppercaseB) {
this.readRadixNumber(2); this.readRadixNumber(2);
return; return;
} }
} }
// Anything else beginning with a digit is an integer, octal // Anything else beginning with a digit is an integer, octal
// number, or float. // number, or float.
case 49: case charCodes.digit1:
case 50: case charCodes.digit2:
case 51: case charCodes.digit3:
case 52: case charCodes.digit4:
case 53: case charCodes.digit5:
case 54: case charCodes.digit6:
case 55: case charCodes.digit7:
case 56: case charCodes.digit8:
case 57: // 1-9 case charCodes.digit9:
this.readNumber(false); this.readNumber(false);
return; return;
// Quotes produce strings. // Quotes produce strings.
case 34: case charCodes.quotationMark:
case 39: // '"', "'" case charCodes.apostrophe:
this.readString(code); this.readString(code);
return; return;
// Operators are parsed inline in tiny state machines. '=' (61) is // Operators are parsed inline in tiny state machines. '=' (charCodes.equalsTo) is
// often referred to. `finishOp` simply skips the amount of // often referred to. `finishOp` simply skips the amount of
// characters it is given as second argument, and returns a token // characters it is given as second argument, and returns a token
// of the type given by its first argument. // of the type given by its first argument.
case 47: // '/' case charCodes.slash:
this.readToken_slash(); this.readToken_slash();
return; return;
case 37: case charCodes.percentSign:
case 42: // '%*' case charCodes.asterisk:
this.readToken_mult_modulo(code); this.readToken_mult_modulo(code);
return; return;
case 124: case charCodes.verticalBar:
case 38: // '|&' case charCodes.ampersand:
this.readToken_pipe_amp(code); this.readToken_pipe_amp(code);
return; return;
case 94: // '^' case charCodes.caret:
this.readToken_caret(); this.readToken_caret();
return; return;
case 43: case charCodes.plusSign:
case 45: // '+-' case charCodes.dash:
this.readToken_plus_min(code); this.readToken_plus_min(code);
return; return;
case 60: case charCodes.lessThan:
case 62: // '<>' case charCodes.greaterThan:
this.readToken_lt_gt(code); this.readToken_lt_gt(code);
return; return;
case 61: case charCodes.equalsTo:
case 33: // '=!' case charCodes.exclamationMark:
this.readToken_eq_excl(code); this.readToken_eq_excl(code);
return; return;
case 126: // '~' case charCodes.tilde:
this.finishOp(tt.tilde, 1); this.finishOp(tt.tilde, 1);
return; return;
} }
@ -853,7 +871,7 @@ export default class Tokenizer extends LocationParser {
if (this.hasPlugin("numericSeparator")) { if (this.hasPlugin("numericSeparator")) {
const prev = this.input.charCodeAt(this.state.pos - 1); const prev = this.input.charCodeAt(this.state.pos - 1);
const next = this.input.charCodeAt(this.state.pos + 1); const next = this.input.charCodeAt(this.state.pos + 1);
if (code === 95) { if (code === charCodes.underscore) {
if (allowedSiblings.indexOf(next) === -1) { if (allowedSiblings.indexOf(next) === -1) {
this.raise(this.state.pos, "Invalid or unexpected token"); this.raise(this.state.pos, "Invalid or unexpected token");
} }
@ -872,12 +890,12 @@ export default class Tokenizer extends LocationParser {
} }
} }
if (code >= 97) { if (code >= charCodes.lowercaseA) {
val = code - 97 + 10; // a val = code - charCodes.lowercaseA + charCodes.lineFeed;
} else if (code >= 65) { } else if (code >= charCodes.uppercaseA) {
val = code - 65 + 10; // A val = code - charCodes.uppercaseA + charCodes.lineFeed;
} else if (code >= 48 && code <= 57) { } else if (charCodes.isDigit(code)) {
val = code - 48; // 0-9 val = code - charCodes.digit0; // 0-9
} else { } else {
val = Infinity; val = Infinity;
} }
@ -906,8 +924,7 @@ export default class Tokenizer extends LocationParser {
} }
if (this.hasPlugin("bigInt")) { if (this.hasPlugin("bigInt")) {
if (this.input.charCodeAt(this.state.pos) === 0x6e) { if (this.input.charCodeAt(this.state.pos) === charCodes.lowercaseN) {
// 'n'
++this.state.pos; ++this.state.pos;
isBigInt = true; isBigInt = true;
} }
@ -930,7 +947,7 @@ export default class Tokenizer extends LocationParser {
readNumber(startsWithDot: boolean): void { readNumber(startsWithDot: boolean): void {
const start = this.state.pos; const start = this.state.pos;
let octal = this.input.charCodeAt(start) === 0x30; // '0' let octal = this.input.charCodeAt(start) === charCodes.digit0;
let isFloat = false; let isFloat = false;
let isBigInt = false; let isBigInt = false;
@ -940,26 +957,28 @@ export default class Tokenizer extends LocationParser {
if (octal && this.state.pos == start + 1) octal = false; // number === 0 if (octal && this.state.pos == start + 1) octal = false; // number === 0
let next = this.input.charCodeAt(this.state.pos); let next = this.input.charCodeAt(this.state.pos);
if (next === 0x2e && !octal) { if (next === charCodes.dot && !octal) {
// '.'
++this.state.pos; ++this.state.pos;
this.readInt(10); this.readInt(10);
isFloat = true; isFloat = true;
next = this.input.charCodeAt(this.state.pos); next = this.input.charCodeAt(this.state.pos);
} }
if ((next === 0x45 || next === 0x65) && !octal) { if (
// 'Ee' (next === charCodes.uppercaseE || next === charCodes.lowercaseE) &&
!octal
) {
next = this.input.charCodeAt(++this.state.pos); next = this.input.charCodeAt(++this.state.pos);
if (next === 0x2b || next === 0x2d) ++this.state.pos; // '+-' if (next === charCodes.plusSign || next === charCodes.dash) {
++this.state.pos;
}
if (this.readInt(10) === null) this.raise(start, "Invalid number"); if (this.readInt(10) === null) this.raise(start, "Invalid number");
isFloat = true; isFloat = true;
next = this.input.charCodeAt(this.state.pos); next = this.input.charCodeAt(this.state.pos);
} }
if (this.hasPlugin("bigInt")) { if (this.hasPlugin("bigInt")) {
if (next === 0x6e) { if (next === charCodes.lowercaseN) {
// 'n'
// disallow floats and legacy octal syntax, new style octal ("0o") is handled in this.readRadixNumber // disallow floats and legacy octal syntax, new style octal ("0o") is handled in this.readRadixNumber
if (isFloat || octal) this.raise(start, "Invalid BigIntLiteral"); if (isFloat || octal) this.raise(start, "Invalid BigIntLiteral");
++this.state.pos; ++this.state.pos;
@ -1000,8 +1019,7 @@ export default class Tokenizer extends LocationParser {
const ch = this.input.charCodeAt(this.state.pos); const ch = this.input.charCodeAt(this.state.pos);
let code; let code;
if (ch === 123) { if (ch === charCodes.leftCurlyBrace) {
// '{'
const codePos = ++this.state.pos; const codePos = ++this.state.pos;
code = this.readHexChar( code = this.readHexChar(
this.input.indexOf("}", this.state.pos) - this.state.pos, this.input.indexOf("}", this.state.pos) - this.state.pos,
@ -1034,8 +1052,7 @@ export default class Tokenizer extends LocationParser {
} }
const ch = this.input.charCodeAt(this.state.pos); const ch = this.input.charCodeAt(this.state.pos);
if (ch === quote) break; if (ch === quote) break;
if (ch === 92) { if (ch === charCodes.backslash) {
// '\'
out += this.input.slice(chunkStart, this.state.pos); out += this.input.slice(chunkStart, this.state.pos);
// $FlowFixMe // $FlowFixMe
out += this.readEscapedChar(false); out += this.readEscapedChar(false);
@ -1063,12 +1080,13 @@ export default class Tokenizer extends LocationParser {
} }
const ch = this.input.charCodeAt(this.state.pos); const ch = this.input.charCodeAt(this.state.pos);
if ( if (
ch === 96 || ch === charCodes.graveAccent ||
(ch === 36 && this.input.charCodeAt(this.state.pos + 1) === 123) (ch === charCodes.dollarSign &&
this.input.charCodeAt(this.state.pos + 1) ===
charCodes.leftCurlyBrace)
) { ) {
// '`', '${'
if (this.state.pos === this.state.start && this.match(tt.template)) { if (this.state.pos === this.state.start && this.match(tt.template)) {
if (ch === 36) { if (ch === charCodes.dollarSign) {
this.state.pos += 2; this.state.pos += 2;
this.finishToken(tt.dollarBraceL); this.finishToken(tt.dollarBraceL);
return; return;
@ -1082,8 +1100,7 @@ export default class Tokenizer extends LocationParser {
this.finishToken(tt.template, containsInvalid ? null : out); this.finishToken(tt.template, containsInvalid ? null : out);
return; return;
} }
if (ch === 92) { if (ch === charCodes.backslash) {
// '\'
out += this.input.slice(chunkStart, this.state.pos); out += this.input.slice(chunkStart, this.state.pos);
const escaped = this.readEscapedChar(true); const escaped = this.readEscapedChar(true);
if (escaped === null) { if (escaped === null) {
@ -1096,9 +1113,11 @@ export default class Tokenizer extends LocationParser {
out += this.input.slice(chunkStart, this.state.pos); out += this.input.slice(chunkStart, this.state.pos);
++this.state.pos; ++this.state.pos;
switch (ch) { switch (ch) {
case 13: case charCodes.carriageReturn:
if (this.input.charCodeAt(this.state.pos) === 10) ++this.state.pos; if (this.input.charCodeAt(this.state.pos) === charCodes.lineFeed) {
case 10: ++this.state.pos;
}
case charCodes.lineFeed:
out += "\n"; out += "\n";
break; break;
default: default:
@ -1121,36 +1140,36 @@ export default class Tokenizer extends LocationParser {
const ch = this.input.charCodeAt(++this.state.pos); const ch = this.input.charCodeAt(++this.state.pos);
++this.state.pos; ++this.state.pos;
switch (ch) { switch (ch) {
case 110: case charCodes.lowercaseN:
return "\n"; // 'n' -> '\n' return "\n";
case 114: case charCodes.lowercaseR:
return "\r"; // 'r' -> '\r' return "\r";
case 120: { case charCodes.lowercaseX: {
// 'x'
const code = this.readHexChar(2, throwOnInvalid); const code = this.readHexChar(2, throwOnInvalid);
return code === null ? null : String.fromCharCode(code); return code === null ? null : String.fromCharCode(code);
} }
case 117: { case charCodes.lowercaseU: {
// 'u'
const code = this.readCodePoint(throwOnInvalid); const code = this.readCodePoint(throwOnInvalid);
return code === null ? null : codePointToString(code); return code === null ? null : codePointToString(code);
} }
case 116: case charCodes.lowercaseT:
return "\t"; // 't' -> '\t' return "\t";
case 98: case charCodes.lowercaseB:
return "\b"; // 'b' -> '\b' return "\b";
case 118: case charCodes.lowercaseV:
return "\u000b"; // 'v' -> '\u000b' return "\u000b";
case 102: case charCodes.lowercaseF:
return "\f"; // 'f' -> '\f' return "\f";
case 13: case charCodes.carriageReturn:
if (this.input.charCodeAt(this.state.pos) === 10) ++this.state.pos; // '\r\n' if (this.input.charCodeAt(this.state.pos) === charCodes.lineFeed) {
case 10: // ' \n' ++this.state.pos;
}
case charCodes.lineFeed:
this.state.lineStart = this.state.pos; this.state.lineStart = this.state.pos;
++this.state.curLine; ++this.state.curLine;
return ""; return "";
default: default:
if (ch >= 48 && ch <= 55) { if (ch >= charCodes.digit0 && ch <= charCodes.digit7) {
const codePos = this.state.pos - 1; const codePos = this.state.pos - 1;
// $FlowFixMe // $FlowFixMe
let octalStr = this.input let octalStr = this.input
@ -1212,15 +1231,13 @@ export default class Tokenizer extends LocationParser {
const ch = this.fullCharCodeAtPos(); const ch = this.fullCharCodeAtPos();
if (isIdentifierChar(ch)) { if (isIdentifierChar(ch)) {
this.state.pos += ch <= 0xffff ? 1 : 2; this.state.pos += ch <= 0xffff ? 1 : 2;
} else if (ch === 92) { } else if (ch === charCodes.backslash) {
// "\"
this.state.containsEsc = true; this.state.containsEsc = true;
word += this.input.slice(chunkStart, this.state.pos); word += this.input.slice(chunkStart, this.state.pos);
const escStart = this.state.pos; const escStart = this.state.pos;
if (this.input.charCodeAt(++this.state.pos) !== 117) { if (this.input.charCodeAt(++this.state.pos) !== charCodes.lowercaseU) {
// "u"
this.raise( this.raise(
this.state.pos, this.state.pos,
"Expecting Unicode escape sequence \\uXXXX", "Expecting Unicode escape sequence \\uXXXX",