cerxes/babel
3
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Remove input and length from state (#9646)

Browse Source
This commit is contained in:
Daniel Tschinder 2019-03-11 00:42:42 -07:00 committed by GitHub
parent ec318d01fa
commit cf4bd8bb8d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 147 additions and 182 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
packages/babel-parser/src/parser/base.js
View File

@ -16,6 +16,10 @@ export default class BaseParser {
// Initialized by Tokenizer // Initialized by Tokenizer
state: State; state: State;
// input and length are not in state as they are constant and we do
// not want to ever copy them, which happens if state gets cloned
input: string;
length: number;
hasPlugin(name: string): boolean { hasPlugin(name: string): boolean {
return this.plugins.has(name); return this.plugins.has(name);

13
packages/babel-parser/src/parser/expression.js
View File

@ -725,7 +725,7 @@ export default class ExpressionParser extends LValParser {
base.name === "async" && base.name === "async" &&
this.state.lastTokEnd === base.end && this.state.lastTokEnd === base.end &&
!this.canInsertSemicolon() && !this.canInsertSemicolon() &&
this.state.input.slice(base.start, base.end) === "async" this.input.slice(base.start, base.end) === "async"
); );
} }
@ -1164,11 +1164,7 @@ export default class ExpressionParser extends LValParser {
const node = this.startNodeAt(startPos, startLoc); const node = this.startNodeAt(startPos, startLoc);
this.addExtra(node, "rawValue", value); this.addExtra(node, "rawValue", value);
this.addExtra( this.addExtra(node, "raw", this.input.slice(startPos, this.state.end));
node,
"raw",
this.state.input.slice(startPos, this.state.end),
);
node.value = value; node.value = value;
this.next(); this.next();
return this.finishNode(node, type); return this.finishNode(node, type);
@ -1390,7 +1386,7 @@ export default class ExpressionParser extends LValParser {
} }
} }
elem.value = { elem.value = {
raw: this.state.input raw: this.input
.slice(this.state.start, this.state.end) .slice(this.state.start, this.state.end)
.replace(/\r\n?/g, "\n"), .replace(/\r\n?/g, "\n"),
cooked: this.state.value, cooked: this.state.value,
@ -2028,8 +2024,7 @@ export default class ExpressionParser extends LValParser {
if ( if (
(name === "class" || name === "function") && (name === "class" || name === "function") &&
(this.state.lastTokEnd !== this.state.lastTokStart + 1 || (this.state.lastTokEnd !== this.state.lastTokStart + 1 ||
this.state.input.charCodeAt(this.state.lastTokStart) !== this.input.charCodeAt(this.state.lastTokStart) !== charCodes.dot)
charCodes.dot)
) { ) {
this.state.context.pop(); this.state.context.pop();
} }

2
packages/babel-parser/src/parser/location.js
View File

@ -21,7 +21,7 @@ export default class LocationParser extends CommentsParser {
code?: string, code?: string,
} = {}, } = {},
): empty { ): empty {
const loc = getLineInfo(this.state.input, pos); const loc = getLineInfo(this.input, pos);
message += ` (${loc.line}:${loc.column})`; message += ` (${loc.line}:${loc.column})`;
// $FlowIgnore // $FlowIgnore
const err: SyntaxError & { pos: number, loc: Position } = new SyntaxError( const err: SyntaxError & { pos: number, loc: Position } = new SyntaxError(

25
packages/babel-parser/src/parser/statement.js
View File

@ -73,7 +73,7 @@ export default class StatementParser extends ExpressionParser {
const directiveLiteral = this.startNodeAt(expr.start, expr.loc.start); const directiveLiteral = this.startNodeAt(expr.start, expr.loc.start);
const directive = this.startNodeAt(stmt.start, stmt.loc.start); const directive = this.startNodeAt(stmt.start, stmt.loc.start);
const raw = this.state.input.slice(expr.start, expr.end); const raw = this.input.slice(expr.start, expr.end);
const val = (directiveLiteral.value = raw.slice(1, -1)); // remove quotes const val = (directiveLiteral.value = raw.slice(1, -1)); // remove quotes
this.addExtra(directiveLiteral, "raw", raw); this.addExtra(directiveLiteral, "raw", raw);
@ -105,10 +105,10 @@ export default class StatementParser extends ExpressionParser {
return false; return false;
} }
skipWhiteSpace.lastIndex = this.state.pos; skipWhiteSpace.lastIndex = this.state.pos;
const skip = skipWhiteSpace.exec(this.state.input); const skip = skipWhiteSpace.exec(this.input);
// $FlowIgnore // $FlowIgnore
const next = this.state.pos + skip[0].length; const next = this.state.pos + skip[0].length;
const nextCh = this.state.input.charCodeAt(next); const nextCh = this.input.charCodeAt(next);
// For ambiguous cases, determine if a LexicalDeclaration (or only a // For ambiguous cases, determine if a LexicalDeclaration (or only a
// Statement) is allowed here. If context is not empty then only a Statement // Statement) is allowed here. If context is not empty then only a Statement
// is allowed. However, `let [` is an explicit negative lookahead for // is allowed. However, `let [` is an explicit negative lookahead for
@ -120,10 +120,10 @@ export default class StatementParser extends ExpressionParser {
if (isIdentifierStart(nextCh)) { if (isIdentifierStart(nextCh)) {
let pos = next + 1; let pos = next + 1;
while (isIdentifierChar(this.state.input.charCodeAt(pos))) { while (isIdentifierChar(this.input.charCodeAt(pos))) {
++pos; ++pos;
} }
const ident = this.state.input.slice(next, pos); const ident = this.input.slice(next, pos);
if (!keywordRelationalOperator.test(ident)) return true; if (!keywordRelationalOperator.test(ident)) return true;
} }
return false; return false;
@ -649,9 +649,7 @@ export default class StatementParser extends ExpressionParser {
parseThrowStatement(node: N.ThrowStatement): N.ThrowStatement { parseThrowStatement(node: N.ThrowStatement): N.ThrowStatement {
this.next(); this.next();
if ( if (
lineBreak.test( lineBreak.test(this.input.slice(this.state.lastTokEnd, this.state.start))
this.state.input.slice(this.state.lastTokEnd, this.state.start),
)
) { ) {
this.raise(this.state.lastTokEnd, "Illegal newline after throw"); this.raise(this.state.lastTokEnd, "Illegal newline after throw");
} }
@ -1747,19 +1745,20 @@ export default class StatementParser extends ExpressionParser {
isAsyncFunction(): boolean { isAsyncFunction(): boolean {
if (!this.isContextual("async")) return false; if (!this.isContextual("async")) return false;
const { input, pos, length } = this.state; const { pos } = this.state;
skipWhiteSpace.lastIndex = pos; skipWhiteSpace.lastIndex = pos;
const skip = skipWhiteSpace.exec(input); const skip = skipWhiteSpace.exec(this.input);
if (!skip || !skip.length) return false; if (!skip || !skip.length) return false;
const next = pos + skip[0].length; const next = pos + skip[0].length;
return ( return (
!lineBreak.test(input.slice(pos, next)) && !lineBreak.test(this.input.slice(pos, next)) &&
input.slice(next, next + 8) === "function" && this.input.slice(next, next + 8) === "function" &&
(next + 8 === length || !isIdentifierChar(input.charCodeAt(next + 8))) (next + 8 === this.length ||
!isIdentifierChar(this.input.charCodeAt(next + 8)))
); );
} }

10
packages/babel-parser/src/parser/util.js
View File

@ -89,7 +89,7 @@ export default class UtilParser extends Tokenizer {
hasPrecedingLineBreak(): boolean { hasPrecedingLineBreak(): boolean {
return lineBreak.test( return lineBreak.test(
this.state.input.slice(this.state.lastTokEnd, this.state.start), this.input.slice(this.state.lastTokEnd, this.state.start),
); );
} }
@ -180,8 +180,8 @@ export default class UtilParser extends Tokenizer {
// Try to find string literal. // Try to find string literal.
skipWhiteSpace.lastIndex = start; skipWhiteSpace.lastIndex = start;
// $FlowIgnore // $FlowIgnore
start += skipWhiteSpace.exec(this.state.input)[0].length; start += skipWhiteSpace.exec(this.input)[0].length;
const match = literal.exec(this.state.input.slice(start)); const match = literal.exec(this.input.slice(start));
if (!match) break; if (!match) break;
if (match[2] === "use strict") return true; if (match[2] === "use strict") return true;
start += match[0].length; start += match[0].length;
@ -189,8 +189,8 @@ export default class UtilParser extends Tokenizer {
// Skip semicolon, if any. // Skip semicolon, if any.
skipWhiteSpace.lastIndex = start; skipWhiteSpace.lastIndex = start;
// $FlowIgnore // $FlowIgnore
start += skipWhiteSpace.exec(this.state.input)[0].length; start += skipWhiteSpace.exec(this.input)[0].length;
if (this.state.input[start] === ";") { if (this.input[start] === ";") {
start++; start++;
} }
} }

22
packages/babel-parser/src/plugins/flow.js
View File

@ -1137,7 +1137,7 @@ export default (superClass: Class<Parser>): Class<Parser> =>
node.types = []; node.types = [];
this.expect(tt.bracketL); this.expect(tt.bracketL);
// We allow trailing commas // We allow trailing commas
while (this.state.pos < this.state.length && !this.match(tt.bracketR)) { while (this.state.pos < this.length && !this.match(tt.bracketR)) {
node.types.push(this.flowParseType()); node.types.push(this.flowParseType());
if (this.match(tt.bracketR)) break; if (this.match(tt.bracketR)) break;
this.expect(tt.comma); this.expect(tt.comma);
@ -1938,7 +1938,7 @@ export default (superClass: Class<Parser>): Class<Parser> =>
// ensure that inside flow types, we bypass the jsx parser plugin // ensure that inside flow types, we bypass the jsx parser plugin
getTokenFromCode(code: number): void { getTokenFromCode(code: number): void {
const next = this.state.input.charCodeAt(this.state.pos + 1); const next = this.input.charCodeAt(this.state.pos + 1);
if (code === charCodes.leftCurlyBrace && next === charCodes.verticalBar) { if (code === charCodes.leftCurlyBrace && next === charCodes.verticalBar) {
return this.finishOp(tt.braceBarL, 2); return this.finishOp(tt.braceBarL, 2);
} else if ( } else if (
@ -2710,7 +2710,7 @@ export default (superClass: Class<Parser>): Class<Parser> =>
} }
readToken_mult_modulo(code: number): void { readToken_mult_modulo(code: number): void {
const next = this.state.input.charCodeAt(this.state.pos + 1); const next = this.input.charCodeAt(this.state.pos + 1);
if ( if (
code === charCodes.asterisk && code === charCodes.asterisk &&
next === charCodes.slash && next === charCodes.slash &&
@ -2726,7 +2726,7 @@ export default (superClass: Class<Parser>): Class<Parser> =>
} }
readToken_pipe_amp(code: number): void { readToken_pipe_amp(code: number): void {
const next = this.state.input.charCodeAt(this.state.pos + 1); const next = this.input.charCodeAt(this.state.pos + 1);
if ( if (
code === charCodes.verticalBar && code === charCodes.verticalBar &&
next === charCodes.rightCurlyBrace next === charCodes.rightCurlyBrace
@ -2762,7 +2762,7 @@ export default (superClass: Class<Parser>): Class<Parser> =>
} }
if (this.state.hasFlowComment) { if (this.state.hasFlowComment) {
const end = this.state.input.indexOf("*-/", (this.state.pos += 2)); const end = this.input.indexOf("*-/", (this.state.pos += 2));
if (end === -1) this.raise(this.state.pos - 2, "Unterminated comment"); if (end === -1) this.raise(this.state.pos - 2, "Unterminated comment");
this.state.pos = end + 3; this.state.pos = end + 3;
return; return;
@ -2776,22 +2776,20 @@ export default (superClass: Class<Parser>): Class<Parser> =>
let shiftToFirstNonWhiteSpace = 2; let shiftToFirstNonWhiteSpace = 2;
while ( while (
[charCodes.space, charCodes.tab].includes( [charCodes.space, charCodes.tab].includes(
this.state.input.charCodeAt(pos + shiftToFirstNonWhiteSpace), this.input.charCodeAt(pos + shiftToFirstNonWhiteSpace),
) )
) { ) {
shiftToFirstNonWhiteSpace++; shiftToFirstNonWhiteSpace++;
} }
const ch2 = this.state.input.charCodeAt(shiftToFirstNonWhiteSpace + pos); const ch2 = this.input.charCodeAt(shiftToFirstNonWhiteSpace + pos);
const ch3 = this.state.input.charCodeAt( const ch3 = this.input.charCodeAt(shiftToFirstNonWhiteSpace + pos + 1);
shiftToFirstNonWhiteSpace + pos + 1,
);
if (ch2 === charCodes.colon && ch3 === charCodes.colon) { if (ch2 === charCodes.colon && ch3 === charCodes.colon) {
return shiftToFirstNonWhiteSpace + 2; // check for /*:: return shiftToFirstNonWhiteSpace + 2; // check for /*::
} }
if ( if (
this.state.input.slice( this.input.slice(
shiftToFirstNonWhiteSpace + pos, shiftToFirstNonWhiteSpace + pos,
shiftToFirstNonWhiteSpace + pos + 12, shiftToFirstNonWhiteSpace + pos + 12,
) === "flow-include" ) === "flow-include"
@ -2805,7 +2803,7 @@ export default (superClass: Class<Parser>): Class<Parser> =>
} }
hasFlowCommentCompletion(): void { hasFlowCommentCompletion(): void {
const end = this.state.input.indexOf("*/", this.state.pos); const end = this.input.indexOf("*/", this.state.pos);
if (end === -1) { if (end === -1) {
this.raise(this.state.pos, "Unterminated comment"); this.raise(this.state.pos, "Unterminated comment");
} }

40
packages/babel-parser/src/plugins/jsx/index.js
View File

@ -81,11 +81,11 @@ export default (superClass: Class<Parser>): Class<Parser> =>
let out = ""; let out = "";
let chunkStart = this.state.pos; let chunkStart = this.state.pos;
for (;;) { for (;;) {
if (this.state.pos >= this.state.length) { if (this.state.pos >= this.length) {
this.raise(this.state.start, "Unterminated JSX contents"); this.raise(this.state.start, "Unterminated JSX contents");
} }
const ch = this.state.input.charCodeAt(this.state.pos); const ch = this.input.charCodeAt(this.state.pos);
switch (ch) { switch (ch) {
case charCodes.lessThan: case charCodes.lessThan:
@ -97,18 +97,18 @@ export default (superClass: Class<Parser>): Class<Parser> =>
} }
return super.getTokenFromCode(ch); return super.getTokenFromCode(ch);
} }
out += this.state.input.slice(chunkStart, this.state.pos); out += this.input.slice(chunkStart, this.state.pos);
return this.finishToken(tt.jsxText, out); return this.finishToken(tt.jsxText, out);
case charCodes.ampersand: case charCodes.ampersand:
out += this.state.input.slice(chunkStart, this.state.pos); out += this.input.slice(chunkStart, this.state.pos);
out += this.jsxReadEntity(); out += this.jsxReadEntity();
chunkStart = this.state.pos; chunkStart = this.state.pos;
break; break;
default: default:
if (isNewLine(ch)) { if (isNewLine(ch)) {
out += this.state.input.slice(chunkStart, this.state.pos); out += this.input.slice(chunkStart, this.state.pos);
out += this.jsxReadNewLine(true); out += this.jsxReadNewLine(true);
chunkStart = this.state.pos; chunkStart = this.state.pos;
} else { } else {
@ -119,12 +119,12 @@ export default (superClass: Class<Parser>): Class<Parser> =>
} }
jsxReadNewLine(normalizeCRLF: boolean): string { jsxReadNewLine(normalizeCRLF: boolean): string {
const ch = this.state.input.charCodeAt(this.state.pos); const ch = this.input.charCodeAt(this.state.pos);
let out; let out;
++this.state.pos; ++this.state.pos;
if ( if (
ch === charCodes.carriageReturn && ch === charCodes.carriageReturn &&
this.state.input.charCodeAt(this.state.pos) === charCodes.lineFeed this.input.charCodeAt(this.state.pos) === charCodes.lineFeed
) { ) {
++this.state.pos; ++this.state.pos;
out = normalizeCRLF ? "\n" : "\r\n"; out = normalizeCRLF ? "\n" : "\r\n";
@ -141,25 +141,25 @@ export default (superClass: Class<Parser>): Class<Parser> =>
let out = ""; let out = "";
let chunkStart = ++this.state.pos; let chunkStart = ++this.state.pos;
for (;;) { for (;;) {
if (this.state.pos >= this.state.length) { if (this.state.pos >= this.length) {
this.raise(this.state.start, "Unterminated string constant"); this.raise(this.state.start, "Unterminated string constant");
} }
const ch = this.state.input.charCodeAt(this.state.pos); const ch = this.input.charCodeAt(this.state.pos);
if (ch === quote) break; if (ch === quote) break;
if (ch === charCodes.ampersand) { if (ch === charCodes.ampersand) {
out += this.state.input.slice(chunkStart, this.state.pos); out += this.input.slice(chunkStart, this.state.pos);
out += this.jsxReadEntity(); out += this.jsxReadEntity();
chunkStart = this.state.pos; chunkStart = this.state.pos;
} else if (isNewLine(ch)) { } else if (isNewLine(ch)) {
out += this.state.input.slice(chunkStart, this.state.pos); out += this.input.slice(chunkStart, this.state.pos);
out += this.jsxReadNewLine(false); out += this.jsxReadNewLine(false);
chunkStart = this.state.pos; chunkStart = this.state.pos;
} else { } else {
++this.state.pos; ++this.state.pos;
} }
} }
out += this.state.input.slice(chunkStart, this.state.pos++); out += this.input.slice(chunkStart, this.state.pos++);
return this.finishToken(tt.string, out); return this.finishToken(tt.string, out);
} }
@ -167,11 +167,11 @@ export default (superClass: Class<Parser>): Class<Parser> =>
let str = ""; let str = "";
let count = 0; let count = 0;
let entity; let entity;
let ch = this.state.input[this.state.pos]; let ch = this.input[this.state.pos];
const startPos = ++this.state.pos; const startPos = ++this.state.pos;
while (this.state.pos < this.state.length && count++ < 10) { while (this.state.pos < this.length && count++ < 10) {
ch = this.state.input[this.state.pos++]; ch = this.input[this.state.pos++];
if (ch === ";") { if (ch === ";") {
if (str[0] === "#") { if (str[0] === "#") {
if (str[1] === "x") { if (str[1] === "x") {
@ -210,11 +210,11 @@ export default (superClass: Class<Parser>): Class<Parser> =>
let ch; let ch;
const start = this.state.pos; const start = this.state.pos;
do { do {
ch = this.state.input.charCodeAt(++this.state.pos); ch = this.input.charCodeAt(++this.state.pos);
} while (isIdentifierChar(ch) || ch === charCodes.dash); } while (isIdentifierChar(ch) || ch === charCodes.dash);
return this.finishToken( return this.finishToken(
tt.jsxName, tt.jsxName,
this.state.input.slice(start, this.state.pos), this.input.slice(start, this.state.pos),
); );
} }
@ -510,8 +510,7 @@ export default (superClass: Class<Parser>): Class<Parser> =>
return this.jsxParseElement(); return this.jsxParseElement();
} else if ( } else if (
this.isRelational("<") && this.isRelational("<") &&
this.state.input.charCodeAt(this.state.pos) !== this.input.charCodeAt(this.state.pos) !== charCodes.exclamationMark
charCodes.exclamationMark
) { ) {
// In case we encounter an lt token here it will always be the start of // In case we encounter an lt token here it will always be the start of
// jsx as the lt sign is not allowed in places that expect an expression // jsx as the lt sign is not allowed in places that expect an expression
@ -552,8 +551,7 @@ export default (superClass: Class<Parser>): Class<Parser> =>
if ( if (
code === charCodes.lessThan && code === charCodes.lessThan &&
this.state.exprAllowed && this.state.exprAllowed &&
this.state.input.charCodeAt(this.state.pos + 1) !== this.input.charCodeAt(this.state.pos + 1) !== charCodes.exclamationMark
charCodes.exclamationMark
) { ) {
++this.state.pos; ++this.state.pos;
return this.finishToken(tt.jsxTagStart); return this.finishToken(tt.jsxTagStart);

3
packages/babel-parser/src/plugins/placeholders.js
View File

@ -82,8 +82,7 @@ export default (superClass: Class<Parser>): Class<Parser> =>
getTokenFromCode(code: number) { getTokenFromCode(code: number) {
if ( if (
code === charCodes.percentSign && code === charCodes.percentSign &&
this.state.input.charCodeAt(this.state.pos + 1) === this.input.charCodeAt(this.state.pos + 1) === charCodes.percentSign
charCodes.percentSign
) { ) {
return this.finishOp(tt.placeholder, 2); return this.finishOp(tt.placeholder, 2);
} }

4
packages/babel-parser/src/tokenizer/context.js
View File

@ -107,9 +107,7 @@ tt._function.updateContext = tt._class.updateContext = function(prevType) {
prevType !== tt._else && prevType !== tt._else &&
!( !(
prevType === tt._return && prevType === tt._return &&
lineBreak.test( lineBreak.test(this.input.slice(this.state.lastTokEnd, this.state.start))
this.state.input.slice(this.state.lastTokEnd, this.state.start),
)
) && ) &&
!( !(
(prevType === tt.colon || prevType === tt.braceL) && (prevType === tt.colon || prevType === tt.braceL) &&

198
packages/babel-parser/src/tokenizer/index.js
View File

@ -116,7 +116,9 @@ export default class Tokenizer extends LocationParser {
constructor(options: Options, input: string) { constructor(options: Options, input: string) {
super(); super();
this.state = new State(); this.state = new State();
this.state.init(options, input); this.state.init(options);
this.input = input;
this.length = input.length;
this.isLookahead = false; this.isLookahead = false;
} }
@ -175,7 +177,7 @@ export default class Tokenizer extends LocationParser {
this.state.pos = this.state.start; this.state.pos = this.state.start;
while (this.state.pos < this.state.lineStart) { while (this.state.pos < this.state.lineStart) {
this.state.lineStart = this.state.lineStart =
this.state.input.lastIndexOf("\n", this.state.lineStart - 2) + 1; this.input.lastIndexOf("\n", this.state.lineStart - 2) + 1;
--this.state.curLine; --this.state.curLine;
} }
this.nextToken(); this.nextToken();
@ -196,7 +198,7 @@ export default class Tokenizer extends LocationParser {
this.state.octalPosition = null; this.state.octalPosition = null;
this.state.start = this.state.pos; this.state.start = this.state.pos;
this.state.startLoc = this.state.curPosition(); this.state.startLoc = this.state.curPosition();
if (this.state.pos >= this.state.length) { if (this.state.pos >= this.length) {
this.finishToken(tt.eof); this.finishToken(tt.eof);
return; return;
} }
@ -204,7 +206,7 @@ export default class Tokenizer extends LocationParser {
if (curContext.override) { if (curContext.override) {
curContext.override(this); curContext.override(this);
} else { } else {
this.getTokenFromCode(this.state.input.codePointAt(this.state.pos)); this.getTokenFromCode(this.input.codePointAt(this.state.pos));
} }
} }
@ -234,14 +236,14 @@ export default class Tokenizer extends LocationParser {
skipBlockComment(): void { skipBlockComment(): void {
const startLoc = this.state.curPosition(); const startLoc = this.state.curPosition();
const start = this.state.pos; const start = this.state.pos;
const end = this.state.input.indexOf("*/", (this.state.pos += 2)); const end = this.input.indexOf("*/", (this.state.pos += 2));
if (end === -1) this.raise(this.state.pos - 2, "Unterminated comment"); if (end === -1) this.raise(this.state.pos - 2, "Unterminated comment");
this.state.pos = end + 2; this.state.pos = end + 2;
lineBreakG.lastIndex = start; lineBreakG.lastIndex = start;
let match; let match;
while ( while (
(match = lineBreakG.exec(this.state.input)) && (match = lineBreakG.exec(this.input)) &&
match.index < this.state.pos match.index < this.state.pos
) { ) {
++this.state.curLine; ++this.state.curLine;
@ -250,7 +252,7 @@ export default class Tokenizer extends LocationParser {
this.pushComment( this.pushComment(
true, true,
this.state.input.slice(start + 2, end), this.input.slice(start + 2, end),
start, start,
this.state.pos, this.state.pos,
startLoc, startLoc,
@ -261,22 +263,22 @@ export default class Tokenizer extends LocationParser {
skipLineComment(startSkip: number): void { skipLineComment(startSkip: number): void {
const start = this.state.pos; const start = this.state.pos;
const startLoc = this.state.curPosition(); const startLoc = this.state.curPosition();
let ch = this.state.input.charCodeAt((this.state.pos += startSkip)); let ch = this.input.charCodeAt((this.state.pos += startSkip));
if (this.state.pos < this.state.length) { if (this.state.pos < this.length) {
while ( while (
ch !== charCodes.lineFeed && ch !== charCodes.lineFeed &&
ch !== charCodes.carriageReturn && ch !== charCodes.carriageReturn &&
ch !== charCodes.lineSeparator && ch !== charCodes.lineSeparator &&
ch !== charCodes.paragraphSeparator && ch !== charCodes.paragraphSeparator &&
++this.state.pos < this.state.length ++this.state.pos < this.length
) { ) {
ch = this.state.input.charCodeAt(this.state.pos); ch = this.input.charCodeAt(this.state.pos);
} }
} }
this.pushComment( this.pushComment(
false, false,
this.state.input.slice(start + startSkip, this.state.pos), this.input.slice(start + startSkip, this.state.pos),
start, start,
this.state.pos, this.state.pos,
startLoc, startLoc,
@ -288,8 +290,8 @@ export default class Tokenizer extends LocationParser {
// whitespace and comments, and. // whitespace and comments, and.
skipSpace(): void { skipSpace(): void {
loop: while (this.state.pos < this.state.length) { loop: while (this.state.pos < this.length) {
const ch = this.state.input.charCodeAt(this.state.pos); const ch = this.input.charCodeAt(this.state.pos);
switch (ch) { switch (ch) {
case charCodes.space: case charCodes.space:
case charCodes.nonBreakingSpace: case charCodes.nonBreakingSpace:
@ -298,8 +300,7 @@ export default class Tokenizer extends LocationParser {
break; break;
case charCodes.carriageReturn: case charCodes.carriageReturn:
if ( if (
this.state.input.charCodeAt(this.state.pos + 1) === this.input.charCodeAt(this.state.pos + 1) === charCodes.lineFeed
charCodes.lineFeed
) { ) {
++this.state.pos; ++this.state.pos;
} }
@ -313,7 +314,7 @@ export default class Tokenizer extends LocationParser {
break; break;
case charCodes.slash: case charCodes.slash:
switch (this.state.input.charCodeAt(this.state.pos + 1)) { switch (this.input.charCodeAt(this.state.pos + 1)) {
case charCodes.asterisk: case charCodes.asterisk:
this.skipBlockComment(); this.skipBlockComment();
break; break;
@ -368,7 +369,7 @@ export default class Tokenizer extends LocationParser {
} }
const nextPos = this.state.pos + 1; const nextPos = this.state.pos + 1;
const next = this.state.input.charCodeAt(nextPos); const next = this.input.charCodeAt(nextPos);
if (next >= charCodes.digit0 && next <= charCodes.digit9) { if (next >= charCodes.digit0 && next <= charCodes.digit9) {
this.raise(this.state.pos, "Unexpected digit after hash token"); this.raise(this.state.pos, "Unexpected digit after hash token");
} }
@ -391,13 +392,13 @@ export default class Tokenizer extends LocationParser {
} }
readToken_dot(): void { readToken_dot(): void {
const next = this.state.input.charCodeAt(this.state.pos + 1); const next = this.input.charCodeAt(this.state.pos + 1);
if (next >= charCodes.digit0 && next <= charCodes.digit9) { if (next >= charCodes.digit0 && next <= charCodes.digit9) {
this.readNumber(true); this.readNumber(true);
return; return;
} }
const next2 = this.state.input.charCodeAt(this.state.pos + 2); const next2 = this.input.charCodeAt(this.state.pos + 2);
if (next === charCodes.dot && next2 === charCodes.dot) { if (next === charCodes.dot && next2 === charCodes.dot) {
this.state.pos += 3; this.state.pos += 3;
this.finishToken(tt.ellipsis); this.finishToken(tt.ellipsis);
@ -415,7 +416,7 @@ export default class Tokenizer extends LocationParser {
return; return;
} }
const next = this.state.input.charCodeAt(this.state.pos + 1); const next = this.input.charCodeAt(this.state.pos + 1);
if (next === charCodes.equalsTo) { if (next === charCodes.equalsTo) {
this.finishOp(tt.assign, 2); this.finishOp(tt.assign, 2);
} else { } else {
@ -424,12 +425,12 @@ export default class Tokenizer extends LocationParser {
} }
readToken_interpreter(): boolean { readToken_interpreter(): boolean {
if (this.state.pos !== 0 || this.state.length < 2) return false; if (this.state.pos !== 0 || this.length < 2) return false;
const start = this.state.pos; const start = this.state.pos;
this.state.pos += 1; this.state.pos += 1;
let ch = this.state.input.charCodeAt(this.state.pos); let ch = this.input.charCodeAt(this.state.pos);
if (ch !== charCodes.exclamationMark) return false; if (ch !== charCodes.exclamationMark) return false;
while ( while (
@ -437,12 +438,12 @@ export default class Tokenizer extends LocationParser {
ch !== charCodes.carriageReturn && ch !== charCodes.carriageReturn &&
ch !== charCodes.lineSeparator && ch !== charCodes.lineSeparator &&
ch !== charCodes.paragraphSeparator && ch !== charCodes.paragraphSeparator &&
++this.state.pos < this.state.length ++this.state.pos < this.length
) { ) {
ch = this.state.input.charCodeAt(this.state.pos); ch = this.input.charCodeAt(this.state.pos);
} }
const value = this.state.input.slice(start + 2, this.state.pos); const value = this.input.slice(start + 2, this.state.pos);
this.finishToken(tt.interpreterDirective, value); this.finishToken(tt.interpreterDirective, value);
@ -453,13 +454,13 @@ export default class Tokenizer extends LocationParser {
// '%*' // '%*'
let type = code === charCodes.asterisk ? tt.star : tt.modulo; let type = code === charCodes.asterisk ? tt.star : tt.modulo;
let width = 1; let width = 1;
let next = this.state.input.charCodeAt(this.state.pos + 1); let next = this.input.charCodeAt(this.state.pos + 1);
const exprAllowed = this.state.exprAllowed; const exprAllowed = this.state.exprAllowed;
// Exponentiation operator ** // Exponentiation operator **
if (code === charCodes.asterisk && next === charCodes.asterisk) { if (code === charCodes.asterisk && next === charCodes.asterisk) {
width++; width++;
next = this.state.input.charCodeAt(this.state.pos + 2); next = this.input.charCodeAt(this.state.pos + 2);
type = tt.exponent; type = tt.exponent;
} }
@ -473,12 +474,10 @@ export default class Tokenizer extends LocationParser {
readToken_pipe_amp(code: number): void { readToken_pipe_amp(code: number): void {
// '||' '&&' '||=' '&&=' // '||' '&&' '||=' '&&='
const next = this.state.input.charCodeAt(this.state.pos + 1); const next = this.input.charCodeAt(this.state.pos + 1);
if (next === code) { if (next === code) {
if ( if (this.input.charCodeAt(this.state.pos + 2) === charCodes.equalsTo) {
this.state.input.charCodeAt(this.state.pos + 2) === charCodes.equalsTo
) {
this.finishOp(tt.assign, 3); this.finishOp(tt.assign, 3);
} else { } else {
this.finishOp( this.finishOp(
@ -510,7 +509,7 @@ export default class Tokenizer extends LocationParser {
readToken_caret(): void { readToken_caret(): void {
// '^' // '^'
const next = this.state.input.charCodeAt(this.state.pos + 1); const next = this.input.charCodeAt(this.state.pos + 1);
if (next === charCodes.equalsTo) { if (next === charCodes.equalsTo) {
this.finishOp(tt.assign, 2); this.finishOp(tt.assign, 2);
} else { } else {
@ -520,17 +519,14 @@ export default class Tokenizer extends LocationParser {
readToken_plus_min(code: number): void { readToken_plus_min(code: number): void {
// '+-' // '+-'
const next = this.state.input.charCodeAt(this.state.pos + 1); const next = this.input.charCodeAt(this.state.pos + 1);
if (next === code) { if (next === code) {
if ( if (
next === charCodes.dash && next === charCodes.dash &&
!this.inModule && !this.inModule &&
this.state.input.charCodeAt(this.state.pos + 2) === this.input.charCodeAt(this.state.pos + 2) === charCodes.greaterThan &&
charCodes.greaterThan && lineBreak.test(this.input.slice(this.state.lastTokEnd, this.state.pos))
lineBreak.test(
this.state.input.slice(this.state.lastTokEnd, this.state.pos),
)
) { ) {
// A `-->` line comment // A `-->` line comment
this.skipLineComment(3); this.skipLineComment(3);
@ -551,20 +547,16 @@ export default class Tokenizer extends LocationParser {
readToken_lt_gt(code: number): void { readToken_lt_gt(code: number): void {
// '<>' // '<>'
const next = this.state.input.charCodeAt(this.state.pos + 1); const next = this.input.charCodeAt(this.state.pos + 1);
let size = 1; let size = 1;
if (next === code) { if (next === code) {
size = size =
code === charCodes.greaterThan && code === charCodes.greaterThan &&
this.state.input.charCodeAt(this.state.pos + 2) === this.input.charCodeAt(this.state.pos + 2) === charCodes.greaterThan
charCodes.greaterThan
? 3 ? 3
: 2; : 2;
if ( if (this.input.charCodeAt(this.state.pos + size) === charCodes.equalsTo) {
this.state.input.charCodeAt(this.state.pos + size) ===
charCodes.equalsTo
) {
this.finishOp(tt.assign, size + 1); this.finishOp(tt.assign, size + 1);
return; return;
} }
@ -576,8 +568,8 @@ export default class Tokenizer extends LocationParser {
next === charCodes.exclamationMark && next === charCodes.exclamationMark &&
code === charCodes.lessThan && code === charCodes.lessThan &&
!this.inModule && !this.inModule &&
this.state.input.charCodeAt(this.state.pos + 2) === charCodes.dash && this.input.charCodeAt(this.state.pos + 2) === charCodes.dash &&
this.state.input.charCodeAt(this.state.pos + 3) === charCodes.dash this.input.charCodeAt(this.state.pos + 3) === charCodes.dash
) { ) {
// `<!--`, an XML-style comment that should be interpreted as a line comment // `<!--`, an XML-style comment that should be interpreted as a line comment
this.skipLineComment(4); this.skipLineComment(4);
@ -596,11 +588,11 @@ export default class Tokenizer extends LocationParser {
readToken_eq_excl(code: number): void { readToken_eq_excl(code: number): void {
// '=!' // '=!'
const next = this.state.input.charCodeAt(this.state.pos + 1); const next = this.input.charCodeAt(this.state.pos + 1);
if (next === charCodes.equalsTo) { if (next === charCodes.equalsTo) {
this.finishOp( this.finishOp(
tt.equality, tt.equality,
this.state.input.charCodeAt(this.state.pos + 2) === charCodes.equalsTo this.input.charCodeAt(this.state.pos + 2) === charCodes.equalsTo
? 3 ? 3
: 2, : 2,
); );
@ -617,8 +609,8 @@ export default class Tokenizer extends LocationParser {
readToken_question(): void { readToken_question(): void {
// '?' // '?'
const next = this.state.input.charCodeAt(this.state.pos + 1); const next = this.input.charCodeAt(this.state.pos + 1);
const next2 = this.state.input.charCodeAt(this.state.pos + 2); const next2 = this.input.charCodeAt(this.state.pos + 2);
if (next === charCodes.questionMark && !this.state.inType) { if (next === charCodes.questionMark && !this.state.inType) {
if (next2 === charCodes.equalsTo) { if (next2 === charCodes.equalsTo) {
// '??=' // '??='
@ -686,7 +678,7 @@ export default class Tokenizer extends LocationParser {
case charCodes.colon: case charCodes.colon:
if ( if (
this.hasPlugin("functionBind") && this.hasPlugin("functionBind") &&
this.state.input.charCodeAt(this.state.pos + 1) === charCodes.colon this.input.charCodeAt(this.state.pos + 1) === charCodes.colon
) { ) {
this.finishOp(tt.doubleColon, 2); this.finishOp(tt.doubleColon, 2);
} else { } else {
@ -705,7 +697,7 @@ export default class Tokenizer extends LocationParser {
return; return;
case charCodes.digit0: { case charCodes.digit0: {
const next = this.state.input.charCodeAt(this.state.pos + 1); const next = this.input.charCodeAt(this.state.pos + 1);
// '0x', '0X' - hex number // '0x', '0X' - hex number
if (next === charCodes.lowercaseX || next === charCodes.uppercaseX) { if (next === charCodes.lowercaseX || next === charCodes.uppercaseX) {
this.readRadixNumber(16); this.readRadixNumber(16);
@ -811,7 +803,7 @@ export default class Tokenizer extends LocationParser {
} }
finishOp(type: TokenType, size: number): void { finishOp(type: TokenType, size: number): void {
const str = this.state.input.slice(this.state.pos, this.state.pos + size); const str = this.input.slice(this.state.pos, this.state.pos + size);
this.state.pos += size; this.state.pos += size;
this.finishToken(type, str); this.finishToken(type, str);
} }
@ -820,10 +812,10 @@ export default class Tokenizer extends LocationParser {
const start = this.state.pos; const start = this.state.pos;
let escaped, inClass; let escaped, inClass;
for (;;) { for (;;) {
if (this.state.pos >= this.state.length) { if (this.state.pos >= this.length) {
this.raise(start, "Unterminated regular expression"); this.raise(start, "Unterminated regular expression");
} }
const ch = this.state.input.charAt(this.state.pos); const ch = this.input.charAt(this.state.pos);
if (lineBreak.test(ch)) { if (lineBreak.test(ch)) {
this.raise(start, "Unterminated regular expression"); this.raise(start, "Unterminated regular expression");
} }
@ -841,14 +833,14 @@ export default class Tokenizer extends LocationParser {
} }
++this.state.pos; ++this.state.pos;
} }
const content = this.state.input.slice(start, this.state.pos); const content = this.input.slice(start, this.state.pos);
++this.state.pos; ++this.state.pos;
let mods = ""; let mods = "";
while (this.state.pos < this.state.length) { while (this.state.pos < this.length) {
const char = this.state.input[this.state.pos]; const char = this.input[this.state.pos];
const charCode = this.state.input.codePointAt(this.state.pos); const charCode = this.input.codePointAt(this.state.pos);
if (VALID_REGEX_FLAGS.has(char)) { if (VALID_REGEX_FLAGS.has(char)) {
if (mods.indexOf(char) > -1) { if (mods.indexOf(char) > -1) {
@ -895,12 +887,12 @@ export default class Tokenizer extends LocationParser {
let total = 0; let total = 0;
for (let i = 0, e = len == null ? Infinity : len; i < e; ++i) { for (let i = 0, e = len == null ? Infinity : len; i < e; ++i) {
const code = this.state.input.charCodeAt(this.state.pos); const code = this.input.charCodeAt(this.state.pos);
let val; let val;
if (this.hasPlugin("numericSeparator")) { if (this.hasPlugin("numericSeparator")) {
const prev = this.state.input.charCodeAt(this.state.pos - 1); const prev = this.input.charCodeAt(this.state.pos - 1);
const next = this.state.input.charCodeAt(this.state.pos + 1); const next = this.input.charCodeAt(this.state.pos + 1);
if (code === charCodes.underscore) { if (code === charCodes.underscore) {
if (allowedSiblings.indexOf(next) === -1) { if (allowedSiblings.indexOf(next) === -1) {
this.raise(this.state.pos, "Invalid or unexpected token"); this.raise(this.state.pos, "Invalid or unexpected token");
@ -954,22 +946,18 @@ export default class Tokenizer extends LocationParser {
} }
if (this.hasPlugin("bigInt")) { if (this.hasPlugin("bigInt")) {
if ( if (this.input.charCodeAt(this.state.pos) === charCodes.lowercaseN) {
this.state.input.charCodeAt(this.state.pos) === charCodes.lowercaseN
) {
++this.state.pos; ++this.state.pos;
isBigInt = true; isBigInt = true;
} }
} }
if (isIdentifierStart(this.state.input.codePointAt(this.state.pos))) { if (isIdentifierStart(this.input.codePointAt(this.state.pos))) {
this.raise(this.state.pos, "Identifier directly after number"); this.raise(this.state.pos, "Identifier directly after number");
} }
if (isBigInt) { if (isBigInt) {
const str = this.state.input const str = this.input.slice(start, this.state.pos).replace(/[_n]/g, "");
.slice(start, this.state.pos)
.replace(/[_n]/g, "");
this.finishToken(tt.bigint, str); this.finishToken(tt.bigint, str);
return; return;
} }
@ -989,7 +977,7 @@ export default class Tokenizer extends LocationParser {
} }
let octal = let octal =
this.state.pos - start >= 2 && this.state.pos - start >= 2 &&
this.state.input.charCodeAt(start) === charCodes.digit0; this.input.charCodeAt(start) === charCodes.digit0;
if (octal) { if (octal) {
if (this.state.strict) { if (this.state.strict) {
this.raise( this.raise(
@ -997,30 +985,30 @@ export default class Tokenizer extends LocationParser {
"Legacy octal literals are not allowed in strict mode", "Legacy octal literals are not allowed in strict mode",
); );
} }
if (/[89]/.test(this.state.input.slice(start, this.state.pos))) { if (/[89]/.test(this.input.slice(start, this.state.pos))) {
octal = false; octal = false;
} }
} }
let next = this.state.input.charCodeAt(this.state.pos); let next = this.input.charCodeAt(this.state.pos);
if (next === charCodes.dot && !octal) { if (next === charCodes.dot && !octal) {
++this.state.pos; ++this.state.pos;
this.readInt(10); this.readInt(10);
isFloat = true; isFloat = true;
next = this.state.input.charCodeAt(this.state.pos); next = this.input.charCodeAt(this.state.pos);
} }
if ( if (
(next === charCodes.uppercaseE || next === charCodes.lowercaseE) && (next === charCodes.uppercaseE || next === charCodes.lowercaseE) &&
!octal !octal
) { ) {
next = this.state.input.charCodeAt(++this.state.pos); next = this.input.charCodeAt(++this.state.pos);
if (next === charCodes.plusSign || next === charCodes.dash) { if (next === charCodes.plusSign || next === charCodes.dash) {
++this.state.pos; ++this.state.pos;
} }
if (this.readInt(10) === null) this.raise(start, "Invalid number"); if (this.readInt(10) === null) this.raise(start, "Invalid number");
isFloat = true; isFloat = true;
next = this.state.input.charCodeAt(this.state.pos); next = this.input.charCodeAt(this.state.pos);
} }
if (this.hasPlugin("bigInt")) { if (this.hasPlugin("bigInt")) {
@ -1032,14 +1020,12 @@ export default class Tokenizer extends LocationParser {
} }
} }
if (isIdentifierStart(this.state.input.codePointAt(this.state.pos))) { if (isIdentifierStart(this.input.codePointAt(this.state.pos))) {
this.raise(this.state.pos, "Identifier directly after number"); this.raise(this.state.pos, "Identifier directly after number");
} }
// remove "_" for numeric literal separator, and "n" for BigInts // remove "_" for numeric literal separator, and "n" for BigInts
const str = this.state.input const str = this.input.slice(start, this.state.pos).replace(/[_n]/g, "");
.slice(start, this.state.pos)
.replace(/[_n]/g, "");
if (isBigInt) { if (isBigInt) {
this.finishToken(tt.bigint, str); this.finishToken(tt.bigint, str);
@ -1053,13 +1039,13 @@ export default class Tokenizer extends LocationParser {
// Read a string value, interpreting backslash-escapes. // Read a string value, interpreting backslash-escapes.
readCodePoint(throwOnInvalid: boolean): number | null { readCodePoint(throwOnInvalid: boolean): number | null {
const ch = this.state.input.charCodeAt(this.state.pos); const ch = this.input.charCodeAt(this.state.pos);
let code; let code;
if (ch === charCodes.leftCurlyBrace) { if (ch === charCodes.leftCurlyBrace) {
const codePos = ++this.state.pos; const codePos = ++this.state.pos;
code = this.readHexChar( code = this.readHexChar(
this.state.input.indexOf("}", this.state.pos) - this.state.pos, this.input.indexOf("}", this.state.pos) - this.state.pos,
throwOnInvalid, throwOnInvalid,
); );
++this.state.pos; ++this.state.pos;
@ -1084,13 +1070,13 @@ export default class Tokenizer extends LocationParser {
let out = "", let out = "",
chunkStart = ++this.state.pos; chunkStart = ++this.state.pos;
for (;;) { for (;;) {
if (this.state.pos >= this.state.length) { if (this.state.pos >= this.length) {
this.raise(this.state.start, "Unterminated string constant"); this.raise(this.state.start, "Unterminated string constant");
} }
const ch = this.state.input.charCodeAt(this.state.pos); const ch = this.input.charCodeAt(this.state.pos);
if (ch === quote) break; if (ch === quote) break;
if (ch === charCodes.backslash) { if (ch === charCodes.backslash) {
out += this.state.input.slice(chunkStart, this.state.pos); out += this.input.slice(chunkStart, this.state.pos);
// $FlowFixMe // $FlowFixMe
out += this.readEscapedChar(false); out += this.readEscapedChar(false);
chunkStart = this.state.pos; chunkStart = this.state.pos;
@ -1106,7 +1092,7 @@ export default class Tokenizer extends LocationParser {
++this.state.pos; ++this.state.pos;
} }
} }
out += this.state.input.slice(chunkStart, this.state.pos++); out += this.input.slice(chunkStart, this.state.pos++);
this.finishToken(tt.string, out); this.finishToken(tt.string, out);
} }
@ -1117,14 +1103,14 @@ export default class Tokenizer extends LocationParser {
chunkStart = this.state.pos, chunkStart = this.state.pos,
containsInvalid = false; containsInvalid = false;
for (;;) { for (;;) {
if (this.state.pos >= this.state.length) { if (this.state.pos >= this.length) {
this.raise(this.state.start, "Unterminated template"); this.raise(this.state.start, "Unterminated template");
} }
const ch = this.state.input.charCodeAt(this.state.pos); const ch = this.input.charCodeAt(this.state.pos);
if ( if (
ch === charCodes.graveAccent || ch === charCodes.graveAccent ||
(ch === charCodes.dollarSign && (ch === charCodes.dollarSign &&
this.state.input.charCodeAt(this.state.pos + 1) === this.input.charCodeAt(this.state.pos + 1) ===
charCodes.leftCurlyBrace) charCodes.leftCurlyBrace)
) { ) {
if (this.state.pos === this.state.start && this.match(tt.template)) { if (this.state.pos === this.state.start && this.match(tt.template)) {
@ -1138,12 +1124,12 @@ export default class Tokenizer extends LocationParser {
return; return;
} }
} }
out += this.state.input.slice(chunkStart, this.state.pos); out += this.input.slice(chunkStart, this.state.pos);
this.finishToken(tt.template, containsInvalid ? null : out); this.finishToken(tt.template, containsInvalid ? null : out);
return; return;
} }
if (ch === charCodes.backslash) { if (ch === charCodes.backslash) {
out += this.state.input.slice(chunkStart, this.state.pos); out += this.input.slice(chunkStart, this.state.pos);
const escaped = this.readEscapedChar(true); const escaped = this.readEscapedChar(true);
if (escaped === null) { if (escaped === null) {
containsInvalid = true; containsInvalid = true;
@ -1152,13 +1138,11 @@ export default class Tokenizer extends LocationParser {
} }
chunkStart = this.state.pos; chunkStart = this.state.pos;
} else if (isNewLine(ch)) { } else if (isNewLine(ch)) {
out += this.state.input.slice(chunkStart, this.state.pos); out += this.input.slice(chunkStart, this.state.pos);
++this.state.pos; ++this.state.pos;
switch (ch) { switch (ch) {
case charCodes.carriageReturn: case charCodes.carriageReturn:
if ( if (this.input.charCodeAt(this.state.pos) === charCodes.lineFeed) {
this.state.input.charCodeAt(this.state.pos) === charCodes.lineFeed
) {
++this.state.pos; ++this.state.pos;
} }
case charCodes.lineFeed: case charCodes.lineFeed:
@ -1181,7 +1165,7 @@ export default class Tokenizer extends LocationParser {
readEscapedChar(inTemplate: boolean): string | null { readEscapedChar(inTemplate: boolean): string | null {
const throwOnInvalid = !inTemplate; const throwOnInvalid = !inTemplate;
const ch = this.state.input.charCodeAt(++this.state.pos); const ch = this.input.charCodeAt(++this.state.pos);
++this.state.pos; ++this.state.pos;
switch (ch) { switch (ch) {
case charCodes.lowercaseN: case charCodes.lowercaseN:
@ -1205,9 +1189,7 @@ export default class Tokenizer extends LocationParser {
case charCodes.lowercaseF: case charCodes.lowercaseF:
return "\f"; return "\f";
case charCodes.carriageReturn: case charCodes.carriageReturn:
if ( if (this.input.charCodeAt(this.state.pos) === charCodes.lineFeed) {
this.state.input.charCodeAt(this.state.pos) === charCodes.lineFeed
) {
++this.state.pos; ++this.state.pos;
} }
case charCodes.lineFeed: case charCodes.lineFeed:
@ -1220,7 +1202,7 @@ export default class Tokenizer extends LocationParser {
if (ch >= charCodes.digit0 && ch <= charCodes.digit7) { if (ch >= charCodes.digit0 && ch <= charCodes.digit7) {
const codePos = this.state.pos - 1; const codePos = this.state.pos - 1;
// $FlowFixMe // $FlowFixMe
let octalStr = this.state.input let octalStr = this.input
.substr(this.state.pos - 1, 3) .substr(this.state.pos - 1, 3)
.match(/^[0-7]+/)[0]; .match(/^[0-7]+/)[0];
let octal = parseInt(octalStr, 8); let octal = parseInt(octalStr, 8);
@ -1277,8 +1259,8 @@ export default class Tokenizer extends LocationParser {
const start = this.state.pos; const start = this.state.pos;
let chunkStart = this.state.pos; let chunkStart = this.state.pos;
while (this.state.pos < this.state.length) { while (this.state.pos < this.length) {
const ch = this.state.input.codePointAt(this.state.pos); const ch = this.input.codePointAt(this.state.pos);
if (isIdentifierChar(ch)) { if (isIdentifierChar(ch)) {
this.state.pos += ch <= 0xffff ? 1 : 2; this.state.pos += ch <= 0xffff ? 1 : 2;
} else if (this.state.isIterator && ch === charCodes.atSign) { } else if (this.state.isIterator && ch === charCodes.atSign) {
@ -1286,14 +1268,12 @@ export default class Tokenizer extends LocationParser {
} else if (ch === charCodes.backslash) { } else if (ch === charCodes.backslash) {
this.state.containsEsc = true; this.state.containsEsc = true;
word += this.state.input.slice(chunkStart, this.state.pos); word += this.input.slice(chunkStart, this.state.pos);
const escStart = this.state.pos; const escStart = this.state.pos;
const identifierCheck = const identifierCheck =
this.state.pos === start ? isIdentifierStart : isIdentifierChar; this.state.pos === start ? isIdentifierStart : isIdentifierChar;
if ( if (this.input.charCodeAt(++this.state.pos) !== charCodes.lowercaseU) {
this.state.input.charCodeAt(++this.state.pos) !== charCodes.lowercaseU
) {
this.raise( this.raise(
this.state.pos, this.state.pos,
"Expecting Unicode escape sequence \\uXXXX", "Expecting Unicode escape sequence \\uXXXX",
@ -1317,7 +1297,7 @@ export default class Tokenizer extends LocationParser {
break; break;
} }
} }
return word + this.state.input.slice(chunkStart, this.state.pos); return word + this.input.slice(chunkStart, this.state.pos);
} }
isIterator(word: string): boolean { isIterator(word: string): boolean {
@ -1366,7 +1346,7 @@ export default class Tokenizer extends LocationParser {
(prevType === tt.name && this.state.exprAllowed) (prevType === tt.name && this.state.exprAllowed)
) { ) {
return lineBreak.test( return lineBreak.test(
this.state.input.slice(this.state.lastTokEnd, this.state.start), this.input.slice(this.state.lastTokEnd, this.state.start),
); );
} }

8
packages/babel-parser/src/tokenizer/state.js
View File

@ -23,9 +23,6 @@ type TopicContextState = {
export default class State { export default class State {
strict: boolean; strict: boolean;
input: string;
length: number;
curLine: number; curLine: number;
// And, if locations are used, the {line, column} object // And, if locations are used, the {line, column} object
@ -33,13 +30,10 @@ export default class State {
startLoc: Position; startLoc: Position;
endLoc: Position; endLoc: Position;
init(options: Options, input: string): void { init(options: Options): void {
this.strict = this.strict =
options.strictMode === false ? false : options.sourceType === "module"; options.strictMode === false ? false : options.sourceType === "module";
this.input = input;
this.length = input.length;
this.curLine = options.startLine; this.curLine = options.startLine;
this.startLoc = this.endLoc = this.curPosition(); this.startLoc = this.endLoc = this.curPosition();
} }