cerxes/babel
3
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Refactor private name tokenizing (#13256)

Browse Source 
* add benchmark

* refactor: create tt.privateName token for private names

* add backward compat privateName = hash + name to Babel 7

* perf: get private name SV from token value

* chore: tweak benchmark file

* chore: update test fixtures

* convert tt.privateName to PrivateIdentifier

* perf: avoid most isPrivateName call

* Update packages/babel-parser/src/parser/expression.js

Co-authored-by: Justin Ridgewell <justin@ridgewell.name>

* perf: use inlinable codePointAtPos

* make prettier happy

Co-authored-by: Justin Ridgewell <justin@ridgewell.name>
This commit is contained in:
Huáng Jùnliàng
2021-05-06 09:46:09 -04:00
committed by GitHub
parent 278193b6f7
commit a387973821
24 changed files with 551 additions and 210 deletions
Download Patch File Download Diff File Expand all files Collapse all files

97
packages/babel-parser/src/parser/expression.js
View File

@@ -29,7 +29,8 @@ import {
isStrictBindReservedWord,
isIdentifierStart,
} from "../util/identifier";
import type { Pos, Position } from "../util/location";
import type { Pos } from "../util/location";
import { Position } from "../util/location";
import * as charCodes from "charcodes";
import {
BIND_OUTSIDE,
@@ -705,18 +706,19 @@ export default class ExpressionParser extends LValParser {
const computed = this.eat(tt.bracketL);
node.object = base;
node.computed = computed;
const privateName =
!computed && this.match(tt.privateName) && this.state.value;
const property = computed
? this.parseExpression()
: this.parseMaybePrivateName(true);
: privateName
? this.parsePrivateName()
: this.parseIdentifier(true);
if (this.isPrivateName(property)) {
if (privateName !== false) {
if (node.object.type === "Super") {
this.raise(startPos, Errors.SuperPrivateField);
}
this.classScope.usePrivateName(
this.getPrivateNameSV(property),
property.start,
);
this.classScope.usePrivateName(privateName, property.start);
}
node.property = property;
@@ -1160,6 +1162,23 @@ export default class ExpressionParser extends LValParser {
}
}
case tt.privateName: {
// https://tc39.es/proposal-private-fields-in-in
// RelationalExpression [In, Yield, Await]
// [+In] PrivateIdentifier in ShiftExpression[?Yield, ?Await]
const start = this.state.start;
const value = this.state.value;
node = this.parsePrivateName();
if (this.match(tt._in)) {
this.expectPlugin("privateIn");
this.classScope.usePrivateName(value, node.start);
} else if (this.hasPlugin("privateIn")) {
this.raise(this.state.start, Errors.PrivateInExpectedIn, value);
} else {
throw this.unexpected(start);
}
return node;
}
case tt.hash: {
if (this.state.inPipeline) {
node = this.startNode();
@@ -1179,32 +1198,6 @@ export default class ExpressionParser extends LValParser {
this.registerTopicReference();
return this.finishNode(node, "PipelinePrimaryTopicReference");
}
// https://tc39.es/proposal-private-fields-in-in
// RelationalExpression [In, Yield, Await]
// [+In] PrivateIdentifier in ShiftExpression[?Yield, ?Await]
const nextCh = this.input.codePointAt(this.state.end);
if (isIdentifierStart(nextCh) || nextCh === charCodes.backslash) {
const start = this.state.start;
// $FlowIgnore It'll either parse a PrivateName or throw.
node = (this.parseMaybePrivateName(true): N.PrivateName);
if (this.match(tt._in)) {
this.expectPlugin("privateIn");
this.classScope.usePrivateName(
this.getPrivateNameSV(node),
node.start,
);
} else if (this.hasPlugin("privateIn")) {
this.raise(
this.state.start,
Errors.PrivateInExpectedIn,
this.getPrivateNameSV(node),
);
} else {
throw this.unexpected(start);
}
return node;
}
}
// fall through
case tt.relational: {
@@ -1305,22 +1298,35 @@ export default class ExpressionParser extends LValParser {
parseMaybePrivateName(
isPrivateNameAllowed: boolean,
): N.PrivateName | N.Identifier {
const isPrivate = this.match(tt.hash);
const isPrivate = this.match(tt.privateName);
if (isPrivate) {
if (!isPrivateNameAllowed) {
this.raise(this.state.pos, Errors.UnexpectedPrivateField);
this.raise(this.state.start + 1, Errors.UnexpectedPrivateField);
}
const node = this.startNode();
this.next();
this.assertNoSpace("Unexpected space between # and identifier");
node.id = this.parseIdentifier(true);
return this.finishNode(node, "PrivateName");
return this.parsePrivateName();
} else {
return this.parseIdentifier(true);
}
}
parsePrivateName(): N.PrivateName {
const node = this.startNode();
const id = this.startNodeAt(
this.state.start + 1,
// The position is hardcoded because we merge `#` and name into a single
// tt.privateName token
new Position(
this.state.curLine,
this.state.start + 1 - this.state.lineStart,
),
);
const name = this.state.value;
this.next(); // eat #name;
node.id = this.createIdentifier(id, name);
return this.finishNode(node, "PrivateName");
}
parseFunctionOrFunctionSent(): N.FunctionExpression | N.MetaProperty {
const node = this.startNode();
@@ -1976,15 +1982,16 @@ export default class ExpressionParser extends LValParser {
const oldInPropertyName = this.state.inPropertyName;
this.state.inPropertyName = true;
// We check if it's valid for it to be a private name when we push it.
const type = this.state.type;
(prop: $FlowFixMe).key =
this.match(tt.num) ||
this.match(tt.string) ||
this.match(tt.bigint) ||
this.match(tt.decimal)
type === tt.num ||
type === tt.string ||
type === tt.bigint ||
type === tt.decimal
? this.parseExprAtom()
: this.parseMaybePrivateName(isPrivateNameAllowed);
if (!this.isPrivateName(prop.key)) {
if (type !== tt.privateName) {
// ClassPrivateProperty is never computed, so we don't assign in that case.
prop.computed = false;
}

76
packages/babel-parser/src/parser/statement.js
View File

@@ -36,6 +36,8 @@ import {
newParameterDeclarationScope,
} from "../util/expression-scope";
import type { SourceType } from "../options";
import { Token } from "../tokenizer";
import { Position } from "../util/location";
const loopLabel = { kind: "loop" },
switchLabel = { kind: "switch" };
@@ -47,6 +49,48 @@ const FUNC_NO_FLAGS = 0b000,
const loneSurrogate = /[\uD800-\uDFFF]/u;
/**
* Convert tt.privateName to tt.hash + tt.name for backward Babel 7 compat.
* For performance reasons this routine mutates `tokens`, it is okay
* here since we execute `parseTopLevel` once for every file.
* @param {*} tokens
* @returns
*/
function babel7CompatTokens(tokens) {
if (!process.env.BABEL_8_BREAKING) {
for (let i = 0; i < tokens.length; i++) {
const token = tokens[i];
if (token.type === tt.privateName) {
const { loc, start, value, end } = token;
const hashEndPos = start + 1;
const hashEndLoc = new Position(loc.start.line, loc.start.column + 1);
tokens.splice(
i,
1,
// $FlowIgnore: hacky way to create token
new Token({
type: tt.hash,
value: "#",
start: start,
end: hashEndPos,
startLoc: loc.start,
endLoc: hashEndLoc,
}),
// $FlowIgnore: hacky way to create token
new Token({
type: tt.name,
value: value,
start: hashEndPos,
end: end,
startLoc: hashEndLoc,
endLoc: loc.end,
}),
);
}
}
}
return tokens;
}
export default class StatementParser extends ExpressionParser {
// ### Statement parsing
@@ -59,7 +103,7 @@ export default class StatementParser extends ExpressionParser {
file.program = this.parseProgram(program);
file.comments = this.state.comments;
if (this.options.tokens) file.tokens = this.tokens;
if (this.options.tokens) file.tokens = babel7CompatTokens(this.tokens);
return this.finishNode(file, "File");
}
@@ -1366,9 +1410,10 @@ export default class StatementParser extends ExpressionParser {
if (this.eat(tt.star)) {
// a generator
method.kind = "method";
const isPrivateName = this.match(tt.privateName);
this.parseClassElementName(method);
if (this.isPrivateName(method.key)) {
if (isPrivateName) {
// Private generator method
this.pushClassPrivateMethod(classBody, privateMethod, true, false);
return;
@@ -1391,8 +1436,8 @@ export default class StatementParser extends ExpressionParser {
}
const containsEsc = this.state.containsEsc;
const isPrivate = this.match(tt.privateName);
const key = this.parseClassElementName(member);
const isPrivate = this.isPrivateName(key);
// Check the key is not a computed expression or string literal.
const isSimple = key.type === "Identifier";
const maybeQuestionTokenStart = this.state.start;
@@ -1453,10 +1498,11 @@ export default class StatementParser extends ExpressionParser {
method.kind = "method";
// The so-called parsed name would have been "async": get the real name.
const isPrivate = this.match(tt.privateName);
this.parseClassElementName(method);
this.parsePostMemberNameModifiers(publicMember);
if (this.isPrivateName(method.key)) {
if (isPrivate) {
// private async method
this.pushClassPrivateMethod(
classBody,
@@ -1488,9 +1534,10 @@ export default class StatementParser extends ExpressionParser {
// a getter or setter
method.kind = key.name;
// The so-called parsed name would have been "get/set": get the real name.
const isPrivate = this.match(tt.privateName);
this.parseClassElementName(publicMethod);
if (this.isPrivateName(method.key)) {
if (isPrivate) {
// private getter/setter
this.pushClassPrivateMethod(classBody, privateMethod, false, false);
} else {
@@ -1522,25 +1569,20 @@ export default class StatementParser extends ExpressionParser {
// https://tc39.es/proposal-class-fields/#prod-ClassElementName
parseClassElementName(member: N.ClassMember): N.Expression | N.Identifier {
const key = this.parsePropertyName(member, /* isPrivateNameAllowed */ true);
const { type, value, start } = this.state;
if (
!member.computed &&
(type === tt.name || type === tt.string) &&
member.static &&
((key: $FlowSubtype<N.Identifier>).name === "prototype" ||
(key: $FlowSubtype<N.StringLiteral>).value === "prototype")
value === "prototype"
) {
this.raise(key.start, Errors.StaticPrototype);
this.raise(start, Errors.StaticPrototype);
}
if (
this.isPrivateName(key) &&
this.getPrivateNameSV(key) === "constructor"
) {
this.raise(key.start, Errors.ConstructorClassPrivateField);
if (type === tt.privateName && value === "constructor") {
this.raise(start, Errors.ConstructorClassPrivateField);
}
return key;
return this.parsePropertyName(member, /* isPrivateNameAllowed */ true);
}
parseClassStaticBlock(

2
packages/babel-parser/src/plugins/typescript/index.js
View File

@@ -210,7 +210,7 @@ export default (superClass: Class<Parser>): Class<Parser> =>
this.match(tt.braceL) ||
this.match(tt.star) ||
this.match(tt.ellipsis) ||
this.match(tt.hash) ||
this.match(tt.privateName) ||
this.isLiteralPropertyName()) &&
!this.hasPrecedingLineBreak()
);

30
packages/babel-parser/src/tokenizer/index.js
View File

@@ -205,6 +205,21 @@ export default class Tokenizer extends ParserErrors {
return this.input.charCodeAt(this.nextTokenStart());
}
codePointAtPos(pos: number): number {
// The implementation is based on
// https://source.chromium.org/chromium/chromium/src/+/master:v8/src/builtins/builtins-string-gen.cc;l=1455;drc=221e331b49dfefadbc6fa40b0c68e6f97606d0b3;bpv=0;bpt=1
// We reimplement `codePointAt` because `codePointAt` is a V8 builtin which is not inlined by TurboFan (as of M91)
// since `input` is mostly ASCII, an inlined `charCodeAt` wins here
let cp = this.input.charCodeAt(pos);
if ((cp & 0xfc00) === 0xd800 && ++pos < this.input.length) {
const trail = this.input.charCodeAt(pos);
if ((trail & 0xfc00) === 0xdc00) {
cp = 0x10000 + ((cp & 0x3ff) << 10) + (trail & 0x3ff);
}
}
return cp;
}
// Toggle strict mode. Re-reads the next number or string to please
// pedantic tests (`"use strict"; 010;` should fail).
@@ -244,7 +259,7 @@ export default class Tokenizer extends ParserErrors {
if (override) {
override(this);
} else {
this.getTokenFromCode(this.input.codePointAt(this.state.pos));
this.getTokenFromCode(this.codePointAtPos(this.state.pos));
}
}
@@ -407,7 +422,7 @@ export default class Tokenizer extends ParserErrors {
}
const nextPos = this.state.pos + 1;
const next = this.input.charCodeAt(nextPos);
const next = this.codePointAtPos(nextPos);
if (next >= charCodes.digit0 && next <= charCodes.digit9) {
throw this.raise(this.state.pos, Errors.UnexpectedDigitAfterHash);
}
@@ -438,6 +453,9 @@ export default class Tokenizer extends ParserErrors {
this.finishToken(tt.bracketHashL);
}
this.state.pos += 2;
} else if (isIdentifierStart(next) || next === charCodes.backslash) {
++this.state.pos;
this.finishToken(tt.privateName, this.readWord1());
} else {
this.finishOp(tt.hash, 1);
}
@@ -952,7 +970,7 @@ export default class Tokenizer extends ParserErrors {
while (this.state.pos < this.length) {
const char = this.input[this.state.pos];
const charCode = this.input.codePointAt(this.state.pos);
const charCode = this.codePointAtPos(this.state.pos);
if (VALID_REGEX_FLAGS.has(char)) {
if (mods.indexOf(char) > -1) {
@@ -1090,7 +1108,7 @@ export default class Tokenizer extends ParserErrors {
throw this.raise(start, Errors.InvalidDecimal);
}
if (isIdentifierStart(this.input.codePointAt(this.state.pos))) {
if (isIdentifierStart(this.codePointAtPos(this.state.pos))) {
throw this.raise(this.state.pos, Errors.NumberIdentifier);
}
@@ -1176,7 +1194,7 @@ export default class Tokenizer extends ParserErrors {
isDecimal = true;
}
if (isIdentifierStart(this.input.codePointAt(this.state.pos))) {
if (isIdentifierStart(this.codePointAtPos(this.state.pos))) {
throw this.raise(this.state.pos, Errors.NumberIdentifier);
}
@@ -1447,7 +1465,7 @@ export default class Tokenizer extends ParserErrors {
let chunkStart = this.state.pos;
while (this.state.pos < this.length) {
const ch = this.input.codePointAt(this.state.pos);
const ch = this.codePointAtPos(this.state.pos);
if (isIdentifierChar(ch)) {
this.state.pos += ch <= 0xffff ? 1 : 2;
} else if (this.state.isIterator && ch === charCodes.atSign) {

1
packages/babel-parser/src/tokenizer/types.js
View File

@@ -90,6 +90,7 @@ export const types: { [name: string]: TokenType } = {
regexp: new TokenType("regexp", { startsExpr }),
string: new TokenType("string", { startsExpr }),
name: new TokenType("name", { startsExpr }),
privateName: new TokenType("#name", { startsExpr }),
eof: new TokenType("eof"),
// Punctuation token types.