cerxes/babel
3
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

fix(parser): correctly parse record and tuple tokens (#13418)

Browse Source 
* fix(parser): correctly parse token location for `#{` and `#[`

* fix: `bar` tokens

* fix: don't parse record and tuple pipe closing tokens as operators
This commit is contained in:
Federico Ciardi 2021-06-09 02:58:48 +02:00 committed by GitHub
parent 56db172b0e
commit a64d08c101
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 301 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
packages/babel-parser/src/tokenizer/index.js
View File

@ -486,6 +486,7 @@ export default class Tokenizer extends ParserErrors {
);
}
this.state.pos += 2;
if (next === charCodes.leftCurlyBrace) {
// #{
this.finishToken(tt.braceHashL);
@ -493,7 +494,6 @@ export default class Tokenizer extends ParserErrors {
// #[
this.finishToken(tt.bracketHashL);
}
this.state.pos += 2;
} else if (isIdentifierStart(next)) {
++this.state.pos;
this.finishToken(tt.privateName, this.readWord1(next));
@ -615,8 +615,8 @@ export default class Tokenizer extends ParserErrors {
Errors.RecordExpressionBarIncorrectEndSyntaxType,
);
}
this.finishOp(tt.braceBarR, 2);
this.state.pos += 2;
this.finishToken(tt.braceBarR);
return;
}
@ -631,8 +631,8 @@ export default class Tokenizer extends ParserErrors {
Errors.TupleExpressionBarIncorrectEndSyntaxType,
);
}
this.finishOp(tt.bracketBarR, 2);
this.state.pos += 2;
this.finishToken(tt.bracketBarR);
return;
}
}
@ -812,8 +812,8 @@ export default class Tokenizer extends ParserErrors {
}
// [|
this.finishToken(tt.bracketBarL);
this.state.pos += 2;
this.finishToken(tt.bracketBarL);
} else {
++this.state.pos;
this.finishToken(tt.bracketL);
@ -836,8 +836,8 @@ export default class Tokenizer extends ParserErrors {
}
// {|
this.finishToken(tt.braceBarL);
this.state.pos += 2;
this.finishToken(tt.braceBarL);
} else {
++this.state.pos;
this.finishToken(tt.braceL);

1
packages/babel-parser/test/fixtures/tokens/record-and-tuple/brace-bar/input.js vendored Normal file
View File

@ -0,0 +1 @@
{||}

4
packages/babel-parser/test/fixtures/tokens/record-and-tuple/brace-bar/options.json vendored Normal file
View File

@ -0,0 +1,4 @@
{
"plugins": [["recordAndTuple", { "syntaxType": "bar" }]],
"tokens": true
}

69
packages/babel-parser/test/fixtures/tokens/record-and-tuple/brace-bar/output.json vendored Normal file
View File

@ -0,0 +1,69 @@
{
"type": "File",
"start":0,"end":4,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":4}},
"program": {
"type": "Program",
"start":0,"end":4,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":4}},
"sourceType": "script",
"interpreter": null,
"body": [
{
"type": "ExpressionStatement",
"start":0,"end":4,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":4}},
"expression": {
"type": "RecordExpression",
"start":0,"end":4,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":4}},
"properties": []
}
}
],
"directives": []
},
"tokens": [
{
"type": {
"label": "{|",
"beforeExpr": true,
"startsExpr": true,
"rightAssociative": false,
"isLoop": false,
"isAssign": false,
"prefix": false,
"postfix": false,
"binop": null,
"updateContext": null
},
"start":0,"end":2,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":2}}
},
{
"type": {
"label": "|}",
"beforeExpr": false,
"startsExpr": false,
"rightAssociative": false,
"isLoop": false,
"isAssign": false,
"prefix": false,
"postfix": false,
"binop": null,
"updateContext": null
},
"start":2,"end":4,"loc":{"start":{"line":1,"column":2},"end":{"line":1,"column":4}}
},
{
"type": {
"label": "eof",
"beforeExpr": false,
"startsExpr": false,
"rightAssociative": false,
"isLoop": false,
"isAssign": false,
"prefix": false,
"postfix": false,
"binop": null,
"updateContext": null
},
"start":4,"end":4,"loc":{"start":{"line":1,"column":4},"end":{"line":1,"column":4}}
}
]
}

1
packages/babel-parser/test/fixtures/tokens/record-and-tuple/brace-hash/input.js vendored Normal file
View File

@ -0,0 +1 @@
#{}

4
packages/babel-parser/test/fixtures/tokens/record-and-tuple/brace-hash/options.json vendored Normal file
View File

@ -0,0 +1,4 @@
{
"plugins": [[ "recordAndTuple", { "syntaxType": "hash" }]],
"tokens": true
}

67
packages/babel-parser/test/fixtures/tokens/record-and-tuple/brace-hash/output.json vendored Normal file
View File

@ -0,0 +1,67 @@
{
"type": "File",
"start":0,"end":3,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":3}},
"program": {
"type": "Program",
"start":0,"end":3,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":3}},
"sourceType": "script",
"interpreter": null,
"body": [
{
"type": "ExpressionStatement",
"start":0,"end":3,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":3}},
"expression": {
"type": "RecordExpression",
"start":0,"end":3,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":3}},
"properties": []
}
}
],
"directives": []
},
"tokens": [
{
"type": {
"label": "#{",
"beforeExpr": true,
"startsExpr": true,
"rightAssociative": false,
"isLoop": false,
"isAssign": false,
"prefix": false,
"postfix": false,
"binop": null
},
"start":0,"end":2,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":2}}
},
{
"type": {
"label": "}",
"beforeExpr": false,
"startsExpr": false,
"rightAssociative": false,
"isLoop": false,
"isAssign": false,
"prefix": false,
"postfix": false,
"binop": null
},
"start":2,"end":3,"loc":{"start":{"line":1,"column":2},"end":{"line":1,"column":3}}
},
{
"type": {
"label": "eof",
"beforeExpr": false,
"startsExpr": false,
"rightAssociative": false,
"isLoop": false,
"isAssign": false,
"prefix": false,
"postfix": false,
"binop": null,
"updateContext": null
},
"start":3,"end":3,"loc":{"start":{"line":1,"column":3},"end":{"line":1,"column":3}}
}
]
}

1
packages/babel-parser/test/fixtures/tokens/record-and-tuple/bracket-bar/input.js vendored Normal file
View File

@ -0,0 +1 @@
[||]

4
packages/babel-parser/test/fixtures/tokens/record-and-tuple/bracket-bar/options.json vendored Normal file
View File

@ -0,0 +1,4 @@
{
"plugins": [["recordAndTuple", { "syntaxType": "bar" }]],
"tokens": true
}

69
packages/babel-parser/test/fixtures/tokens/record-and-tuple/bracket-bar/output.json vendored Normal file
View File

@ -0,0 +1,69 @@
{
"type": "File",
"start":0,"end":4,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":4}},
"program": {
"type": "Program",
"start":0,"end":4,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":4}},
"sourceType": "script",
"interpreter": null,
"body": [
{
"type": "ExpressionStatement",
"start":0,"end":4,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":4}},
"expression": {
"type": "TupleExpression",
"start":0,"end":4,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":4}},
"elements": []
}
}
],
"directives": []
},
"tokens": [
{
"type": {
"label": "[|",
"beforeExpr": true,
"startsExpr": true,
"rightAssociative": false,
"isLoop": false,
"isAssign": false,
"prefix": false,
"postfix": false,
"binop": null,
"updateContext": null
},
"start":0,"end":2,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":2}}
},
{
"type": {
"label": "|]",
"beforeExpr": false,
"startsExpr": false,
"rightAssociative": false,
"isLoop": false,
"isAssign": false,
"prefix": false,
"postfix": false,
"binop": null,
"updateContext": null
},
"start":2,"end":4,"loc":{"start":{"line":1,"column":2},"end":{"line":1,"column":4}}
},
{
"type": {
"label": "eof",
"beforeExpr": false,
"startsExpr": false,
"rightAssociative": false,
"isLoop": false,
"isAssign": false,
"prefix": false,
"postfix": false,
"binop": null,
"updateContext": null
},
"start":4,"end":4,"loc":{"start":{"line":1,"column":4},"end":{"line":1,"column":4}}
}
]
}

1
packages/babel-parser/test/fixtures/tokens/record-and-tuple/bracket-hash/input.js vendored Normal file
View File

@ -0,0 +1 @@
#[]

4
packages/babel-parser/test/fixtures/tokens/record-and-tuple/bracket-hash/options.json vendored Normal file
View File

@ -0,0 +1,4 @@
{
"plugins": [[ "recordAndTuple", { "syntaxType": "hash" }]],
"tokens": true
}

69
packages/babel-parser/test/fixtures/tokens/record-and-tuple/bracket-hash/output.json vendored Normal file
View File

@ -0,0 +1,69 @@
{
"type": "File",
"start":0,"end":3,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":3}},
"program": {
"type": "Program",
"start":0,"end":3,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":3}},
"sourceType": "script",
"interpreter": null,
"body": [
{
"type": "ExpressionStatement",
"start":0,"end":3,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":3}},
"expression": {
"type": "TupleExpression",
"start":0,"end":3,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":3}},
"elements": []
}
}
],
"directives": []
},
"tokens": [
{
"type": {
"label": "#[",
"beforeExpr": true,
"startsExpr": true,
"rightAssociative": false,
"isLoop": false,
"isAssign": false,
"prefix": false,
"postfix": false,
"binop": null,
"updateContext": null
},
"start":0,"end":2,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":2}}
},
{
"type": {
"label": "]",
"beforeExpr": false,
"startsExpr": false,
"rightAssociative": false,
"isLoop": false,
"isAssign": false,
"prefix": false,
"postfix": false,
"binop": null,
"updateContext": null
},
"start":2,"end":3,"loc":{"start":{"line":1,"column":2},"end":{"line":1,"column":3}}
},
{
"type": {
"label": "eof",
"beforeExpr": false,
"startsExpr": false,
"rightAssociative": false,
"isLoop": false,
"isAssign": false,
"prefix": false,
"postfix": false,
"binop": null,
"updateContext": null
},
"start":3,"end":3,"loc":{"start":{"line":1,"column":3},"end":{"line":1,"column":3}}
}
]
}