diff --git a/README.md b/README.md index 06fb2c67db..4a7420d451 100644 --- a/README.md +++ b/README.md @@ -73,6 +73,9 @@ object referring to that same position. one-based line and zero-based column numbers in `{line, column}` form. Default is `false`. +- **onToken**: If a function is passed for this option, each found + token will be passed in format that `tokenize()` method provides. + - **onComment**: If a function is passed for this option, whenever a comment is encountered the function will be called with the following parameters: diff --git a/acorn.js b/acorn.js index e506ad4fe1..3a93936fc7 100644 --- a/acorn.js +++ b/acorn.js @@ -74,6 +74,12 @@ // line being 1-based and column 0-based) will be attached to the // nodes. locations: false, + // A function can be passed as `onToken` option, which will + // cause Acorn to call that function with object in the same + // format as is used in tokenize(). Note that you are not + // allowed to call the parser from the callback—that will + // corrupt its internal state. + onToken: null, // A function can be passed as `onComment` option, which will // cause Acorn to call that function with `(block, text, start, // end)` parameters whenever a comment is skipped. `block` is a @@ -526,6 +532,16 @@ if (type !== _bquote || inTemplate) skipSpace(); tokVal = val; tokRegexpAllowed = type.beforeExpr; + if (options.onToken) { + options.onToken({ + start: tokStart, + end: tokEnd, + startLoc: tokStartLoc, + endLoc: tokEndLoc, + type: tokType, + value: tokVal + }); + } } function skipBlockComment() { diff --git a/test/tests.js b/test/tests.js index 2a0cda4b4c..ee9dd107ee 100644 --- a/test/tests.js +++ b/test/tests.js @@ -5,6 +5,7 @@ if (typeof exports != "undefined") { var test = require("./driver.js").test; var testFail = require("./driver.js").testFail; var testAssert = require("./driver.js").testAssert; + var acorn = require(".."); } test("this\n", { @@ -28650,3 +28651,104 @@ testFail("for(const x = 0;;);", "Unexpected token (1:4)", {ecmaVersion: 6}); if (comments != 1) return "Comment after strict counted twice."; }, {onComment: function() {++comments;}}); })(); + +(function() { + var tokTypes = acorn.tokTypes; + + var actualTokens = [], + expectedTokens = [ + { + start: 0, + end: 3, + startLoc: {line: 1, column: 0}, + endLoc: {line: 1, column: 3}, + type: tokTypes._var, + value: "var" + }, + { + start: 4, + end: 5, + startLoc: {line: 1, column: 4}, + endLoc: {line: 1, column: 5}, + type: tokTypes.name, + value: "x" + }, + { + start: 6, + end: 7, + startLoc: {line: 1, column: 6}, + endLoc: {line: 1, column: 7}, + type: tokTypes.eq, + value: "=" + }, + { + start: 8, + end: 9, + startLoc: {line: 1, column: 8}, + endLoc: {line: 1, column: 9}, + type: tokTypes.parenL, + value: undefined + }, + { + start: 9, + end: 10, + startLoc: {line: 1, column: 9}, + endLoc: {line: 1, column: 10}, + type: tokTypes.num, + value: 1 + }, + { + start: 11, + end: 12, + startLoc: {line: 1, column: 11}, + endLoc: {line: 1, column: 12}, + type: { + binop: 9, + prefix: true, + beforeExpr: true + }, + value: "+" + }, + { + start: 13, + end: 14, + startLoc: {line: 1, column: 13}, + endLoc: {line: 1, column: 14}, + type: tokTypes.num, + value: 2 + }, + { + start: 14, + end: 15, + startLoc: {line: 1, column: 14}, + endLoc: {line: 1, column: 15}, + type: tokTypes.parenR, + value: undefined + }, + { + start: 15, + end: 15, + startLoc: {line: 1, column: 15}, + endLoc: {line: 1, column: 15}, + type: tokTypes.eof, + value: undefined + } + ]; + testAssert('var x = (1 + 2)', function assert(ast) { + if (actualTokens.length !== expectedTokens.length) { + return JSON.stringify(actualTokens) + " !== " + JSON.stringify(expectedTokens); + } else { + for (var i=0, n=actualTokens.length; i < n; i++) { + var actualToken = JSON.stringify(actualTokens[i]); + var expectedToken = JSON.stringify(expectedTokens[i]); + if (actualToken !== expectedToken) + return actualToken + ' !== ' + expectedToken; + } + } + }, { + locations: true, + onToken: function(token) { + actualTokens.push(token); + } + }); +})(); \ No newline at end of file