Added onToken support.

README.md

@@ -73,6 +73,9 @@ object referring to that same position.
   one-based line and zero-based column numbers in `{line, column}`
   form. Default is `false`.
 
+- **onToken**: If a function is passed for this option, each found
+  token will be passed in format that `tokenize()` method provides.
+
 - **onComment**: If a function is passed for this option, whenever a
   comment is encountered the function will be called with the
   following parameters:

acorn.js

@@ -74,6 +74,12 @@
     // line being 1-based and column 0-based) will be attached to the
     // nodes.
     locations: false,
+    // A function can be passed as `onToken` option, which will
+    // cause Acorn to call that function with object in the same
+    // format as is used in tokenize(). Note that you are not
+    // allowed to call the parser from the callback—that will
+    // corrupt its internal state.
+    onToken: null,
     // A function can be passed as `onComment` option, which will
     // cause Acorn to call that function with `(block, text, start,
     // end)` parameters whenever a comment is skipped. `block` is a
@@ -526,6 +532,16 @@
     if (type !== _bquote || inTemplate) skipSpace();
     tokVal = val;
     tokRegexpAllowed = type.beforeExpr;
+    if (options.onToken) {
+      options.onToken({
+        start: tokStart,
+        end: tokEnd,
+        startLoc: tokStartLoc,
+        endLoc: tokEndLoc,
+        type: tokType,
+        value: tokVal
+      });
+    }
   }
 
   function skipBlockComment() {

test/tests.js

@@ -5,6 +5,7 @@ if (typeof exports != "undefined") {
   var test = require("./driver.js").test;
   var testFail = require("./driver.js").testFail;
   var testAssert = require("./driver.js").testAssert;
+  var acorn = require("..");
 }
 
 test("this\n", {
@@ -28650,3 +28651,104 @@ testFail("for(const x = 0;;);", "Unexpected token (1:4)", {ecmaVersion: 6});
     if (comments != 1) return "Comment after strict counted twice.";
   }, {onComment: function() {++comments;}});
 })();
+
+(function() {
+  var tokTypes = acorn.tokTypes;
+
+  var actualTokens = [],
+      expectedTokens = [
+        {
+          start: 0,
+          end: 3,
+          startLoc: {line: 1, column: 0},
+          endLoc: {line: 1, column: 3},
+          type: tokTypes._var,
+          value: "var"
+        },
+        {
+          start: 4,
+          end: 5,
+          startLoc: {line: 1, column: 4},
+          endLoc: {line: 1, column: 5},
+          type: tokTypes.name,
+          value: "x"
+        },
+        {
+          start: 6,
+          end: 7,
+          startLoc: {line: 1, column: 6},
+          endLoc: {line: 1, column: 7},
+          type: tokTypes.eq,
+          value: "="
+        },
+        {
+          start: 8,
+          end: 9,
+          startLoc: {line: 1, column: 8},
+          endLoc: {line: 1, column: 9},
+          type: tokTypes.parenL,
+          value: undefined
+        },
+        {
+          start: 9,
+          end: 10,
+          startLoc: {line: 1, column: 9},
+          endLoc: {line: 1, column: 10},
+          type: tokTypes.num,
+          value: 1
+        },
+        {
+          start: 11,
+          end: 12,
+          startLoc: {line: 1, column: 11},
+          endLoc: {line: 1, column: 12},
+          type: {
+            binop: 9,
+            prefix: true,
+            beforeExpr: true
+          },
+          value: "+"
+        },
+        {
+          start: 13,
+          end: 14,
+          startLoc: {line: 1, column: 13},
+          endLoc: {line: 1, column: 14},
+          type: tokTypes.num,
+          value: 2
+        },
+        {
+          start: 14,
+          end: 15,
+          startLoc: {line: 1, column: 14},
+          endLoc: {line: 1, column: 15},
+          type: tokTypes.parenR,
+          value: undefined
+        },
+        {
+          start: 15,
+          end: 15,
+          startLoc: {line: 1, column: 15},
+          endLoc: {line: 1, column: 15},
+          type: tokTypes.eof,
+          value: undefined
+        }
+      ];
+  testAssert('var x = (1 + 2)', function assert(ast) {
+    if (actualTokens.length !== expectedTokens.length) {
+      return JSON.stringify(actualTokens) + " !== " + JSON.stringify(expectedTokens);
+    } else {
+      for (var i=0, n=actualTokens.length; i < n; i++) {
+        var actualToken = JSON.stringify(actualTokens[i]);
+        var expectedToken = JSON.stringify(expectedTokens[i]);
+        if (actualToken !== expectedToken)
+          return actualToken + ' !== ' + expectedToken;
+      }
+    }
+  }, {
+    locations: true,
+    onToken: function(token) {
+      actualTokens.push(token);
+    }
+  });
+})();