Made tokenize() compliant with ES6 iterables for easier processing.

README.md

@@ -162,6 +162,17 @@ token, and returns a `{start, end, type, value}` object (with added
 `loc` property when the `locations` option is enabled and `range`
 property when the `ranges` option is enabled).
 
+In ES6 environment, returned result can be used as any other protocol-compliant iterable:
+
+```javascript
+for (let token of acorn.tokenize(str)) {
+  // iterate over the tokens
+}
+
+// transform code to array of tokens:
+var tokens = [...acorn.tokenize(str)];
+```
+
 **tokTypes** holds an object mapping names to the token type objects
 that end up in the `type` properties of tokens.
 

acorn.js

@@ -243,6 +243,19 @@
       skipSpace();
     };
     getToken.current = function() { return new Token(); };
+    if (typeof Symbol !== 'undefined') {
+      getToken[Symbol.iterator] = function () {
+        return {
+          next: function () {
+            var token = getToken();
+            return {
+              done: token.type === _eof,
+              value: token
+            };
+          }
+        };
+      };
+    }
     getToken.options = options;
     return getToken;
   };