diff --git a/src/tokenizer/index.js b/src/tokenizer/index.js index c2248cb4fb..c8b06ef0d6 100644 --- a/src/tokenizer/index.js +++ b/src/tokenizer/index.js @@ -56,7 +56,9 @@ export default class Tokenizer { // Move to the next token next() { - this.state.tokens.push(new Token(this.state)); + if (!this.isLookahead) { + this.state.tokens.push(new Token(this.state)); + } this.state.lastTokEnd = this.state.end; this.state.lastTokStart = this.state.start; @@ -102,9 +104,13 @@ export default class Tokenizer { lookahead() { var old = this.state; - this.state = old.clone(); + this.state = old.clone(true); + + this.isLookahead = true; this.next(); - var curr = this.state.clone(); + this.isLookahead = false; + + var curr = this.state.clone(true); this.state = old; return curr; } @@ -172,8 +178,10 @@ export default class Tokenizer { range: [start, end] }; - this.state.tokens.push(comment); - this.state.comments.push(comment); + if (!this.isLookahead) { + this.state.tokens.push(comment); + this.state.comments.push(comment); + } this.addComment(comment); } diff --git a/src/tokenizer/state.js b/src/tokenizer/state.js index d5099ccaeb..4765017de3 100644 --- a/src/tokenizer/state.js +++ b/src/tokenizer/state.js @@ -70,11 +70,15 @@ export default class State { return new Position(this.curLine, this.pos - this.lineStart); } - clone() { + clone(skipArrays?) { var state = new State; for (var key in this) { var val = this[key]; - if (Array.isArray(val)) val = val.slice(); + + if (!skipArrays && Array.isArray(val)) { + val = val.slice(); + } + state[key] = val; } return state;