generators/js-tokens.js - external/github.com/WebKit/JetStream - Git at Google

 // The MIT License (MIT)

 // Copyright (c) 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023 Simon Lydell

 // Permission is hereby granted, free of charge, to any person obtaining a copy
 // of this software and associated documentation files (the "Software"), to deal
 // in the Software without restriction, including without limitation the rights
 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 // copies of the Software, and to permit persons to whom the Software is
 // furnished to do so, subject to the following conditions:

 // The above copyright notice and this permission notice shall be included in
 // all copies or substantial portions of the Software.

 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 // THE SOFTWARE.

 "use strict";

 function jsTokensWrapperFunction() {
     var Identifier, JSXIdentifier, JSXPunctuator, JSXString, JSXText, KeywordsWithExpressionAfter, KeywordsWithNoLineTerminatorAfter, LineTerminatorSequence, MultiLineComment, Newline, NumericLiteral, Punctuator, RegularExpressionLiteral, SingleLineComment, StringLiteral, Template, TokensNotPrecedingObjectLiteral, TokensPrecedingExpression, WhiteSpace, jsTokens;
     RegularExpressionLiteral = /\/(?![*\/])(?:\[(?:(?![\]\\]).|\\.)*\]|(?![\/\\]).|\\.)*(\/[$_\u200C\u200D\p{ID_Continue}]*|\\)?/yu;
     Punctuator = /--|\+\+|=>|\.{3}|\??\.(?!\d)|(?:&&|\|\||\?\?|[+\-%&|^]|\*{1,2}|<{1,2}|>{1,3}|!=?|={1,2}|\/(?![\/*]))=?|[?~,:;[\](){}]/y;
     Identifier = /(\x23?)(?=[$_\p{ID_Start}\\])(?:[$_\u200C\u200D\p{ID_Continue}]|\\u[\da-fA-F]{4}|\\u\{[\da-fA-F]+\})+/yu;
     StringLiteral = /(['"])(?:(?!\1)[^\\\n\r]|\\(?:\r\n|[^]))*(\1)?/y;
     NumericLiteral = /(?:0[xX][\da-fA-F](?:_?[\da-fA-F])*|0[oO][0-7](?:_?[0-7])*|0[bB][01](?:_?[01])*)n?|0n|[1-9](?:_?\d)*n|(?:(?:0(?!\d)|0\d*[89]\d*|[1-9](?:_?\d)*)(?:\.(?:\d(?:_?\d)*)?)?|\.\d(?:_?\d)*)(?:[eE][+-]?\d(?:_?\d)*)?|0[0-7]+/y;
     Template = /[`}](?:[^`\\$]|\\[^]|\$(?!\{))*(`|\$\{)?/y;
     WhiteSpace = /[\t\v\f\ufeff\p{Zs}]+/yu;
     LineTerminatorSequence = /\r?\n|[\r\u2028\u2029]/y;
     MultiLineComment = /\/\*(?:[^*]|\*(?!\/))*(\*\/)?/y;
     SingleLineComment = /\/\/.*/y;
     JSXPunctuator = /[<>.:={}]|\/(?![\/*])/y;
     JSXIdentifier = /[$_\p{ID_Start}][$_\u200C\u200D\p{ID_Continue}-]*/yu;
     JSXString = /(['"])(?:(?!\1)[^])*(\1)?/y;
     JSXText = /[^<>{}]+/y;
     TokensPrecedingExpression = /^(?:[\/+-]|\.{3}|\?(?:InterpolationIn(?:JSX|Template)|NoLineTerminatorHere|NonExpressionParenEnd|UnaryIncDec))?$|[{}([,;<>=*%&|^!~?:]$/;
     TokensNotPrecedingObjectLiteral = /^(?:=>|[;\]){}]|else|\?(?:NoLineTerminatorHere|NonExpressionParenEnd))?$/;
     KeywordsWithExpressionAfter = /^(?:await|case|default|delete|do|else|instanceof|new|return|throw|typeof|void|yield)$/;
     KeywordsWithNoLineTerminatorAfter = /^(?:return|throw|yield)$/;
     Newline = RegExp(LineTerminatorSequence.source);
     return function*(input, {jsx = false} = {}) {
         var braces, firstCodePoint, isExpression, lastIndex, lastSignificantToken, length, match, mode, nextLastIndex, nextLastSignificantToken, parenNesting, postfixIncDec, punctuator, stack;
         ({length} = input);
         lastIndex = 0;
         lastSignificantToken = "";
         stack = [
             {tag: "JS"}
         ];
         braces = [];
         parenNesting = 0;
         postfixIncDec = false;
         while (lastIndex < length) {
             mode = stack[stack.length - 1];
             switch (mode.tag) {
                 case "JS":
                 case "JSNonExpressionParen":
                 case "InterpolationInTemplate":
                 case "InterpolationInJSX":
                     if (input[lastIndex] === "/" && (TokensPrecedingExpression.test(lastSignificantToken) || KeywordsWithExpressionAfter.test(lastSignificantToken))) {
                         RegularExpressionLiteral.lastIndex = lastIndex;
                         if (match = RegularExpressionLiteral.exec(input)) {
                             lastIndex = RegularExpressionLiteral.lastIndex;
                             lastSignificantToken = match[0];
                             postfixIncDec = true;
                             yield ({
                                 type: "RegularExpressionLiteral",
                                 value: match[0],
                                 closed: match[1] !== void 0 && match[1] !== "\\"
                             });
                             continue;
                         }
                     }
                     Punctuator.lastIndex = lastIndex;
                     if (match = Punctuator.exec(input)) {
                         punctuator = match[0];
                         nextLastIndex = Punctuator.lastIndex;
                         nextLastSignificantToken = punctuator;
                         switch (punctuator) {
                             case "(":
                                 if (lastSignificantToken === "?NonExpressionParenKeyword") {
                                     stack.push({
                                         tag: "JSNonExpressionParen",
                                         nesting: parenNesting
                                     });
                                 }
                                 parenNesting++;
                                 postfixIncDec = false;
                                 break;
                             case ")":
                                 parenNesting--;
                                 postfixIncDec = true;
                                 if (mode.tag === "JSNonExpressionParen" && parenNesting === mode.nesting) {
                                     stack.pop();
                                     nextLastSignificantToken = "?NonExpressionParenEnd";
                                     postfixIncDec = false;
                                 }
                                 break;
                             case "{":
                                 Punctuator.lastIndex = 0;
                                 isExpression = !TokensNotPrecedingObjectLiteral.test(lastSignificantToken) && (TokensPrecedingExpression.test(lastSignificantToken) || KeywordsWithExpressionAfter.test(lastSignificantToken));
                                 braces.push(isExpression);
                                 postfixIncDec = false;
                                 break;
                             case "}":
                                 switch (mode.tag) {
                                     case "InterpolationInTemplate":
                                         if (braces.length === mode.nesting) {
                                             Template.lastIndex = lastIndex;
                                             match = Template.exec(input);
                                             lastIndex = Template.lastIndex;
                                             lastSignificantToken = match[0];
                                             if (match[1] === "${") {
                                                 lastSignificantToken = "?InterpolationInTemplate";
                                                 postfixIncDec = false;
                                                 yield ({
                                                     type: "TemplateMiddle",
                                                     value: match[0]
                                                 });
                                             } else {
                                                 stack.pop();
                                                 postfixIncDec = true;
                                                 yield ({
                                                     type: "TemplateTail",
                                                     value: match[0],
                                                     closed: match[1] === "`"
                                                 });
                                             }
                                             continue;
                                         }
                                         break;
                                     case "InterpolationInJSX":
                                         if (braces.length === mode.nesting) {
                                             stack.pop();
                                             lastIndex += 1;
                                             lastSignificantToken = "}";
                                             yield ({
                                                 type: "JSXPunctuator",
                                                 value: "}"
                                             });
                                             continue;
                                         }
                                 }
                                 postfixIncDec = braces.pop();
                                 nextLastSignificantToken = postfixIncDec ? "?ExpressionBraceEnd" : "}";
                                 break;
                             case "]":
                                 postfixIncDec = true;
                                 break;
                             case "++":
                             case "--":
                                 nextLastSignificantToken = postfixIncDec ? "?PostfixIncDec" : "?UnaryIncDec";
                                 break;
                             case "<":
                                 if (jsx && (TokensPrecedingExpression.test(lastSignificantToken) || KeywordsWithExpressionAfter.test(lastSignificantToken))) {
                                     stack.push({tag: "JSXTag"});
                                     lastIndex += 1;
                                     lastSignificantToken = "<";
                                     yield ({
                                         type: "JSXPunctuator",
                                         value: punctuator
                                     });
                                     continue;
                                 }
                                 postfixIncDec = false;
                                 break;
                             default:
                                 postfixIncDec = false;
                         }
                         lastIndex = nextLastIndex;
                         lastSignificantToken = nextLastSignificantToken;
                         yield ({
                             type: "Punctuator",
                             value: punctuator
                         });
                         continue;
                     }
                     Identifier.lastIndex = lastIndex;
                     if (match = Identifier.exec(input)) {
                         lastIndex = Identifier.lastIndex;
                         nextLastSignificantToken = match[0];
                         switch (match[0]) {
                             case "for":
                             case "if":
                             case "while":
                             case "with":
                                 if (lastSignificantToken !== "." && lastSignificantToken !== "?.") {
                                     nextLastSignificantToken = "?NonExpressionParenKeyword";
                                 }
                         }
                         lastSignificantToken = nextLastSignificantToken;
                         postfixIncDec = !KeywordsWithExpressionAfter.test(match[0]);
                         yield ({
                             type: match[1] === "#" ? "PrivateIdentifier" : "IdentifierName",
                             value: match[0]
                         });
                         continue;
                     }
                     StringLiteral.lastIndex = lastIndex;
                     if (match = StringLiteral.exec(input)) {
                         lastIndex = StringLiteral.lastIndex;
                         lastSignificantToken = match[0];
                         postfixIncDec = true;
                         yield ({
                             type: "StringLiteral",
                             value: match[0],
                             closed: match[2] !== void 0
                         });
                         continue;
                     }
                     NumericLiteral.lastIndex = lastIndex;
                     if (match = NumericLiteral.exec(input)) {
                         lastIndex = NumericLiteral.lastIndex;
                         lastSignificantToken = match[0];
                         postfixIncDec = true;
                         yield ({
                             type: "NumericLiteral",
                             value: match[0]
                         });
                         continue;
                     }
                     Template.lastIndex = lastIndex;
                     if (match = Template.exec(input)) {
                         lastIndex = Template.lastIndex;
                         lastSignificantToken = match[0];
                         if (match[1] === "${") {
                             lastSignificantToken = "?InterpolationInTemplate";
                             stack.push({
                                 tag: "InterpolationInTemplate",
                                 nesting: braces.length
                             });
                             postfixIncDec = false;
                             yield ({
                                 type: "TemplateHead",
                                 value: match[0]
                             });
                         } else {
                             postfixIncDec = true;
                             yield ({
                                 type: "NoSubstitutionTemplate",
                                 value: match[0],
                                 closed: match[1] === "`"
                             });
                         }
                         continue;
                     }
                     break;
                 case "JSXTag":
                 case "JSXTagEnd":
                     JSXPunctuator.lastIndex = lastIndex;
                     if (match = JSXPunctuator.exec(input)) {
                         lastIndex = JSXPunctuator.lastIndex;
                         nextLastSignificantToken = match[0];
                         switch (match[0]) {
                             case "<":
                                 stack.push({tag: "JSXTag"});
                                 break;
                             case ">":
                                 stack.pop();
                                 if (lastSignificantToken === "/" || mode.tag === "JSXTagEnd") {
                                     nextLastSignificantToken = "?JSX";
                                     postfixIncDec = true;
                                 } else {
                                     stack.push({tag: "JSXChildren"});
                                 }
                                 break;
                             case "{":
                                 stack.push({
                                     tag: "InterpolationInJSX",
                                     nesting: braces.length
                                 });
                                 nextLastSignificantToken = "?InterpolationInJSX";
                                 postfixIncDec = false;
                                 break;
                             case "/":
                                 if (lastSignificantToken === "<") {
                                     stack.pop();
                                     if (stack[stack.length - 1].tag === "JSXChildren") {
                                         stack.pop();
                                     }
                                     stack.push({tag: "JSXTagEnd"});
                                 }
                         }
                         lastSignificantToken = nextLastSignificantToken;
                         yield ({
                             type: "JSXPunctuator",
                             value: match[0]
                         });
                         continue;
                     }
                     JSXIdentifier.lastIndex = lastIndex;
                     if (match = JSXIdentifier.exec(input)) {
                         lastIndex = JSXIdentifier.lastIndex;
                         lastSignificantToken = match[0];
                         yield ({
                             type: "JSXIdentifier",
                             value: match[0]
                         });
                         continue;
                     }
                     JSXString.lastIndex = lastIndex;
                     if (match = JSXString.exec(input)) {
                         lastIndex = JSXString.lastIndex;
                         lastSignificantToken = match[0];
                         yield ({
                             type: "JSXString",
                             value: match[0],
                             closed: match[2] !== void 0
                         });
                         continue;
                     }
                     break;
                 case "JSXChildren":
                     JSXText.lastIndex = lastIndex;
                     if (match = JSXText.exec(input)) {
                         lastIndex = JSXText.lastIndex;
                         lastSignificantToken = match[0];
                         yield ({
                             type: "JSXText",
                             value: match[0]
                         });
                         continue;
                     }
                     switch (input[lastIndex]) {
                         case "<":
                             stack.push({tag: "JSXTag"});
                             lastIndex++;
                             lastSignificantToken = "<";
                             yield ({
                                 type: "JSXPunctuator",
                                 value: "<"
                             });
                             continue;
                         case "{":
                             stack.push({
                                 tag: "InterpolationInJSX",
                                 nesting: braces.length
                             });
                             lastIndex++;
                             lastSignificantToken = "?InterpolationInJSX";
                             postfixIncDec = false;
                             yield ({
                                 type: "JSXPunctuator",
                                 value: "{"
                             });
                             continue;
                     }
             }
             WhiteSpace.lastIndex = lastIndex;
             if (match = WhiteSpace.exec(input)) {
                 lastIndex = WhiteSpace.lastIndex;
                 yield ({
                     type: "WhiteSpace",
                     value: match[0]
                 });
                 continue;
             }
             LineTerminatorSequence.lastIndex = lastIndex;
             if (match = LineTerminatorSequence.exec(input)) {
                 lastIndex = LineTerminatorSequence.lastIndex;
                 postfixIncDec = false;
                 if (KeywordsWithNoLineTerminatorAfter.test(lastSignificantToken)) {
                     lastSignificantToken = "?NoLineTerminatorHere";
                 }
                 yield ({
                     type: "LineTerminatorSequence",
                     value: match[0]
                 });
                 continue;
             }
             MultiLineComment.lastIndex = lastIndex;
             if (match = MultiLineComment.exec(input)) {
                 lastIndex = MultiLineComment.lastIndex;
                 if (Newline.test(match[0])) {
                     postfixIncDec = false;
                     if (KeywordsWithNoLineTerminatorAfter.test(lastSignificantToken)) {
                         lastSignificantToken = "?NoLineTerminatorHere";
                     }
                 }
                 yield ({
                     type: "MultiLineComment",
                     value: match[0],
                     closed: match[1] !== void 0
                 });
                 continue;
             }
             SingleLineComment.lastIndex = lastIndex;
             if (match = SingleLineComment.exec(input)) {
                 lastIndex = SingleLineComment.lastIndex;
                 postfixIncDec = false;
                 yield ({
                     type: "SingleLineComment",
                     value: match[0]
                 });
                 continue;
             }
             firstCodePoint = String.fromCodePoint(input.codePointAt(lastIndex));
             lastIndex += firstCodePoint.length;
             lastSignificantToken = firstCodePoint;
             postfixIncDec = false;
             yield ({
                 type: mode.tag.startsWith("JSX") ? "JSXInvalid" : "Invalid",
                 value: firstCodePoint
             });
         }
     };
 };

 const jsTokensSourceCode = jsTokensWrapperFunction.toString();
 const jsxSourceCode = `

 function Comment(props) {
   return (
     <div className="Comment">
       <div className="UserInfo">
         <Avatar user={props.author} />
         <div className="UserInfo-name">
           {props.author.name}
         </div>
       </div>
       <div className="Comment-text">
         {props.text}
       </div>
       <div className="Comment-date">
         {formatDate(props.date)}
       </div>
     </div>
   );
 }

 `;

 const jsTokens = jsTokensWrapperFunction();

 class Benchmark {
     runIteration() {
         this.tokenCount = 0;

         for (var i = 0; i < 25; i++) {
             for (const token of jsTokens(jsTokensSourceCode))
                 this.tokenCount++;
         }

         for (var i = 0; i < 250; i++) {
             for (const token of jsTokens(jsxSourceCode, { jsx: true }))
                 this.tokenCount++;
         }
     }

     validate() {
         if (this.tokenCount !== 113975)
             throw new Error(`this.tokenCount of ${this.tokenCount} is invalid!`);
     }
 }
	// The MIT License (MIT)

	// Copyright (c) 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023 Simon Lydell

	// Permission is hereby granted, free of charge, to any person obtaining a copy
	// of this software and associated documentation files (the "Software"), to deal
	// in the Software without restriction, including without limitation the rights
	// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
	// copies of the Software, and to permit persons to whom the Software is
	// furnished to do so, subject to the following conditions:

	// The above copyright notice and this permission notice shall be included in
	// all copies or substantial portions of the Software.

	// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
	// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
	// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
	// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
	// THE SOFTWARE.

	"use strict";

	function jsTokensWrapperFunction() {
	var Identifier, JSXIdentifier, JSXPunctuator, JSXString, JSXText, KeywordsWithExpressionAfter, KeywordsWithNoLineTerminatorAfter, LineTerminatorSequence, MultiLineComment, Newline, NumericLiteral, Punctuator, RegularExpressionLiteral, SingleLineComment, StringLiteral, Template, TokensNotPrecedingObjectLiteral, TokensPrecedingExpression, WhiteSpace, jsTokens;
	RegularExpressionLiteral = /\/(?![\/])(?:\[(?:(?![\]\\]).\|\\.)\]\|(?![\/\\]).\|\\.)(\/[$_\u200C\u200D\p{ID_Continue}]\|\\)?/yu;
	Punctuator = /--\|\+\+\|=>\|\.{3}\|\??\.(?!\d)\|(?:&&\|\\|\\|\|\?\?\|[+\-%&\|^]\|\{1,2}\|<{1,2}\|>{1,3}\|!=?\|={1,2}\|\/(?![\/]))=?\|[?~,:;[\](){}]/y;
	Identifier = /(\x23?)(?=[$_\p{ID_Start}\\])(?:[$_\u200C\u200D\p{ID_Continue}]\|\\u[\da-fA-F]{4}\|\\u\{[\da-fA-F]+\})+/yu;
	StringLiteral = /(['"])(?:(?!\1)[^\\\n\r]\|\\(?:\r\n\|[^]))*(\1)?/y;
	NumericLiteral = /(?:0[xX][\da-fA-F](?:_?[\da-fA-F])\|0[oO][0-7](?:_?[0-7])\|0[bB][01](?:_?[01]))n?\|0n\|[1-9](?:_?\d)n\|(?:(?:0(?!\d)\|0\d[89]\d\|[1-9](?:_?\d))(?:\.(?:\d(?:_?\d))?)?\|\.\d(?:_?\d))(?:[eE][+-]?\d(?:_?\d))?\|0[0-7]+/y;
	Template = /[`}](?:[^`\\$]\|\\[^]\|\$(?!\{))*(`\|\$\{)?/y;
	WhiteSpace = /[\t\v\f\ufeff\p{Zs}]+/yu;
	LineTerminatorSequence = /\r?\n\|[\r\u2028\u2029]/y;
	MultiLineComment = /\/\(?:[^]\|\(?!\/))(\*\/)?/y;
	SingleLineComment = /\/\/.*/y;
	JSXPunctuator = /[<>.:={}]\|\/(?![\/*])/y;
	JSXIdentifier = /[$_\p{ID_Start}][$_\u200C\u200D\p{ID_Continue}-]*/yu;
	JSXString = /(['"])(?:(?!\1)[^])*(\1)?/y;
	JSXText = /[^<>{}]+/y;
	TokensPrecedingExpression = /^(?:[\/+-]\|\.{3}\|\?(?:InterpolationIn(?:JSX\|Template)\|NoLineTerminatorHere\|NonExpressionParenEnd\|UnaryIncDec))?$\|[{}([,;<>=*%&\|^!~?:]$/;
	TokensNotPrecedingObjectLiteral = /^(?:=>\|[;\]){}]\|else\|\?(?:NoLineTerminatorHere\|NonExpressionParenEnd))?$/;
	KeywordsWithExpressionAfter = /^(?:await\|case\|default\|delete\|do\|else\|instanceof\|new\|return\|throw\|typeof\|void\|yield)$/;
	KeywordsWithNoLineTerminatorAfter = /^(?:return\|throw\|yield)$/;
	Newline = RegExp(LineTerminatorSequence.source);
	return function*(input, {jsx = false} = {}) {
	var braces, firstCodePoint, isExpression, lastIndex, lastSignificantToken, length, match, mode, nextLastIndex, nextLastSignificantToken, parenNesting, postfixIncDec, punctuator, stack;
	({length} = input);
	lastIndex = 0;
	lastSignificantToken = "";
	stack = [
	{tag: "JS"}
	];
	braces = [];
	parenNesting = 0;
	postfixIncDec = false;
	while (lastIndex < length) {
	mode = stack[stack.length - 1];
	switch (mode.tag) {
	case "JS":
	case "JSNonExpressionParen":
	case "InterpolationInTemplate":
	case "InterpolationInJSX":
	if (input[lastIndex] === "/" && (TokensPrecedingExpression.test(lastSignificantToken) \|\| KeywordsWithExpressionAfter.test(lastSignificantToken))) {
	RegularExpressionLiteral.lastIndex = lastIndex;
	if (match = RegularExpressionLiteral.exec(input)) {
	lastIndex = RegularExpressionLiteral.lastIndex;
	lastSignificantToken = match[0];
	postfixIncDec = true;
	yield ({
	type: "RegularExpressionLiteral",
	value: match[0],
	closed: match[1] !== void 0 && match[1] !== "\\"
	});
	continue;
	}
	}
	Punctuator.lastIndex = lastIndex;
	if (match = Punctuator.exec(input)) {
	punctuator = match[0];
	nextLastIndex = Punctuator.lastIndex;
	nextLastSignificantToken = punctuator;
	switch (punctuator) {
	case "(":
	if (lastSignificantToken === "?NonExpressionParenKeyword") {
	stack.push({
	tag: "JSNonExpressionParen",
	nesting: parenNesting
	});
	}
	parenNesting++;
	postfixIncDec = false;
	break;
	case ")":
	parenNesting--;
	postfixIncDec = true;
	if (mode.tag === "JSNonExpressionParen" && parenNesting === mode.nesting) {
	stack.pop();
	nextLastSignificantToken = "?NonExpressionParenEnd";
	postfixIncDec = false;
	}
	break;
	case "{":
	Punctuator.lastIndex = 0;
	isExpression = !TokensNotPrecedingObjectLiteral.test(lastSignificantToken) && (TokensPrecedingExpression.test(lastSignificantToken) \|\| KeywordsWithExpressionAfter.test(lastSignificantToken));
	braces.push(isExpression);
	postfixIncDec = false;
	break;
	case "}":
	switch (mode.tag) {
	case "InterpolationInTemplate":
	if (braces.length === mode.nesting) {
	Template.lastIndex = lastIndex;
	match = Template.exec(input);
	lastIndex = Template.lastIndex;
	lastSignificantToken = match[0];
	if (match[1] === "${") {
	lastSignificantToken = "?InterpolationInTemplate";
	postfixIncDec = false;
	yield ({
	type: "TemplateMiddle",
	value: match[0]
	});
	} else {
	stack.pop();
	postfixIncDec = true;
	yield ({
	type: "TemplateTail",
	value: match[0],
	closed: match[1] === "`"
	});
	}
	continue;
	}
	break;
	case "InterpolationInJSX":
	if (braces.length === mode.nesting) {
	stack.pop();
	lastIndex += 1;
	lastSignificantToken = "}";
	yield ({
	type: "JSXPunctuator",
	value: "}"
	});
	continue;
	}
	}
	postfixIncDec = braces.pop();
	nextLastSignificantToken = postfixIncDec ? "?ExpressionBraceEnd" : "}";
	break;
	case "]":
	postfixIncDec = true;
	break;
	case "++":
	case "--":
	nextLastSignificantToken = postfixIncDec ? "?PostfixIncDec" : "?UnaryIncDec";
	break;
	case "<":
	if (jsx && (TokensPrecedingExpression.test(lastSignificantToken) \|\| KeywordsWithExpressionAfter.test(lastSignificantToken))) {
	stack.push({tag: "JSXTag"});
	lastIndex += 1;
	lastSignificantToken = "<";
	yield ({
	type: "JSXPunctuator",
	value: punctuator
	});
	continue;
	}
	postfixIncDec = false;
	break;
	default:
	postfixIncDec = false;
	}
	lastIndex = nextLastIndex;
	lastSignificantToken = nextLastSignificantToken;
	yield ({
	type: "Punctuator",
	value: punctuator
	});
	continue;
	}
	Identifier.lastIndex = lastIndex;
	if (match = Identifier.exec(input)) {
	lastIndex = Identifier.lastIndex;
	nextLastSignificantToken = match[0];
	switch (match[0]) {
	case "for":
	case "if":
	case "while":
	case "with":
	if (lastSignificantToken !== "." && lastSignificantToken !== "?.") {
	nextLastSignificantToken = "?NonExpressionParenKeyword";
	}
	}
	lastSignificantToken = nextLastSignificantToken;
	postfixIncDec = !KeywordsWithExpressionAfter.test(match[0]);
	yield ({
	type: match[1] === "#" ? "PrivateIdentifier" : "IdentifierName",
	value: match[0]
	});
	continue;
	}
	StringLiteral.lastIndex = lastIndex;
	if (match = StringLiteral.exec(input)) {
	lastIndex = StringLiteral.lastIndex;
	lastSignificantToken = match[0];
	postfixIncDec = true;
	yield ({
	type: "StringLiteral",
	value: match[0],
	closed: match[2] !== void 0
	});
	continue;
	}
	NumericLiteral.lastIndex = lastIndex;
	if (match = NumericLiteral.exec(input)) {
	lastIndex = NumericLiteral.lastIndex;
	lastSignificantToken = match[0];
	postfixIncDec = true;
	yield ({
	type: "NumericLiteral",
	value: match[0]
	});
	continue;
	}
	Template.lastIndex = lastIndex;
	if (match = Template.exec(input)) {
	lastIndex = Template.lastIndex;
	lastSignificantToken = match[0];
	if (match[1] === "${") {
	lastSignificantToken = "?InterpolationInTemplate";
	stack.push({
	tag: "InterpolationInTemplate",
	nesting: braces.length
	});
	postfixIncDec = false;
	yield ({
	type: "TemplateHead",
	value: match[0]
	});
	} else {
	postfixIncDec = true;
	yield ({
	type: "NoSubstitutionTemplate",
	value: match[0],
	closed: match[1] === "`"
	});
	}
	continue;
	}
	break;
	case "JSXTag":
	case "JSXTagEnd":
	JSXPunctuator.lastIndex = lastIndex;
	if (match = JSXPunctuator.exec(input)) {
	lastIndex = JSXPunctuator.lastIndex;
	nextLastSignificantToken = match[0];
	switch (match[0]) {
	case "<":
	stack.push({tag: "JSXTag"});
	break;
	case ">":
	stack.pop();
	if (lastSignificantToken === "/" \|\| mode.tag === "JSXTagEnd") {
	nextLastSignificantToken = "?JSX";
	postfixIncDec = true;
	} else {
	stack.push({tag: "JSXChildren"});
	}
	break;
	case "{":
	stack.push({
	tag: "InterpolationInJSX",
	nesting: braces.length
	});
	nextLastSignificantToken = "?InterpolationInJSX";
	postfixIncDec = false;
	break;
	case "/":
	if (lastSignificantToken === "<") {
	stack.pop();
	if (stack[stack.length - 1].tag === "JSXChildren") {
	stack.pop();
	}
	stack.push({tag: "JSXTagEnd"});
	}
	}
	lastSignificantToken = nextLastSignificantToken;
	yield ({
	type: "JSXPunctuator",
	value: match[0]
	});
	continue;
	}
	JSXIdentifier.lastIndex = lastIndex;
	if (match = JSXIdentifier.exec(input)) {
	lastIndex = JSXIdentifier.lastIndex;
	lastSignificantToken = match[0];
	yield ({
	type: "JSXIdentifier",
	value: match[0]
	});
	continue;
	}
	JSXString.lastIndex = lastIndex;
	if (match = JSXString.exec(input)) {
	lastIndex = JSXString.lastIndex;
	lastSignificantToken = match[0];
	yield ({
	type: "JSXString",
	value: match[0],
	closed: match[2] !== void 0
	});
	continue;
	}
	break;
	case "JSXChildren":
	JSXText.lastIndex = lastIndex;
	if (match = JSXText.exec(input)) {
	lastIndex = JSXText.lastIndex;
	lastSignificantToken = match[0];
	yield ({
	type: "JSXText",
	value: match[0]
	});
	continue;
	}
	switch (input[lastIndex]) {
	case "<":
	stack.push({tag: "JSXTag"});
	lastIndex++;
	lastSignificantToken = "<";
	yield ({
	type: "JSXPunctuator",
	value: "<"
	});
	continue;
	case "{":
	stack.push({
	tag: "InterpolationInJSX",
	nesting: braces.length
	});
	lastIndex++;
	lastSignificantToken = "?InterpolationInJSX";
	postfixIncDec = false;
	yield ({
	type: "JSXPunctuator",
	value: "{"
	});
	continue;
	}
	}
	WhiteSpace.lastIndex = lastIndex;
	if (match = WhiteSpace.exec(input)) {
	lastIndex = WhiteSpace.lastIndex;
	yield ({
	type: "WhiteSpace",
	value: match[0]
	});
	continue;
	}
	LineTerminatorSequence.lastIndex = lastIndex;
	if (match = LineTerminatorSequence.exec(input)) {
	lastIndex = LineTerminatorSequence.lastIndex;
	postfixIncDec = false;
	if (KeywordsWithNoLineTerminatorAfter.test(lastSignificantToken)) {
	lastSignificantToken = "?NoLineTerminatorHere";
	}
	yield ({
	type: "LineTerminatorSequence",
	value: match[0]
	});
	continue;
	}
	MultiLineComment.lastIndex = lastIndex;
	if (match = MultiLineComment.exec(input)) {
	lastIndex = MultiLineComment.lastIndex;
	if (Newline.test(match[0])) {
	postfixIncDec = false;
	if (KeywordsWithNoLineTerminatorAfter.test(lastSignificantToken)) {
	lastSignificantToken = "?NoLineTerminatorHere";
	}
	}
	yield ({
	type: "MultiLineComment",
	value: match[0],
	closed: match[1] !== void 0
	});
	continue;
	}
	SingleLineComment.lastIndex = lastIndex;
	if (match = SingleLineComment.exec(input)) {
	lastIndex = SingleLineComment.lastIndex;
	postfixIncDec = false;
	yield ({
	type: "SingleLineComment",
	value: match[0]
	});
	continue;
	}
	firstCodePoint = String.fromCodePoint(input.codePointAt(lastIndex));
	lastIndex += firstCodePoint.length;
	lastSignificantToken = firstCodePoint;
	postfixIncDec = false;
	yield ({
	type: mode.tag.startsWith("JSX") ? "JSXInvalid" : "Invalid",
	value: firstCodePoint
	});
	}
	};
	};

	const jsTokensSourceCode = jsTokensWrapperFunction.toString();
	const jsxSourceCode = `

	function Comment(props) {
	return (
	<div className="Comment">
	<div className="UserInfo">
	<Avatar user={props.author} />
	<div className="UserInfo-name">
	{props.author.name}
	</div>
	</div>
	<div className="Comment-text">
	{props.text}
	</div>
	<div className="Comment-date">
	{formatDate(props.date)}
	</div>
	</div>
	);
	}

	`;

	const jsTokens = jsTokensWrapperFunction();

	class Benchmark {
	runIteration() {
	this.tokenCount = 0;

	for (var i = 0; i < 25; i++) {
	for (const token of jsTokens(jsTokensSourceCode))
	this.tokenCount++;
	}

	for (var i = 0; i < 250; i++) {
	for (const token of jsTokens(jsxSourceCode, { jsx: true }))
	this.tokenCount++;
	}
	}

	validate() {
	if (this.tokenCount !== 113975)
	throw new Error(`this.tokenCount of ${this.tokenCount} is invalid!`);
	}
	}