1// Package js is an ECMAScript5.1 lexer following the specifications at http://www.ecma-international.org/ecma-262/5.1/.
2package js
3
4import (
5 "unicode"
6 "unicode/utf8"
7
8 "github.com/tdewolff/parse/v2"
9)
10
11var identifierStart = []*unicode.RangeTable{unicode.Lu, unicode.Ll, unicode.Lt, unicode.Lm, unicode.Lo, unicode.Nl, unicode.Other_ID_Start}
12var identifierContinue = []*unicode.RangeTable{unicode.Lu, unicode.Ll, unicode.Lt, unicode.Lm, unicode.Lo, unicode.Nl, unicode.Mn, unicode.Mc, unicode.Nd, unicode.Pc, unicode.Other_ID_Continue}
13
14// IsIdentifierStart returns true if the byte-slice start is the start of an identifier
15func IsIdentifierStart(b []byte) bool {
16 r, _ := utf8.DecodeRune(b)
17 return r == '$' || r == '\\' || r == '_' || unicode.IsOneOf(identifierStart, r)
18}
19
20// IsIdentifierContinue returns true if the byte-slice start is a continuation of an identifier
21func IsIdentifierContinue(b []byte) bool {
22 r, _ := utf8.DecodeRune(b)
23 return r == '$' || r == '\\' || r == '\u200C' || r == '\u200D' || unicode.IsOneOf(identifierContinue, r)
24}
25
26// IsIdentifierEnd returns true if the byte-slice end is a start or continuation of an identifier
27func IsIdentifierEnd(b []byte) bool {
28 r, _ := utf8.DecodeLastRune(b)
29 return r == '$' || r == '\\' || r == '\u200C' || r == '\u200D' || unicode.IsOneOf(identifierContinue, r)
30}
31
32////////////////////////////////////////////////////////////////
33
34// Lexer is the state for the lexer.
35type Lexer struct {
36 r *parse.Input
37 err error
38 prevLineTerminator bool
39 prevNumericLiteral bool
40 level int
41 templateLevels []int
42}
43
44// NewLexer returns a new Lexer for a given io.Reader.
45func NewLexer(r *parse.Input) *Lexer {
46 return &Lexer{
47 r: r,
48 prevLineTerminator: true,
49 level: 0,
50 templateLevels: []int{},
51 }
52}
53
54// Err returns the error encountered during lexing, this is often io.EOF but also other errors can be returned.
55func (l *Lexer) Err() error {
56 if l.err != nil {
57 return l.err
58 }
59 return l.r.Err()
60}
61
62// RegExp reparses the input stream for a regular expression. It is assumed that we just received DivToken or DivEqToken with Next(). This function will go back and read that as a regular expression.
63func (l *Lexer) RegExp() (TokenType, []byte) {
64 if 0 < l.r.Offset() && l.r.Peek(-1) == '/' {
65 l.r.Move(-1)
66 } else if 1 < l.r.Offset() && l.r.Peek(-1) == '=' && l.r.Peek(-2) == '/' {
67 l.r.Move(-2)
68 } else {
69 l.err = parse.NewErrorLexer(l.r, "expected / or /=")
70 return ErrorToken, nil
71 }
72 l.r.Skip() // trick to set start = pos
73
74 if l.consumeRegExpToken() {
75 return RegExpToken, l.r.Shift()
76 }
77 l.err = parse.NewErrorLexer(l.r, "unexpected EOF or newline")
78 return ErrorToken, nil
79}
80
81// Next returns the next Token. It returns ErrorToken when an error was encountered. Using Err() one can retrieve the error message.
82func (l *Lexer) Next() (TokenType, []byte) {
83 prevLineTerminator := l.prevLineTerminator
84 l.prevLineTerminator = false
85
86 prevNumericLiteral := l.prevNumericLiteral
87 l.prevNumericLiteral = false
88
89 // study on 50x jQuery shows:
90 // spaces: 20k
91 // alpha: 16k
92 // newlines: 14.4k
93 // operators: 4k
94 // numbers and dot: 3.6k
95 // (): 3.4k
96 // {}: 1.8k
97 // []: 0.9k
98 // "': 1k
99 // semicolon: 2.4k
100 // colon: 0.8k
101 // comma: 2.4k
102 // slash: 1.4k
103 // `~: almost 0
104
105 c := l.r.Peek(0)
106 switch c {
107 case ' ', '\t', '\v', '\f':
108 l.r.Move(1)
109 for l.consumeWhitespace() {
110 }
111 l.prevLineTerminator = prevLineTerminator
112 return WhitespaceToken, l.r.Shift()
113 case '\n', '\r':
114 l.r.Move(1)
115 for l.consumeLineTerminator() {
116 }
117 l.prevLineTerminator = true
118 return LineTerminatorToken, l.r.Shift()
119 case '>', '=', '!', '+', '*', '%', '&', '|', '^', '~', '?':
120 if tt := l.consumeOperatorToken(); tt != ErrorToken {
121 return tt, l.r.Shift()
122 }
123 case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '.':
124 if tt := l.consumeNumericToken(); tt != ErrorToken || l.r.Pos() != 0 {
125 l.prevNumericLiteral = true
126 return tt, l.r.Shift()
127 } else if c == '.' {
128 l.r.Move(1)
129 if l.r.Peek(0) == '.' && l.r.Peek(1) == '.' {
130 l.r.Move(2)
131 return EllipsisToken, l.r.Shift()
132 }
133 return DotToken, l.r.Shift()
134 }
135 case ',':
136 l.r.Move(1)
137 return CommaToken, l.r.Shift()
138 case ';':
139 l.r.Move(1)
140 return SemicolonToken, l.r.Shift()
141 case '(':
142 l.level++
143 l.r.Move(1)
144 return OpenParenToken, l.r.Shift()
145 case ')':
146 l.level--
147 l.r.Move(1)
148 return CloseParenToken, l.r.Shift()
149 case '/':
150 if tt := l.consumeCommentToken(); tt != ErrorToken {
151 return tt, l.r.Shift()
152 } else if tt := l.consumeOperatorToken(); tt != ErrorToken {
153 return tt, l.r.Shift()
154 }
155 case '{':
156 l.level++
157 l.r.Move(1)
158 return OpenBraceToken, l.r.Shift()
159 case '}':
160 l.level--
161 if len(l.templateLevels) != 0 && l.level == l.templateLevels[len(l.templateLevels)-1] {
162 return l.consumeTemplateToken(), l.r.Shift()
163 }
164 l.r.Move(1)
165 return CloseBraceToken, l.r.Shift()
166 case ':':
167 l.r.Move(1)
168 return ColonToken, l.r.Shift()
169 case '\'', '"':
170 return l.consumeStringToken(), l.r.Shift()
171 case ']':
172 l.r.Move(1)
173 return CloseBracketToken, l.r.Shift()
174 case '[':
175 l.r.Move(1)
176 return OpenBracketToken, l.r.Shift()
177 case '<', '-':
178 if l.consumeHTMLLikeCommentToken(prevLineTerminator) {
179 return CommentToken, l.r.Shift()
180 } else if tt := l.consumeOperatorToken(); tt != ErrorToken {
181 return tt, l.r.Shift()
182 }
183 case '`':
184 l.templateLevels = append(l.templateLevels, l.level)
185 return l.consumeTemplateToken(), l.r.Shift()
186 case '#':
187 l.r.Move(1)
188 if l.consumeIdentifierToken() {
189 return PrivateIdentifierToken, l.r.Shift()
190 }
191 return ErrorToken, nil
192 default:
193 if l.consumeIdentifierToken() {
194 if prevNumericLiteral {
195 l.err = parse.NewErrorLexer(l.r, "unexpected identifier after number")
196 return ErrorToken, nil
197 } else if keyword, ok := Keywords[string(l.r.Lexeme())]; ok {
198 return keyword, l.r.Shift()
199 }
200 return IdentifierToken, l.r.Shift()
201 }
202 if 0xC0 <= c {
203 if l.consumeWhitespace() {
204 for l.consumeWhitespace() {
205 }
206 l.prevLineTerminator = prevLineTerminator
207 return WhitespaceToken, l.r.Shift()
208 } else if l.consumeLineTerminator() {
209 for l.consumeLineTerminator() {
210 }
211 l.prevLineTerminator = true
212 return LineTerminatorToken, l.r.Shift()
213 }
214 } else if c == 0 && l.r.Err() != nil {
215 return ErrorToken, nil
216 }
217 }
218
219 r, _ := l.r.PeekRune(0)
220 l.err = parse.NewErrorLexer(l.r, "unexpected %s", parse.Printable(r))
221 return ErrorToken, l.r.Shift()
222}
223
224////////////////////////////////////////////////////////////////
225
226/*
227The following functions follow the specifications at http://www.ecma-international.org/ecma-262/5.1/
228*/
229
230func (l *Lexer) consumeWhitespace() bool {
231 c := l.r.Peek(0)
232 if c == ' ' || c == '\t' || c == '\v' || c == '\f' {
233 l.r.Move(1)
234 return true
235 } else if 0xC0 <= c {
236 if r, n := l.r.PeekRune(0); r == '\u00A0' || r == '\uFEFF' || unicode.Is(unicode.Zs, r) {
237 l.r.Move(n)
238 return true
239 }
240 }
241 return false
242}
243
244func (l *Lexer) isLineTerminator() bool {
245 c := l.r.Peek(0)
246 if c == '\n' || c == '\r' {
247 return true
248 } else if c == 0xE2 && l.r.Peek(1) == 0x80 && (l.r.Peek(2) == 0xA8 || l.r.Peek(2) == 0xA9) {
249 return true
250 }
251 return false
252}
253
254func (l *Lexer) consumeLineTerminator() bool {
255 c := l.r.Peek(0)
256 if c == '\n' {
257 l.r.Move(1)
258 return true
259 } else if c == '\r' {
260 if l.r.Peek(1) == '\n' {
261 l.r.Move(2)
262 } else {
263 l.r.Move(1)
264 }
265 return true
266 } else if c == 0xE2 && l.r.Peek(1) == 0x80 && (l.r.Peek(2) == 0xA8 || l.r.Peek(2) == 0xA9) {
267 l.r.Move(3)
268 return true
269 }
270 return false
271}
272
273func (l *Lexer) consumeDigit() bool {
274 if c := l.r.Peek(0); c >= '0' && c <= '9' {
275 l.r.Move(1)
276 return true
277 }
278 return false
279}
280
281func (l *Lexer) consumeHexDigit() bool {
282 if c := l.r.Peek(0); (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F') {
283 l.r.Move(1)
284 return true
285 }
286 return false
287}
288
289func (l *Lexer) consumeBinaryDigit() bool {
290 if c := l.r.Peek(0); c == '0' || c == '1' {
291 l.r.Move(1)
292 return true
293 }
294 return false
295}
296
297func (l *Lexer) consumeOctalDigit() bool {
298 if c := l.r.Peek(0); c >= '0' && c <= '7' {
299 l.r.Move(1)
300 return true
301 }
302 return false
303}
304
305func (l *Lexer) consumeUnicodeEscape() bool {
306 if l.r.Peek(0) != '\\' || l.r.Peek(1) != 'u' {
307 return false
308 }
309 mark := l.r.Pos()
310 l.r.Move(2)
311 if c := l.r.Peek(0); c == '{' {
312 l.r.Move(1)
313 if l.consumeHexDigit() {
314 for l.consumeHexDigit() {
315 }
316 if c := l.r.Peek(0); c == '}' {
317 l.r.Move(1)
318 return true
319 }
320 }
321 l.r.Rewind(mark)
322 return false
323 } else if !l.consumeHexDigit() || !l.consumeHexDigit() || !l.consumeHexDigit() || !l.consumeHexDigit() {
324 l.r.Rewind(mark)
325 return false
326 }
327 return true
328}
329
330func (l *Lexer) consumeSingleLineComment() {
331 for {
332 c := l.r.Peek(0)
333 if c == '\r' || c == '\n' || c == 0 && l.r.Err() != nil {
334 break
335 } else if 0xC0 <= c {
336 if r, _ := l.r.PeekRune(0); r == '\u2028' || r == '\u2029' {
337 break
338 }
339 }
340 l.r.Move(1)
341 }
342}
343
344////////////////////////////////////////////////////////////////
345
346func (l *Lexer) consumeHTMLLikeCommentToken(prevLineTerminator bool) bool {
347 c := l.r.Peek(0)
348 if c == '<' && l.r.Peek(1) == '!' && l.r.Peek(2) == '-' && l.r.Peek(3) == '-' {
349 // opening HTML-style single line comment
350 l.r.Move(4)
351 l.consumeSingleLineComment()
352 return true
353 } else if prevLineTerminator && c == '-' && l.r.Peek(1) == '-' && l.r.Peek(2) == '>' {
354 // closing HTML-style single line comment
355 // (only if current line didn't contain any meaningful tokens)
356 l.r.Move(3)
357 l.consumeSingleLineComment()
358 return true
359 }
360 return false
361}
362
363func (l *Lexer) consumeCommentToken() TokenType {
364 c := l.r.Peek(1)
365 if c == '/' {
366 // single line comment
367 l.r.Move(2)
368 l.consumeSingleLineComment()
369 return CommentToken
370 } else if c == '*' {
371 l.r.Move(2)
372 tt := CommentToken
373 for {
374 c := l.r.Peek(0)
375 if c == '*' && l.r.Peek(1) == '/' {
376 l.r.Move(2)
377 break
378 } else if c == 0 && l.r.Err() != nil {
379 break
380 } else if l.consumeLineTerminator() {
381 l.prevLineTerminator = true
382 tt = CommentLineTerminatorToken
383 } else {
384 l.r.Move(1)
385 }
386 }
387 return tt
388 }
389 return ErrorToken
390}
391
392var opTokens = map[byte]TokenType{
393 '=': EqToken,
394 '!': NotToken,
395 '<': LtToken,
396 '>': GtToken,
397 '+': AddToken,
398 '-': SubToken,
399 '*': MulToken,
400 '/': DivToken,
401 '%': ModToken,
402 '&': BitAndToken,
403 '|': BitOrToken,
404 '^': BitXorToken,
405 '~': BitNotToken,
406 '?': QuestionToken,
407}
408
409var opEqTokens = map[byte]TokenType{
410 '=': EqEqToken,
411 '!': NotEqToken,
412 '<': LtEqToken,
413 '>': GtEqToken,
414 '+': AddEqToken,
415 '-': SubEqToken,
416 '*': MulEqToken,
417 '/': DivEqToken,
418 '%': ModEqToken,
419 '&': BitAndEqToken,
420 '|': BitOrEqToken,
421 '^': BitXorEqToken,
422}
423
424var opOpTokens = map[byte]TokenType{
425 '<': LtLtToken,
426 '+': IncrToken,
427 '-': DecrToken,
428 '*': ExpToken,
429 '&': AndToken,
430 '|': OrToken,
431 '?': NullishToken,
432}
433
434var opOpEqTokens = map[byte]TokenType{
435 '<': LtLtEqToken,
436 '*': ExpEqToken,
437 '&': AndEqToken,
438 '|': OrEqToken,
439 '?': NullishEqToken,
440}
441
442func (l *Lexer) consumeOperatorToken() TokenType {
443 c := l.r.Peek(0)
444 l.r.Move(1)
445 if l.r.Peek(0) == '=' {
446 l.r.Move(1)
447 if l.r.Peek(0) == '=' && (c == '!' || c == '=') {
448 l.r.Move(1)
449 if c == '!' {
450 return NotEqEqToken
451 }
452 return EqEqEqToken
453 }
454 return opEqTokens[c]
455 } else if l.r.Peek(0) == c && (c == '+' || c == '-' || c == '*' || c == '&' || c == '|' || c == '?' || c == '<') {
456 l.r.Move(1)
457 if l.r.Peek(0) == '=' && c != '+' && c != '-' {
458 l.r.Move(1)
459 return opOpEqTokens[c]
460 }
461 return opOpTokens[c]
462 } else if c == '?' && l.r.Peek(0) == '.' && (l.r.Peek(1) < '0' || l.r.Peek(1) > '9') {
463 l.r.Move(1)
464 return OptChainToken
465 } else if c == '=' && l.r.Peek(0) == '>' {
466 l.r.Move(1)
467 return ArrowToken
468 } else if c == '>' && l.r.Peek(0) == '>' {
469 l.r.Move(1)
470 if l.r.Peek(0) == '>' {
471 l.r.Move(1)
472 if l.r.Peek(0) == '=' {
473 l.r.Move(1)
474 return GtGtGtEqToken
475 }
476 return GtGtGtToken
477 } else if l.r.Peek(0) == '=' {
478 l.r.Move(1)
479 return GtGtEqToken
480 }
481 return GtGtToken
482 }
483 return opTokens[c]
484}
485
486func (l *Lexer) consumeIdentifierToken() bool {
487 c := l.r.Peek(0)
488 if identifierStartTable[c] {
489 l.r.Move(1)
490 } else if 0xC0 <= c {
491 if r, n := l.r.PeekRune(0); unicode.IsOneOf(identifierStart, r) {
492 l.r.Move(n)
493 } else {
494 return false
495 }
496 } else if !l.consumeUnicodeEscape() {
497 return false
498 }
499 for {
500 c := l.r.Peek(0)
501 if identifierTable[c] {
502 l.r.Move(1)
503 } else if 0xC0 <= c {
504 if r, n := l.r.PeekRune(0); r == '\u200C' || r == '\u200D' || unicode.IsOneOf(identifierContinue, r) {
505 l.r.Move(n)
506 } else {
507 break
508 }
509 } else if !l.consumeUnicodeEscape() {
510 break
511 }
512 }
513 return true
514}
515
516func (l *Lexer) consumeNumericSeparator(f func() bool) bool {
517 if l.r.Peek(0) != '_' {
518 return false
519 }
520 l.r.Move(1)
521 if !f() {
522 l.r.Move(-1)
523 return false
524 }
525 return true
526}
527
528func (l *Lexer) consumeNumericToken() TokenType {
529 // assume to be on 0 1 2 3 4 5 6 7 8 9 .
530 first := l.r.Peek(0)
531 if first == '0' {
532 l.r.Move(1)
533 if l.r.Peek(0) == 'x' || l.r.Peek(0) == 'X' {
534 l.r.Move(1)
535 if l.consumeHexDigit() {
536 for l.consumeHexDigit() || l.consumeNumericSeparator(l.consumeHexDigit) {
537 }
538 return HexadecimalToken
539 }
540 l.err = parse.NewErrorLexer(l.r, "invalid hexadecimal number")
541 return ErrorToken
542 } else if l.r.Peek(0) == 'b' || l.r.Peek(0) == 'B' {
543 l.r.Move(1)
544 if l.consumeBinaryDigit() {
545 for l.consumeBinaryDigit() || l.consumeNumericSeparator(l.consumeBinaryDigit) {
546 }
547 return BinaryToken
548 }
549 l.err = parse.NewErrorLexer(l.r, "invalid binary number")
550 return ErrorToken
551 } else if l.r.Peek(0) == 'o' || l.r.Peek(0) == 'O' {
552 l.r.Move(1)
553 if l.consumeOctalDigit() {
554 for l.consumeOctalDigit() || l.consumeNumericSeparator(l.consumeOctalDigit) {
555 }
556 return OctalToken
557 }
558 l.err = parse.NewErrorLexer(l.r, "invalid octal number")
559 return ErrorToken
560 } else if l.r.Peek(0) == 'n' {
561 l.r.Move(1)
562 return BigIntToken
563 } else if '0' <= l.r.Peek(0) && l.r.Peek(0) <= '9' {
564 l.err = parse.NewErrorLexer(l.r, "legacy octal numbers are not supported")
565 return ErrorToken
566 }
567 } else if first != '.' {
568 for l.consumeDigit() || l.consumeNumericSeparator(l.consumeDigit) {
569 }
570 }
571 // we have parsed a 0 or an integer number
572 c := l.r.Peek(0)
573 if c == '.' {
574 l.r.Move(1)
575 if l.consumeDigit() {
576 for l.consumeDigit() || l.consumeNumericSeparator(l.consumeDigit) {
577 }
578 c = l.r.Peek(0)
579 } else if first == '.' {
580 // number starts with a dot and must be followed by digits
581 l.r.Move(-1)
582 return ErrorToken // may be dot or ellipsis
583 } else {
584 c = l.r.Peek(0)
585 }
586 } else if c == 'n' {
587 l.r.Move(1)
588 return BigIntToken
589 }
590 if c == 'e' || c == 'E' {
591 l.r.Move(1)
592 c = l.r.Peek(0)
593 if c == '+' || c == '-' {
594 l.r.Move(1)
595 }
596 if !l.consumeDigit() {
597 l.err = parse.NewErrorLexer(l.r, "invalid number")
598 return ErrorToken
599 }
600 for l.consumeDigit() || l.consumeNumericSeparator(l.consumeDigit) {
601 }
602 }
603 return DecimalToken
604}
605
606func (l *Lexer) consumeStringToken() TokenType {
607 // assume to be on ' or "
608 delim := l.r.Peek(0)
609 l.r.Move(1)
610 for {
611 c := l.r.Peek(0)
612 if c == delim {
613 l.r.Move(1)
614 break
615 } else if c == '\\' {
616 l.r.Move(1)
617 if !l.consumeLineTerminator() {
618 if c := l.r.Peek(0); c == delim || c == '\\' {
619 l.r.Move(1)
620 }
621 }
622 continue
623 } else if c == '\n' || c == '\r' || c == 0 && l.r.Err() != nil {
624 l.err = parse.NewErrorLexer(l.r, "unterminated string literal")
625 return ErrorToken
626 }
627 l.r.Move(1)
628 }
629 return StringToken
630}
631
632func (l *Lexer) consumeRegExpToken() bool {
633 // assume to be on /
634 l.r.Move(1)
635 inClass := false
636 for {
637 c := l.r.Peek(0)
638 if !inClass && c == '/' {
639 l.r.Move(1)
640 break
641 } else if c == '[' {
642 inClass = true
643 } else if c == ']' {
644 inClass = false
645 } else if c == '\\' {
646 l.r.Move(1)
647 if l.isLineTerminator() || l.r.Peek(0) == 0 && l.r.Err() != nil {
648 return false
649 }
650 } else if l.isLineTerminator() || c == 0 && l.r.Err() != nil {
651 return false
652 }
653 l.r.Move(1)
654 }
655 // flags
656 for {
657 c := l.r.Peek(0)
658 if identifierTable[c] {
659 l.r.Move(1)
660 } else if 0xC0 <= c {
661 if r, n := l.r.PeekRune(0); r == '\u200C' || r == '\u200D' || unicode.IsOneOf(identifierContinue, r) {
662 l.r.Move(n)
663 } else {
664 break
665 }
666 } else {
667 break
668 }
669 }
670 return true
671}
672
673func (l *Lexer) consumeTemplateToken() TokenType {
674 // assume to be on ` or } when already within template
675 continuation := l.r.Peek(0) == '}'
676 l.r.Move(1)
677 for {
678 c := l.r.Peek(0)
679 if c == '`' {
680 l.templateLevels = l.templateLevels[:len(l.templateLevels)-1]
681 l.r.Move(1)
682 if continuation {
683 return TemplateEndToken
684 }
685 return TemplateToken
686 } else if c == '$' && l.r.Peek(1) == '{' {
687 l.level++
688 l.r.Move(2)
689 if continuation {
690 return TemplateMiddleToken
691 }
692 return TemplateStartToken
693 } else if c == '\\' {
694 l.r.Move(1)
695 if c := l.r.Peek(0); c != 0 {
696 l.r.Move(1)
697 }
698 continue
699 } else if c == 0 && l.r.Err() != nil {
700 l.err = parse.NewErrorLexer(l.r, "unterminated template literal")
701 return ErrorToken
702 }
703 l.r.Move(1)
704 }
705}
706
707var identifierStartTable = [256]bool{
708 // ASCII
709 false, false, false, false, false, false, false, false,
710 false, false, false, false, false, false, false, false,
711 false, false, false, false, false, false, false, false,
712 false, false, false, false, false, false, false, false,
713
714 false, false, false, false, true, false, false, false, // $
715 false, false, false, false, false, false, false, false,
716 false, false, false, false, false, false, false, false,
717 false, false, false, false, false, false, false, false,
718
719 false, true, true, true, true, true, true, true, // A, B, C, D, E, F, G
720 true, true, true, true, true, true, true, true, // H, I, J, K, L, M, N, O
721 true, true, true, true, true, true, true, true, // P, Q, R, S, T, U, V, W
722 true, true, true, false, false, false, false, true, // X, Y, Z, _
723
724 false, true, true, true, true, true, true, true, // a, b, c, d, e, f, g
725 true, true, true, true, true, true, true, true, // h, i, j, k, l, m, n, o
726 true, true, true, true, true, true, true, true, // p, q, r, s, t, u, v, w
727 true, true, true, false, false, false, false, false, // x, y, z
728
729 // non-ASCII
730 false, false, false, false, false, false, false, false,
731 false, false, false, false, false, false, false, false,
732 false, false, false, false, false, false, false, false,
733 false, false, false, false, false, false, false, false,
734
735 false, false, false, false, false, false, false, false,
736 false, false, false, false, false, false, false, false,
737 false, false, false, false, false, false, false, false,
738 false, false, false, false, false, false, false, false,
739
740 false, false, false, false, false, false, false, false,
741 false, false, false, false, false, false, false, false,
742 false, false, false, false, false, false, false, false,
743 false, false, false, false, false, false, false, false,
744
745 false, false, false, false, false, false, false, false,
746 false, false, false, false, false, false, false, false,
747 false, false, false, false, false, false, false, false,
748 false, false, false, false, false, false, false, false,
749}
750
751var identifierTable = [256]bool{
752 // ASCII
753 false, false, false, false, false, false, false, false,
754 false, false, false, false, false, false, false, false,
755 false, false, false, false, false, false, false, false,
756 false, false, false, false, false, false, false, false,
757
758 false, false, false, false, true, false, false, false, // $
759 false, false, false, false, false, false, false, false,
760 true, true, true, true, true, true, true, true, // 0, 1, 2, 3, 4, 5, 6, 7
761 true, true, false, false, false, false, false, false, // 8, 9
762
763 false, true, true, true, true, true, true, true, // A, B, C, D, E, F, G
764 true, true, true, true, true, true, true, true, // H, I, J, K, L, M, N, O
765 true, true, true, true, true, true, true, true, // P, Q, R, S, T, U, V, W
766 true, true, true, false, false, false, false, true, // X, Y, Z, _
767
768 false, true, true, true, true, true, true, true, // a, b, c, d, e, f, g
769 true, true, true, true, true, true, true, true, // h, i, j, k, l, m, n, o
770 true, true, true, true, true, true, true, true, // p, q, r, s, t, u, v, w
771 true, true, true, false, false, false, false, false, // x, y, z
772
773 // non-ASCII
774 false, false, false, false, false, false, false, false,
775 false, false, false, false, false, false, false, false,
776 false, false, false, false, false, false, false, false,
777 false, false, false, false, false, false, false, false,
778
779 false, false, false, false, false, false, false, false,
780 false, false, false, false, false, false, false, false,
781 false, false, false, false, false, false, false, false,
782 false, false, false, false, false, false, false, false,
783
784 false, false, false, false, false, false, false, false,
785 false, false, false, false, false, false, false, false,
786 false, false, false, false, false, false, false, false,
787 false, false, false, false, false, false, false, false,
788
789 false, false, false, false, false, false, false, false,
790 false, false, false, false, false, false, false, false,
791 false, false, false, false, false, false, false, false,
792 false, false, false, false, false, false, false, false,
793}