Debian Codegraph @

modules/lsp-runtime-lib/src/main/fmt/json/Tokenizer.cpp

raw
1 /*
2 * Copyright (C) 2020 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2020 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-runtime-lib
6 * Created on: 14 окт. 2019 г.
7 *
8 * lsp-runtime-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-runtime-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-runtime-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #include <lsp-plug.in/stdlib/math.h>
23 #include <lsp-plug.in/fmt/json/Tokenizer.h>
24 #include <wctype.h>
25
26 namespace lsp
27 {
28 namespace json
29 {
30 static const char *ecma_reserved[] =
31 {
32 "Infinity",
33 "abstract",
34 "arguments",
35 "await",
36 "boolean",
37 "break",
38 "byte",
39 "case",
40 "catch",
41 "char",
42 "class",
43 "const",
44 "continue",
45 "debugger",
46 "default",
47 "delete",
48 "do",
49 "double",
50 "instanceof",
51 "interface",
52 "else",
53 "enum",
54 "eval",
55 "export",
56 "extends",
57 "false",
58 "final",
59 "finally",
60 "float",
61 "for",
62 "function",
63 "goto",
64 "if",
65 "implements",
66 "import",
67 "in",
68 "int",
69 "let",
70 "long",
71 "native",
72 "new",
73 "null",
74 "package",
75 "private",
76 "protected",
77 "public",
78 "return",
79 "short",
80 "static",
81 "super",
82 "switch",
83 "synchronized",
84 "this",
85 "throw",
86 "transient",
87 "try",
88 "true",
89 "typeof",
90 "var",
91 "void",
92 "volatile",
93 "while",
94 "with",
95 "yield"
96 };
97
98 Tokenizer::Tokenizer(io::IInSequence *in)
99 {
100 pIn = in;
101 cCurrent = -1;
102 enToken = JT_UNKNOWN;
103 nError = STATUS_OK;
104 fValue = 0;
105 iValue = 0;
106 nUnget = 0;
107 vPending = NULL;
108 nPending = 0;
109 nCapacity = 0;
110 }
111
112 Tokenizer::~Tokenizer()
113 {
114 pIn = NULL;
115 if (vPending != NULL)
116 {
117 ::free(vPending);
118 vPending = NULL;
119 }
120 nPending = 0;
121 nCapacity = 0;
122 }
123
124 token_t Tokenizer::set_error(status_t code)
125 {
126 nError = code;
127 return enToken = JT_ERROR;
128 }
129
130 lsp_swchar_t Tokenizer::skip_whitespace()
131 {
132 if (cCurrent < 0)
133 cCurrent = pIn->read();
134
135 while (true)
136 {
137 // Skip whitespace
138 if (::iswspace(cCurrent) || (::iswblank(cCurrent)))
139 cCurrent = pIn->read();
140 else
141 return cCurrent;
142 }
143 }
144
145 lsp_swchar_t Tokenizer::lookup()
146 {
147 if (cCurrent < 0)
148 cCurrent = pIn->read();
149 return cCurrent;
150 }
151
152 token_t Tokenizer::commit(token_t token)
153 {
154 if (cCurrent < 0)
155 return set_error(STATUS_BAD_STATE);
156
157 if (!sValue.append(cCurrent))
158 return set_error(STATUS_NO_MEM);
159
160 enToken = token;
161 cCurrent = -1;
162 return token;
163 }
164
165 lsp_swchar_t Tokenizer::commit_lookup(token_t token)
166 {
167 token_t x = commit(token);
168 return (x != JT_ERROR) ? lookup() : -1;
169 }
170
171 bool Tokenizer::is_identifier_start(lsp_wchar_t ch)
172 {
173 if (::iswupper(ch))
174 return true;
175 if (::iswlower(ch))
176 return true;
177 return (ch == '_') || (ch == '$') || (ch == '\\');
178 }
179
180 bool Tokenizer::is_identifier(lsp_wchar_t ch)
181 {
182 if (::iswupper(ch))
183 return true;
184 if (::iswlower(ch))
185 return true;
186 if (::iswdigit(ch))
187 return true;
188 return (ch == '_') || (ch == '$');
189 }
190
191 bool Tokenizer::is_reserved_word(const LSPString *text)
192 {
193 ssize_t first = 0, last = sizeof(ecma_reserved) / sizeof (const char *) - 1;
194 while (first <= last)
195 {
196 ssize_t center = (first + last) >> 1;
197 int cmp = text->compare_to_ascii(ecma_reserved[center]);
198 if (cmp < 0)
199 last = center - 1;
200 else if (cmp > 0)
201 first = center + 1;
202 else
203 return true;
204 }
205
206 return false;
207 }
208
209 bool Tokenizer::is_valid_identifier(const LSPString *text)
210 {
211 size_t len = text->length();
212 if (len <= 0)
213 return false;
214 if (!is_identifier_start(text->char_at(0)))
215 return false;
216 for (size_t i=1; i<len; ++i)
217 {
218 if (!is_identifier(text->char_at(i)))
219 return false;
220 }
221
222 return !is_reserved_word(text);
223 }
224
225 bool Tokenizer::parse_digit(int *digit, lsp_wchar_t ch, int radix)
226 {
227 int res;
228 if ((ch >= '0') && (ch <= '9'))
229 res = ch - '0';
230 else if ((ch >= 'a') && (ch <= 'f'))
231 res = ch - 'a' + 10;
232 else if ((ch >= 'A') && (ch <= 'F'))
233 res = ch - 'A' + 10;
234 else
235 return false;
236
237 if (res >= radix)
238 return false;
239
240 *digit = res;
241 return true;
242 }
243
244 status_t Tokenizer::add_pending_character(lsp_utf16_t ch)
245 {
246 if (nPending >= nCapacity)
247 {
248 size_t cap = ((nPending + 0x10) & ~0xf) + nCapacity;
249 lsp_utf16_t *buf = reinterpret_cast<lsp_utf16_t *>(::realloc(vPending, sizeof(lsp_utf16_t) * cap));
250 if (buf == NULL)
251 return STATUS_NO_MEM;
252 vPending = buf;
253 }
254
255 vPending[nPending++] = ch;
256 return STATUS_OK;
257 }
258
259 status_t Tokenizer::commit_pending_characters()
260 {
261 // Are there any pending characters?
262 if (nPending <= 0)
263 return STATUS_OK;
264
265 // Initialize temporary string
266 LSPString tmp;
267 bool res = tmp.set_utf16(vPending, nPending);
268 nPending = 0;
269 if (!res)
270 return STATUS_BAD_FORMAT;
271
272 // Append string to result
273 return (sValue.append(&tmp)) ? STATUS_OK : STATUS_NO_MEM;
274 }
275
276 token_t Tokenizer::parse_string(token_t type)
277 {
278 sValue.clear();
279 skip(type);
280
281 status_t res;
282 bool protector = false;
283
284 // Parse string
285 while (true)
286 {
287 // Read character
288 lsp_swchar_t c = lookup();
289 if (c < 0)
290 return set_error(-c);
291
292 if (protector)
293 {
294 protector = false;
295
296 if ((c == 'u') || (c == 'U'))
297 {
298 token_t tok = parse_unicode_escape_sequence(type);
299 if (tok == JT_ERROR)
300 return tok;
301 }
302 else if ((c == 'x') || (c == 'X'))
303 {
304 token_t tok = parse_hexadecimal_escape_sequence(type);
305 if (tok == JT_ERROR)
306 return tok;
307 }
308 else
309 {
310 // Commit previously pending unicode characters
311 if ((res = commit_pending_characters()) != STATUS_OK)
312 return set_error(res);
313
314 switch (c)
315 {
316 // Escaped characters: ' " \ b f n r t v
317 case 'b': c = '\b'; break;
318 case 'f': c = '\f'; break;
319 case 'n': c = '\n'; break;
320 case 'r': c = '\r'; break;
321 case 't': c = '\t'; break;
322 case 'v': c = '\v'; break;
323 case '0': c = '\0'; break;
324 case '\n':
325 skip(type); // Skip end of line
326 c = lookup();
327 if (c < 0)
328 {
329 if (c != -STATUS_EOF)
330 return set_error(-c);
331 }
332 else if (c == '\r')
333 skip(type);
334 c = -1;
335 break;
336
337 case '\r':
338 case 0x2028:
339 case 0x2029:
340 skip(type);
341 c = -1; // Line terminator
342 break;
343
344 default: // Any other characters just omit the protector character in ECMA script
345 break;
346 }
347
348 // Append character (if it is present)
349 if (c >= 0)
350 {
351 if (!sValue.append(c))
352 return set_error(STATUS_NO_MEM);
353 skip(type);
354 }
355 }
356 }
357 else if (c == '\\')
358 {
359 skip(type);
360 protector = true;
361 }
362 else
363 {
364 // Commit previously pending unicode characters
365 if ((res = commit_pending_characters()) != STATUS_OK)
366 return set_error(res);
367
368 switch (c)
369 {
370 case '\n':
371 return set_error(STATUS_BAD_TOKEN);
372 case '\"':
373 if (type == JT_DQ_STRING)
374 return skip(type);
375 break;
376 case '\'':
377 if (type == JT_SQ_STRING)
378 return skip(type);
379 break;
380 default:
381 break;
382 }
383
384 if ((type = commit(type)) == JT_ERROR)
385 return type;
386 }
387 }
388
389 return enToken = type;
390 }
391
392 token_t Tokenizer::parse_unicode_escape_sequence(token_t type)
393 {
394 // Seek for 'u' or 'U'
395 lsp_swchar_t c = lookup();
396 if (c < 0)
397 return set_error(-c);
398 else if ((c != 'u') && (c != 'U'))
399 return set_error(STATUS_BAD_TOKEN);
400 skip(type);
401
402 // Read 4 mandatory digits
403 int digit = 0;
404 lsp_utf16_t cp = 0;
405 for (size_t i=0; i<4; ++i)
406 {
407 c = lookup();
408 if (c < 0)
409 return set_error(-c);
410
411 skip(type);
412 if (!parse_digit(&digit, c, 16))
413 return set_error(STATUS_BAD_TOKEN);
414
415 cp = (cp << 4) + digit;
416 }
417
418 // All is fine, store, truncate value and return result
419 status_t res = add_pending_character(cp);
420 return (res != STATUS_OK) ? set_error(res) : enToken = type;
421 }
422
423 token_t Tokenizer::parse_hexadecimal_escape_sequence(token_t type)
424 {
425 // Seek for 'u' or 'U'
426 lsp_swchar_t c = lookup();
427 if (c < 0)
428 return set_error(-c);
429 else if ((c != 'x') && (c != 'X'))
430 return set_error(STATUS_BAD_TOKEN);
431 skip(type);
432
433 // Read 4 mandatory digits
434 int digit = 0;
435 size_t cp = 0;
436 for (size_t i=0; i<2; ++i)
437 {
438 c = lookup();
439 if (c < 0)
440 return set_error(-c);
441
442 skip(type);
443 if (!parse_digit(&digit, c, 16))
444 return set_error(STATUS_BAD_TOKEN);
445
446 cp = (cp << 4) + digit;
447 }
448
449 // All is fine, store, truncate value and return result
450 status_t res = add_pending_character(cp);
451 return (res != STATUS_OK) ? set_error(res) : enToken = type;
452 }
453
454 token_t Tokenizer::parse_identifier()
455 {
456 // Commit identifier's start character
457 status_t res;
458 token_t tok;
459
460 while (true)
461 {
462 // Read character
463 lsp_swchar_t c = lookup();
464 if (c < 0)
465 {
466 if (c != -STATUS_EOF)
467 return set_error(-c);
468
469 // Commit pending characters first
470 res = commit_pending_characters();
471 if (res != STATUS_OK)
472 return set_error(res);
473 break;
474 }
475
476 // Check that character is an identifier
477 if (!is_identifier(c))
478 {
479 if (c != '\\')
480 break;
481
482 skip(JT_IDENTIFIER);
483 tok = parse_unicode_escape_sequence(JT_IDENTIFIER);
484 }
485 else
486 {
487 // Commit pending unicode characters first
488 res = commit_pending_characters();
489 if (res != STATUS_OK)
490 return set_error(res);
491
492 // Now, commit pending character
493 tok = commit(JT_IDENTIFIER);
494 }
495
496 if (tok == JT_ERROR)
497 return tok;
498 }
499
500 // Analyze identifier
501 if (sValue.equals_ascii("true"))
502 return enToken = JT_TRUE;
503 else if (sValue.equals_ascii("false"))
504 return enToken = JT_FALSE;
505 else if (sValue.equals_ascii("null"))
506 return enToken = JT_NULL;
507 else if (sValue.equals_ascii("NaN"))
508 {
509 fValue = NAN;
510 return enToken = JT_DOUBLE;
511 }
512 else if (sValue.equals_ascii("Infinity"))
513 {
514 fValue = INFINITY;
515 return enToken = JT_DOUBLE;
516 }
517 else if (is_reserved_word(&sValue))
518 return enToken = JT_RESERVED;
519
520 return enToken = JT_IDENTIFIER;
521 }
522
523 token_t Tokenizer::parse_single_line_comment()
524 {
525 // Reset the length of string literal
526 sValue.set_length(0);
527 skip(JT_SL_COMMENT);
528
529 while (true)
530 {
531 // Read character
532 lsp_swchar_t c = lookup();
533 if (c < 0)
534 return (c == -STATUS_EOF) ? JT_SL_COMMENT : set_error(-c);
535
536 // Analyze character
537 switch (c) {
538 case '\n':
539 {
540 status_t res = commit_pending_characters();
541 return (res == STATUS_OK) ? skip(JT_SL_COMMENT) : set_error(STATUS_BAD_TOKEN);
542 }
543 case '\\':
544 {
545 skip(JT_SL_COMMENT);
546 token_t tok = parse_unicode_escape_sequence(JT_SL_COMMENT);
547 if (tok == JT_ERROR)
548 return tok;
549 break;
550 }
551 default:
552 {
553 status_t res = commit_pending_characters();
554 if (res != STATUS_OK)
555 return set_error(STATUS_BAD_TOKEN);
556
557 token_t tok = commit(JT_SL_COMMENT);
558 if (tok == JT_ERROR)
559 return tok;
560 break;
561 }
562 }
563 }
564
565 return JT_UNKNOWN;
566 }
567
568 token_t Tokenizer::parse_multiline_comment()
569 {
570 // Reset the length of string literal
571 sValue.set_length(0);
572 skip(JT_SL_COMMENT);
573
574 lsp_swchar_t last = -1;
575 while (true)
576 {
577 // Read character
578 lsp_swchar_t c = lookup();
579 if (c < 0)
580 return set_error(-c);
581
582 // Check state
583 if (last == '*')
584 {
585 if (c == '/')
586 {
587 // Remove last stored character (asterisk)
588 sValue.remove_last();
589 return skip(JT_ML_COMMENT);
590 }
591 }
592 else if (last == '\n')
593 {
594 if (c == '\r')
595 {
596 skip(JT_ML_COMMENT);
597 continue;
598 }
599 last = -1;
600 }
601
602 // Append current character to the comment
603 switch (c) {
604 case '\\':
605 {
606 skip(JT_ML_COMMENT);
607 token_t tok = parse_unicode_escape_sequence(JT_ML_COMMENT);
608 if (tok == JT_ERROR)
609 return tok;
610 break;
611 }
612
613 default:
614 {
615 status_t res = commit_pending_characters();
616 if (res != STATUS_OK)
617 return set_error(STATUS_BAD_TOKEN);
618
619 token_t tok = commit(JT_ML_COMMENT);
620 if (tok == JT_ERROR)
621 return tok;
622 }
623 }
624
625 // Remember last character
626 last = c;
627 }
628
629 return JT_UNKNOWN;
630 }
631
632 token_t Tokenizer::parse_number()
633 {
634 enum flags_t
635 {
636 F_NEGATIVE = 1 << 0,
637 F_SIGN = 1 << 1,
638 F_INTEGER = 1 << 2,
639 F_INT = 1 << 3,
640 F_DOT = 1 << 4,
641 F_FRAC = 1 << 5,
642 F_EXP = 1 << 6,
643 F_ESIGN = 1 << 7,
644 F_ENEGATIVE = 1 << 8
645 };
646
647 lsp_swchar_t c = lookup();
648 size_t flags = 0;
649 int radix = 10;
650 int digit = 0;
651 ssize_t ivalue = 0;
652 double ifrac = 0;
653 double ifpow = 1.0;
654 double rradix = 0.1;
655 ssize_t iexp = 0;
656
657 // Has sign?
658 if (c == '-')
659 {
660 flags |= F_NEGATIVE | F_SIGN;
661 c = commit_lookup(JT_UNKNOWN);
662 }
663 else if (c == '+')
664 {
665 flags |= F_SIGN;
666 c = commit_lookup(JT_UNKNOWN);
667 }
668
669 // Has prefix ?
670 if (c == '0')
671 {
672 c = commit_lookup(JT_UNKNOWN);
673 switch (c)
674 {
675 case 'x': // hexadecimal
676 case 'X':
677 radix = 16;
678 rradix = 0.0625;
679 c = commit_lookup(JT_UNKNOWN);
680 flags |= F_INTEGER;
681 break;
682 default:
683 flags |= F_INT;
684 break;
685 }
686 }
687 else if ((c == 'I') || (c == 'N')) // Infinity or NaN?
688 {
689 LSPString tmp;
690 tmp.swap(&sValue);
691 token_t tok = parse_identifier();
692 if (!tmp.append(&sValue))
693 return set_error(STATUS_NO_MEM);
694 sValue.swap(&tmp);
695 if (tok != JT_DOUBLE)
696 return enToken = JT_UNKNOWN;
697 if (flags & F_NEGATIVE)
698 fValue = -fValue;
699 return tok;
700 }
701
702 // Read the integer part
703 while (parse_digit(&digit, c, radix))
704 {
705 ivalue = ivalue*radix + digit;
706 flags |= F_INT;
707 c = commit_lookup(JT_DECIMAL);
708 }
709
710 // Is integer only?
711 if (flags & F_INTEGER)
712 {
713 if (!(flags & F_INT)) // There should be at least one integer character
714 return enToken = JT_UNKNOWN;
715 iValue = (flags & F_NEGATIVE) ? -ivalue : ivalue;
716 return enToken = (radix != 16) ? JT_DECIMAL : JT_HEXADECIMAL;
717 }
718
719 // Has a fraction part?
720 if (c == '.')
721 {
722 flags |= F_DOT;
723 c = commit_lookup(JT_DOUBLE);
724
725 while (parse_digit(&digit, c, radix))
726 {
727 ifpow *= rradix;
728 ifrac += digit * ifpow;
729 flags |= F_FRAC;
730 c = commit_lookup(JT_DOUBLE);
731 }
732 }
733
734 // Is there at least INT or FRAC part defined?
735 if ((flags & (F_INT | F_FRAC)) == 0)
736 return enToken = JT_UNKNOWN;
737
738 // Has an exponent part?
739 if ((c == 'e') || (c == 'E'))
740 {
741 c = commit_lookup(JT_DOUBLE);
742
743 // Has sign?
744 if (c == '-')
745 {
746 flags |= F_ENEGATIVE | F_ESIGN;
747 c = commit_lookup(JT_UNKNOWN);
748 }
749 else if (c == '+')
750 {
751 flags |= F_ESIGN;
752 c = commit_lookup(JT_UNKNOWN);
753 }
754
755 // Parse exponent
756 while (parse_digit(&digit, c, radix))
757 {
758 iexp = iexp*radix + digit;
759 flags |= F_EXP;
760 c = commit_lookup(JT_DOUBLE);
761 }
762
763 // Analyze post-condition: if exponent sign is defined,
764 // the exponent value also should be defined
765 if ((flags & (F_ESIGN | F_EXP)) == F_ESIGN)
766 return enToken = JT_UNKNOWN;
767 else if (flags & F_ENEGATIVE)
768 iexp = -iexp;
769 }
770
771 // Ensure that the next character is not an identifier character (ECMA)
772 c = lookup();
773 if (c < 0)
774 {
775 if (c != (-STATUS_EOF))
776 return set_error(-c);
777 }
778 else if (is_identifier_start(c))
779 return set_error(STATUS_BAD_FORMAT);
780
781 // Now analyze parsing state
782 if ((flags & (F_INT | F_FRAC | F_EXP | F_DOT)) == F_INT)
783 {
784 iValue = (flags & F_NEGATIVE) ? -ivalue : ivalue;
785 return enToken = JT_DECIMAL;
786 }
787
788 // Form the floating-point value
789 double fv = (double(ivalue) + ifrac) * pow(radix, iexp);
790 fValue = (flags & F_NEGATIVE) ? -fv : fv;
791 return enToken = JT_DOUBLE;
792 }
793
794 token_t Tokenizer::get_token(bool get)
795 {
796 // Pre-checks
797 if (!get)
798 return enToken;
799 else if (nUnget > 0)
800 {
801 --nUnget;
802 return enToken;
803 }
804
805 // Skip whitespaces
806 lsp_swchar_t c = skip_whitespace();
807 if (c < 0)
808 {
809 enToken = (c == -STATUS_EOF) ? JT_EOF : JT_ERROR;
810 nError = -c;
811 return enToken;
812 }
813
814 sValue.set_length(0);
815
816 switch (c)
817 {
818 case '[': return commit(JT_LQ_BRACE);
819 case ']': return commit(JT_RQ_BRACE);
820 case '{': return commit(JT_LC_BRACE);
821 case '}': return commit(JT_RC_BRACE);
822 case ':': return commit(JT_COLON);
823 case ',': return commit(JT_COMMA);
824 case '\'': return parse_string(JT_SQ_STRING);
825 case '\"': return parse_string(JT_DQ_STRING);
826 case '/':
827 c = commit_lookup(JT_UNKNOWN);
828 if (c == '/')
829 return parse_single_line_comment();
830 else if (c == '*')
831 return parse_multiline_comment();
832 commit(JT_UNKNOWN);
833 break;
834
835 default:
836 if (is_identifier_start(c))
837 return parse_identifier();
838 return parse_number();
839 }
840
841 return enToken = JT_UNKNOWN;
842 }
843
844 } /* namespace json */
845 } /* namespace lsp */