Debian Codegraph @

modules/lsp-runtime-lib/src/main/expr/Tokenizer.cpp

raw
1 /*
2 * Copyright (C) 2023 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2023 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-runtime-lib
6 * Created on: 16 сент. 2019 г.
7 *
8 * lsp-runtime-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-runtime-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-runtime-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #include <lsp-plug.in/expr/Tokenizer.h>
23 #include <lsp-plug.in/io/charset.h>
24 #include <lsp-plug.in/stdlib/math.h>
25
26 namespace lsp
27 {
28 namespace expr
29 {
30 typedef struct bareword_t {
31 const char *text;
32 token_t token;
33 } bareword_t;
34
35 static const bareword_t barewords[] =
36 {
37 { "abs", TT_ABS },
38 { "acos", TT_ACOS },
39 { "add", TT_ADDSYM },
40 { "and", TT_AND },
41 { "arccos", TT_ACOS },
42 { "arcsin", TT_ASIN },
43 { "arctg", TT_ATAN },
44 { "asin", TT_ASIN },
45 { "atan", TT_ATAN },
46 { "band", TT_BAND },
47 { "bnot", TT_BNOT },
48 { "bool", TT_BOOL },
49 { "bor", TT_BOR },
50 { "bxor", TT_BXOR },
51 { "cmp", TT_CMP },
52 { "cos", TT_COS },
53 { "db", TT_DB },
54 { "deg", TT_DEG },
55 { "div", TT_DIV },
56 { "e", TT_E },
57 { "eq", TT_EQ },
58 { "ex", TT_EX },
59 { "exp", TT_EXP },
60 { "false", TT_FALSE },
61 { "float", TT_FLOAT },
62 { "fmod", TT_FMOD },
63 { "fp", TT_FLOAT },
64 { "ge", TT_GREATER_EQ },
65 { "gt", TT_GREATER },
66 { "iadd", TT_IADD },
67 { "icmp", TT_ICMP },
68 { "idiv", TT_IDIV },
69 { "ie", TT_IEQ },
70 { "ieq", TT_IEQ },
71 { "ige", TT_IGREATER_EQ },
72 { "igt", TT_IGREATER },
73 { "ile", TT_ILESS_EQ },
74 { "ilt", TT_ILESS },
75 { "imod", TT_IMOD },
76 { "imul", TT_IMUL },
77 { "ine", TT_INOT_EQ },
78 { "inge", TT_ILESS },
79 { "ingt", TT_ILESS_EQ },
80 { "inle", TT_IGREATER },
81 { "inlt", TT_IGREATER_EQ },
82 { "int", TT_INT },
83 { "isub", TT_ISUB },
84 { "lc", TT_SLWR },
85 { "le", TT_LESS_EQ },
86 { "lg", TT_LOGD },
87 { "ln", TT_LOGE },
88 { "log", TT_LOGE },
89 { "log10", TT_LOGD },
90 { "log2", TT_LOG2 },
91 { "logb", TT_LOG2 },
92 { "logd", TT_LOGD },
93 { "loge", TT_LOGE },
94 { "lt", TT_LESS },
95 { "mod", TT_IMOD },
96 { "mul", TT_MUL },
97 { "ne", TT_NOT_EQ },
98 { "nge", TT_LESS },
99 { "ngt", TT_LESS_EQ },
100 { "nle", TT_GREATER },
101 { "nlt", TT_GREATER_EQ },
102 { "not", TT_NOT },
103 { "null", TT_NULL },
104 { "or", TT_OR },
105 { "pi", TT_PI },
106 { "pow", TT_POW },
107 { "rad", TT_RAD },
108 { "sc", TT_SCAT },
109 { "scat", TT_SCAT },
110 { "sin", TT_SIN },
111 { "sl", TT_SLEN },
112 { "slen", TT_SLEN },
113 { "slwr", TT_SLWR },
114 { "sqrt", TT_SQRT },
115 { "sr", TT_SREP },
116 { "srep", TT_SREP },
117 { "srev", TT_SREV },
118 { "str", TT_STR },
119 { "sub", TT_SUBSYM },
120 { "supr", TT_SUPR },
121 { "tan", TT_TAN },
122 { "tg", TT_TAN },
123 { "true", TT_TRUE },
124 { "uc", TT_SUPR },
125 { "undef", TT_UNDEF },
126 { "xor", TT_XOR }
127 };
128
129 Tokenizer::Tokenizer(io::IInSequence *in)
130 {
131 pIn = in;
132 cCurrent = -1;
133 enToken = TT_UNKNOWN;
134 nError = STATUS_OK;
135 fValue = 0;
136 iValue = 0;
137 nUnget = 0;
138 }
139
140 Tokenizer::~Tokenizer()
141 {
142 pIn = NULL;
143 }
144
145 lsp_swchar_t Tokenizer::lookup()
146 {
147 if (cCurrent < 0)
148 cCurrent = pIn->read();
149 return cCurrent;
150 }
151
152 token_t Tokenizer::set_error(status_t code)
153 {
154 nError = code;
155 return enToken = TT_ERROR;
156 }
157
158 token_t Tokenizer::commit(token_t token)
159 {
160 if (cCurrent < 0)
161 return set_error(STATUS_BAD_STATE);
162
163 if (!sValue.append(cCurrent))
164 return set_error(STATUS_NO_MEM);
165
166 enToken = token;
167 cCurrent = -1;
168 return token;
169 }
170
171 lsp_swchar_t Tokenizer::commit_lookup(token_t token)
172 {
173 token_t x = commit(token);
174 return (x != TT_ERROR) ? lookup() : -1;
175 }
176
177 lsp_swchar_t Tokenizer::skip_whitespace()
178 {
179 if (cCurrent < 0)
180 cCurrent = pIn->read();
181
182 while (true)
183 {
184 // Skip whitespace
185 switch (cCurrent)
186 {
187 case ' ':
188 case '\t':
189 case '\n':
190 case '\r':
191 cCurrent = pIn->read();
192 continue;
193 default:
194 return cCurrent;
195 }
196 }
197 }
198
199 bool Tokenizer::is_identifier_first(lsp_wchar_t ch)
200 {
201 if ((ch >= 'a') && (ch <= 'z'))
202 return true;
203 if ((ch >= 'A') && (ch <= 'Z'))
204 return true;
205 return ch == '_';
206 }
207
208 bool Tokenizer::is_hex(lsp_wchar_t ch)
209 {
210 if ((ch >= '0') && (ch <= '9'))
211 return true;
212 if ((ch >= 'a') && (ch <= 'f'))
213 return true;
214 if ((ch >= 'A') && (ch <= 'F'))
215 return true;
216 return false;
217 }
218
219 bool Tokenizer::is_identifier_next(lsp_wchar_t ch)
220 {
221 if ((ch >= 'a') && (ch <= 'z'))
222 return true;
223 if ((ch >= 'A') && (ch <= 'Z'))
224 return true;
225 if ((ch >= '0') && (ch <= '9'))
226 return true;
227 return ch == '_';
228 }
229
230 bool Tokenizer::parse_digit(int *digit, lsp_wchar_t ch, int radix)
231 {
232 int res;
233 if ((ch >= '0') && (ch <= '9'))
234 res = ch - '0';
235 else if ((ch >= 'a') && (ch <= 'f'))
236 res = ch - 'a' + 10;
237 else if ((ch >= 'A') && (ch <= 'F'))
238 res = ch - 'A' + 10;
239 else if (ch == '_')
240 res = -1;
241 else
242 return false;
243
244 if (res >= radix)
245 return false;
246
247 *digit = res;
248 return true;
249 }
250
251 token_t Tokenizer::lookup_identifier(token_t type)
252 {
253 if (cCurrent < 0)
254 cCurrent = pIn->read();
255
256 if (!is_identifier_first(cCurrent))
257 return enToken;
258
259 sValue.clear();
260 do
261 {
262 // Append character
263 if (!sValue.append(cCurrent))
264 return set_error(STATUS_NO_MEM);
265
266 // Read next character
267 if ((cCurrent = pIn->read()) < 0)
268 {
269 if (cCurrent == -STATUS_EOF)
270 break;
271 return set_error(-cCurrent);
272 }
273 } while (is_identifier_next(cCurrent));
274
275 return enToken = type;
276 }
277
278 token_t Tokenizer::lookup_string()
279 {
280 sValue.clear();
281
282 while (cCurrent == '\'')
283 {
284 bool protector = false;
285 bool scan = true;
286
287 // Parse string
288 do
289 {
290 // Read character
291 if ((cCurrent = pIn->read()) < 0)
292 return set_error(-cCurrent);
293
294 if (protector)
295 {
296 switch (cCurrent)
297 {
298 case '\\':
299 if (!sValue.append('\\'))
300 return set_error(STATUS_NO_MEM);
301 break;
302 case 'n':
303 if (!sValue.append('\n'))
304 return set_error(STATUS_NO_MEM);
305 break;
306 case 't':
307 if (!sValue.append('\t'))
308 return set_error(STATUS_NO_MEM);
309 break;
310 case 'r':
311 if (!sValue.append('\r'))
312 return set_error(STATUS_NO_MEM);
313 break;
314 case '\'':
315 if (!sValue.append('\''))
316 return set_error(STATUS_NO_MEM);
317 break;
318 default:
319 if (!sValue.append('\\'))
320 return set_error(STATUS_NO_MEM);
321 if (!sValue.append(cCurrent))
322 return set_error(STATUS_NO_MEM);
323 break;
324 }
325 protector = false;
326 }
327 else
328 {
329 switch (cCurrent)
330 {
331 case '\\':
332 protector = true;
333 break;
334 case '\'':
335 scan = false;
336 cCurrent = -1;
337 break;
338 default:
339 if (!sValue.append(cCurrent))
340 return set_error(STATUS_NO_MEM);
341 break;
342 }
343 }
344 } while (scan);
345
346 // Skip whitespace after string
347 skip_whitespace();
348 }
349
350 return enToken = TT_STRING;
351 }
352
353 token_t Tokenizer::lookup_color()
354 {
355 if (cCurrent < 0)
356 cCurrent = pIn->read();
357
358 if (!is_hex(cCurrent))
359 return enToken;
360
361 do {
362 // Append character
363 if (!sValue.append(cCurrent))
364 return set_error(STATUS_NO_MEM);
365
366 // Read next character
367 if ((cCurrent = pIn->read()) < 0)
368 {
369 if (cCurrent == -STATUS_EOF)
370 break;
371 return set_error(-cCurrent);
372 }
373 } while (is_hex(cCurrent));
374
375 return enToken = TT_COLOR;
376 }
377
378 token_t Tokenizer::lookup_number()
379 {
380 enum flags_t
381 {
382 F_NEGATIVE = 1 << 0,
383 F_SIGN = 1 << 1,
384 F_INT = 1 << 2,
385 F_DOT = 1 << 3,
386 F_FRAC = 1 << 4,
387 F_EXP = 1 << 5,
388 F_ESIGN = 1 << 6,
389 F_ENEGATIVE = 1 << 7
390 };
391
392 lsp_swchar_t c = lookup();
393 size_t flags = 0;
394 int radix = 10;
395 int digit = 0;
396 ssize_t ivalue = 0;
397 double ifrac = 0;
398 double ifpow = 1.0;
399 double rradix = 0.1;
400 ssize_t iexp = 0;
401
402 // Has sign?
403 if (c == '-')
404 {
405 flags |= F_NEGATIVE | F_SIGN;
406 c = commit_lookup(TT_UNKNOWN);
407 }
408 else if (c == '+')
409 {
410 flags |= F_SIGN;
411 c = commit_lookup(TT_UNKNOWN);
412 }
413
414 // Has prefix ?
415 if (c == '0')
416 {
417 c = commit_lookup(TT_UNKNOWN);
418 switch (c)
419 {
420 case 'b': // binary
421 case 'B':
422 radix = 2;
423 rradix = 0.5;
424 c = commit_lookup(TT_UNKNOWN);
425 break;
426 case 'o': // octal
427 case 'O':
428 radix = 8;
429 rradix = 0.125;
430 c = commit_lookup(TT_UNKNOWN);
431 break;
432 case 'd': // decimal
433 case 'D':
434 // radix = 10; // assigned above
435 // rradix = 0.1; // assigned above
436 c = commit_lookup(TT_UNKNOWN);
437 break;
438 case 'x': // hexadecimal
439 case 'X':
440 radix = 16;
441 rradix = 0.0625;
442 c = commit_lookup(TT_UNKNOWN);
443 break;
444 default:
445 flags |= F_INT;
446 break;
447 }
448 }
449
450 // Read the integer part
451 while (parse_digit(&digit, c, radix))
452 {
453 if (digit < 0) // Skip separator
454 {
455 if (!(flags & F_INT)) // Allow separators only after the first digit
456 return commit(TT_UNKNOWN);
457 }
458 else
459 {
460 ivalue = ivalue*radix + digit;
461 flags |= F_INT;
462 }
463 c = commit_lookup(TT_IVALUE);
464 }
465
466 // Has a fraction part?
467 if (c == '.')
468 {
469 flags |= F_DOT;
470 c = commit_lookup(TT_FVALUE);
471
472 while (parse_digit(&digit, c, radix))
473 {
474 if (digit < 0) // Skip separator
475 {
476 if (!(flags & F_FRAC)) // Allow separators only after the first digit
477 return commit(TT_UNKNOWN);
478 }
479 else
480 {
481 ifpow *= rradix;
482 ifrac += digit * ifpow;
483 flags |= F_FRAC;
484 }
485 c = commit_lookup(TT_FVALUE);
486 }
487 }
488
489 // Is there at least INT or FRAC part defined?
490 if ((flags & (F_INT | F_FRAC)) == 0)
491 {
492 if ((flags & (F_SIGN | F_DOT)) == F_SIGN)
493 return enToken = (flags & F_NEGATIVE) ? TT_SUB : TT_ADD;
494 return enToken = TT_UNKNOWN;
495 }
496
497 // Has an exponent part?
498 if ((c == 'e') || (c == 'E') || (c == 'p') || (c == 'P'))
499 {
500 c = commit_lookup(TT_FVALUE);
501
502 // Has sign?
503 if (c == '-')
504 {
505 flags |= F_ENEGATIVE | F_ESIGN;
506 c = commit_lookup(TT_UNKNOWN);
507 }
508 else if (c == '+')
509 {
510 flags |= F_ESIGN;
511 c = commit_lookup(TT_UNKNOWN);
512 }
513
514 // Parse exponent
515 while (parse_digit(&digit, c, radix))
516 {
517 if (digit < 0) // Skip separator
518 {
519 if (!(flags & F_EXP)) // Allow separators only after the first digit
520 return commit(TT_UNKNOWN);
521 }
522 else
523 {
524 iexp = iexp*radix + digit;
525 flags |= F_EXP;
526 }
527 c = commit_lookup(TT_FVALUE);
528 }
529
530 // Analyze post-condition: if exponent sign is defined,
531 // the exponent value also should be defined
532 if ((flags & (F_ESIGN | F_EXP)) == F_ESIGN)
533 return enToken = TT_UNKNOWN;
534 else if (flags & F_ENEGATIVE)
535 iexp = -iexp;
536 }
537
538 // Now analyze parsing state
539 if ((flags & (F_INT | F_FRAC | F_EXP | F_DOT)) == F_INT)
540 {
541 iValue = (flags & F_NEGATIVE) ? -ivalue : ivalue;
542 return enToken = TT_IVALUE;
543 }
544
545 // Form the floating-point value
546 double fv = (double(ivalue) + ifrac) * pow(radix, iexp);
547 fValue = (flags & F_NEGATIVE) ? -fv : fv;
548 return enToken = TT_FVALUE;
549 }
550
551 token_t Tokenizer::decode_bareword()
552 {
553 const char *text = sValue.get_utf8();
554 ssize_t first = 0, last = sizeof(barewords)/sizeof(bareword_t) - 1;
555 while (first <= last)
556 {
557 ssize_t center = (first + last) >> 1;
558 const bareword_t *bw = &barewords[center];
559 int cmp = ::strcasecmp(text, bw->text);
560
561 if (cmp < 0)
562 last = center - 1;
563 else if (cmp > 0)
564 first = center + 1;
565 else
566 {
567 enToken = bw->token;
568 break;
569 }
570 }
571
572 return enToken;
573 }
574
575 token_t Tokenizer::get_token(size_t flags)
576 {
577 // Pre-checks
578 if (!(flags & TF_GET))
579 return enToken;
580 else if (nUnget > 0)
581 {
582 --nUnget;
583 return enToken;
584 }
585
586 // Skip whitespaces
587 lsp_swchar_t c = skip_whitespace();
588 if (c < 0)
589 {
590 enToken = (c == -STATUS_EOF) ? TT_EOF : TT_ERROR;
591 nError = -c;
592 return enToken;
593 }
594
595 sValue.set_length(0);
596
597 // Now analyze first character
598 switch (c)
599 {
600 // Non-alpha
601 case '(': // TT_LBRACE
602 return commit(TT_LBRACE);
603 case ')': // TT_RBRACE
604 return commit(TT_RBRACE);
605 case '[': // TT_LQBRACE
606 return commit(TT_LQBRACE);
607 case ']': // TT_RQBRACE
608 return commit(TT_RQBRACE);
609 case '{': // TT_LCBRACE
610 return commit(TT_LCBRACE);
611 case '}': // TT_RCBRACE
612 return commit(TT_RCBRACE);
613
614 case '#': // TT_SHARP
615 if (!(flags & TF_COLOR))
616 return commit(TT_SHARP);
617
618 commit_lookup(TT_SHARP);
619 return lookup_color();
620
621 case '@': // TT_AT
622 if (!(flags & TF_COLOR))
623 return commit(TT_AT);
624
625 commit_lookup(TT_AT);
626 return lookup_color();
627
628 case '*': // TT_MUL, TT_POW
629 {
630 commit(TT_MUL);
631 if ((c = lookup()) == '*') // Accept both variants: ^ and ^^
632 commit(TT_POW);
633 return enToken;
634 }
635 case '/': // TT_DIV
636 return commit(TT_DIV);
637 case '%': // TT_MOD
638 return commit(TT_IMOD);
639 case '~': // TT_BNOT
640 return commit(TT_BNOT);
641 case '?': // TT_QUESTION
642 return commit(TT_QUESTION);
643 case '&': // TT_AND
644 c = commit_lookup(TT_AND);
645 if (c == '&') // Accept both variants: & and &&
646 commit(TT_AND);
647 return enToken;
648
649 case '|': // TT_OR
650 c = commit_lookup(TT_OR);
651 if (c == '|') // Accept both variants: | and ||
652 commit(TT_OR);
653 return enToken;
654
655 case '!': // TT_NOT, TT_NOT_EQ
656 c = commit_lookup(TT_NOT);
657 if (c == '=') // Check for !=
658 commit(TT_NOT_EQ);
659 return enToken;
660
661 case '^': // TT_XOR
662 c = commit_lookup(TT_XOR);
663 if (c == '^') // Accept both variants: ^ and ^^
664 commit(TT_XOR);
665 return enToken;
666
667 case '<': // TT_LESS, TT_LESS_EQ, TT_NOT_EQ, TT_CMP
668 c = commit_lookup(TT_LESS);
669 if (c == '=') // <=
670 {
671 c = commit_lookup(TT_LESS_EQ);
672 if (c == '>')
673 commit(TT_CMP);
674 }
675 else if (c == '>') // <>
676 commit(TT_NOT_EQ);
677
678 return enToken;
679
680 case '>': // TT_GREATER, TT_GREATER_EQ
681 c = commit_lookup(TT_GREATER);
682 if (c == '=') // Check for >=
683 commit(TT_GREATER_EQ);
684 return enToken;
685
686 case '=': // TT_EQ
687 c = commit_lookup(TT_EQ);
688 if (c == '=') // Accept both variants: = and ==
689 commit(TT_EQ);
690 return enToken;
691
692 case ';': // TT_SEMICOLON
693 return commit(TT_SEMICOLON);
694
695 case ',': // TT_COMMA
696 return commit(TT_COMMA);
697
698 case ':': // TT_COLON, TT_IDENTIFIER
699 {
700 commit(TT_COLON);
701 return lookup_identifier(TT_IDENTIFIER);
702 }
703
704 case '\'': // TT_STRING
705 return lookup_string();
706
707 // Defaults
708 default: // TT_VALUE
709 {
710 // Test character
711 if (flags & TF_XSIGN)
712 {
713 if (c == '+')
714 return commit(TT_ADD);
715 else if (c == '-')
716 return commit(TT_SUB);
717 }
718
719 if (!is_identifier_first(c))
720 return lookup_number();
721
722 if (lookup_identifier(TT_BAREWORD) != TT_BAREWORD)
723 return enToken;
724
725 // Do not decode barewords if flags require keyword to exclude
726 return (flags & TF_XKEYWORDS) ? enToken : decode_bareword();
727 }
728 }
729
730 return enToken = TT_UNKNOWN;
731 }
732
733 } /* namespace expr */
734 } /* namespace lsp */