Debian Codegraph @

modules/lsp-runtime-lib/src/test/utest/expr/tokenizer.cpp

raw
1 /*
2 * Copyright (C) 2020 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2020 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-runtime-lib
6 * Created on: 17 сен. 2019 г.
7 *
8 * lsp-runtime-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-runtime-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-runtime-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #include <lsp-plug.in/test-fw/utest.h>
23 #include <lsp-plug.in/test-fw/helpers.h>
24 #include <lsp-plug.in/expr/token.h>
25 #include <lsp-plug.in/expr/Tokenizer.h>
26 #include <lsp-plug.in/io/InStringSequence.h>
27
28 namespace lsp
29 {
30 using namespace lsp::expr;
31 }
32
33 UTEST_BEGIN("runtime.expr", tokenizer)
34
35 void ck_token(Tokenizer &t, const char *s, token_t token)
36 {
37 printf(" checking token: %s\n", s);
38 token_t tok = t.get_token(TF_GET);
39 UTEST_ASSERT_MSG(tok == token, "Error testing token: %s", s);
40 UTEST_ASSERT_MSG(t.text_value()->equals_ascii(s), "Error testing token: %s", s);
41 }
42
43 void ck_bareword(Tokenizer &t, const char *s)
44 {
45 printf(" checking token: %s\n", s);
46 token_t tok = t.get_token(TF_GET | TF_XKEYWORDS);
47 UTEST_ASSERT_MSG(tok == TT_BAREWORD, "Error testing token: %s", s);
48 UTEST_ASSERT_MSG(t.text_value()->equals_ascii(s), "Error testing token: %s", s);
49 }
50
51 void ck_color(Tokenizer &t, const char *s)
52 {
53 printf(" checking token: %s\n", s);
54 token_t tok = t.get_token(TF_GET | TF_COLOR);
55 UTEST_ASSERT_MSG(tok == TT_COLOR, "Error testing token: %s", s);
56 UTEST_ASSERT_MSG(t.text_value()->equals_ascii(s), "Error testing token: %s", s);
57 }
58
59 void ck_int(Tokenizer &t, ssize_t value)
60 {
61 printf(" checking integer: %d\n", int(value));
62 token_t tok = t.get_token(TF_GET);
63 UTEST_ASSERT_MSG(tok == TT_IVALUE, "Error testing token: not IVALUE");
64 UTEST_ASSERT_MSG(t.int_value() == value, "Error testing token: %d != %d", int(t.int_value()), int(value));
65 }
66
67 void ck_float(Tokenizer &t, double value)
68 {
69 printf(" checking float: %f\n", double(value));
70 token_t tok = t.get_token(TF_GET);
71 UTEST_ASSERT_MSG(tok == TT_FVALUE, "Error testing token: not FVALUE");
72 UTEST_ASSERT_MSG(float_equals_relative(t.float_value(), value), "Error testing token: %f != %f", t.float_value(), value);
73 }
74
75 void ck_invalid(const char *s, token_t token)
76 {
77 printf(" checking invalid token: %s\n", s);
78
79 io::InStringSequence sq;
80 UTEST_ASSERT(sq.wrap(s, "UTF-8") == STATUS_OK);
81
82 Tokenizer t(&sq);
83 token_t tok = t.get_token(TF_GET);
84 UTEST_ASSERT(tok == token);
85 }
86
87 void test_sign_tokens()
88 {
89 static const char *tokens =
90 "( ) [ ] { } & && | || ! ^ ^^ ~ + - * ** / % < > <= >= != <> = == <=> ? : ; , # @ ( )";
91
92 io::InStringSequence sq;
93 UTEST_ASSERT(sq.wrap(tokens, "UTF-8") == STATUS_OK);
94
95 Tokenizer t(&sq);
96
97 ck_token(t, "(", TT_LBRACE);
98 ck_token(t, ")", TT_RBRACE);
99 ck_token(t, "[", TT_LQBRACE);
100 ck_token(t, "]", TT_RQBRACE);
101 ck_token(t, "{", TT_LCBRACE);
102 ck_token(t, "}", TT_RCBRACE);
103 ck_token(t, "&", TT_AND);
104 ck_token(t, "&&", TT_AND);
105 ck_token(t, "|", TT_OR);
106 ck_token(t, "||", TT_OR);
107 ck_token(t, "!", TT_NOT);
108 ck_token(t, "^", TT_XOR);
109 ck_token(t, "^^", TT_XOR);
110 ck_token(t, "~", TT_BNOT);
111 ck_token(t, "+", TT_ADD);
112 ck_token(t, "-", TT_SUB);
113 ck_token(t, "*", TT_MUL);
114 ck_token(t, "**", TT_POW);
115 ck_token(t, "/", TT_DIV);
116 ck_token(t, "%", TT_IMOD);
117 ck_token(t, "<", TT_LESS);
118 ck_token(t, ">", TT_GREATER);
119 ck_token(t, "<=", TT_LESS_EQ);
120 ck_token(t, ">=", TT_GREATER_EQ);
121 ck_token(t, "!=", TT_NOT_EQ);
122 ck_token(t, "<>", TT_NOT_EQ);
123 ck_token(t, "=", TT_EQ);
124 ck_token(t, "==", TT_EQ);
125 ck_token(t, "<=>", TT_CMP);
126 ck_token(t, "?", TT_QUESTION);
127 ck_token(t, ":", TT_COLON);
128 ck_token(t, ";", TT_SEMICOLON);
129 ck_token(t, ",", TT_COMMA);
130 ck_token(t, "#", TT_SHARP);
131 ck_token(t, "@", TT_AT);
132 ck_token(t, "(", TT_LBRACE);
133 ck_token(t, ")", TT_RBRACE);
134
135 UTEST_ASSERT(t.get_token(TF_GET) == TT_EOF);
136 }
137
138 void test_text_tokens()
139 {
140 static const char *tokens =
141 "true false null undef pi e "
142 "and or not xor "
143 "band bor bnot bxor "
144 "add sub mul pow div fmod "
145 "iadd isub imul idiv imod mod "
146 "sc scat sr srep sl slen uc supr lc slwr srev "
147 "lt nge gt nle le ngt ge nlt ne eq cmp icmp "
148 "ilt inge igt inle ile ingt ige inlt ine ie ieq "
149 "int float fp bool str "
150 "ex db "
151 "bareword "
152 "sin cos tan tg asin acos atan arcsin arccos arctg "
153 "ln log loge lg logd log10 logb log2 "
154 "exp sqrt rad deg abs "
155 "true false null " // Parse this as barewords
156 "@112233 #123 " // Parse this as colors
157 ;
158
159 io::InStringSequence sq;
160 UTEST_ASSERT(sq.wrap(tokens, "UTF-8") == STATUS_OK);
161
162 Tokenizer t(&sq);
163
164 ck_token(t, "true", TT_TRUE);
165 ck_token(t, "false", TT_FALSE);
166 ck_token(t, "null", TT_NULL);
167 ck_token(t, "undef", TT_UNDEF);
168 ck_token(t, "pi", TT_PI);
169 ck_token(t, "e", TT_E);
170
171 ck_token(t, "and", TT_AND);
172 ck_token(t, "or", TT_OR);
173 ck_token(t, "not", TT_NOT);
174 ck_token(t, "xor", TT_XOR);
175
176 ck_token(t, "band", TT_BAND);
177 ck_token(t, "bor", TT_BOR);
178 ck_token(t, "bnot", TT_BNOT);
179 ck_token(t, "bxor", TT_BXOR);
180
181 ck_token(t, "add", TT_ADDSYM);
182 ck_token(t, "sub", TT_SUBSYM);
183 ck_token(t, "mul", TT_MUL);
184 ck_token(t, "pow", TT_POW);
185 ck_token(t, "div", TT_DIV);
186 ck_token(t, "fmod", TT_FMOD);
187
188 ck_token(t, "iadd", TT_IADD);
189 ck_token(t, "isub", TT_ISUB);
190 ck_token(t, "imul", TT_IMUL);
191 ck_token(t, "idiv", TT_IDIV);
192 ck_token(t, "imod", TT_IMOD);
193 ck_token(t, "mod", TT_IMOD);
194
195 ck_token(t, "sc", TT_SCAT);
196 ck_token(t, "scat", TT_SCAT);
197 ck_token(t, "sr", TT_SREP);
198 ck_token(t, "srep", TT_SREP);
199 ck_token(t, "sl", TT_SLEN);
200 ck_token(t, "slen", TT_SLEN);
201 ck_token(t, "uc", TT_SUPR);
202 ck_token(t, "supr", TT_SUPR);
203 ck_token(t, "lc", TT_SLWR);
204 ck_token(t, "slwr", TT_SLWR);
205 ck_token(t, "srev", TT_SREV);
206
207 ck_token(t, "lt", TT_LESS);
208 ck_token(t, "nge", TT_LESS);
209 ck_token(t, "gt", TT_GREATER);
210 ck_token(t, "nle", TT_GREATER);
211 ck_token(t, "le", TT_LESS_EQ);
212 ck_token(t, "ngt", TT_LESS_EQ);
213 ck_token(t, "ge", TT_GREATER_EQ);
214 ck_token(t, "nlt", TT_GREATER_EQ);
215 ck_token(t, "ne", TT_NOT_EQ);
216 ck_token(t, "eq", TT_EQ);
217 ck_token(t, "cmp", TT_CMP);
218 ck_token(t, "icmp", TT_ICMP);
219
220 ck_token(t, "ilt", TT_ILESS);
221 ck_token(t, "inge", TT_ILESS);
222 ck_token(t, "igt", TT_IGREATER);
223 ck_token(t, "inle", TT_IGREATER);
224 ck_token(t, "ile", TT_ILESS_EQ);
225 ck_token(t, "ingt", TT_ILESS_EQ);
226 ck_token(t, "ige", TT_IGREATER_EQ);
227 ck_token(t, "inlt", TT_IGREATER_EQ);
228 ck_token(t, "ine", TT_INOT_EQ);
229 ck_token(t, "ie", TT_IEQ);
230 ck_token(t, "ieq", TT_IEQ);
231
232 ck_token(t, "int", TT_INT);
233 ck_token(t, "float", TT_FLOAT);
234 ck_token(t, "fp", TT_FLOAT);
235 ck_token(t, "bool", TT_BOOL);
236 ck_token(t, "str", TT_STR);
237
238 ck_token(t, "ex", TT_EX);
239 ck_token(t, "db", TT_DB);
240 ck_token(t, "bareword", TT_BAREWORD);
241
242 ck_token(t, "sin", TT_SIN);
243 ck_token(t, "cos", TT_COS);
244 ck_token(t, "tan", TT_TAN);
245 ck_token(t, "tg", TT_TAN);
246 ck_token(t, "asin", TT_ASIN);
247 ck_token(t, "acos", TT_ACOS);
248 ck_token(t, "atan", TT_ATAN);
249 ck_token(t, "arcsin", TT_ASIN);
250 ck_token(t, "arccos", TT_ACOS);
251 ck_token(t, "arctg", TT_ATAN);
252
253 ck_token(t, "ln", TT_LOGE);
254 ck_token(t, "log", TT_LOGE);
255 ck_token(t, "loge", TT_LOGE);
256 ck_token(t, "lg", TT_LOGD);
257 ck_token(t, "logd", TT_LOGD);
258 ck_token(t, "log10", TT_LOGD);
259 ck_token(t, "logb", TT_LOG2);
260 ck_token(t, "log2", TT_LOG2);
261
262 ck_token(t, "exp", TT_EXP);
263 ck_token(t, "sqrt", TT_SQRT);
264 ck_token(t, "rad", TT_RAD);
265 ck_token(t, "deg", TT_DEG);
266 ck_token(t, "abs", TT_ABS);
267
268 ck_bareword(t, "true");
269 ck_bareword(t, "false");
270 ck_bareword(t, "null");
271
272 ck_color(t, "@112233");
273 ck_color(t, "#123");
274
275 UTEST_ASSERT(t.get_token(TF_GET) == TT_EOF);
276 }
277
278 void test_identifier_tokens()
279 {
280 static const char *tokens =
281 ": :_ :A :abc :aBc :abc123 :i123:xyz"
282 ;
283
284 io::InStringSequence sq;
285 UTEST_ASSERT(sq.wrap(tokens, "UTF-8") == STATUS_OK);
286
287 Tokenizer t(&sq);
288
289 ck_token(t, ":", TT_COLON);
290 ck_token(t, "_", TT_IDENTIFIER);
291 ck_token(t, "A", TT_IDENTIFIER);
292 ck_token(t, "abc", TT_IDENTIFIER);
293 ck_token(t, "aBc", TT_IDENTIFIER);
294 ck_token(t, "abc123", TT_IDENTIFIER);
295 ck_token(t, "i123", TT_IDENTIFIER);
296 ck_token(t, "xyz", TT_IDENTIFIER);
297
298 UTEST_ASSERT(t.get_token(TF_GET) == TT_EOF);
299 }
300
301 void test_numeric_tokens()
302 {
303 static const char *tokens =
304 " "
305 "+ - 0 0.0 +1 -1 +1.0 -1.0 +1.123 -1.321 "
306 "0b0101 0b0101.0 0b0101.11 "
307 "0o17 0o7.0 0o7.7 "
308 "0d129 0d9.0 0d9.9 "
309 "0x1f 0x1f.0 0x1f.1 "
310 "1.e 2.0e .3e 4.5e 6.7e1 8.9e+1 1.0e-1 .2e+1 .3e-1 "
311 "0b101.0e-10 -0b101.0e+10 "
312 "0x5.0p-2 -0x5.0p+2 "
313 ;
314
315 io::InStringSequence sq;
316 UTEST_ASSERT(sq.wrap(tokens, "UTF-8") == STATUS_OK);
317
318 Tokenizer t(&sq);
319
320 ck_token(t, "+", TT_ADD);
321 ck_token(t, "-", TT_SUB);
322 ck_int(t, 0);
323 ck_float(t, 0.0);
324 ck_int(t, +1);
325 ck_int(t, -1);
326 ck_float(t, +1.0);
327 ck_float(t, -1.0);
328 ck_float(t, +1.123);
329 ck_float(t, -1.321);
330
331 ck_int(t, 5);
332 ck_float(t, 5.0);
333 ck_float(t, 5.75);
334
335 ck_int(t, 15);
336 ck_float(t, 7.0);
337 ck_float(t, 7.875);
338
339 ck_int(t, 129);
340 ck_float(t, 9.0);
341 ck_float(t, 9.9);
342
343 ck_int(t, 0x1f);
344 ck_float(t, 31.0);
345 ck_float(t, 31.0625);
346
347 ck_float(t, 1.0);
348 ck_float(t, 2.0);
349 ck_float(t, 0.3);
350 ck_float(t, 4.5);
351 ck_float(t, 67.0);
352 ck_float(t, 89.0);
353 ck_float(t, 0.1);
354 ck_float(t, 2.0);
355 ck_float(t, 0.03);
356
357 ck_float(t, 1.25);
358 ck_float(t, -20.0);
359
360 ck_float(t, 0.01953125);
361 ck_float(t, -1280.0);
362
363 UTEST_ASSERT(t.get_token(TF_GET) == TT_EOF);
364 }
365
366 void test_string_tokens()
367 {
368 static const char *tokens =
369 "'' 1 '\\n' 2 '\\r\\t' 3 "
370 "'123 ' '456 ' '789' 4 'end \\protector'5 '\\\':xc\\\''";
371 ;
372
373 io::InStringSequence sq;
374 UTEST_ASSERT(sq.wrap(tokens, "UTF-8") == STATUS_OK);
375
376 Tokenizer t(&sq);
377
378 ck_token(t, "", TT_STRING);
379 ck_int(t, 1);
380 ck_token(t, "\n", TT_STRING);
381 ck_int(t, 2);
382 ck_token(t, "\r\t", TT_STRING);
383 ck_int(t, 3);
384 ck_token(t, "123 456 789", TT_STRING);
385 ck_int(t, 4);
386 ck_token(t, "end \\protector", TT_STRING);
387 ck_int(t, 5);
388 ck_token(t, "':xc'", TT_STRING);
389
390 UTEST_ASSERT(t.get_token(TF_GET) == TT_EOF);
391 }
392
393 void test_invalid_tokens()
394 {
395 ck_invalid(".", TT_UNKNOWN);
396 ck_invalid(".e+", TT_UNKNOWN);
397 ck_invalid("\'", TT_ERROR);
398 ck_invalid("\'\' \'", TT_ERROR);
399 }
400
401 void test_expression_tokens()
402 {
403 static const char *tokens =
404 "((:a eq :b) or (:a eq :c+:d[ssel])) * 10 + (:b ine :c) ? 1 db : 2.0 db";
405
406 io::InStringSequence sq;
407 UTEST_ASSERT(sq.wrap(tokens, "UTF-8") == STATUS_OK);
408
409 Tokenizer t(&sq);
410
411 ck_token(t, "(", TT_LBRACE);
412 ck_token(t, "(", TT_LBRACE);
413 ck_token(t, "a", TT_IDENTIFIER);
414 ck_token(t, "eq", TT_EQ);
415 ck_token(t, "b", TT_IDENTIFIER);
416 ck_token(t, ")", TT_RBRACE);
417
418 ck_token(t, "or", TT_OR);
419
420 ck_token(t, "(", TT_LBRACE);
421 ck_token(t, "a", TT_IDENTIFIER);
422 ck_token(t, "eq", TT_EQ);
423 ck_token(t, "c", TT_IDENTIFIER);
424 ck_token(t, "+", TT_ADD);
425 ck_token(t, "d", TT_IDENTIFIER);
426 ck_token(t, "[", TT_LQBRACE);
427 ck_token(t, "ssel", TT_BAREWORD);
428 ck_token(t, "]", TT_RQBRACE);
429 ck_token(t, ")", TT_RBRACE);
430 ck_token(t, ")", TT_RBRACE);
431
432 ck_token(t, "*", TT_MUL);
433 ck_int(t, 10);
434 ck_token(t, "+", TT_ADD);
435
436 ck_token(t, "(", TT_LBRACE);
437 ck_token(t, "b", TT_IDENTIFIER);
438 ck_token(t, "ine", TT_INOT_EQ);
439 ck_token(t, "c", TT_IDENTIFIER);
440 ck_token(t, ")", TT_RBRACE);
441
442 ck_token(t, "?", TT_QUESTION);
443 ck_int(t, 1);
444 ck_token(t, "db", TT_DB);
445 ck_token(t, ":", TT_COLON);
446 ck_float(t, 2.0);
447 ck_token(t, "db", TT_DB);
448
449 UTEST_ASSERT(t.get_token(TF_GET) == TT_EOF);
450 }
451
452 UTEST_MAIN
453 {
454 printf("Testing basic sign tokens...\n");
455 test_sign_tokens();
456 printf("Testing basic text tokens...\n");
457 test_text_tokens();
458 printf("Testing identifier tokens...\n");
459 test_identifier_tokens();
460 printf("Testing numeric tokens...\n");
461 test_numeric_tokens();
462 printf("Testing string tokens...\n");
463 test_string_tokens();
464 printf("Testing invalid tokens...\n");
465 test_invalid_tokens();
466
467 printf("Testing expression tokens...\n");
468 test_expression_tokens();
469 }
470
471 UTEST_END
472
473