/* * Copyright (C) 2020 Linux Studio Plugins Project * (C) 2020 Vladimir Sadovnikov * * This file is part of lsp-runtime-lib * Created on: 15 окт. 2019 г. * * lsp-runtime-lib is free software: you can redistribute it and/or modify * it under the terms of the GNU Lesser General Public License as published by * the Free Software Foundation, either version 3 of the License, or * any later version. * * lsp-runtime-lib is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public License * along with lsp-runtime-lib. If not, see . */ #include #include #include #include #include #include //#include using namespace lsp; using namespace lsp::json; UTEST_BEGIN("runtime.fmt.json", tokenizer) void ck_token(Tokenizer &t, const char *s, token_t token) { printf(" checking token: %s\n", s); token_t tok = t.get_token(true); UTEST_ASSERT_MSG(tok == token, "Error testing token: %s", s); LSPString tmp; UTEST_ASSERT(tmp.set_utf8(s)); UTEST_ASSERT_MSG(t.text_value()->equals(&tmp), "Error testing token: %s, got: %s", s, t.text_value()->get_utf8()); } void ck_int(Tokenizer &t, ssize_t value) { printf(" checking integer: %d\n", int(value)); token_t tok = t.get_token(true); UTEST_ASSERT_MSG((tok == JT_DECIMAL) || (tok == JT_HEXADECIMAL), "Error testing token: not decimal/hexadecimal value"); UTEST_ASSERT_MSG(t.int_value() == value, "Error testing token: %d != %d", int(t.int_value()), int(value)); } void ck_float(Tokenizer &t, double value) { printf(" checking float: %f\n", double(value)); token_t tok = t.get_token(true); UTEST_ASSERT_MSG(tok == JT_DOUBLE, "Error testing token: not double value"); UTEST_ASSERT_MSG(float_equals_relative(t.float_value(), value), "Error testing token: %f != %f", t.float_value(), value); } void ck_invalid(const char *s, token_t token) { printf(" checking invalid token: %s\n", s); io::InStringSequence sq; UTEST_ASSERT(sq.wrap(s) == STATUS_OK); Tokenizer t(&sq); token_t tok = t.get_token(true); UTEST_ASSERT(tok == token); } void test_basic_tokens() { static const char *tokens = " [ ] { } : , 'sq_string' \"dq_string\" " " identifier true false null break " " Infinity NaN " " //comment\n\r /*Multiline\n\rcomment*/ "; io::InStringSequence sq; UTEST_ASSERT(sq.wrap(tokens, "UTF-8") == STATUS_OK); Tokenizer t(&sq); ck_token(t, "[", JT_LQ_BRACE); ck_token(t, "]", JT_RQ_BRACE); ck_token(t, "{", JT_LC_BRACE); ck_token(t, "}", JT_RC_BRACE); ck_token(t, ":", JT_COLON); ck_token(t, ",", JT_COMMA); ck_token(t, "sq_string", JT_SQ_STRING); ck_token(t, "dq_string", JT_DQ_STRING); ck_token(t, "identifier", JT_IDENTIFIER); ck_token(t, "true", JT_TRUE); ck_token(t, "false", JT_FALSE); ck_token(t, "null", JT_NULL); ck_token(t, "break", JT_RESERVED); ck_token(t, "Infinity", JT_DOUBLE); ck_token(t, "NaN", JT_DOUBLE); ck_token(t, "comment", JT_SL_COMMENT); ck_token(t, "Multiline\ncomment", JT_ML_COMMENT); UTEST_ASSERT(t.get_token(true) == JT_EOF); } void test_string_tokens() { static const char *tokens = // "'Lorem ipsum dolor sit amet, \\\n\rconsectetur adipiscing elit.'\n\r" // "'\\n\\t\\r\\f\"\\\"\\\'\\v\\b\\f'\n\r" "'\\x20\\X41\\u000a\\U000D\\u042F\\uD83C\\uDFBC'\n\r" "'\\x20\\X41\\u000a\\U000D\\u042F\\uD83C\\uDFBCx'\n\r" "\"\\A\\C/\\D\\C\'\\\'\\\"\"\n\r" "\"\\0\"\n\r"; io::InStringSequence sq; UTEST_ASSERT(sq.wrap(tokens, "UTF-8") == STATUS_OK); Tokenizer t(&sq); // ck_token(t, "Lorem ipsum dolor sit amet, consectetur adipiscing elit.", JT_SQ_STRING); // ck_token(t, "\n\t\r\f\"\"\'\v\b\f", JT_SQ_STRING); ck_token(t, " A\n\rЯ\xF0\x9F\x8E\xBC", JT_SQ_STRING); ck_token(t, " A\n\rЯ\xF0\x9F\x8E\xBCx", JT_SQ_STRING); ck_token(t, "AC/DC\'\'\"", JT_DQ_STRING); UTEST_ASSERT(t.get_token(true) == JT_DQ_STRING); UTEST_ASSERT(t.text_value()->length() == 1); UTEST_ASSERT(t.text_value()->char_at(0) == '\0'); UTEST_ASSERT(t.get_token(true) == JT_EOF); } void test_identifier_tokens() { static const char *tokens = "Identifier_001\n\r" // "Identifier\\u0041\\u0042\n\r" // "Identifier\\u0041\\u0042C\n\r" "$valid_identifier$\n\r" "\\u042F8"; io::InStringSequence sq; UTEST_ASSERT(sq.wrap(tokens, "UTF-8") == STATUS_OK); Tokenizer t(&sq); // char *locale = ::setlocale(LC_ALL, NULL); // ::setlocale(LC_ALL, "ru_RU.UTF-8"); ck_token(t, "Identifier_001", JT_IDENTIFIER); // ck_token(t, "IdentifierAB", JT_IDENTIFIER); // ck_token(t, "IdentifierABC", JT_IDENTIFIER); ck_token(t, "$valid_identifier$", JT_IDENTIFIER); ck_token(t, "Я8", JT_IDENTIFIER); UTEST_ASSERT(t.get_token(true) == JT_EOF); // ::setlocale(LC_ALL, locale); } void test_numeric_tokens() { static const char *tokens = "0\n\r" "10\n\r" "+20\n\r" "-20\n\r" "0x10\n\r" "+0x20\n\r" "-0x20\n\r" "123.\n\r" ".123\n\r" "12.34\n\r" "12.34E+1\n\r" "12.34e-1\n\r" "1e-1\n\r" "2e+1\n\r" "+Infinity\n\r" "-Infinity\n\r" "+NaN\n\r" "-NaN\n\r" ; io::InStringSequence sq; UTEST_ASSERT(sq.wrap(tokens, "UTF-8") == STATUS_OK); Tokenizer t(&sq); ck_int(t, 0); ck_int(t, 10); ck_int(t, +20); ck_int(t, -20); ck_int(t, 0x10); ck_int(t, 0x20); ck_int(t, -0x20); ck_float(t, 123.0); ck_float(t, 0.123); ck_float(t, 12.34); ck_float(t, 12.34E+1); ck_float(t, 12.34E-1); ck_float(t, 1E-1); ck_float(t, 2E+1); UTEST_ASSERT(t.get_token(true) == JT_DOUBLE); UTEST_ASSERT(isinf(t.float_value()) && (t.float_value() > 0.0)); UTEST_ASSERT(t.get_token(true) == JT_DOUBLE); UTEST_ASSERT(isinf(t.float_value()) && (t.float_value() < 0.0)); UTEST_ASSERT(t.get_token(true) == JT_DOUBLE); UTEST_ASSERT(isnan(t.float_value())); UTEST_ASSERT(t.get_token(true) == JT_DOUBLE); UTEST_ASSERT(isnan(t.float_value())); UTEST_ASSERT(t.get_token(true) == JT_EOF); } void test_unicode_comments() { static const char *tokens = "// \\u0041\\u0042\n\r" "/* \\u0041\\u0042 */\n\r" "//\\uD83C\\uDFBC\n\r" "/*\\uD83C\\uDFBC*/\n\r" "//\\uD83C\\uDFBCX\n\r" "/*\\uD83C\\uDFBCX*/\n\r" ; io::InStringSequence sq; UTEST_ASSERT(sq.wrap(tokens, "UTF-8") == STATUS_OK); Tokenizer t(&sq); ck_token(t, " AB", JT_SL_COMMENT); ck_token(t, " AB ", JT_ML_COMMENT); ck_token(t, "\xF0\x9F\x8E\xBC", JT_SL_COMMENT); ck_token(t, "\xF0\x9F\x8E\xBC", JT_ML_COMMENT); ck_token(t, "\xF0\x9F\x8E\xBCX", JT_SL_COMMENT); ck_token(t, "\xF0\x9F\x8E\xBCX", JT_ML_COMMENT); } void test_invalid_tokens() { ck_invalid("\"", JT_ERROR); ck_invalid("\'", JT_ERROR); ck_invalid("\\", JT_ERROR); ck_invalid("\"\\\"", JT_ERROR); ck_invalid("'\\x1'", JT_ERROR); ck_invalid("'\\u1'", JT_ERROR); ck_invalid("'\\u12'", JT_ERROR); ck_invalid("'\\u123'", JT_ERROR); ck_invalid("\\u123", JT_ERROR); ck_invalid("\\u12", JT_ERROR); ck_invalid("'\n\r'", JT_ERROR); ck_invalid("+ 1", JT_UNKNOWN); ck_invalid("- 1", JT_UNKNOWN); ck_invalid(".", JT_UNKNOWN); ck_invalid("12identifier", JT_ERROR); ck_invalid("/* test comment", JT_ERROR); } UTEST_MAIN { printf("Testing basic tokens...\n"); // test_basic_tokens(); printf("Testing string tokens...\n"); test_string_tokens(); printf("Testing identifier tokens...\n"); test_identifier_tokens(); printf("Testing numeric tokens...\n"); test_numeric_tokens(); printf("Testing unicode comments...\n"); test_unicode_comments(); printf("Testing invalid tokens...\n"); test_invalid_tokens(); } UTEST_END