/** * This file is part of the Detox package. * * Copyright (c) Doug Harple * * For the full copyright and license information, please view the LICENSE * file that was distributed with this source code. */ #include #include #include #include "builtin_table.h" #include "clean_utf_8.h" #include "detox_struct.h" #include "table.h" #include "unit_struct.h" #define DATA_COUNT 21 static struct test_filename data[DATA_COUNT] = { // // Note: // - Both tables, `a` and `b` are based on the builtin unicode table. // - `a` has the default translation set to `_` // - `b` has the default translation set to NULL // // legacy { .filename = "safe string", .expected_a = "safe string", .expected_b = "safe string", }, { .filename = "\u00AE reg", .expected_a = "_ reg", .expected_b = "\u00AE reg", }, { .filename = "\u00A9 copy", .expected_a = "_ copy", .expected_b = "\u00A9 copy", }, { .filename = "\u00C6 capital AE", .expected_a = "_ capital AE", .expected_b = "\u00C6 capital AE", }, { .filename = "\u00DE capital thorn", .expected_a = "_ capital thorn", .expected_b = "\u00DE capital thorn", }, { .filename = "\u014A capital ENG", .expected_a = "_ capital ENG", .expected_b = "\u014A capital ENG", }, { .filename = "\u0172 capital U with Ogonek", .expected_a = "_ capital U with Ogonek", .expected_b = "\u0172 capital U with Ogonek", }, // github issue 40 // + full coverage - test default translation rules { .filename = "_ () whatever", .expected_a = "_ () whatever", .expected_b = "_ () whatever", }, { .filename = "\u4000 ignore", .expected_a = "_ ignore", .expected_b = "\u4000 ignore", }, { .filename = "\x7F delete", .expected_a = "_ delete", .expected_b = "_ delete", }, // 2 byte UTF-8 encoded 0x00 (null) { .filename = "\xC0\x80 null", .expected_a = "_hidden_null_ null", .expected_b = "_hidden_null_ null", }, // 2 byte UTF-8 encoded 0x20 (space) { .filename = "\xC0\xA0 space", .expected_a = " space", .expected_b = " space", }, // 2 byte UTF-8 encoded 0x30 (zero) { .filename = "\xC0\xB0 zero", .expected_a = "0 zero", .expected_b = "0 zero", }, // 2 byte UTF-8 encoded 0x7E (tilde) { .filename = "\xC1\xBE tilde", .expected_a = "~ tilde", .expected_b = "~ tilde", }, // 2 byte UTF-8 encoded 0x7F (tilde) { .filename = "\xC1\xBF delete", .expected_a = "_ delete", .expected_b = "_ delete", }, // github issue 33 { .filename = "\U00010348 hwair", .expected_a = "_ hwair", .expected_b = "\U00010348 hwair", }, // confirm invalid single-byte character sequence { .filename = "\x80 invalid", .expected_a = "_ invalid", .expected_b = "_ invalid", }, // phishing { .filename = "\u03A1a\u200E\u200EyPa\u1963\u200E : Y\u200E\u200Eo\u1959r \u200Ea\u1974\u1974o\u200E\u1959\u1952t \u200Eis \u1974\u1959\u200E\u200Err\u1971\u200E\u200E\u1952t\u1963y\u200E \u1963o\u200E\u200E\u1974k\u1971d\u200E\u200E for \u200Es\u1971\u1974\u1959\u200E\u200Erity \u200Er\u1971as\u200E\u200Eo\u1952s. P\u200E\u200E\u1963\u1971a\u200Es\u1971 \u1974\u1963\u200E\u200Ei\u1974k \u200E\u200Eo\u1952 t\u200Eh\u1971 \u1963i\u200E\u200E\u1952k b\u1971\u200E\u200E\u1963ow t\u200Eo \u1959\u1952\u1963\u200E\u200Eo\u1974k \u200Eyo\u200E\u1959r a\u200E\u200E\u1974\u1974o\u200E\u1959\u1952t", .expected_a = "_ayPa_ : Yo_r a__o__t is __rr__t_y _o_k_d for s___rity r_aso_s. P__as_ __i_k o_ th_ _i_k b__ow to ___o_k yo_r a__o__t", .expected_b = "\u03A1ayPa\u1963 : Yo\u1959r a\u1974\u1974o\u1959\u1952t is \u1974\u1959rr\u1971\u1952t\u1963y \u1963o\u1974k\u1971d for s\u1971\u1974\u1959rity r\u1971aso\u1952s. P\u1963\u1971as\u1971 \u1974\u1963i\u1974k o\u1952 th\u1971 \u1963i\u1952k b\u1971\u1963ow to \u1959\u1952\u1963o\u1974k yo\u1959r a\u1974\u1974o\u1959\u1952t", }, { .filename = "C\u200E\u200EAS\u200EE \u200EID :\u200E E", .expected_a = "CASE ID : E", .expected_b = "CASE ID : E", }, // github issue 106 { .filename = "\u2044 fraction slash https:\u2044\u2044www.google.com\u2044whatever\u2044.desktop", .expected_a = "_ fraction slash https:__www.google.com_whatever_.desktop", .expected_b = "_ fraction slash https:__www.google.com_whatever_.desktop", }, // github issue 116 { .filename = "Screen Recording 2024-04-24 at 12.22.20\u202FPM.mov", .expected_a = "Screen Recording 2024-04-24 at 12.22.20 PM.mov", .expected_b = "Screen Recording 2024-04-24 at 12.22.20 PM.mov", }, }; #test test_clean_utf_8 table_t *table_a; table_t *table_b; char *output; int i; table_a = load_builtin_unicode_table(); table_b = load_builtin_unicode_table(); table_a->default_translation = strdup("_"); table_b->default_translation = NULL; // confirm that our a/b test is still valid ck_assert(table_get(table_b, 0x4000) == NULL); ck_assert(table_get(table_b, 0x0080) == NULL); // legacy tests for (i = 0; i < DATA_COUNT; i++) { output = clean_utf_8(data[i].filename, table_a); ck_assert_str_eq(output, data[i].expected_a); output = clean_utf_8(data[i].filename, table_b); ck_assert_str_eq(output, data[i].expected_b); } #test test_clean_utf_8_null char *output; // confirm NULL works output = clean_utf_8(NULL, NULL); ck_assert(output == NULL); #test-exit(1) test_clean_utf_8_missing_table clean_utf_8("what", NULL); #test test_clean_utf_8_invalid table_t *table; char *output; table = load_builtin_unicode_table(); table->default_translation = strdup("_"); // test an invalid UTF-8 sequence // the cleaner should replace the invalid sequence with an underscore output = clean_utf_8("\xC0" "blah", table); ck_assert_str_eq(output, "_blah"); output = clean_utf_8("blah" "\xC0", table); ck_assert_str_eq(output, "blah_"); #test test_clean_utf_8_beyond_unicode_max table_t *table; char *output; table = load_builtin_unicode_table(); table->default_translation = strdup("_"); // 3-byte encoding a lower ASCII "6" output = clean_utf_8("\xE0\x80\xB6" "blah", table); ck_assert_str_eq(output, "6blah"); // 4-byte encoding a lower ASCII "6" output = clean_utf_8("\xF0\x80\x80\xB6" "blah", table); ck_assert_str_eq(output, "6blah"); // 5-byte encoding a lower ASCII "6" output = clean_utf_8("\xF8\x80\x80\x80\xB6" "blah", table); ck_assert_str_eq(output, "6blah"); // 6-byte encoding a lower ASCII "6" output = clean_utf_8("\xF8\x80\x80\x80\xB6" "blah", table); ck_assert_str_eq(output, "6blah"); // 6-byte max on Ubuntu 20.04 // Unicode 0x7FFFFFFF output = clean_utf_8("\xFD\xBF\xBF\xBF\xBF\xBF" "blah", table); ck_assert_str_eq(output, "_blah");