| 1 |
/** |
| 2 |
* This file is part of the Detox package. |
| 3 |
* |
| 4 |
* Copyright (c) Doug Harple <[email protected]> |
| 5 |
* |
| 6 |
* For the full copyright and license information, please view the LICENSE |
| 7 |
* file that was distributed with this source code. |
| 8 |
*/ |
| 9 |
|
| 10 |
#include <stdlib.h> |
| 11 |
#include <stdio.h> |
| 12 |
#include <string.h> |
| 13 |
|
| 14 |
#include "builtin_table.h" |
| 15 |
#include "clean_utf_8.h" |
| 16 |
#include "detox_struct.h" |
| 17 |
#include "table.h" |
| 18 |
|
| 19 |
#include "unit_struct.h" |
| 20 |
|
| 21 |
#define DATA_COUNT 21 |
| 22 |
static struct test_filename data[DATA_COUNT] = { |
| 23 |
// |
| 24 |
// Note: |
| 25 |
// - Both tables, `a` and `b` are based on the builtin unicode table. |
| 26 |
// - `a` has the default translation set to `_` |
| 27 |
// - `b` has the default translation set to NULL |
| 28 |
// |
| 29 |
|
| 30 |
// legacy |
| 31 |
|
| 32 |
{ |
| 33 |
.filename = "safe string", |
| 34 |
.expected_a = "safe string", |
| 35 |
.expected_b = "safe string", |
| 36 |
}, |
| 37 |
{ |
| 38 |
.filename = "\u00AE reg", |
| 39 |
.expected_a = "_ reg", |
| 40 |
.expected_b = "\u00AE reg", |
| 41 |
}, |
| 42 |
{ |
| 43 |
.filename = "\u00A9 copy", |
| 44 |
.expected_a = "_ copy", |
| 45 |
.expected_b = "\u00A9 copy", |
| 46 |
}, |
| 47 |
{ |
| 48 |
.filename = "\u00C6 capital AE", |
| 49 |
.expected_a = "_ capital AE", |
| 50 |
.expected_b = "\u00C6 capital AE", |
| 51 |
}, |
| 52 |
{ |
| 53 |
.filename = "\u00DE capital thorn", |
| 54 |
.expected_a = "_ capital thorn", |
| 55 |
.expected_b = "\u00DE capital thorn", |
| 56 |
}, |
| 57 |
{ |
| 58 |
.filename = "\u014A capital ENG", |
| 59 |
.expected_a = "_ capital ENG", |
| 60 |
.expected_b = "\u014A capital ENG", |
| 61 |
}, |
| 62 |
{ |
| 63 |
.filename = "\u0172 capital U with Ogonek", |
| 64 |
.expected_a = "_ capital U with Ogonek", |
| 65 |
.expected_b = "\u0172 capital U with Ogonek", |
| 66 |
}, |
| 67 |
|
| 68 |
// github issue 40 |
| 69 |
// + full coverage - test default translation rules |
| 70 |
|
| 71 |
{ |
| 72 |
.filename = "_ () whatever", |
| 73 |
.expected_a = "_ () whatever", |
| 74 |
.expected_b = "_ () whatever", |
| 75 |
}, |
| 76 |
{ |
| 77 |
.filename = "\u4000 ignore", |
| 78 |
.expected_a = "_ ignore", |
| 79 |
.expected_b = "\u4000 ignore", |
| 80 |
}, |
| 81 |
{ |
| 82 |
.filename = "\x7F delete", |
| 83 |
.expected_a = "_ delete", |
| 84 |
.expected_b = "_ delete", |
| 85 |
}, |
| 86 |
|
| 87 |
// 2 byte UTF-8 encoded 0x00 (null) |
| 88 |
{ |
| 89 |
.filename = "\xC0\x80 null", |
| 90 |
.expected_a = "_hidden_null_ null", |
| 91 |
.expected_b = "_hidden_null_ null", |
| 92 |
}, |
| 93 |
|
| 94 |
// 2 byte UTF-8 encoded 0x20 (space) |
| 95 |
{ |
| 96 |
.filename = "\xC0\xA0 space", |
| 97 |
.expected_a = " space", |
| 98 |
.expected_b = " space", |
| 99 |
}, |
| 100 |
|
| 101 |
// 2 byte UTF-8 encoded 0x30 (zero) |
| 102 |
{ |
| 103 |
.filename = "\xC0\xB0 zero", |
| 104 |
.expected_a = "0 zero", |
| 105 |
.expected_b = "0 zero", |
| 106 |
}, |
| 107 |
|
| 108 |
// 2 byte UTF-8 encoded 0x7E (tilde) |
| 109 |
{ |
| 110 |
.filename = "\xC1\xBE tilde", |
| 111 |
.expected_a = "~ tilde", |
| 112 |
.expected_b = "~ tilde", |
| 113 |
}, |
| 114 |
|
| 115 |
// 2 byte UTF-8 encoded 0x7F (tilde) |
| 116 |
{ |
| 117 |
.filename = "\xC1\xBF delete", |
| 118 |
.expected_a = "_ delete", |
| 119 |
.expected_b = "_ delete", |
| 120 |
}, |
| 121 |
|
| 122 |
// github issue 33 |
| 123 |
|
| 124 |
{ |
| 125 |
.filename = "\U00010348 hwair", |
| 126 |
.expected_a = "_ hwair", |
| 127 |
.expected_b = "\U00010348 hwair", |
| 128 |
}, |
| 129 |
|
| 130 |
// confirm invalid single-byte character sequence |
| 131 |
|
| 132 |
{ |
| 133 |
.filename = "\x80 invalid", |
| 134 |
.expected_a = "_ invalid", |
| 135 |
.expected_b = "_ invalid", |
| 136 |
}, |
| 137 |
|
| 138 |
// phishing |
| 139 |
|
| 140 |
{ |
| 141 |
.filename = "\u03A1a\u200E\u200EyPa\u1963\u200E : Y\u200E\u200Eo\u1959r \u200Ea\u1974\u1974o\u200E\u1959\u1952t \u200Eis \u1974\u1959\u200E\u200Err\u1971\u200E\u200E\u1952t\u1963y\u200E \u1963o\u200E\u200E\u1974k\u1971d\u200E\u200E for \u200Es\u1971\u1974\u1959\u200E\u200Erity \u200Er\u1971as\u200E\u200Eo\u1952s. P\u200E\u200E\u1963\u1971a\u200Es\u1971 \u1974\u1963\u200E\u200Ei\u1974k \u200E\u200Eo\u1952 t\u200Eh\u1971 \u1963i\u200E\u200E\u1952k b\u1971\u200E\u200E\u1963ow t\u200Eo \u1959\u1952\u1963\u200E\u200Eo\u1974k \u200Eyo\u200E\u1959r a\u200E\u200E\u1974\u1974o\u200E\u1959\u1952t", |
| 142 |
.expected_a = "_ayPa_ : Yo_r a__o__t is __rr__t_y _o_k_d for s___rity r_aso_s. P__as_ __i_k o_ th_ _i_k b__ow to ___o_k yo_r a__o__t", |
| 143 |
.expected_b = "\u03A1ayPa\u1963 : Yo\u1959r a\u1974\u1974o\u1959\u1952t is \u1974\u1959rr\u1971\u1952t\u1963y \u1963o\u1974k\u1971d for s\u1971\u1974\u1959rity r\u1971aso\u1952s. P\u1963\u1971as\u1971 \u1974\u1963i\u1974k o\u1952 th\u1971 \u1963i\u1952k b\u1971\u1963ow to \u1959\u1952\u1963o\u1974k yo\u1959r a\u1974\u1974o\u1959\u1952t", |
| 144 |
}, |
| 145 |
|
| 146 |
{ |
| 147 |
.filename = "C\u200E\u200EAS\u200EE \u200EID :\u200E E", |
| 148 |
.expected_a = "CASE ID : E", |
| 149 |
.expected_b = "CASE ID : E", |
| 150 |
}, |
| 151 |
|
| 152 |
// github issue 106 |
| 153 |
|
| 154 |
{ |
| 155 |
.filename = "\u2044 fraction slash https:\u2044\u2044www.google.com\u2044whatever\u2044.desktop", |
| 156 |
.expected_a = "_ fraction slash https:__www.google.com_whatever_.desktop", |
| 157 |
.expected_b = "_ fraction slash https:__www.google.com_whatever_.desktop", |
| 158 |
}, |
| 159 |
|
| 160 |
// github issue 116 |
| 161 |
|
| 162 |
{ |
| 163 |
.filename = "Screen Recording 2024-04-24 at 12.22.20\u202FPM.mov", |
| 164 |
.expected_a = "Screen Recording 2024-04-24 at 12.22.20 PM.mov", |
| 165 |
.expected_b = "Screen Recording 2024-04-24 at 12.22.20 PM.mov", |
| 166 |
}, |
| 167 |
}; |
| 168 |
|
| 169 |
#test test_clean_utf_8 |
| 170 |
table_t *table_a; |
| 171 |
table_t *table_b; |
| 172 |
char *output; |
| 173 |
int i; |
| 174 |
|
| 175 |
table_a = load_builtin_unicode_table(); |
| 176 |
table_b = load_builtin_unicode_table(); |
| 177 |
|
| 178 |
table_a->default_translation = strdup("_"); |
| 179 |
table_b->default_translation = NULL; |
| 180 |
|
| 181 |
// confirm that our a/b test is still valid |
| 182 |
ck_assert(table_get(table_b, 0x4000) == NULL); |
| 183 |
ck_assert(table_get(table_b, 0x0080) == NULL); |
| 184 |
|
| 185 |
// legacy tests |
| 186 |
for (i = 0; i < DATA_COUNT; i++) { |
| 187 |
output = clean_utf_8(data[i].filename, table_a); |
| 188 |
ck_assert_str_eq(output, data[i].expected_a); |
| 189 |
|
| 190 |
output = clean_utf_8(data[i].filename, table_b); |
| 191 |
ck_assert_str_eq(output, data[i].expected_b); |
| 192 |
} |
| 193 |
|
| 194 |
#test test_clean_utf_8_null |
| 195 |
char *output; |
| 196 |
|
| 197 |
// confirm NULL works |
| 198 |
output = clean_utf_8(NULL, NULL); |
| 199 |
ck_assert(output == NULL); |
| 200 |
|
| 201 |
#test-exit(1) test_clean_utf_8_missing_table |
| 202 |
clean_utf_8("what", NULL); |
| 203 |
|
| 204 |
#test test_clean_utf_8_invalid |
| 205 |
table_t *table; |
| 206 |
char *output; |
| 207 |
|
| 208 |
table = load_builtin_unicode_table(); |
| 209 |
table->default_translation = strdup("_"); |
| 210 |
|
| 211 |
// test an invalid UTF-8 sequence |
| 212 |
// the cleaner should replace the invalid sequence with an underscore |
| 213 |
|
| 214 |
output = clean_utf_8("\xC0" "blah", table); |
| 215 |
ck_assert_str_eq(output, "_blah"); |
| 216 |
|
| 217 |
output = clean_utf_8("blah" "\xC0", table); |
| 218 |
ck_assert_str_eq(output, "blah_"); |
| 219 |
|
| 220 |
#test test_clean_utf_8_beyond_unicode_max |
| 221 |
table_t *table; |
| 222 |
char *output; |
| 223 |
|
| 224 |
table = load_builtin_unicode_table(); |
| 225 |
table->default_translation = strdup("_"); |
| 226 |
|
| 227 |
// 3-byte encoding a lower ASCII "6" |
| 228 |
|
| 229 |
output = clean_utf_8("\xE0\x80\xB6" "blah", table); |
| 230 |
ck_assert_str_eq(output, "6blah"); |
| 231 |
|
| 232 |
// 4-byte encoding a lower ASCII "6" |
| 233 |
|
| 234 |
output = clean_utf_8("\xF0\x80\x80\xB6" "blah", table); |
| 235 |
ck_assert_str_eq(output, "6blah"); |
| 236 |
|
| 237 |
// 5-byte encoding a lower ASCII "6" |
| 238 |
|
| 239 |
output = clean_utf_8("\xF8\x80\x80\x80\xB6" "blah", table); |
| 240 |
ck_assert_str_eq(output, "6blah"); |
| 241 |
|
| 242 |
// 6-byte encoding a lower ASCII "6" |
| 243 |
|
| 244 |
output = clean_utf_8("\xF8\x80\x80\x80\xB6" "blah", table); |
| 245 |
ck_assert_str_eq(output, "6blah"); |
| 246 |
|
| 247 |
// 6-byte max on Ubuntu 20.04 |
| 248 |
|
| 249 |
// Unicode 0x7FFFFFFF |
| 250 |
output = clean_utf_8("\xFD\xBF\xBF\xBF\xBF\xBF" "blah", table); |
| 251 |
ck_assert_str_eq(output, "_blah"); |