Debian Codegraph @

tests/unit/test_clean_utf_8.template

raw
1 /**
2 * This file is part of the Detox package.
3 *
4 * Copyright (c) Doug Harple <[email protected]>
5 *
6 * For the full copyright and license information, please view the LICENSE
7 * file that was distributed with this source code.
8 */
9
10 #include <stdlib.h>
11 #include <stdio.h>
12 #include <string.h>
13
14 #include "builtin_table.h"
15 #include "clean_utf_8.h"
16 #include "detox_struct.h"
17 #include "table.h"
18
19 #include "unit_struct.h"
20
21 #define DATA_COUNT 21
22 static struct test_filename data[DATA_COUNT] = {
23 //
24 // Note:
25 // - Both tables, `a` and `b` are based on the builtin unicode table.
26 // - `a` has the default translation set to `_`
27 // - `b` has the default translation set to NULL
28 //
29
30 // legacy
31
32 {
33 .filename = "safe string",
34 .expected_a = "safe string",
35 .expected_b = "safe string",
36 },
37 {
38 .filename = "\u00AE reg",
39 .expected_a = "_ reg",
40 .expected_b = "\u00AE reg",
41 },
42 {
43 .filename = "\u00A9 copy",
44 .expected_a = "_ copy",
45 .expected_b = "\u00A9 copy",
46 },
47 {
48 .filename = "\u00C6 capital AE",
49 .expected_a = "_ capital AE",
50 .expected_b = "\u00C6 capital AE",
51 },
52 {
53 .filename = "\u00DE capital thorn",
54 .expected_a = "_ capital thorn",
55 .expected_b = "\u00DE capital thorn",
56 },
57 {
58 .filename = "\u014A capital ENG",
59 .expected_a = "_ capital ENG",
60 .expected_b = "\u014A capital ENG",
61 },
62 {
63 .filename = "\u0172 capital U with Ogonek",
64 .expected_a = "_ capital U with Ogonek",
65 .expected_b = "\u0172 capital U with Ogonek",
66 },
67
68 // github issue 40
69 // + full coverage - test default translation rules
70
71 {
72 .filename = "_ () whatever",
73 .expected_a = "_ () whatever",
74 .expected_b = "_ () whatever",
75 },
76 {
77 .filename = "\u4000 ignore",
78 .expected_a = "_ ignore",
79 .expected_b = "\u4000 ignore",
80 },
81 {
82 .filename = "\x7F delete",
83 .expected_a = "_ delete",
84 .expected_b = "_ delete",
85 },
86
87 // 2 byte UTF-8 encoded 0x00 (null)
88 {
89 .filename = "\xC0\x80 null",
90 .expected_a = "_hidden_null_ null",
91 .expected_b = "_hidden_null_ null",
92 },
93
94 // 2 byte UTF-8 encoded 0x20 (space)
95 {
96 .filename = "\xC0\xA0 space",
97 .expected_a = " space",
98 .expected_b = " space",
99 },
100
101 // 2 byte UTF-8 encoded 0x30 (zero)
102 {
103 .filename = "\xC0\xB0 zero",
104 .expected_a = "0 zero",
105 .expected_b = "0 zero",
106 },
107
108 // 2 byte UTF-8 encoded 0x7E (tilde)
109 {
110 .filename = "\xC1\xBE tilde",
111 .expected_a = "~ tilde",
112 .expected_b = "~ tilde",
113 },
114
115 // 2 byte UTF-8 encoded 0x7F (tilde)
116 {
117 .filename = "\xC1\xBF delete",
118 .expected_a = "_ delete",
119 .expected_b = "_ delete",
120 },
121
122 // github issue 33
123
124 {
125 .filename = "\U00010348 hwair",
126 .expected_a = "_ hwair",
127 .expected_b = "\U00010348 hwair",
128 },
129
130 // confirm invalid single-byte character sequence
131
132 {
133 .filename = "\x80 invalid",
134 .expected_a = "_ invalid",
135 .expected_b = "_ invalid",
136 },
137
138 // phishing
139
140 {
141 .filename = "\u03A1a\u200E\u200EyPa\u1963\u200E : Y\u200E\u200Eo\u1959r \u200Ea\u1974\u1974o\u200E\u1959\u1952t \u200Eis \u1974\u1959\u200E\u200Err\u1971\u200E\u200E\u1952t\u1963y\u200E \u1963o\u200E\u200E\u1974k\u1971d\u200E\u200E for \u200Es\u1971\u1974\u1959\u200E\u200Erity \u200Er\u1971as\u200E\u200Eo\u1952s. P\u200E\u200E\u1963\u1971a\u200Es\u1971 \u1974\u1963\u200E\u200Ei\u1974k \u200E\u200Eo\u1952 t\u200Eh\u1971 \u1963i\u200E\u200E\u1952k b\u1971\u200E\u200E\u1963ow t\u200Eo \u1959\u1952\u1963\u200E\u200Eo\u1974k \u200Eyo\u200E\u1959r a\u200E\u200E\u1974\u1974o\u200E\u1959\u1952t",
142 .expected_a = "_ayPa_ : Yo_r a__o__t is __rr__t_y _o_k_d for s___rity r_aso_s. P__as_ __i_k o_ th_ _i_k b__ow to ___o_k yo_r a__o__t",
143 .expected_b = "\u03A1ayPa\u1963 : Yo\u1959r a\u1974\u1974o\u1959\u1952t is \u1974\u1959rr\u1971\u1952t\u1963y \u1963o\u1974k\u1971d for s\u1971\u1974\u1959rity r\u1971aso\u1952s. P\u1963\u1971as\u1971 \u1974\u1963i\u1974k o\u1952 th\u1971 \u1963i\u1952k b\u1971\u1963ow to \u1959\u1952\u1963o\u1974k yo\u1959r a\u1974\u1974o\u1959\u1952t",
144 },
145
146 {
147 .filename = "C\u200E\u200EAS\u200EE \u200EID :\u200E E",
148 .expected_a = "CASE ID : E",
149 .expected_b = "CASE ID : E",
150 },
151
152 // github issue 106
153
154 {
155 .filename = "\u2044 fraction slash https:\u2044\u2044www.google.com\u2044whatever\u2044.desktop",
156 .expected_a = "_ fraction slash https:__www.google.com_whatever_.desktop",
157 .expected_b = "_ fraction slash https:__www.google.com_whatever_.desktop",
158 },
159
160 // github issue 116
161
162 {
163 .filename = "Screen Recording 2024-04-24 at 12.22.20\u202FPM.mov",
164 .expected_a = "Screen Recording 2024-04-24 at 12.22.20 PM.mov",
165 .expected_b = "Screen Recording 2024-04-24 at 12.22.20 PM.mov",
166 },
167 };
168
169 #test test_clean_utf_8
170 table_t *table_a;
171 table_t *table_b;
172 char *output;
173 int i;
174
175 table_a = load_builtin_unicode_table();
176 table_b = load_builtin_unicode_table();
177
178 table_a->default_translation = strdup("_");
179 table_b->default_translation = NULL;
180
181 // confirm that our a/b test is still valid
182 ck_assert(table_get(table_b, 0x4000) == NULL);
183 ck_assert(table_get(table_b, 0x0080) == NULL);
184
185 // legacy tests
186 for (i = 0; i < DATA_COUNT; i++) {
187 output = clean_utf_8(data[i].filename, table_a);
188 ck_assert_str_eq(output, data[i].expected_a);
189
190 output = clean_utf_8(data[i].filename, table_b);
191 ck_assert_str_eq(output, data[i].expected_b);
192 }
193
194 #test test_clean_utf_8_null
195 char *output;
196
197 // confirm NULL works
198 output = clean_utf_8(NULL, NULL);
199 ck_assert(output == NULL);
200
201 #test-exit(1) test_clean_utf_8_missing_table
202 clean_utf_8("what", NULL);
203
204 #test test_clean_utf_8_invalid
205 table_t *table;
206 char *output;
207
208 table = load_builtin_unicode_table();
209 table->default_translation = strdup("_");
210
211 // test an invalid UTF-8 sequence
212 // the cleaner should replace the invalid sequence with an underscore
213
214 output = clean_utf_8("\xC0" "blah", table);
215 ck_assert_str_eq(output, "_blah");
216
217 output = clean_utf_8("blah" "\xC0", table);
218 ck_assert_str_eq(output, "blah_");
219
220 #test test_clean_utf_8_beyond_unicode_max
221 table_t *table;
222 char *output;
223
224 table = load_builtin_unicode_table();
225 table->default_translation = strdup("_");
226
227 // 3-byte encoding a lower ASCII "6"
228
229 output = clean_utf_8("\xE0\x80\xB6" "blah", table);
230 ck_assert_str_eq(output, "6blah");
231
232 // 4-byte encoding a lower ASCII "6"
233
234 output = clean_utf_8("\xF0\x80\x80\xB6" "blah", table);
235 ck_assert_str_eq(output, "6blah");
236
237 // 5-byte encoding a lower ASCII "6"
238
239 output = clean_utf_8("\xF8\x80\x80\x80\xB6" "blah", table);
240 ck_assert_str_eq(output, "6blah");
241
242 // 6-byte encoding a lower ASCII "6"
243
244 output = clean_utf_8("\xF8\x80\x80\x80\xB6" "blah", table);
245 ck_assert_str_eq(output, "6blah");
246
247 // 6-byte max on Ubuntu 20.04
248
249 // Unicode 0x7FFFFFFF
250 output = clean_utf_8("\xFD\xBF\xBF\xBF\xBF\xBF" "blah", table);
251 ck_assert_str_eq(output, "_blah");