Debian Codegraph @

src/clean_utf_8.c

raw
1 /**
2 * This file is part of the Detox package.
3 *
4 * Copyright (c) Doug Harple <[email protected]>
5 *
6 * For the full copyright and license information, please view the LICENSE
7 * file that was distributed with this source code.
8 */
9
10 #include "config.h"
11
12 #include <stdio.h>
13 #include <stdlib.h>
14 #include <string.h>
15
16 #include "clean_utf_8.h"
17 #include "table.h"
18 #include "wrapped.h"
19
20 #define UNICODE_MAX_VALUE 0x10FFFF
21
22 static char *null_replacement = "_hidden_null_";
23 static char *invalid_replacement = "_";
24
25 #define UTF_8_ENCODED_MASK 0xC0
26 #define UTF_8_ENCODED_START 0xC0
27 #define UTF_8_ENCODED_CONT 0x80
28
29 #define UTF_8_ENCODED_6_BYTES_MASK 0xFE
30 #define UTF_8_ENCODED_6_BYTES 0xFC
31
32 #define UTF_8_ENCODED_5_BYTES_MASK 0xFC
33 #define UTF_8_ENCODED_5_BYTES 0xF8
34
35 #define UTF_8_ENCODED_4_BYTES_MASK 0xF8
36 #define UTF_8_ENCODED_4_BYTES 0xF0
37
38 #define UTF_8_ENCODED_3_BYTES_MASK 0xF0
39 #define UTF_8_ENCODED_3_BYTES 0xE0
40
41 #define UTF_8_ENCODED_2_BYTES_MASK 0xE0
42 #define UTF_8_ENCODED_2_BYTES 0xC0
43
44 #define UPPER_BIT 0x80
45
46 #define check_width(chr, size) if ((chr & UTF_8_ENCODED_ ## size ## _BYTES_MASK) == UTF_8_ENCODED_ ## size ## _BYTES) { return size; }
47 #define is_upper_bit_set(chr) ((chr & UPPER_BIT) == UPPER_BIT)
48 #define unpack_cont(chr) ((unsigned char) chr & ~UTF_8_ENCODED_MASK)
49 #define unpack_start(chr, size) ((unsigned char) chr & ~UTF_8_ENCODED_ ## size ## _BYTES_MASK)
50
51 /**
52 * Translates UTF-8 characters (Unicode Translation Format - 8 Bit) into
53 * Unicode and then runs the transliteration table.
54 *
55 * @param filename The filename to clean.
56 * @param table_t The transliteration table to use.
57 *
58 * @return The updated filename or NULL if an error occurred.
59 */
60 char *clean_utf_8(char *filename, table_t *table)
61 {
62 char *output, *input_walk, *output_walk, *replace_walk;
63 unsigned int new_value;
64 int expected_chars;
65 int characters_eaten;
66 int utf_8_width;
67 int failed;
68
69 if (filename == NULL) {
70 return NULL;
71 }
72
73 if (table == NULL) {
74 fprintf(stderr, "internal error\n");
75 exit(EXIT_FAILURE);
76 }
77
78 output = wrapped_malloc((strlen(filename) * table->max_data_length) + 1);
79
80 input_walk = filename;
81 output_walk = output;
82
83 while (*input_walk != '\0') {
84 utf_8_width = get_utf_8_width(*input_walk);
85
86 switch (utf_8_width) {
87 case 1: // 0aaaaaaa
88 new_value = (unsigned char) input_walk[0];
89 break;
90
91 case 2: // 110aaaaa 10bbbbbb
92 new_value = unpack_start(*input_walk, 2);
93 break;
94
95 case 3: // 1110aaaa 10bbbbbb 10cccccc
96 new_value = unpack_start(*input_walk, 3);
97 break;
98
99 case 4: // 11110aaa 10bbbbbb 10cccccc 10dddddd
100 new_value = unpack_start(*input_walk, 4);
101 break;
102
103 case 5: // 111110aa 10bbbbbb 10cccccc 10dddddd 10eeeeee
104 new_value = unpack_start(*input_walk, 5);
105 break;
106
107 case 6: // 1111110a 10bbbbbb 10cccccc 10dddddd 10ffffff
108 new_value = unpack_start(*input_walk, 6);
109 break;
110
111 default: // -1
112 fprintf(stderr, "detox: warning: invalid encoding\n");
113 *output_walk++ = invalid_replacement[0];
114 input_walk++;
115 continue;
116 }
117
118 expected_chars = utf_8_width - 1;
119 characters_eaten = utf_8_width;
120
121 failed = 0;
122
123 while (expected_chars > 0) {
124 new_value <<= 6;
125
126 input_walk++;
127
128 if (*input_walk == '\0') {
129 fprintf(stderr, "detox: warning: UTF-8 sequence ended unexpectedly (null)\n");
130 failed = 1;
131 break;
132 }
133
134 if (!is_utf_8_cont(*input_walk)) {
135 fprintf(stderr, "detox: warning: UTF-8 sequence ended unexpectedly (missing con't)\n");
136 failed = 1;
137 break;
138 }
139
140 new_value += unpack_cont(*input_walk);
141
142 expected_chars--;
143 }
144
145 if (failed) {
146 *output_walk++ = invalid_replacement[0];
147 continue;
148 }
149
150 #ifdef DEBUG
151 if (utf_8_width > 1) {
152 fprintf(stderr, "detox: debug: found character 0x%04x, width: %d\n", new_value, utf_8_width);
153 }
154 #endif
155
156 input_walk++;
157
158 replace_walk = table_get(table, new_value);
159
160 //
161 // Never allow a NULL encoded into 2+ byte UTF-8 to persist.
162 //
163
164 if (replace_walk == NULL && new_value == 0) {
165 fprintf(stderr, "detox: warning: UTF-8 encoded NULL encountered\n");
166 replace_walk = null_replacement;
167 }
168
169 //
170 // If we're off the top, take off, nuke it from orbit. It's the only
171 // way to be sure.
172 //
173
174 if (replace_walk == NULL && new_value > UNICODE_MAX_VALUE) {
175 fprintf(stderr, "detox: warning: unicode character 0x%08X is invalid\n", new_value);
176 replace_walk = invalid_replacement;
177 }
178
179 if (replace_walk == NULL) {
180 replace_walk = table->default_translation;
181 }
182
183 if (replace_walk == NULL) {
184
185 /*
186 * Null translation == leave it alone
187 */
188 input_walk -= characters_eaten;
189
190 while (characters_eaten > 0) {
191 *output_walk++ = *input_walk++;
192 characters_eaten--;
193 }
194
195 continue;
196 }
197
198 while (*replace_walk != '\0') {
199 *output_walk++ = *replace_walk++;
200 }
201 }
202
203 *output_walk = '\0';
204
205 return output;
206 }
207
208 /**
209 * Returns the width, in bytes, of a UTF-8 character, based on the start
210 * character.
211 *
212 * @param c The character to examine.
213 *
214 * @return An integer between 1 and 6. If the character is invalid, -1 will be
215 * returned.
216 */
217 int get_utf_8_width(char c)
218 {
219 if (is_utf_8_start(c)) {
220 check_width(c, 2);
221 check_width(c, 3);
222 check_width(c, 4);
223 check_width(c, 5);
224 check_width(c, 6);
225 }
226
227 if (is_upper_bit_set(c)) {
228 return -1;
229 }
230
231 return 1;
232 }
233
234 /**
235 * Returns true if the character is part of a UTF-8 sequence, but not the first
236 * byte.
237 *
238 * @param c The character to examine.
239 *
240 * @return bool
241 */
242 int is_utf_8_cont(char c)
243 {
244 return ((c & UTF_8_ENCODED_MASK) == UTF_8_ENCODED_CONT);
245 }
246
247 /**
248 * Returns true if the character is the start of a UTF-8 sequence.
249 *
250 * @param c The character to examine.
251 *
252 * @return bool
253 */
254 int is_utf_8_start(char c)
255 {
256 return ((c & UTF_8_ENCODED_MASK) == UTF_8_ENCODED_START);
257 }