| 1 |
/** |
| 2 |
* This file is part of the Detox package. |
| 3 |
* |
| 4 |
* Copyright (c) Doug Harple <[email protected]> |
| 5 |
* |
| 6 |
* For the full copyright and license information, please view the LICENSE |
| 7 |
* file that was distributed with this source code. |
| 8 |
*/ |
| 9 |
|
| 10 |
#include "config.h" |
| 11 |
|
| 12 |
#include <stdio.h> |
| 13 |
#include <stdlib.h> |
| 14 |
#include <string.h> |
| 15 |
|
| 16 |
#include "clean_utf_8.h" |
| 17 |
#include "table.h" |
| 18 |
#include "wrapped.h" |
| 19 |
|
| 20 |
#define UNICODE_MAX_VALUE 0x10FFFF |
| 21 |
|
| 22 |
static char *null_replacement = "_hidden_null_"; |
| 23 |
static char *invalid_replacement = "_"; |
| 24 |
|
| 25 |
#define UTF_8_ENCODED_MASK 0xC0 |
| 26 |
#define UTF_8_ENCODED_START 0xC0 |
| 27 |
#define UTF_8_ENCODED_CONT 0x80 |
| 28 |
|
| 29 |
#define UTF_8_ENCODED_6_BYTES_MASK 0xFE |
| 30 |
#define UTF_8_ENCODED_6_BYTES 0xFC |
| 31 |
|
| 32 |
#define UTF_8_ENCODED_5_BYTES_MASK 0xFC |
| 33 |
#define UTF_8_ENCODED_5_BYTES 0xF8 |
| 34 |
|
| 35 |
#define UTF_8_ENCODED_4_BYTES_MASK 0xF8 |
| 36 |
#define UTF_8_ENCODED_4_BYTES 0xF0 |
| 37 |
|
| 38 |
#define UTF_8_ENCODED_3_BYTES_MASK 0xF0 |
| 39 |
#define UTF_8_ENCODED_3_BYTES 0xE0 |
| 40 |
|
| 41 |
#define UTF_8_ENCODED_2_BYTES_MASK 0xE0 |
| 42 |
#define UTF_8_ENCODED_2_BYTES 0xC0 |
| 43 |
|
| 44 |
#define UPPER_BIT 0x80 |
| 45 |
|
| 46 |
#define check_width(chr, size) if ((chr & UTF_8_ENCODED_ ## size ## _BYTES_MASK) == UTF_8_ENCODED_ ## size ## _BYTES) { return size; } |
| 47 |
#define is_upper_bit_set(chr) ((chr & UPPER_BIT) == UPPER_BIT) |
| 48 |
#define unpack_cont(chr) ((unsigned char) chr & ~UTF_8_ENCODED_MASK) |
| 49 |
#define unpack_start(chr, size) ((unsigned char) chr & ~UTF_8_ENCODED_ ## size ## _BYTES_MASK) |
| 50 |
|
| 51 |
/** |
| 52 |
* Translates UTF-8 characters (Unicode Translation Format - 8 Bit) into |
| 53 |
* Unicode and then runs the transliteration table. |
| 54 |
* |
| 55 |
* @param filename The filename to clean. |
| 56 |
* @param table_t The transliteration table to use. |
| 57 |
* |
| 58 |
* @return The updated filename or NULL if an error occurred. |
| 59 |
*/ |
| 60 |
char *clean_utf_8(char *filename, table_t *table) |
| 61 |
{ |
| 62 |
char *output, *input_walk, *output_walk, *replace_walk; |
| 63 |
unsigned int new_value; |
| 64 |
int expected_chars; |
| 65 |
int characters_eaten; |
| 66 |
int utf_8_width; |
| 67 |
int failed; |
| 68 |
|
| 69 |
if (filename == NULL) { |
| 70 |
return NULL; |
| 71 |
} |
| 72 |
|
| 73 |
if (table == NULL) { |
| 74 |
fprintf(stderr, "internal error\n"); |
| 75 |
exit(EXIT_FAILURE); |
| 76 |
} |
| 77 |
|
| 78 |
output = wrapped_malloc((strlen(filename) * table->max_data_length) + 1); |
| 79 |
|
| 80 |
input_walk = filename; |
| 81 |
output_walk = output; |
| 82 |
|
| 83 |
while (*input_walk != '\0') { |
| 84 |
utf_8_width = get_utf_8_width(*input_walk); |
| 85 |
|
| 86 |
switch (utf_8_width) { |
| 87 |
case 1: // 0aaaaaaa |
| 88 |
new_value = (unsigned char) input_walk[0]; |
| 89 |
break; |
| 90 |
|
| 91 |
case 2: // 110aaaaa 10bbbbbb |
| 92 |
new_value = unpack_start(*input_walk, 2); |
| 93 |
break; |
| 94 |
|
| 95 |
case 3: // 1110aaaa 10bbbbbb 10cccccc |
| 96 |
new_value = unpack_start(*input_walk, 3); |
| 97 |
break; |
| 98 |
|
| 99 |
case 4: // 11110aaa 10bbbbbb 10cccccc 10dddddd |
| 100 |
new_value = unpack_start(*input_walk, 4); |
| 101 |
break; |
| 102 |
|
| 103 |
case 5: // 111110aa 10bbbbbb 10cccccc 10dddddd 10eeeeee |
| 104 |
new_value = unpack_start(*input_walk, 5); |
| 105 |
break; |
| 106 |
|
| 107 |
case 6: // 1111110a 10bbbbbb 10cccccc 10dddddd 10ffffff |
| 108 |
new_value = unpack_start(*input_walk, 6); |
| 109 |
break; |
| 110 |
|
| 111 |
default: // -1 |
| 112 |
fprintf(stderr, "detox: warning: invalid encoding\n"); |
| 113 |
*output_walk++ = invalid_replacement[0]; |
| 114 |
input_walk++; |
| 115 |
continue; |
| 116 |
} |
| 117 |
|
| 118 |
expected_chars = utf_8_width - 1; |
| 119 |
characters_eaten = utf_8_width; |
| 120 |
|
| 121 |
failed = 0; |
| 122 |
|
| 123 |
while (expected_chars > 0) { |
| 124 |
new_value <<= 6; |
| 125 |
|
| 126 |
input_walk++; |
| 127 |
|
| 128 |
if (*input_walk == '\0') { |
| 129 |
fprintf(stderr, "detox: warning: UTF-8 sequence ended unexpectedly (null)\n"); |
| 130 |
failed = 1; |
| 131 |
break; |
| 132 |
} |
| 133 |
|
| 134 |
if (!is_utf_8_cont(*input_walk)) { |
| 135 |
fprintf(stderr, "detox: warning: UTF-8 sequence ended unexpectedly (missing con't)\n"); |
| 136 |
failed = 1; |
| 137 |
break; |
| 138 |
} |
| 139 |
|
| 140 |
new_value += unpack_cont(*input_walk); |
| 141 |
|
| 142 |
expected_chars--; |
| 143 |
} |
| 144 |
|
| 145 |
if (failed) { |
| 146 |
*output_walk++ = invalid_replacement[0]; |
| 147 |
continue; |
| 148 |
} |
| 149 |
|
| 150 |
#ifdef DEBUG |
| 151 |
if (utf_8_width > 1) { |
| 152 |
fprintf(stderr, "detox: debug: found character 0x%04x, width: %d\n", new_value, utf_8_width); |
| 153 |
} |
| 154 |
#endif |
| 155 |
|
| 156 |
input_walk++; |
| 157 |
|
| 158 |
replace_walk = table_get(table, new_value); |
| 159 |
|
| 160 |
// |
| 161 |
// Never allow a NULL encoded into 2+ byte UTF-8 to persist. |
| 162 |
// |
| 163 |
|
| 164 |
if (replace_walk == NULL && new_value == 0) { |
| 165 |
fprintf(stderr, "detox: warning: UTF-8 encoded NULL encountered\n"); |
| 166 |
replace_walk = null_replacement; |
| 167 |
} |
| 168 |
|
| 169 |
// |
| 170 |
// If we're off the top, take off, nuke it from orbit. It's the only |
| 171 |
// way to be sure. |
| 172 |
// |
| 173 |
|
| 174 |
if (replace_walk == NULL && new_value > UNICODE_MAX_VALUE) { |
| 175 |
fprintf(stderr, "detox: warning: unicode character 0x%08X is invalid\n", new_value); |
| 176 |
replace_walk = invalid_replacement; |
| 177 |
} |
| 178 |
|
| 179 |
if (replace_walk == NULL) { |
| 180 |
replace_walk = table->default_translation; |
| 181 |
} |
| 182 |
|
| 183 |
if (replace_walk == NULL) { |
| 184 |
|
| 185 |
/* |
| 186 |
* Null translation == leave it alone |
| 187 |
*/ |
| 188 |
input_walk -= characters_eaten; |
| 189 |
|
| 190 |
while (characters_eaten > 0) { |
| 191 |
*output_walk++ = *input_walk++; |
| 192 |
characters_eaten--; |
| 193 |
} |
| 194 |
|
| 195 |
continue; |
| 196 |
} |
| 197 |
|
| 198 |
while (*replace_walk != '\0') { |
| 199 |
*output_walk++ = *replace_walk++; |
| 200 |
} |
| 201 |
} |
| 202 |
|
| 203 |
*output_walk = '\0'; |
| 204 |
|
| 205 |
return output; |
| 206 |
} |
| 207 |
|
| 208 |
/** |
| 209 |
* Returns the width, in bytes, of a UTF-8 character, based on the start |
| 210 |
* character. |
| 211 |
* |
| 212 |
* @param c The character to examine. |
| 213 |
* |
| 214 |
* @return An integer between 1 and 6. If the character is invalid, -1 will be |
| 215 |
* returned. |
| 216 |
*/ |
| 217 |
int get_utf_8_width(char c) |
| 218 |
{ |
| 219 |
if (is_utf_8_start(c)) { |
| 220 |
check_width(c, 2); |
| 221 |
check_width(c, 3); |
| 222 |
check_width(c, 4); |
| 223 |
check_width(c, 5); |
| 224 |
check_width(c, 6); |
| 225 |
} |
| 226 |
|
| 227 |
if (is_upper_bit_set(c)) { |
| 228 |
return -1; |
| 229 |
} |
| 230 |
|
| 231 |
return 1; |
| 232 |
} |
| 233 |
|
| 234 |
/** |
| 235 |
* Returns true if the character is part of a UTF-8 sequence, but not the first |
| 236 |
* byte. |
| 237 |
* |
| 238 |
* @param c The character to examine. |
| 239 |
* |
| 240 |
* @return bool |
| 241 |
*/ |
| 242 |
int is_utf_8_cont(char c) |
| 243 |
{ |
| 244 |
return ((c & UTF_8_ENCODED_MASK) == UTF_8_ENCODED_CONT); |
| 245 |
} |
| 246 |
|
| 247 |
/** |
| 248 |
* Returns true if the character is the start of a UTF-8 sequence. |
| 249 |
* |
| 250 |
* @param c The character to examine. |
| 251 |
* |
| 252 |
* @return bool |
| 253 |
*/ |
| 254 |
int is_utf_8_start(char c) |
| 255 |
{ |
| 256 |
return ((c & UTF_8_ENCODED_MASK) == UTF_8_ENCODED_START); |
| 257 |
} |