| 1 |
/* |
| 2 |
local: some stuff for localisation, safe terminal printout |
| 3 |
|
| 4 |
This is about determining if we got UTF-8 locale and |
| 5 |
checking output terminal properties, along with subsequent string |
| 6 |
transformations for safe printout. |
| 7 |
|
| 8 |
copyright 2008-2021 by the mpg123 project - free software under the terms of the LGPL 2.1 |
| 9 |
see COPYING and AUTHORS files in distribution or http://mpg123.org |
| 10 |
initially written by Thomas Orgis, based on a patch by Thorsten Glaser. |
| 11 |
*/ |
| 12 |
|
| 13 |
// wchar stuff |
| 14 |
#define _XOPEN_SOURCE 600 |
| 15 |
#define _POSIX_C_SOURCE 200112L |
| 16 |
|
| 17 |
#include "config.h" |
| 18 |
|
| 19 |
#ifdef HAVE_LOCALE_H |
| 20 |
#include <locale.h> |
| 21 |
#endif |
| 22 |
#ifdef HAVE_LANGINFO_H |
| 23 |
#include <langinfo.h> |
| 24 |
#endif |
| 25 |
#include "compat/compat.h" |
| 26 |
|
| 27 |
#include "local.h" |
| 28 |
|
| 29 |
#ifdef HAVE_WCHAR_H |
| 30 |
#include <wchar.h> |
| 31 |
#endif |
| 32 |
#ifdef HAVE_WCTYPE_H |
| 33 |
#include <wctype.h> |
| 34 |
#endif |
| 35 |
|
| 36 |
#ifdef WIN32 |
| 37 |
#define WIN32_LEAN_AND_MEAN 1 |
| 38 |
#include <windows.h> |
| 39 |
#include <wincon.h> |
| 40 |
#endif |
| 41 |
|
| 42 |
#include "common/debug.h" |
| 43 |
|
| 44 |
int utf8force = 0; // enforce UTF-8 workings |
| 45 |
int utf8env = 0; // produce UTF-8 text output |
| 46 |
int utf8loc = 0; // have actual UTF-8 locale (so that mbstowcs() works) |
| 47 |
|
| 48 |
//static int term_is_fun = -1; |
| 49 |
|
| 50 |
static const char joker_symbol = '?'; |
| 51 |
static const char *uni_repl = "\xef\xbf\xbd"; |
| 52 |
static const int uni_repl_len = 3; |
| 53 |
|
| 54 |
/* Check some language variable for UTF-8-ness. */ |
| 55 |
static int is_utf8(const char *lang); |
| 56 |
|
| 57 |
void check_locale(void) |
| 58 |
{ |
| 59 |
if(utf8force) |
| 60 |
utf8env = 1; |
| 61 |
else |
| 62 |
{ |
| 63 |
const char *cp; |
| 64 |
|
| 65 |
/* Check for env vars in proper oder. */ |
| 66 |
if((cp = getenv("LC_ALL")) == NULL && (cp = getenv("LC_CTYPE")) == NULL) |
| 67 |
cp = getenv("LANG"); |
| 68 |
|
| 69 |
if(is_utf8(cp)) |
| 70 |
utf8env = 1; |
| 71 |
} |
| 72 |
|
| 73 |
#if defined(HAVE_SETLOCALE) && defined(LC_CTYPE) |
| 74 |
/* To query, we need to set from environment... */ |
| 75 |
if( |
| 76 |
is_utf8(setlocale(LC_CTYPE, "")) |
| 77 |
// If enforced, try to set an UTF-8 locale that hopefully exists. |
| 78 |
|| (utf8force && is_utf8(setlocale(LC_CTYPE, "C.UTF-8"))) |
| 79 |
|| (utf8force && is_utf8(setlocale(LC_CTYPE, "en_US.UTF-8"))) |
| 80 |
) |
| 81 |
{ |
| 82 |
utf8env = 1; |
| 83 |
utf8loc = 1; |
| 84 |
} |
| 85 |
#endif |
| 86 |
#if defined(HAVE_NL_LANGINFO) && defined(CODESET) |
| 87 |
/* ...langinfo works after we set a locale, eh? So it makes sense after setlocale, if only. */ |
| 88 |
if(is_utf8(nl_langinfo(CODESET))) |
| 89 |
{ |
| 90 |
utf8env = 1; |
| 91 |
utf8loc = 1; |
| 92 |
} |
| 93 |
#endif |
| 94 |
|
| 95 |
debug2("UTF-8 env %i: locale: %i", utf8env, utf8loc); |
| 96 |
} |
| 97 |
|
| 98 |
static int is_utf8(const char *lang) |
| 99 |
{ |
| 100 |
if(lang == NULL) return 0; |
| 101 |
|
| 102 |
/* Now, if the variable mentions UTF-8 anywhere, in some variation, the locale is UTF-8. */ |
| 103 |
if( strstr(lang, "UTF-8") || strstr(lang, "utf-8") |
| 104 |
|| strstr(lang, "UTF8") || strstr(lang, "utf8") ) |
| 105 |
return 1; |
| 106 |
else |
| 107 |
return 0; |
| 108 |
} |
| 109 |
|
| 110 |
// Moved encoding stuff over from metaprint.c and removed references to libmpg123, |
| 111 |
// meaning no mpg123_string for you! |
| 112 |
|
| 113 |
int unknown2utf8(char **dest, const char *source, int len) |
| 114 |
{ |
| 115 |
if(!dest) |
| 116 |
return -1; |
| 117 |
if(!source) |
| 118 |
{ |
| 119 |
*dest = INT123_safer_realloc(*dest, 0); |
| 120 |
return -1; |
| 121 |
} |
| 122 |
size_t count = len < 0 ? strlen(source) : (size_t)len; |
| 123 |
// Make a somewhat proper UTF-8 string out of this. Testing for valid |
| 124 |
// UTF-8 is futile. It will be some unspecified legacy 8-bit encoding. |
| 125 |
// I am keeping C0 chars, but replace everything above 7 bits with |
| 126 |
// the Unicode replacement character as most custom 8-bit encodings |
| 127 |
// placed some symbols into the C1 range, we just don't know which. |
| 128 |
size_t ulen = 1; // trailing zero |
| 129 |
for(size_t i=0; i<count; ++i) |
| 130 |
{ |
| 131 |
unsigned char c = ((unsigned char*)source)[i]; |
| 132 |
if(!c) |
| 133 |
break; |
| 134 |
ulen += c >= 0x80 ? uni_repl_len : 1; |
| 135 |
} |
| 136 |
|
| 137 |
if(NULL == (*dest = INT123_safer_realloc(*dest, ulen))) |
| 138 |
return -1; |
| 139 |
|
| 140 |
unsigned char *p = (unsigned char*)*dest; |
| 141 |
for(size_t i=0; i<count; ++i) |
| 142 |
{ |
| 143 |
unsigned char c = ((unsigned char*)source)[i]; |
| 144 |
if(!c) |
| 145 |
break; |
| 146 |
if(c >= 0x80) |
| 147 |
{ |
| 148 |
for(int r=0; r<uni_repl_len; ++r) |
| 149 |
*p++ = uni_repl[r]; |
| 150 |
} |
| 151 |
else |
| 152 |
*p++ = c; |
| 153 |
} |
| 154 |
*p = 0; |
| 155 |
return 0; |
| 156 |
} |
| 157 |
|
| 158 |
static void ascii_space(unsigned char *c, int *wasspace) |
| 159 |
{ |
| 160 |
switch(*c) |
| 161 |
{ |
| 162 |
case '\f': |
| 163 |
case '\r': |
| 164 |
case '\n': |
| 165 |
case '\t': |
| 166 |
case '\v': |
| 167 |
if(!*wasspace) |
| 168 |
*c = ' '; // Will be dropped by < 0x20 check otherwise. |
| 169 |
*wasspace = 1; |
| 170 |
break; |
| 171 |
default: |
| 172 |
*wasspace = 0; |
| 173 |
} |
| 174 |
} |
| 175 |
|
| 176 |
// Filter C1 control chars, using c2lead state. |
| 177 |
#define ASCII_C1(c, append) \ |
| 178 |
if(c2lead) \ |
| 179 |
{ \ |
| 180 |
if((c) >= 0x80 && (c) <= 0x9f) \ |
| 181 |
{ \ |
| 182 |
c2lead = 0; \ |
| 183 |
continue; \ |
| 184 |
} \ |
| 185 |
else \ |
| 186 |
{ \ |
| 187 |
append; \ |
| 188 |
} \ |
| 189 |
} \ |
| 190 |
c2lead = ((c) == 0xc2); \ |
| 191 |
if(c2lead) \ |
| 192 |
continue; |
| 193 |
|
| 194 |
// return: strlen+1 of result, 0 on error |
| 195 |
// If ret is 0, *dest will be freed and NULL. |
| 196 |
|
| 197 |
static size_t utf8_ascii_work(char **dest_, const char *source |
| 198 |
, int keep_nonprint) |
| 199 |
{ |
| 200 |
if(!dest_) |
| 201 |
return 0; |
| 202 |
if(!source) |
| 203 |
{ |
| 204 |
*dest_ = INT123_safer_realloc(*dest_, 0); |
| 205 |
return 0; |
| 206 |
} |
| 207 |
|
| 208 |
char *dest = *dest_; |
| 209 |
size_t source_fill = strlen(source)+1; |
| 210 |
size_t spos = 0; |
| 211 |
size_t dlen = 1; // At least a zero. |
| 212 |
unsigned char *p; |
| 213 |
|
| 214 |
// Find length of ASCII string (count non-continuation bytes). |
| 215 |
// Do _not_ change this to mpg123_strlen()! |
| 216 |
// It needs to match the loop below. |
| 217 |
// No UTF-8 continuation byte 0x10??????, nor control char. |
| 218 |
#define ASCII_PRINT_SOMETHING(c) \ |
| 219 |
(((c) & 0xc0) != 0x80 && (keep_nonprint || ((c) != 0x7f && (c) >= 0x20))) |
| 220 |
int c2lead = 0; |
| 221 |
int wasspace = 0; |
| 222 |
for(spos=0; spos < source_fill; ++spos) |
| 223 |
{ |
| 224 |
unsigned char c = ((unsigned char*)source)[spos]; |
| 225 |
if(!keep_nonprint) |
| 226 |
ascii_space(&c, &wasspace); |
| 227 |
ASCII_C1(c, ++dlen); |
| 228 |
if(ASCII_PRINT_SOMETHING(c)) |
| 229 |
++dlen; |
| 230 |
} |
| 231 |
// Do nothing with nothing or if allocation fails. Neatly catches overflow |
| 232 |
// of ++dlen. |
| 233 |
if(!dlen || !(dest=INT123_safer_realloc(dest, dlen))) |
| 234 |
goto utf8_ascii_bad; |
| 235 |
p = (unsigned char*)dest; |
| 236 |
c2lead = 0; |
| 237 |
wasspace = 0; |
| 238 |
for(spos=0; spos < source_fill; ++spos) |
| 239 |
{ |
| 240 |
unsigned char c = ((unsigned char*)source)[spos]; |
| 241 |
if(!keep_nonprint) |
| 242 |
ascii_space(&c, &wasspace); |
| 243 |
ASCII_C1(c, *p++ = joker_symbol) |
| 244 |
if(!ASCII_PRINT_SOMETHING(c)) |
| 245 |
continue; |
| 246 |
else if(c & 0x80) // UTF-8 lead byte 0x11?????? |
| 247 |
c = joker_symbol; |
| 248 |
*p++ = c; |
| 249 |
} |
| 250 |
#undef ASCII_PRINT_SOMETHING |
| 251 |
// Always close the string. |
| 252 |
if(dlen) |
| 253 |
dest[dlen-1] = 0; |
| 254 |
goto utf8_ascii_end; |
| 255 |
utf8_ascii_bad: |
| 256 |
dest = INT123_safer_realloc(dest, 0); |
| 257 |
utf8_ascii_end: |
| 258 |
*dest_ = dest; |
| 259 |
return dest ? strlen(dest)+1 : 0; |
| 260 |
} |
| 261 |
|
| 262 |
// Reduce UTF-8 data to 7-bit ASCII, dropping non-printable characters. |
| 263 |
// Non-printable ASCII == everything below 0x20 (space), including |
| 264 |
// line breaks. |
| 265 |
// Also: 0x7f (DEL) and the C1 chars. The C0 and C1 chars should just be |
| 266 |
// dropped, not rendered. Or should they? |
| 267 |
static size_t utf8_ascii_print(char **dest, const char *source) |
| 268 |
{ |
| 269 |
return utf8_ascii_work(dest, source, 0); |
| 270 |
} |
| 271 |
|
| 272 |
// Same as above, but keeping non-printable and control chars in the |
| 273 |
// 7 bit realm. |
| 274 |
static size_t utf8_ascii(char **dest, const char *source) |
| 275 |
{ |
| 276 |
return utf8_ascii_work(dest, source, 1); |
| 277 |
} |
| 278 |
|
| 279 |
size_t utf8outstr(char **dest_, const char *source, int to_terminal) |
| 280 |
{ |
| 281 |
if(!dest_) |
| 282 |
return 0; |
| 283 |
if(!source) |
| 284 |
{ |
| 285 |
*dest_ = INT123_safer_realloc(*dest_, 0); |
| 286 |
return 0; |
| 287 |
} |
| 288 |
char *dest = *dest_; |
| 289 |
size_t width = 0; |
| 290 |
size_t source_fill = strlen(source)+1; |
| 291 |
|
| 292 |
if(utf8env) |
| 293 |
{ |
| 294 |
#if defined(HAVE_MBSTOWCS) && defined(HAVE_WCSWIDTH) && \ |
| 295 |
defined(HAVE_ISWPRINT) && defined(HAVE_WCSTOMBS) |
| 296 |
if(utf8loc && to_terminal) |
| 297 |
{ |
| 298 |
// Best case scenario: Convert to wide string, filter, |
| 299 |
// compute printing width. |
| 300 |
size_t wcharlen = mbstowcs(NULL, source, 0); |
| 301 |
if(wcharlen == (size_t)-1) |
| 302 |
goto utf8outstr_bad; |
| 303 |
if(wcharlen+1 > SIZE_MAX/sizeof(wchar_t)) |
| 304 |
goto utf8outstr_bad; |
| 305 |
wchar_t *pre = malloc(sizeof(wchar_t)*(wcharlen+1)); |
| 306 |
wchar_t *flt = malloc(sizeof(wchar_t)*(wcharlen+1)); |
| 307 |
if(!pre || !flt) |
| 308 |
{ |
| 309 |
free(flt); |
| 310 |
free(pre); |
| 311 |
goto utf8outstr_bad; |
| 312 |
} |
| 313 |
if(mbstowcs(pre, source, wcharlen+1) == wcharlen) |
| 314 |
{ |
| 315 |
size_t nwl = 0; |
| 316 |
int wasspace = 0; |
| 317 |
for(size_t i=0; i<wcharlen; ++i) |
| 318 |
{ |
| 319 |
// Turn any funky space sequence (including line breaks) into |
| 320 |
// one normal space. |
| 321 |
if(iswspace(pre[i]) && pre[i] != ' ') |
| 322 |
{ |
| 323 |
if(!wasspace) |
| 324 |
flt[nwl++] = ' '; |
| 325 |
wasspace = 1; |
| 326 |
} else // Anything non-printing is skipped. |
| 327 |
{ |
| 328 |
if(iswprint(pre[i])) |
| 329 |
flt[nwl++] = pre[i]; |
| 330 |
wasspace = 0; |
| 331 |
} |
| 332 |
} |
| 333 |
flt[nwl] = 0; |
| 334 |
int columns = wcswidth(flt, nwl); |
| 335 |
size_t bytelen = wcstombs(NULL, flt, 0); |
| 336 |
if( |
| 337 |
columns >= 0 && bytelen != (size_t)-1 |
| 338 |
&& (dest=INT123_safer_realloc(dest, bytelen+1)) |
| 339 |
&& wcstombs(dest, flt, bytelen+1) == bytelen |
| 340 |
){ |
| 341 |
width = columns; |
| 342 |
} |
| 343 |
else |
| 344 |
dest=INT123_safer_realloc(dest, 0); |
| 345 |
} |
| 346 |
free(flt); |
| 347 |
free(pre); |
| 348 |
} |
| 349 |
else |
| 350 |
#endif |
| 351 |
if(to_terminal) |
| 352 |
{ |
| 353 |
// Only filter C0 and C1 control characters. |
| 354 |
// That is, 0x01 to 0x19 (keeping 0x20, space) and 0x7f (DEL) to 0x9f. |
| 355 |
// Since the input and output is UTF-8, we'll keep that intact. |
| 356 |
// C1 is mapped to 0xc280 till 0xc29f. |
| 357 |
dest = INT123_safer_realloc(dest, source_fill); |
| 358 |
if(!dest) |
| 359 |
goto utf8outstr_bad; |
| 360 |
size_t dest_fill = 0; |
| 361 |
int c2lead = 0; |
| 362 |
int wasspace = 0; |
| 363 |
unsigned char *p = (unsigned char*)dest; |
| 364 |
for(size_t i=0; i<source_fill; ++i) |
| 365 |
{ |
| 366 |
unsigned char c = ((unsigned char*)source)[i]; |
| 367 |
ascii_space(&c, &wasspace); |
| 368 |
ASCII_C1(c, *p++ = 0xc2) |
| 369 |
if(c && c < 0x20) |
| 370 |
continue; // no C0 control chars, except space |
| 371 |
if(c == 0x7f) |
| 372 |
continue; // also no DEL |
| 373 |
*p++ = c; |
| 374 |
if(!c) |
| 375 |
break; // Up to zero is enough. |
| 376 |
// Assume each 7 bit char and each sequence start make one character. |
| 377 |
// So only continuation bytes need to be ignored. |
| 378 |
if((c & 0xc0) != 0x80) |
| 379 |
++width; |
| 380 |
} |
| 381 |
// Make damn sure that it ends. |
| 382 |
dest_fill = (char*)p - dest; |
| 383 |
dest[dest_fill-1] = 0; |
| 384 |
} else |
| 385 |
{ |
| 386 |
dest = INT123_safer_realloc(dest, source_fill); |
| 387 |
if(!dest) |
| 388 |
goto utf8outstr_bad; |
| 389 |
size_t dest_fill = 0; |
| 390 |
unsigned char *p = (unsigned char*)dest; |
| 391 |
for(size_t i=0; i<source_fill; ++i) |
| 392 |
{ |
| 393 |
unsigned char c = ((unsigned char*)source)[i]; |
| 394 |
*p++ = c; |
| 395 |
if(!c) |
| 396 |
break; // Up to zero is enough. |
| 397 |
// Actual width should not matter that much for non-terminal, |
| 398 |
// as we should use less formatting in that case, but anyway. |
| 399 |
if((c & 0xc0) != 0x80) |
| 400 |
++width; |
| 401 |
} |
| 402 |
dest_fill = (char*)p - dest; |
| 403 |
dest[dest_fill-1] = 0; |
| 404 |
} |
| 405 |
} else if(to_terminal) |
| 406 |
{ |
| 407 |
// Last resort: just 7-bit ASCII. |
| 408 |
width = to_terminal |
| 409 |
? utf8_ascii_print(&dest, source) |
| 410 |
: utf8_ascii(&dest, source); |
| 411 |
if(!width) |
| 412 |
goto utf8outstr_bad; |
| 413 |
--width; |
| 414 |
} |
| 415 |
|
| 416 |
goto utf8outstr_end; |
| 417 |
utf8outstr_bad: |
| 418 |
dest = INT123_safer_realloc(dest, 0); |
| 419 |
width = 0; |
| 420 |
utf8outstr_end: |
| 421 |
*dest_ = dest; |
| 422 |
return width; |
| 423 |
} |
| 424 |
|
| 425 |
#undef ASCII_C1 |
| 426 |
|
| 427 |
// I tried saving some malloc using provided work buffers, but |
| 428 |
// realized that the path of Unicode transformations is so full |
| 429 |
// of them regardless. |
| 430 |
// Can this include all the necessary logic? |
| 431 |
// - If UTF-8 input: Use utf8outstr(), which includes terminal switch. |
| 432 |
// - If not: |
| 433 |
// -- If terminal: construct safe UTF-8, pass on to outstr(). |
| 434 |
// -- If not: assume env encoding, unprocessed string that came |
| 435 |
// from the environment. |
| 436 |
|
| 437 |
int outstr(char **dest, const char *str, int is_utf8, int is_term) |
| 438 |
{ |
| 439 |
if(!dest) |
| 440 |
return -1; |
| 441 |
if(!str) |
| 442 |
{ |
| 443 |
*dest = INT123_safer_realloc(*dest, 0); |
| 444 |
return -1; |
| 445 |
} |
| 446 |
int ret = 0; |
| 447 |
if(is_utf8 || utf8env) |
| 448 |
{ |
| 449 |
utf8outstr(dest, str, is_term); |
| 450 |
if(*dest) |
| 451 |
ret = -1; |
| 452 |
} else if(is_term) |
| 453 |
{ |
| 454 |
char *usrc = NULL; |
| 455 |
ret = unknown2utf8(&usrc, str, -1); |
| 456 |
if(!ret) |
| 457 |
{ |
| 458 |
utf8outstr(dest, usrc, is_term); |
| 459 |
if(!*dest) |
| 460 |
ret = -1; |
| 461 |
} |
| 462 |
free(usrc); |
| 463 |
} else |
| 464 |
{ |
| 465 |
*dest = INT123_compat_strdup(str); |
| 466 |
if(!*dest) |
| 467 |
ret = -1; |
| 468 |
} |
| 469 |
return ret; |
| 470 |
} |
| 471 |
|
| 472 |
int print_outstr(FILE *out, const char *str, int is_utf8, int is_term) |
| 473 |
{ |
| 474 |
int ret = 0; |
| 475 |
if(!str) |
| 476 |
return -1; |
| 477 |
char *outbuf = NULL; |
| 478 |
ret = outstr(&outbuf, str, is_utf8, is_term); |
| 479 |
if(outbuf) |
| 480 |
{ |
| 481 |
ret = fprintf(out, "%s", outbuf); |
| 482 |
free(outbuf); |
| 483 |
} |
| 484 |
return ret; |
| 485 |
} |