Debian Codegraph @

modules/lsp-runtime-lib/src/main/io/charset.cpp

raw
1 /*
2 * Copyright (C) 2020 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2020 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-runtime-lib
6 * Created on: 18 июн. 2018 г.
7 *
8 * lsp-runtime-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-runtime-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-runtime-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #include <lsp-plug.in/common/status.h>
23 #include <lsp-plug.in/common/endian.h>
24 #include <lsp-plug.in/stdlib/string.h>
25 #include <lsp-plug.in/stdlib/stdio.h>
26 #include <lsp-plug.in/io/charset.h>
27
28 #include <errno.h>
29 #include <stdlib.h>
30 #include <wctype.h>
31
32 #ifdef PLATFORM_WINDOWS
33 #include <windows.h>
34 #include <winnls.h>
35 #endif /* PLATFORM_WINDOWS */
36
37 namespace lsp
38 {
39 #if defined(PLATFORM_WINDOWS)
40 typedef struct codepage_t
41 {
42 const char *name;
43 size_t codepage;
44 } codepage_t;
45
46 // This is a generated list of codepages supported by Windows,
47 // see script: scripts/perl/core/oi/charset/gen_cp.pl
48 static const codepage_t win_codepages[] = {
49 { "037", 37 },
50 { "10000", 10000 },
51 { "10001", 10001 },
52 { "10002", 10002 },
53 { "10003", 10003 },
54 { "10004", 10004 },
55 { "10005", 10005 },
56 { "10006", 10006 },
57 { "10007", 10007 },
58 { "10008", 10008 },
59 { "10010", 10010 },
60 { "10017", 10017 },
61 { "10021", 10021 },
62 { "10029", 10029 },
63 { "10079", 10079 },
64 { "10081", 10081 },
65 { "10082", 10082 },
66 { "1026", 1026 },
67 { "1047", 1047 },
68 { "1140", 1140 },
69 { "1141", 1141 },
70 { "1142", 1142 },
71 { "1143", 1143 },
72 { "1144", 1144 },
73 { "1145", 1145 },
74 { "1146", 1146 },
75 { "1147", 1147 },
76 { "1148", 1148 },
77 { "1149", 1149 },
78 { "1200", 1200 },
79 { "12000", 12000 },
80 { "12001", 12001 },
81 { "1201", 1201 },
82 { "1250", 1250 },
83 { "1251", 1251 },
84 { "1252", 1252 },
85 { "1253", 1253 },
86 { "1254", 1254 },
87 { "1255", 1255 },
88 { "1256", 1256 },
89 { "1257", 1257 },
90 { "1258", 1258 },
91 { "1361", 1361 },
92 { "20000", 20000 },
93 { "20001", 20001 },
94 { "20002", 20002 },
95 { "20003", 20003 },
96 { "20004", 20004 },
97 { "20005", 20005 },
98 { "20105", 20105 },
99 { "20106", 20106 },
100 { "20107", 20107 },
101 { "20108", 20108 },
102 { "20127", 20127 },
103 { "20261", 20261 },
104 { "20269", 20269 },
105 { "20273", 20273 },
106 { "20277", 20277 },
107 { "20278", 20278 },
108 { "20280", 20280 },
109 { "20284", 20284 },
110 { "20285", 20285 },
111 { "20290", 20290 },
112 { "20297", 20297 },
113 { "20420", 20420 },
114 { "20423", 20423 },
115 { "20424", 20424 },
116 { "20833", 20833 },
117 { "20838", 20838 },
118 { "20866", 20866 },
119 { "20871", 20871 },
120 { "20880", 20880 },
121 { "20905", 20905 },
122 { "20924", 20924 },
123 { "20932", 20932 },
124 { "20936", 20936 },
125 { "20949", 20949 },
126 { "21025", 21025 },
127 { "21027", 21027 },
128 { "21866", 21866 },
129 { "28591", 28591 },
130 { "28592", 28592 },
131 { "28593", 28593 },
132 { "28594", 28594 },
133 { "28595", 28595 },
134 { "28596", 28596 },
135 { "28597", 28597 },
136 { "28598", 28598 },
137 { "28599", 28599 },
138 { "28603", 28603 },
139 { "28605", 28605 },
140 { "29001", 29001 },
141 { "37", 37 },
142 { "38598", 38598 },
143 { "437", 437 },
144 { "500", 500 },
145 { "50220", 50220 },
146 { "50221", 50221 },
147 { "50222", 50222 },
148 { "50225", 50225 },
149 { "50227", 50227 },
150 { "50229", 50229 },
151 { "50930", 50930 },
152 { "50931", 50931 },
153 { "50933", 50933 },
154 { "50935", 50935 },
155 { "50936", 50936 },
156 { "50937", 50937 },
157 { "50939", 50939 },
158 { "51932", 51932 },
159 { "51936", 51936 },
160 { "51949", 51949 },
161 { "51950", 51950 },
162 { "52936", 52936 },
163 { "54936", 54936 },
164 { "57002", 57002 },
165 { "57003", 57003 },
166 { "57004", 57004 },
167 { "57005", 57005 },
168 { "57006", 57006 },
169 { "57007", 57007 },
170 { "57008", 57008 },
171 { "57009", 57009 },
172 { "57010", 57010 },
173 { "57011", 57011 },
174 { "65000", 65000 },
175 { "65001", 65001 },
176 { "708", 708 },
177 { "709", 709 },
178 { "710", 710 },
179 { "720", 720 },
180 { "737", 737 },
181 { "775", 775 },
182 { "850", 850 },
183 { "852", 852 },
184 { "855", 855 },
185 { "857", 857 },
186 { "858", 858 },
187 { "860", 860 },
188 { "861", 861 },
189 { "862", 862 },
190 { "863", 863 },
191 { "864", 864 },
192 { "865", 865 },
193 { "866", 866 },
194 { "869", 869 },
195 { "870", 870 },
196 { "874", 874 },
197 { "875", 875 },
198 { "932", 932 },
199 { "936", 936 },
200 { "949", 949 },
201 { "950", 950 },
202 { "asmo-708", 708 },
203 { "asmo708", 708 },
204 { "big5", 950 },
205 { "cp-1025", 21025 },
206 { "cp-1250", 1250 },
207 { "cp-1251", 1251 },
208 { "cp-1252", 1252 },
209 { "cp-1253", 1253 },
210 { "cp-1254", 1254 },
211 { "cp-1255", 1255 },
212 { "cp-1256", 1256 },
213 { "cp-1257", 1257 },
214 { "cp-1258", 1258 },
215 { "cp-21027", 21027 },
216 { "cp-50229", 50229 },
217 { "cp-50930", 50930 },
218 { "cp-50931", 50931 },
219 { "cp-50933", 50933 },
220 { "cp-50935", 50935 },
221 { "cp-50936", 50936 },
222 { "cp-50937", 50937 },
223 { "cp-50939", 50939 },
224 { "cp-51950", 51950 },
225 { "cp-709", 709 },
226 { "cp-710", 710 },
227 { "cp-866", 866 },
228 { "cp-874", 874 },
229 { "cp-875", 875 },
230 { "cp1025", 21025 },
231 { "cp1250", 1250 },
232 { "cp1251", 1251 },
233 { "cp1252", 1252 },
234 { "cp1253", 1253 },
235 { "cp1254", 1254 },
236 { "cp1255", 1255 },
237 { "cp1256", 1256 },
238 { "cp1257", 1257 },
239 { "cp1258", 1258 },
240 { "cp21027", 21027 },
241 { "cp50229", 50229 },
242 { "cp50930", 50930 },
243 { "cp50931", 50931 },
244 { "cp50933", 50933 },
245 { "cp50935", 50935 },
246 { "cp50936", 50936 },
247 { "cp50937", 50937 },
248 { "cp50939", 50939 },
249 { "cp51950", 51950 },
250 { "cp709", 709 },
251 { "cp710", 710 },
252 { "cp866", 866 },
253 { "cp874", 874 },
254 { "cp875", 875 },
255 { "csiso2022jp", 50221 },
256 { "dos-720", 720 },
257 { "dos-862", 862 },
258 { "dos720", 720 },
259 { "dos862", 862 },
260 { "euc-cn", 51936 },
261 { "euc-jp", 51932 },
262 { "euc-kr", 51949 },
263 { "gb18030", 54936 },
264 { "gb2312", 936 },
265 { "hz-gb-2312", 52936 },
266 { "hz-gb2312", 52936 },
267 { "ibm-thai", 20838 },
268 { "ibm00858", 858 },
269 { "ibm00924", 20924 },
270 { "ibm01047", 1047 },
271 { "ibm01140", 1140 },
272 { "ibm01141", 1141 },
273 { "ibm01142", 1142 },
274 { "ibm01143", 1143 },
275 { "ibm01144", 1144 },
276 { "ibm01145", 1145 },
277 { "ibm01146", 1146 },
278 { "ibm01147", 1147 },
279 { "ibm01148", 1148 },
280 { "ibm01149", 1149 },
281 { "ibm037", 37 },
282 { "ibm1026", 1026 },
283 { "ibm273", 20273 },
284 { "ibm277", 20277 },
285 { "ibm278", 20278 },
286 { "ibm280", 20280 },
287 { "ibm284", 20284 },
288 { "ibm285", 20285 },
289 { "ibm290", 20290 },
290 { "ibm297", 20297 },
291 { "ibm420", 20420 },
292 { "ibm423", 20423 },
293 { "ibm424", 20424 },
294 { "ibm437", 437 },
295 { "ibm500", 500 },
296 { "ibm737", 737 },
297 { "ibm775", 775 },
298 { "ibm850", 850 },
299 { "ibm852", 852 },
300 { "ibm855", 855 },
301 { "ibm857", 857 },
302 { "ibm860", 860 },
303 { "ibm861", 861 },
304 { "ibm863", 863 },
305 { "ibm864", 864 },
306 { "ibm865", 865 },
307 { "ibm869", 869 },
308 { "ibm870", 870 },
309 { "ibm871", 20871 },
310 { "ibm880", 20880 },
311 { "ibm905", 20905 },
312 { "iso-2022-jp", 50222 },
313 { "iso-2022-kr", 50225 },
314 { "iso-2022jp", 50222 },
315 { "iso-2022kr", 50225 },
316 { "iso-8859-1", 28591 },
317 { "iso-8859-13", 28603 },
318 { "iso-8859-15", 28605 },
319 { "iso-8859-2", 28592 },
320 { "iso-8859-3", 28593 },
321 { "iso-8859-4", 28594 },
322 { "iso-8859-5", 28595 },
323 { "iso-8859-6", 28596 },
324 { "iso-8859-7", 28597 },
325 { "iso-8859-8", 28598 },
326 { "iso-8859-8-i", 38598 },
327 { "iso-8859-8i", 38598 },
328 { "iso-8859-9", 28599 },
329 { "iso2022-jp", 50222 },
330 { "iso2022-kr", 50225 },
331 { "iso2022jp", 50222 },
332 { "iso2022kr", 50225 },
333 { "iso8859-1", 28591 },
334 { "iso8859-13", 28603 },
335 { "iso8859-15", 28605 },
336 { "iso8859-2", 28592 },
337 { "iso8859-3", 28593 },
338 { "iso8859-4", 28594 },
339 { "iso8859-5", 28595 },
340 { "iso8859-6", 28596 },
341 { "iso8859-7", 28597 },
342 { "iso8859-8", 28598 },
343 { "iso8859-8-i", 38598 },
344 { "iso8859-8i", 38598 },
345 { "iso8859-9", 28599 },
346 { "johab", 1361 },
347 { "koi8-r", 20866 },
348 { "koi8-u", 21866 },
349 { "koi8r", 20866 },
350 { "koi8u", 21866 },
351 { "ks-c-5601-1987", 949 },
352 { "ks_c_5601-1987", 949 },
353 { "macintosh", 10000 },
354 { "shift-jis", 932 },
355 { "shift_jis", 932 },
356 { "unicodefffe", 1201 },
357 { "us-ascii", 20127 },
358 { "utf-16", 1200 },
359 { "utf-16be", 1201 },
360 { "utf-16le", 1200 },
361 { "utf-32", 12000 },
362 { "utf-32be", 12001 },
363 { "utf-32le", 12000 },
364 { "utf-7", 65000 },
365 { "utf-8", 65001 },
366 { "utf16", 1200 },
367 { "utf16-be", 1201 },
368 { "utf16-le", 1200 },
369 { "utf16be", 1201 },
370 { "utf16le", 1200 },
371 { "utf32", 12000 },
372 { "utf32-be", 12001 },
373 { "utf32-le", 12000 },
374 { "utf32be", 12001 },
375 { "utf32le", 12000 },
376 { "utf7", 65000 },
377 { "utf8", 65001 },
378 { "windows-1250", 1250 },
379 { "windows-1251", 1251 },
380 { "windows-1252", 1252 },
381 { "windows-1253", 1253 },
382 { "windows-1254", 1254 },
383 { "windows-1255", 1255 },
384 { "windows-1256", 1256 },
385 { "windows-1257", 1257 },
386 { "windows-1258", 1258 },
387 { "windows-874", 874 },
388 { "windows1250", 1250 },
389 { "windows1251", 1251 },
390 { "windows1252", 1252 },
391 { "windows1253", 1253 },
392 { "windows1254", 1254 },
393 { "windows1255", 1255 },
394 { "windows1256", 1256 },
395 { "windows1257", 1257 },
396 { "windows1258", 1258 },
397 { "windows874", 874 },
398 { "x-chinese-cns", 20000 },
399 { "x-chinese-eten", 20002 },
400 { "x-chinese_cns", 20000 },
401 { "x-cp-20001", 20001 },
402 { "x-cp-20003", 20003 },
403 { "x-cp-20004", 20004 },
404 { "x-cp-20005", 20005 },
405 { "x-cp-20261", 20261 },
406 { "x-cp-20269", 20269 },
407 { "x-cp-20936", 20936 },
408 { "x-cp-20949", 20949 },
409 { "x-cp-50227", 50227 },
410 { "x-cp20001", 20001 },
411 { "x-cp20003", 20003 },
412 { "x-cp20004", 20004 },
413 { "x-cp20005", 20005 },
414 { "x-cp20261", 20261 },
415 { "x-cp20269", 20269 },
416 { "x-cp20936", 20936 },
417 { "x-cp20949", 20949 },
418 { "x-cp50227", 50227 },
419 { "x-ebcdic-koreanextended", 20833 },
420 { "x-europa", 29001 },
421 { "x-ia5", 20105 },
422 { "x-ia5-german", 20106 },
423 { "x-ia5-norwegian", 20108 },
424 { "x-ia5-swedish", 20107 },
425 { "x-ia5german", 20106 },
426 { "x-ia5norwegian", 20108 },
427 { "x-ia5swedish", 20107 },
428 { "x-iscii-as", 57006 },
429 { "x-iscii-be", 57003 },
430 { "x-iscii-de", 57002 },
431 { "x-iscii-gu", 57010 },
432 { "x-iscii-ka", 57008 },
433 { "x-iscii-ma", 57009 },
434 { "x-iscii-or", 57007 },
435 { "x-iscii-pa", 57011 },
436 { "x-iscii-ta", 57004 },
437 { "x-iscii-te", 57005 },
438 { "x-mac-arabic", 10004 },
439 { "x-mac-ce", 10029 },
440 { "x-mac-chinesesimp", 10008 },
441 { "x-mac-chinesetrad", 10002 },
442 { "x-mac-croatian", 10082 },
443 { "x-mac-cyrillic", 10007 },
444 { "x-mac-greek", 10006 },
445 { "x-mac-hebrew", 10005 },
446 { "x-mac-icelandic", 10079 },
447 { "x-mac-japanese", 10001 },
448 { "x-mac-korean", 10003 },
449 { "x-mac-romanian", 10010 },
450 { "x-mac-thai", 10021 },
451 { "x-mac-turkish", 10081 },
452 { "x-mac-ukrainian", 10017 },
453 { "x_chinese-eten", 20002 }
454 };
455
456 ssize_t get_codepage(LCID locale, bool ansi)
457 {
458 char buf[32];
459
460 int res = GetLocaleInfoA(locale, (ansi) ? LOCALE_IDEFAULTANSICODEPAGE : LOCALE_IDEFAULTCODEPAGE, buf, sizeof(buf)-1);
461 if (res == 0)
462 {
463 switch (GetLastError())
464 {
465 case ERROR_INSUFFICIENT_BUFFER:
466 return -STATUS_NO_MEM;
467 case ERROR_INVALID_FLAGS:
468 case ERROR_INVALID_PARAMETER:
469 return -STATUS_BAD_ARGUMENTS;
470 default:
471 return -STATUS_UNKNOWN_ERR;
472 }
473 }
474
475 errno = 0;
476 ssize_t cp_num = strtol(buf, NULL, 10);
477 if (errno != 0)
478 return -STATUS_UNSUPPORTED_FORMAT;
479 return cp_num;
480 }
481
482 ssize_t codepage_from_name(const char *charset)
483 {
484 if (charset != NULL)
485 {
486 // Do lower-case the character set
487 size_t n = strlen(charset) + 1;
488 char *lower = static_cast<char *>(alloca(n));
489 for (size_t i=0; i<n; ++i)
490 lower[i] = tolower(charset[i]);
491
492 // Perform binary search of character set
493 size_t first = 0, last = sizeof(win_codepages)/sizeof(codepage_t);
494 while (first < last)
495 {
496 size_t middle = (first + last) >> 1;
497 int n = strcmp(lower, win_codepages[middle].name);
498 if (n == 0)
499 return win_codepages[middle].codepage;
500 else if (n < 0)
501 last = middle;
502 else
503 first = middle + 1;
504 }
505
506 return -1;
507 }
508
509 // printf("LOCALE_CUSTOM_DEFAULT = %d\n", int(get_codepage(LOCALE_CUSTOM_DEFAULT)));
510 // printf("LOCALE_USER_DEFAULT = %d\n", int(get_codepage(LOCALE_USER_DEFAULT)));
511 // printf("LOCALE_SYSTEM_DEFAULT = %d\n", int(get_codepage(LOCALE_SYSTEM_DEFAULT)));
512 // printf("LOCALE_CUSTOM_UNSPECIFIED = %d\n", int(get_codepage(LOCALE_CUSTOM_UNSPECIFIED)));
513 // printf("LOCALE_CUSTOM_UI_DEFAULT = %d\n", int(get_codepage(LOCALE_CUSTOM_UI_DEFAULT)));
514 // printf("LOCALE_INVARIANT = %d\n", int(get_codepage(LOCALE_INVARIANT)));
515 // printf("GetConsoleWindow() = %d\n", int(GetConsoleWindow()));
516 // printf("GetConsoleOutputCP() = %d\n", int(GetConsoleOutputCP()));
517 // fflush(stdout);
518
519 // Obtain system character set
520 //ssize_t cp = (GetConsoleWindow() != 0) ? GetConsoleOutputCP() : get_codepage(LOCALE_CUSTOM_DEFAULT);
521 ssize_t cp = get_codepage(LOCALE_CUSTOM_DEFAULT);
522 if (cp < 0)
523 cp = get_codepage(LOCALE_USER_DEFAULT);
524 if (cp < 0)
525 cp = get_codepage(LOCALE_SYSTEM_DEFAULT);
526 return cp;
527 }
528
529 #else
530 iconv_t init_iconv_to_wchar_t(const char *charset)
531 {
532 // Fetch system character set if it is not set
533 if (charset == NULL)
534 {
535 // Assume default locale being UTF-8
536 charset = "UTF-8";
537
538 // Get system locale
539 char *current = setlocale(LC_CTYPE, NULL);
540 if (current == NULL)
541 return iconv_t(-1);
542
543 // Scan for character set presence
544 current = strchr(current, '.');
545 if (current != NULL)
546 {
547 ++current;
548 if (strlen(current) > 0)
549 charset = current;
550 }
551 }
552
553 // Open conversion
554 iconv_t res = iconv_open(__IF_LEBE("UTF-32LE", "UTF-32BE"), charset);
555 if (res != iconv_t(-1))
556 return res;
557
558 res = iconv_open(__IF_LEBE("UTF-32LE", "UTF-32BE"), "UTF-8");
559 if (res != iconv_t(-1))
560 return res;
561
562 return iconv_open("WCHAR_T", "UTF-8");
563 }
564
565 iconv_t init_iconv_from_wchar_t(const char *charset)
566 {
567 // Fetch system charset if it is not set
568 if (charset == NULL)
569 {
570 // Assume default locale being UTF-8
571 charset = "UTF-8";
572
573 // Get system locale
574 char *current = setlocale(LC_CTYPE, NULL);
575 if (current == NULL)
576 return iconv_t(-1);
577
578 // Scan for character set presence
579 current = strchr(current, '.');
580 if (current != NULL)
581 {
582 ++current;
583 if (strlen(current) > 0)
584 charset = current;
585 }
586 }
587
588 // Open conversion
589 iconv_t res = iconv_open(charset, __IF_LEBE("UTF-32LE", "UTF-32BE"));
590 if (res != iconv_t(-1))
591 return res;
592
593 res = iconv_open("UTF-8", __IF_LEBE("UTF-32LE", "UTF-32BE"));
594 if (res != iconv_t(-1))
595 return res;
596
597 return iconv_open("UTF-8", "WCHAR_T");
598 }
599 #endif
600
601 //-------------------------------------------------------------------------
602 // UTF-16 helper routines
603 lsp_utf32_t read_utf16le_codepoint(const lsp_utf16_t **str)
604 {
605 uint32_t cp, sc;
606 const lsp_utf16_t *s = *str;
607
608 cp = LE_TO_CPU(*(s++));
609 if (cp == 0)
610 return cp;
611
612 sc = cp & 0xfc00;
613 if (sc == 0xd800) // cp = Surrogate high
614 {
615 sc = LE_TO_CPU(*s);
616 if ((sc & 0xfc00) == 0xdc00)
617 {
618 ++s;
619 cp = 0x10000 + (((cp & 0x3ff) << 10) | (sc & 0x3ff));
620 }
621 else
622 cp = 0xfffd;
623 }
624 else if (sc == 0xdc00) // Surrogate low?
625 {
626 sc = LE_TO_CPU(*s);
627 if ((sc & 0xfc00) == 0xd800)
628 {
629 ++s;
630 cp = 0x10000 + (((sc & 0x3ff) << 10) | (cp & 0x3ff));
631 }
632 else
633 cp = 0xfffd;
634 }
635
636 *str = s;
637 return cp;
638 }
639
640 lsp_utf32_t read_utf16be_codepoint(const lsp_utf16_t **str)
641 {
642 uint32_t cp, sc;
643 const lsp_utf16_t *s = *str;
644
645 cp = BE_TO_CPU(*(s++));
646 if (cp == 0)
647 return cp;
648
649 sc = cp & 0xfc00;
650 if (sc == 0xd800) // cp = Surrogate high
651 {
652 sc = BE_TO_CPU(*s);
653 if ((sc & 0xfc00) == 0xdc00)
654 {
655 ++s;
656 cp = 0x10000 + (((cp & 0x3ff) << 10) | (sc & 0x3ff));
657 }
658 else
659 cp = 0xfffd;
660 }
661 else if (sc == 0xdc00) // Surrogate low?
662 {
663 sc = BE_TO_CPU(*s);
664 if ((sc & 0xfc00) == 0xd800)
665 {
666 ++s;
667 cp = 0x10000 + (((sc & 0x3ff) << 10) | (cp & 0x3ff));
668 }
669 else
670 cp = 0xfffd;
671 }
672
673 *str = s;
674 return cp;
675 }
676
677 lsp_utf32_t read_utf16le_streaming(const lsp_utf16_t **str, size_t *nsrc, bool force)
678 {
679 if (*nsrc <= 0)
680 return LSP_UTF32_EOF;
681
682 uint32_t cp, sc;
683 const lsp_utf16_t *s = *str;
684
685 cp = LE_TO_CPU(*(s++));
686 sc = cp & 0xfc00;
687 if (sc == 0xd800) // cp = Surrogate high
688 {
689 if (*nsrc > 1)
690 sc = LE_TO_CPU(*s);
691 else if (force)
692 sc = 0;
693 else
694 return LSP_UTF32_EOF;
695
696 if ((sc & 0xfc00) == 0xdc00)
697 {
698 ++s;
699 cp = 0x10000 + (((cp & 0x3ff) << 10) | (sc & 0x3ff));
700 }
701 else
702 cp = 0xfffd;
703 }
704 else if (sc == 0xdc00) // Surrogate low?
705 {
706 if (*nsrc > 1)
707 sc = LE_TO_CPU(*s);
708 else if (force)
709 sc = 0;
710 else
711 return LSP_UTF32_EOF;
712
713 if ((sc & 0xfc00) == 0xd800)
714 {
715 ++s;
716 cp = 0x10000 + (((sc & 0x3ff) << 10) | (cp & 0x3ff));
717 }
718 else
719 cp = 0xfffd;
720 }
721
722 *nsrc -= (s - *str);
723 *str = s;
724 return cp;
725 }
726
727 lsp_utf32_t read_utf16be_streaming(const lsp_utf16_t **str, size_t *nsrc, bool force)
728 {
729 if (*nsrc <= 0)
730 return LSP_UTF32_EOF;
731
732 uint32_t cp, sc;
733 const lsp_utf16_t *s = *str;
734
735 cp = BE_TO_CPU(*(s++));
736 sc = cp & 0xfc00;
737 if (sc == 0xd800) // cp = Surrogate high
738 {
739 if (*nsrc > 1)
740 sc = BE_TO_CPU(*s);
741 else if (force)
742 sc = 0;
743 else
744 return LSP_UTF32_EOF;
745
746 if ((sc & 0xfc00) == 0xdc00)
747 {
748 ++s;
749 cp = 0x10000 + (((cp & 0x3ff) << 10) | (sc & 0x3ff));
750 }
751 else
752 cp = 0xfffd;
753 }
754 else if (sc == 0xdc00) // Surrogate low?
755 {
756 if (*nsrc > 1)
757 sc = BE_TO_CPU(*s);
758 else if (force)
759 sc = 0;
760 else
761 return LSP_UTF32_EOF;
762
763 if ((sc & 0xfc00) == 0xd800)
764 {
765 ++s;
766 cp = 0x10000 + (((sc & 0x3ff) << 10) | (cp & 0x3ff));
767 }
768 else
769 cp = 0xfffd;
770 }
771
772 *nsrc -= (s - *str);
773 *str = s;
774 return cp;
775 }
776
777 inline size_t sizeof_utf16(lsp_utf32_t cp)
778 {
779 return (cp < 0x10000) ? 2 : 4;
780 }
781
782 inline size_t count_utf16(lsp_utf32_t cp)
783 {
784 return (cp < 0x10000) ? 1 : 2;
785 }
786
787 void write_utf16le_codepoint(lsp_utf16_t **str, lsp_utf32_t cp)
788 {
789 lsp_utf16_t *dst = *str;
790 if (cp < 0x10000)
791 *(dst++) = CPU_TO_LE(lsp_utf16_t(cp));
792 else
793 {
794 cp -= 0x10000;
795 dst[0] = CPU_TO_LE(lsp_utf16_t(0xd800 | (cp >> 10)));
796 dst[1] = CPU_TO_LE(lsp_utf16_t(0xdc00 | (cp & 0x3ff)));
797 dst += 2;
798 }
799 *str = dst;
800 }
801
802 void write_utf16be_codepoint(lsp_utf16_t **str, lsp_utf32_t cp)
803 {
804 lsp_utf16_t *dst = *str;
805 if (cp < 0x10000)
806 *(dst++) = CPU_TO_BE(lsp_utf16_t(cp));
807 else
808 {
809 cp -= 0x10000;
810 dst[0] = CPU_TO_BE(lsp_utf16_t(0xd800 | (cp >> 10)));
811 dst[1] = CPU_TO_BE(lsp_utf16_t(0xdc00 | (cp & 0x3ff)));
812 dst += 2;
813 }
814 *str = dst;
815 }
816
817 //-------------------------------------------------------------------------
818 // UTF-8 helper routines
819 lsp_utf32_t read_utf8_codepoint(const char **str)
820 {
821 lsp_utf32_t cp, sp;
822 size_t bytes;
823 const char *s = *str;
824
825 // Decode primary byte
826 cp = uint8_t(*s);
827 if (cp <= 0x7f)
828 {
829 *str = (cp == 0) ? s : s+1;
830 return cp;
831 }
832
833 ++s;
834 if ((cp & 0xe0) == 0xc0) // 2 bytes: 110xxxxx 10xxxxxx
835 {
836 cp &= 0x1f;
837 bytes = (cp >= 0x02) ? 1 : 0;
838 }
839 else if ((cp & 0xf0) == 0xe0) // 3 bytes: 1110xxxx 10xxxxxx 10xxxxxx
840 {
841 cp &= 0x0f;
842 bytes = (cp) ? 2 : 0;
843 }
844 else if ((cp & 0xf8) == 0xf0) // 4 bytes: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
845 {
846 cp &= 0x07;
847 bytes = 3;
848 }
849 else
850 bytes = 0;
851
852 // Invalid first byte sequence?
853 if (!bytes)
854 {
855 *str = s;
856 return 0xfffd;
857 }
858
859 // Decode extension bytes
860 for (size_t i=0; i<bytes; ++i)
861 {
862 sp = uint8_t(*s);
863 if ((sp & 0xc0) != 0x80) // Invalid sequence?
864 {
865 *str = (sp == 0) ? s : s+1;
866 return 0xfffd;
867 }
868 cp = (cp << 6) | (sp & 0x3f);
869 ++s;
870 }
871
872 if ((bytes == 3) && (cp < 0x10000)) // Check that 4-byte sequence is valid
873 cp = 0xfffd;
874 else if ((cp >= 0xd800) && (cp < 0xe000)) // Check for surrogates
875 cp = 0xfffd;
876
877 *str = s;
878 return cp;
879 }
880
881 lsp_utf32_t read_utf8_streaming(const char **str, size_t *nsrc, bool force)
882 {
883 if (*nsrc <= 0)
884 return LSP_UTF32_EOF;
885
886 lsp_utf32_t cp, sp;
887 size_t bytes;
888 const char *s = *str;
889
890 // Decode primary byte
891 cp = uint8_t(*s);
892 if (cp <= 0x7f)
893 {
894 *str = (cp == 0) ? s : s+1;
895 --(*nsrc);
896 return cp;
897 }
898
899 // Multi-byte sequence
900 ++s;
901 if ((cp & 0xe0) == 0xc0) // 2 bytes: 110xxxxx 10xxxxxx
902 {
903 cp &= 0x1f;
904 bytes = (cp >= 0x02) ? 1 : 0;
905 }
906 else if ((cp & 0xf0) == 0xe0) // 3 bytes: 1110xxxx 10xxxxxx 10xxxxxx
907 {
908 cp &= 0x0f;
909 bytes = (cp) ? 2 : 0;
910 }
911 else if ((cp & 0xf8) == 0xf0) // 4 bytes: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
912 {
913 cp &= 0x07;
914 bytes = 3;
915 }
916 else
917 bytes = 0;
918
919 // Invalid first byte sequence?
920 if (!bytes)
921 {
922 *str = s;
923 --(*nsrc);
924 return 0xfffd;
925 }
926 else if (bytes >= *nsrc)
927 {
928 if (force)
929 {
930 *nsrc = 0;
931 return 0xfffd;
932 }
933 return LSP_UTF32_EOF;
934 }
935
936 // Decode extension bytes
937 for (size_t i=0; i<bytes; ++i)
938 {
939 sp = uint8_t(*s);
940 if ((sp & 0xc0) != 0x80) // Invalid sequence?
941 {
942 if (sp == 0)
943 ++s;
944 *nsrc -= (s - *str);
945 *str = s;
946 return 0xfffd;
947 }
948 cp = (cp << 6) | (sp & 0x3f);
949 ++s;
950 }
951
952 if ((bytes == 3) && (cp < 0x10000)) // Check that 4-byte sequence is valid
953 cp = 0xfffd;
954 else if ((cp >= 0xd800) && (cp < 0xe000)) // Check for surrogates
955 cp = 0xfffd;
956
957 *nsrc -= (s - *str);
958 *str = s;
959 return cp;
960 }
961
962 inline size_t sizeof_utf8(lsp_utf32_t cp)
963 {
964 if (cp >= 0x800)
965 return ((cp < 0x10000) || (cp >= 0x200000)) ? 3 : 4;
966 else
967 return (cp >= 0x80) ? 2 : 1;
968 }
969
970 inline size_t count_utf8(lsp_utf32_t cp)
971 {
972 if (cp >= 0x800)
973 return ((cp < 0x10000) || (cp >= 0x200000)) ? 3 : 4;
974 else
975 return (cp >= 0x80) ? 2 : 1;
976 }
977
978 void write_utf8_codepoint(char **str, lsp_utf32_t cp)
979 {
980 char *dst = *str;
981 if (cp >= 0x800) // 3-4 bytes
982 {
983 if (cp < 0x10000) // 3 bytes
984 {
985 dst[0] = (cp >> 12) | 0xe0;
986 dst[1] = ((cp >> 6) & 0x3f) | 0x80;
987 dst[2] = (cp & 0x3f) | 0x80;
988 dst += 3;
989 }
990 else if (cp < 0x200000) // 4 bytes
991 {
992 dst[0] = (cp >> 16) | 0xf0;
993 dst[1] = ((cp >> 12) & 0x3f) | 0x80;
994 dst[2] = ((cp >> 6) & 0x3f) | 0x80;
995 dst[3] = (cp & 0x3f) | 0x80;
996 dst += 4;
997 }
998 else // Invalid character, emit 3 bytes of 0xfffd code point value
999 {
1000 dst[0] = 0xef;
1001 dst[1] = 0xbf;
1002 dst[2] = 0xbd;
1003 dst += 3;
1004 }
1005 }
1006 else // 1-2 bytes
1007 {
1008 if (cp >= 0x80) // 2 bytes
1009 {
1010 dst[0] = (cp >> 6) | 0xc0;
1011 dst[1] = (cp & 0x3f) | 0x80;
1012 dst += 2;
1013 }
1014 else // 1 byte
1015 *(dst++) = char(cp);
1016 }
1017 *str = dst;
1018 }
1019
1020 //-------------------------------------------------------------------------
1021 // UTF-8 non-streaming routines
1022 lsp_utf16_t *utf8_to_utf16le(const char *str)
1023 {
1024 // Estimate number of bytes
1025 lsp_utf32_t cp;
1026 size_t bytes = 0;
1027 const char *p = str;
1028 do
1029 {
1030 cp = read_utf8_codepoint(&p);
1031 bytes += sizeof_utf16(cp);
1032 } while (cp != 0);
1033
1034 // Allocate memory
1035 lsp_utf16_t *utf16 = reinterpret_cast<lsp_utf16_t *>(::malloc(bytes));
1036 if (utf16 == NULL)
1037 return NULL;
1038
1039 // Perform encoding
1040 lsp_utf16_t *dst = utf16;
1041 p = str;
1042 while ((cp = read_utf8_codepoint(&p)) != 0)
1043 write_utf16le_codepoint(&dst, cp);
1044 *dst = 0;
1045
1046 return utf16;
1047 }
1048
1049 lsp_utf16_t *utf8_to_utf16be(const char *str)
1050 {
1051 // Estimate number of bytes
1052 lsp_utf32_t cp;
1053 size_t bytes = 0;
1054 const char *p = str;
1055 do
1056 {
1057 cp = read_utf8_codepoint(&p);
1058 bytes += sizeof_utf16(cp);
1059 } while (cp != 0);
1060
1061 // Allocate memory
1062 lsp_utf16_t *utf16 = reinterpret_cast<lsp_utf16_t *>(::malloc(bytes));
1063 if (utf16 == NULL)
1064 return NULL;
1065
1066 // Perform encoding
1067 lsp_utf16_t *dst = utf16;
1068 p = str;
1069 while ((cp = read_utf8_codepoint(&p)) != 0)
1070 write_utf16be_codepoint(&dst, cp);
1071 *dst = 0;
1072
1073 return utf16;
1074 }
1075
1076 size_t utf8_to_utf16le(lsp_utf16_t *dst, const char *str, size_t count)
1077 {
1078 lsp_utf32_t cp;
1079 size_t items = 0;
1080 do
1081 {
1082 cp = read_utf8_codepoint(&str);
1083 items += count_utf16(cp);
1084 if (items > count)
1085 return 0;
1086
1087 write_utf16le_codepoint(&dst, cp);
1088 } while (cp != 0);
1089
1090 return items;
1091 }
1092
1093 size_t utf8_to_utf16be(lsp_utf16_t *dst, const char *str, size_t count)
1094 {
1095 lsp_utf32_t cp;
1096 size_t items = 0;
1097 do
1098 {
1099 cp = read_utf8_codepoint(&str);
1100 items += count_utf16(cp);
1101 if (items > count)
1102 return 0;
1103
1104 write_utf16be_codepoint(&dst, cp);
1105 } while (cp != 0);
1106
1107 return items;
1108 }
1109
1110 lsp_utf32_t *utf8_to_utf32le(const char *str)
1111 {
1112 // Estimate number of bytes
1113 lsp_utf32_t cp;
1114 size_t bytes = 0;
1115 const char *p = str;
1116 do
1117 {
1118 cp = read_utf8_codepoint(&p);
1119 bytes += sizeof(lsp_utf32_t);
1120 } while (cp != 0);
1121
1122 // Allocate memory
1123 lsp_utf32_t *utf32 = reinterpret_cast<lsp_utf32_t *>(::malloc(bytes));
1124 if (utf32 == NULL)
1125 return NULL;
1126
1127 // Perform encoding
1128 lsp_utf32_t *dst = utf32;
1129 p = str;
1130 while ((cp = read_utf8_codepoint(&p)) != 0)
1131 *(dst++) = CPU_TO_LE(cp);
1132 *dst = 0;
1133
1134 return utf32;
1135 }
1136
1137 lsp_utf32_t *utf8_to_utf32be(const char *str)
1138 {
1139 // Estimate number of bytes
1140 lsp_utf32_t cp;
1141 size_t bytes = 0;
1142 const char *p = str;
1143 do
1144 {
1145 cp = read_utf8_codepoint(&p);
1146 bytes += sizeof(lsp_utf32_t);
1147 } while (cp != 0);
1148
1149 // Allocate memory
1150 lsp_utf32_t *utf32 = reinterpret_cast<lsp_utf32_t *>(::malloc(bytes));
1151 if (utf32 == NULL)
1152 return NULL;
1153
1154 // Perform encoding
1155 lsp_utf32_t *dst = utf32;
1156 p = str;
1157 while ((cp = read_utf8_codepoint(&p)) != 0)
1158 *(dst++) = CPU_TO_BE(cp);
1159 *dst = 0;
1160
1161 return utf32;
1162 }
1163
1164 size_t utf8_to_utf32le(lsp_utf32_t *dst, const char *str, size_t count)
1165 {
1166 lsp_utf32_t cp;
1167 size_t items = 0;
1168 do
1169 {
1170 cp = read_utf8_codepoint(&str);
1171 if (++items > count)
1172 return 0;
1173 *(dst++) = CPU_TO_LE(cp);
1174 } while (cp != 0);
1175
1176 return items;
1177 }
1178
1179 size_t utf8_to_utf32be(lsp_utf32_t *dst, const char *str, size_t count)
1180 {
1181 lsp_utf32_t cp;
1182 size_t items = 0;
1183 do
1184 {
1185 cp = read_utf8_codepoint(&str);
1186 if (++items > count)
1187 return 0;
1188 *(dst++) = CPU_TO_BE(cp);
1189 } while (cp != 0);
1190
1191 return items;
1192 }
1193
1194 //-------------------------------------------------------------------------
1195 // UTF-16 non-streaming routines
1196 char *utf16le_to_utf8(const lsp_utf16_t *str)
1197 {
1198 // Estimate number of bytes
1199 lsp_utf32_t cp;
1200 size_t bytes = 0;
1201 const lsp_utf16_t *p = str;
1202 do
1203 {
1204 cp = read_utf16le_codepoint(&p);
1205 bytes += sizeof_utf8(cp);
1206 } while (cp != 0);
1207
1208 // Allocate memory
1209 char *utf8 = reinterpret_cast<char *>(::malloc(bytes));
1210 if (utf8 == NULL)
1211 return NULL;
1212
1213 // Now perform encoding
1214 char *dst = utf8;
1215 p = str;
1216 while ((cp = read_utf16le_codepoint(&p)) != 0)
1217 write_utf8_codepoint(&dst, cp);
1218 *dst = '\0';
1219
1220 return utf8;
1221 }
1222
1223 char *utf16be_to_utf8(const lsp_utf16_t *str)
1224 {
1225 // Estimate number of bytes
1226 lsp_utf32_t cp;
1227 size_t bytes = 0;
1228 const lsp_utf16_t *p = str;
1229 do
1230 {
1231 cp = read_utf16be_codepoint(&p);
1232 bytes += sizeof_utf8(cp);
1233 } while (cp != 0);
1234
1235 // Allocate memory
1236 char *utf8 = reinterpret_cast<char *>(::malloc(bytes));
1237 if (utf8 == NULL)
1238 return NULL;
1239
1240 // Now perform encoding
1241 char *dst = utf8;
1242 p = str;
1243 while ((cp = read_utf16be_codepoint(&p)) != 0)
1244 write_utf8_codepoint(&dst, cp);
1245 *dst = '\0';
1246
1247 return utf8;
1248 }
1249
1250 size_t utf16le_to_utf8(char *dst, const lsp_utf16_t *str, size_t count)
1251 {
1252 lsp_utf32_t cp;
1253 size_t items = 0;
1254 do
1255 {
1256 cp = read_utf16le_codepoint(&str);
1257 items += count_utf8(cp);
1258 if (items > count)
1259 return 0;
1260 write_utf8_codepoint(&dst, cp);
1261 } while (cp != 0);
1262
1263 return items;
1264 }
1265
1266 size_t utf16be_to_utf8(char *dst, const lsp_utf16_t *str, size_t count)
1267 {
1268 lsp_utf32_t cp;
1269 size_t items = 0;
1270 do
1271 {
1272 cp = read_utf16be_codepoint(&str);
1273 items += count_utf8(cp);
1274 if (items > count)
1275 return 0;
1276 write_utf8_codepoint(&dst, cp);
1277 } while (cp != 0);
1278
1279 return items;
1280 }
1281
1282 lsp_utf32_t *utf16le_to_utf32le(const lsp_utf16_t *str)
1283 {
1284 // Estimate number of bytes
1285 lsp_utf32_t cp;
1286 size_t bytes = 0;
1287 const lsp_utf16_t *p = str;
1288 do
1289 {
1290 cp = read_utf16le_codepoint(&p);
1291 bytes += sizeof(lsp_utf32_t);
1292 } while (cp != 0);
1293
1294 // Allocate memory
1295 lsp_utf32_t *utf32 = reinterpret_cast<lsp_utf32_t *>(::malloc(bytes));
1296 if (utf32 == NULL)
1297 return NULL;
1298
1299 // Perform encoding
1300 p = str;
1301 lsp_utf32_t *dst= utf32;
1302 while ((cp = read_utf16le_codepoint(&p)) != 0)
1303 *(dst++) = CPU_TO_LE(cp);
1304 *dst = 0;
1305
1306 return utf32;
1307 }
1308
1309 lsp_utf32_t *utf16le_to_utf32be(const lsp_utf16_t *str)
1310 {
1311 // Estimate number of bytes
1312 lsp_utf32_t cp;
1313 size_t bytes = 0;
1314 const lsp_utf16_t *p = str;
1315 do
1316 {
1317 cp = read_utf16le_codepoint(&p);
1318 bytes += sizeof(lsp_utf32_t);
1319 } while (cp != 0);
1320
1321 // Allocate memory
1322 lsp_utf32_t *utf32 = reinterpret_cast<lsp_utf32_t *>(::malloc(bytes));
1323 if (utf32 == NULL)
1324 return NULL;
1325
1326 // Perform encoding
1327 p = str;
1328 lsp_utf32_t *dst= utf32;
1329 while ((cp = read_utf16le_codepoint(&p)) != 0)
1330 *(dst++) = CPU_TO_BE(cp);
1331 *dst = 0;
1332
1333 return utf32;
1334 }
1335
1336 lsp_utf32_t *utf16be_to_utf32le(const lsp_utf16_t *str)
1337 {
1338 // Estimate number of bytes
1339 lsp_utf32_t cp;
1340 size_t bytes = 0;
1341 const lsp_utf16_t *p = str;
1342 do
1343 {
1344 cp = read_utf16be_codepoint(&p);
1345 bytes += sizeof(lsp_utf32_t);
1346 } while (cp != 0);
1347
1348 // Allocate memory
1349 lsp_utf32_t *utf32 = reinterpret_cast<lsp_utf32_t *>(::malloc(bytes));
1350 if (utf32 == NULL)
1351 return NULL;
1352
1353 // Perform encoding
1354 p = str;
1355 lsp_utf32_t *dst= utf32;
1356 while ((cp = read_utf16be_codepoint(&p)) != 0)
1357 *(dst++) = CPU_TO_LE(cp);
1358 *dst = 0;
1359
1360 return utf32;
1361 }
1362
1363 lsp_utf32_t *utf16be_to_utf32be(const lsp_utf16_t *str)
1364 {
1365 // Estimate number of bytes
1366 lsp_utf32_t cp;
1367 size_t bytes = 0;
1368 const lsp_utf16_t *p = str;
1369 do
1370 {
1371 cp = read_utf16be_codepoint(&p);
1372 bytes += sizeof(lsp_utf32_t);
1373 } while (cp != 0);
1374
1375 // Allocate memory
1376 lsp_utf32_t *utf32 = reinterpret_cast<lsp_utf32_t *>(::malloc(bytes));
1377 if (utf32 == NULL)
1378 return NULL;
1379
1380 // Perform encoding
1381 p = str;
1382 lsp_utf32_t *dst= utf32;
1383 while ((cp = read_utf16be_codepoint(&p)) != 0)
1384 *(dst++) = CPU_TO_BE(cp);
1385 *dst = 0;
1386
1387 return utf32;
1388 }
1389
1390 size_t utf16le_to_utf32le(lsp_utf32_t *dst, const lsp_utf16_t *str, size_t count)
1391 {
1392 lsp_utf32_t cp;
1393 size_t items = 0;
1394 do
1395 {
1396 cp = read_utf16le_codepoint(&str);
1397 if (++items > count)
1398 return 0;
1399 *(dst++) = CPU_TO_LE(cp);
1400 } while (cp != 0);
1401
1402 return items;
1403 }
1404
1405 size_t utf16le_to_utf32be(lsp_utf32_t *dst, const lsp_utf16_t *str, size_t count)
1406 {
1407 lsp_utf32_t cp;
1408 size_t items = 0;
1409 do
1410 {
1411 cp = read_utf16le_codepoint(&str);
1412 if (++items > count)
1413 return 0;
1414 *(dst++) = CPU_TO_BE(cp);
1415 } while (cp != 0);
1416
1417 return items;
1418 }
1419
1420 size_t utf16be_to_utf32le(lsp_utf32_t *dst, const lsp_utf16_t *str, size_t count)
1421 {
1422 lsp_utf32_t cp;
1423 size_t items = 0;
1424 do
1425 {
1426 cp = read_utf16be_codepoint(&str);
1427 if (++items > count)
1428 return 0;
1429 *(dst++) = CPU_TO_LE(cp);
1430 } while (cp != 0);
1431
1432 return items;
1433 }
1434
1435 size_t utf16be_to_utf32be(lsp_utf32_t *dst, const lsp_utf16_t *str, size_t count)
1436 {
1437 lsp_utf32_t cp;
1438 size_t items = 0;
1439 do
1440 {
1441 cp = read_utf16be_codepoint(&str);
1442 if (++items > count)
1443 return 0;
1444 *(dst++) = CPU_TO_BE(cp);
1445 } while (cp != 0);
1446
1447 return items;
1448 }
1449
1450 //-------------------------------------------------------------------------
1451 // UTF-32 non-streaming routines
1452 char *utf32le_to_utf8(const lsp_utf32_t *str)
1453 {
1454 lsp_utf32_t cp;
1455 size_t bytes = 0;
1456 const lsp_utf32_t *p = str;
1457
1458 // Estimate length
1459 do
1460 {
1461 cp = LE_TO_CPU(*(p++));
1462 bytes += sizeof_utf8(cp);
1463 } while (cp != 0);
1464
1465 // Allocate memory
1466 char *utf8 = reinterpret_cast<char *>(::malloc(bytes));
1467 if (utf8 == NULL)
1468 return NULL;
1469
1470 // Perform encoding
1471 p = str;
1472 char *dst = utf8;
1473 while ((cp = *(p++)) != 0)
1474 write_utf8_codepoint(&dst, cp);
1475
1476 *dst = 0;
1477 return utf8;
1478 }
1479
1480 char *utf32be_to_utf8(const lsp_utf32_t *str)
1481 {
1482 lsp_utf32_t cp;
1483 size_t bytes = 0;
1484 const lsp_utf32_t *p = str;
1485
1486 // Estimate length
1487 do
1488 {
1489 cp = BE_TO_CPU(*(p++));
1490 bytes += sizeof_utf8(cp);
1491 } while (cp != 0);
1492
1493 // Allocate memory
1494 char *utf8 = reinterpret_cast<char *>(::malloc(bytes));
1495 if (utf8 == NULL)
1496 return NULL;
1497
1498 // Perform encoding
1499 p = str;
1500 char *dst = utf8;
1501 while ((cp = *(p++)) != 0)
1502 write_utf8_codepoint(&dst, cp);
1503
1504 *dst = 0;
1505 return utf8;
1506 }
1507
1508 size_t utf32le_to_utf8(char *dst, const lsp_utf32_t *src, size_t count)
1509 {
1510 lsp_utf32_t cp;
1511 size_t items = 0;
1512 do
1513 {
1514 cp = LE_TO_CPU(*(src++));
1515 items += count_utf8(cp);
1516 if (items > count)
1517 return 0;
1518 write_utf8_codepoint(&dst, cp);
1519 } while (cp != 0);
1520
1521 return items;
1522 }
1523
1524 size_t utf32be_to_utf8(char *dst, const lsp_utf32_t *src, size_t count)
1525 {
1526 lsp_utf32_t cp;
1527 size_t items = 0;
1528 do
1529 {
1530 cp = BE_TO_CPU(*(src++));
1531 items += count_utf8(cp);
1532 if (items > count)
1533 return 0;
1534 write_utf8_codepoint(&dst, cp);
1535 } while (cp != 0);
1536
1537 return items;
1538 }
1539
1540 lsp_utf16_t *utf32le_to_utf16le(const lsp_utf32_t *str)
1541 {
1542 lsp_utf32_t cp;
1543 size_t bytes = 0;
1544 const lsp_utf32_t *p = str;
1545
1546 // Estimate length
1547 do
1548 {
1549 cp = LE_TO_CPU(*(p++));
1550 bytes += sizeof_utf16(cp);
1551 } while (cp != 0);
1552
1553 // Allocate memory
1554 lsp_utf16_t *utf16 = reinterpret_cast<lsp_utf16_t *>(::malloc(bytes));
1555 if (utf16 == NULL)
1556 return NULL;
1557
1558 // Perform encoding
1559 p = str;
1560 lsp_utf16_t *dst= utf16;
1561 while ((cp = *(p++)) != 0)
1562 write_utf16le_codepoint(&dst, cp);
1563
1564 *dst = 0;
1565 return utf16;
1566 }
1567
1568 lsp_utf16_t *utf32le_to_utf16be(const lsp_utf32_t *str)
1569 {
1570 lsp_utf32_t cp;
1571 size_t bytes = 0;
1572 const lsp_utf32_t *p = str;
1573
1574 // Estimate length
1575 do
1576 {
1577 cp = LE_TO_CPU(*(p++));
1578 bytes += sizeof_utf16(cp);
1579 } while (cp != 0);
1580
1581 // Allocate memory
1582 lsp_utf16_t *utf16 = reinterpret_cast<lsp_utf16_t *>(::malloc(bytes));
1583 if (utf16 == NULL)
1584 return NULL;
1585
1586 // Perform encoding
1587 p = str;
1588 lsp_utf16_t *dst= utf16;
1589 while ((cp = *(p++)) != 0)
1590 write_utf16be_codepoint(&dst, cp);
1591
1592 *dst = 0;
1593 return utf16;
1594 }
1595
1596 lsp_utf16_t *utf32be_to_utf16le(const lsp_utf32_t *str)
1597 {
1598 lsp_utf32_t cp;
1599 size_t bytes = 0;
1600 const lsp_utf32_t *p = str;
1601
1602 // Estimate length
1603 do
1604 {
1605 cp = BE_TO_CPU(*(p++));
1606 bytes += sizeof_utf16(cp);
1607 } while (cp != 0);
1608
1609 // Allocate memory
1610 lsp_utf16_t *utf16 = reinterpret_cast<lsp_utf16_t *>(::malloc(bytes));
1611 if (utf16 == NULL)
1612 return NULL;
1613
1614 // Perform encoding
1615 p = str;
1616 lsp_utf16_t *dst= utf16;
1617 while ((cp = *(p++)) != 0)
1618 write_utf16le_codepoint(&dst, cp);
1619
1620 *dst = 0;
1621 return utf16;
1622 }
1623
1624 lsp_utf16_t *utf32be_to_utf16be(const lsp_utf32_t *str)
1625 {
1626 lsp_utf32_t cp;
1627 size_t bytes = 0;
1628 const lsp_utf32_t *p = str;
1629
1630 // Estimate length
1631 do
1632 {
1633 cp = BE_TO_CPU(*(p++));
1634 bytes += sizeof_utf16(cp);
1635 } while (cp != 0);
1636
1637 // Allocate memory
1638 lsp_utf16_t *utf16 = reinterpret_cast<lsp_utf16_t *>(::malloc(bytes));
1639 if (utf16 == NULL)
1640 return NULL;
1641
1642 // Perform encoding
1643 p = str;
1644 lsp_utf16_t *dst= utf16;
1645 while ((cp = *(p++)) != 0)
1646 write_utf16be_codepoint(&dst, cp);
1647
1648 *dst = 0;
1649 return utf16;
1650 }
1651
1652 size_t utf32le_to_utf16le(lsp_utf16_t *dst, const lsp_utf32_t *str, size_t count)
1653 {
1654 lsp_utf32_t cp;
1655 size_t items = 0;
1656
1657 // Estimate length
1658 do
1659 {
1660 cp = LE_TO_CPU(*(str++));
1661 items += count_utf16(cp);
1662 if (items > count)
1663 return 0;
1664 write_utf16le_codepoint(&dst, cp);
1665 } while (cp != 0);
1666
1667 return items;
1668 }
1669
1670 size_t utf32le_to_utf16be(lsp_utf16_t *dst, const lsp_utf32_t *str, size_t count)
1671 {
1672 lsp_utf32_t cp;
1673 size_t items = 0;
1674
1675 // Estimate length
1676 do
1677 {
1678 cp = LE_TO_CPU(*(str++));
1679 items += count_utf16(cp);
1680 if (items > count)
1681 return 0;
1682 write_utf16be_codepoint(&dst, cp);
1683 } while (cp != 0);
1684
1685 return items;
1686 }
1687
1688 size_t utf32be_to_utf16le(lsp_utf16_t *dst, const lsp_utf32_t *str, size_t count)
1689 {
1690 lsp_utf32_t cp;
1691 size_t items = 0;
1692
1693 // Estimate length
1694 do
1695 {
1696 cp = BE_TO_CPU(*(str++));
1697 items += count_utf16(cp);
1698 if (items > count)
1699 return 0;
1700 write_utf16le_codepoint(&dst, cp);
1701 } while (cp != 0);
1702
1703 return items;
1704 }
1705
1706 size_t utf32be_to_utf16be(lsp_utf16_t *dst, const lsp_utf32_t *str, size_t count)
1707 {
1708 lsp_utf32_t cp;
1709 size_t items = 0;
1710
1711 // Estimate length
1712 do
1713 {
1714 cp = BE_TO_CPU(*(str++));
1715 items += count_utf16(cp);
1716 if (items > count)
1717 return 0;
1718 write_utf16be_codepoint(&dst, cp);
1719 } while (cp != 0);
1720
1721 return items;
1722 }
1723
1724 //-------------------------------------------------------------------------
1725 // UTF-8 streaming routines
1726 size_t utf8_to_utf16le(lsp_utf16_t *dst, size_t *ndst, const char *src, size_t *nsrc, bool force)
1727 {
1728 lsp_utf32_t cp;
1729 size_t processed = 0;
1730
1731 while (*ndst > 0)
1732 {
1733 // Read code point
1734 size_t nin = *nsrc;
1735 cp = read_utf8_streaming(&src, &nin, force);
1736 if (cp == LSP_UTF32_EOF) // No data ?
1737 break;
1738
1739 // Encode code point
1740 size_t nout = count_utf16(cp);
1741 if (nout > *ndst)
1742 break;
1743 write_utf16le_codepoint(&dst, cp);
1744 *nsrc = nin;
1745 *ndst -= nout;
1746
1747 // Update statistics
1748 ++processed;
1749 }
1750
1751 return processed;
1752 }
1753
1754 size_t utf8_to_utf16be(lsp_utf16_t *dst, size_t *ndst, const char *src, size_t *nsrc, bool force)
1755 {
1756 lsp_utf32_t cp;
1757 size_t processed = 0;
1758
1759 while (*ndst > 0)
1760 {
1761 // Read code point
1762 size_t nin = *nsrc;
1763 cp = read_utf8_streaming(&src, &nin, force);
1764 if (cp == LSP_UTF32_EOF) // No data ?
1765 break;
1766
1767 // Encode code point
1768 size_t nout = count_utf16(cp);
1769 if (nout > *ndst)
1770 break;
1771 write_utf16be_codepoint(&dst, cp);
1772 *nsrc = nin;
1773 *ndst -= nout;
1774
1775 // Update statistics
1776 ++processed;
1777 }
1778
1779 return processed;
1780 }
1781
1782 size_t utf8_to_utf32le(lsp_utf32_t *dst, size_t *ndst, const char *src, size_t *nsrc, bool force)
1783 {
1784 lsp_utf32_t cp;
1785 size_t processed = 0;
1786
1787 while (*ndst > 0)
1788 {
1789 // Read code point
1790 size_t nin = *nsrc;
1791 cp = read_utf8_streaming(&src, &nin, force);
1792 if (cp == LSP_UTF32_EOF) // No data ?
1793 break;
1794
1795 // Encode code point
1796 *(dst++) = CPU_TO_LE(cp);
1797 *nsrc = nin;
1798 --(*ndst);
1799
1800 // Update statistics
1801 ++processed;
1802 }
1803
1804 return processed;
1805 }
1806
1807 size_t utf8_to_utf32be(lsp_utf32_t *dst, size_t *ndst, const char *src, size_t *nsrc, bool force)
1808 {
1809 lsp_utf32_t cp;
1810 size_t processed = 0;
1811
1812 while (*ndst > 0)
1813 {
1814 // Read code point
1815 size_t nin = *nsrc;
1816 cp = read_utf8_streaming(&src, &nin, force);
1817 if (cp == LSP_UTF32_EOF) // No data ?
1818 break;
1819
1820 // Encode code point
1821 *(dst++) = CPU_TO_BE(cp);
1822 *nsrc = nin;
1823 --(*ndst);
1824
1825 // Update statistics
1826 ++processed;
1827 }
1828
1829 return processed;
1830 }
1831
1832 //-------------------------------------------------------------------------
1833 // UTF-16 streaming routines
1834 size_t utf16le_to_utf8(char *dst, size_t *ndst, const lsp_utf16_t *src, size_t *nsrc, bool force)
1835 {
1836 lsp_utf32_t cp;
1837 size_t processed = 0;
1838
1839 while (*ndst > 0)
1840 {
1841 // Read code point
1842 size_t nin = *nsrc;
1843 cp = read_utf16le_streaming(&src, &nin, force);
1844 if (cp == LSP_UTF32_EOF) // No data ?
1845 break;
1846
1847 // Encode code point
1848 size_t nout = count_utf8(cp);
1849 if (nout > *ndst)
1850 break;
1851 write_utf8_codepoint(&dst, cp);
1852 *nsrc = nin;
1853 *ndst -= nout;
1854
1855 // Update statistics
1856 ++processed;
1857 }
1858
1859 return processed;
1860 }
1861
1862 size_t utf16be_to_utf8(char *dst, size_t *ndst, const lsp_utf16_t *src, size_t *nsrc, bool force)
1863 {
1864 lsp_utf32_t cp;
1865 size_t processed = 0;
1866
1867 while (*ndst > 0)
1868 {
1869 // Read code point
1870 size_t nin = *nsrc;
1871 cp = read_utf16be_streaming(&src, &nin, force);
1872 if (cp == LSP_UTF32_EOF) // No data ?
1873 break;
1874
1875 // Encode code point
1876 size_t nout = count_utf8(cp);
1877 if (nout > *ndst)
1878 break;
1879 write_utf8_codepoint(&dst, cp);
1880 *nsrc = nin;
1881 *ndst -= nout;
1882
1883 // Update statistics
1884 ++processed;
1885 }
1886
1887 return processed;
1888 }
1889
1890 size_t utf16le_to_utf32le(lsp_utf32_t *dst, size_t *ndst, const lsp_utf16_t *src, size_t *nsrc, bool force)
1891 {
1892 lsp_utf32_t cp;
1893 size_t processed = 0;
1894
1895 while (*ndst > 0)
1896 {
1897 // Read code point
1898 size_t nin = *nsrc;
1899 cp = read_utf16le_streaming(&src, &nin, force);
1900 if (cp == LSP_UTF32_EOF) // No data ?
1901 break;
1902
1903 // Encode code point
1904 *(dst++) = CPU_TO_LE(cp);
1905 *nsrc = nin;
1906 --(*ndst);
1907
1908 // Update statistics
1909 ++processed;
1910 }
1911
1912 return processed;
1913 }
1914
1915 size_t utf16be_to_utf32le(lsp_utf32_t *dst, size_t *ndst, const lsp_utf16_t *src, size_t *nsrc, bool force)
1916 {
1917 lsp_utf32_t cp;
1918 size_t processed = 0;
1919
1920 while (*ndst > 0)
1921 {
1922 // Read code point
1923 size_t nin = *nsrc;
1924 cp = read_utf16be_streaming(&src, &nin, force);
1925 if (cp == LSP_UTF32_EOF) // No data ?
1926 break;
1927
1928 // Encode code point
1929 *(dst++) = CPU_TO_LE(cp);
1930 *nsrc = nin;
1931 --(*ndst);
1932
1933 // Update statistics
1934 ++processed;
1935 }
1936
1937 return processed;
1938 }
1939
1940 size_t utf16le_to_utf32be(lsp_utf32_t *dst, size_t *ndst, const lsp_utf16_t *src, size_t *nsrc, bool force)
1941 {
1942 lsp_utf32_t cp;
1943 size_t processed = 0;
1944
1945 while (*ndst > 0)
1946 {
1947 // Read code point
1948 size_t nin = *nsrc;
1949 cp = read_utf16le_streaming(&src, &nin, force);
1950 if (cp == LSP_UTF32_EOF) // No data ?
1951 break;
1952
1953 // Encode code point
1954 *(dst++) = CPU_TO_BE(cp);
1955 *nsrc = nin;
1956 --(*ndst);
1957
1958 // Update statistics
1959 ++processed;
1960 }
1961
1962 return processed;
1963 }
1964
1965 size_t utf16be_to_utf32be(lsp_utf32_t *dst, size_t *ndst, const lsp_utf16_t *src, size_t *nsrc, bool force)
1966 {
1967 lsp_utf32_t cp;
1968 size_t processed = 0;
1969
1970 while (*ndst > 0)
1971 {
1972 // Read code point
1973 size_t nin = *nsrc;
1974 cp = read_utf16be_streaming(&src, &nin, force);
1975 if (cp == LSP_UTF32_EOF) // No data ?
1976 break;
1977
1978 // Encode code point
1979 *(dst++) = CPU_TO_BE(cp);
1980 *nsrc = nin;
1981 --(*ndst);
1982
1983 // Update statistics
1984 ++processed;
1985 }
1986
1987 return processed;
1988 }
1989
1990 //-------------------------------------------------------------------------
1991 // UTF-32 streaming routines
1992 size_t utf32le_to_utf8(char *dst, size_t *ndst, const lsp_utf32_t *src, size_t *nsrc, bool force)
1993 {
1994 lsp_utf32_t cp;
1995 size_t processed = 0;
1996
1997 while (*ndst > 0)
1998 {
1999 // Read code point
2000 if (*nsrc <= 0)
2001 break;
2002 cp = LE_TO_CPU(*(src++));
2003
2004 // Encode code point
2005 size_t nout = count_utf8(cp);
2006 if (nout > *ndst)
2007 break;
2008 write_utf8_codepoint(&dst, cp);
2009 --(*nsrc);
2010 *ndst -= nout;
2011
2012 // Update statistics
2013 ++processed;
2014 }
2015
2016 return processed;
2017 }
2018
2019 size_t utf32be_to_utf8(char *dst, size_t *ndst, const lsp_utf32_t *src, size_t *nsrc, bool force)
2020 {
2021 lsp_utf32_t cp;
2022 size_t processed = 0;
2023
2024 while (*ndst > 0)
2025 {
2026 // Read code point
2027 if (*nsrc <= 0)
2028 break;
2029 cp = BE_TO_CPU(*(src++));
2030
2031 // Encode code point
2032 size_t nout = count_utf8(cp);
2033 if (nout > *ndst)
2034 break;
2035 write_utf8_codepoint(&dst, cp);
2036 --(*nsrc);
2037 *ndst -= nout;
2038
2039 // Update statistics
2040 ++processed;
2041 }
2042
2043 return processed;
2044 }
2045
2046 size_t utf32le_to_utf16le(lsp_utf16_t *dst, size_t *ndst, const lsp_utf32_t *src, size_t *nsrc, bool force)
2047 {
2048 lsp_utf32_t cp;
2049 size_t processed = 0;
2050
2051 while (*ndst > 0)
2052 {
2053 // Read code point
2054 if (*nsrc <= 0)
2055 break;
2056 cp = LE_TO_CPU(*(src++));
2057
2058 // Encode code point
2059 size_t nout = count_utf16(cp);
2060 if (nout > *ndst)
2061 break;
2062 write_utf16le_codepoint(&dst, cp);
2063 --(*nsrc);
2064 *ndst -= nout;
2065
2066 // Update statistics
2067 ++processed;
2068 }
2069
2070 return processed;
2071 }
2072
2073 size_t utf32le_to_utf16be(lsp_utf16_t *dst, size_t *ndst, const lsp_utf32_t *src, size_t *nsrc, bool force)
2074 {
2075 lsp_utf32_t cp;
2076 size_t processed = 0;
2077
2078 while (*ndst > 0)
2079 {
2080 // Read code point
2081 if (*nsrc <= 0)
2082 break;
2083 cp = LE_TO_CPU(*(src++));
2084
2085 // Encode code point
2086 size_t nout = count_utf16(cp);
2087 if (nout > *ndst)
2088 break;
2089 write_utf16be_codepoint(&dst, cp);
2090 --(*nsrc);
2091 *ndst -= nout;
2092
2093 // Update statistics
2094 ++processed;
2095 }
2096
2097 return processed;
2098 }
2099
2100 size_t utf32be_to_utf16le(lsp_utf16_t *dst, size_t *ndst, const lsp_utf32_t *src, size_t *nsrc, bool force)
2101 {
2102 lsp_utf32_t cp;
2103 size_t processed = 0;
2104
2105 while (*ndst > 0)
2106 {
2107 // Read code point
2108 if (*nsrc <= 0)
2109 break;
2110 cp = BE_TO_CPU(*(src++));
2111
2112 // Encode code point
2113 size_t nout = count_utf16(cp);
2114 if (nout > *ndst)
2115 break;
2116 write_utf16le_codepoint(&dst, cp);
2117 --(*nsrc);
2118 *ndst -= nout;
2119
2120 // Update statistics
2121 ++processed;
2122 }
2123
2124 return processed;
2125 }
2126
2127 size_t utf32be_to_utf16be(lsp_utf16_t *dst, size_t *ndst, const lsp_utf32_t *src, size_t *nsrc, bool force)
2128 {
2129 lsp_utf32_t cp;
2130 size_t processed = 0;
2131
2132 while (*ndst > 0)
2133 {
2134 // Read code point
2135 if (*nsrc <= 0)
2136 break;
2137 cp = BE_TO_CPU(*(src++));
2138
2139 // Encode code point
2140 size_t nout = count_utf16(cp);
2141 if (nout > *ndst)
2142 break;
2143 write_utf16be_codepoint(&dst, cp);
2144 --(*nsrc);
2145 *ndst -= nout;
2146
2147 // Update statistics
2148 ++processed;
2149 }
2150
2151 return processed;
2152 }
2153
2154 #if defined(PLATFORM_WINDOWS)
2155 static ssize_t multibyte_to_widechar_utf16le(LPCCH src, size_t *nsrc, LPWSTR dst, size_t *ndst)
2156 {
2157 lsp_wchar_t cp;
2158 ssize_t nconv = 0;
2159 const lsp_utf16_t *xsrc = reinterpret_cast<const lsp_utf16_t *>(src);
2160 size_t nin = (*nsrc) >> 1;
2161 size_t nout = *ndst;
2162
2163 while (nin > 0)
2164 {
2165 // Read code point
2166 size_t xin = nin;
2167 cp = read_utf16le_streaming(&xsrc, &xin, false);
2168 if (cp == LSP_UTF32_EOF) // No data ?
2169 break;
2170
2171 // Check that we have enough space
2172 size_t len = count_utf16(cp);
2173 if (nout < len)
2174 break;
2175
2176 // Write code point
2177 write_utf16_codepoint(&dst, cp);
2178 nin = xin;
2179 nout -= len;
2180 nconv += len;
2181 }
2182
2183 *nsrc = ((*nsrc) & 1) + (nin << 1);
2184 *ndst = nout;
2185 return nconv;
2186 }
2187
2188 static ssize_t multibyte_to_widechar_utf16be(LPCCH src, size_t *nsrc, LPWSTR dst, size_t *ndst)
2189 {
2190 lsp_wchar_t cp;
2191 ssize_t nconv = 0;
2192 const lsp_utf16_t *xsrc = reinterpret_cast<const lsp_utf16_t *>(src);
2193 size_t nin = (*nsrc) >> 1;
2194 size_t nout = *ndst;
2195
2196 while (nin > 0)
2197 {
2198 // Read code point
2199 size_t xin = nin;
2200 cp = read_utf16be_streaming(&xsrc, &xin, false);
2201 if (cp == LSP_UTF32_EOF) // No data ?
2202 break;
2203
2204 // Check that we have enough space
2205 size_t len = count_utf16(cp);
2206 if (nout < len)
2207 break;
2208
2209 // Write code point
2210 write_utf16_codepoint(&dst, cp);
2211 nin = xin;
2212 nout -= len;
2213 nconv += len;
2214 }
2215
2216 *nsrc = ((*nsrc) & 1) + (nin << 1);
2217 *ndst = nout;
2218 return nconv;
2219 }
2220
2221 static ssize_t est_multibyte_to_widechar_utf16le(LPCCH src, size_t nsrc)
2222 {
2223 lsp_wchar_t cp;
2224 ssize_t nconv = 0;
2225 const lsp_utf16_t *xsrc = reinterpret_cast<const lsp_utf16_t *>(src);
2226 nsrc >>= 1;
2227
2228 while (nsrc > 0)
2229 {
2230 // Read code point
2231 cp = read_utf16le_streaming(&xsrc, &nsrc, false);
2232 if (cp == LSP_UTF32_EOF) // No data ?
2233 break;
2234
2235 // Check that we have enough space
2236 nconv += count_utf16(cp);
2237 }
2238
2239 return nconv;
2240 }
2241
2242 static ssize_t est_multibyte_to_widechar_utf16be(LPCCH src, size_t nsrc)
2243 {
2244 lsp_wchar_t cp;
2245 ssize_t nconv = 0;
2246 const lsp_utf16_t *xsrc = reinterpret_cast<const lsp_utf16_t *>(src);
2247 nsrc >>= 1;
2248
2249 while (nsrc > 0)
2250 {
2251 // Read code point
2252 cp = read_utf16le_streaming(&xsrc, &nsrc, false);
2253 if (cp == LSP_UTF32_EOF) // No data ?
2254 break;
2255
2256 // Check that we have enough space
2257 nconv += count_utf16(cp);
2258 }
2259
2260 return nconv;
2261 }
2262
2263 static ssize_t multibyte_to_widechar_utf32le(LPCCH src, size_t *nsrc, LPWSTR dst, size_t *ndst)
2264 {
2265 lsp_wchar_t cp;
2266 ssize_t nconv = 0;
2267 const lsp_utf32_t *xsrc = reinterpret_cast<const lsp_utf32_t *>(src);
2268 size_t nin = (*nsrc) >> 2;
2269 size_t nout = *ndst;
2270
2271 while (nin > 0)
2272 {
2273 // Read code point
2274 cp = LE_TO_CPU(*(xsrc++));
2275
2276 // Check that we have enough space
2277 size_t len = count_utf16(cp);
2278 if (nout < len)
2279 break;
2280
2281 // Write code point
2282 write_utf16_codepoint(&dst, cp);
2283 nin -= 1;
2284 nout -= len;
2285 nconv += len;
2286 }
2287
2288 *nsrc = ((*nsrc) & 3) + (nin << 2);
2289 *ndst = nout;
2290 return nconv;
2291 }
2292
2293 static ssize_t multibyte_to_widechar_utf32be(LPCCH src, size_t *nsrc, LPWSTR dst, size_t *ndst)
2294 {
2295 lsp_wchar_t cp;
2296 ssize_t nconv = 0;
2297 const lsp_utf32_t *xsrc = reinterpret_cast<const lsp_utf32_t *>(src);
2298 size_t nin = (*nsrc) >> 2;
2299 size_t nout = *ndst;
2300
2301 while (nin > 0)
2302 {
2303 // Read code point
2304 cp = BE_TO_CPU(*(xsrc++));
2305
2306 // Check that we have enough space
2307 size_t len = count_utf16(cp);
2308 if (nout < len)
2309 break;
2310
2311 // Write code point
2312 write_utf16_codepoint(&dst, cp);
2313 nin -= 1;
2314 nout -= len;
2315 nconv += len;
2316 }
2317
2318 *nsrc = ((*nsrc) & 3) + (nin << 2);
2319 *ndst = nout;
2320 return nconv;
2321 }
2322
2323 static ssize_t est_multibyte_to_widechar_utf32le(LPCCH src, size_t nsrc)
2324 {
2325 lsp_wchar_t cp;
2326 ssize_t nconv = 0;
2327 const lsp_utf32_t *xsrc = reinterpret_cast<const lsp_utf32_t *>(src);
2328 nsrc >>= 2;
2329
2330 while (nsrc > 0)
2331 {
2332 // Read code point
2333 cp = LE_TO_CPU(*(xsrc++));
2334 nconv += count_utf16(cp);
2335 }
2336
2337 return nconv;
2338 }
2339
2340 static ssize_t est_multibyte_to_widechar_utf32be(LPCCH src, size_t nsrc)
2341 {
2342 lsp_wchar_t cp;
2343 ssize_t nconv = 0;
2344 const lsp_utf32_t *xsrc = reinterpret_cast<const lsp_utf32_t *>(src);
2345 nsrc >>= 2;
2346
2347 while (nsrc > 0)
2348 {
2349 // Read code point
2350 cp = BE_TO_CPU(*(xsrc++));
2351 nconv += count_utf16(cp);
2352 }
2353
2354 return nconv;
2355 }
2356
2357 ssize_t multibyte_to_widechar(size_t cp, const char *src, size_t *nsrc, lsp_utf16_t *dst, size_t *ndst)
2358 {
2359 ssize_t nconv;
2360
2361 switch (cp)
2362 {
2363 case 1200: // UTF-16LE
2364 nconv = ((dst == NULL) || (ndst == NULL) || (ssize_t(*ndst) <= 0)) ?
2365 est_multibyte_to_widechar_utf16le(src, *nsrc) :
2366 multibyte_to_widechar_utf16le(src, nsrc, dst, ndst);
2367 break;
2368 case 1201: // UTF-16BE
2369 nconv = ((dst == NULL) || (ndst == NULL) || (ssize_t(*ndst) <= 0)) ?
2370 est_multibyte_to_widechar_utf16be(src, *nsrc) :
2371 multibyte_to_widechar_utf16be(src, nsrc, dst, ndst);
2372 break;
2373 case 12000: // UTF-32LE
2374 nconv = ((dst == NULL) || (ndst == NULL) || (ssize_t(*ndst) <= 0)) ?
2375 est_multibyte_to_widechar_utf32le(src, *nsrc) :
2376 multibyte_to_widechar_utf32le(src, nsrc, dst, ndst);
2377 break;
2378 case 12001: // UTF-32BE
2379 nconv = ((dst == NULL) || (ndst == NULL) || (ssize_t(*ndst) <= 0)) ?
2380 est_multibyte_to_widechar_utf32be(src, *nsrc) :
2381 multibyte_to_widechar_utf32be(src, nsrc, dst, ndst);
2382 break;
2383 default:
2384 // We need just to estimate the size?
2385 if ((dst == NULL) || (ndst == NULL) || (ssize_t(*ndst) <= 0))
2386 return ::MultiByteToWideChar(cp, 0, src, *nsrc, 0, 0);
2387
2388 // Do the conversion
2389 nconv = ::MultiByteToWideChar(cp, 0, src, *nsrc, dst, *ndst);
2390 if (nconv == 0)
2391 {
2392 switch (GetLastError())
2393 {
2394 case ERROR_SUCCESS:
2395 return 0;
2396 case ERROR_INSUFFICIENT_BUFFER:
2397 return -STATUS_NO_MEM;
2398 case ERROR_INVALID_FLAGS:
2399 case ERROR_INVALID_PARAMETER:
2400 return -STATUS_BAD_STATE;
2401 case ERROR_NO_UNICODE_TRANSLATION:
2402 return -STATUS_BAD_LOCALE;
2403 default:
2404 return -STATUS_UNKNOWN_ERR;
2405 }
2406 }
2407
2408 // There are converted characters, analyze output
2409 // If function meets invalid sequence, it replaces the code point with such magic value
2410 // We should know if function has failed
2411 if (dst[nconv-1] == 0xfffd)
2412 --nconv;
2413
2414 if (nconv > 0)
2415 {
2416 // Estimate number of bytes decoded (yep, this is dumb but no way...)
2417 ssize_t nbytes = ::WideCharToMultiByte(cp, 0, dst, nconv, NULL, 0, 0, 0);
2418 if (nbytes <= 0)
2419 return -STATUS_IO_ERROR;
2420
2421 *nsrc -= nbytes;
2422 *ndst -= nconv;
2423 }
2424
2425 break;
2426 }
2427
2428 return nconv;
2429 }
2430
2431 static ssize_t widechar_to_multibyte_utf16le(const lsp_utf16_t *src, size_t *nsrc, char *dst, size_t *ndst)
2432 {
2433 lsp_wchar_t cp;
2434 ssize_t nconv = 0;
2435 size_t nin = *nsrc;
2436 size_t nout = *ndst;
2437 lsp_utf16_t *xdst = reinterpret_cast<lsp_utf16_t *>(dst);
2438
2439 while (nin > 0)
2440 {
2441 size_t xin = nin;
2442 cp = read_utf16_streaming(&src, &xin, false);
2443 if (cp == LSP_UTF32_EOF) // No data ?
2444 break;
2445
2446 // Check that we have enough space
2447 size_t len = sizeof_utf16(cp);
2448 if (nout < len)
2449 break;
2450
2451 // Write code point
2452 write_utf16le_codepoint(&xdst, cp);
2453 nin = xin;
2454 nout -= len;
2455 nconv += len;
2456 }
2457
2458 *nsrc = nin;
2459 *ndst = nout;
2460 return nconv;
2461 }
2462
2463 static ssize_t widechar_to_multibyte_utf16be(const lsp_utf16_t *src, size_t *nsrc, char *dst, size_t *ndst)
2464 {
2465 lsp_wchar_t cp;
2466 ssize_t nconv = 0;
2467 size_t nin = *nsrc;
2468 size_t nout = *ndst;
2469 lsp_utf16_t *xdst = reinterpret_cast<lsp_utf16_t *>(dst);
2470
2471 while (nin > 0)
2472 {
2473 size_t xin = nin;
2474 cp = read_utf16_streaming(&src, &xin, false);
2475 if (cp == LSP_UTF32_EOF) // No data ?
2476 break;
2477
2478 // Check that we have enough space
2479 size_t len = sizeof_utf16(cp);
2480 if (nout < len)
2481 break;
2482
2483 // Write code point
2484 write_utf16be_codepoint(&xdst, cp);
2485 nin = xin;
2486 nout -= len;
2487 nconv += len;
2488 }
2489
2490 *nsrc = nin;
2491 *ndst = nout;
2492 return nconv;
2493 }
2494
2495 static ssize_t est_widechar_to_multibyte_utf16(const lsp_utf16_t *src, size_t nsrc)
2496 {
2497 lsp_wchar_t cp;
2498 ssize_t nconv = 0;
2499
2500 while (nsrc > 0)
2501 {
2502 cp = read_utf16_streaming(&src, &nsrc, false);
2503 if (cp == LSP_UTF32_EOF) // No data ?
2504 break;
2505
2506 // Check that we have enough space
2507 nconv += sizeof_utf16(cp);
2508 }
2509
2510 return nconv;
2511 }
2512
2513 static ssize_t widechar_to_multibyte_utf32le(const lsp_utf16_t *src, size_t *nsrc, char *dst, size_t *ndst)
2514 {
2515 lsp_wchar_t cp;
2516 ssize_t nconv = 0;
2517 size_t nin = *nsrc;
2518 size_t nout = *ndst;
2519 lsp_utf32_t *xdst = reinterpret_cast<lsp_utf32_t *>(dst);
2520
2521 while (nin > 0)
2522 {
2523 size_t xin = nin;
2524 cp = read_utf16_streaming(&src, &xin, false);
2525 if (cp == LSP_UTF32_EOF) // No data ?
2526 break;
2527
2528 // Check that we have enough space
2529 if (nout < sizeof(lsp_utf32_t))
2530 break;
2531
2532 // Write code point
2533 *(xdst++) = CPU_TO_LE(cp);
2534 nin = xin;
2535 nout -= sizeof(lsp_utf32_t);
2536 nconv += sizeof(lsp_utf32_t);
2537 }
2538
2539 *nsrc = nin;
2540 *ndst = nout;
2541 return nconv;
2542 }
2543
2544 static ssize_t widechar_to_multibyte_utf32be(const lsp_utf16_t *src, size_t *nsrc, char *dst, size_t *ndst)
2545 {
2546 lsp_wchar_t cp;
2547 ssize_t nconv = 0;
2548 size_t nin = *nsrc;
2549 size_t nout = *ndst;
2550 lsp_utf32_t *xdst = reinterpret_cast<lsp_utf32_t *>(dst);
2551
2552 while (nin > 0)
2553 {
2554 size_t xin = nin;
2555 cp = read_utf16_streaming(&src, &xin, false);
2556 if (cp == LSP_UTF32_EOF) // No data ?
2557 break;
2558
2559 // Check that we have enough space
2560 if (nout < sizeof(lsp_utf32_t))
2561 break;
2562
2563 // Write code point
2564 *(xdst++) = CPU_TO_BE(cp);
2565 nin = xin;
2566 nout -= sizeof(lsp_utf32_t);
2567 nconv += sizeof(lsp_utf32_t);
2568 }
2569
2570 *nsrc = nin;
2571 *ndst = nout;
2572 return nconv;
2573 }
2574
2575 static ssize_t est_widechar_to_multibyte_utf32(const lsp_utf16_t *src, size_t nsrc)
2576 {
2577 lsp_wchar_t cp;
2578 ssize_t nconv = 0;
2579
2580 while (nsrc > 0)
2581 {
2582 cp = read_utf16_streaming(&src, &nsrc, false);
2583 if (cp == LSP_UTF32_EOF) // No data ?
2584 break;
2585 nconv += sizeof(lsp_utf32_t);
2586 }
2587
2588 return nconv;
2589 }
2590
2591 static size_t widechar_to_multibyte_split(const lsp_utf16_t *src, size_t limit)
2592 {
2593 // Estimate the middle of an array
2594 size_t half = limit >> 1;
2595 if (half <= 0)
2596 return half;
2597
2598 // Now scan valid code points until we reach the end of array
2599 lsp_wchar_t cp;
2600 limit = half;
2601 while (true)
2602 {
2603 cp = read_utf16_streaming(&src, &limit, false);
2604 if (cp == LSP_UTF32_EOF) // No data ?
2605 break;
2606 }
2607
2608 // Return the result as middle of array without remained points in limit
2609 return half - limit;
2610 }
2611
2612 ssize_t widechar_to_multibyte(size_t cp, const lsp_utf16_t *src, size_t *nsrc, char *dst, size_t *ndst)
2613 {
2614 ssize_t nconv;
2615
2616 switch (cp)
2617 {
2618 case 1200: // UTF-16LE
2619 nconv = ((dst == NULL) || (ndst == NULL) || (ssize_t(*ndst) <= 0)) ?
2620 est_widechar_to_multibyte_utf16(src, *nsrc) :
2621 widechar_to_multibyte_utf16le(src, nsrc, dst, ndst);
2622 break;
2623 case 1201: // UTF-16BE
2624 nconv = ((dst == NULL) || (ndst == NULL) || (ssize_t(*ndst) <= 0)) ?
2625 est_widechar_to_multibyte_utf16(src, *nsrc) :
2626 widechar_to_multibyte_utf16be(src, nsrc, dst, ndst);
2627 break;
2628 case 12000: // UTF-32LE
2629 nconv = ((dst == NULL) || (ndst == NULL) || (ssize_t(*ndst) <= 0)) ?
2630 est_widechar_to_multibyte_utf32(src, *nsrc) :
2631 widechar_to_multibyte_utf32le(src, nsrc, dst, ndst);
2632 break;
2633 case 12001: // UTF-32BE
2634 nconv = ((dst == NULL) || (ndst == NULL) || (ssize_t(*ndst) <= 0)) ?
2635 est_widechar_to_multibyte_utf32(src, *nsrc) :
2636 widechar_to_multibyte_utf32be(src, nsrc, dst, ndst);
2637 break;
2638 default:
2639 {
2640 // Just estimate number of characters?
2641 if ((dst == NULL) || (ndst == NULL) || (ssize_t(*ndst) <= 0))
2642 {
2643 nconv = ::WideCharToMultiByte(cp, 0, src, *nsrc, NULL, 0, 0, FALSE);
2644 if (nconv == 0)
2645 {
2646 switch (::GetLastError())
2647 {
2648 case ERROR_SUCCESS:
2649 return 0;
2650 case ERROR_INSUFFICIENT_BUFFER:
2651 return -STATUS_NO_MEM;
2652 case ERROR_INVALID_FLAGS:
2653 case ERROR_INVALID_PARAMETER:
2654 return -STATUS_BAD_STATE;
2655 case ERROR_NO_UNICODE_TRANSLATION:
2656 return -STATUS_BAD_LOCALE;
2657 default:
2658 return -STATUS_UNKNOWN_ERR;
2659 }
2660 }
2661 return nconv;
2662 }
2663
2664 // Perform first try
2665 size_t xnsrc = *nsrc;
2666 nconv = ::WideCharToMultiByte(cp, 0, src, xnsrc, dst, *ndst, 0, FALSE);
2667
2668 // Do while conversion is unsuccessful
2669 while (nconv <= 0)
2670 {
2671 // There was a fail, analyze it
2672 switch (::GetLastError())
2673 {
2674 case ERROR_SUCCESS:
2675 return 0;
2676 case ERROR_INSUFFICIENT_BUFFER:
2677 break; // Will retry with twice lesser input buffer
2678 case ERROR_INVALID_FLAGS:
2679 case ERROR_INVALID_PARAMETER:
2680 return -STATUS_BAD_STATE;
2681 case ERROR_NO_UNICODE_TRANSLATION:
2682 return -STATUS_BAD_LOCALE;
2683 default:
2684 return -STATUS_UNKNOWN_ERR;
2685 }
2686
2687 // Try to twice reduce the buffer size, validate data for surrogates
2688 xnsrc = widechar_to_multibyte_split(src, xnsrc);
2689 if (xnsrc <= 0)
2690 break;
2691
2692 // Perform next conversion try with lesser buffer
2693 nconv = ::WideCharToMultiByte(cp, 0, src, xnsrc, dst, *ndst, 0, FALSE);
2694 }
2695
2696 *ndst -= nconv;
2697 *nsrc -= xnsrc;
2698 }
2699 break;
2700 }
2701
2702 return nconv;
2703 }
2704 #endif /* PLATFORM_WINDOWS */
2705
2706 #ifndef ARCH_LE
2707 int wchar_cmp(const lsp_wchar_t *s1, const lsp_wchar_t *s2, size_t count)
2708 {
2709 while (count--)
2710 {
2711 int32_t retval = int32_t(*(s1++)) - int32_t(*(s2++));
2712 if (retval != 0)
2713 return (retval > 0) ? 1 : -1;
2714 }
2715 return 0;
2716 }
2717 #endif
2718
2719 int wchar_casecmp(const lsp_wchar_t *s1, const lsp_wchar_t *s2, size_t count)
2720 {
2721 while (count--)
2722 {
2723 int32_t retval = int32_t(to_lower(*(s1++))) - int32_t(to_lower(*(s2++)));
2724 if (retval != 0)
2725 return (retval > 0) ? 1 : -1;
2726 }
2727 return 0;
2728 }
2729
2730 lsp_wchar_t to_lower(lsp_wchar_t c)
2731 {
2732 // Latin codepoints
2733 if (c <= 0x7f)
2734 {
2735 if ((c >= 'A') && (c <= 'Z'))
2736 return c - 'A' + 'a';
2737
2738 return c;
2739 }
2740
2741 // Cyrillic + Cyrillic supplement codepoints
2742 if ((c >= 0x400) && (c <= 0x52f))
2743 {
2744 if (c <= 0x40f)
2745 return c + 0x50;
2746 if (c <= 0x42f)
2747 return c + 0x20;
2748 if (!(c & 0x01))
2749 {
2750 if ((c >= 0x460) && (c <= 0x481))
2751 return c + 1;
2752 if (c >= 0x48a)
2753 return c + 1;
2754 }
2755
2756 return c;
2757 }
2758
2759 // Cyrillic extended B
2760 if ((c >= 0xa640) && (c <= 0xa69f))
2761 {
2762 if (!(c & 0x01))
2763 {
2764 if (c <= 0xa66d)
2765 return c + 1;
2766 if ((c >= 0xa680) && (c <= 0xa69b))
2767 return c + 1;
2768 }
2769 return c;
2770 }
2771
2772 // Other possible locales
2773 return towlower(c);
2774 }
2775
2776 lsp_wchar_t to_upper(lsp_wchar_t c)
2777 {
2778 // Latin codepoints
2779 if (c <= 0x7f)
2780 {
2781 if ((c >= 'a') && (c <= 'z'))
2782 return c - 'a' + 'A';
2783
2784 return c;
2785 }
2786
2787 // Cyrillic + Cyrillic supplement codepoints
2788 if ((c >= 0x400) && (c <= 0x52f))
2789 {
2790 if ((c >= 0x450) && (c <= 0x45f))
2791 return c - 0x50;
2792 if ((c >= 0x430) && (c <= 0x44f))
2793 return c - 0x20;
2794 if (c & 0x01)
2795 {
2796 if ((c >= 0x460) && (c <= 0x481))
2797 return c - 1;
2798 if (c >= 0x48a)
2799 return c - 1;
2800 }
2801
2802 return c;
2803 }
2804
2805 // Cyrillic extended B
2806 if ((c >= 0xa640) && (c <= 0xa69f))
2807 {
2808 if (c & 0x01)
2809 {
2810 if (c <= 0xa66d)
2811 return c - 1;
2812 if ((c >= 0xa680) && (c <= 0xa69b))
2813 return c - 1;
2814 }
2815
2816 return c;
2817 }
2818
2819 // Other possible locales
2820 return towupper(c);
2821 }
2822
2823 } /* namespace lsp */