/*
* Copyright (C) 2020 Linux Studio Plugins Project
* (C) 2020 Vladimir Sadovnikov
*
* This file is part of lsp-runtime-lib
* Created on: 18 июн. 2018 г.
*
* lsp-runtime-lib is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* any later version.
*
* lsp-runtime-lib is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with lsp-runtime-lib. If not, see .
*/
#include
#include
#include
#include
#include
#include
#include
#include
#ifdef PLATFORM_WINDOWS
#include
#include
#endif /* PLATFORM_WINDOWS */
namespace lsp
{
#if defined(PLATFORM_WINDOWS)
typedef struct codepage_t
{
const char *name;
size_t codepage;
} codepage_t;
// This is a generated list of codepages supported by Windows,
// see script: scripts/perl/core/oi/charset/gen_cp.pl
static const codepage_t win_codepages[] = {
{ "037", 37 },
{ "10000", 10000 },
{ "10001", 10001 },
{ "10002", 10002 },
{ "10003", 10003 },
{ "10004", 10004 },
{ "10005", 10005 },
{ "10006", 10006 },
{ "10007", 10007 },
{ "10008", 10008 },
{ "10010", 10010 },
{ "10017", 10017 },
{ "10021", 10021 },
{ "10029", 10029 },
{ "10079", 10079 },
{ "10081", 10081 },
{ "10082", 10082 },
{ "1026", 1026 },
{ "1047", 1047 },
{ "1140", 1140 },
{ "1141", 1141 },
{ "1142", 1142 },
{ "1143", 1143 },
{ "1144", 1144 },
{ "1145", 1145 },
{ "1146", 1146 },
{ "1147", 1147 },
{ "1148", 1148 },
{ "1149", 1149 },
{ "1200", 1200 },
{ "12000", 12000 },
{ "12001", 12001 },
{ "1201", 1201 },
{ "1250", 1250 },
{ "1251", 1251 },
{ "1252", 1252 },
{ "1253", 1253 },
{ "1254", 1254 },
{ "1255", 1255 },
{ "1256", 1256 },
{ "1257", 1257 },
{ "1258", 1258 },
{ "1361", 1361 },
{ "20000", 20000 },
{ "20001", 20001 },
{ "20002", 20002 },
{ "20003", 20003 },
{ "20004", 20004 },
{ "20005", 20005 },
{ "20105", 20105 },
{ "20106", 20106 },
{ "20107", 20107 },
{ "20108", 20108 },
{ "20127", 20127 },
{ "20261", 20261 },
{ "20269", 20269 },
{ "20273", 20273 },
{ "20277", 20277 },
{ "20278", 20278 },
{ "20280", 20280 },
{ "20284", 20284 },
{ "20285", 20285 },
{ "20290", 20290 },
{ "20297", 20297 },
{ "20420", 20420 },
{ "20423", 20423 },
{ "20424", 20424 },
{ "20833", 20833 },
{ "20838", 20838 },
{ "20866", 20866 },
{ "20871", 20871 },
{ "20880", 20880 },
{ "20905", 20905 },
{ "20924", 20924 },
{ "20932", 20932 },
{ "20936", 20936 },
{ "20949", 20949 },
{ "21025", 21025 },
{ "21027", 21027 },
{ "21866", 21866 },
{ "28591", 28591 },
{ "28592", 28592 },
{ "28593", 28593 },
{ "28594", 28594 },
{ "28595", 28595 },
{ "28596", 28596 },
{ "28597", 28597 },
{ "28598", 28598 },
{ "28599", 28599 },
{ "28603", 28603 },
{ "28605", 28605 },
{ "29001", 29001 },
{ "37", 37 },
{ "38598", 38598 },
{ "437", 437 },
{ "500", 500 },
{ "50220", 50220 },
{ "50221", 50221 },
{ "50222", 50222 },
{ "50225", 50225 },
{ "50227", 50227 },
{ "50229", 50229 },
{ "50930", 50930 },
{ "50931", 50931 },
{ "50933", 50933 },
{ "50935", 50935 },
{ "50936", 50936 },
{ "50937", 50937 },
{ "50939", 50939 },
{ "51932", 51932 },
{ "51936", 51936 },
{ "51949", 51949 },
{ "51950", 51950 },
{ "52936", 52936 },
{ "54936", 54936 },
{ "57002", 57002 },
{ "57003", 57003 },
{ "57004", 57004 },
{ "57005", 57005 },
{ "57006", 57006 },
{ "57007", 57007 },
{ "57008", 57008 },
{ "57009", 57009 },
{ "57010", 57010 },
{ "57011", 57011 },
{ "65000", 65000 },
{ "65001", 65001 },
{ "708", 708 },
{ "709", 709 },
{ "710", 710 },
{ "720", 720 },
{ "737", 737 },
{ "775", 775 },
{ "850", 850 },
{ "852", 852 },
{ "855", 855 },
{ "857", 857 },
{ "858", 858 },
{ "860", 860 },
{ "861", 861 },
{ "862", 862 },
{ "863", 863 },
{ "864", 864 },
{ "865", 865 },
{ "866", 866 },
{ "869", 869 },
{ "870", 870 },
{ "874", 874 },
{ "875", 875 },
{ "932", 932 },
{ "936", 936 },
{ "949", 949 },
{ "950", 950 },
{ "asmo-708", 708 },
{ "asmo708", 708 },
{ "big5", 950 },
{ "cp-1025", 21025 },
{ "cp-1250", 1250 },
{ "cp-1251", 1251 },
{ "cp-1252", 1252 },
{ "cp-1253", 1253 },
{ "cp-1254", 1254 },
{ "cp-1255", 1255 },
{ "cp-1256", 1256 },
{ "cp-1257", 1257 },
{ "cp-1258", 1258 },
{ "cp-21027", 21027 },
{ "cp-50229", 50229 },
{ "cp-50930", 50930 },
{ "cp-50931", 50931 },
{ "cp-50933", 50933 },
{ "cp-50935", 50935 },
{ "cp-50936", 50936 },
{ "cp-50937", 50937 },
{ "cp-50939", 50939 },
{ "cp-51950", 51950 },
{ "cp-709", 709 },
{ "cp-710", 710 },
{ "cp-866", 866 },
{ "cp-874", 874 },
{ "cp-875", 875 },
{ "cp1025", 21025 },
{ "cp1250", 1250 },
{ "cp1251", 1251 },
{ "cp1252", 1252 },
{ "cp1253", 1253 },
{ "cp1254", 1254 },
{ "cp1255", 1255 },
{ "cp1256", 1256 },
{ "cp1257", 1257 },
{ "cp1258", 1258 },
{ "cp21027", 21027 },
{ "cp50229", 50229 },
{ "cp50930", 50930 },
{ "cp50931", 50931 },
{ "cp50933", 50933 },
{ "cp50935", 50935 },
{ "cp50936", 50936 },
{ "cp50937", 50937 },
{ "cp50939", 50939 },
{ "cp51950", 51950 },
{ "cp709", 709 },
{ "cp710", 710 },
{ "cp866", 866 },
{ "cp874", 874 },
{ "cp875", 875 },
{ "csiso2022jp", 50221 },
{ "dos-720", 720 },
{ "dos-862", 862 },
{ "dos720", 720 },
{ "dos862", 862 },
{ "euc-cn", 51936 },
{ "euc-jp", 51932 },
{ "euc-kr", 51949 },
{ "gb18030", 54936 },
{ "gb2312", 936 },
{ "hz-gb-2312", 52936 },
{ "hz-gb2312", 52936 },
{ "ibm-thai", 20838 },
{ "ibm00858", 858 },
{ "ibm00924", 20924 },
{ "ibm01047", 1047 },
{ "ibm01140", 1140 },
{ "ibm01141", 1141 },
{ "ibm01142", 1142 },
{ "ibm01143", 1143 },
{ "ibm01144", 1144 },
{ "ibm01145", 1145 },
{ "ibm01146", 1146 },
{ "ibm01147", 1147 },
{ "ibm01148", 1148 },
{ "ibm01149", 1149 },
{ "ibm037", 37 },
{ "ibm1026", 1026 },
{ "ibm273", 20273 },
{ "ibm277", 20277 },
{ "ibm278", 20278 },
{ "ibm280", 20280 },
{ "ibm284", 20284 },
{ "ibm285", 20285 },
{ "ibm290", 20290 },
{ "ibm297", 20297 },
{ "ibm420", 20420 },
{ "ibm423", 20423 },
{ "ibm424", 20424 },
{ "ibm437", 437 },
{ "ibm500", 500 },
{ "ibm737", 737 },
{ "ibm775", 775 },
{ "ibm850", 850 },
{ "ibm852", 852 },
{ "ibm855", 855 },
{ "ibm857", 857 },
{ "ibm860", 860 },
{ "ibm861", 861 },
{ "ibm863", 863 },
{ "ibm864", 864 },
{ "ibm865", 865 },
{ "ibm869", 869 },
{ "ibm870", 870 },
{ "ibm871", 20871 },
{ "ibm880", 20880 },
{ "ibm905", 20905 },
{ "iso-2022-jp", 50222 },
{ "iso-2022-kr", 50225 },
{ "iso-2022jp", 50222 },
{ "iso-2022kr", 50225 },
{ "iso-8859-1", 28591 },
{ "iso-8859-13", 28603 },
{ "iso-8859-15", 28605 },
{ "iso-8859-2", 28592 },
{ "iso-8859-3", 28593 },
{ "iso-8859-4", 28594 },
{ "iso-8859-5", 28595 },
{ "iso-8859-6", 28596 },
{ "iso-8859-7", 28597 },
{ "iso-8859-8", 28598 },
{ "iso-8859-8-i", 38598 },
{ "iso-8859-8i", 38598 },
{ "iso-8859-9", 28599 },
{ "iso2022-jp", 50222 },
{ "iso2022-kr", 50225 },
{ "iso2022jp", 50222 },
{ "iso2022kr", 50225 },
{ "iso8859-1", 28591 },
{ "iso8859-13", 28603 },
{ "iso8859-15", 28605 },
{ "iso8859-2", 28592 },
{ "iso8859-3", 28593 },
{ "iso8859-4", 28594 },
{ "iso8859-5", 28595 },
{ "iso8859-6", 28596 },
{ "iso8859-7", 28597 },
{ "iso8859-8", 28598 },
{ "iso8859-8-i", 38598 },
{ "iso8859-8i", 38598 },
{ "iso8859-9", 28599 },
{ "johab", 1361 },
{ "koi8-r", 20866 },
{ "koi8-u", 21866 },
{ "koi8r", 20866 },
{ "koi8u", 21866 },
{ "ks-c-5601-1987", 949 },
{ "ks_c_5601-1987", 949 },
{ "macintosh", 10000 },
{ "shift-jis", 932 },
{ "shift_jis", 932 },
{ "unicodefffe", 1201 },
{ "us-ascii", 20127 },
{ "utf-16", 1200 },
{ "utf-16be", 1201 },
{ "utf-16le", 1200 },
{ "utf-32", 12000 },
{ "utf-32be", 12001 },
{ "utf-32le", 12000 },
{ "utf-7", 65000 },
{ "utf-8", 65001 },
{ "utf16", 1200 },
{ "utf16-be", 1201 },
{ "utf16-le", 1200 },
{ "utf16be", 1201 },
{ "utf16le", 1200 },
{ "utf32", 12000 },
{ "utf32-be", 12001 },
{ "utf32-le", 12000 },
{ "utf32be", 12001 },
{ "utf32le", 12000 },
{ "utf7", 65000 },
{ "utf8", 65001 },
{ "windows-1250", 1250 },
{ "windows-1251", 1251 },
{ "windows-1252", 1252 },
{ "windows-1253", 1253 },
{ "windows-1254", 1254 },
{ "windows-1255", 1255 },
{ "windows-1256", 1256 },
{ "windows-1257", 1257 },
{ "windows-1258", 1258 },
{ "windows-874", 874 },
{ "windows1250", 1250 },
{ "windows1251", 1251 },
{ "windows1252", 1252 },
{ "windows1253", 1253 },
{ "windows1254", 1254 },
{ "windows1255", 1255 },
{ "windows1256", 1256 },
{ "windows1257", 1257 },
{ "windows1258", 1258 },
{ "windows874", 874 },
{ "x-chinese-cns", 20000 },
{ "x-chinese-eten", 20002 },
{ "x-chinese_cns", 20000 },
{ "x-cp-20001", 20001 },
{ "x-cp-20003", 20003 },
{ "x-cp-20004", 20004 },
{ "x-cp-20005", 20005 },
{ "x-cp-20261", 20261 },
{ "x-cp-20269", 20269 },
{ "x-cp-20936", 20936 },
{ "x-cp-20949", 20949 },
{ "x-cp-50227", 50227 },
{ "x-cp20001", 20001 },
{ "x-cp20003", 20003 },
{ "x-cp20004", 20004 },
{ "x-cp20005", 20005 },
{ "x-cp20261", 20261 },
{ "x-cp20269", 20269 },
{ "x-cp20936", 20936 },
{ "x-cp20949", 20949 },
{ "x-cp50227", 50227 },
{ "x-ebcdic-koreanextended", 20833 },
{ "x-europa", 29001 },
{ "x-ia5", 20105 },
{ "x-ia5-german", 20106 },
{ "x-ia5-norwegian", 20108 },
{ "x-ia5-swedish", 20107 },
{ "x-ia5german", 20106 },
{ "x-ia5norwegian", 20108 },
{ "x-ia5swedish", 20107 },
{ "x-iscii-as", 57006 },
{ "x-iscii-be", 57003 },
{ "x-iscii-de", 57002 },
{ "x-iscii-gu", 57010 },
{ "x-iscii-ka", 57008 },
{ "x-iscii-ma", 57009 },
{ "x-iscii-or", 57007 },
{ "x-iscii-pa", 57011 },
{ "x-iscii-ta", 57004 },
{ "x-iscii-te", 57005 },
{ "x-mac-arabic", 10004 },
{ "x-mac-ce", 10029 },
{ "x-mac-chinesesimp", 10008 },
{ "x-mac-chinesetrad", 10002 },
{ "x-mac-croatian", 10082 },
{ "x-mac-cyrillic", 10007 },
{ "x-mac-greek", 10006 },
{ "x-mac-hebrew", 10005 },
{ "x-mac-icelandic", 10079 },
{ "x-mac-japanese", 10001 },
{ "x-mac-korean", 10003 },
{ "x-mac-romanian", 10010 },
{ "x-mac-thai", 10021 },
{ "x-mac-turkish", 10081 },
{ "x-mac-ukrainian", 10017 },
{ "x_chinese-eten", 20002 }
};
ssize_t get_codepage(LCID locale, bool ansi)
{
char buf[32];
int res = GetLocaleInfoA(locale, (ansi) ? LOCALE_IDEFAULTANSICODEPAGE : LOCALE_IDEFAULTCODEPAGE, buf, sizeof(buf)-1);
if (res == 0)
{
switch (GetLastError())
{
case ERROR_INSUFFICIENT_BUFFER:
return -STATUS_NO_MEM;
case ERROR_INVALID_FLAGS:
case ERROR_INVALID_PARAMETER:
return -STATUS_BAD_ARGUMENTS;
default:
return -STATUS_UNKNOWN_ERR;
}
}
errno = 0;
ssize_t cp_num = strtol(buf, NULL, 10);
if (errno != 0)
return -STATUS_UNSUPPORTED_FORMAT;
return cp_num;
}
ssize_t codepage_from_name(const char *charset)
{
if (charset != NULL)
{
// Do lower-case the character set
size_t n = strlen(charset) + 1;
char *lower = static_cast(alloca(n));
for (size_t i=0; i> 1;
int n = strcmp(lower, win_codepages[middle].name);
if (n == 0)
return win_codepages[middle].codepage;
else if (n < 0)
last = middle;
else
first = middle + 1;
}
return -1;
}
// printf("LOCALE_CUSTOM_DEFAULT = %d\n", int(get_codepage(LOCALE_CUSTOM_DEFAULT)));
// printf("LOCALE_USER_DEFAULT = %d\n", int(get_codepage(LOCALE_USER_DEFAULT)));
// printf("LOCALE_SYSTEM_DEFAULT = %d\n", int(get_codepage(LOCALE_SYSTEM_DEFAULT)));
// printf("LOCALE_CUSTOM_UNSPECIFIED = %d\n", int(get_codepage(LOCALE_CUSTOM_UNSPECIFIED)));
// printf("LOCALE_CUSTOM_UI_DEFAULT = %d\n", int(get_codepage(LOCALE_CUSTOM_UI_DEFAULT)));
// printf("LOCALE_INVARIANT = %d\n", int(get_codepage(LOCALE_INVARIANT)));
// printf("GetConsoleWindow() = %d\n", int(GetConsoleWindow()));
// printf("GetConsoleOutputCP() = %d\n", int(GetConsoleOutputCP()));
// fflush(stdout);
// Obtain system character set
//ssize_t cp = (GetConsoleWindow() != 0) ? GetConsoleOutputCP() : get_codepage(LOCALE_CUSTOM_DEFAULT);
ssize_t cp = get_codepage(LOCALE_CUSTOM_DEFAULT);
if (cp < 0)
cp = get_codepage(LOCALE_USER_DEFAULT);
if (cp < 0)
cp = get_codepage(LOCALE_SYSTEM_DEFAULT);
return cp;
}
#else
iconv_t init_iconv_to_wchar_t(const char *charset)
{
// Fetch system character set if it is not set
if (charset == NULL)
{
// Assume default locale being UTF-8
charset = "UTF-8";
// Get system locale
char *current = setlocale(LC_CTYPE, NULL);
if (current == NULL)
return iconv_t(-1);
// Scan for character set presence
current = strchr(current, '.');
if (current != NULL)
{
++current;
if (strlen(current) > 0)
charset = current;
}
}
// Open conversion
iconv_t res = iconv_open(__IF_LEBE("UTF-32LE", "UTF-32BE"), charset);
if (res != iconv_t(-1))
return res;
res = iconv_open(__IF_LEBE("UTF-32LE", "UTF-32BE"), "UTF-8");
if (res != iconv_t(-1))
return res;
return iconv_open("WCHAR_T", "UTF-8");
}
iconv_t init_iconv_from_wchar_t(const char *charset)
{
// Fetch system charset if it is not set
if (charset == NULL)
{
// Assume default locale being UTF-8
charset = "UTF-8";
// Get system locale
char *current = setlocale(LC_CTYPE, NULL);
if (current == NULL)
return iconv_t(-1);
// Scan for character set presence
current = strchr(current, '.');
if (current != NULL)
{
++current;
if (strlen(current) > 0)
charset = current;
}
}
// Open conversion
iconv_t res = iconv_open(charset, __IF_LEBE("UTF-32LE", "UTF-32BE"));
if (res != iconv_t(-1))
return res;
res = iconv_open("UTF-8", __IF_LEBE("UTF-32LE", "UTF-32BE"));
if (res != iconv_t(-1))
return res;
return iconv_open("UTF-8", "WCHAR_T");
}
#endif
//-------------------------------------------------------------------------
// UTF-16 helper routines
lsp_utf32_t read_utf16le_codepoint(const lsp_utf16_t **str)
{
uint32_t cp, sc;
const lsp_utf16_t *s = *str;
cp = LE_TO_CPU(*(s++));
if (cp == 0)
return cp;
sc = cp & 0xfc00;
if (sc == 0xd800) // cp = Surrogate high
{
sc = LE_TO_CPU(*s);
if ((sc & 0xfc00) == 0xdc00)
{
++s;
cp = 0x10000 + (((cp & 0x3ff) << 10) | (sc & 0x3ff));
}
else
cp = 0xfffd;
}
else if (sc == 0xdc00) // Surrogate low?
{
sc = LE_TO_CPU(*s);
if ((sc & 0xfc00) == 0xd800)
{
++s;
cp = 0x10000 + (((sc & 0x3ff) << 10) | (cp & 0x3ff));
}
else
cp = 0xfffd;
}
*str = s;
return cp;
}
lsp_utf32_t read_utf16be_codepoint(const lsp_utf16_t **str)
{
uint32_t cp, sc;
const lsp_utf16_t *s = *str;
cp = BE_TO_CPU(*(s++));
if (cp == 0)
return cp;
sc = cp & 0xfc00;
if (sc == 0xd800) // cp = Surrogate high
{
sc = BE_TO_CPU(*s);
if ((sc & 0xfc00) == 0xdc00)
{
++s;
cp = 0x10000 + (((cp & 0x3ff) << 10) | (sc & 0x3ff));
}
else
cp = 0xfffd;
}
else if (sc == 0xdc00) // Surrogate low?
{
sc = BE_TO_CPU(*s);
if ((sc & 0xfc00) == 0xd800)
{
++s;
cp = 0x10000 + (((sc & 0x3ff) << 10) | (cp & 0x3ff));
}
else
cp = 0xfffd;
}
*str = s;
return cp;
}
lsp_utf32_t read_utf16le_streaming(const lsp_utf16_t **str, size_t *nsrc, bool force)
{
if (*nsrc <= 0)
return LSP_UTF32_EOF;
uint32_t cp, sc;
const lsp_utf16_t *s = *str;
cp = LE_TO_CPU(*(s++));
sc = cp & 0xfc00;
if (sc == 0xd800) // cp = Surrogate high
{
if (*nsrc > 1)
sc = LE_TO_CPU(*s);
else if (force)
sc = 0;
else
return LSP_UTF32_EOF;
if ((sc & 0xfc00) == 0xdc00)
{
++s;
cp = 0x10000 + (((cp & 0x3ff) << 10) | (sc & 0x3ff));
}
else
cp = 0xfffd;
}
else if (sc == 0xdc00) // Surrogate low?
{
if (*nsrc > 1)
sc = LE_TO_CPU(*s);
else if (force)
sc = 0;
else
return LSP_UTF32_EOF;
if ((sc & 0xfc00) == 0xd800)
{
++s;
cp = 0x10000 + (((sc & 0x3ff) << 10) | (cp & 0x3ff));
}
else
cp = 0xfffd;
}
*nsrc -= (s - *str);
*str = s;
return cp;
}
lsp_utf32_t read_utf16be_streaming(const lsp_utf16_t **str, size_t *nsrc, bool force)
{
if (*nsrc <= 0)
return LSP_UTF32_EOF;
uint32_t cp, sc;
const lsp_utf16_t *s = *str;
cp = BE_TO_CPU(*(s++));
sc = cp & 0xfc00;
if (sc == 0xd800) // cp = Surrogate high
{
if (*nsrc > 1)
sc = BE_TO_CPU(*s);
else if (force)
sc = 0;
else
return LSP_UTF32_EOF;
if ((sc & 0xfc00) == 0xdc00)
{
++s;
cp = 0x10000 + (((cp & 0x3ff) << 10) | (sc & 0x3ff));
}
else
cp = 0xfffd;
}
else if (sc == 0xdc00) // Surrogate low?
{
if (*nsrc > 1)
sc = BE_TO_CPU(*s);
else if (force)
sc = 0;
else
return LSP_UTF32_EOF;
if ((sc & 0xfc00) == 0xd800)
{
++s;
cp = 0x10000 + (((sc & 0x3ff) << 10) | (cp & 0x3ff));
}
else
cp = 0xfffd;
}
*nsrc -= (s - *str);
*str = s;
return cp;
}
inline size_t sizeof_utf16(lsp_utf32_t cp)
{
return (cp < 0x10000) ? 2 : 4;
}
inline size_t count_utf16(lsp_utf32_t cp)
{
return (cp < 0x10000) ? 1 : 2;
}
void write_utf16le_codepoint(lsp_utf16_t **str, lsp_utf32_t cp)
{
lsp_utf16_t *dst = *str;
if (cp < 0x10000)
*(dst++) = CPU_TO_LE(lsp_utf16_t(cp));
else
{
cp -= 0x10000;
dst[0] = CPU_TO_LE(lsp_utf16_t(0xd800 | (cp >> 10)));
dst[1] = CPU_TO_LE(lsp_utf16_t(0xdc00 | (cp & 0x3ff)));
dst += 2;
}
*str = dst;
}
void write_utf16be_codepoint(lsp_utf16_t **str, lsp_utf32_t cp)
{
lsp_utf16_t *dst = *str;
if (cp < 0x10000)
*(dst++) = CPU_TO_BE(lsp_utf16_t(cp));
else
{
cp -= 0x10000;
dst[0] = CPU_TO_BE(lsp_utf16_t(0xd800 | (cp >> 10)));
dst[1] = CPU_TO_BE(lsp_utf16_t(0xdc00 | (cp & 0x3ff)));
dst += 2;
}
*str = dst;
}
//-------------------------------------------------------------------------
// UTF-8 helper routines
lsp_utf32_t read_utf8_codepoint(const char **str)
{
lsp_utf32_t cp, sp;
size_t bytes;
const char *s = *str;
// Decode primary byte
cp = uint8_t(*s);
if (cp <= 0x7f)
{
*str = (cp == 0) ? s : s+1;
return cp;
}
++s;
if ((cp & 0xe0) == 0xc0) // 2 bytes: 110xxxxx 10xxxxxx
{
cp &= 0x1f;
bytes = (cp >= 0x02) ? 1 : 0;
}
else if ((cp & 0xf0) == 0xe0) // 3 bytes: 1110xxxx 10xxxxxx 10xxxxxx
{
cp &= 0x0f;
bytes = (cp) ? 2 : 0;
}
else if ((cp & 0xf8) == 0xf0) // 4 bytes: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
{
cp &= 0x07;
bytes = 3;
}
else
bytes = 0;
// Invalid first byte sequence?
if (!bytes)
{
*str = s;
return 0xfffd;
}
// Decode extension bytes
for (size_t i=0; i= 0xd800) && (cp < 0xe000)) // Check for surrogates
cp = 0xfffd;
*str = s;
return cp;
}
lsp_utf32_t read_utf8_streaming(const char **str, size_t *nsrc, bool force)
{
if (*nsrc <= 0)
return LSP_UTF32_EOF;
lsp_utf32_t cp, sp;
size_t bytes;
const char *s = *str;
// Decode primary byte
cp = uint8_t(*s);
if (cp <= 0x7f)
{
*str = (cp == 0) ? s : s+1;
--(*nsrc);
return cp;
}
// Multi-byte sequence
++s;
if ((cp & 0xe0) == 0xc0) // 2 bytes: 110xxxxx 10xxxxxx
{
cp &= 0x1f;
bytes = (cp >= 0x02) ? 1 : 0;
}
else if ((cp & 0xf0) == 0xe0) // 3 bytes: 1110xxxx 10xxxxxx 10xxxxxx
{
cp &= 0x0f;
bytes = (cp) ? 2 : 0;
}
else if ((cp & 0xf8) == 0xf0) // 4 bytes: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
{
cp &= 0x07;
bytes = 3;
}
else
bytes = 0;
// Invalid first byte sequence?
if (!bytes)
{
*str = s;
--(*nsrc);
return 0xfffd;
}
else if (bytes >= *nsrc)
{
if (force)
{
*nsrc = 0;
return 0xfffd;
}
return LSP_UTF32_EOF;
}
// Decode extension bytes
for (size_t i=0; i= 0xd800) && (cp < 0xe000)) // Check for surrogates
cp = 0xfffd;
*nsrc -= (s - *str);
*str = s;
return cp;
}
inline size_t sizeof_utf8(lsp_utf32_t cp)
{
if (cp >= 0x800)
return ((cp < 0x10000) || (cp >= 0x200000)) ? 3 : 4;
else
return (cp >= 0x80) ? 2 : 1;
}
inline size_t count_utf8(lsp_utf32_t cp)
{
if (cp >= 0x800)
return ((cp < 0x10000) || (cp >= 0x200000)) ? 3 : 4;
else
return (cp >= 0x80) ? 2 : 1;
}
void write_utf8_codepoint(char **str, lsp_utf32_t cp)
{
char *dst = *str;
if (cp >= 0x800) // 3-4 bytes
{
if (cp < 0x10000) // 3 bytes
{
dst[0] = (cp >> 12) | 0xe0;
dst[1] = ((cp >> 6) & 0x3f) | 0x80;
dst[2] = (cp & 0x3f) | 0x80;
dst += 3;
}
else if (cp < 0x200000) // 4 bytes
{
dst[0] = (cp >> 16) | 0xf0;
dst[1] = ((cp >> 12) & 0x3f) | 0x80;
dst[2] = ((cp >> 6) & 0x3f) | 0x80;
dst[3] = (cp & 0x3f) | 0x80;
dst += 4;
}
else // Invalid character, emit 3 bytes of 0xfffd code point value
{
dst[0] = 0xef;
dst[1] = 0xbf;
dst[2] = 0xbd;
dst += 3;
}
}
else // 1-2 bytes
{
if (cp >= 0x80) // 2 bytes
{
dst[0] = (cp >> 6) | 0xc0;
dst[1] = (cp & 0x3f) | 0x80;
dst += 2;
}
else // 1 byte
*(dst++) = char(cp);
}
*str = dst;
}
//-------------------------------------------------------------------------
// UTF-8 non-streaming routines
lsp_utf16_t *utf8_to_utf16le(const char *str)
{
// Estimate number of bytes
lsp_utf32_t cp;
size_t bytes = 0;
const char *p = str;
do
{
cp = read_utf8_codepoint(&p);
bytes += sizeof_utf16(cp);
} while (cp != 0);
// Allocate memory
lsp_utf16_t *utf16 = reinterpret_cast(::malloc(bytes));
if (utf16 == NULL)
return NULL;
// Perform encoding
lsp_utf16_t *dst = utf16;
p = str;
while ((cp = read_utf8_codepoint(&p)) != 0)
write_utf16le_codepoint(&dst, cp);
*dst = 0;
return utf16;
}
lsp_utf16_t *utf8_to_utf16be(const char *str)
{
// Estimate number of bytes
lsp_utf32_t cp;
size_t bytes = 0;
const char *p = str;
do
{
cp = read_utf8_codepoint(&p);
bytes += sizeof_utf16(cp);
} while (cp != 0);
// Allocate memory
lsp_utf16_t *utf16 = reinterpret_cast(::malloc(bytes));
if (utf16 == NULL)
return NULL;
// Perform encoding
lsp_utf16_t *dst = utf16;
p = str;
while ((cp = read_utf8_codepoint(&p)) != 0)
write_utf16be_codepoint(&dst, cp);
*dst = 0;
return utf16;
}
size_t utf8_to_utf16le(lsp_utf16_t *dst, const char *str, size_t count)
{
lsp_utf32_t cp;
size_t items = 0;
do
{
cp = read_utf8_codepoint(&str);
items += count_utf16(cp);
if (items > count)
return 0;
write_utf16le_codepoint(&dst, cp);
} while (cp != 0);
return items;
}
size_t utf8_to_utf16be(lsp_utf16_t *dst, const char *str, size_t count)
{
lsp_utf32_t cp;
size_t items = 0;
do
{
cp = read_utf8_codepoint(&str);
items += count_utf16(cp);
if (items > count)
return 0;
write_utf16be_codepoint(&dst, cp);
} while (cp != 0);
return items;
}
lsp_utf32_t *utf8_to_utf32le(const char *str)
{
// Estimate number of bytes
lsp_utf32_t cp;
size_t bytes = 0;
const char *p = str;
do
{
cp = read_utf8_codepoint(&p);
bytes += sizeof(lsp_utf32_t);
} while (cp != 0);
// Allocate memory
lsp_utf32_t *utf32 = reinterpret_cast(::malloc(bytes));
if (utf32 == NULL)
return NULL;
// Perform encoding
lsp_utf32_t *dst = utf32;
p = str;
while ((cp = read_utf8_codepoint(&p)) != 0)
*(dst++) = CPU_TO_LE(cp);
*dst = 0;
return utf32;
}
lsp_utf32_t *utf8_to_utf32be(const char *str)
{
// Estimate number of bytes
lsp_utf32_t cp;
size_t bytes = 0;
const char *p = str;
do
{
cp = read_utf8_codepoint(&p);
bytes += sizeof(lsp_utf32_t);
} while (cp != 0);
// Allocate memory
lsp_utf32_t *utf32 = reinterpret_cast(::malloc(bytes));
if (utf32 == NULL)
return NULL;
// Perform encoding
lsp_utf32_t *dst = utf32;
p = str;
while ((cp = read_utf8_codepoint(&p)) != 0)
*(dst++) = CPU_TO_BE(cp);
*dst = 0;
return utf32;
}
size_t utf8_to_utf32le(lsp_utf32_t *dst, const char *str, size_t count)
{
lsp_utf32_t cp;
size_t items = 0;
do
{
cp = read_utf8_codepoint(&str);
if (++items > count)
return 0;
*(dst++) = CPU_TO_LE(cp);
} while (cp != 0);
return items;
}
size_t utf8_to_utf32be(lsp_utf32_t *dst, const char *str, size_t count)
{
lsp_utf32_t cp;
size_t items = 0;
do
{
cp = read_utf8_codepoint(&str);
if (++items > count)
return 0;
*(dst++) = CPU_TO_BE(cp);
} while (cp != 0);
return items;
}
//-------------------------------------------------------------------------
// UTF-16 non-streaming routines
char *utf16le_to_utf8(const lsp_utf16_t *str)
{
// Estimate number of bytes
lsp_utf32_t cp;
size_t bytes = 0;
const lsp_utf16_t *p = str;
do
{
cp = read_utf16le_codepoint(&p);
bytes += sizeof_utf8(cp);
} while (cp != 0);
// Allocate memory
char *utf8 = reinterpret_cast(::malloc(bytes));
if (utf8 == NULL)
return NULL;
// Now perform encoding
char *dst = utf8;
p = str;
while ((cp = read_utf16le_codepoint(&p)) != 0)
write_utf8_codepoint(&dst, cp);
*dst = '\0';
return utf8;
}
char *utf16be_to_utf8(const lsp_utf16_t *str)
{
// Estimate number of bytes
lsp_utf32_t cp;
size_t bytes = 0;
const lsp_utf16_t *p = str;
do
{
cp = read_utf16be_codepoint(&p);
bytes += sizeof_utf8(cp);
} while (cp != 0);
// Allocate memory
char *utf8 = reinterpret_cast(::malloc(bytes));
if (utf8 == NULL)
return NULL;
// Now perform encoding
char *dst = utf8;
p = str;
while ((cp = read_utf16be_codepoint(&p)) != 0)
write_utf8_codepoint(&dst, cp);
*dst = '\0';
return utf8;
}
size_t utf16le_to_utf8(char *dst, const lsp_utf16_t *str, size_t count)
{
lsp_utf32_t cp;
size_t items = 0;
do
{
cp = read_utf16le_codepoint(&str);
items += count_utf8(cp);
if (items > count)
return 0;
write_utf8_codepoint(&dst, cp);
} while (cp != 0);
return items;
}
size_t utf16be_to_utf8(char *dst, const lsp_utf16_t *str, size_t count)
{
lsp_utf32_t cp;
size_t items = 0;
do
{
cp = read_utf16be_codepoint(&str);
items += count_utf8(cp);
if (items > count)
return 0;
write_utf8_codepoint(&dst, cp);
} while (cp != 0);
return items;
}
lsp_utf32_t *utf16le_to_utf32le(const lsp_utf16_t *str)
{
// Estimate number of bytes
lsp_utf32_t cp;
size_t bytes = 0;
const lsp_utf16_t *p = str;
do
{
cp = read_utf16le_codepoint(&p);
bytes += sizeof(lsp_utf32_t);
} while (cp != 0);
// Allocate memory
lsp_utf32_t *utf32 = reinterpret_cast(::malloc(bytes));
if (utf32 == NULL)
return NULL;
// Perform encoding
p = str;
lsp_utf32_t *dst= utf32;
while ((cp = read_utf16le_codepoint(&p)) != 0)
*(dst++) = CPU_TO_LE(cp);
*dst = 0;
return utf32;
}
lsp_utf32_t *utf16le_to_utf32be(const lsp_utf16_t *str)
{
// Estimate number of bytes
lsp_utf32_t cp;
size_t bytes = 0;
const lsp_utf16_t *p = str;
do
{
cp = read_utf16le_codepoint(&p);
bytes += sizeof(lsp_utf32_t);
} while (cp != 0);
// Allocate memory
lsp_utf32_t *utf32 = reinterpret_cast(::malloc(bytes));
if (utf32 == NULL)
return NULL;
// Perform encoding
p = str;
lsp_utf32_t *dst= utf32;
while ((cp = read_utf16le_codepoint(&p)) != 0)
*(dst++) = CPU_TO_BE(cp);
*dst = 0;
return utf32;
}
lsp_utf32_t *utf16be_to_utf32le(const lsp_utf16_t *str)
{
// Estimate number of bytes
lsp_utf32_t cp;
size_t bytes = 0;
const lsp_utf16_t *p = str;
do
{
cp = read_utf16be_codepoint(&p);
bytes += sizeof(lsp_utf32_t);
} while (cp != 0);
// Allocate memory
lsp_utf32_t *utf32 = reinterpret_cast(::malloc(bytes));
if (utf32 == NULL)
return NULL;
// Perform encoding
p = str;
lsp_utf32_t *dst= utf32;
while ((cp = read_utf16be_codepoint(&p)) != 0)
*(dst++) = CPU_TO_LE(cp);
*dst = 0;
return utf32;
}
lsp_utf32_t *utf16be_to_utf32be(const lsp_utf16_t *str)
{
// Estimate number of bytes
lsp_utf32_t cp;
size_t bytes = 0;
const lsp_utf16_t *p = str;
do
{
cp = read_utf16be_codepoint(&p);
bytes += sizeof(lsp_utf32_t);
} while (cp != 0);
// Allocate memory
lsp_utf32_t *utf32 = reinterpret_cast(::malloc(bytes));
if (utf32 == NULL)
return NULL;
// Perform encoding
p = str;
lsp_utf32_t *dst= utf32;
while ((cp = read_utf16be_codepoint(&p)) != 0)
*(dst++) = CPU_TO_BE(cp);
*dst = 0;
return utf32;
}
size_t utf16le_to_utf32le(lsp_utf32_t *dst, const lsp_utf16_t *str, size_t count)
{
lsp_utf32_t cp;
size_t items = 0;
do
{
cp = read_utf16le_codepoint(&str);
if (++items > count)
return 0;
*(dst++) = CPU_TO_LE(cp);
} while (cp != 0);
return items;
}
size_t utf16le_to_utf32be(lsp_utf32_t *dst, const lsp_utf16_t *str, size_t count)
{
lsp_utf32_t cp;
size_t items = 0;
do
{
cp = read_utf16le_codepoint(&str);
if (++items > count)
return 0;
*(dst++) = CPU_TO_BE(cp);
} while (cp != 0);
return items;
}
size_t utf16be_to_utf32le(lsp_utf32_t *dst, const lsp_utf16_t *str, size_t count)
{
lsp_utf32_t cp;
size_t items = 0;
do
{
cp = read_utf16be_codepoint(&str);
if (++items > count)
return 0;
*(dst++) = CPU_TO_LE(cp);
} while (cp != 0);
return items;
}
size_t utf16be_to_utf32be(lsp_utf32_t *dst, const lsp_utf16_t *str, size_t count)
{
lsp_utf32_t cp;
size_t items = 0;
do
{
cp = read_utf16be_codepoint(&str);
if (++items > count)
return 0;
*(dst++) = CPU_TO_BE(cp);
} while (cp != 0);
return items;
}
//-------------------------------------------------------------------------
// UTF-32 non-streaming routines
char *utf32le_to_utf8(const lsp_utf32_t *str)
{
lsp_utf32_t cp;
size_t bytes = 0;
const lsp_utf32_t *p = str;
// Estimate length
do
{
cp = LE_TO_CPU(*(p++));
bytes += sizeof_utf8(cp);
} while (cp != 0);
// Allocate memory
char *utf8 = reinterpret_cast(::malloc(bytes));
if (utf8 == NULL)
return NULL;
// Perform encoding
p = str;
char *dst = utf8;
while ((cp = *(p++)) != 0)
write_utf8_codepoint(&dst, cp);
*dst = 0;
return utf8;
}
char *utf32be_to_utf8(const lsp_utf32_t *str)
{
lsp_utf32_t cp;
size_t bytes = 0;
const lsp_utf32_t *p = str;
// Estimate length
do
{
cp = BE_TO_CPU(*(p++));
bytes += sizeof_utf8(cp);
} while (cp != 0);
// Allocate memory
char *utf8 = reinterpret_cast(::malloc(bytes));
if (utf8 == NULL)
return NULL;
// Perform encoding
p = str;
char *dst = utf8;
while ((cp = *(p++)) != 0)
write_utf8_codepoint(&dst, cp);
*dst = 0;
return utf8;
}
size_t utf32le_to_utf8(char *dst, const lsp_utf32_t *src, size_t count)
{
lsp_utf32_t cp;
size_t items = 0;
do
{
cp = LE_TO_CPU(*(src++));
items += count_utf8(cp);
if (items > count)
return 0;
write_utf8_codepoint(&dst, cp);
} while (cp != 0);
return items;
}
size_t utf32be_to_utf8(char *dst, const lsp_utf32_t *src, size_t count)
{
lsp_utf32_t cp;
size_t items = 0;
do
{
cp = BE_TO_CPU(*(src++));
items += count_utf8(cp);
if (items > count)
return 0;
write_utf8_codepoint(&dst, cp);
} while (cp != 0);
return items;
}
lsp_utf16_t *utf32le_to_utf16le(const lsp_utf32_t *str)
{
lsp_utf32_t cp;
size_t bytes = 0;
const lsp_utf32_t *p = str;
// Estimate length
do
{
cp = LE_TO_CPU(*(p++));
bytes += sizeof_utf16(cp);
} while (cp != 0);
// Allocate memory
lsp_utf16_t *utf16 = reinterpret_cast(::malloc(bytes));
if (utf16 == NULL)
return NULL;
// Perform encoding
p = str;
lsp_utf16_t *dst= utf16;
while ((cp = *(p++)) != 0)
write_utf16le_codepoint(&dst, cp);
*dst = 0;
return utf16;
}
lsp_utf16_t *utf32le_to_utf16be(const lsp_utf32_t *str)
{
lsp_utf32_t cp;
size_t bytes = 0;
const lsp_utf32_t *p = str;
// Estimate length
do
{
cp = LE_TO_CPU(*(p++));
bytes += sizeof_utf16(cp);
} while (cp != 0);
// Allocate memory
lsp_utf16_t *utf16 = reinterpret_cast(::malloc(bytes));
if (utf16 == NULL)
return NULL;
// Perform encoding
p = str;
lsp_utf16_t *dst= utf16;
while ((cp = *(p++)) != 0)
write_utf16be_codepoint(&dst, cp);
*dst = 0;
return utf16;
}
lsp_utf16_t *utf32be_to_utf16le(const lsp_utf32_t *str)
{
lsp_utf32_t cp;
size_t bytes = 0;
const lsp_utf32_t *p = str;
// Estimate length
do
{
cp = BE_TO_CPU(*(p++));
bytes += sizeof_utf16(cp);
} while (cp != 0);
// Allocate memory
lsp_utf16_t *utf16 = reinterpret_cast(::malloc(bytes));
if (utf16 == NULL)
return NULL;
// Perform encoding
p = str;
lsp_utf16_t *dst= utf16;
while ((cp = *(p++)) != 0)
write_utf16le_codepoint(&dst, cp);
*dst = 0;
return utf16;
}
lsp_utf16_t *utf32be_to_utf16be(const lsp_utf32_t *str)
{
lsp_utf32_t cp;
size_t bytes = 0;
const lsp_utf32_t *p = str;
// Estimate length
do
{
cp = BE_TO_CPU(*(p++));
bytes += sizeof_utf16(cp);
} while (cp != 0);
// Allocate memory
lsp_utf16_t *utf16 = reinterpret_cast(::malloc(bytes));
if (utf16 == NULL)
return NULL;
// Perform encoding
p = str;
lsp_utf16_t *dst= utf16;
while ((cp = *(p++)) != 0)
write_utf16be_codepoint(&dst, cp);
*dst = 0;
return utf16;
}
size_t utf32le_to_utf16le(lsp_utf16_t *dst, const lsp_utf32_t *str, size_t count)
{
lsp_utf32_t cp;
size_t items = 0;
// Estimate length
do
{
cp = LE_TO_CPU(*(str++));
items += count_utf16(cp);
if (items > count)
return 0;
write_utf16le_codepoint(&dst, cp);
} while (cp != 0);
return items;
}
size_t utf32le_to_utf16be(lsp_utf16_t *dst, const lsp_utf32_t *str, size_t count)
{
lsp_utf32_t cp;
size_t items = 0;
// Estimate length
do
{
cp = LE_TO_CPU(*(str++));
items += count_utf16(cp);
if (items > count)
return 0;
write_utf16be_codepoint(&dst, cp);
} while (cp != 0);
return items;
}
size_t utf32be_to_utf16le(lsp_utf16_t *dst, const lsp_utf32_t *str, size_t count)
{
lsp_utf32_t cp;
size_t items = 0;
// Estimate length
do
{
cp = BE_TO_CPU(*(str++));
items += count_utf16(cp);
if (items > count)
return 0;
write_utf16le_codepoint(&dst, cp);
} while (cp != 0);
return items;
}
size_t utf32be_to_utf16be(lsp_utf16_t *dst, const lsp_utf32_t *str, size_t count)
{
lsp_utf32_t cp;
size_t items = 0;
// Estimate length
do
{
cp = BE_TO_CPU(*(str++));
items += count_utf16(cp);
if (items > count)
return 0;
write_utf16be_codepoint(&dst, cp);
} while (cp != 0);
return items;
}
//-------------------------------------------------------------------------
// UTF-8 streaming routines
size_t utf8_to_utf16le(lsp_utf16_t *dst, size_t *ndst, const char *src, size_t *nsrc, bool force)
{
lsp_utf32_t cp;
size_t processed = 0;
while (*ndst > 0)
{
// Read code point
size_t nin = *nsrc;
cp = read_utf8_streaming(&src, &nin, force);
if (cp == LSP_UTF32_EOF) // No data ?
break;
// Encode code point
size_t nout = count_utf16(cp);
if (nout > *ndst)
break;
write_utf16le_codepoint(&dst, cp);
*nsrc = nin;
*ndst -= nout;
// Update statistics
++processed;
}
return processed;
}
size_t utf8_to_utf16be(lsp_utf16_t *dst, size_t *ndst, const char *src, size_t *nsrc, bool force)
{
lsp_utf32_t cp;
size_t processed = 0;
while (*ndst > 0)
{
// Read code point
size_t nin = *nsrc;
cp = read_utf8_streaming(&src, &nin, force);
if (cp == LSP_UTF32_EOF) // No data ?
break;
// Encode code point
size_t nout = count_utf16(cp);
if (nout > *ndst)
break;
write_utf16be_codepoint(&dst, cp);
*nsrc = nin;
*ndst -= nout;
// Update statistics
++processed;
}
return processed;
}
size_t utf8_to_utf32le(lsp_utf32_t *dst, size_t *ndst, const char *src, size_t *nsrc, bool force)
{
lsp_utf32_t cp;
size_t processed = 0;
while (*ndst > 0)
{
// Read code point
size_t nin = *nsrc;
cp = read_utf8_streaming(&src, &nin, force);
if (cp == LSP_UTF32_EOF) // No data ?
break;
// Encode code point
*(dst++) = CPU_TO_LE(cp);
*nsrc = nin;
--(*ndst);
// Update statistics
++processed;
}
return processed;
}
size_t utf8_to_utf32be(lsp_utf32_t *dst, size_t *ndst, const char *src, size_t *nsrc, bool force)
{
lsp_utf32_t cp;
size_t processed = 0;
while (*ndst > 0)
{
// Read code point
size_t nin = *nsrc;
cp = read_utf8_streaming(&src, &nin, force);
if (cp == LSP_UTF32_EOF) // No data ?
break;
// Encode code point
*(dst++) = CPU_TO_BE(cp);
*nsrc = nin;
--(*ndst);
// Update statistics
++processed;
}
return processed;
}
//-------------------------------------------------------------------------
// UTF-16 streaming routines
size_t utf16le_to_utf8(char *dst, size_t *ndst, const lsp_utf16_t *src, size_t *nsrc, bool force)
{
lsp_utf32_t cp;
size_t processed = 0;
while (*ndst > 0)
{
// Read code point
size_t nin = *nsrc;
cp = read_utf16le_streaming(&src, &nin, force);
if (cp == LSP_UTF32_EOF) // No data ?
break;
// Encode code point
size_t nout = count_utf8(cp);
if (nout > *ndst)
break;
write_utf8_codepoint(&dst, cp);
*nsrc = nin;
*ndst -= nout;
// Update statistics
++processed;
}
return processed;
}
size_t utf16be_to_utf8(char *dst, size_t *ndst, const lsp_utf16_t *src, size_t *nsrc, bool force)
{
lsp_utf32_t cp;
size_t processed = 0;
while (*ndst > 0)
{
// Read code point
size_t nin = *nsrc;
cp = read_utf16be_streaming(&src, &nin, force);
if (cp == LSP_UTF32_EOF) // No data ?
break;
// Encode code point
size_t nout = count_utf8(cp);
if (nout > *ndst)
break;
write_utf8_codepoint(&dst, cp);
*nsrc = nin;
*ndst -= nout;
// Update statistics
++processed;
}
return processed;
}
size_t utf16le_to_utf32le(lsp_utf32_t *dst, size_t *ndst, const lsp_utf16_t *src, size_t *nsrc, bool force)
{
lsp_utf32_t cp;
size_t processed = 0;
while (*ndst > 0)
{
// Read code point
size_t nin = *nsrc;
cp = read_utf16le_streaming(&src, &nin, force);
if (cp == LSP_UTF32_EOF) // No data ?
break;
// Encode code point
*(dst++) = CPU_TO_LE(cp);
*nsrc = nin;
--(*ndst);
// Update statistics
++processed;
}
return processed;
}
size_t utf16be_to_utf32le(lsp_utf32_t *dst, size_t *ndst, const lsp_utf16_t *src, size_t *nsrc, bool force)
{
lsp_utf32_t cp;
size_t processed = 0;
while (*ndst > 0)
{
// Read code point
size_t nin = *nsrc;
cp = read_utf16be_streaming(&src, &nin, force);
if (cp == LSP_UTF32_EOF) // No data ?
break;
// Encode code point
*(dst++) = CPU_TO_LE(cp);
*nsrc = nin;
--(*ndst);
// Update statistics
++processed;
}
return processed;
}
size_t utf16le_to_utf32be(lsp_utf32_t *dst, size_t *ndst, const lsp_utf16_t *src, size_t *nsrc, bool force)
{
lsp_utf32_t cp;
size_t processed = 0;
while (*ndst > 0)
{
// Read code point
size_t nin = *nsrc;
cp = read_utf16le_streaming(&src, &nin, force);
if (cp == LSP_UTF32_EOF) // No data ?
break;
// Encode code point
*(dst++) = CPU_TO_BE(cp);
*nsrc = nin;
--(*ndst);
// Update statistics
++processed;
}
return processed;
}
size_t utf16be_to_utf32be(lsp_utf32_t *dst, size_t *ndst, const lsp_utf16_t *src, size_t *nsrc, bool force)
{
lsp_utf32_t cp;
size_t processed = 0;
while (*ndst > 0)
{
// Read code point
size_t nin = *nsrc;
cp = read_utf16be_streaming(&src, &nin, force);
if (cp == LSP_UTF32_EOF) // No data ?
break;
// Encode code point
*(dst++) = CPU_TO_BE(cp);
*nsrc = nin;
--(*ndst);
// Update statistics
++processed;
}
return processed;
}
//-------------------------------------------------------------------------
// UTF-32 streaming routines
size_t utf32le_to_utf8(char *dst, size_t *ndst, const lsp_utf32_t *src, size_t *nsrc, bool force)
{
lsp_utf32_t cp;
size_t processed = 0;
while (*ndst > 0)
{
// Read code point
if (*nsrc <= 0)
break;
cp = LE_TO_CPU(*(src++));
// Encode code point
size_t nout = count_utf8(cp);
if (nout > *ndst)
break;
write_utf8_codepoint(&dst, cp);
--(*nsrc);
*ndst -= nout;
// Update statistics
++processed;
}
return processed;
}
size_t utf32be_to_utf8(char *dst, size_t *ndst, const lsp_utf32_t *src, size_t *nsrc, bool force)
{
lsp_utf32_t cp;
size_t processed = 0;
while (*ndst > 0)
{
// Read code point
if (*nsrc <= 0)
break;
cp = BE_TO_CPU(*(src++));
// Encode code point
size_t nout = count_utf8(cp);
if (nout > *ndst)
break;
write_utf8_codepoint(&dst, cp);
--(*nsrc);
*ndst -= nout;
// Update statistics
++processed;
}
return processed;
}
size_t utf32le_to_utf16le(lsp_utf16_t *dst, size_t *ndst, const lsp_utf32_t *src, size_t *nsrc, bool force)
{
lsp_utf32_t cp;
size_t processed = 0;
while (*ndst > 0)
{
// Read code point
if (*nsrc <= 0)
break;
cp = LE_TO_CPU(*(src++));
// Encode code point
size_t nout = count_utf16(cp);
if (nout > *ndst)
break;
write_utf16le_codepoint(&dst, cp);
--(*nsrc);
*ndst -= nout;
// Update statistics
++processed;
}
return processed;
}
size_t utf32le_to_utf16be(lsp_utf16_t *dst, size_t *ndst, const lsp_utf32_t *src, size_t *nsrc, bool force)
{
lsp_utf32_t cp;
size_t processed = 0;
while (*ndst > 0)
{
// Read code point
if (*nsrc <= 0)
break;
cp = LE_TO_CPU(*(src++));
// Encode code point
size_t nout = count_utf16(cp);
if (nout > *ndst)
break;
write_utf16be_codepoint(&dst, cp);
--(*nsrc);
*ndst -= nout;
// Update statistics
++processed;
}
return processed;
}
size_t utf32be_to_utf16le(lsp_utf16_t *dst, size_t *ndst, const lsp_utf32_t *src, size_t *nsrc, bool force)
{
lsp_utf32_t cp;
size_t processed = 0;
while (*ndst > 0)
{
// Read code point
if (*nsrc <= 0)
break;
cp = BE_TO_CPU(*(src++));
// Encode code point
size_t nout = count_utf16(cp);
if (nout > *ndst)
break;
write_utf16le_codepoint(&dst, cp);
--(*nsrc);
*ndst -= nout;
// Update statistics
++processed;
}
return processed;
}
size_t utf32be_to_utf16be(lsp_utf16_t *dst, size_t *ndst, const lsp_utf32_t *src, size_t *nsrc, bool force)
{
lsp_utf32_t cp;
size_t processed = 0;
while (*ndst > 0)
{
// Read code point
if (*nsrc <= 0)
break;
cp = BE_TO_CPU(*(src++));
// Encode code point
size_t nout = count_utf16(cp);
if (nout > *ndst)
break;
write_utf16be_codepoint(&dst, cp);
--(*nsrc);
*ndst -= nout;
// Update statistics
++processed;
}
return processed;
}
#if defined(PLATFORM_WINDOWS)
static ssize_t multibyte_to_widechar_utf16le(LPCCH src, size_t *nsrc, LPWSTR dst, size_t *ndst)
{
lsp_wchar_t cp;
ssize_t nconv = 0;
const lsp_utf16_t *xsrc = reinterpret_cast(src);
size_t nin = (*nsrc) >> 1;
size_t nout = *ndst;
while (nin > 0)
{
// Read code point
size_t xin = nin;
cp = read_utf16le_streaming(&xsrc, &xin, false);
if (cp == LSP_UTF32_EOF) // No data ?
break;
// Check that we have enough space
size_t len = count_utf16(cp);
if (nout < len)
break;
// Write code point
write_utf16_codepoint(&dst, cp);
nin = xin;
nout -= len;
nconv += len;
}
*nsrc = ((*nsrc) & 1) + (nin << 1);
*ndst = nout;
return nconv;
}
static ssize_t multibyte_to_widechar_utf16be(LPCCH src, size_t *nsrc, LPWSTR dst, size_t *ndst)
{
lsp_wchar_t cp;
ssize_t nconv = 0;
const lsp_utf16_t *xsrc = reinterpret_cast(src);
size_t nin = (*nsrc) >> 1;
size_t nout = *ndst;
while (nin > 0)
{
// Read code point
size_t xin = nin;
cp = read_utf16be_streaming(&xsrc, &xin, false);
if (cp == LSP_UTF32_EOF) // No data ?
break;
// Check that we have enough space
size_t len = count_utf16(cp);
if (nout < len)
break;
// Write code point
write_utf16_codepoint(&dst, cp);
nin = xin;
nout -= len;
nconv += len;
}
*nsrc = ((*nsrc) & 1) + (nin << 1);
*ndst = nout;
return nconv;
}
static ssize_t est_multibyte_to_widechar_utf16le(LPCCH src, size_t nsrc)
{
lsp_wchar_t cp;
ssize_t nconv = 0;
const lsp_utf16_t *xsrc = reinterpret_cast(src);
nsrc >>= 1;
while (nsrc > 0)
{
// Read code point
cp = read_utf16le_streaming(&xsrc, &nsrc, false);
if (cp == LSP_UTF32_EOF) // No data ?
break;
// Check that we have enough space
nconv += count_utf16(cp);
}
return nconv;
}
static ssize_t est_multibyte_to_widechar_utf16be(LPCCH src, size_t nsrc)
{
lsp_wchar_t cp;
ssize_t nconv = 0;
const lsp_utf16_t *xsrc = reinterpret_cast(src);
nsrc >>= 1;
while (nsrc > 0)
{
// Read code point
cp = read_utf16le_streaming(&xsrc, &nsrc, false);
if (cp == LSP_UTF32_EOF) // No data ?
break;
// Check that we have enough space
nconv += count_utf16(cp);
}
return nconv;
}
static ssize_t multibyte_to_widechar_utf32le(LPCCH src, size_t *nsrc, LPWSTR dst, size_t *ndst)
{
lsp_wchar_t cp;
ssize_t nconv = 0;
const lsp_utf32_t *xsrc = reinterpret_cast(src);
size_t nin = (*nsrc) >> 2;
size_t nout = *ndst;
while (nin > 0)
{
// Read code point
cp = LE_TO_CPU(*(xsrc++));
// Check that we have enough space
size_t len = count_utf16(cp);
if (nout < len)
break;
// Write code point
write_utf16_codepoint(&dst, cp);
nin -= 1;
nout -= len;
nconv += len;
}
*nsrc = ((*nsrc) & 3) + (nin << 2);
*ndst = nout;
return nconv;
}
static ssize_t multibyte_to_widechar_utf32be(LPCCH src, size_t *nsrc, LPWSTR dst, size_t *ndst)
{
lsp_wchar_t cp;
ssize_t nconv = 0;
const lsp_utf32_t *xsrc = reinterpret_cast(src);
size_t nin = (*nsrc) >> 2;
size_t nout = *ndst;
while (nin > 0)
{
// Read code point
cp = BE_TO_CPU(*(xsrc++));
// Check that we have enough space
size_t len = count_utf16(cp);
if (nout < len)
break;
// Write code point
write_utf16_codepoint(&dst, cp);
nin -= 1;
nout -= len;
nconv += len;
}
*nsrc = ((*nsrc) & 3) + (nin << 2);
*ndst = nout;
return nconv;
}
static ssize_t est_multibyte_to_widechar_utf32le(LPCCH src, size_t nsrc)
{
lsp_wchar_t cp;
ssize_t nconv = 0;
const lsp_utf32_t *xsrc = reinterpret_cast(src);
nsrc >>= 2;
while (nsrc > 0)
{
// Read code point
cp = LE_TO_CPU(*(xsrc++));
nconv += count_utf16(cp);
}
return nconv;
}
static ssize_t est_multibyte_to_widechar_utf32be(LPCCH src, size_t nsrc)
{
lsp_wchar_t cp;
ssize_t nconv = 0;
const lsp_utf32_t *xsrc = reinterpret_cast(src);
nsrc >>= 2;
while (nsrc > 0)
{
// Read code point
cp = BE_TO_CPU(*(xsrc++));
nconv += count_utf16(cp);
}
return nconv;
}
ssize_t multibyte_to_widechar(size_t cp, const char *src, size_t *nsrc, lsp_utf16_t *dst, size_t *ndst)
{
ssize_t nconv;
switch (cp)
{
case 1200: // UTF-16LE
nconv = ((dst == NULL) || (ndst == NULL) || (ssize_t(*ndst) <= 0)) ?
est_multibyte_to_widechar_utf16le(src, *nsrc) :
multibyte_to_widechar_utf16le(src, nsrc, dst, ndst);
break;
case 1201: // UTF-16BE
nconv = ((dst == NULL) || (ndst == NULL) || (ssize_t(*ndst) <= 0)) ?
est_multibyte_to_widechar_utf16be(src, *nsrc) :
multibyte_to_widechar_utf16be(src, nsrc, dst, ndst);
break;
case 12000: // UTF-32LE
nconv = ((dst == NULL) || (ndst == NULL) || (ssize_t(*ndst) <= 0)) ?
est_multibyte_to_widechar_utf32le(src, *nsrc) :
multibyte_to_widechar_utf32le(src, nsrc, dst, ndst);
break;
case 12001: // UTF-32BE
nconv = ((dst == NULL) || (ndst == NULL) || (ssize_t(*ndst) <= 0)) ?
est_multibyte_to_widechar_utf32be(src, *nsrc) :
multibyte_to_widechar_utf32be(src, nsrc, dst, ndst);
break;
default:
// We need just to estimate the size?
if ((dst == NULL) || (ndst == NULL) || (ssize_t(*ndst) <= 0))
return ::MultiByteToWideChar(cp, 0, src, *nsrc, 0, 0);
// Do the conversion
nconv = ::MultiByteToWideChar(cp, 0, src, *nsrc, dst, *ndst);
if (nconv == 0)
{
switch (GetLastError())
{
case ERROR_SUCCESS:
return 0;
case ERROR_INSUFFICIENT_BUFFER:
return -STATUS_NO_MEM;
case ERROR_INVALID_FLAGS:
case ERROR_INVALID_PARAMETER:
return -STATUS_BAD_STATE;
case ERROR_NO_UNICODE_TRANSLATION:
return -STATUS_BAD_LOCALE;
default:
return -STATUS_UNKNOWN_ERR;
}
}
// There are converted characters, analyze output
// If function meets invalid sequence, it replaces the code point with such magic value
// We should know if function has failed
if (dst[nconv-1] == 0xfffd)
--nconv;
if (nconv > 0)
{
// Estimate number of bytes decoded (yep, this is dumb but no way...)
ssize_t nbytes = ::WideCharToMultiByte(cp, 0, dst, nconv, NULL, 0, 0, 0);
if (nbytes <= 0)
return -STATUS_IO_ERROR;
*nsrc -= nbytes;
*ndst -= nconv;
}
break;
}
return nconv;
}
static ssize_t widechar_to_multibyte_utf16le(const lsp_utf16_t *src, size_t *nsrc, char *dst, size_t *ndst)
{
lsp_wchar_t cp;
ssize_t nconv = 0;
size_t nin = *nsrc;
size_t nout = *ndst;
lsp_utf16_t *xdst = reinterpret_cast(dst);
while (nin > 0)
{
size_t xin = nin;
cp = read_utf16_streaming(&src, &xin, false);
if (cp == LSP_UTF32_EOF) // No data ?
break;
// Check that we have enough space
size_t len = sizeof_utf16(cp);
if (nout < len)
break;
// Write code point
write_utf16le_codepoint(&xdst, cp);
nin = xin;
nout -= len;
nconv += len;
}
*nsrc = nin;
*ndst = nout;
return nconv;
}
static ssize_t widechar_to_multibyte_utf16be(const lsp_utf16_t *src, size_t *nsrc, char *dst, size_t *ndst)
{
lsp_wchar_t cp;
ssize_t nconv = 0;
size_t nin = *nsrc;
size_t nout = *ndst;
lsp_utf16_t *xdst = reinterpret_cast(dst);
while (nin > 0)
{
size_t xin = nin;
cp = read_utf16_streaming(&src, &xin, false);
if (cp == LSP_UTF32_EOF) // No data ?
break;
// Check that we have enough space
size_t len = sizeof_utf16(cp);
if (nout < len)
break;
// Write code point
write_utf16be_codepoint(&xdst, cp);
nin = xin;
nout -= len;
nconv += len;
}
*nsrc = nin;
*ndst = nout;
return nconv;
}
static ssize_t est_widechar_to_multibyte_utf16(const lsp_utf16_t *src, size_t nsrc)
{
lsp_wchar_t cp;
ssize_t nconv = 0;
while (nsrc > 0)
{
cp = read_utf16_streaming(&src, &nsrc, false);
if (cp == LSP_UTF32_EOF) // No data ?
break;
// Check that we have enough space
nconv += sizeof_utf16(cp);
}
return nconv;
}
static ssize_t widechar_to_multibyte_utf32le(const lsp_utf16_t *src, size_t *nsrc, char *dst, size_t *ndst)
{
lsp_wchar_t cp;
ssize_t nconv = 0;
size_t nin = *nsrc;
size_t nout = *ndst;
lsp_utf32_t *xdst = reinterpret_cast(dst);
while (nin > 0)
{
size_t xin = nin;
cp = read_utf16_streaming(&src, &xin, false);
if (cp == LSP_UTF32_EOF) // No data ?
break;
// Check that we have enough space
if (nout < sizeof(lsp_utf32_t))
break;
// Write code point
*(xdst++) = CPU_TO_LE(cp);
nin = xin;
nout -= sizeof(lsp_utf32_t);
nconv += sizeof(lsp_utf32_t);
}
*nsrc = nin;
*ndst = nout;
return nconv;
}
static ssize_t widechar_to_multibyte_utf32be(const lsp_utf16_t *src, size_t *nsrc, char *dst, size_t *ndst)
{
lsp_wchar_t cp;
ssize_t nconv = 0;
size_t nin = *nsrc;
size_t nout = *ndst;
lsp_utf32_t *xdst = reinterpret_cast(dst);
while (nin > 0)
{
size_t xin = nin;
cp = read_utf16_streaming(&src, &xin, false);
if (cp == LSP_UTF32_EOF) // No data ?
break;
// Check that we have enough space
if (nout < sizeof(lsp_utf32_t))
break;
// Write code point
*(xdst++) = CPU_TO_BE(cp);
nin = xin;
nout -= sizeof(lsp_utf32_t);
nconv += sizeof(lsp_utf32_t);
}
*nsrc = nin;
*ndst = nout;
return nconv;
}
static ssize_t est_widechar_to_multibyte_utf32(const lsp_utf16_t *src, size_t nsrc)
{
lsp_wchar_t cp;
ssize_t nconv = 0;
while (nsrc > 0)
{
cp = read_utf16_streaming(&src, &nsrc, false);
if (cp == LSP_UTF32_EOF) // No data ?
break;
nconv += sizeof(lsp_utf32_t);
}
return nconv;
}
static size_t widechar_to_multibyte_split(const lsp_utf16_t *src, size_t limit)
{
// Estimate the middle of an array
size_t half = limit >> 1;
if (half <= 0)
return half;
// Now scan valid code points until we reach the end of array
lsp_wchar_t cp;
limit = half;
while (true)
{
cp = read_utf16_streaming(&src, &limit, false);
if (cp == LSP_UTF32_EOF) // No data ?
break;
}
// Return the result as middle of array without remained points in limit
return half - limit;
}
ssize_t widechar_to_multibyte(size_t cp, const lsp_utf16_t *src, size_t *nsrc, char *dst, size_t *ndst)
{
ssize_t nconv;
switch (cp)
{
case 1200: // UTF-16LE
nconv = ((dst == NULL) || (ndst == NULL) || (ssize_t(*ndst) <= 0)) ?
est_widechar_to_multibyte_utf16(src, *nsrc) :
widechar_to_multibyte_utf16le(src, nsrc, dst, ndst);
break;
case 1201: // UTF-16BE
nconv = ((dst == NULL) || (ndst == NULL) || (ssize_t(*ndst) <= 0)) ?
est_widechar_to_multibyte_utf16(src, *nsrc) :
widechar_to_multibyte_utf16be(src, nsrc, dst, ndst);
break;
case 12000: // UTF-32LE
nconv = ((dst == NULL) || (ndst == NULL) || (ssize_t(*ndst) <= 0)) ?
est_widechar_to_multibyte_utf32(src, *nsrc) :
widechar_to_multibyte_utf32le(src, nsrc, dst, ndst);
break;
case 12001: // UTF-32BE
nconv = ((dst == NULL) || (ndst == NULL) || (ssize_t(*ndst) <= 0)) ?
est_widechar_to_multibyte_utf32(src, *nsrc) :
widechar_to_multibyte_utf32be(src, nsrc, dst, ndst);
break;
default:
{
// Just estimate number of characters?
if ((dst == NULL) || (ndst == NULL) || (ssize_t(*ndst) <= 0))
{
nconv = ::WideCharToMultiByte(cp, 0, src, *nsrc, NULL, 0, 0, FALSE);
if (nconv == 0)
{
switch (::GetLastError())
{
case ERROR_SUCCESS:
return 0;
case ERROR_INSUFFICIENT_BUFFER:
return -STATUS_NO_MEM;
case ERROR_INVALID_FLAGS:
case ERROR_INVALID_PARAMETER:
return -STATUS_BAD_STATE;
case ERROR_NO_UNICODE_TRANSLATION:
return -STATUS_BAD_LOCALE;
default:
return -STATUS_UNKNOWN_ERR;
}
}
return nconv;
}
// Perform first try
size_t xnsrc = *nsrc;
nconv = ::WideCharToMultiByte(cp, 0, src, xnsrc, dst, *ndst, 0, FALSE);
// Do while conversion is unsuccessful
while (nconv <= 0)
{
// There was a fail, analyze it
switch (::GetLastError())
{
case ERROR_SUCCESS:
return 0;
case ERROR_INSUFFICIENT_BUFFER:
break; // Will retry with twice lesser input buffer
case ERROR_INVALID_FLAGS:
case ERROR_INVALID_PARAMETER:
return -STATUS_BAD_STATE;
case ERROR_NO_UNICODE_TRANSLATION:
return -STATUS_BAD_LOCALE;
default:
return -STATUS_UNKNOWN_ERR;
}
// Try to twice reduce the buffer size, validate data for surrogates
xnsrc = widechar_to_multibyte_split(src, xnsrc);
if (xnsrc <= 0)
break;
// Perform next conversion try with lesser buffer
nconv = ::WideCharToMultiByte(cp, 0, src, xnsrc, dst, *ndst, 0, FALSE);
}
*ndst -= nconv;
*nsrc -= xnsrc;
}
break;
}
return nconv;
}
#endif /* PLATFORM_WINDOWS */
#ifndef ARCH_LE
int wchar_cmp(const lsp_wchar_t *s1, const lsp_wchar_t *s2, size_t count)
{
while (count--)
{
int32_t retval = int32_t(*(s1++)) - int32_t(*(s2++));
if (retval != 0)
return (retval > 0) ? 1 : -1;
}
return 0;
}
#endif
int wchar_casecmp(const lsp_wchar_t *s1, const lsp_wchar_t *s2, size_t count)
{
while (count--)
{
int32_t retval = int32_t(to_lower(*(s1++))) - int32_t(to_lower(*(s2++)));
if (retval != 0)
return (retval > 0) ? 1 : -1;
}
return 0;
}
lsp_wchar_t to_lower(lsp_wchar_t c)
{
// Latin codepoints
if (c <= 0x7f)
{
if ((c >= 'A') && (c <= 'Z'))
return c - 'A' + 'a';
return c;
}
// Cyrillic + Cyrillic supplement codepoints
if ((c >= 0x400) && (c <= 0x52f))
{
if (c <= 0x40f)
return c + 0x50;
if (c <= 0x42f)
return c + 0x20;
if (!(c & 0x01))
{
if ((c >= 0x460) && (c <= 0x481))
return c + 1;
if (c >= 0x48a)
return c + 1;
}
return c;
}
// Cyrillic extended B
if ((c >= 0xa640) && (c <= 0xa69f))
{
if (!(c & 0x01))
{
if (c <= 0xa66d)
return c + 1;
if ((c >= 0xa680) && (c <= 0xa69b))
return c + 1;
}
return c;
}
// Other possible locales
return towlower(c);
}
lsp_wchar_t to_upper(lsp_wchar_t c)
{
// Latin codepoints
if (c <= 0x7f)
{
if ((c >= 'a') && (c <= 'z'))
return c - 'a' + 'A';
return c;
}
// Cyrillic + Cyrillic supplement codepoints
if ((c >= 0x400) && (c <= 0x52f))
{
if ((c >= 0x450) && (c <= 0x45f))
return c - 0x50;
if ((c >= 0x430) && (c <= 0x44f))
return c - 0x20;
if (c & 0x01)
{
if ((c >= 0x460) && (c <= 0x481))
return c - 1;
if (c >= 0x48a)
return c - 1;
}
return c;
}
// Cyrillic extended B
if ((c >= 0xa640) && (c <= 0xa69f))
{
if (c & 0x01)
{
if (c <= 0xa66d)
return c - 1;
if ((c >= 0xa680) && (c <= 0xa69b))
return c - 1;
}
return c;
}
// Other possible locales
return towupper(c);
}
} /* namespace lsp */