Debian Codegraph @

common/mptString.h

raw
1 /*
2 * mptString.h
3 * ----------
4 * Purpose: Small string-related utilities, number and message formatting.
5 * Notes : Currently none.
6 * Authors: OpenMPT Devs
7 * The OpenMPT source code is released under the BSD license. Read LICENSE for more details.
8 */
9
10
11 #pragma once
12
13 #include "openmpt/all/BuildSettings.hpp"
14
15 #include "mpt/string/types.hpp"
16 #include "mpt/string/utility.hpp"
17 #include "mpt/string_transcode/transcode.hpp"
18
19 #include <algorithm>
20 #include <string>
21 #include <string_view>
22 #include <utility>
23
24 #include <cstddef>
25 #if defined(MODPLUG_TRACKER)
26 #include <cwctype>
27 #endif // MODPLUG_TRACKER
28
29
30
31 OPENMPT_NAMESPACE_BEGIN
32
33
34 namespace mpt
35 {
36
37
38
39 enum class CharsetEnum {
40
41 UTF8,
42
43 ASCII, // strictly 7-bit ASCII
44
45 ISO8859_1,
46 ISO8859_15,
47
48 CP437,
49 CP737,
50 CP775,
51 CP850,
52 CP852,
53 CP855,
54 CP857,
55 CP860,
56 CP861,
57 CP862,
58 CP863,
59 CP864,
60 CP865,
61 CP866,
62 CP869,
63 CP874,
64
65 CP437AMS,
66 CP437AMS2,
67
68 Windows1252,
69
70 Amiga,
71 RISC_OS,
72 AtariST,
73
74 ISO8859_1_no_C1,
75 ISO8859_15_no_C1,
76 Amiga_no_C1,
77
78 #if defined(MPT_ENABLE_CHARSET_LOCALE)
79 Locale, // CP_ACP on windows, current C locale otherwise
80 #endif // MPT_ENABLE_CHARSET_LOCALE
81
82 };
83
84 namespace CharsetTable
85 {
86
87 #define C(x) (mpt::char_value((x)))
88
89 // AMS1 actually only supports ASCII plus the modified control characters and no high chars at all.
90 // Just default to CP437 for those to keep things simple.
91 inline constexpr char32_t CP437AMS[256] = {
92 C(' '),0x0001,0x0002,0x0003,0x00e4,0x0005,0x00e5,0x0007,0x0008,0x0009,0x000a,0x000b,0x000c,0x000d,0x00c4,0x00c5, // differs from CP437
93 0x0010,0x0011,0x0012,0x0013,0x00f6,0x0015,0x0016,0x0017,0x0018,0x00d6,0x001a,0x001b,0x001c,0x001d,0x001e,0x001f, // differs from CP437
94 0x0020,0x0021,0x0022,0x0023,0x0024,0x0025,0x0026,0x0027,0x0028,0x0029,0x002a,0x002b,0x002c,0x002d,0x002e,0x002f,
95 0x0030,0x0031,0x0032,0x0033,0x0034,0x0035,0x0036,0x0037,0x0038,0x0039,0x003a,0x003b,0x003c,0x003d,0x003e,0x003f,
96 0x0040,0x0041,0x0042,0x0043,0x0044,0x0045,0x0046,0x0047,0x0048,0x0049,0x004a,0x004b,0x004c,0x004d,0x004e,0x004f,
97 0x0050,0x0051,0x0052,0x0053,0x0054,0x0055,0x0056,0x0057,0x0058,0x0059,0x005a,0x005b,0x005c,0x005d,0x005e,0x005f,
98 0x0060,0x0061,0x0062,0x0063,0x0064,0x0065,0x0066,0x0067,0x0068,0x0069,0x006a,0x006b,0x006c,0x006d,0x006e,0x006f,
99 0x0070,0x0071,0x0072,0x0073,0x0074,0x0075,0x0076,0x0077,0x0078,0x0079,0x007a,0x007b,0x007c,0x007d,0x007e,0x2302,
100 0x00c7,0x00fc,0x00e9,0x00e2,0x00e4,0x00e0,0x00e5,0x00e7,0x00ea,0x00eb,0x00e8,0x00ef,0x00ee,0x00ec,0x00c4,0x00c5,
101 0x00c9,0x00e6,0x00c6,0x00f4,0x00f6,0x00f2,0x00fb,0x00f9,0x00ff,0x00d6,0x00dc,0x00a2,0x00a3,0x00a5,0x20a7,0x0192,
102 0x00e1,0x00ed,0x00f3,0x00fa,0x00f1,0x00d1,0x00aa,0x00ba,0x00bf,0x2310,0x00ac,0x00bd,0x00bc,0x00a1,0x00ab,0x00bb,
103 0x2591,0x2592,0x2593,0x2502,0x2524,0x2561,0x2562,0x2556,0x2555,0x2563,0x2551,0x2557,0x255d,0x255c,0x255b,0x2510,
104 0x2514,0x2534,0x252c,0x251c,0x2500,0x253c,0x255e,0x255f,0x255a,0x2554,0x2569,0x2566,0x2560,0x2550,0x256c,0x2567,
105 0x2568,0x2564,0x2565,0x2559,0x2558,0x2552,0x2553,0x256b,0x256a,0x2518,0x250c,0x2588,0x2584,0x258c,0x2590,0x2580,
106 0x03b1,0x00df,0x0393,0x03c0,0x03a3,0x03c3,0x00b5,0x03c4,0x03a6,0x0398,0x03a9,0x03b4,0x221e,0x03c6,0x03b5,0x2229,
107 0x2261,0x00b1,0x2265,0x2264,0x2320,0x2321,0x00f7,0x2248,0x00b0,0x2219,0x00b7,0x221a,0x207f,0x00b2,0x25a0,0x00a0
108 };
109
110 // AMS2: Looking at Velvet Studio's bitmap font (TPIC32.PCX), these appear to be the only supported non-ASCII chars.
111 inline constexpr char32_t CP437AMS2[256] = {
112 C(' '),0x00a9,0x221a,0x00b7,C('0'),C('1'),C('2'),C('3'),C('4'),C('5'),C('6'),C('7'),C('8'),C('9'),C('A'),C('B'), // differs from CP437
113 C('C'),C('D'),C('E'),C('F'),C(' '),0x00a7,C(' '),C(' '),C(' '),C(' '),C(' '),C(' '),C(' '),C(' '),C(' '),C(' '), // differs from CP437
114 0x0020,0x0021,0x0022,0x0023,0x0024,0x0025,0x0026,0x0027,0x0028,0x0029,0x002a,0x002b,0x002c,0x002d,0x002e,0x002f,
115 0x0030,0x0031,0x0032,0x0033,0x0034,0x0035,0x0036,0x0037,0x0038,0x0039,0x003a,0x003b,0x003c,0x003d,0x003e,0x003f,
116 0x0040,0x0041,0x0042,0x0043,0x0044,0x0045,0x0046,0x0047,0x0048,0x0049,0x004a,0x004b,0x004c,0x004d,0x004e,0x004f,
117 0x0050,0x0051,0x0052,0x0053,0x0054,0x0055,0x0056,0x0057,0x0058,0x0059,0x005a,0x005b,0x005c,0x005d,0x005e,0x005f,
118 0x0060,0x0061,0x0062,0x0063,0x0064,0x0065,0x0066,0x0067,0x0068,0x0069,0x006a,0x006b,0x006c,0x006d,0x006e,0x006f,
119 0x0070,0x0071,0x0072,0x0073,0x0074,0x0075,0x0076,0x0077,0x0078,0x0079,0x007a,0x007b,0x007c,0x007d,0x007e,0x2302,
120 0x00c7,0x00fc,0x00e9,0x00e2,0x00e4,0x00e0,0x00e5,0x00e7,0x00ea,0x00eb,0x00e8,0x00ef,0x00ee,0x00ec,0x00c4,0x00c5,
121 0x00c9,0x00e6,0x00c6,0x00f4,0x00f6,0x00f2,0x00fb,0x00f9,0x00ff,0x00d6,0x00dc,0x00a2,0x00a3,0x00a5,0x20a7,0x0192,
122 0x00e1,0x00ed,0x00f3,0x00fa,0x00f1,0x00d1,0x00aa,0x00ba,0x00bf,0x2310,0x00ac,0x00bd,0x00bc,0x00a1,0x00ab,0x00bb,
123 0x2591,0x2592,0x2593,0x2502,0x2524,0x2561,0x2562,0x2556,0x2555,0x2563,0x2551,0x2557,0x255d,0x255c,0x255b,0x2510,
124 0x2514,0x2534,0x252c,0x251c,0x2500,0x253c,0x255e,0x255f,0x255a,0x2554,0x2569,0x2566,0x2560,0x2550,0x256c,0x2567,
125 0x2568,0x2564,0x2565,0x2559,0x2558,0x2552,0x2553,0x256b,0x256a,0x2518,0x250c,0x2588,0x2584,0x258c,0x2590,0x2580,
126 0x03b1,0x00df,0x0393,0x03c0,0x03a3,0x03c3,0x00b5,0x03c4,0x03a6,0x0398,0x03a9,0x03b4,0x221e,0x03c6,0x03b5,0x2229,
127 0x2261,0x00b1,0x2265,0x2264,0x2320,0x2321,0x00f7,0x2248,0x00b0,0x2219,0x00b7,0x221a,0x207f,0x00b2,0x25a0,0x00a0
128 };
129
130 #undef C
131
132 } // namespace CharsetTable
133
134
135
136 struct CharsetTranscoder
137 {
138
139 private:
140
141 CharsetEnum m_Charset;
142
143 public:
144
145 constexpr CharsetEnum GetCharset() const noexcept
146 {
147 return m_Charset;
148 }
149
150 constexpr CharsetTranscoder(CharsetEnum charset) noexcept
151 : m_Charset(charset)
152 {
153 return;
154 }
155
156 constexpr operator CharsetEnum() const noexcept
157 {
158 return m_Charset;
159 }
160
161 // templated on 8bit strings because of type-safe variants
162 template <typename Tdststring>
163 inline Tdststring encode(const mpt::widestring &src) const
164 {
165 static_assert(sizeof(typename Tdststring::value_type) == sizeof(char));
166 static_assert(mpt::is_character<typename Tdststring::value_type>::value);
167 switch(m_Charset)
168 {
169 #if defined(MPT_ENABLE_CHARSET_LOCALE)
170 case CharsetEnum::Locale: return mpt::encode<Tdststring>(mpt::logical_encoding::locale, src); break;
171 #endif
172 case CharsetEnum::UTF8: return mpt::encode<Tdststring>(mpt::common_encoding::utf8, src); break;
173 case CharsetEnum::ASCII: return mpt::encode<Tdststring>(mpt::common_encoding::ascii, src); break;
174 case CharsetEnum::ISO8859_1: return mpt::encode<Tdststring>(mpt::common_encoding::iso8859_1, src); break;
175 case CharsetEnum::ISO8859_15: return mpt::encode<Tdststring>(mpt::common_encoding::iso8859_15, src); break;
176 case CharsetEnum::CP437: return mpt::encode<Tdststring>(mpt::common_encoding::cp437, src); break;
177 case CharsetEnum::CP737: return mpt::encode<Tdststring>(mpt::common_encoding::cp737, src); break;
178 case CharsetEnum::CP775: return mpt::encode<Tdststring>(mpt::common_encoding::cp775, src); break;
179 case CharsetEnum::CP850: return mpt::encode<Tdststring>(mpt::common_encoding::cp850, src); break;
180 case CharsetEnum::CP852: return mpt::encode<Tdststring>(mpt::common_encoding::cp852, src); break;
181 case CharsetEnum::CP855: return mpt::encode<Tdststring>(mpt::common_encoding::cp855, src); break;
182 case CharsetEnum::CP857: return mpt::encode<Tdststring>(mpt::common_encoding::cp857, src); break;
183 case CharsetEnum::CP860: return mpt::encode<Tdststring>(mpt::common_encoding::cp860, src); break;
184 case CharsetEnum::CP861: return mpt::encode<Tdststring>(mpt::common_encoding::cp861, src); break;
185 case CharsetEnum::CP862: return mpt::encode<Tdststring>(mpt::common_encoding::cp862, src); break;
186 case CharsetEnum::CP863: return mpt::encode<Tdststring>(mpt::common_encoding::cp863, src); break;
187 case CharsetEnum::CP864: return mpt::encode<Tdststring>(mpt::common_encoding::cp864, src); break;
188 case CharsetEnum::CP865: return mpt::encode<Tdststring>(mpt::common_encoding::cp865, src); break;
189 case CharsetEnum::CP866: return mpt::encode<Tdststring>(mpt::common_encoding::cp866, src); break;
190 case CharsetEnum::CP869: return mpt::encode<Tdststring>(mpt::common_encoding::cp869, src); break;
191 case CharsetEnum::CP874: return mpt::encode<Tdststring>(mpt::common_encoding::cp874, src); break;
192 case CharsetEnum::CP437AMS: return mpt::encode<Tdststring>(CharsetTable::CP437AMS, src); break;
193 case CharsetEnum::CP437AMS2: return mpt::encode<Tdststring>(CharsetTable::CP437AMS2, src); break;
194 case CharsetEnum::Windows1252: return mpt::encode<Tdststring>(mpt::common_encoding::windows1252, src); break;
195 case CharsetEnum::Amiga: return mpt::encode<Tdststring>(mpt::common_encoding::amiga, src); break;
196 case CharsetEnum::RISC_OS: return mpt::encode<Tdststring>(mpt::common_encoding::riscos, src); break;
197 case CharsetEnum::AtariST: return mpt::encode<Tdststring>(mpt::common_encoding::atarist, src); break;
198 case CharsetEnum::ISO8859_1_no_C1: return mpt::encode<Tdststring>(mpt::common_encoding::iso8859_1_no_c1, src); break;
199 case CharsetEnum::ISO8859_15_no_C1: return mpt::encode<Tdststring>(mpt::common_encoding::iso8859_15_no_c1, src); break;
200 case CharsetEnum::Amiga_no_C1: return mpt::encode<Tdststring>(mpt::common_encoding::amiga_no_c1, src); break;
201 }
202 return Tdststring();
203 }
204
205 // templated on 8bit strings because of type-safe variants
206 template <typename Tsrcstring>
207 inline mpt::widestring decode(const Tsrcstring &src) const
208 {
209 static_assert(sizeof(typename Tsrcstring::value_type) == sizeof(char));
210 static_assert(mpt::is_character<typename Tsrcstring::value_type>::value);
211 switch(m_Charset)
212 {
213 #if defined(MPT_ENABLE_CHARSET_LOCALE)
214 case CharsetEnum::Locale: return mpt::decode<Tsrcstring>(mpt::logical_encoding::locale, src); break;
215 #endif
216 case CharsetEnum::UTF8: return mpt::decode<Tsrcstring>(mpt::common_encoding::utf8, src); break;
217 case CharsetEnum::ASCII: return mpt::decode<Tsrcstring>(mpt::common_encoding::ascii, src); break;
218 case CharsetEnum::ISO8859_1: return mpt::decode<Tsrcstring>(mpt::common_encoding::iso8859_1, src); break;
219 case CharsetEnum::ISO8859_15: return mpt::decode<Tsrcstring>(mpt::common_encoding::iso8859_15, src); break;
220 case CharsetEnum::CP437: return mpt::decode<Tsrcstring>(mpt::common_encoding::cp437, src); break;
221 case CharsetEnum::CP737: return mpt::decode<Tsrcstring>(mpt::common_encoding::cp737, src); break;
222 case CharsetEnum::CP775: return mpt::decode<Tsrcstring>(mpt::common_encoding::cp775, src); break;
223 case CharsetEnum::CP850: return mpt::decode<Tsrcstring>(mpt::common_encoding::cp850, src); break;
224 case CharsetEnum::CP852: return mpt::decode<Tsrcstring>(mpt::common_encoding::cp852, src); break;
225 case CharsetEnum::CP855: return mpt::decode<Tsrcstring>(mpt::common_encoding::cp855, src); break;
226 case CharsetEnum::CP857: return mpt::decode<Tsrcstring>(mpt::common_encoding::cp857, src); break;
227 case CharsetEnum::CP860: return mpt::decode<Tsrcstring>(mpt::common_encoding::cp860, src); break;
228 case CharsetEnum::CP861: return mpt::decode<Tsrcstring>(mpt::common_encoding::cp861, src); break;
229 case CharsetEnum::CP862: return mpt::decode<Tsrcstring>(mpt::common_encoding::cp862, src); break;
230 case CharsetEnum::CP863: return mpt::decode<Tsrcstring>(mpt::common_encoding::cp863, src); break;
231 case CharsetEnum::CP864: return mpt::decode<Tsrcstring>(mpt::common_encoding::cp864, src); break;
232 case CharsetEnum::CP865: return mpt::decode<Tsrcstring>(mpt::common_encoding::cp865, src); break;
233 case CharsetEnum::CP866: return mpt::decode<Tsrcstring>(mpt::common_encoding::cp866, src); break;
234 case CharsetEnum::CP869: return mpt::decode<Tsrcstring>(mpt::common_encoding::cp869, src); break;
235 case CharsetEnum::CP874: return mpt::decode<Tsrcstring>(mpt::common_encoding::cp874, src); break;
236 case CharsetEnum::CP437AMS: return mpt::decode<Tsrcstring>(CharsetTable::CP437AMS, src); break;
237 case CharsetEnum::CP437AMS2: return mpt::decode<Tsrcstring>(CharsetTable::CP437AMS2, src); break;
238 case CharsetEnum::Windows1252: return mpt::decode<Tsrcstring>(mpt::common_encoding::windows1252, src); break;
239 case CharsetEnum::Amiga: return mpt::decode<Tsrcstring>(mpt::common_encoding::amiga, src); break;
240 case CharsetEnum::RISC_OS: return mpt::decode<Tsrcstring>(mpt::common_encoding::riscos, src); break;
241 case CharsetEnum::AtariST: return mpt::decode<Tsrcstring>(mpt::common_encoding::atarist, src); break;
242 case CharsetEnum::ISO8859_1_no_C1: return mpt::decode<Tsrcstring>(mpt::common_encoding::iso8859_1_no_c1, src); break;
243 case CharsetEnum::ISO8859_15_no_C1: return mpt::decode<Tsrcstring>(mpt::common_encoding::iso8859_15_no_c1, src); break;
244 case CharsetEnum::Amiga_no_C1: return mpt::decode<Tsrcstring>(mpt::common_encoding::amiga_no_c1, src); break;
245 }
246 return mpt::widestring();
247 }
248
249 };
250
251
252
253 struct Charset
254 : public CharsetTranscoder
255 {
256
257 constexpr Charset(mpt::CharsetEnum charset) noexcept
258 : CharsetTranscoder(charset)
259 {
260 return;
261 }
262
263 constexpr Charset(mpt::CharsetTranscoder charset) noexcept
264 : CharsetTranscoder(charset.GetCharset())
265 {
266 return;
267 }
268
269 static inline constexpr auto UTF8 = mpt::CharsetTranscoder{ mpt::CharsetEnum::UTF8 };
270 static inline constexpr auto ASCII = mpt::CharsetTranscoder{ mpt::CharsetEnum::ASCII };
271 static inline constexpr auto ISO8859_1 = mpt::CharsetTranscoder{ mpt::CharsetEnum::ISO8859_1 };
272 static inline constexpr auto ISO8859_15 = mpt::CharsetTranscoder{ mpt::CharsetEnum::ISO8859_15 };
273 static inline constexpr auto CP437 = mpt::CharsetTranscoder{ mpt::CharsetEnum::CP437 };
274 static inline constexpr auto CP737 = mpt::CharsetTranscoder{ mpt::CharsetEnum::CP737 };
275 static inline constexpr auto CP775 = mpt::CharsetTranscoder{ mpt::CharsetEnum::CP775 };
276 static inline constexpr auto CP850 = mpt::CharsetTranscoder{ mpt::CharsetEnum::CP850 };
277 static inline constexpr auto CP852 = mpt::CharsetTranscoder{ mpt::CharsetEnum::CP852 };
278 static inline constexpr auto CP855 = mpt::CharsetTranscoder{ mpt::CharsetEnum::CP855 };
279 static inline constexpr auto CP857 = mpt::CharsetTranscoder{ mpt::CharsetEnum::CP857 };
280 static inline constexpr auto CP860 = mpt::CharsetTranscoder{ mpt::CharsetEnum::CP860 };
281 static inline constexpr auto CP861 = mpt::CharsetTranscoder{ mpt::CharsetEnum::CP861 };
282 static inline constexpr auto CP862 = mpt::CharsetTranscoder{ mpt::CharsetEnum::CP862 };
283 static inline constexpr auto CP863 = mpt::CharsetTranscoder{ mpt::CharsetEnum::CP863 };
284 static inline constexpr auto CP864 = mpt::CharsetTranscoder{ mpt::CharsetEnum::CP864 };
285 static inline constexpr auto CP865 = mpt::CharsetTranscoder{ mpt::CharsetEnum::CP865 };
286 static inline constexpr auto CP866 = mpt::CharsetTranscoder{ mpt::CharsetEnum::CP866 };
287 static inline constexpr auto CP869 = mpt::CharsetTranscoder{ mpt::CharsetEnum::CP869 };
288 static inline constexpr auto CP874 = mpt::CharsetTranscoder{ mpt::CharsetEnum::CP874 };
289 static inline constexpr auto CP437AMS = mpt::CharsetTranscoder{ mpt::CharsetEnum::CP437AMS };
290 static inline constexpr auto CP437AMS2 = mpt::CharsetTranscoder{ mpt::CharsetEnum::CP437AMS2 };
291 static inline constexpr auto Windows1252 = mpt::CharsetTranscoder{ mpt::CharsetEnum::Windows1252 };
292 static inline constexpr auto Amiga = mpt::CharsetTranscoder{ mpt::CharsetEnum::Amiga };
293 static inline constexpr auto RISC_OS = mpt::CharsetTranscoder{ mpt::CharsetEnum::RISC_OS };
294 static inline constexpr auto AtariST = mpt::CharsetTranscoder{ mpt::CharsetEnum::AtariST };
295 static inline constexpr auto ISO8859_1_no_C1 = mpt::CharsetTranscoder{ mpt::CharsetEnum::ISO8859_1_no_C1 };
296 static inline constexpr auto ISO8859_15_no_C1 = mpt::CharsetTranscoder{ mpt::CharsetEnum::ISO8859_15_no_C1 };
297 static inline constexpr auto Amiga_no_C1 = mpt::CharsetTranscoder{ mpt::CharsetEnum::Amiga_no_C1 };
298
299 #if defined(MPT_ENABLE_CHARSET_LOCALE)
300 static inline constexpr auto Locale = mpt::CharsetTranscoder{ mpt::CharsetEnum::Locale };
301 #endif // MPT_ENABLE_CHARSET_LOCALE
302
303 };
304
305
306
307
308
309 // Checks if the std::string represents an UTF8 string.
310 // This is currently implemented as converting to std::wstring and back assuming UTF8 both ways,
311 // and comparing the result to the original string.
312 // Caveats:
313 // - can give false negatives because of possible unicode normalization during conversion
314 // - can give false positives if the 8bit encoding contains high-ascii only in valid utf8 groups
315 // - slow because of double conversion
316 inline bool IsUTF8(const std::string &str)
317 {
318 return mpt::is_utf8(str);
319 }
320
321
322
323 template <typename Tsrc>
324 inline mpt::ustring ToUnicode(Tsrc &&str)
325 {
326 return mpt::transcode<mpt::ustring>(std::forward<Tsrc>(str));
327 }
328 template <typename Tsrc, typename Tencoding>
329 inline mpt::ustring ToUnicode(Tencoding &&from, Tsrc &&str)
330 {
331 return mpt::transcode<mpt::ustring>(std::forward<Tencoding>(from), std::forward<Tsrc>(str));
332 }
333
334 #if !defined(MPT_COMPILER_QUIRK_NO_WCHAR)
335 // Convert to a wide character string.
336 // The wide encoding is UTF-16 or UTF-32, based on sizeof(wchar_t).
337 // If str does not contain any invalid characters, this conversion is lossless.
338 // Invalid source bytes will be replaced by some replacement character or string.
339 template <typename Tsrc>
340 inline std::wstring ToWide(Tsrc &&str)
341 {
342 return mpt::transcode<std::wstring>(std::forward<Tsrc>(str));
343 }
344 template <typename Tsrc, typename Tencoding>
345 inline std::wstring ToWide(Tencoding &&from, Tsrc &&str)
346 {
347 return mpt::transcode<std::wstring>(std::forward<Tencoding>(from), std::forward<Tsrc>(str));
348 }
349 #endif
350
351 // Convert to a string encoded in the 'to'-specified character set.
352 // If str does not contain any invalid characters,
353 // this conversion will be lossless iff, and only iff,
354 // 'to' is UTF8.
355 // Invalid source bytes or characters that are not representable in the
356 // destination charset will be replaced by some replacement character or string.
357 template <typename Tsrc, typename Tencoding>
358 inline std::string ToCharset(Tencoding &&to, Tsrc &&str)
359 {
360 return mpt::transcode<std::string>(std::forward<Tencoding>(to), std::forward<Tsrc>(str));
361 }
362 template <typename Tsrc, typename Tto, typename Tfrom>
363 inline std::string ToCharset(Tto &&to, Tfrom &&from, Tsrc &&str)
364 {
365 return mpt::transcode<std::string>(std::forward<Tto>(to), std::forward<Tfrom>(from), std::forward<Tsrc>(str));
366 }
367
368 #if defined(MPT_ENABLE_CHARSET_LOCALE)
369 template <typename Tsrc>
370 inline mpt::lstring ToLocale(Tsrc &&str)
371 {
372 return mpt::transcode<mpt::lstring>(std::forward<Tsrc>(str));
373 }
374 template <typename Tsrc, typename Tencoding>
375 inline mpt::lstring ToLocale(Tencoding &&from, Tsrc &&str)
376 {
377 return mpt::transcode<mpt::lstring>(std::forward<Tencoding>(from), std::forward<Tsrc>(str));
378 }
379 #endif // MPT_ENABLE_CHARSET_LOCALE
380
381 #if MPT_OS_WINDOWS
382 template <typename Tsrc>
383 inline mpt::winstring ToWin(Tsrc &&str)
384 {
385 return mpt::transcode<mpt::winstring>(std::forward<Tsrc>(str));
386 }
387 template <typename Tsrc, typename Tencoding>
388 inline mpt::winstring ToWin(Tencoding &&from, Tsrc &&str)
389 {
390 return mpt::transcode<mpt::winstring>(std::forward<Tencoding>(from), std::forward<Tsrc>(str));
391 }
392 #endif // MPT_OS_WINDOWS
393
394 #if defined(MPT_WITH_MFC)
395 template <typename Tsrc>
396 inline CString ToCString(Tsrc &&str)
397 {
398 return mpt::transcode<CString>(std::forward<Tsrc>(str));
399 }
400 template <typename Tsrc, typename Tencoding>
401 inline CString ToCString(Tencoding &&from, Tsrc &&str)
402 {
403 return mpt::transcode<CString>(std::forward<Tencoding>(from), std::forward<Tsrc>(str));
404 }
405 #endif // MPT_WITH_MFC
406
407
408
409 #define UC_(x) MPT_UCHAR(x)
410 #define UL_(x) MPT_ULITERAL(x)
411 #define UV_(x) MPT_USTRINGVIEW(x)
412 #define U_(x) MPT_USTRING(x)
413
414
415
416 // The MPT_UTF8 allows specifying UTF8 char arrays.
417 // The resulting type is mpt::ustring and the construction might require runtime translation,
418 // i.e. it is NOT generally available at compile time.
419 // Use explicit UTF8 encoding,
420 // i.e. U+00FC (LATIN SMALL LETTER U WITH DIAERESIS) would be written as "\xC3\xBC".
421 #define MPT_UTF8(x) mpt::transcode<mpt::ustring>(mpt::common_encoding::utf8, x)
422
423
424
425 template <typename Tsrc, typename Tencoding>
426 inline mpt::ustring ToUnicode(uint16 codepage, Tencoding &&fallback, Tsrc &&str)
427 {
428 #if MPT_OS_WINDOWS && !defined(MPT_COMPILER_QUIRK_NO_WCHAR)
429 mpt::ustring result;
430 std::optional<mpt::common_encoding> charset = mpt::optional_encoding_from_codepage(codepage);
431 if(charset.has_value())
432 {
433 #if defined(MPT_LIBCXX_QUIRK_NO_OPTIONAL_VALUE)
434 result = mpt::transcode<mpt::ustring>(*charset, std::forward<Tsrc>(str));
435 #else
436 result = mpt::transcode<mpt::ustring>(charset.value(), std::forward<Tsrc>(str));
437 #endif
438 } else if(mpt::has_codepage(static_cast<UINT>(codepage)))
439 {
440 result = mpt::transcode<mpt::ustring>(static_cast<UINT>(codepage), std::forward<Tsrc>(str));
441 } else
442 {
443 result = mpt::transcode<mpt::ustring>(std::forward<Tencoding>(fallback), std::forward<Tsrc>(str));
444 }
445 return result;
446 #else // !MPT_OS_WINDOWS
447 std::optional<mpt::common_encoding> charset = mpt::optional_encoding_from_codepage(codepage);
448 #if defined(MPT_LIBCXX_QUIRK_NO_OPTIONAL_VALUE)
449 return charset.has_value() ? mpt::transcode<mpt::ustring>(charset.value(), std::forward<Tsrc>(str)) : mpt::transcode<mpt::ustring>(std::forward<Tencoding>(fallback), std::forward<Tsrc>(str));
450 #else
451 return charset.has_value() ? mpt::transcode<mpt::ustring>(*charset, std::forward<Tsrc>(str)) : mpt::transcode<mpt::ustring>(std::forward<Tencoding>(fallback), std::forward<Tsrc>(str));
452 #endif
453 #endif // MPT_OS_WINDOWS
454 }
455
456
457
458
459
460 inline char ToLowerCaseAscii(char c)
461 {
462 return mpt::to_lower_ascii(c);
463 }
464
465 inline char ToUpperCaseAscii(char c)
466 {
467 return mpt::to_upper_ascii(c);
468 }
469
470 inline std::string ToLowerCaseAscii(std::string s)
471 {
472 std::transform(s.begin(), s.end(), s.begin(), static_cast<char(*)(char)>(&mpt::ToLowerCaseAscii));
473 return s;
474 }
475
476 inline std::string ToUpperCaseAscii(std::string s)
477 {
478 std::transform(s.begin(), s.end(), s.begin(), static_cast<char(*)(char)>(&mpt::ToUpperCaseAscii));
479 return s;
480 }
481
482 inline int CompareNoCaseAscii(const char *a, const char *b, std::size_t n)
483 {
484 while(n--)
485 {
486 unsigned char ac = mpt::char_value(mpt::ToLowerCaseAscii(*a));
487 unsigned char bc = mpt::char_value(mpt::ToLowerCaseAscii(*b));
488 if(ac != bc)
489 {
490 return ac < bc ? -1 : 1;
491 } else if(!ac && !bc)
492 {
493 return 0;
494 }
495 ++a;
496 ++b;
497 }
498 return 0;
499 }
500
501 inline int CompareNoCaseAscii(std::string_view a, std::string_view b)
502 {
503 for(std::size_t i = 0; i < std::min(a.length(), b.length()); ++i)
504 {
505 unsigned char ac = mpt::char_value(mpt::ToLowerCaseAscii(a[i]));
506 unsigned char bc = mpt::char_value(mpt::ToLowerCaseAscii(b[i]));
507 if(ac != bc)
508 {
509 return ac < bc ? -1 : 1;
510 } else if(!ac && !bc)
511 {
512 return 0;
513 }
514 }
515 if(a.length() == b.length())
516 {
517 return 0;
518 }
519 return a.length() < b.length() ? -1 : 1;
520 }
521
522 inline int CompareNoCaseAscii(const std::string &a, const std::string &b)
523 {
524 return CompareNoCaseAscii(std::string_view(a), std::string_view(b));
525 }
526
527
528 #if defined(MODPLUG_TRACKER)
529
530 inline mpt::ustring ToLowerCase(const mpt::ustring &s)
531 {
532 #if defined(MPT_WITH_MFC)
533 #if defined(UNICODE)
534 return mpt::transcode<mpt::ustring>(mpt::transcode<CString>(s).MakeLower());
535 #else // !UNICODE
536 return mpt::transcode<mpt::ustring>(mpt::transcode<CStringW>(s).MakeLower());
537 #endif // UNICODE
538 #else // !MPT_WITH_MFC
539 std::wstring ws = mpt::transcode<std::wstring>(s);
540 std::transform(ws.begin(), ws.end(), ws.begin(), &std::towlower);
541 return mpt::transcode<mpt::ustring>(ws);
542 #endif // MPT_WITH_MFC
543 }
544
545 inline mpt::ustring ToUpperCase(const mpt::ustring &s)
546 {
547 #if defined(MPT_WITH_MFC)
548 #if defined(UNICODE)
549 return mpt::transcode<mpt::ustring>(mpt::transcode<CString>(s).MakeUpper());
550 #else // !UNICODE
551 return mpt::transcode<mpt::ustring>(mpt::transcode<CStringW>(s).MakeUpper());
552 #endif // UNICODE
553 #else // !MPT_WITH_MFC
554 std::wstring ws = mpt::transcode<std::wstring>(s);
555 std::transform(ws.begin(), ws.end(), ws.begin(), &std::towupper);
556 return mpt::transcode<mpt::ustring>(ws);
557 #endif // MPT_WITH_MFC
558 }
559
560 #endif // MODPLUG_TRACKER
561
562
563
564
565
566 } // namespace mpt
567
568
569
570
571
572 // The AnyString types are meant to be used as function argument types only,
573 // and only during the transition phase to all-unicode strings in the whole codebase.
574 // Using an AnyString type as function argument avoids the need to overload a function for all the
575 // different string types that we currently have.
576 // Warning: These types will silently do charset conversions. Only use them when this can be tolerated.
577
578 // BasicAnyString is convertable to mpt::ustring and constructable from any string at all.
579 template <mpt::CharsetEnum charset = mpt::Charset::UTF8, bool tryUTF8 = true>
580 class BasicAnyString : public mpt::ustring
581 {
582
583 private:
584
585 static mpt::ustring From8bit(std::string str)
586 {
587 if constexpr(charset == mpt::CharsetEnum::UTF8)
588 {
589 return mpt::transcode<mpt::ustring>(mpt::common_encoding::utf8, std::move(str));
590 } else
591 {
592 // auto utf8 detection
593 if constexpr(tryUTF8)
594 {
595 if(mpt::is_utf8(str))
596 {
597 return mpt::transcode<mpt::ustring>(mpt::common_encoding::utf8, std::move(str));
598 } else
599 {
600 return mpt::transcode<mpt::ustring>(mpt::CharsetTranscoder(charset), std::move(str));
601 }
602 } else
603 {
604 return mpt::transcode<mpt::ustring>(mpt::CharsetTranscoder(charset), std::move(str));
605 }
606 }
607 }
608
609 public:
610
611 // 8 bit
612 BasicAnyString(const char *str)
613 : mpt::ustring(From8bit(str ? str : std::string()))
614 {
615 return;
616 }
617 BasicAnyString(std::string str)
618 : mpt::ustring(From8bit(std::move(str)))
619 {
620 return;
621 }
622
623 template <typename Tstring>
624 BasicAnyString(Tstring &&str)
625 : mpt::ustring(mpt::transcode<mpt::ustring>(std::forward<Tstring>(str)))
626 {
627 return;
628 }
629
630 };
631
632 // AnyUnicodeString is convertable to mpt::ustring and constructable from any known encoding
633 class AnyUnicodeString : public mpt::ustring
634 {
635
636 public:
637
638 template <typename Tstring>
639 AnyUnicodeString(Tstring &&str)
640 : mpt::ustring(mpt::transcode<mpt::ustring>(std::forward<Tstring>(str)))
641 {
642 return;
643 }
644
645 };
646
647 // AnyString
648 // Try to do the smartest auto-magic we can do.
649 #if defined(MPT_ENABLE_CHARSET_LOCALE)
650 using AnyString = BasicAnyString<mpt::CharsetEnum::Locale, true>;
651 #elif MPT_OS_WINDOWS
652 using AnyString = BasicAnyString<mpt::CharsetEnum::Windows1252, true>;
653 #else
654 using AnyString = BasicAnyString<mpt::CharsetEnum::ISO8859_1, true>;
655 #endif
656
657 // AnyStringLocale
658 // char-based strings are assumed to be in locale encoding.
659 #if defined(MPT_ENABLE_CHARSET_LOCALE)
660 using AnyStringLocale = BasicAnyString<mpt::CharsetEnum::Locale, false>;
661 #else
662 using AnyStringLocale = BasicAnyString<mpt::CharsetEnum::UTF8, false>;
663 #endif
664
665 // AnyStringUTF8orLocale
666 // char-based strings are tried in UTF8 first, if this fails, locale is used.
667 #if defined(MPT_ENABLE_CHARSET_LOCALE)
668 using AnyStringUTF8orLocale = BasicAnyString<mpt::CharsetEnum::Locale, true>;
669 #else
670 using AnyStringUTF8orLocale = BasicAnyString<mpt::CharsetEnum::UTF8, false>;
671 #endif
672
673 // AnyStringUTF8
674 // char-based strings are assumed to be in UTF8.
675 using AnyStringUTF8 = BasicAnyString<mpt::CharsetEnum::UTF8, false>;
676
677
678
679 OPENMPT_NAMESPACE_END
680
681
682
683 template <OPENMPT_NAMESPACE::mpt::CharsetEnum charset, bool tryUTF8>
684 struct mpt::make_string_type<OPENMPT_NAMESPACE::BasicAnyString<charset, tryUTF8>> {
685 using type = mpt::ustring;
686 };
687
688 template <>
689 struct mpt::make_string_type<OPENMPT_NAMESPACE::AnyUnicodeString> {
690 using type = mpt::ustring;
691 };