Debian Codegraph @

modules/lsp-runtime-lib/src/main/io/CharsetDecoder.cpp

raw
1 /*
2 * Copyright (C) 2020 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2020 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-runtime-lib
6 * Created on: 8 мар. 2019 г.
7 *
8 * lsp-runtime-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-runtime-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-runtime-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #include <lsp-plug.in/common/debug.h>
23 #include <lsp-plug.in/io/CharsetDecoder.h>
24 #include <lsp-plug.in/stdlib/stdlib.h>
25 #include <errno.h>
26
27 #define DATA_BUFSIZE 0x1000
28
29 namespace lsp
30 {
31 namespace io
32 {
33
34 CharsetDecoder::CharsetDecoder()
35 {
36 bBuffer = NULL;
37 bBufHead = NULL;
38 bBufTail = NULL;
39 cBuffer = NULL;
40 cBufHead = NULL;
41 cBufTail = NULL;
42
43 #if defined(PLATFORM_WINDOWS)
44 xBuffer = NULL;
45 nCodePage = uint32_t(-1);
46 #else
47 hIconv = iconv_t(-1);
48 #endif /* PLATFORM_WINDOWS */
49 }
50
51 CharsetDecoder::~CharsetDecoder()
52 {
53 close();
54 }
55
56 status_t CharsetDecoder::init(const char *charset)
57 {
58 #if defined(PLATFORM_WINDOWS)
59 if (nCodePage != uint32_t(-1))
60 return STATUS_BAD_STATE;
61
62 ssize_t cp = codepage_from_name(charset);
63 if (cp < 0)
64 return STATUS_BAD_LOCALE;
65 nCodePage = cp;
66 #else
67 if (hIconv != iconv_t(-1))
68 return STATUS_BAD_STATE;
69
70 iconv_t handle = init_iconv_to_wchar_t(charset);
71 if (handle == iconv_t(-1))
72 return STATUS_BAD_LOCALE;
73 hIconv = handle;
74 #endif /* PLATFORM_WINDOWS */
75
76 // Allocate buffer
77 uint8_t *buf= reinterpret_cast<uint8_t *>(::malloc(
78 DATA_BUFSIZE // The byte buffer size
79 + sizeof(lsp_wchar_t) * DATA_BUFSIZE * 2 // The temporary buffer size
80 #if defined(PLATFORM_WINDOWS)
81 + sizeof(lsp_utf16_t) * DATA_BUFSIZE * 2
82 #endif /* PLATFORM_WINDOWS */
83 ));
84 if (buf == NULL)
85 {
86 close();
87 return STATUS_NO_MEM;
88 }
89
90 bBuffer = buf;
91 bBufHead = bBuffer;
92 bBufTail = bBuffer;
93 buf += DATA_BUFSIZE;
94 cBuffer = reinterpret_cast<lsp_wchar_t *>(buf);
95 cBufHead = cBuffer;
96 cBufTail = cBuffer;
97
98 #if defined(PLATFORM_WINDOWS)
99 buf += sizeof(lsp_wchar_t) * DATA_BUFSIZE * 2;
100 xBuffer = reinterpret_cast<lsp_utf16_t *>(buf);
101 #endif /* PLATFORM_WINDOWS */
102
103 return STATUS_OK;
104 }
105
106 void CharsetDecoder::close()
107 {
108 if (bBuffer != NULL)
109 {
110 free(bBuffer);
111
112 bBuffer = NULL;
113 bBufHead = NULL;
114 bBufTail = NULL;
115 cBuffer = NULL;
116 cBufHead = NULL;
117 cBufTail = NULL;
118 }
119
120 #ifdef PLATFORM_WINDOWS
121 xBuffer = NULL;
122 nCodePage = uint32_t(-1);
123 #else
124 if (hIconv != iconv_t(-1))
125 {
126 ::iconv_close(hIconv);
127 hIconv = iconv_t(-1);
128 }
129 #endif /* PLATFORM_WINDOWS */
130 }
131 #if 0
132 ssize_t CharsetDecoder::decode(lsp_wchar_t **outbuf, size_t *outleft, void **inbuf, size_t *inleft)
133 {
134 size_t nconv;
135
136 #if defined(PLATFORM_WINDOWS)
137 CHAR *xinbuf = reinterpret_cast<CHAR *>(*inbuf);
138 lsp_wchar_t *xoutbuf= *outbuf;
139 size_t xinleft = *inleft;
140 size_t xoutleft = *outleft;
141 nconv = 0;
142
143 while (xoutleft > 0)
144 {
145 // Is there a data in wchar_t buffer?
146 size_t nbuf = cBufTail - cBufHead;
147 if (nbuf > 0)
148 {
149 size_t nsrc = nbuf;
150 nbuf = utf16_to_utf32(xoutbuf, &xoutleft, cBufHead, &nsrc, false);
151 if (nbuf <= 0)
152 break;
153
154 nconv += nbuf;
155 xoutbuf += nbuf;
156 cBufHead += nbuf;
157 continue;
158 }
159
160 // Fill the rest space with converted UTF-16 data
161 // Each input character can take up to 2 UTF-16 characters, prevent from buffer overflows
162 // We can manipulate only with input buffer size because otherwise we will
163 // get a conversion error from dump MultiByteToWideChar routine
164 // character buffer is guaranteed to be empty
165 size_t xinamount = (xinleft > DATA_BUFSIZE) ? DATA_BUFSIZE : xinleft;
166 size_t bufcw = DATA_BUFSIZE*2;
167 size_t xincw = xinamount;
168
169 ssize_t nchars = multibyte_to_widechar(nCodePage, xinbuf, &xincw, cBuffer, &bufcw);
170 if (nchars <= 0)
171 {
172 if (nconv <= 0)
173 return nchars;
174 break;
175 }
176
177 // Update pointers and data
178 xinamount -= xincw;
179 cBufHead = cBuffer;
180 cBufTail = &cBuffer[nchars];
181 xinbuf += xinamount;
182 xinleft -= xinamount;
183 }
184
185 // Update pointers and values
186 *outbuf = xoutbuf;
187 *outleft = xoutleft;
188 *inbuf = reinterpret_cast<void *>(xinbuf);
189 *inleft = xinleft;
190 #else
191 char *xinbuf = reinterpret_cast<char *>(*inbuf);
192 char *xoutbuf = reinterpret_cast<char *>(*outbuf);
193 size_t xinleft = *inleft;
194 size_t xoutleft = *outleft * sizeof(lsp_wchar_t);
195
196 // Perform conversion
197 nconv = ::iconv(hIconv, &xinbuf, &xinleft, &xoutbuf, &xoutleft);
198 if (nconv == size_t(-1))
199 {
200 int code = errno;
201 switch (code)
202 {
203 case E2BIG:
204 case EINVAL:
205 nconv = *outleft - (xoutleft/sizeof(lsp_wchar_t));
206 break;
207 default:
208 return -STATUS_BAD_FORMAT;
209 }
210 }
211
212 // Update pointers and values
213 *outbuf = reinterpret_cast<lsp_wchar_t *>(xoutbuf);
214 *outleft = xoutleft / sizeof(lsp_wchar_t);
215 *inbuf = reinterpret_cast<void *>(xinbuf);
216 *inleft = xinleft;
217 #endif /* PLATFORM_WINDOWS */
218
219 return nconv;
220 }
221 #endif
222
223 size_t CharsetDecoder::prepare_buffer()
224 {
225 size_t bufsz = bBufTail - bBufHead;
226 if (bufsz > (DATA_BUFSIZE >> 1))
227 return 0;
228 else if (bBufHead != bBuffer)
229 {
230 if (bufsz > 0)
231 ::memmove(bBuffer, bBufHead, bufsz);
232
233 bBufHead = bBuffer;
234 bBufTail = &bBuffer[bufsz];
235 }
236 return DATA_BUFSIZE - bufsz;
237 }
238
239 ssize_t CharsetDecoder::decode_buffer()
240 {
241 // Prepare buffer
242 size_t bufsz = cBufTail - cBufHead;
243 if (bufsz > DATA_BUFSIZE)
244 return bufsz;
245 else if (cBufHead != cBuffer)
246 {
247 if (bufsz > 0)
248 ::memmove(cBuffer, cBufHead, bufsz * sizeof(lsp_wchar_t));
249
250 cBufHead = cBuffer;
251 cBufTail = &cBuffer[bufsz];
252 }
253
254 // Is there any data in byte buffer?
255 size_t xinleft = bBufTail - bBufHead;
256 if (xinleft <= 0)
257 return bufsz;
258
259 // Now we can surely decode DATA_BUFSIZE characters
260 #ifdef PLATFORM_WINDOWS
261 // Round 1: Perform native -> UTF-16 decoding
262 char *xinbuf = reinterpret_cast<char *>(bBufHead);
263 size_t nsrc = xinleft;
264 size_t ndst = DATA_BUFSIZE*2;
265 ssize_t nbytes = multibyte_to_widechar(nCodePage, xinbuf, &nsrc, xBuffer, &ndst);
266 if (nbytes <= 0)
267 return nbytes;
268 uint8_t *bhead = &bBufHead[xinleft - nsrc];
269
270 // Round 2: Perform UTF-16 -> UTF-32 decoding
271 nsrc = DATA_BUFSIZE*2 - ndst;
272 ndst = DATA_BUFSIZE;
273 ssize_t nchars = utf16_to_utf32(cBufTail, &ndst, xBuffer, &nsrc, false);
274 if (nchars <= 0)
275 return nchars;
276
277 bBufHead = bhead;
278 cBufTail += DATA_BUFSIZE - ndst;
279 #else
280 char *xinbuf = reinterpret_cast<char *>(bBufHead);
281 char *xoutbuf = reinterpret_cast<char *>(cBufTail);
282 bufsz = DATA_BUFSIZE * sizeof(lsp_wchar_t);
283 size_t old_xinleft = xinleft;
284
285 // Perform conversion
286 size_t nconv = ::iconv(hIconv, &xinbuf, &xinleft, &xoutbuf, &bufsz);
287 if (nconv == size_t(-1))
288 {
289 int code = errno;
290 switch (code)
291 {
292 case E2BIG:
293 case EINVAL:
294 break;
295 case EILSEQ:
296 if (old_xinleft > xinleft)
297 break;
298 return -STATUS_BAD_FORMAT;
299
300 default:
301 return -STATUS_BAD_FORMAT;
302 }
303 }
304
305 bBufHead = reinterpret_cast<uint8_t *>(xinbuf);
306 cBufTail = reinterpret_cast<lsp_wchar_t *>(xoutbuf);
307 #endif
308 return cBufTail - cBufHead;
309 }
310
311 lsp_swchar_t CharsetDecoder::fetch()
312 {
313 if (bBuffer == NULL)
314 return -STATUS_CLOSED;
315
316 // Is there any data in character buffer
317 if (cBufTail > cBufHead)
318 return *(cBufHead++);
319
320 ssize_t nchars = decode_buffer();
321 if (nchars > 0)
322 return *(cBufHead++);
323 return (nchars < 0) ? lsp_swchar_t(nchars) : -STATUS_EOF;
324 }
325
326 ssize_t CharsetDecoder::fetch(lsp_wchar_t *outbuf, size_t count)
327 {
328 if (bBuffer == NULL)
329 return -STATUS_CLOSED;
330 else if (outbuf == NULL)
331 return -STATUS_BAD_ARGUMENTS;
332
333 // Compute the amount of data to read
334 size_t processed = 0;
335
336 // Perform read
337 while (processed < count)
338 {
339 // Perform decoding
340 ssize_t nchars = decode_buffer();
341 if (nchars <= 0)
342 {
343 if (processed > 0)
344 break;
345 return nchars;
346 }
347
348 // Copy data to output buffer
349 ssize_t to_copy = count - processed;
350 if (nchars > to_copy)
351 nchars = to_copy;
352 ::memcpy(outbuf, cBufHead, nchars * sizeof(lsp_wchar_t));
353
354 // Update state
355 cBufHead += nchars;
356 processed += nchars;
357 outbuf += nchars;
358 }
359
360 return processed;
361 }
362
363 ssize_t CharsetDecoder::fetch(LSPString *out, size_t count)
364 {
365 if (bBuffer == NULL)
366 return -STATUS_CLOSED;
367 else if (out == NULL)
368 return -STATUS_BAD_ARGUMENTS;
369
370 // Compute the amount of data to read
371 size_t processed = 0;
372 if (!count)
373 count = DATA_BUFSIZE*2;
374
375 // Perform read
376 while (processed < count)
377 {
378 // Perform decoding
379 ssize_t nchars = decode_buffer();
380 if (nchars <= 0)
381 {
382 if (processed > 0)
383 break;
384 return nchars;
385 }
386
387 // Copy data to output buffer
388 ssize_t to_copy = count - processed;
389 if (nchars > to_copy)
390 nchars = to_copy;
391 if (!out->append(cBufHead, nchars))
392 return -STATUS_NO_MEM;
393
394 // Update state
395 cBufHead += nchars;
396 processed += nchars;
397 }
398
399 return processed;
400 }
401
402 ssize_t CharsetDecoder::fetch(IOutSequence *out, size_t count)
403 {
404 if (bBuffer == NULL)
405 return -STATUS_CLOSED;
406 else if (out == NULL)
407 return -STATUS_BAD_ARGUMENTS;
408
409 // Compute the amount of data to read
410 size_t processed = 0;
411 if (!count)
412 count = DATA_BUFSIZE*2;
413
414 // Perform read
415 while (processed < count)
416 {
417 // Perform decoding
418 ssize_t nchars = decode_buffer();
419 if (nchars <= 0)
420 {
421 if (processed > 0)
422 break;
423 return nchars;
424 }
425
426 // Write data to output sequence
427 ssize_t to_copy = count - processed;
428 if (nchars > to_copy)
429 nchars = to_copy;
430 nchars = out->write(cBufHead, nchars);
431 if (nchars < 0)
432 {
433 if (processed > 0)
434 break;
435 return nchars;
436 }
437
438 // Update state
439 cBufHead += nchars;
440 processed += nchars;
441 }
442
443 return processed;
444 }
445
446 ssize_t CharsetDecoder::fill(const void *buf, size_t count)
447 {
448 if (bBuffer == NULL)
449 return -STATUS_CLOSED;
450 else if (buf == NULL)
451 return -STATUS_BAD_ARGUMENTS;
452
453 size_t bufsz = prepare_buffer();
454 if (bufsz <= 0)
455 return bufsz;
456
457 if (count > bufsz)
458 count = bufsz;
459 ::memcpy(&bBufTail, buf, count);
460 bBufTail += count;
461 return count;
462 }
463
464 ssize_t CharsetDecoder::fill(File *fd, size_t count)
465 {
466 if (bBuffer == NULL)
467 return -STATUS_CLOSED;
468 else if (fd == NULL)
469 return -STATUS_BAD_ARGUMENTS;
470
471 // Is there a space in the buffer for reading?
472 size_t bufsz = prepare_buffer();
473 if (bufsz <= 0)
474 return bufsz;
475
476 // Compute the amount of data to read
477 size_t read = 0;
478 if ((!count) || (count > bufsz))
479 count = bufsz;
480
481 // Perform read
482 do
483 {
484 ssize_t nread = fd->read(bBufTail, count - read);
485 if (nread <= 0)
486 {
487 if (read > 0) // Ignore error if there is data on the input
488 break;
489 return nread;
490 }
491
492 bBufTail += nread;
493 read += nread;
494 }
495 while (read < count);
496
497 return read;
498 }
499
500 ssize_t CharsetDecoder::fill(IInStream *is, size_t count)
501 {
502 if (bBuffer == NULL)
503 return -STATUS_CLOSED;
504 else if (is == NULL)
505 return -STATUS_BAD_ARGUMENTS;
506
507 // Is there a space in the buffer for reading?
508 size_t bufsz = prepare_buffer();
509 if (bufsz <= 0)
510 return bufsz;
511
512 // Compute the amount of data to read
513 size_t read = 0;
514 if ((!count) || (count > bufsz))
515 count = bufsz;
516
517 // Perform read
518 do
519 {
520 ssize_t nread = is->read(bBufTail, count - read);
521 if (nread <= 0)
522 {
523 if (read > 0) // Ignore error if there is data on the input
524 break;
525 return nread;
526 }
527
528 bBufTail += nread;
529 read += nread;
530 }
531 while (read < count);
532
533 return read;
534 }
535 } /* namespace io */
536 } /* namespace lsp */