Debian Codegraph @

ext/charconv/charconv.c

raw
1 /*
2 * charconv.c - character code conversion library
3 *
4 * Copyright (c) 2000-2024 Shiro Kawai <[email protected]>
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 *
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 *
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * 3. Neither the name of the authors nor the names of its contributors
18 * may be used to endorse or promote products derived from this
19 * software without specific prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
27 * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
28 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
29 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
30 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
31 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 */
33
34 #include <string.h>
35 #include <errno.h>
36 #include <gauche.h>
37 #include <gauche/extend.h>
38 #include <gauche/priv/configP.h>
39 #include <gauche/priv/charP.h>
40 #include <gauche/priv/portP.h>
41 #include "charconv.h"
42
43 #define DEFAULT_CONVERSION_BUFFER_SIZE 1024
44 #define MINIMUM_CONVERSION_BUFFER_SIZE 16
45
46 typedef struct conv_guess_rec {
47 const char *codeName;
48 ScmCodeGuessingProc proc;
49 void *data;
50 struct conv_guess_rec *next;
51 } conv_guess;
52
53 /* anchor of the chain of conversion guessing procedures */
54 static struct {
55 conv_guess *procs;
56 ScmInternalMutex mutex;
57 } guess;
58
59 #define CONV_INFO(port) ((ScmConvInfo*)(PORT_BUF(port)->data))
60
61 /* external library to delegate conversion */
62 static const ScmPrimitiveParameter *ext_conv = NULL;
63 static ScmObj sym_iconv; /* symbol 'iconv */
64
65 #define USE_ICONV_P() \
66 (SCM_EQ(Scm_PrimitiveParameterRef(Scm_VM(), ext_conv), sym_iconv))
67
68 /*------------------------------------------------------------
69 * Query
70 */
71
72 /* Auxiliary function */
73 const char* Scm_GetCESName(ScmObj code, const char *argname)
74 {
75 const char *c = NULL;
76 if (SCM_UNBOUNDP(code) || SCM_FALSEP(code)) {
77 c = Scm_SupportedCharacterEncodings()[0];
78 } else if (SCM_STRINGP(code)) {
79 c = Scm_GetStringConst(SCM_STRING(code));
80 } else if (SCM_SYMBOLP(code)) {
81 c = Scm_GetStringConst(SCM_SYMBOL_NAME(code));
82 } else {
83 Scm_Error("string, symbol or #f is required for %s, but got %S",
84 argname, code);
85 }
86 return c;
87 }
88
89 int Scm_ConversionSupportedP(const char *from, const char *to,
90 u_long flags SCM_UNUSED)
91 {
92 ScmConvInfo *cinfo = jconv_open(to, from, USE_ICONV_P());
93 if (cinfo == NULL) return FALSE;
94 jconv_close(cinfo);
95 return TRUE;
96 }
97
98 void Scm_RegisterCodeGuessingProc(const char *code,
99 ScmCodeGuessingProc proc,
100 void *data)
101 {
102 conv_guess *rec = SCM_NEW(conv_guess);
103 rec->codeName = code;
104 rec->proc = proc;
105 rec->data = data;
106 (void)SCM_INTERNAL_MUTEX_LOCK(guess.mutex);
107 rec->next = guess.procs;
108 guess.procs = rec;
109 (void)SCM_INTERNAL_MUTEX_UNLOCK(guess.mutex);
110 }
111
112 static conv_guess *findGuessingProc(const char *code)
113 {
114 conv_guess *rec;
115 (void)SCM_INTERNAL_MUTEX_LOCK(guess.mutex);
116 for (rec = guess.procs; rec; rec = rec->next) {
117 if (strcasecmp(rec->codeName, code) == 0) break;
118 }
119 (void)SCM_INTERNAL_MUTEX_UNLOCK(guess.mutex);
120 return rec;
121 }
122
123 static int conv_fileno(ScmPort *port)
124 {
125 ScmConvInfo *cinfo = CONV_INFO(port);
126 return Scm_PortFileNo(cinfo->remote);
127 }
128
129 static int conv_ready(ScmPort *port)
130 {
131 ScmConvInfo *cinfo = CONV_INFO(port);
132 /* This isn't accurate, but for now ... */
133 return Scm_CharReady(cinfo->remote);
134 }
135
136 static ScmObj conv_name(int dir, ScmPort *remote, const char *from, const char *to)
137 {
138 ScmObj out = Scm_MakeOutputStringPort(TRUE);
139 Scm_Printf(SCM_PORT(out), "[conv(%s->%s) %s %S]",
140 from, to, (dir == SCM_PORT_INPUT? "from" : "to"),
141 Scm_PortName(remote));
142 return Scm_GetOutputStringUnsafe(SCM_PORT(out), 0);
143 }
144
145 /*------------------------------------------------------------
146 * Input conversion
147 *
148 * <-- Buffered port <--- filler <--(cinfo->buf)--- getz(remote)
149 */
150
151 static ScmSize conv_input_filler(ScmPort *port, ScmSize mincnt SCM_UNUSED)
152 {
153 ScmConvInfo *cinfo = CONV_INFO(port);
154 const char *inbuf = cinfo->buf;
155 char *outbuf = PORT_BUF(port)->end;
156
157 if (cinfo->remoteClosed) return 0;
158
159 /* Fill the input buffer. There may be some remaining bytes in the
160 inbuf from the last conversion (insize), so we try to fill the
161 rest. */
162 ScmSize insize = cinfo->ptr - cinfo->buf;
163 ScmSize nread = Scm_Getz(cinfo->ptr, cinfo->bufsiz - insize, cinfo->remote);
164 if (nread <= 0) {
165 /* input reached EOF. finish the output state */
166 if (insize == 0) {
167 ScmSize outroom = Scm_PortBufferRoom(port);
168 ScmSize result = jconv_reset(cinfo, outbuf, outroom);
169 if (result == OUTPUT_NOT_ENOUGH) {
170 /* The port buffer doesn't have enough space to contain the
171 finishing sequence. Its unusual, for the port buffer
172 must be almost empty at this time, and the finishing
173 sequence is usually just a few bytes.
174 We signal an error. */
175 Scm_Error("couldn't flush the ending escape sequence in the character encoding conversion port (%s -> %s). possibly an implementation error",
176 cinfo->fromCode, cinfo->toCode);
177 }
178 if (cinfo->ownerp) {
179 Scm_ClosePort(cinfo->remote);
180 cinfo->remoteClosed = TRUE;
181 }
182 #ifdef JCONV_DEBUG
183 fprintf(stderr, "<= r=%ld (reset), out(%p)%ld\n",
184 result, outbuf, outroom);
185 #endif
186 return (ScmSize)result;
187 }
188 } else {
189 insize += nread;
190 }
191
192 /* Conversion. */
193 ScmSize inroom = insize;
194 ScmSize outroom = Scm_PortBufferRoom(port);
195
196 #ifdef JCONV_DEBUG
197 fprintf(stderr, "=> in(%p)%ld out(%p)%ld\n", inbuf, insize, outbuf, outroom);
198 #endif
199 ScmSize result = jconv(cinfo, &inbuf, &inroom, &outbuf, &outroom);
200 #ifdef JCONV_DEBUG
201 fprintf(stderr, "<= r=%ld, in(%p)%ld out(%p)%ld\n",
202 result, inbuf, inroom, outbuf, outroom);
203 #endif
204 /* we've got an error. */
205 if (result == INPUT_NOT_ENOUGH || result == OUTPUT_NOT_ENOUGH) {
206 /* Conversion stopped due to an incomplete character at the
207 end of the input buffer, or the output buffer is full.
208 We shift the unconverted bytes to the beginning of input
209 buffer. */
210 memmove(cinfo->buf, cinfo->buf+insize-inroom, inroom);
211 cinfo->ptr = cinfo->buf + inroom;
212 return cinfo->bufsiz - (ScmSize)outroom;
213 } else if (result == ILLEGAL_SEQUENCE || result == NO_OUTPUT_CHAR) {
214 /* input sequence can't be represented by the destination CES. */
215 if (cinfo->replacep) {
216 if (outroom < cinfo->replaceSize) {
217 /* We don't have enough room to place replace char, so
218 we stop before the bad char and let it be handled
219 in the next callback. */
220 memmove(cinfo->buf, cinfo->buf+insize-inroom, inroom);
221 cinfo->ptr = cinfo->buf + inroom;
222 return cinfo->bufsiz - (ScmSize)outroom;
223 } else {
224 memmove(cinfo->buf, cinfo->buf+insize-inroom+1, inroom-1);
225 cinfo->ptr = cinfo->buf + inroom - 1;
226 memcpy(outbuf, cinfo->replaceSeq, cinfo->replaceSize);
227 return cinfo->bufsiz - (ScmSize)(outroom - cinfo->replaceSize);
228 }
229 } else {
230 ScmSize cnt = inroom >= 6 ? 6 : (ScmSize)inroom;
231 ScmObj s = Scm_MakeString(cinfo->buf+insize-inroom, cnt, cnt,
232 SCM_STRING_COPYING|SCM_STRING_INCOMPLETE);
233 Scm_PortError(port, SCM_PORT_ERROR_DECODING,
234 "invalid character sequence in the input stream: %S ...", s);
235 }
236 }
237
238 /* Conversion is done completely. */
239 /* NB: There are cases that some bytes are left in the input buffer
240 even iconv returns positive value. We need to shift those bytes. */
241 if (inroom > 0) {
242 memmove(cinfo->buf, cinfo->buf+insize-inroom, inroom);
243 cinfo->ptr = cinfo->buf + inroom;
244 return cinfo->bufsiz - (ScmSize)outroom;
245 } else {
246 cinfo->ptr = cinfo->buf;
247 return cinfo->bufsiz - (ScmSize)outroom;
248 }
249 }
250
251 static void conv_input_closer(ScmPort *p)
252 {
253 ScmConvInfo *cinfo = CONV_INFO(p);
254 jconv_close(cinfo);
255 if (cinfo->ownerp) {
256 Scm_ClosePort(cinfo->remote);
257 cinfo->remoteClosed = TRUE;
258 }
259 }
260
261 ScmObj Scm_MakeInputConversionPort(ScmPort *fromPort,
262 const char *fromCode,
263 const char *toCode,
264 ScmSize bufsiz,
265 u_long flags)
266 {
267 char *inbuf = NULL;
268 ScmSize preread = 0;
269
270 if (!SCM_IPORTP(fromPort))
271 Scm_Error("input port required, but got %S", fromPort);
272
273 if (bufsiz <= 0) bufsiz = DEFAULT_CONVERSION_BUFFER_SIZE;
274 if (bufsiz <= MINIMUM_CONVERSION_BUFFER_SIZE) {
275 bufsiz = MINIMUM_CONVERSION_BUFFER_SIZE;
276 }
277 conv_guess *guess = findGuessingProc(fromCode);
278 if (guess) {
279 const char *guessed;
280
281 inbuf = SCM_NEW_ATOMIC2(char *, bufsiz);
282 preread = Scm_Getz(inbuf, bufsiz, fromPort);
283 if (preread <= 0) {
284 /* Input buffer is already empty or unreadable.
285 Determining character code is not necessary.
286 We just return a dummy empty port. */
287 return Scm_MakeInputStringPort(SCM_STRING(SCM_MAKE_STR("")), FALSE);
288 }
289 guessed = guess->proc(inbuf, preread, guess->data);
290 if (guessed == NULL)
291 Scm_Error("%s: failed to guess input encoding", fromCode);
292 fromCode = guessed;
293 }
294
295 ScmConvInfo *cinfo = jconv_open(toCode, fromCode, USE_ICONV_P());
296 if (cinfo == NULL) {
297 Scm_Error("conversion from code %s to code %s is not supported",
298 fromCode, toCode);
299 }
300 cinfo->remote = fromPort;
301 cinfo->ownerp = flags & CVPORT_OWNER;
302 cinfo->bufsiz = bufsiz;
303 cinfo->remoteClosed = FALSE;
304 if (preread > 0) {
305 cinfo->buf = inbuf;
306 cinfo->ptr = inbuf + preread;
307 } else {
308 cinfo->buf = SCM_NEW_ATOMIC2(char *, cinfo->bufsiz);
309 cinfo->ptr = cinfo->buf;
310 }
311
312 if (flags & CVPORT_REPLACE) {
313 jconv_set_replacement(cinfo);
314 }
315
316 ScmPortBuffer bufrec;
317 memset(&bufrec, 0, sizeof(bufrec));
318 bufrec.size = cinfo->bufsiz;
319 bufrec.buffer = SCM_NEW_ATOMIC2(char *, cinfo->bufsiz);
320 bufrec.mode = SCM_PORT_BUFFER_FULL;
321 bufrec.filler = conv_input_filler;
322 bufrec.flusher = NULL;
323 bufrec.closer = conv_input_closer;
324 bufrec.ready = conv_ready;
325 bufrec.filenum = conv_fileno;
326 bufrec.data = (void*)cinfo;
327
328 ScmObj name = conv_name(SCM_PORT_INPUT, fromPort, fromCode, toCode);
329 return Scm_MakeBufferedPort(SCM_CLASS_PORT, name, SCM_PORT_INPUT, TRUE, &bufrec);
330 }
331
332 /* a special case of input conversion port --- coding-aware port coversion.
333 this function is called via Scm_CodingAwarePortHook from
334 src/port.c */
335
336 static ScmPort *coding_aware_conv(ScmPort *src, const char *encoding)
337 {
338 return SCM_PORT(Scm_MakeInputConversionPort(src,
339 encoding,
340 Scm_SupportedCharacterEncodings()[0],
341 0, CVPORT_OWNER));
342 }
343
344 /*------------------------------------------------------------
345 * Output conversion
346 *
347 * Buffered port ----> flusher -->(cinfo->buf)--> putz(remote)
348 */
349
350 /* NB: Glibc-2.1.2's iconv() has a bug in SJIS handling. If output
351 * is in SJIS and output buffer overflows in the middle of two-byte
352 * sequence, it leaves the first byte in the output buffer as if
353 * it were valid converted character, while the input buffer pointer
354 * stops just before the unconverted character, as supposed.
355 * There's no way to detect that unless I scan the output by myself
356 * to see the last byte of conversion is invalid or not.
357 *
358 * As a workaround, I flush the output buffer more frequently than
359 * needed, avoiding the situation that the output buffer overflow.
360 * Hoping the bugs are fixed in the future release of glibc.
361 */
362
363 #define GLIBC_2_1_ICONV_BUG
364
365 static void conv_output_closer(ScmPort *port)
366 {
367 ScmConvInfo *cinfo = CONV_INFO(port);
368
369 /* if there's remaining bytes in buf, send them to the remote port. */
370 if (cinfo->ptr > cinfo->buf) {
371 Scm_Putz(cinfo->buf, (int)(cinfo->ptr - cinfo->buf), cinfo->remote);
372 cinfo->ptr = cinfo->buf;
373 }
374 /* sends out the closing sequence, if any */
375 ScmSize r = jconv_reset(cinfo, cinfo->buf, cinfo->bufsiz);
376 #ifdef JCONV_DEBUG
377 fprintf(stderr, "<= r=%ld(reset), buf(%p)\n",
378 r, cinfo->buf);
379 #endif
380 if (r < 0) {
381 Scm_Error("something wrong in resetting output character encoding conversion (%s -> %s). possibly an implementation error.",
382 cinfo->fromCode, cinfo->toCode);
383 }
384 if (r > 0) {
385 Scm_Putz(cinfo->buf, (ScmSize)r, cinfo->remote);
386 }
387 /* flush remove port */
388 Scm_Flush(cinfo->remote);
389 if (cinfo->ownerp) {
390 Scm_ClosePort(cinfo->remote);
391 cinfo->remoteClosed = TRUE;
392 }
393 jconv_close(cinfo);
394 }
395
396 static ScmSize conv_output_flusher(ScmPort *port, ScmSize cnt, int forcep)
397 {
398 ScmConvInfo *cinfo = CONV_INFO(port);
399 ScmSize inroom = Scm_PortBufferAvail(port);
400 ScmSize len = inroom;
401 const char *inbuf = PORT_BUF(port)->buffer;
402
403 for (;;) {
404 /* Conversion. */
405 char *outbuf = cinfo->ptr;
406 ScmSize outroom = cinfo->bufsiz - (cinfo->ptr - cinfo->buf);
407 #ifdef JCONV_DEBUG
408 fprintf(stderr, "=> in(%p,%p)%ld out(%p,%p)%ld\n",
409 inbuf, inbuf+len, inroom,
410 cinfo->buf, cinfo->ptr, outroom);
411 #endif
412 ScmSize result = jconv(cinfo, &inbuf, &inroom, &outbuf, &outroom);
413 #ifdef JCONV_DEBUG
414 fprintf(stderr, "<= r=%ld, in(%p)%ld out(%p)%ld\n",
415 result, inbuf, inroom, outbuf, outroom);
416 #endif
417 if (result == INPUT_NOT_ENOUGH) {
418 #ifndef GLIBC_2_1_ICONV_BUG
419 /* Conversion stopped due to an incomplete character at the
420 end of the input buffer. We just return # of bytes
421 flushed. (Shifting unconverted characters is done by
422 buffered port routine) */
423 cinfo->ptr = outbuf;
424 #else
425 /* See the above notes. We always flush the output buffer
426 here, so that we can avoid output buffer overrun. */
427 Scm_Putz(cinfo->buf, (ScmSize)(outbuf - cinfo->buf), cinfo->remote);
428 cinfo->ptr = cinfo->buf;
429 #endif
430 return (ScmSize)(len - inroom);
431 } else if (result == OUTPUT_NOT_ENOUGH) {
432 /* Output buffer got full. Flush it, and continue
433 conversion. */
434 Scm_Putz(cinfo->buf, (ScmSize)(outbuf - cinfo->buf), cinfo->remote);
435 cinfo->ptr = cinfo->buf;
436 continue;
437 } else if (result == ILLEGAL_SEQUENCE || result == NO_OUTPUT_CHAR) {
438 /* it's likely that input contains invalid sequence.
439 TODO: we should handle this case gracefully. */
440 Scm_PortError(port, SCM_PORT_ERROR_ENCODING,
441 "cannot encode a character to the output stream");
442 return 0; /* dummy */
443 } else {
444 #ifndef GLIBC_2_1_ICONV_BUG
445 /* Conversion is done completely. Update outptr. */
446 cinfo->ptr = outbuf;
447 #else
448 /* See the above notes. We always flush the output buffer here,
449 so that we can avoid output buffer overrun. */
450 Scm_Putz(cinfo->buf, (ScmSize)(outbuf - cinfo->buf), cinfo->remote);
451 cinfo->ptr = cinfo->buf;
452 #endif
453 if (forcep && len - inroom != cnt) continue;
454 return len - inroom;
455 }
456 }
457 }
458
459 ScmObj Scm_MakeOutputConversionPort(ScmPort *toPort,
460 const char *toCode,
461 const char *fromCode,
462 ScmSize bufsiz, u_long flags)
463 {
464 if (!SCM_OPORTP(toPort))
465 Scm_Error("output port required, but got %S", toPort);
466
467 if (bufsiz <= 0) bufsiz = DEFAULT_CONVERSION_BUFFER_SIZE;
468 if (bufsiz <= MINIMUM_CONVERSION_BUFFER_SIZE) {
469 bufsiz = MINIMUM_CONVERSION_BUFFER_SIZE;
470 }
471
472 ScmConvInfo *cinfo = jconv_open(toCode, fromCode, USE_ICONV_P());
473 if (cinfo == NULL) {
474 Scm_Error("conversion from code %s to code %s is not supported",
475 fromCode, toCode);
476 }
477 cinfo->remote = toPort;
478 cinfo->ownerp = flags & CVPORT_OWNER;
479 cinfo->bufsiz = (bufsiz > 0)? bufsiz : DEFAULT_CONVERSION_BUFFER_SIZE;
480 cinfo->remoteClosed = FALSE;
481 cinfo->buf = SCM_NEW_ATOMIC2(char *, cinfo->bufsiz);
482 cinfo->ptr = cinfo->buf;
483
484 if (flags & CVPORT_REPLACE) {
485 jconv_set_replacement(cinfo);
486 }
487
488 ScmPortBuffer bufrec;
489 memset(&bufrec, 0, sizeof(bufrec));
490 bufrec.size = cinfo->bufsiz;
491 bufrec.buffer = SCM_NEW_ATOMIC2(char *, cinfo->bufsiz);
492 bufrec.mode = SCM_PORT_BUFFER_FULL;
493 bufrec.filler = NULL;
494 bufrec.flusher = conv_output_flusher;
495 bufrec.closer = conv_output_closer;
496 bufrec.ready = conv_ready;
497 bufrec.filenum = conv_fileno;
498 bufrec.data = (void*)cinfo;
499
500 ScmObj name = conv_name(SCM_PORT_OUTPUT, toPort, fromCode, toCode);
501 return Scm_MakeBufferedPort(SCM_CLASS_PORT, name, SCM_PORT_OUTPUT, TRUE, &bufrec);
502 }
503
504 /*------------------------------------------------------------
505 * Direct interface for code guessing
506 */
507 const char *Scm_GuessCES(const char *code, const char *buf, ScmSize buflen)
508 {
509 conv_guess *guess = findGuessingProc(code);
510 if (guess == NULL)
511 Scm_Error("unknown code guessing scheme: %s", code);
512 return guess->proc(buf, buflen, guess->data);
513 }
514
515 /*====================================================================
516 * Initialization
517 */
518 extern void Scm_Init_convaux(void);
519 extern void Scm_Init_convguess(void);
520
521 SCM_EXTENSION_ENTRY void Scm_Init_gauche__charconv(void)
522 {
523 SCM_INIT_EXTENSION(gauche__charconv);
524 guess.procs = NULL;
525
526 Scm_Init_convguess();
527 Scm_Init_convaux();
528
529 ScmModule *mod = SCM_MODULE(SCM_FIND_MODULE("gauche.charconv", 0));
530 sym_iconv = SCM_INTERN("iconv");
531 ext_conv = Scm_BindPrimitiveParameter(mod, "external-conversion-library",
532 sym_iconv, 0);
533
534 Scm__InstallCodingAwarePortHook(coding_aware_conv);
535 }