Debian Codegraph @

ext/charconv/jconv.c

raw
1 /*
2 * jconv.c - alternative japanese code conversion routines
3 *
4 * Copyright (c) 2000-2024 Shiro Kawai <[email protected]>
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 *
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 *
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * 3. Neither the name of the authors nor the names of its contributors
18 * may be used to endorse or promote products derived from this
19 * software without specific prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
27 * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
28 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
29 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
30 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
31 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 */
33
34 /* Some iconv() implementations don't support japanese character encodings,
35 * or have problems handling them. This code provides an alternative way
36 * to convert these encodings.
37 */
38
39 /* This file handles conversion among UTF8, Shift-JIS, EUC_JP, and ISO2022JP.
40 * Shift-JIS and EUC_JP are based on JIS X 0213:2000. ISO2022JP partially
41 * handles ISO2022-JP-3 as well.
42 */
43
44 #include <ctype.h>
45 #include "charconv.h"
46 #include "jconv_tab.h"
47 #include "latin_tab.h"
48
49 #define INCHK(n) do{if ((int)inroom < (n)) return INPUT_NOT_ENOUGH;}while(0)
50 #define OUTCHK(n) do{if ((int)outroom < (n)) return OUTPUT_NOT_ENOUGH;}while(0)
51
52 #define ERRP(n) ((n) < 0)
53
54 #define INTERMEDIATE_BUF_SIZE 6
55
56 /* Fill outptr with substitution character. Can return jconv error code. */
57 static inline int do_subst(ScmConvInfo *cinfo,
58 char *outptr,
59 ScmSize outroom,
60 ScmSize *outchars)
61 {
62 if (cinfo->replaceSize == 0) {
63 return NO_OUTPUT_CHAR;
64 }
65 OUTCHK(cinfo->replaceSize);
66 for (int i = 0; i < cinfo->replaceSize; i++) {
67 outptr[i] = cinfo->replaceSeq[i];
68 }
69 *outchars = cinfo->replaceSize;
70 return cinfo->replaceSize;
71 }
72
73 #define DO_SUBST \
74 do { \
75 int i = do_subst(cinfo, outptr, outroom, outchars); \
76 if (i < 0) return i; \
77 } while (0)
78
79 /******************************************************************
80 *
81 * Single-unit handling routines
82 *
83 * This section defines routines that converts single input unit
84 * to single output unit, optionally affecting the state.
85 * A unit is usually a character, but sometimes one input character
86 * may be mapped to more than one output characters, or a sequence of
87 * input characters is mapped to one output character.
88 *
89 * The routine returns the number of input octets consumed, and
90 * sets the number of output octets emitted in *outchars.
91 * If an errornous condition occurs, it returns one of the following
92 * error code, and not update *outchars.
93 *
94 * ILLEGAL_SEQUENCE - Input contains illegal sequence.
95 * INPUT_NOT_ENOUGH - Input sequence ends prematurely.
96 * OUTPUT_NOT_ENOUGH - Output buffer is too small.
97 * NO_OUTPUT_CHAR - Input unit can't be represented in output CES.
98 *
99 *****************************************************************/
100
101 /*=================================================================
102 * EUC-JP
103 */
104
105 /* EUC_JISX0213 -> Shift_JIS
106 *
107 * Mapping anormalities
108 *
109 * 0x80--0xa0 except 0x8e and 0x8f : C1 region.
110 * Doesn't have corresponding SJIS bytes,
111 * so mapped to substitution char.
112 * 0xff : reserved byte. mapped to substitution char.
113 *
114 * Conversion scheme
115 * 0x00-0x7f : corresponding ASCII range.
116 * 0x80--0x8d : substitution char.
117 * 0x8e : leading byte of JISX 0201 kana
118 * 0x8f : leading byte of JISX 0212 or JISX 0213 plane 2
119 * 0x90--0xa0 : substitution char.
120 * 0xa1--0xfe : first byte (e1) of JISX 0213 plane 1
121 * 0xff : substitution char
122 *
123 * For double or trible-byte character, subsequent byte has to be in
124 * the range between 0xa1 and 0xfe inclusive. If not, it is replaced
125 * for the substitution character.
126 *
127 * If the first byte is in the range of 0xa1--0xfe, two bytes (e1, e2)
128 * is mapped to SJIS (s1, s2) by:
129 *
130 * s1 = (e1 - 0xa0 + 0x101)/2 if 0xa1 <= e1 <= 0xde
131 * (e1 - 0xa0 + 0x181)/2 if 0xdf <= e1 <= 0xfe
132 * s2 = (e2 - 0xa0 + 0x3f) if odd?(e1) && 0xa1 <= e2 <= 0xdf
133 * (e2 - 0xa0 + 0x40) if odd?(e1) && 0xe0 <= e2 <= 0xfe
134 * (e2 - 0xa0 + 0x9e) if even?(e1)
135 *
136 * If the first byte is 0x8f, the second byte (e1) and the third byte
137 * (e2) is mapped to SJIS (s1, s2) by:
138 * if (0xee <= e1 <= 0xfe) s1 = (e1 - 0xa0 + 0x19b)/2
139 * otherwise, follow the table:
140 * e1 == 0xa1 or 0xa8 => s1 = 0xf0
141 * e1 == 0xa3 or 0xa4 => s1 = 0xf1
142 * e1 == 0xa5 or 0xac => s1 = 0xf2
143 * e1 == 0xae or 0xad => s1 = 0xf3
144 * e1 == 0xaf => s1 = 0xf4
145 * If e1 is other value, it is JISX0212; we use substitution char.
146 * s2 is mapped with the same rule above.
147 */
148
149 static ScmSize eucj_sjis(ScmConvInfo *cinfo SCM_UNUSED,
150 const char *inptr, ScmSize inroom,
151 char *outptr, ScmSize outroom,
152 ScmSize *outchars)
153 {
154 unsigned char e1 = inptr[0];
155 if (e1 <= 0x7f) {
156 outptr[0] = e1;
157 *outchars = 1;
158 return 1;
159 }
160 if (e1 >= 0xa1 && e1 <= 0xfe) {
161 /* double byte char (JISX 0213 plane 1) */
162 unsigned char s1, s2;
163 INCHK(2);
164 unsigned char e2 = inptr[1];
165 if (e2 < 0xa1 || e2 == 0xff) {
166 DO_SUBST;
167 return 2;
168 }
169 OUTCHK(2);
170 if (e1 <= 0xde) s1 = (e1 - 0xa0 + 0x101)/2;
171 else s1 = (e1 - 0xa0 + 0x181)/2;
172 if (e1%2 == 0) {
173 s2 = e2 - 0xa0 + 0x9e;
174 } else {
175 if (e2 <= 0xdf) s2 = e2 - 0xa0 + 0x3f;
176 else s2 = e2 - 0xa0 + 0x40;
177 }
178 outptr[0] = s1;
179 outptr[1] = s2;
180 *outchars = 2;
181 return 2;
182 }
183 if (e1 == 0x8e) {
184 /* JISX 0201 kana */
185 INCHK(2);
186 unsigned char e2 = inptr[1];
187 if (e2 < 0xa1 || e2 == 0xff) {
188 DO_SUBST;
189 } else {
190 outptr[0] = e2;
191 *outchars = 1;
192 }
193 return 2;
194 }
195 if (e1 == 0x8f) {
196 /* triple byte char */
197 unsigned char s1, s2;
198 static const unsigned char cvt[] = { 0xf0, 0, 0xf1, 0xf1, 0xf2, 0, 0, 0xf0, 0, 0, 0, 0xf2, 0xf3, 0xf3, 0xf4 };
199
200 INCHK(3);
201 OUTCHK(2);
202 e1 = inptr[1];
203 unsigned char e2 = inptr[2];
204 if (e1 < 0xa1 || e1 == 0xff || e2 < 0xa1 || e2 == 0xff) {
205 DO_SUBST;
206 return 3;
207 }
208 if (e1 >= 0xee) {
209 s1 = (e1 - 0xa0 + 0x19b)/2;
210 } else if (e1 >= 0xb0) {
211 DO_SUBST;
212 return 3;
213 } else {
214 s1 = cvt[e1-0xa1];
215 if (s1 == 0) {
216 DO_SUBST;
217 return 3;
218 }
219 }
220 if (e1%2 == 0) {
221 s2 = e2 - 0xa0 + 0x9e;
222 } else {
223 if (e2 < 0xdf) s2 = e2 - 0xa0 + 0x3f;
224 else s2 = e2 - 0xa0 + 0x40;
225 }
226 outptr[0] = s1;
227 outptr[1] = s2;
228 *outchars = 2;
229 return 3;
230 }
231 /* no corresponding char */
232 DO_SUBST;
233 return 1;
234 }
235
236 /* [EUC_JP -> UTF8 conversion]
237 *
238 * Conversion strategy:
239 * If euc0 is in ASCII range, or C1 range except 0x8e or 0x8f, map it as is.
240 * If euc0 is 0x8e, use JISX0201-KANA table.
241 * If euc0 is 0x8f, use JISX0213 plane 2 table.
242 * If euc0 is in [0xa1-0xfe], use JISX0213 plane1 table.
243 * If euc0 is 0xa0 or 0xff, return ILLEGAL_SEQUENCE.
244 *
245 * JISX0213 plane2 table is consisted by a 2-level tree. The first-level
246 * returns an index to the second-level table by (euc1 - 0xa1). Only the
247 * range of JISX0213 defined region is converted; JISX0212 region will be
248 * mapped to the substitution char.
249 */
250
251 #include "eucj2ucs.c"
252
253 /* UTF8 utility. Similar stuff is included in gauche/char_utf_8.h
254 if the native encoding is UTF8, but not otherwise.
255 So I include them here as well. */
256
257 void jconv_ucs4_to_utf8(unsigned int ucs, char *cp)
258 {
259 if (ucs < 0x80) {
260 *cp = ucs;
261 }
262 else if (ucs < 0x800) {
263 *cp++ = ((ucs>>6)&0x1f) | 0xc0;
264 *cp = (ucs&0x3f) | 0x80;
265 }
266 else if (ucs < 0x10000) {
267 *cp++ = ((ucs>>12)&0x0f) | 0xe0;
268 *cp++ = ((ucs>>6)&0x3f) | 0x80;
269 *cp = (ucs&0x3f) | 0x80;
270 }
271 else if (ucs < 0x200000) {
272 *cp++ = ((ucs>>18)&0x07) | 0xf0;
273 *cp++ = ((ucs>>12)&0x3f) | 0x80;
274 *cp++ = ((ucs>>6)&0x3f) | 0x80;
275 *cp = (ucs&0x3f) | 0x80;
276 }
277 else if (ucs < 0x4000000) {
278 *cp++ = ((ucs>>24)&0x03) | 0xf8;
279 *cp++ = ((ucs>>18)&0x3f) | 0x80;
280 *cp++ = ((ucs>>12)&0x3f) | 0x80;
281 *cp++ = ((ucs>>6)&0x3f) | 0x80;
282 *cp = (ucs&0x3f) | 0x80;
283 } else {
284 *cp++ = ((ucs>>30)&0x1) | 0xfc;
285 *cp++ = ((ucs>>24)&0x3f) | 0x80;
286 *cp++ = ((ucs>>18)&0x3f) | 0x80;
287 *cp++ = ((ucs>>12)&0x3f) | 0x80;
288 *cp++ = ((ucs>>6)&0x3f) | 0x80;
289 *cp++ = (ucs&0x3f) | 0x80;
290 }
291 }
292
293 /* Returns # of input chars, or negative error code on error */
294 int jconv_utf8_to_ucs4(const char *cp, ScmSize size, ScmChar *ucs)
295 {
296 u_char u0 = cp[0];
297 if (u0 < 0x80) {
298 *ucs = u0;
299 return 1;
300 } else if (u0 < 0xc0) {
301 return ILLEGAL_SEQUENCE;
302 } else if (u0 < 0xe0) {
303 if (size < 2) return INPUT_NOT_ENOUGH;
304 u_char u1 = cp[1];
305 ScmChar ch = ((u0 & 0x1f) << 6) | (u1 & 0x3f);
306 if (ch < 0x80) return ILLEGAL_SEQUENCE;
307 *ucs = ch;
308 return 2;
309 } else if (u0 < 0xf0) {
310 if (size < 3) return INPUT_NOT_ENOUGH;
311 u_char u1 = cp[1], u2 = cp[2];
312 ScmChar ch = ((u0 & 0x0f) << 12) | ((u1 & 0x3f) << 6) | (u2 & 0x3f);
313 if (ch < 0x800) return ILLEGAL_SEQUENCE;
314 *ucs = ch;
315 return 3;
316 } else if (u0 < 0xf8) {
317 if (size < 4) return INPUT_NOT_ENOUGH;
318 u_char u1 = cp[1], u2 = cp[2], u3 = cp[3];
319 ScmChar ch = ((u0 & 0x07) << 18) | ((u1 & 0x3f) << 12)
320 | ((u2 & 0x3f) << 6) | (u3 & 0x3f);
321 if (ch < 0x10000) return ILLEGAL_SEQUENCE;
322 *ucs = ch;
323 return 4;
324 } else if (u0 < 0xfc) {
325 if (size < 5) return INPUT_NOT_ENOUGH;
326 u_char u1 = cp[1], u2 = cp[2], u3 = cp[3], u4 = cp[4];
327 ScmChar ch = ((u0 & 0x03) << 24) | ((u1 & 0x3f) << 18)
328 | ((u2 & 0x3f) << 12) | ((u3 & 0x3f) << 6) | (u4 & 0x3f);
329 if (ch < 0x8000000) return ILLEGAL_SEQUENCE;
330 *ucs = ch;
331 return 5;
332 } else if (u0 < 0xfe) {
333 if (size < 6) return INPUT_NOT_ENOUGH;
334 u_char u1 = cp[1], u2 = cp[2], u3 = cp[3], u4 = cp[4], u5 = cp[5];
335 ScmChar ch = ((u0 & 0x01) << 30) | ((u1 & 0x3f) << 24)
336 | ((u2 & 0x3f) << 18) | ((u3 & 0x3f) << 12)
337 | ((u4 & 0x3f) << 6) | (u5 & 0x3f);
338 *ucs = ch;
339 return 6;
340 } else {
341 return ILLEGAL_SEQUENCE;
342 }
343 }
344
345 /* Given 'encoded' ucs, emit utf8. 'Encoded' ucs is the entry of the
346 conversion table. If ucs >= 0x100000, it is composed by two UCS2
347 character. Otherwise, it is one UCS4 character. */
348 static inline ScmSize eucj_utf8_emit_utf(unsigned int ucs, ScmSize inchars,
349 char *outptr, ScmSize outroom,
350 ScmSize *outchars)
351 {
352 if (ucs < 0x100000) {
353 int outreq = UCS2UTF_NBYTES(ucs);
354 OUTCHK(outreq);
355 jconv_ucs4_to_utf8(ucs, outptr);
356 *outchars = outreq;
357 } else {
358 /* we need two UCS characters */
359 unsigned int ucs0 = (ucs >> 16) & 0xffff;
360 unsigned int ucs1 = ucs & 0xfff;
361 int outreq0 = UCS2UTF_NBYTES(ucs0);
362 int outreq1 = UCS2UTF_NBYTES(ucs1);
363 OUTCHK(outreq0+outreq1);
364 jconv_ucs4_to_utf8(ucs0, outptr);
365 jconv_ucs4_to_utf8(ucs1, outptr+outreq0);
366 *outchars = outreq0+outreq1;
367 }
368 return inchars;
369 }
370
371 static ScmSize eucj_utf8(ScmConvInfo *cinfo SCM_UNUSED,
372 const char *inptr, ScmSize inroom,
373 char *outptr, ScmSize outroom, ScmSize *outchars)
374 {
375 unsigned char e0 = (unsigned char)inptr[0];
376 if (e0 < 0xa0) {
377 if (e0 == 0x8e) {
378 /* JIS X 0201 KANA */
379 INCHK(2);
380 unsigned char e1 = (unsigned char)inptr[1];
381 if (e1 < 0xa1 || e1 > 0xdf) return ILLEGAL_SEQUENCE;
382 unsigned int ucs = 0xff61 + (e1 - 0xa1);
383 return eucj_utf8_emit_utf(ucs, 2, outptr, outroom, outchars);
384 }
385 else if (e0 == 0x8f) {
386 /* JIS X 0213 plane 2 */
387 int index;
388
389 INCHK(3);
390 unsigned char e1 = (unsigned char)inptr[1];
391 unsigned char e2 = (unsigned char)inptr[2];
392 if (e1 < 0xa1 || e1 > 0xfe || e2 < 0xa1 || e2 > 0xfe) {
393 return ILLEGAL_SEQUENCE;
394 }
395 index = euc_jisx0213_2_index[e1 - 0xa1];
396 if (index < 0) {
397 DO_SUBST;
398 return 3;
399 }
400 unsigned int ucs = euc_jisx0213_2_to_ucs2[index][e2 - 0xa1];
401 if (ucs != 0) {
402 return eucj_utf8_emit_utf(ucs, 3, outptr, outroom, outchars);
403 }
404 DO_SUBST;
405 return 3;
406 }
407 else {
408 /* ASCII or C1 region */
409 outptr[0] = e0;
410 *outchars = 1;
411 return 1;
412 }
413 }
414 if (e0 > 0xa0 && e0 < 0xff) {
415 /* JIS X 0213 plane 1 */
416 INCHK(2);
417 unsigned char e1 = (unsigned char)inptr[1];
418 if (e1 < 0xa1 || e1 > 0xfe) return ILLEGAL_SEQUENCE;
419 unsigned int ucs = euc_jisx0213_1_to_ucs2[e0 - 0xa1][e1 - 0xa1];
420 if (ucs != 0) {
421 return eucj_utf8_emit_utf(ucs, 2, outptr, outroom, outchars);
422 }
423 DO_SUBST;
424 return 2;
425 }
426 /* e0 == 0xa0 */
427 DO_SUBST;
428 return 1;
429 }
430
431 /* EUC_JP -> ISO8859-1 */
432 static ScmSize eucj_lat1(ScmConvInfo *cinfo,
433 const char *inptr, ScmSize inroom,
434 char *outptr, ScmSize outroom, ScmSize *outchars)
435 {
436 char u[6];
437 ScmSize nu;
438 ScmSize r = eucj_utf8(cinfo, inptr, inroom, u, 6, &nu);
439 if (r < 0) return r;
440 ScmChar ch;
441 ScmSize r2 = jconv_utf8_to_ucs4(u, nu, &ch);
442 if (r2 < 0) return r2;
443 if (ch < 0x100) {
444 *outptr = ch;
445 *outchars = 1;
446 } else {
447 DO_SUBST;
448 }
449 return r;
450 }
451
452 /* EUC_JP -> ISO2022JP(-3)
453 *
454 * For now, I follow the strategy of iso2022jp-3-compatible behavior.
455 */
456
457 /* ensure the current state is newstate. returns # of output chars.
458 may return OUTPUT_NOT_ENOUGH. */
459 static ScmSize jis_ensure_state(ScmConvInfo *cinfo, int newstate,
460 ScmSize outbytes,
461 char *outptr, ScmSize outroom)
462 {
463 const char *escseq = NULL;
464 ScmSize esclen = 0;
465
466 if (cinfo->ostate == newstate) {
467 OUTCHK(outbytes);
468 return 0;
469 }
470 switch (newstate) {
471 case JIS_ASCII:
472 escseq = "\033(B"; esclen = 3; break;
473 case JIS_KANA:
474 escseq = "\033(I"; esclen = 3; break;
475 case JIS_0213_1:
476 escseq = "\033$B"; esclen = 3; break;
477 case JIS_0213_2:
478 escseq = "\033$(P"; esclen = 4; break;
479 case JIS_0212:
480 escseq = "\033$(D"; esclen = 4; break;
481 default:
482 /* Can't be here */
483 Scm_Panic("something wrong in jis_ensure_state: implementation error?");
484 return 0; /* dummy */
485 }
486 OUTCHK(esclen + outbytes);
487 memcpy(outptr, escseq, esclen);
488 cinfo->ostate = newstate;
489 return esclen;
490 }
491
492 static ScmSize eucj_jis(ScmConvInfo *cinfo, const char *inptr, ScmSize inroom,
493 char *outptr, ScmSize outroom, ScmSize *outchars)
494 {
495 unsigned char e0 = inptr[0];
496 if (e0 < 0x80) {
497 ScmSize outoffset = jis_ensure_state(cinfo, JIS_ASCII, 1, outptr, outroom);
498 if (ERRP(outoffset)) return outoffset;
499 outptr[outoffset] = e0;
500 *outchars = outoffset+1;
501 return 1;
502 } else if (e0 == 0x8e) {
503 INCHK(2);
504 unsigned char e1 = inptr[1];
505 if (e1 > 0xa0 && e1 < 0xff) {
506 ScmSize outoffset = jis_ensure_state(cinfo, JIS_KANA, 1, outptr, outroom);
507 if (ERRP(outoffset)) return outoffset;
508 outptr[outoffset] = e1 - 0x80;
509 *outchars = outoffset+1;
510 return 2;
511 }
512 } else if (e0 == 0x8f) {
513 INCHK(3);
514 e0 = inptr[1];
515 unsigned char e1 = inptr[2];
516 if (e0 > 0xa0 && e0 < 0xff && e1 > 0xa0 && e1 < 0xff) {
517 int newstate = JIS_0212;
518 switch (e0) {
519 case 0xa1:; case 0xa3:; case 0xa4:; case 0xa5:;
520 case 0xa8:; case 0xac:; case 0xad:; case 0xae:; case 0xaf:;
521 newstate = JIS_0213_2; break;
522 default:
523 if (e0 >= 0xee) newstate = JIS_0213_2;
524 }
525 ScmSize outoffset = jis_ensure_state(cinfo, newstate, 2, outptr, outroom);
526 outptr[outoffset] = e0 - 0x80;
527 outptr[outoffset+1] = e1 - 0x80;
528 *outchars = outoffset+1;
529 return 3;
530 }
531 } else if (e0 > 0xa0 && e0 < 0xff) {
532 INCHK(2);
533 unsigned char e1 = inptr[1];
534 if (e1 > 0xa0 && e1 < 0xff) {
535 ScmSize outoffset = jis_ensure_state(cinfo, JIS_0213_1, 2, outptr, outroom);
536 if (ERRP(outoffset)) return outoffset;
537 outptr[outoffset] = e0 - 0x80;
538 outptr[outoffset+1] = e1 - 0x80;
539 *outchars = outoffset+2;
540 return 2;
541 }
542 }
543 return ILLEGAL_SEQUENCE;
544 }
545
546
547 /* EUC-JP -> ASCII */
548 static ScmSize eucj_ascii(ScmConvInfo *cinfo,
549 const char *inptr, ScmSize inroom,
550 char *outptr, ScmSize outroom,
551 ScmSize *outchars)
552 {
553 unsigned char e1 = inptr[0];
554 if (e1 <= 0x7f) {
555 outptr[0] = e1;
556 *outchars = 1;
557 return 1;
558 }
559 if (e1 >= 0xa1 && e1 <= 0xfe) {
560 /* double byte char (JISX 0213 plane 1) */
561 INCHK(2);
562 DO_SUBST;
563 return 2;
564 }
565 if (e1 == 0x8e) {
566 INCHK(2);
567 DO_SUBST;
568 return 2;
569 }
570 if (e1 == 0x8f) {
571 INCHK(3);
572 DO_SUBST;
573 return 3;
574 }
575 DO_SUBST;
576 return 1;
577 }
578
579 /*=================================================================
580 * Shift JIS
581 */
582
583 /* Shift_JISX0213 -> EUC-JP
584 *
585 * Mapping anormalities
586 *
587 * 0x5c, 0x7e : Shift_JISX0213 mapping table maps 0x5c to U+00A5
588 * (YEN SIGN) and 0x7e to U+203E (OVERLINE). But mapping so
589 * breaks the program code written in Shift JIS. I map them
590 * to the corresponding ASCII chars.
591 * 0xfd, 0xfe, 0xff : These are reserved bytes. Apple uses these
592 * bytes for vendor extension:
593 * 0xfd - U+00A9 COPYRIGHT SIGN |EUC A9A6 |JISX0213
594 * 0xfe - U+2122 TRADE MARK SIGN |EUC 8FA2EF|JISX0212
595 * 0xff - U+2026 HORIZONTAL ELLIPSIS|EUC A1C4 |JISX0208
596 * This is a one-direction mapping.
597 * 0x80, 0xa0 : These are reserved bytes. Replaced to the
598 * one-byte substitution character of destination encoding.
599 *
600 * Conversion scheme
601 * 0x00-0x7f : corresponding ASCII range.
602 * 0x80 : substitution character
603 * 0x81 -- 0x9f : first byte (s1) of double byte range for JIS X 0213 m=1
604 * 0xa0 : substitution character
605 * 0xa1 -- 0xdf : JISX 0201 kana = s1-0x80
606 * 0xe0 -- 0xef : first byte (s1) of double byte range for JIS X 0213 m=1
607 * 0xf0 -- 0xfc : first byte (s1) of double byte range for JIS X 0213 m=2
608 * 0xfd : U+00A9, EUC A9A6, JISX0213 (1, 0x09, 0x06)
609 * 0xfe : U+2122, EUC 8FA2EF, JISX0212
610 * 0xff : U+2026, EUC A1C4, JISX0208 (1, 0x01, 0x24)
611 *
612 * For double-byte character, second byte s2 must be in the range of
613 * 0x40 <= s2 <= 0x7e or 0x80 <= s2 <= 0xfc. Otherwise, double-byte
614 * substitution character is used.
615 *
616 * two bytes (s1, s2) maps to JIS X 0213 (m, k, t) by
617 * m = 1 if s1 <= 0xef, 2 otherwise
618 * k = (s1-0x80)*2 - ((s2 < 0x9f)? 1 : 0) if s1 <= 0x9f
619 * (s1-0xc0)*2 - ((s2 < 0x9f)? 1 : 0) if 0xe0 <= s1 <= 0xef
620 * (s1-0x9e)*2 - ((s2 < 0x89)? 1 : 0) if s1 >= 0xf5
621 * otherwise, use the following table
622 * s1 k (s2>=0x80, s2<0x80)
623 * 0xf0 (0x01, 0x08)
624 * 0xf1 (0x03, 0x04)
625 * 0xf2 (0x05, 0x0c)
626 * 0xf3 (0x0e, 0x0d)
627 * 0xf4 (0x0f, 0x4e)
628 * t = s2-0x3f if s2 < 0x7f
629 * s2-0x40 if s2 < 0x9f
630 * s2-0x9e otherwise
631 *
632 * JIS X 0213 to EUC-JP is a straightfoward conversion.
633 */
634
635 static ScmSize sjis_eucj(ScmConvInfo *cinfo SCM_UNUSED,
636 const char *inptr, ScmSize inroom,
637 char *outptr, ScmSize outroom,
638 ScmSize *outchars)
639 {
640 static const unsigned char cvt[] = { 0xa1, 0xa8, 0xa3, 0xa4, 0xa5, 0xac, 0xae, 0xad, 0xaf, 0xee };
641
642 unsigned char s1 = inptr[0];
643 if (s1 <= 0x7f) {
644 *outptr = s1;
645 *outchars = 1;
646 return 1;
647 }
648 if ((s1 > 0x80 && s1 < 0xa0) || (s1 >= 0xe0 && s1 <= 0xfc)) {
649 /* Double byte char */
650 unsigned char m, e1, e2;
651 INCHK(2);
652 unsigned char s2 = inptr[1];
653 if (s2 < 0x40 || s2 > 0xfc) {
654 DO_SUBST;
655 return 2;
656 }
657
658 if (s1 <= 0x9f) {
659 OUTCHK(2);
660 m = 1;
661 e1 = (s1-0x80)*2 + 0xa0 - ((s2 < 0x9f)? 1 : 0);
662 } else if (s1 <= 0xef) {
663 OUTCHK(2);
664 m = 1;
665 e1 = (s1-0xc0)*2 + 0xa0 - ((s2 < 0x9f)? 1 : 0);
666 } else if (s1 >= 0xf5) {
667 OUTCHK(3);
668 m = 2;
669 e1 = (s1-0xf5)*2 + 0x50 + 0xa0 - ((s2 < 0x9f)? 1 : 0);
670 } else {
671 OUTCHK(3);
672 m = 2;
673 e1 = cvt[(s1-0xf0)*2+((s2 < 0x9f)? 1 : 0)];
674 }
675
676 if (s2 < 0x7f) {
677 e2 = s2 - 0x3f + 0xa0;
678 } else if (s2 < 0x9f) {
679 e2 = s2 - 0x40 + 0xa0;
680 } else {
681 e2 = s2 - 0x9e + 0xa0;
682 }
683 if (m == 1) {
684 outptr[0] = e1;
685 outptr[1] = e2;
686 *outchars = 2;
687 } else {
688 outptr[0] = 0x8f;
689 outptr[1] = e1;
690 outptr[2] = e2;
691 *outchars = 3;
692 }
693 return 2;
694 }
695 if (s1 >= 0xa1 && s1 <= 0xdf) {
696 /* JISX0201 KANA */
697 OUTCHK(2);
698 outptr[0] = 0x8e;
699 outptr[1] = s1;
700 *outchars = 2;
701 return 1;
702 }
703 if (s1 == 0xfd) {
704 /* copyright mark */
705 OUTCHK(2);
706 outptr[0] = 0xa9;
707 outptr[1] = 0xa6;
708 *outchars = 2;
709 return 1;
710 }
711 if (s1 == 0xfe) {
712 /* trademark sign. this is not in JISX0213, but in JISX0212. */
713 OUTCHK(3);
714 outptr[0] = 0x8f;
715 outptr[1] = 0xa2;
716 outptr[2] = 0xef;
717 *outchars = 3;
718 return 1;
719 }
720 if (s1 == 0xff) {
721 /* horizontal ellipsis. */
722 OUTCHK(2);
723 outptr[0] = 0xa1;
724 outptr[1] = 0xc4;
725 *outchars = 2;
726 return 1;
727 }
728
729 /* s1 == 0x80 or 0xa0 */
730 DO_SUBST;
731 return 2;
732 }
733
734 /* SJIS -> ASCII */
735
736 static ScmSize sjis_ascii(ScmConvInfo *cinfo,
737 const char *inptr, ScmSize inroom,
738 char *outptr, ScmSize outroom,
739 ScmSize *outchars)
740 {
741 unsigned char s1 = inptr[0];
742 if (s1 <= 0x7f) {
743 outptr[0] = s1;
744 *outchars = 1;
745 return 1;
746 }
747 if ((s1 > 0x80 && s1 < 0xa0) || (s1 >= 0xe0 && s1 < 0xfc)) {
748 INCHK(2);
749 DO_SUBST;
750 *outchars = cinfo->replaceSize;
751 return 2;
752 }
753 else {
754 DO_SUBST;
755 *outchars = cinfo->replaceSize;
756 return 1;
757 }
758 }
759
760 /*=================================================================
761 * UTF8
762 */
763
764 /* Conversion between UTF8 and EUC_JP is based on the table found at
765 * http://isweb11.infoseek.co.jp/computer/wakaba/table/jis-note.ja.html
766 *
767 * There are some characters in JISX0213 that can't be represented
768 * in a single Unicode character, but can be with a combining character.
769 * In such case, EUC_JP to UTF8 conversion uses combining character,
770 * but UTF8 to EUC_JP conversion translates the combining character into
771 * another character. For example, a single JISX0213 katakana 'nga'
772 * (hiragana "ka" with han-dakuon mark) will translates to Unicode
773 * U+304B+309A (HIRAGANA LETTER KA + COMBINING KATAKANA-HIRAGANA SEMI-VOICED
774 * SOUND MARK). When this sequence is converted to EUC_JP again, it
775 * becomes EUCJ 0xA4AB + 0xA1AC. This is an implementation limitation,
776 * and should be removed in later release.
777 */
778
779 /* [UTF8 -> EUC_JP conversion]
780 *
781 * EUC-JP has the corresponding characters to the wide range of
782 * UCS characters.
783 *
784 * UCS4 character # of EUC_JP characters
785 * ---------------------------------------
786 * U+0000+0xxx 564
787 * U+0000+1xxx 6
788 * U+0000+2xxx 321
789 * U+0000+3xxx 422
790 * U+0000+4xxx 347
791 * U+0000+5xxx 1951
792 * U+0000+6xxx 2047
793 * U+0000+7xxx 1868
794 * U+0000+8xxx 1769
795 * U+0000+9xxx 1583
796 * U+0000+fxxx 241
797 * U+0002+xxxx 302
798 *
799 * It is so wide and so sparse that naive lookup table implementation from
800 * UCS to EUC can be space-wasting. I use hierarchical table with some
801 * ad-hoc heuristics. Since the hierarchical table is used, I directly
802 * translates UTF8 to EUC_JP, without converting it to UCS4.
803 *
804 * Strategy outline: say input consists of bytes named u0, u1, ....
805 *
806 * u0 <= 0x7f : ASCII range
807 * u0 in [0xc2-0xd1] : UTF8 uses 2 bytes. Some mappings within this range
808 * is either very regular or very small, and they are
809 * hardcoded. Other mappings uses table lookup.
810 * u0 == 0xe1 : UTF8 uses 3 bytes. There are only 6 characters in this
811 * range, and it is hardcoded.
812 * u0 in [0xe2-0xe9, 0xef] : Large number of characters are in this range.
813 * Two-level table of 64 entries each is used to dispatch the
814 * characters.
815 * u0 == 0xf0 : UTF8 uses 4 bytes. u1 is in [0xa0-0xaa]. u2 and u3 is
816 * used for dispatch table of 64 entries each.
817 *
818 * The final table entry is unsigned short. 0x0000 means no corresponding
819 * character is defined in EUC_JP. >=0x8000 is the EUC_JP character itself.
820 * < 0x8000 means the character is in G3 plane; 0x8f should be preceded,
821 * and 0x8000 must be added to the value.
822 */
823
824 #include "ucs2eucj.c"
825
826 /* Emit given euc char */
827 static inline ScmSize utf2euc_emit_euc(unsigned short euc,
828 ScmSize inchars,
829 char *outptr,
830 ScmSize outroom,
831 ScmSize *outchars)
832 {
833 if (euc < 0x8000) {
834 OUTCHK(3);
835 outptr[0] = 0x8f;
836 outptr[1] = (euc >> 8) + 0x80;
837 outptr[2] = euc & 0xff;
838 *outchars = 3;
839 } else {
840 OUTCHK(2);
841 outptr[0] = (euc >> 8);
842 outptr[1] = euc & 0xff;
843 *outchars = 2;
844 }
845 return inchars;
846 }
847
848 /* handle 2-byte UTF8 sequence. 0xc0 <= u0 <= 0xdf */
849 static inline ScmSize utf2euc_2(ScmConvInfo *cinfo SCM_UNUSED, unsigned char u0,
850 const char *inptr, ScmSize inroom,
851 char *outptr, ScmSize outroom,
852 ScmSize *outchars)
853 {
854 const unsigned short *etab = NULL;
855
856 INCHK(2);
857 unsigned char u1 = (unsigned char)inptr[1];
858 if (u1 < 0x80 || u1 >= 0xc0) return ILLEGAL_SEQUENCE;
859
860 switch (u0) {
861 case 0xc2: etab = utf2euc_c2; break;
862 case 0xc3: etab = utf2euc_c3; break;
863 case 0xc4: etab = utf2euc_c4; break;
864 case 0xc5: etab = utf2euc_c5; break;
865 case 0xc6:
866 if (u1 == 0x93) { /* U+0193 -> euc ABA9 */
867 return utf2euc_emit_euc(0xaba9, 2, outptr, outroom, outchars);
868 } else break;
869 case 0xc7: etab = utf2euc_c7; break;
870 case 0xc9: etab = utf2euc_c9; break;
871 case 0xca: etab = utf2euc_ca; break;
872 case 0xcb: etab = utf2euc_cb; break;
873 case 0xcc: etab = utf2euc_cc; break;
874 case 0xcd:
875 if (u1 == 0xa1) { /* U+0361 -> euc ABD2 */
876 return utf2euc_emit_euc(0xabd2, 2, outptr, outroom, outchars);
877 } else break;
878 case 0xce: etab = utf2euc_ce; break;
879 case 0xcf: etab = utf2euc_cf; break;
880 case 0xd0: etab = utf2euc_d0; break;
881 case 0xd1: etab = utf2euc_d1; break;
882 default:
883 break;
884 }
885 if (etab != NULL) {
886 /* table lookup */
887 unsigned short euc = etab[u1-0x80];
888 if (euc != 0) {
889 return utf2euc_emit_euc(euc, 2, outptr, outroom, outchars);
890 }
891 }
892 DO_SUBST;
893 return 2;
894 }
895
896 /* handle 3-byte UTF8 sequence. 0xe0 <= u0 <= 0xef */
897 static inline ScmSize utf2euc_3(ScmConvInfo *cinfo SCM_UNUSED, unsigned char u0,
898 const char *inptr, ScmSize inroom,
899 char *outptr, ScmSize outroom,
900 ScmSize *outchars)
901 {
902 const unsigned char *tab1 = NULL;
903 const unsigned short (*tab2)[64] = NULL;
904
905 INCHK(3);
906 unsigned char u1 = (unsigned char)inptr[1];
907 unsigned char u2 = (unsigned char)inptr[2];
908
909 switch (u0) {
910 case 0xe1: /* special case : there's only 6 chars */
911 {
912 unsigned short euc = 0;
913 if (u1 == 0xb8) {
914 if (u2 == 0xbe) euc = 0xa8f2;
915 else if (u2 == 0xbf) euc = 0xa8f3;
916 } else if (u1 == 0xbd) {
917 if (u2 == 0xb0) euc = 0xabc6;
918 else if (u2 == 0xb1) euc = 0xabc7;
919 else if (u2 == 0xb2) euc = 0xabd0;
920 else if (u2 == 0xb3) euc = 0xabd1;
921 }
922 return utf2euc_emit_euc(euc, 3, outptr, outroom, outchars);
923 }
924 case 0xe2: tab1 = utf2euc_e2; tab2 = utf2euc_e2_xx; break;
925 case 0xe3: tab1 = utf2euc_e3; tab2 = utf2euc_e3_xx; break;
926 case 0xe4: tab1 = utf2euc_e4; tab2 = utf2euc_e4_xx; break;
927 case 0xe5: tab1 = utf2euc_e5; tab2 = utf2euc_e5_xx; break;
928 case 0xe6: tab1 = utf2euc_e6; tab2 = utf2euc_e6_xx; break;
929 case 0xe7: tab1 = utf2euc_e7; tab2 = utf2euc_e7_xx; break;
930 case 0xe8: tab1 = utf2euc_e8; tab2 = utf2euc_e8_xx; break;
931 case 0xe9: tab1 = utf2euc_e9; tab2 = utf2euc_e9_xx; break;
932 case 0xef: tab1 = utf2euc_ef; tab2 = utf2euc_ef_xx; break;
933 default:
934 break;
935 }
936 if (tab1 != NULL) {
937 unsigned char ind = tab1[u1-0x80];
938 if (ind != 0) {
939 unsigned short euc = tab2[ind-1][u2-0x80];
940 if (euc != 0) {
941 return utf2euc_emit_euc(euc, 3, outptr, outroom, outchars);
942 }
943 }
944 }
945 DO_SUBST;
946 return 3;
947 }
948
949 /* handle 4-byte UTF8 sequence. u0 == 0xf0, 0xa0 <= u1 <= 0xaa */
950 static inline ScmSize utf2euc_4(ScmConvInfo *cinfo SCM_UNUSED, unsigned char u0,
951 const char *inptr, ScmSize inroom,
952 char *outptr, ScmSize outroom,
953 ScmSize *outchars)
954 {
955 const unsigned short *tab = NULL;
956
957 INCHK(4);
958 if (u0 != 0xf0) {
959 DO_SUBST;
960 return 4;
961 }
962 unsigned char u1 = (unsigned char)inptr[1];
963 unsigned char u2 = (unsigned char)inptr[2];
964 unsigned char u3 = (unsigned char)inptr[3];
965
966 switch (u1) {
967 case 0xa0: tab = utf2euc_f0_a0; break;
968 case 0xa1: tab = utf2euc_f0_a1; break;
969 case 0xa2: tab = utf2euc_f0_a2; break;
970 case 0xa3: tab = utf2euc_f0_a3; break;
971 case 0xa4: tab = utf2euc_f0_a4; break;
972 case 0xa5: tab = utf2euc_f0_a5; break;
973 case 0xa6: tab = utf2euc_f0_a6; break;
974 case 0xa7: tab = utf2euc_f0_a7; break;
975 case 0xa8: tab = utf2euc_f0_a8; break;
976 case 0xa9: tab = utf2euc_f0_a9; break;
977 case 0xaa: tab = utf2euc_f0_aa; break;
978 default:
979 break;
980 }
981 if (tab != NULL) {
982 unsigned short u2u3 = u2*256 + u3;
983 for (int i=0; tab[i]; i+=2) {
984 if (tab[i] == u2u3) {
985 unsigned short euc = tab[i+1];
986 if (euc != 0) {
987 return utf2euc_emit_euc(euc, 4, outptr, outroom, outchars);
988 }
989 }
990 }
991 }
992 DO_SUBST;
993 return 4;
994 }
995
996 /* Body of UTF8 -> EUC_JP conversion */
997 static ScmSize utf8_eucj(ScmConvInfo *cinfo,
998 const char *inptr, ScmSize inroom,
999 char *outptr, ScmSize outroom,
1000 ScmSize *outchars)
1001 {
1002 unsigned char u0 = (unsigned char)inptr[0];
1003
1004 if (u0 <= 0x7f) {
1005 *outptr = u0;
1006 *outchars = 1;
1007 return 1;
1008 }
1009 if (u0 <= 0xbf) {
1010 /* invalid UTF8 sequence */
1011 return ILLEGAL_SEQUENCE;
1012 }
1013 if (u0 <= 0xdf) {
1014 /* 2-byte UTF8 sequence */
1015 return utf2euc_2(cinfo, u0, inptr, inroom, outptr, outroom, outchars);
1016 }
1017 if (u0 <= 0xef) {
1018 /* 3-byte UTF8 sequence */
1019 return utf2euc_3(cinfo, u0, inptr, inroom, outptr, outroom, outchars);
1020 }
1021 if (u0 <= 0xf7) {
1022 /* 4-byte UTF8 sequence */
1023 return utf2euc_4(cinfo, u0, inptr, inroom, outptr, outroom, outchars);
1024 }
1025 if (u0 <= 0xfb) {
1026 /* 5-byte UTF8 sequence */
1027 INCHK(5);
1028 DO_SUBST;
1029 return 5;
1030 }
1031 if (u0 <= 0xfd) {
1032 /* 6-byte UTF8 sequence */
1033 INCHK(6);
1034 DO_SUBST;
1035 return 6;
1036 }
1037 return ILLEGAL_SEQUENCE;
1038 }
1039
1040 /* UTF8 -> UTF16 */
1041 static ScmSize utf8_utf16(ScmConvInfo *cinfo,
1042 const char *inptr, ScmSize inroom,
1043 char *outptr, ScmSize outroom,
1044 ScmSize *outchars)
1045 {
1046 ScmSize reqsize = 0;
1047 int ostate = cinfo->ostate;
1048 int need_bom = FALSE;
1049 ScmChar ch;
1050
1051 if (ostate == UTF_DEFAULT) {
1052 reqsize += 2;
1053 need_bom = TRUE;
1054 ostate = UTF_BE;
1055 }
1056 int r = jconv_utf8_to_ucs4(inptr, inroom, &ch);
1057 if (r < 0) return r;
1058 if (ch < 0x10000) reqsize += 2;
1059 else reqsize += 4;
1060
1061 OUTCHK(reqsize);
1062 if (need_bom) {
1063 if (ostate == UTF_BE) {
1064 outptr[0] = 0xfe;
1065 outptr[1] = 0xff;
1066 } else {
1067 outptr[1] = 0xfe;
1068 outptr[0] = 0xff;
1069 }
1070 outptr += 2;
1071 }
1072 if (ch < 0x10000) {
1073 char u[2];
1074 u[0] = (ch >> 8) & 0xff;
1075 u[1] = ch & 0xff;
1076 if (ostate == UTF_BE) {
1077 outptr[0] = u[0];
1078 outptr[1] = u[1];
1079 } else {
1080 outptr[1] = u[0];
1081 outptr[0] = u[1];
1082 }
1083 } else {
1084 ch -= 0x10000;
1085 char u[2];
1086 u[0] = 0xd8 + ((ch >> 18) & 0x03);
1087 u[1] = (ch >> 10) & 0xff;
1088 if (ostate == UTF_BE) {
1089 outptr[0] = u[0];
1090 outptr[1] = u[1];
1091 } else {
1092 outptr[1] = u[0];
1093 outptr[0] = u[1];
1094 }
1095 u[0] = 0xdc + ((ch >> 8) & 0x03);
1096 u[1] = ch & 0xff;
1097 if (ostate == UTF_BE) {
1098 outptr[2] = u[0];
1099 outptr[3] = u[1];
1100 } else {
1101 outptr[3] = u[0];
1102 outptr[2] = u[1];
1103 }
1104 }
1105 cinfo->ostate = ostate;
1106 *outchars = reqsize;
1107 return r;
1108 }
1109
1110 /* UTF8 -> UTF32 */
1111 static ScmSize utf8_utf32(ScmConvInfo *cinfo,
1112 const char *inptr, ScmSize inroom,
1113 char *outptr, ScmSize outroom,
1114 ScmSize *outchars)
1115 {
1116 ScmSize reqsize = 0;
1117 int ostate = cinfo->ostate;
1118 int need_bom = FALSE;
1119 ScmChar ch;
1120
1121 if (ostate == UTF_DEFAULT) {
1122 reqsize += 4;
1123 need_bom = TRUE;
1124 ostate = UTF_BE;
1125 }
1126 int r = jconv_utf8_to_ucs4(inptr, inroom, &ch);
1127 if (r < 0) return r;
1128 reqsize += 4;
1129
1130 OUTCHK(reqsize);
1131 if (need_bom) {
1132 if (ostate == UTF_BE) {
1133 outptr[0] = 0;
1134 outptr[1] = 0;
1135 outptr[2] = 0xfe;
1136 outptr[3] = 0xff;
1137 } else {
1138 outptr[3] = 0;
1139 outptr[2] = 0;
1140 outptr[1] = 0xfe;
1141 outptr[0] = 0xff;
1142 }
1143 outptr += 4;
1144 }
1145 if (ostate == UTF_BE) {
1146 outptr[0] = (ch >> 24) & 0xff;
1147 outptr[1] = (ch >> 16) & 0xff;
1148 outptr[2] = (ch >> 8) & 0xff;
1149 outptr[3] = ch & 0xff;
1150 } else {
1151 outptr[3] = (ch >> 24) & 0xff;
1152 outptr[2] = (ch >> 16) & 0xff;
1153 outptr[1] = (ch >> 8) & 0xff;
1154 outptr[0] = ch & 0xff;
1155 }
1156 cinfo->ostate = ostate;
1157 *outchars = reqsize;
1158 return r;
1159 }
1160
1161 /* UTF8 -> Latin1 */
1162 static ScmSize utf8_lat1(ScmConvInfo *cinfo,
1163 const char *inptr, ScmSize inroom,
1164 char *outptr, ScmSize outroom,
1165 ScmSize *outchars)
1166 {
1167 ScmChar ch;
1168 int r = jconv_utf8_to_ucs4(inptr, inroom, &ch);
1169 if (r < 0) return r;
1170 if (ch < 0x100) {
1171 *outptr = ch;
1172 *outchars = 1;
1173 } else {
1174 DO_SUBST;
1175 }
1176 return r;
1177 }
1178
1179 /* UTF8 -> ASCII */
1180 static ScmSize utf8_ascii(ScmConvInfo *cinfo,
1181 const char *inptr, ScmSize inroom,
1182 char *outptr, ScmSize outroom,
1183 ScmSize *outchars)
1184 {
1185 ScmChar ch;
1186 int r = jconv_utf8_to_ucs4(inptr, inroom, &ch);
1187 if (r < 0) return r;
1188 if (ch < 0x80) {
1189 *outptr = ch;
1190 *outchars = 1;
1191 } else {
1192 DO_SUBST;
1193 }
1194 return r;
1195 }
1196
1197 /* UTF8BOM -> UTF8 */
1198 static ScmSize utf8bom_utf8(ScmConvInfo *cinfo,
1199 const char *inptr, ScmSize inroom,
1200 char *outptr, ScmSize outroom SCM_UNUSED,
1201 ScmSize *outchars)
1202 {
1203 if (cinfo->istate == UTF_OPTIONAL_BOM) {
1204 if ((u_char)inptr[0] == 0xef) {
1205 INCHK(3);
1206 if ((u_char)inptr[1] == 0xbb
1207 && (u_char)inptr[2] == 0xbf) {
1208 cinfo->istate = UTF_DEFAULT;
1209 *outchars = 0;
1210 return 3;
1211 }
1212 }
1213 }
1214 /* FALLTHROUGH */
1215 outptr[0] = inptr[0];
1216 *outchars = 1;
1217 return 1;
1218 }
1219
1220 /*=================================================================
1221 * UTF16
1222 */
1223
1224 /* For now, we first convert it to utf8, for we already have the table
1225 directly supports utf8. Theoretically though, having ucs4 to
1226 jis table would speed it up. */
1227
1228 static ScmSize utf16_utf8(ScmConvInfo *cinfo,
1229 const char *inptr, ScmSize inroom,
1230 char *outptr, ScmSize outroom,
1231 ScmSize *outchars)
1232 {
1233 INCHK(2);
1234 int istate = cinfo->istate;
1235 ScmSize inread = 0;
1236 if (istate == UTF_DEFAULT) {
1237 if ((u_char)inptr[0] == 0xfe && (u_char)inptr[1] == 0xff) {
1238 inptr += 2;
1239 inroom -= 2;
1240 inread += 2;
1241 INCHK(2);
1242 istate = UTF_BE;
1243 } else if ((u_char)inptr[0] == 0xff && (u_char)inptr[1] == 0xfe) {
1244 inptr += 2;
1245 inroom -= 2;
1246 inread += 2;
1247 INCHK(2);
1248 istate = UTF_LE;
1249 } else {
1250 /* Arbitrary choice */
1251 istate = UTF_BE;
1252 }
1253 }
1254
1255 u_char u[2];
1256 if (istate == UTF_BE) {
1257 u[0] = inptr[0];
1258 u[1] = inptr[1];
1259 } else {
1260 u[0] = inptr[1];
1261 u[1] = inptr[0];
1262 }
1263
1264 ScmChar ch;
1265
1266 if ((u[0] & 0xfc) == 0xd8) {
1267 /* surrogate */
1268 inptr += 2;
1269 inroom -= 2;
1270 INCHK(2);
1271 u_char v[2];
1272 if (istate == UTF_BE) {
1273 v[0] = inptr[0];
1274 v[1] = inptr[1];
1275 } else {
1276 v[0] = inptr[1];
1277 v[1] = inptr[0];
1278 }
1279 if ((v[0] & 0xfc) == 0xdc) {
1280 ch = (((u[0] & 0x03) << 18)
1281 | (u[1] << 10)
1282 | ((v[0] & 0x03) << 8)
1283 | v[1])
1284 + 0x10000;
1285 inread += 4;
1286 } else {
1287 /* We only have first half of a surrogate pair.
1288 We leave the second character in the input, and try to
1289 substitute the first. */
1290 DO_SUBST;
1291 cinfo->istate = istate;
1292 return inread;
1293 }
1294 } else if ((u[0] & 0xfc) == 0xdc) {
1295 /* Stray second half of a surrogate pair. */
1296 DO_SUBST;
1297 cinfo->istate = istate;
1298 return inread;
1299 } else {
1300 inread += 2;
1301 ch = (u[0] << 8) + u[1];
1302 }
1303
1304 int outreq = UCS2UTF_NBYTES(ch);
1305 OUTCHK(outreq);
1306 jconv_ucs4_to_utf8(ch, outptr);
1307 cinfo->istate = istate;
1308 *outchars = outreq;
1309 return inread;
1310 }
1311
1312 /* This handles BOM stuff. It is pretty twisted, for we need to keep the
1313 internal state consistent even when we return an error. */
1314 static ScmSize utf16_utf16(ScmConvInfo *cinfo,
1315 const char *inptr, ScmSize inroom,
1316 char *outptr, ScmSize outroom,
1317 ScmSize *outchars)
1318 {
1319 ScmSize consumed = 0;
1320 ScmSize emitted = 0;
1321
1322 if (cinfo->istate == UTF_DEFAULT || cinfo->ostate == UTF_DEFAULT) {
1323 /* We come here only at the beginning. */
1324 int istate = 0;
1325
1326 if (cinfo->istate == UTF_DEFAULT) {
1327 INCHK(2);
1328 if ((u_char)inptr[0] == 0xfe && (u_char)inptr[1] == 0xff) {
1329 consumed += 2;
1330 istate = UTF_BE;
1331 inptr += 2;
1332 inroom -= 2;
1333 } else if ((u_char)inptr[0] == 0xff && (u_char)inptr[1] == 0xfe) {
1334 consumed += 2;
1335 istate = UTF_LE;
1336 inptr += 2;
1337 inroom -= 2;
1338 } else {
1339 istate = UTF_BE;
1340 }
1341 }
1342 INCHK(2);
1343 if (cinfo->ostate == UTF_DEFAULT) {
1344 OUTCHK(4);
1345 outptr[0] = 0xfe;
1346 outptr[1] = 0xff;
1347 outptr += 2;
1348 outroom -= 2;
1349 emitted += 2;
1350 cinfo->ostate = UTF_BE;
1351 } else {
1352 OUTCHK(2);
1353 }
1354 cinfo->istate = istate;
1355 } else {
1356 INCHK(2);
1357 OUTCHK(2);
1358 }
1359
1360 char u[2];
1361 if (cinfo->istate == UTF_BE) {
1362 u[0] = inptr[0];
1363 u[1] = inptr[1];
1364 } else {
1365 u[1] = inptr[0];
1366 u[0] = inptr[1];
1367 }
1368 if (cinfo->ostate == UTF_BE) {
1369 outptr[0] = u[0];
1370 outptr[1] = u[1];
1371 } else {
1372 outptr[1] = u[0];
1373 outptr[0] = u[1];
1374 }
1375 *outchars = emitted + 2;
1376 return consumed + 2;
1377 }
1378
1379 /*=================================================================
1380 * UTF32
1381 */
1382
1383 static ScmSize utf32_utf8(ScmConvInfo *cinfo,
1384 const char *inptr, ScmSize inroom,
1385 char *outptr, ScmSize outroom,
1386 ScmSize *outchars)
1387 {
1388 INCHK(4);
1389 int istate = cinfo->istate;
1390 ScmSize inread = 0;
1391 if (istate == UTF_DEFAULT) {
1392 if ((u_char)inptr[0] == 0
1393 && (u_char)inptr[1] == 0
1394 && (u_char)inptr[2] == 0xfe
1395 && (u_char)inptr[3] == 0xff) {
1396 inptr += 4;
1397 inroom -= 4;
1398 inread += 4;
1399 INCHK(4);
1400 istate = UTF_BE;
1401 } else if ((u_char)inptr[0] == 0xff
1402 && (u_char)inptr[1] == 0xfe
1403 && (u_char)inptr[2] == 0
1404 && (u_char)inptr[3] == 0) {
1405 inptr += 4;
1406 inroom -= 4;
1407 inread += 4;
1408 INCHK(4);
1409 istate = UTF_LE;
1410 } else {
1411 /* Arbitrary choice */
1412 istate = UTF_BE;
1413 }
1414 }
1415
1416 u_char u[4];
1417 if (istate == UTF_BE) {
1418 u[0] = inptr[0];
1419 u[1] = inptr[1];
1420 u[2] = inptr[2];
1421 u[3] = inptr[3];
1422 } else {
1423 u[0] = inptr[3];
1424 u[1] = inptr[2];
1425 u[2] = inptr[1];
1426 u[3] = inptr[0];
1427 }
1428 inread += 4;
1429
1430 ScmChar ch = (u[0] << 24) | (u[1] << 16) | (u[2] << 8) | u[3];
1431
1432 int outreq = UCS2UTF_NBYTES(ch);
1433 OUTCHK(outreq);
1434 jconv_ucs4_to_utf8(ch, outptr);
1435 cinfo->istate = istate;
1436 *outchars = outreq;
1437 return inread;
1438 }
1439
1440 /* This handles BOM stuff. */
1441 static ScmSize utf32_utf32(ScmConvInfo *cinfo,
1442 const char *inptr, ScmSize inroom,
1443 char *outptr, ScmSize outroom,
1444 ScmSize *outchars)
1445 {
1446 ScmSize consumed = 0;
1447 ScmSize emitted = 0;
1448
1449 if (cinfo->istate == UTF_DEFAULT || cinfo->ostate == UTF_DEFAULT) {
1450 /* We come here only at the beginning. */
1451 int istate = 0;
1452
1453 if (cinfo->istate == UTF_DEFAULT) {
1454 INCHK(4);
1455 if ((u_char)inptr[0] == 0
1456 && (u_char)inptr[1] == 0
1457 && (u_char)inptr[2] == 0xfe
1458 && (u_char)inptr[3] == 0xff) {
1459 consumed += 4;
1460 istate = UTF_BE;
1461 inptr += 4;
1462 inroom -= 4;
1463 } else if ((u_char)inptr[0] == 0xff
1464 && (u_char)inptr[1] == 0xfe
1465 && (u_char)inptr[2] == 0
1466 && (u_char)inptr[3] == 0) {
1467 consumed += 4;
1468 istate = UTF_LE;
1469 inptr += 4;
1470 inroom -= 4;
1471 } else {
1472 istate = UTF_BE;
1473 }
1474 }
1475 INCHK(4);
1476 if (cinfo->ostate == UTF_DEFAULT) {
1477 OUTCHK(8);
1478 outptr[0] = 0;
1479 outptr[1] = 0;
1480 outptr[2] = 0xfe;
1481 outptr[3] = 0xff;
1482 outptr += 4;
1483 outroom -= 4;
1484 emitted += 4;
1485 cinfo->ostate = UTF_BE;
1486 } else {
1487 OUTCHK(4);
1488 }
1489 cinfo->istate = istate;
1490 } else {
1491 INCHK(4);
1492 OUTCHK(4);
1493 }
1494
1495 char u[4];
1496 if (cinfo->istate == UTF_BE) {
1497 u[0] = inptr[0];
1498 u[1] = inptr[1];
1499 u[2] = inptr[2];
1500 u[3] = inptr[3];
1501 } else {
1502 u[3] = inptr[0];
1503 u[2] = inptr[1];
1504 u[1] = inptr[2];
1505 u[0] = inptr[3];
1506 }
1507 if (cinfo->ostate == UTF_BE) {
1508 outptr[0] = u[0];
1509 outptr[1] = u[1];
1510 outptr[2] = u[2];
1511 outptr[3] = u[3];
1512 } else {
1513 outptr[3] = u[0];
1514 outptr[2] = u[1];
1515 outptr[1] = u[2];
1516 outptr[0] = u[3];
1517 }
1518 *outchars = emitted + 4;
1519 return consumed + 4;
1520 }
1521
1522 /*=================================================================
1523 * ISO2022-JP
1524 */
1525
1526 /* ISO2022-JP{-1(,2),3} -> EUC_JP
1527 * Strategy: accepts as many possibilities as possible.
1528 * The following escape sequence is recognized:
1529 * (See Lunde, CJKV information processing, O'Reilly, pp.155--158)
1530 *
1531 * <ESC> ( B ASCII
1532 * <ESC> ( J JIS-Roman
1533 * <ESC> ( H JIS-Roman (for compatibility)
1534 * <ESC> ( I Half-width katakana (JIS X 0201 kana)
1535 * <ESC> $ @ JIS C 6226-1978 (78JIS)
1536 * <ESC> $ B JIS X 0208-1983 (83JIS)
1537 * <ESC> $ ( D JIS X 0212-1990
1538 * <ESC> $ ( O JIS X 0213:2000 plane 1
1539 * <ESC> $ ( P JIS X 0213:2000 plane 2
1540 * <ESC> & @ <ESC> $ B JIS X 0208-1990, JIS X 0208:1997
1541 * 0x0e JIS7 half-width katakana shift-out
1542 * 0x0f JIS7 half-width katakana shift-in
1543 *
1544 * The state is reset to ASCII whenever newline character is read.
1545 *
1546 * The following escape sequences defined in ISO2022-JP-2 are recognized,
1547 * but all the characters within the sequence will be replaced by '?'.
1548 *
1549 * <ESC> $ A (GB2312-80) unsupported
1550 * <ESC> $ ( C (KS X 1001:1992) unsupported
1551 * <ESC> . A (ISO8859-1:1998) unsupported
1552 * <ESC> . F (ISO8859-7:1998) unsupported
1553 *
1554 * If other escape sequence is seen, the converter returns ILLEGAL_SEQUENCE.
1555 *
1556 * JIS8 kana is allowed.
1557 */
1558
1559 /* deal with escape sequence. escape byte itself is already consumed.
1560 returns # of input bytes consumed by the escape sequence,
1561 or an error code. cinfo->istate is updated accordingly. */
1562 static ScmSize jis_esc(ScmConvInfo *cinfo, const char *inptr, ScmSize inroom)
1563 {
1564 INCHK(2);
1565 unsigned char j1 = inptr[0];
1566 unsigned char j2 = inptr[1];
1567 switch (j1) {
1568 case '(':
1569 switch (j2) {
1570 case 'B': cinfo->istate = JIS_ASCII; break;
1571 case 'J': cinfo->istate = JIS_ROMAN; break;
1572 case 'H': cinfo->istate = JIS_ROMAN; break;
1573 case 'I': cinfo->istate = JIS_KANA; break;
1574 default: return ILLEGAL_SEQUENCE;
1575 }
1576 return 2;
1577 case '$':
1578 switch (j2) {
1579 case '@': cinfo->istate = JIS_78; break;
1580 case 'B': cinfo->istate = JIS_0213_1; break;
1581 case 'A': cinfo->istate = JIS_UNKNOWN; break;
1582 case '(':
1583 {
1584 INCHK(3);
1585 switch (inptr[2]) {
1586 case 'D': cinfo->istate = JIS_0212; break;
1587 case 'O': cinfo->istate = JIS_0213_1; break;
1588 case 'P': cinfo->istate = JIS_0213_2; break;
1589 case 'C': cinfo->istate = JIS_UNKNOWN; break;
1590 default: return ILLEGAL_SEQUENCE;
1591 }
1592 return 3;
1593 break;
1594 }
1595 default: return ILLEGAL_SEQUENCE;
1596 }
1597 return 2;
1598 case '&':
1599 {
1600 INCHK(6);
1601 if (inptr[2] == '@' && inptr[3] == 0x1b && inptr[4] == '$'
1602 && inptr[5] == 'B') {
1603 cinfo->istate = JIS_0213_1;
1604 return 5;
1605 } else {
1606 return ILLEGAL_SEQUENCE;
1607 }
1608 }
1609 case '.':
1610 switch (inptr[2]) {
1611 case 'A':/*fallthrough*/;
1612 case 'F': cinfo->istate = JIS_UNKNOWN; break;
1613 default: return ILLEGAL_SEQUENCE;
1614 }
1615 return 2;
1616 default: return ILLEGAL_SEQUENCE;
1617 }
1618 }
1619
1620 /* main routine for iso2022-jp -> euc_jp */
1621 static ScmSize jis_eucj(ScmConvInfo *cinfo, const char *inptr, ScmSize inroom,
1622 char *outptr, ScmSize outroom, ScmSize *outchars)
1623 {
1624 ScmSize inoffset = 0;
1625
1626 unsigned char j0 = inptr[inoffset];
1627 /* skip escape sequence */
1628 while (j0 == 0x1b) {
1629 inoffset++;
1630 ScmSize r = jis_esc(cinfo, inptr+inoffset, inroom-inoffset);
1631 if (ERRP(r)) return r;
1632 inoffset += r;
1633 if (inoffset >= inroom) {
1634 *outchars = 0;
1635 return inoffset;
1636 }
1637 j0 = inptr[inoffset];
1638 }
1639
1640 if (j0 == '\n' || j0 == '\r') {
1641 cinfo->istate = JIS_ASCII;
1642 outptr[0] = j0;
1643 *outchars = 1;
1644 return 1+inoffset;
1645 } else if (j0 < 0x20) {
1646 outptr[0] = j0;
1647 *outchars = 1;
1648 return 1+inoffset;
1649 } else if (j0 >= 0xa1 && j0 <= 0xdf) {
1650 /* JIS8 kana */
1651 OUTCHK(2);
1652 outptr[0] = 0x8e;
1653 outptr[1] = j0;
1654 *outchars = 2;
1655 return 1+inoffset;
1656 } else {
1657 switch (cinfo->istate) {
1658 case JIS_ROMAN:
1659 /* jis-roman and ascii differs on 0x5c and 0x7e -- for now,
1660 I ignore the difference. */
1661 /* FALLTHROUGH */
1662 case JIS_ASCII:
1663 outptr[0] = j0;
1664 *outchars = 1;
1665 return 1+inoffset;
1666 case JIS_KANA:
1667 OUTCHK(2);
1668 outptr[0] = 0x8e;
1669 outptr[1] = j0 + 0x80;
1670 *outchars = 2;
1671 return 1+inoffset;
1672 case JIS_78:
1673 /* for now, I ignore the difference between JIS78 and JIS83 */
1674 /* FALLTHROUGH */
1675 case JIS_0213_1: {
1676 INCHK(inoffset+2);
1677 OUTCHK(2);
1678 unsigned char j1 = inptr[inoffset+1];
1679 outptr[0] = j0 + 0x80;
1680 outptr[1] = j1 + 0x80;
1681 *outchars = 2;
1682 return 2+inoffset;
1683 }
1684 case JIS_0212:
1685 /* jis x 0212 and jis x 0213 plane 2 are different character sets,
1686 but uses the same conversion scheme. */
1687 /* FALLTHROUGH */
1688 case JIS_0213_2: {
1689 INCHK(inoffset+2);
1690 OUTCHK(3);
1691 unsigned char j1 = inptr[inoffset+1];
1692 outptr[0] = 0x8f;
1693 outptr[1] = j0 + 0x80;
1694 outptr[2] = j1 + 0x80;
1695 *outchars = 3;
1696 return 2+inoffset;
1697 }
1698 case JIS_UNKNOWN:
1699 DO_SUBST;
1700 return 1+inoffset;
1701 default:
1702 /* Can't be here */
1703 Scm_Panic("internal state of ISO2022-JP -> EUC_JP got messed up (%d). Implementation error?", cinfo->istate);
1704 }
1705 }
1706 return ILLEGAL_SEQUENCE;
1707 }
1708
1709 /* reset proc */
1710 static ScmSize jis_reset(ScmConvInfo *cinfo, char *outptr, ScmSize outroom)
1711 {
1712 if (outptr == NULL) {
1713 /* just reset */
1714 cinfo->ostate = JIS_ASCII;
1715 return 0;
1716 } else {
1717 if (cinfo->ostate == JIS_ASCII) return 0;
1718 if (outroom < 3) return OUTPUT_NOT_ENOUGH;
1719 outptr[0] = 0x1b;
1720 outptr[1] = '(';
1721 outptr[2] = 'B';
1722 cinfo->ostate = JIS_ASCII;
1723 return 3;
1724 }
1725 }
1726
1727 /*=================================================================
1728 * ISO8859-1
1729 */
1730
1731 static ScmSize lat1_utf8(ScmConvInfo *cinfo SCM_UNUSED,
1732 const char *inptr,
1733 ScmSize inroom SCM_UNUSED,
1734 char *outptr,
1735 ScmSize outroom,
1736 ScmSize *outchars)
1737 {
1738 unsigned char c = inptr[0];
1739 if (c <= 0x7f) {
1740 outptr[0] = c;
1741 *outchars = 1;
1742 } else {
1743 OUTCHK(2);
1744 outptr[0] = 0xc0 + (c >> 6);
1745 outptr[1] = 0x80 + (c & 0x3f);
1746 *outchars = 2;
1747 }
1748 return 1;
1749 }
1750
1751 static ScmSize lat1_ascii(ScmConvInfo *cinfo,
1752 const char *inptr,
1753 ScmSize inroom SCM_UNUSED,
1754 char *outptr,
1755 ScmSize outroom,
1756 ScmSize *outchars)
1757 {
1758 unsigned char c = inptr[0];
1759 if (c <= 0x7f) {
1760 outptr[0] = c;
1761 *outchars = 1;
1762 } else {
1763 DO_SUBST;
1764 }
1765 return 1;
1766 }
1767
1768 /*=================================================================
1769 * ASCII
1770 */
1771
1772 /* ASCII -> X */
1773
1774 static ScmSize ascii_x(ScmConvInfo *cinfo SCM_UNUSED,
1775 const char *inptr,
1776 ScmSize inroom SCM_UNUSED,
1777 char *outptr,
1778 ScmSize outroom SCM_UNUSED,
1779 ScmSize *outchars)
1780 {
1781 outptr[0] = inptr[0];
1782 *outchars = 1;
1783 return 1;
1784 }
1785
1786 /*=================================================================
1787 * Placeholder
1788 */
1789
1790 static ScmSize ident(ScmConvInfo *cinfo SCM_UNUSED,
1791 const char *inptr SCM_UNUSED,
1792 ScmSize inroom SCM_UNUSED,
1793 char *outptr SCM_UNUSED,
1794 ScmSize outroom SCM_UNUSED,
1795 ScmSize *outchars SCM_UNUSED)
1796 {
1797 return 0;
1798 }
1799
1800 /******************************************************************
1801 *
1802 * Actual conversion
1803 *
1804 */
1805
1806 /* map canonical code designator to inconv and outconv. the order of
1807 entry must match with the above designators.
1808 conv_converter[incode][outcode] returns the appropriate combiniation
1809 of routines.
1810 NB: It is tedious to maintain this table; we'll eventually generate
1811 this from some DSL.
1812 */
1813 struct conv_converter_rec {
1814 ScmConvProc *conv;
1815 ScmConvReset *reset;
1816 int istate; /* initial input state */
1817 int ostate; /* initial output state */
1818 };
1819
1820 /* map convesion name to the canonical code */
1821 struct conv_support_rec {
1822 const char *name;
1823 int code;
1824 };
1825
1826 #include "jconv_tab.c"
1827 #include "latin_tab.c"
1828
1829 static int conv_name_match(const char *s, const char *t)
1830 {
1831 const char *p, *q;
1832 for (p=s, q=t; *p && *q; p++) {
1833 if (*p == '-' || *p == '_') {
1834 continue; /* ignore '-' and '_' */
1835 } else {
1836 if (tolower(*p) != tolower(*q)) return FALSE;
1837 q++;
1838 }
1839 }
1840 if (*p || *q) return FALSE;
1841 return TRUE;
1842 }
1843
1844 static int conv_name_find(const char *name)
1845 {
1846 struct conv_support_rec *cvtab = conv_supports;
1847 for (; cvtab->name; cvtab++) {
1848 if (conv_name_match(name, cvtab->name)) {
1849 return cvtab->code;
1850 }
1851 }
1852 return -1;
1853 }
1854
1855 /* Internal conversion handler. */
1856
1857 /* when we can just pass-through input to output */
1858 static ScmSize jconv_ident(ScmConvInfo *cinfo SCM_UNUSED, const char **iptr,
1859 ScmSize *iroom, char **optr, ScmSize *oroom)
1860 {
1861 ScmSize inroom = *iroom, outroom = *oroom;
1862 #ifdef JCONV_DEBUG
1863 fprintf(stderr, "jconv_ident %s->%s\n", cinfo->fromCode, cinfo->toCode);
1864 #endif
1865 if (inroom <= outroom) {
1866 memcpy(*optr, *iptr, inroom);
1867 *optr += inroom;
1868 *iptr += inroom;
1869 *iroom = 0;
1870 *oroom -= inroom;
1871 return inroom;
1872 } else {
1873 memcpy(*optr, *iptr, outroom);
1874 *optr += outroom;
1875 *iptr += outroom;
1876 *iroom -= outroom;
1877 *oroom = 0;
1878 return OUTPUT_NOT_ENOUGH;
1879 }
1880 }
1881
1882 /* calling conversion routine for each char */
1883 static ScmSize jconv_1tier(ScmConvInfo *cinfo, const char **iptr,
1884 ScmSize *iroom, char **optr, ScmSize *oroom)
1885 {
1886 ScmConvProc *cvt = cinfo->convert;
1887 const char *inp = *iptr;
1888 char *outp = *optr;
1889 int inr = (int)*iroom, outr = (int)*oroom;
1890 ScmSize converted = 0;
1891
1892 #ifdef JCONV_DEBUG
1893 fprintf(stderr, "jconv_1tier %s->%s\n", cinfo->fromCode, cinfo->toCode);
1894 #endif
1895 SCM_ASSERT(cvt != NULL);
1896 while (inr > 0 && outr > 0) {
1897 ScmSize outchars;
1898 ScmSize inchars = cvt(cinfo, inp, inr, outp, outr, &outchars);
1899 if (ERRP(inchars)) {
1900 converted = inchars;
1901 break;
1902 } else {
1903 converted += inchars;
1904 inp += inchars;
1905 inr -= (int)inchars;
1906 outp += outchars;
1907 outr -= (int)outchars;
1908 }
1909 }
1910 *iptr = inp;
1911 *iroom = inr;
1912 *optr = outp;
1913 *oroom = outr;
1914 return converted;
1915 }
1916
1917 /* When we delegate conversion to iconv(3) */
1918 #ifdef HAVE_ICONV_H
1919 /* NB: although iconv manages states, we need to keep track of whether
1920 * we're sure in default status (JIS_ASCII) or not (we use JIS_UNKNOWN for it).
1921 * It's because jconv_iconv_reset will be called twice if there is any
1922 * reset sequence; the first call should emit the sequence, but the second
1923 * call shouldn't.
1924 */
1925 static ScmSize jconv_iconv(ScmConvInfo *cinfo, const char **iptr, ScmSize *iroom,
1926 char **optr, ScmSize *oroom)
1927 {
1928 #ifdef JCONV_DEBUG
1929 fprintf(stderr, "jconv_iconv %s->%s\n", cinfo->fromCode, cinfo->toCode);
1930 #endif
1931 size_t ir = *iroom, or = *oroom;
1932 size_t r = iconv(cinfo->handle, (char **)iptr, &ir, optr, &or);
1933 *iroom = ir;
1934 *oroom = or;
1935 cinfo->ostate = JIS_UNKNOWN;
1936 if (r == (size_t)-1) {
1937 if (errno == EINVAL) return INPUT_NOT_ENOUGH;
1938 if (errno == E2BIG) return OUTPUT_NOT_ENOUGH;
1939 return ILLEGAL_SEQUENCE;
1940 } else {
1941 return (ScmSize)r;
1942 }
1943 }
1944
1945 /* reset routine for iconv */
1946 static ScmSize jconv_iconv_reset(ScmConvInfo *cinfo, char *optr, ScmSize oroom)
1947 {
1948 ScmSize oroom_prev = oroom;
1949 if (cinfo->ostate == JIS_ASCII) return 0;
1950 size_t or = oroom;
1951 size_t r = iconv(cinfo->handle, NULL, 0, &optr, &or);
1952 if (r == (size_t)-1) {
1953 if (errno == E2BIG) return OUTPUT_NOT_ENOUGH;
1954 Scm_Panic("jconv_iconv_reset: unknown error number %d\n", errno);
1955 }
1956 cinfo->ostate = JIS_ASCII;
1957 return oroom_prev - (ScmSize)or;
1958 }
1959 #endif /*HAVE_ICONV_H*/
1960
1961 /*------------------------------------------------------------------
1962 * JCONV_OPEN
1963 * Returns ScmConvInfo, setting up some fields.
1964 * If no conversion is possible, returns NULL.
1965 */
1966 ScmConvInfo *jconv_open(const char *toCode, const char *fromCode,
1967 int useIconv)
1968 {
1969 ScmConvHandler *handler = NULL;
1970 ScmConvProc *convert = NULL;
1971 ScmConvReset *reset = NULL;
1972 int istate = 0, ostate = 0;
1973 iconv_t handle = (iconv_t)-1;
1974
1975 int incode = conv_name_find(fromCode);
1976 int outcode = conv_name_find(toCode);
1977
1978 if (incode >= 0 && outcode >= 0) {
1979 convert = conv_converter[incode][outcode].conv;
1980 reset = conv_converter[incode][outcode].reset;
1981 istate = conv_converter[incode][outcode].istate;
1982 ostate = conv_converter[incode][outcode].ostate;
1983 }
1984
1985 if (convert == NULL) {
1986 if (conv_name_match(fromCode, toCode)) {
1987 handler = jconv_ident;
1988 } else if (useIconv) {
1989 #ifdef HAVE_ICONV_H
1990 /* try iconv */
1991 handle = iconv_open(toCode, fromCode);
1992 if (handle == (iconv_t)-1) return NULL;
1993 handler = jconv_iconv;
1994 reset = jconv_iconv_reset;
1995 #else /*!HAVE_ICONV_H*/
1996 return NULL;
1997 #endif
1998 } else {
1999 return NULL;
2000 }
2001 } else if (convert == ident) {
2002 handler = jconv_ident;
2003 } else {
2004 handler = jconv_1tier;
2005 }
2006
2007 ScmConvInfo *cinfo;
2008 cinfo = SCM_NEW(ScmConvInfo);
2009 cinfo->jconv = handler;
2010 cinfo->convert = convert;
2011 cinfo->reset = reset;
2012 cinfo->handle = handle;
2013 cinfo->toCode = toCode;
2014 cinfo->istate = istate;
2015 cinfo->ostate = ostate;
2016 cinfo->fromCode = fromCode;
2017 /* The replacement settings can be modified by jconv_set_replacement */
2018 cinfo->replacep = FALSE;
2019 cinfo->replaceSize = 0;
2020 cinfo->replaceSeq = NULL;
2021 return cinfo;
2022 }
2023
2024 /*------------------------------------------------------------------
2025 * JCONV_SET_REPLACEMENT
2026 * Setting up replacement sequence according to the toCode.
2027 */
2028 void jconv_set_replacement(ScmConvInfo *cinfo)
2029 {
2030 static ScmObj ces_replacement_proc = SCM_UNDEFINED;
2031 SCM_BIND_PROC(ces_replacement_proc, "%ces-replacement",
2032 Scm_FindModule(SCM_SYMBOL(SCM_INTERN("gauche.charconv")), 0));
2033 ScmObj replacements = Scm_ApplyRec1(ces_replacement_proc,
2034 SCM_MAKE_STR(cinfo->toCode));
2035 ScmSize i = Scm_Length(replacements);
2036 if (i > 0) {
2037 cinfo->replacep = TRUE;
2038 cinfo->replaceSize = i;
2039 char *replaceSeq = SCM_NEW_ATOMIC_ARRAY(char, i);
2040 for (int j = 0; j < i; j++) {
2041 SCM_ASSERT(SCM_PAIRP(replacements));
2042 replaceSeq[j] = SCM_INT_VALUE(SCM_CAR(replacements));
2043 replacements = SCM_CDR(replacements);
2044 }
2045 cinfo->replaceSeq = replaceSeq;
2046 }
2047 }
2048
2049 /*------------------------------------------------------------------
2050 * JCONV_CLOSE
2051 */
2052 int jconv_close(ScmConvInfo *cinfo)
2053 {
2054 int r = 0;
2055 #ifdef HAVE_ICONV_H
2056 if (cinfo->handle != (iconv_t)-1) {
2057 r = iconv_close(cinfo->handle);
2058 cinfo->handle = (iconv_t)-1;
2059 }
2060 #endif /*HAVE_ICONV_H*/
2061 return r;
2062 }
2063
2064 /*------------------------------------------------------------------
2065 * JCONV - main conversion routine
2066 */
2067 ScmSize jconv(ScmConvInfo *cinfo,
2068 const char **inptr, ScmSize *inroom,
2069 char **outptr, ScmSize *outroom)
2070 {
2071 SCM_ASSERT(cinfo->jconv != NULL);
2072 return cinfo->jconv(cinfo, inptr, inroom, outptr, outroom);
2073 }
2074
2075 /*------------------------------------------------------------------
2076 * JCONV_RESET - reset
2077 */
2078 ScmSize jconv_reset(ScmConvInfo *cinfo, char *outptr, ScmSize outroom)
2079 {
2080 if (cinfo->reset) {
2081 return cinfo->reset(cinfo, outptr, outroom);
2082 } else {
2083 return 0;
2084 }
2085 }