/* * jconv.c - alternative japanese code conversion routines * * Copyright (c) 2000-2024 Shiro Kawai * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * 3. Neither the name of the authors nor the names of its contributors * may be used to endorse or promote products derived from this * software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /* Some iconv() implementations don't support japanese character encodings, * or have problems handling them. This code provides an alternative way * to convert these encodings. */ /* This file handles conversion among UTF8, Shift-JIS, EUC_JP, and ISO2022JP. * Shift-JIS and EUC_JP are based on JIS X 0213:2000. ISO2022JP partially * handles ISO2022-JP-3 as well. */ #include #include "charconv.h" #include "jconv_tab.h" #include "latin_tab.h" #define INCHK(n) do{if ((int)inroom < (n)) return INPUT_NOT_ENOUGH;}while(0) #define OUTCHK(n) do{if ((int)outroom < (n)) return OUTPUT_NOT_ENOUGH;}while(0) #define ERRP(n) ((n) < 0) #define INTERMEDIATE_BUF_SIZE 6 /* Fill outptr with substitution character. Can return jconv error code. */ static inline int do_subst(ScmConvInfo *cinfo, char *outptr, ScmSize outroom, ScmSize *outchars) { if (cinfo->replaceSize == 0) { return NO_OUTPUT_CHAR; } OUTCHK(cinfo->replaceSize); for (int i = 0; i < cinfo->replaceSize; i++) { outptr[i] = cinfo->replaceSeq[i]; } *outchars = cinfo->replaceSize; return cinfo->replaceSize; } #define DO_SUBST \ do { \ int i = do_subst(cinfo, outptr, outroom, outchars); \ if (i < 0) return i; \ } while (0) /****************************************************************** * * Single-unit handling routines * * This section defines routines that converts single input unit * to single output unit, optionally affecting the state. * A unit is usually a character, but sometimes one input character * may be mapped to more than one output characters, or a sequence of * input characters is mapped to one output character. * * The routine returns the number of input octets consumed, and * sets the number of output octets emitted in *outchars. * If an errornous condition occurs, it returns one of the following * error code, and not update *outchars. * * ILLEGAL_SEQUENCE - Input contains illegal sequence. * INPUT_NOT_ENOUGH - Input sequence ends prematurely. * OUTPUT_NOT_ENOUGH - Output buffer is too small. * NO_OUTPUT_CHAR - Input unit can't be represented in output CES. * *****************************************************************/ /*================================================================= * EUC-JP */ /* EUC_JISX0213 -> Shift_JIS * * Mapping anormalities * * 0x80--0xa0 except 0x8e and 0x8f : C1 region. * Doesn't have corresponding SJIS bytes, * so mapped to substitution char. * 0xff : reserved byte. mapped to substitution char. * * Conversion scheme * 0x00-0x7f : corresponding ASCII range. * 0x80--0x8d : substitution char. * 0x8e : leading byte of JISX 0201 kana * 0x8f : leading byte of JISX 0212 or JISX 0213 plane 2 * 0x90--0xa0 : substitution char. * 0xa1--0xfe : first byte (e1) of JISX 0213 plane 1 * 0xff : substitution char * * For double or trible-byte character, subsequent byte has to be in * the range between 0xa1 and 0xfe inclusive. If not, it is replaced * for the substitution character. * * If the first byte is in the range of 0xa1--0xfe, two bytes (e1, e2) * is mapped to SJIS (s1, s2) by: * * s1 = (e1 - 0xa0 + 0x101)/2 if 0xa1 <= e1 <= 0xde * (e1 - 0xa0 + 0x181)/2 if 0xdf <= e1 <= 0xfe * s2 = (e2 - 0xa0 + 0x3f) if odd?(e1) && 0xa1 <= e2 <= 0xdf * (e2 - 0xa0 + 0x40) if odd?(e1) && 0xe0 <= e2 <= 0xfe * (e2 - 0xa0 + 0x9e) if even?(e1) * * If the first byte is 0x8f, the second byte (e1) and the third byte * (e2) is mapped to SJIS (s1, s2) by: * if (0xee <= e1 <= 0xfe) s1 = (e1 - 0xa0 + 0x19b)/2 * otherwise, follow the table: * e1 == 0xa1 or 0xa8 => s1 = 0xf0 * e1 == 0xa3 or 0xa4 => s1 = 0xf1 * e1 == 0xa5 or 0xac => s1 = 0xf2 * e1 == 0xae or 0xad => s1 = 0xf3 * e1 == 0xaf => s1 = 0xf4 * If e1 is other value, it is JISX0212; we use substitution char. * s2 is mapped with the same rule above. */ static ScmSize eucj_sjis(ScmConvInfo *cinfo SCM_UNUSED, const char *inptr, ScmSize inroom, char *outptr, ScmSize outroom, ScmSize *outchars) { unsigned char e1 = inptr[0]; if (e1 <= 0x7f) { outptr[0] = e1; *outchars = 1; return 1; } if (e1 >= 0xa1 && e1 <= 0xfe) { /* double byte char (JISX 0213 plane 1) */ unsigned char s1, s2; INCHK(2); unsigned char e2 = inptr[1]; if (e2 < 0xa1 || e2 == 0xff) { DO_SUBST; return 2; } OUTCHK(2); if (e1 <= 0xde) s1 = (e1 - 0xa0 + 0x101)/2; else s1 = (e1 - 0xa0 + 0x181)/2; if (e1%2 == 0) { s2 = e2 - 0xa0 + 0x9e; } else { if (e2 <= 0xdf) s2 = e2 - 0xa0 + 0x3f; else s2 = e2 - 0xa0 + 0x40; } outptr[0] = s1; outptr[1] = s2; *outchars = 2; return 2; } if (e1 == 0x8e) { /* JISX 0201 kana */ INCHK(2); unsigned char e2 = inptr[1]; if (e2 < 0xa1 || e2 == 0xff) { DO_SUBST; } else { outptr[0] = e2; *outchars = 1; } return 2; } if (e1 == 0x8f) { /* triple byte char */ unsigned char s1, s2; static const unsigned char cvt[] = { 0xf0, 0, 0xf1, 0xf1, 0xf2, 0, 0, 0xf0, 0, 0, 0, 0xf2, 0xf3, 0xf3, 0xf4 }; INCHK(3); OUTCHK(2); e1 = inptr[1]; unsigned char e2 = inptr[2]; if (e1 < 0xa1 || e1 == 0xff || e2 < 0xa1 || e2 == 0xff) { DO_SUBST; return 3; } if (e1 >= 0xee) { s1 = (e1 - 0xa0 + 0x19b)/2; } else if (e1 >= 0xb0) { DO_SUBST; return 3; } else { s1 = cvt[e1-0xa1]; if (s1 == 0) { DO_SUBST; return 3; } } if (e1%2 == 0) { s2 = e2 - 0xa0 + 0x9e; } else { if (e2 < 0xdf) s2 = e2 - 0xa0 + 0x3f; else s2 = e2 - 0xa0 + 0x40; } outptr[0] = s1; outptr[1] = s2; *outchars = 2; return 3; } /* no corresponding char */ DO_SUBST; return 1; } /* [EUC_JP -> UTF8 conversion] * * Conversion strategy: * If euc0 is in ASCII range, or C1 range except 0x8e or 0x8f, map it as is. * If euc0 is 0x8e, use JISX0201-KANA table. * If euc0 is 0x8f, use JISX0213 plane 2 table. * If euc0 is in [0xa1-0xfe], use JISX0213 plane1 table. * If euc0 is 0xa0 or 0xff, return ILLEGAL_SEQUENCE. * * JISX0213 plane2 table is consisted by a 2-level tree. The first-level * returns an index to the second-level table by (euc1 - 0xa1). Only the * range of JISX0213 defined region is converted; JISX0212 region will be * mapped to the substitution char. */ #include "eucj2ucs.c" /* UTF8 utility. Similar stuff is included in gauche/char_utf_8.h if the native encoding is UTF8, but not otherwise. So I include them here as well. */ void jconv_ucs4_to_utf8(unsigned int ucs, char *cp) { if (ucs < 0x80) { *cp = ucs; } else if (ucs < 0x800) { *cp++ = ((ucs>>6)&0x1f) | 0xc0; *cp = (ucs&0x3f) | 0x80; } else if (ucs < 0x10000) { *cp++ = ((ucs>>12)&0x0f) | 0xe0; *cp++ = ((ucs>>6)&0x3f) | 0x80; *cp = (ucs&0x3f) | 0x80; } else if (ucs < 0x200000) { *cp++ = ((ucs>>18)&0x07) | 0xf0; *cp++ = ((ucs>>12)&0x3f) | 0x80; *cp++ = ((ucs>>6)&0x3f) | 0x80; *cp = (ucs&0x3f) | 0x80; } else if (ucs < 0x4000000) { *cp++ = ((ucs>>24)&0x03) | 0xf8; *cp++ = ((ucs>>18)&0x3f) | 0x80; *cp++ = ((ucs>>12)&0x3f) | 0x80; *cp++ = ((ucs>>6)&0x3f) | 0x80; *cp = (ucs&0x3f) | 0x80; } else { *cp++ = ((ucs>>30)&0x1) | 0xfc; *cp++ = ((ucs>>24)&0x3f) | 0x80; *cp++ = ((ucs>>18)&0x3f) | 0x80; *cp++ = ((ucs>>12)&0x3f) | 0x80; *cp++ = ((ucs>>6)&0x3f) | 0x80; *cp++ = (ucs&0x3f) | 0x80; } } /* Returns # of input chars, or negative error code on error */ int jconv_utf8_to_ucs4(const char *cp, ScmSize size, ScmChar *ucs) { u_char u0 = cp[0]; if (u0 < 0x80) { *ucs = u0; return 1; } else if (u0 < 0xc0) { return ILLEGAL_SEQUENCE; } else if (u0 < 0xe0) { if (size < 2) return INPUT_NOT_ENOUGH; u_char u1 = cp[1]; ScmChar ch = ((u0 & 0x1f) << 6) | (u1 & 0x3f); if (ch < 0x80) return ILLEGAL_SEQUENCE; *ucs = ch; return 2; } else if (u0 < 0xf0) { if (size < 3) return INPUT_NOT_ENOUGH; u_char u1 = cp[1], u2 = cp[2]; ScmChar ch = ((u0 & 0x0f) << 12) | ((u1 & 0x3f) << 6) | (u2 & 0x3f); if (ch < 0x800) return ILLEGAL_SEQUENCE; *ucs = ch; return 3; } else if (u0 < 0xf8) { if (size < 4) return INPUT_NOT_ENOUGH; u_char u1 = cp[1], u2 = cp[2], u3 = cp[3]; ScmChar ch = ((u0 & 0x07) << 18) | ((u1 & 0x3f) << 12) | ((u2 & 0x3f) << 6) | (u3 & 0x3f); if (ch < 0x10000) return ILLEGAL_SEQUENCE; *ucs = ch; return 4; } else if (u0 < 0xfc) { if (size < 5) return INPUT_NOT_ENOUGH; u_char u1 = cp[1], u2 = cp[2], u3 = cp[3], u4 = cp[4]; ScmChar ch = ((u0 & 0x03) << 24) | ((u1 & 0x3f) << 18) | ((u2 & 0x3f) << 12) | ((u3 & 0x3f) << 6) | (u4 & 0x3f); if (ch < 0x8000000) return ILLEGAL_SEQUENCE; *ucs = ch; return 5; } else if (u0 < 0xfe) { if (size < 6) return INPUT_NOT_ENOUGH; u_char u1 = cp[1], u2 = cp[2], u3 = cp[3], u4 = cp[4], u5 = cp[5]; ScmChar ch = ((u0 & 0x01) << 30) | ((u1 & 0x3f) << 24) | ((u2 & 0x3f) << 18) | ((u3 & 0x3f) << 12) | ((u4 & 0x3f) << 6) | (u5 & 0x3f); *ucs = ch; return 6; } else { return ILLEGAL_SEQUENCE; } } /* Given 'encoded' ucs, emit utf8. 'Encoded' ucs is the entry of the conversion table. If ucs >= 0x100000, it is composed by two UCS2 character. Otherwise, it is one UCS4 character. */ static inline ScmSize eucj_utf8_emit_utf(unsigned int ucs, ScmSize inchars, char *outptr, ScmSize outroom, ScmSize *outchars) { if (ucs < 0x100000) { int outreq = UCS2UTF_NBYTES(ucs); OUTCHK(outreq); jconv_ucs4_to_utf8(ucs, outptr); *outchars = outreq; } else { /* we need two UCS characters */ unsigned int ucs0 = (ucs >> 16) & 0xffff; unsigned int ucs1 = ucs & 0xfff; int outreq0 = UCS2UTF_NBYTES(ucs0); int outreq1 = UCS2UTF_NBYTES(ucs1); OUTCHK(outreq0+outreq1); jconv_ucs4_to_utf8(ucs0, outptr); jconv_ucs4_to_utf8(ucs1, outptr+outreq0); *outchars = outreq0+outreq1; } return inchars; } static ScmSize eucj_utf8(ScmConvInfo *cinfo SCM_UNUSED, const char *inptr, ScmSize inroom, char *outptr, ScmSize outroom, ScmSize *outchars) { unsigned char e0 = (unsigned char)inptr[0]; if (e0 < 0xa0) { if (e0 == 0x8e) { /* JIS X 0201 KANA */ INCHK(2); unsigned char e1 = (unsigned char)inptr[1]; if (e1 < 0xa1 || e1 > 0xdf) return ILLEGAL_SEQUENCE; unsigned int ucs = 0xff61 + (e1 - 0xa1); return eucj_utf8_emit_utf(ucs, 2, outptr, outroom, outchars); } else if (e0 == 0x8f) { /* JIS X 0213 plane 2 */ int index; INCHK(3); unsigned char e1 = (unsigned char)inptr[1]; unsigned char e2 = (unsigned char)inptr[2]; if (e1 < 0xa1 || e1 > 0xfe || e2 < 0xa1 || e2 > 0xfe) { return ILLEGAL_SEQUENCE; } index = euc_jisx0213_2_index[e1 - 0xa1]; if (index < 0) { DO_SUBST; return 3; } unsigned int ucs = euc_jisx0213_2_to_ucs2[index][e2 - 0xa1]; if (ucs != 0) { return eucj_utf8_emit_utf(ucs, 3, outptr, outroom, outchars); } DO_SUBST; return 3; } else { /* ASCII or C1 region */ outptr[0] = e0; *outchars = 1; return 1; } } if (e0 > 0xa0 && e0 < 0xff) { /* JIS X 0213 plane 1 */ INCHK(2); unsigned char e1 = (unsigned char)inptr[1]; if (e1 < 0xa1 || e1 > 0xfe) return ILLEGAL_SEQUENCE; unsigned int ucs = euc_jisx0213_1_to_ucs2[e0 - 0xa1][e1 - 0xa1]; if (ucs != 0) { return eucj_utf8_emit_utf(ucs, 2, outptr, outroom, outchars); } DO_SUBST; return 2; } /* e0 == 0xa0 */ DO_SUBST; return 1; } /* EUC_JP -> ISO8859-1 */ static ScmSize eucj_lat1(ScmConvInfo *cinfo, const char *inptr, ScmSize inroom, char *outptr, ScmSize outroom, ScmSize *outchars) { char u[6]; ScmSize nu; ScmSize r = eucj_utf8(cinfo, inptr, inroom, u, 6, &nu); if (r < 0) return r; ScmChar ch; ScmSize r2 = jconv_utf8_to_ucs4(u, nu, &ch); if (r2 < 0) return r2; if (ch < 0x100) { *outptr = ch; *outchars = 1; } else { DO_SUBST; } return r; } /* EUC_JP -> ISO2022JP(-3) * * For now, I follow the strategy of iso2022jp-3-compatible behavior. */ /* ensure the current state is newstate. returns # of output chars. may return OUTPUT_NOT_ENOUGH. */ static ScmSize jis_ensure_state(ScmConvInfo *cinfo, int newstate, ScmSize outbytes, char *outptr, ScmSize outroom) { const char *escseq = NULL; ScmSize esclen = 0; if (cinfo->ostate == newstate) { OUTCHK(outbytes); return 0; } switch (newstate) { case JIS_ASCII: escseq = "\033(B"; esclen = 3; break; case JIS_KANA: escseq = "\033(I"; esclen = 3; break; case JIS_0213_1: escseq = "\033$B"; esclen = 3; break; case JIS_0213_2: escseq = "\033$(P"; esclen = 4; break; case JIS_0212: escseq = "\033$(D"; esclen = 4; break; default: /* Can't be here */ Scm_Panic("something wrong in jis_ensure_state: implementation error?"); return 0; /* dummy */ } OUTCHK(esclen + outbytes); memcpy(outptr, escseq, esclen); cinfo->ostate = newstate; return esclen; } static ScmSize eucj_jis(ScmConvInfo *cinfo, const char *inptr, ScmSize inroom, char *outptr, ScmSize outroom, ScmSize *outchars) { unsigned char e0 = inptr[0]; if (e0 < 0x80) { ScmSize outoffset = jis_ensure_state(cinfo, JIS_ASCII, 1, outptr, outroom); if (ERRP(outoffset)) return outoffset; outptr[outoffset] = e0; *outchars = outoffset+1; return 1; } else if (e0 == 0x8e) { INCHK(2); unsigned char e1 = inptr[1]; if (e1 > 0xa0 && e1 < 0xff) { ScmSize outoffset = jis_ensure_state(cinfo, JIS_KANA, 1, outptr, outroom); if (ERRP(outoffset)) return outoffset; outptr[outoffset] = e1 - 0x80; *outchars = outoffset+1; return 2; } } else if (e0 == 0x8f) { INCHK(3); e0 = inptr[1]; unsigned char e1 = inptr[2]; if (e0 > 0xa0 && e0 < 0xff && e1 > 0xa0 && e1 < 0xff) { int newstate = JIS_0212; switch (e0) { case 0xa1:; case 0xa3:; case 0xa4:; case 0xa5:; case 0xa8:; case 0xac:; case 0xad:; case 0xae:; case 0xaf:; newstate = JIS_0213_2; break; default: if (e0 >= 0xee) newstate = JIS_0213_2; } ScmSize outoffset = jis_ensure_state(cinfo, newstate, 2, outptr, outroom); outptr[outoffset] = e0 - 0x80; outptr[outoffset+1] = e1 - 0x80; *outchars = outoffset+1; return 3; } } else if (e0 > 0xa0 && e0 < 0xff) { INCHK(2); unsigned char e1 = inptr[1]; if (e1 > 0xa0 && e1 < 0xff) { ScmSize outoffset = jis_ensure_state(cinfo, JIS_0213_1, 2, outptr, outroom); if (ERRP(outoffset)) return outoffset; outptr[outoffset] = e0 - 0x80; outptr[outoffset+1] = e1 - 0x80; *outchars = outoffset+2; return 2; } } return ILLEGAL_SEQUENCE; } /* EUC-JP -> ASCII */ static ScmSize eucj_ascii(ScmConvInfo *cinfo, const char *inptr, ScmSize inroom, char *outptr, ScmSize outroom, ScmSize *outchars) { unsigned char e1 = inptr[0]; if (e1 <= 0x7f) { outptr[0] = e1; *outchars = 1; return 1; } if (e1 >= 0xa1 && e1 <= 0xfe) { /* double byte char (JISX 0213 plane 1) */ INCHK(2); DO_SUBST; return 2; } if (e1 == 0x8e) { INCHK(2); DO_SUBST; return 2; } if (e1 == 0x8f) { INCHK(3); DO_SUBST; return 3; } DO_SUBST; return 1; } /*================================================================= * Shift JIS */ /* Shift_JISX0213 -> EUC-JP * * Mapping anormalities * * 0x5c, 0x7e : Shift_JISX0213 mapping table maps 0x5c to U+00A5 * (YEN SIGN) and 0x7e to U+203E (OVERLINE). But mapping so * breaks the program code written in Shift JIS. I map them * to the corresponding ASCII chars. * 0xfd, 0xfe, 0xff : These are reserved bytes. Apple uses these * bytes for vendor extension: * 0xfd - U+00A9 COPYRIGHT SIGN |EUC A9A6 |JISX0213 * 0xfe - U+2122 TRADE MARK SIGN |EUC 8FA2EF|JISX0212 * 0xff - U+2026 HORIZONTAL ELLIPSIS|EUC A1C4 |JISX0208 * This is a one-direction mapping. * 0x80, 0xa0 : These are reserved bytes. Replaced to the * one-byte substitution character of destination encoding. * * Conversion scheme * 0x00-0x7f : corresponding ASCII range. * 0x80 : substitution character * 0x81 -- 0x9f : first byte (s1) of double byte range for JIS X 0213 m=1 * 0xa0 : substitution character * 0xa1 -- 0xdf : JISX 0201 kana = s1-0x80 * 0xe0 -- 0xef : first byte (s1) of double byte range for JIS X 0213 m=1 * 0xf0 -- 0xfc : first byte (s1) of double byte range for JIS X 0213 m=2 * 0xfd : U+00A9, EUC A9A6, JISX0213 (1, 0x09, 0x06) * 0xfe : U+2122, EUC 8FA2EF, JISX0212 * 0xff : U+2026, EUC A1C4, JISX0208 (1, 0x01, 0x24) * * For double-byte character, second byte s2 must be in the range of * 0x40 <= s2 <= 0x7e or 0x80 <= s2 <= 0xfc. Otherwise, double-byte * substitution character is used. * * two bytes (s1, s2) maps to JIS X 0213 (m, k, t) by * m = 1 if s1 <= 0xef, 2 otherwise * k = (s1-0x80)*2 - ((s2 < 0x9f)? 1 : 0) if s1 <= 0x9f * (s1-0xc0)*2 - ((s2 < 0x9f)? 1 : 0) if 0xe0 <= s1 <= 0xef * (s1-0x9e)*2 - ((s2 < 0x89)? 1 : 0) if s1 >= 0xf5 * otherwise, use the following table * s1 k (s2>=0x80, s2<0x80) * 0xf0 (0x01, 0x08) * 0xf1 (0x03, 0x04) * 0xf2 (0x05, 0x0c) * 0xf3 (0x0e, 0x0d) * 0xf4 (0x0f, 0x4e) * t = s2-0x3f if s2 < 0x7f * s2-0x40 if s2 < 0x9f * s2-0x9e otherwise * * JIS X 0213 to EUC-JP is a straightfoward conversion. */ static ScmSize sjis_eucj(ScmConvInfo *cinfo SCM_UNUSED, const char *inptr, ScmSize inroom, char *outptr, ScmSize outroom, ScmSize *outchars) { static const unsigned char cvt[] = { 0xa1, 0xa8, 0xa3, 0xa4, 0xa5, 0xac, 0xae, 0xad, 0xaf, 0xee }; unsigned char s1 = inptr[0]; if (s1 <= 0x7f) { *outptr = s1; *outchars = 1; return 1; } if ((s1 > 0x80 && s1 < 0xa0) || (s1 >= 0xe0 && s1 <= 0xfc)) { /* Double byte char */ unsigned char m, e1, e2; INCHK(2); unsigned char s2 = inptr[1]; if (s2 < 0x40 || s2 > 0xfc) { DO_SUBST; return 2; } if (s1 <= 0x9f) { OUTCHK(2); m = 1; e1 = (s1-0x80)*2 + 0xa0 - ((s2 < 0x9f)? 1 : 0); } else if (s1 <= 0xef) { OUTCHK(2); m = 1; e1 = (s1-0xc0)*2 + 0xa0 - ((s2 < 0x9f)? 1 : 0); } else if (s1 >= 0xf5) { OUTCHK(3); m = 2; e1 = (s1-0xf5)*2 + 0x50 + 0xa0 - ((s2 < 0x9f)? 1 : 0); } else { OUTCHK(3); m = 2; e1 = cvt[(s1-0xf0)*2+((s2 < 0x9f)? 1 : 0)]; } if (s2 < 0x7f) { e2 = s2 - 0x3f + 0xa0; } else if (s2 < 0x9f) { e2 = s2 - 0x40 + 0xa0; } else { e2 = s2 - 0x9e + 0xa0; } if (m == 1) { outptr[0] = e1; outptr[1] = e2; *outchars = 2; } else { outptr[0] = 0x8f; outptr[1] = e1; outptr[2] = e2; *outchars = 3; } return 2; } if (s1 >= 0xa1 && s1 <= 0xdf) { /* JISX0201 KANA */ OUTCHK(2); outptr[0] = 0x8e; outptr[1] = s1; *outchars = 2; return 1; } if (s1 == 0xfd) { /* copyright mark */ OUTCHK(2); outptr[0] = 0xa9; outptr[1] = 0xa6; *outchars = 2; return 1; } if (s1 == 0xfe) { /* trademark sign. this is not in JISX0213, but in JISX0212. */ OUTCHK(3); outptr[0] = 0x8f; outptr[1] = 0xa2; outptr[2] = 0xef; *outchars = 3; return 1; } if (s1 == 0xff) { /* horizontal ellipsis. */ OUTCHK(2); outptr[0] = 0xa1; outptr[1] = 0xc4; *outchars = 2; return 1; } /* s1 == 0x80 or 0xa0 */ DO_SUBST; return 2; } /* SJIS -> ASCII */ static ScmSize sjis_ascii(ScmConvInfo *cinfo, const char *inptr, ScmSize inroom, char *outptr, ScmSize outroom, ScmSize *outchars) { unsigned char s1 = inptr[0]; if (s1 <= 0x7f) { outptr[0] = s1; *outchars = 1; return 1; } if ((s1 > 0x80 && s1 < 0xa0) || (s1 >= 0xe0 && s1 < 0xfc)) { INCHK(2); DO_SUBST; *outchars = cinfo->replaceSize; return 2; } else { DO_SUBST; *outchars = cinfo->replaceSize; return 1; } } /*================================================================= * UTF8 */ /* Conversion between UTF8 and EUC_JP is based on the table found at * http://isweb11.infoseek.co.jp/computer/wakaba/table/jis-note.ja.html * * There are some characters in JISX0213 that can't be represented * in a single Unicode character, but can be with a combining character. * In such case, EUC_JP to UTF8 conversion uses combining character, * but UTF8 to EUC_JP conversion translates the combining character into * another character. For example, a single JISX0213 katakana 'nga' * (hiragana "ka" with han-dakuon mark) will translates to Unicode * U+304B+309A (HIRAGANA LETTER KA + COMBINING KATAKANA-HIRAGANA SEMI-VOICED * SOUND MARK). When this sequence is converted to EUC_JP again, it * becomes EUCJ 0xA4AB + 0xA1AC. This is an implementation limitation, * and should be removed in later release. */ /* [UTF8 -> EUC_JP conversion] * * EUC-JP has the corresponding characters to the wide range of * UCS characters. * * UCS4 character # of EUC_JP characters * --------------------------------------- * U+0000+0xxx 564 * U+0000+1xxx 6 * U+0000+2xxx 321 * U+0000+3xxx 422 * U+0000+4xxx 347 * U+0000+5xxx 1951 * U+0000+6xxx 2047 * U+0000+7xxx 1868 * U+0000+8xxx 1769 * U+0000+9xxx 1583 * U+0000+fxxx 241 * U+0002+xxxx 302 * * It is so wide and so sparse that naive lookup table implementation from * UCS to EUC can be space-wasting. I use hierarchical table with some * ad-hoc heuristics. Since the hierarchical table is used, I directly * translates UTF8 to EUC_JP, without converting it to UCS4. * * Strategy outline: say input consists of bytes named u0, u1, .... * * u0 <= 0x7f : ASCII range * u0 in [0xc2-0xd1] : UTF8 uses 2 bytes. Some mappings within this range * is either very regular or very small, and they are * hardcoded. Other mappings uses table lookup. * u0 == 0xe1 : UTF8 uses 3 bytes. There are only 6 characters in this * range, and it is hardcoded. * u0 in [0xe2-0xe9, 0xef] : Large number of characters are in this range. * Two-level table of 64 entries each is used to dispatch the * characters. * u0 == 0xf0 : UTF8 uses 4 bytes. u1 is in [0xa0-0xaa]. u2 and u3 is * used for dispatch table of 64 entries each. * * The final table entry is unsigned short. 0x0000 means no corresponding * character is defined in EUC_JP. >=0x8000 is the EUC_JP character itself. * < 0x8000 means the character is in G3 plane; 0x8f should be preceded, * and 0x8000 must be added to the value. */ #include "ucs2eucj.c" /* Emit given euc char */ static inline ScmSize utf2euc_emit_euc(unsigned short euc, ScmSize inchars, char *outptr, ScmSize outroom, ScmSize *outchars) { if (euc < 0x8000) { OUTCHK(3); outptr[0] = 0x8f; outptr[1] = (euc >> 8) + 0x80; outptr[2] = euc & 0xff; *outchars = 3; } else { OUTCHK(2); outptr[0] = (euc >> 8); outptr[1] = euc & 0xff; *outchars = 2; } return inchars; } /* handle 2-byte UTF8 sequence. 0xc0 <= u0 <= 0xdf */ static inline ScmSize utf2euc_2(ScmConvInfo *cinfo SCM_UNUSED, unsigned char u0, const char *inptr, ScmSize inroom, char *outptr, ScmSize outroom, ScmSize *outchars) { const unsigned short *etab = NULL; INCHK(2); unsigned char u1 = (unsigned char)inptr[1]; if (u1 < 0x80 || u1 >= 0xc0) return ILLEGAL_SEQUENCE; switch (u0) { case 0xc2: etab = utf2euc_c2; break; case 0xc3: etab = utf2euc_c3; break; case 0xc4: etab = utf2euc_c4; break; case 0xc5: etab = utf2euc_c5; break; case 0xc6: if (u1 == 0x93) { /* U+0193 -> euc ABA9 */ return utf2euc_emit_euc(0xaba9, 2, outptr, outroom, outchars); } else break; case 0xc7: etab = utf2euc_c7; break; case 0xc9: etab = utf2euc_c9; break; case 0xca: etab = utf2euc_ca; break; case 0xcb: etab = utf2euc_cb; break; case 0xcc: etab = utf2euc_cc; break; case 0xcd: if (u1 == 0xa1) { /* U+0361 -> euc ABD2 */ return utf2euc_emit_euc(0xabd2, 2, outptr, outroom, outchars); } else break; case 0xce: etab = utf2euc_ce; break; case 0xcf: etab = utf2euc_cf; break; case 0xd0: etab = utf2euc_d0; break; case 0xd1: etab = utf2euc_d1; break; default: break; } if (etab != NULL) { /* table lookup */ unsigned short euc = etab[u1-0x80]; if (euc != 0) { return utf2euc_emit_euc(euc, 2, outptr, outroom, outchars); } } DO_SUBST; return 2; } /* handle 3-byte UTF8 sequence. 0xe0 <= u0 <= 0xef */ static inline ScmSize utf2euc_3(ScmConvInfo *cinfo SCM_UNUSED, unsigned char u0, const char *inptr, ScmSize inroom, char *outptr, ScmSize outroom, ScmSize *outchars) { const unsigned char *tab1 = NULL; const unsigned short (*tab2)[64] = NULL; INCHK(3); unsigned char u1 = (unsigned char)inptr[1]; unsigned char u2 = (unsigned char)inptr[2]; switch (u0) { case 0xe1: /* special case : there's only 6 chars */ { unsigned short euc = 0; if (u1 == 0xb8) { if (u2 == 0xbe) euc = 0xa8f2; else if (u2 == 0xbf) euc = 0xa8f3; } else if (u1 == 0xbd) { if (u2 == 0xb0) euc = 0xabc6; else if (u2 == 0xb1) euc = 0xabc7; else if (u2 == 0xb2) euc = 0xabd0; else if (u2 == 0xb3) euc = 0xabd1; } return utf2euc_emit_euc(euc, 3, outptr, outroom, outchars); } case 0xe2: tab1 = utf2euc_e2; tab2 = utf2euc_e2_xx; break; case 0xe3: tab1 = utf2euc_e3; tab2 = utf2euc_e3_xx; break; case 0xe4: tab1 = utf2euc_e4; tab2 = utf2euc_e4_xx; break; case 0xe5: tab1 = utf2euc_e5; tab2 = utf2euc_e5_xx; break; case 0xe6: tab1 = utf2euc_e6; tab2 = utf2euc_e6_xx; break; case 0xe7: tab1 = utf2euc_e7; tab2 = utf2euc_e7_xx; break; case 0xe8: tab1 = utf2euc_e8; tab2 = utf2euc_e8_xx; break; case 0xe9: tab1 = utf2euc_e9; tab2 = utf2euc_e9_xx; break; case 0xef: tab1 = utf2euc_ef; tab2 = utf2euc_ef_xx; break; default: break; } if (tab1 != NULL) { unsigned char ind = tab1[u1-0x80]; if (ind != 0) { unsigned short euc = tab2[ind-1][u2-0x80]; if (euc != 0) { return utf2euc_emit_euc(euc, 3, outptr, outroom, outchars); } } } DO_SUBST; return 3; } /* handle 4-byte UTF8 sequence. u0 == 0xf0, 0xa0 <= u1 <= 0xaa */ static inline ScmSize utf2euc_4(ScmConvInfo *cinfo SCM_UNUSED, unsigned char u0, const char *inptr, ScmSize inroom, char *outptr, ScmSize outroom, ScmSize *outchars) { const unsigned short *tab = NULL; INCHK(4); if (u0 != 0xf0) { DO_SUBST; return 4; } unsigned char u1 = (unsigned char)inptr[1]; unsigned char u2 = (unsigned char)inptr[2]; unsigned char u3 = (unsigned char)inptr[3]; switch (u1) { case 0xa0: tab = utf2euc_f0_a0; break; case 0xa1: tab = utf2euc_f0_a1; break; case 0xa2: tab = utf2euc_f0_a2; break; case 0xa3: tab = utf2euc_f0_a3; break; case 0xa4: tab = utf2euc_f0_a4; break; case 0xa5: tab = utf2euc_f0_a5; break; case 0xa6: tab = utf2euc_f0_a6; break; case 0xa7: tab = utf2euc_f0_a7; break; case 0xa8: tab = utf2euc_f0_a8; break; case 0xa9: tab = utf2euc_f0_a9; break; case 0xaa: tab = utf2euc_f0_aa; break; default: break; } if (tab != NULL) { unsigned short u2u3 = u2*256 + u3; for (int i=0; tab[i]; i+=2) { if (tab[i] == u2u3) { unsigned short euc = tab[i+1]; if (euc != 0) { return utf2euc_emit_euc(euc, 4, outptr, outroom, outchars); } } } } DO_SUBST; return 4; } /* Body of UTF8 -> EUC_JP conversion */ static ScmSize utf8_eucj(ScmConvInfo *cinfo, const char *inptr, ScmSize inroom, char *outptr, ScmSize outroom, ScmSize *outchars) { unsigned char u0 = (unsigned char)inptr[0]; if (u0 <= 0x7f) { *outptr = u0; *outchars = 1; return 1; } if (u0 <= 0xbf) { /* invalid UTF8 sequence */ return ILLEGAL_SEQUENCE; } if (u0 <= 0xdf) { /* 2-byte UTF8 sequence */ return utf2euc_2(cinfo, u0, inptr, inroom, outptr, outroom, outchars); } if (u0 <= 0xef) { /* 3-byte UTF8 sequence */ return utf2euc_3(cinfo, u0, inptr, inroom, outptr, outroom, outchars); } if (u0 <= 0xf7) { /* 4-byte UTF8 sequence */ return utf2euc_4(cinfo, u0, inptr, inroom, outptr, outroom, outchars); } if (u0 <= 0xfb) { /* 5-byte UTF8 sequence */ INCHK(5); DO_SUBST; return 5; } if (u0 <= 0xfd) { /* 6-byte UTF8 sequence */ INCHK(6); DO_SUBST; return 6; } return ILLEGAL_SEQUENCE; } /* UTF8 -> UTF16 */ static ScmSize utf8_utf16(ScmConvInfo *cinfo, const char *inptr, ScmSize inroom, char *outptr, ScmSize outroom, ScmSize *outchars) { ScmSize reqsize = 0; int ostate = cinfo->ostate; int need_bom = FALSE; ScmChar ch; if (ostate == UTF_DEFAULT) { reqsize += 2; need_bom = TRUE; ostate = UTF_BE; } int r = jconv_utf8_to_ucs4(inptr, inroom, &ch); if (r < 0) return r; if (ch < 0x10000) reqsize += 2; else reqsize += 4; OUTCHK(reqsize); if (need_bom) { if (ostate == UTF_BE) { outptr[0] = 0xfe; outptr[1] = 0xff; } else { outptr[1] = 0xfe; outptr[0] = 0xff; } outptr += 2; } if (ch < 0x10000) { char u[2]; u[0] = (ch >> 8) & 0xff; u[1] = ch & 0xff; if (ostate == UTF_BE) { outptr[0] = u[0]; outptr[1] = u[1]; } else { outptr[1] = u[0]; outptr[0] = u[1]; } } else { ch -= 0x10000; char u[2]; u[0] = 0xd8 + ((ch >> 18) & 0x03); u[1] = (ch >> 10) & 0xff; if (ostate == UTF_BE) { outptr[0] = u[0]; outptr[1] = u[1]; } else { outptr[1] = u[0]; outptr[0] = u[1]; } u[0] = 0xdc + ((ch >> 8) & 0x03); u[1] = ch & 0xff; if (ostate == UTF_BE) { outptr[2] = u[0]; outptr[3] = u[1]; } else { outptr[3] = u[0]; outptr[2] = u[1]; } } cinfo->ostate = ostate; *outchars = reqsize; return r; } /* UTF8 -> UTF32 */ static ScmSize utf8_utf32(ScmConvInfo *cinfo, const char *inptr, ScmSize inroom, char *outptr, ScmSize outroom, ScmSize *outchars) { ScmSize reqsize = 0; int ostate = cinfo->ostate; int need_bom = FALSE; ScmChar ch; if (ostate == UTF_DEFAULT) { reqsize += 4; need_bom = TRUE; ostate = UTF_BE; } int r = jconv_utf8_to_ucs4(inptr, inroom, &ch); if (r < 0) return r; reqsize += 4; OUTCHK(reqsize); if (need_bom) { if (ostate == UTF_BE) { outptr[0] = 0; outptr[1] = 0; outptr[2] = 0xfe; outptr[3] = 0xff; } else { outptr[3] = 0; outptr[2] = 0; outptr[1] = 0xfe; outptr[0] = 0xff; } outptr += 4; } if (ostate == UTF_BE) { outptr[0] = (ch >> 24) & 0xff; outptr[1] = (ch >> 16) & 0xff; outptr[2] = (ch >> 8) & 0xff; outptr[3] = ch & 0xff; } else { outptr[3] = (ch >> 24) & 0xff; outptr[2] = (ch >> 16) & 0xff; outptr[1] = (ch >> 8) & 0xff; outptr[0] = ch & 0xff; } cinfo->ostate = ostate; *outchars = reqsize; return r; } /* UTF8 -> Latin1 */ static ScmSize utf8_lat1(ScmConvInfo *cinfo, const char *inptr, ScmSize inroom, char *outptr, ScmSize outroom, ScmSize *outchars) { ScmChar ch; int r = jconv_utf8_to_ucs4(inptr, inroom, &ch); if (r < 0) return r; if (ch < 0x100) { *outptr = ch; *outchars = 1; } else { DO_SUBST; } return r; } /* UTF8 -> ASCII */ static ScmSize utf8_ascii(ScmConvInfo *cinfo, const char *inptr, ScmSize inroom, char *outptr, ScmSize outroom, ScmSize *outchars) { ScmChar ch; int r = jconv_utf8_to_ucs4(inptr, inroom, &ch); if (r < 0) return r; if (ch < 0x80) { *outptr = ch; *outchars = 1; } else { DO_SUBST; } return r; } /* UTF8BOM -> UTF8 */ static ScmSize utf8bom_utf8(ScmConvInfo *cinfo, const char *inptr, ScmSize inroom, char *outptr, ScmSize outroom SCM_UNUSED, ScmSize *outchars) { if (cinfo->istate == UTF_OPTIONAL_BOM) { if ((u_char)inptr[0] == 0xef) { INCHK(3); if ((u_char)inptr[1] == 0xbb && (u_char)inptr[2] == 0xbf) { cinfo->istate = UTF_DEFAULT; *outchars = 0; return 3; } } } /* FALLTHROUGH */ outptr[0] = inptr[0]; *outchars = 1; return 1; } /*================================================================= * UTF16 */ /* For now, we first convert it to utf8, for we already have the table directly supports utf8. Theoretically though, having ucs4 to jis table would speed it up. */ static ScmSize utf16_utf8(ScmConvInfo *cinfo, const char *inptr, ScmSize inroom, char *outptr, ScmSize outroom, ScmSize *outchars) { INCHK(2); int istate = cinfo->istate; ScmSize inread = 0; if (istate == UTF_DEFAULT) { if ((u_char)inptr[0] == 0xfe && (u_char)inptr[1] == 0xff) { inptr += 2; inroom -= 2; inread += 2; INCHK(2); istate = UTF_BE; } else if ((u_char)inptr[0] == 0xff && (u_char)inptr[1] == 0xfe) { inptr += 2; inroom -= 2; inread += 2; INCHK(2); istate = UTF_LE; } else { /* Arbitrary choice */ istate = UTF_BE; } } u_char u[2]; if (istate == UTF_BE) { u[0] = inptr[0]; u[1] = inptr[1]; } else { u[0] = inptr[1]; u[1] = inptr[0]; } ScmChar ch; if ((u[0] & 0xfc) == 0xd8) { /* surrogate */ inptr += 2; inroom -= 2; INCHK(2); u_char v[2]; if (istate == UTF_BE) { v[0] = inptr[0]; v[1] = inptr[1]; } else { v[0] = inptr[1]; v[1] = inptr[0]; } if ((v[0] & 0xfc) == 0xdc) { ch = (((u[0] & 0x03) << 18) | (u[1] << 10) | ((v[0] & 0x03) << 8) | v[1]) + 0x10000; inread += 4; } else { /* We only have first half of a surrogate pair. We leave the second character in the input, and try to substitute the first. */ DO_SUBST; cinfo->istate = istate; return inread; } } else if ((u[0] & 0xfc) == 0xdc) { /* Stray second half of a surrogate pair. */ DO_SUBST; cinfo->istate = istate; return inread; } else { inread += 2; ch = (u[0] << 8) + u[1]; } int outreq = UCS2UTF_NBYTES(ch); OUTCHK(outreq); jconv_ucs4_to_utf8(ch, outptr); cinfo->istate = istate; *outchars = outreq; return inread; } /* This handles BOM stuff. It is pretty twisted, for we need to keep the internal state consistent even when we return an error. */ static ScmSize utf16_utf16(ScmConvInfo *cinfo, const char *inptr, ScmSize inroom, char *outptr, ScmSize outroom, ScmSize *outchars) { ScmSize consumed = 0; ScmSize emitted = 0; if (cinfo->istate == UTF_DEFAULT || cinfo->ostate == UTF_DEFAULT) { /* We come here only at the beginning. */ int istate = 0; if (cinfo->istate == UTF_DEFAULT) { INCHK(2); if ((u_char)inptr[0] == 0xfe && (u_char)inptr[1] == 0xff) { consumed += 2; istate = UTF_BE; inptr += 2; inroom -= 2; } else if ((u_char)inptr[0] == 0xff && (u_char)inptr[1] == 0xfe) { consumed += 2; istate = UTF_LE; inptr += 2; inroom -= 2; } else { istate = UTF_BE; } } INCHK(2); if (cinfo->ostate == UTF_DEFAULT) { OUTCHK(4); outptr[0] = 0xfe; outptr[1] = 0xff; outptr += 2; outroom -= 2; emitted += 2; cinfo->ostate = UTF_BE; } else { OUTCHK(2); } cinfo->istate = istate; } else { INCHK(2); OUTCHK(2); } char u[2]; if (cinfo->istate == UTF_BE) { u[0] = inptr[0]; u[1] = inptr[1]; } else { u[1] = inptr[0]; u[0] = inptr[1]; } if (cinfo->ostate == UTF_BE) { outptr[0] = u[0]; outptr[1] = u[1]; } else { outptr[1] = u[0]; outptr[0] = u[1]; } *outchars = emitted + 2; return consumed + 2; } /*================================================================= * UTF32 */ static ScmSize utf32_utf8(ScmConvInfo *cinfo, const char *inptr, ScmSize inroom, char *outptr, ScmSize outroom, ScmSize *outchars) { INCHK(4); int istate = cinfo->istate; ScmSize inread = 0; if (istate == UTF_DEFAULT) { if ((u_char)inptr[0] == 0 && (u_char)inptr[1] == 0 && (u_char)inptr[2] == 0xfe && (u_char)inptr[3] == 0xff) { inptr += 4; inroom -= 4; inread += 4; INCHK(4); istate = UTF_BE; } else if ((u_char)inptr[0] == 0xff && (u_char)inptr[1] == 0xfe && (u_char)inptr[2] == 0 && (u_char)inptr[3] == 0) { inptr += 4; inroom -= 4; inread += 4; INCHK(4); istate = UTF_LE; } else { /* Arbitrary choice */ istate = UTF_BE; } } u_char u[4]; if (istate == UTF_BE) { u[0] = inptr[0]; u[1] = inptr[1]; u[2] = inptr[2]; u[3] = inptr[3]; } else { u[0] = inptr[3]; u[1] = inptr[2]; u[2] = inptr[1]; u[3] = inptr[0]; } inread += 4; ScmChar ch = (u[0] << 24) | (u[1] << 16) | (u[2] << 8) | u[3]; int outreq = UCS2UTF_NBYTES(ch); OUTCHK(outreq); jconv_ucs4_to_utf8(ch, outptr); cinfo->istate = istate; *outchars = outreq; return inread; } /* This handles BOM stuff. */ static ScmSize utf32_utf32(ScmConvInfo *cinfo, const char *inptr, ScmSize inroom, char *outptr, ScmSize outroom, ScmSize *outchars) { ScmSize consumed = 0; ScmSize emitted = 0; if (cinfo->istate == UTF_DEFAULT || cinfo->ostate == UTF_DEFAULT) { /* We come here only at the beginning. */ int istate = 0; if (cinfo->istate == UTF_DEFAULT) { INCHK(4); if ((u_char)inptr[0] == 0 && (u_char)inptr[1] == 0 && (u_char)inptr[2] == 0xfe && (u_char)inptr[3] == 0xff) { consumed += 4; istate = UTF_BE; inptr += 4; inroom -= 4; } else if ((u_char)inptr[0] == 0xff && (u_char)inptr[1] == 0xfe && (u_char)inptr[2] == 0 && (u_char)inptr[3] == 0) { consumed += 4; istate = UTF_LE; inptr += 4; inroom -= 4; } else { istate = UTF_BE; } } INCHK(4); if (cinfo->ostate == UTF_DEFAULT) { OUTCHK(8); outptr[0] = 0; outptr[1] = 0; outptr[2] = 0xfe; outptr[3] = 0xff; outptr += 4; outroom -= 4; emitted += 4; cinfo->ostate = UTF_BE; } else { OUTCHK(4); } cinfo->istate = istate; } else { INCHK(4); OUTCHK(4); } char u[4]; if (cinfo->istate == UTF_BE) { u[0] = inptr[0]; u[1] = inptr[1]; u[2] = inptr[2]; u[3] = inptr[3]; } else { u[3] = inptr[0]; u[2] = inptr[1]; u[1] = inptr[2]; u[0] = inptr[3]; } if (cinfo->ostate == UTF_BE) { outptr[0] = u[0]; outptr[1] = u[1]; outptr[2] = u[2]; outptr[3] = u[3]; } else { outptr[3] = u[0]; outptr[2] = u[1]; outptr[1] = u[2]; outptr[0] = u[3]; } *outchars = emitted + 4; return consumed + 4; } /*================================================================= * ISO2022-JP */ /* ISO2022-JP{-1(,2),3} -> EUC_JP * Strategy: accepts as many possibilities as possible. * The following escape sequence is recognized: * (See Lunde, CJKV information processing, O'Reilly, pp.155--158) * * ( B ASCII * ( J JIS-Roman * ( H JIS-Roman (for compatibility) * ( I Half-width katakana (JIS X 0201 kana) * $ @ JIS C 6226-1978 (78JIS) * $ B JIS X 0208-1983 (83JIS) * $ ( D JIS X 0212-1990 * $ ( O JIS X 0213:2000 plane 1 * $ ( P JIS X 0213:2000 plane 2 * & @ $ B JIS X 0208-1990, JIS X 0208:1997 * 0x0e JIS7 half-width katakana shift-out * 0x0f JIS7 half-width katakana shift-in * * The state is reset to ASCII whenever newline character is read. * * The following escape sequences defined in ISO2022-JP-2 are recognized, * but all the characters within the sequence will be replaced by '?'. * * $ A (GB2312-80) unsupported * $ ( C (KS X 1001:1992) unsupported * . A (ISO8859-1:1998) unsupported * . F (ISO8859-7:1998) unsupported * * If other escape sequence is seen, the converter returns ILLEGAL_SEQUENCE. * * JIS8 kana is allowed. */ /* deal with escape sequence. escape byte itself is already consumed. returns # of input bytes consumed by the escape sequence, or an error code. cinfo->istate is updated accordingly. */ static ScmSize jis_esc(ScmConvInfo *cinfo, const char *inptr, ScmSize inroom) { INCHK(2); unsigned char j1 = inptr[0]; unsigned char j2 = inptr[1]; switch (j1) { case '(': switch (j2) { case 'B': cinfo->istate = JIS_ASCII; break; case 'J': cinfo->istate = JIS_ROMAN; break; case 'H': cinfo->istate = JIS_ROMAN; break; case 'I': cinfo->istate = JIS_KANA; break; default: return ILLEGAL_SEQUENCE; } return 2; case '$': switch (j2) { case '@': cinfo->istate = JIS_78; break; case 'B': cinfo->istate = JIS_0213_1; break; case 'A': cinfo->istate = JIS_UNKNOWN; break; case '(': { INCHK(3); switch (inptr[2]) { case 'D': cinfo->istate = JIS_0212; break; case 'O': cinfo->istate = JIS_0213_1; break; case 'P': cinfo->istate = JIS_0213_2; break; case 'C': cinfo->istate = JIS_UNKNOWN; break; default: return ILLEGAL_SEQUENCE; } return 3; break; } default: return ILLEGAL_SEQUENCE; } return 2; case '&': { INCHK(6); if (inptr[2] == '@' && inptr[3] == 0x1b && inptr[4] == '$' && inptr[5] == 'B') { cinfo->istate = JIS_0213_1; return 5; } else { return ILLEGAL_SEQUENCE; } } case '.': switch (inptr[2]) { case 'A':/*fallthrough*/; case 'F': cinfo->istate = JIS_UNKNOWN; break; default: return ILLEGAL_SEQUENCE; } return 2; default: return ILLEGAL_SEQUENCE; } } /* main routine for iso2022-jp -> euc_jp */ static ScmSize jis_eucj(ScmConvInfo *cinfo, const char *inptr, ScmSize inroom, char *outptr, ScmSize outroom, ScmSize *outchars) { ScmSize inoffset = 0; unsigned char j0 = inptr[inoffset]; /* skip escape sequence */ while (j0 == 0x1b) { inoffset++; ScmSize r = jis_esc(cinfo, inptr+inoffset, inroom-inoffset); if (ERRP(r)) return r; inoffset += r; if (inoffset >= inroom) { *outchars = 0; return inoffset; } j0 = inptr[inoffset]; } if (j0 == '\n' || j0 == '\r') { cinfo->istate = JIS_ASCII; outptr[0] = j0; *outchars = 1; return 1+inoffset; } else if (j0 < 0x20) { outptr[0] = j0; *outchars = 1; return 1+inoffset; } else if (j0 >= 0xa1 && j0 <= 0xdf) { /* JIS8 kana */ OUTCHK(2); outptr[0] = 0x8e; outptr[1] = j0; *outchars = 2; return 1+inoffset; } else { switch (cinfo->istate) { case JIS_ROMAN: /* jis-roman and ascii differs on 0x5c and 0x7e -- for now, I ignore the difference. */ /* FALLTHROUGH */ case JIS_ASCII: outptr[0] = j0; *outchars = 1; return 1+inoffset; case JIS_KANA: OUTCHK(2); outptr[0] = 0x8e; outptr[1] = j0 + 0x80; *outchars = 2; return 1+inoffset; case JIS_78: /* for now, I ignore the difference between JIS78 and JIS83 */ /* FALLTHROUGH */ case JIS_0213_1: { INCHK(inoffset+2); OUTCHK(2); unsigned char j1 = inptr[inoffset+1]; outptr[0] = j0 + 0x80; outptr[1] = j1 + 0x80; *outchars = 2; return 2+inoffset; } case JIS_0212: /* jis x 0212 and jis x 0213 plane 2 are different character sets, but uses the same conversion scheme. */ /* FALLTHROUGH */ case JIS_0213_2: { INCHK(inoffset+2); OUTCHK(3); unsigned char j1 = inptr[inoffset+1]; outptr[0] = 0x8f; outptr[1] = j0 + 0x80; outptr[2] = j1 + 0x80; *outchars = 3; return 2+inoffset; } case JIS_UNKNOWN: DO_SUBST; return 1+inoffset; default: /* Can't be here */ Scm_Panic("internal state of ISO2022-JP -> EUC_JP got messed up (%d). Implementation error?", cinfo->istate); } } return ILLEGAL_SEQUENCE; } /* reset proc */ static ScmSize jis_reset(ScmConvInfo *cinfo, char *outptr, ScmSize outroom) { if (outptr == NULL) { /* just reset */ cinfo->ostate = JIS_ASCII; return 0; } else { if (cinfo->ostate == JIS_ASCII) return 0; if (outroom < 3) return OUTPUT_NOT_ENOUGH; outptr[0] = 0x1b; outptr[1] = '('; outptr[2] = 'B'; cinfo->ostate = JIS_ASCII; return 3; } } /*================================================================= * ISO8859-1 */ static ScmSize lat1_utf8(ScmConvInfo *cinfo SCM_UNUSED, const char *inptr, ScmSize inroom SCM_UNUSED, char *outptr, ScmSize outroom, ScmSize *outchars) { unsigned char c = inptr[0]; if (c <= 0x7f) { outptr[0] = c; *outchars = 1; } else { OUTCHK(2); outptr[0] = 0xc0 + (c >> 6); outptr[1] = 0x80 + (c & 0x3f); *outchars = 2; } return 1; } static ScmSize lat1_ascii(ScmConvInfo *cinfo, const char *inptr, ScmSize inroom SCM_UNUSED, char *outptr, ScmSize outroom, ScmSize *outchars) { unsigned char c = inptr[0]; if (c <= 0x7f) { outptr[0] = c; *outchars = 1; } else { DO_SUBST; } return 1; } /*================================================================= * ASCII */ /* ASCII -> X */ static ScmSize ascii_x(ScmConvInfo *cinfo SCM_UNUSED, const char *inptr, ScmSize inroom SCM_UNUSED, char *outptr, ScmSize outroom SCM_UNUSED, ScmSize *outchars) { outptr[0] = inptr[0]; *outchars = 1; return 1; } /*================================================================= * Placeholder */ static ScmSize ident(ScmConvInfo *cinfo SCM_UNUSED, const char *inptr SCM_UNUSED, ScmSize inroom SCM_UNUSED, char *outptr SCM_UNUSED, ScmSize outroom SCM_UNUSED, ScmSize *outchars SCM_UNUSED) { return 0; } /****************************************************************** * * Actual conversion * */ /* map canonical code designator to inconv and outconv. the order of entry must match with the above designators. conv_converter[incode][outcode] returns the appropriate combiniation of routines. NB: It is tedious to maintain this table; we'll eventually generate this from some DSL. */ struct conv_converter_rec { ScmConvProc *conv; ScmConvReset *reset; int istate; /* initial input state */ int ostate; /* initial output state */ }; /* map convesion name to the canonical code */ struct conv_support_rec { const char *name; int code; }; #include "jconv_tab.c" #include "latin_tab.c" static int conv_name_match(const char *s, const char *t) { const char *p, *q; for (p=s, q=t; *p && *q; p++) { if (*p == '-' || *p == '_') { continue; /* ignore '-' and '_' */ } else { if (tolower(*p) != tolower(*q)) return FALSE; q++; } } if (*p || *q) return FALSE; return TRUE; } static int conv_name_find(const char *name) { struct conv_support_rec *cvtab = conv_supports; for (; cvtab->name; cvtab++) { if (conv_name_match(name, cvtab->name)) { return cvtab->code; } } return -1; } /* Internal conversion handler. */ /* when we can just pass-through input to output */ static ScmSize jconv_ident(ScmConvInfo *cinfo SCM_UNUSED, const char **iptr, ScmSize *iroom, char **optr, ScmSize *oroom) { ScmSize inroom = *iroom, outroom = *oroom; #ifdef JCONV_DEBUG fprintf(stderr, "jconv_ident %s->%s\n", cinfo->fromCode, cinfo->toCode); #endif if (inroom <= outroom) { memcpy(*optr, *iptr, inroom); *optr += inroom; *iptr += inroom; *iroom = 0; *oroom -= inroom; return inroom; } else { memcpy(*optr, *iptr, outroom); *optr += outroom; *iptr += outroom; *iroom -= outroom; *oroom = 0; return OUTPUT_NOT_ENOUGH; } } /* calling conversion routine for each char */ static ScmSize jconv_1tier(ScmConvInfo *cinfo, const char **iptr, ScmSize *iroom, char **optr, ScmSize *oroom) { ScmConvProc *cvt = cinfo->convert; const char *inp = *iptr; char *outp = *optr; int inr = (int)*iroom, outr = (int)*oroom; ScmSize converted = 0; #ifdef JCONV_DEBUG fprintf(stderr, "jconv_1tier %s->%s\n", cinfo->fromCode, cinfo->toCode); #endif SCM_ASSERT(cvt != NULL); while (inr > 0 && outr > 0) { ScmSize outchars; ScmSize inchars = cvt(cinfo, inp, inr, outp, outr, &outchars); if (ERRP(inchars)) { converted = inchars; break; } else { converted += inchars; inp += inchars; inr -= (int)inchars; outp += outchars; outr -= (int)outchars; } } *iptr = inp; *iroom = inr; *optr = outp; *oroom = outr; return converted; } /* When we delegate conversion to iconv(3) */ #ifdef HAVE_ICONV_H /* NB: although iconv manages states, we need to keep track of whether * we're sure in default status (JIS_ASCII) or not (we use JIS_UNKNOWN for it). * It's because jconv_iconv_reset will be called twice if there is any * reset sequence; the first call should emit the sequence, but the second * call shouldn't. */ static ScmSize jconv_iconv(ScmConvInfo *cinfo, const char **iptr, ScmSize *iroom, char **optr, ScmSize *oroom) { #ifdef JCONV_DEBUG fprintf(stderr, "jconv_iconv %s->%s\n", cinfo->fromCode, cinfo->toCode); #endif size_t ir = *iroom, or = *oroom; size_t r = iconv(cinfo->handle, (char **)iptr, &ir, optr, &or); *iroom = ir; *oroom = or; cinfo->ostate = JIS_UNKNOWN; if (r == (size_t)-1) { if (errno == EINVAL) return INPUT_NOT_ENOUGH; if (errno == E2BIG) return OUTPUT_NOT_ENOUGH; return ILLEGAL_SEQUENCE; } else { return (ScmSize)r; } } /* reset routine for iconv */ static ScmSize jconv_iconv_reset(ScmConvInfo *cinfo, char *optr, ScmSize oroom) { ScmSize oroom_prev = oroom; if (cinfo->ostate == JIS_ASCII) return 0; size_t or = oroom; size_t r = iconv(cinfo->handle, NULL, 0, &optr, &or); if (r == (size_t)-1) { if (errno == E2BIG) return OUTPUT_NOT_ENOUGH; Scm_Panic("jconv_iconv_reset: unknown error number %d\n", errno); } cinfo->ostate = JIS_ASCII; return oroom_prev - (ScmSize)or; } #endif /*HAVE_ICONV_H*/ /*------------------------------------------------------------------ * JCONV_OPEN * Returns ScmConvInfo, setting up some fields. * If no conversion is possible, returns NULL. */ ScmConvInfo *jconv_open(const char *toCode, const char *fromCode, int useIconv) { ScmConvHandler *handler = NULL; ScmConvProc *convert = NULL; ScmConvReset *reset = NULL; int istate = 0, ostate = 0; iconv_t handle = (iconv_t)-1; int incode = conv_name_find(fromCode); int outcode = conv_name_find(toCode); if (incode >= 0 && outcode >= 0) { convert = conv_converter[incode][outcode].conv; reset = conv_converter[incode][outcode].reset; istate = conv_converter[incode][outcode].istate; ostate = conv_converter[incode][outcode].ostate; } if (convert == NULL) { if (conv_name_match(fromCode, toCode)) { handler = jconv_ident; } else if (useIconv) { #ifdef HAVE_ICONV_H /* try iconv */ handle = iconv_open(toCode, fromCode); if (handle == (iconv_t)-1) return NULL; handler = jconv_iconv; reset = jconv_iconv_reset; #else /*!HAVE_ICONV_H*/ return NULL; #endif } else { return NULL; } } else if (convert == ident) { handler = jconv_ident; } else { handler = jconv_1tier; } ScmConvInfo *cinfo; cinfo = SCM_NEW(ScmConvInfo); cinfo->jconv = handler; cinfo->convert = convert; cinfo->reset = reset; cinfo->handle = handle; cinfo->toCode = toCode; cinfo->istate = istate; cinfo->ostate = ostate; cinfo->fromCode = fromCode; /* The replacement settings can be modified by jconv_set_replacement */ cinfo->replacep = FALSE; cinfo->replaceSize = 0; cinfo->replaceSeq = NULL; return cinfo; } /*------------------------------------------------------------------ * JCONV_SET_REPLACEMENT * Setting up replacement sequence according to the toCode. */ void jconv_set_replacement(ScmConvInfo *cinfo) { static ScmObj ces_replacement_proc = SCM_UNDEFINED; SCM_BIND_PROC(ces_replacement_proc, "%ces-replacement", Scm_FindModule(SCM_SYMBOL(SCM_INTERN("gauche.charconv")), 0)); ScmObj replacements = Scm_ApplyRec1(ces_replacement_proc, SCM_MAKE_STR(cinfo->toCode)); ScmSize i = Scm_Length(replacements); if (i > 0) { cinfo->replacep = TRUE; cinfo->replaceSize = i; char *replaceSeq = SCM_NEW_ATOMIC_ARRAY(char, i); for (int j = 0; j < i; j++) { SCM_ASSERT(SCM_PAIRP(replacements)); replaceSeq[j] = SCM_INT_VALUE(SCM_CAR(replacements)); replacements = SCM_CDR(replacements); } cinfo->replaceSeq = replaceSeq; } } /*------------------------------------------------------------------ * JCONV_CLOSE */ int jconv_close(ScmConvInfo *cinfo) { int r = 0; #ifdef HAVE_ICONV_H if (cinfo->handle != (iconv_t)-1) { r = iconv_close(cinfo->handle); cinfo->handle = (iconv_t)-1; } #endif /*HAVE_ICONV_H*/ return r; } /*------------------------------------------------------------------ * JCONV - main conversion routine */ ScmSize jconv(ScmConvInfo *cinfo, const char **inptr, ScmSize *inroom, char **outptr, ScmSize *outroom) { SCM_ASSERT(cinfo->jconv != NULL); return cinfo->jconv(cinfo, inptr, inroom, outptr, outroom); } /*------------------------------------------------------------------ * JCONV_RESET - reset */ ScmSize jconv_reset(ScmConvInfo *cinfo, char *outptr, ScmSize outroom) { if (cinfo->reset) { return cinfo->reset(cinfo, outptr, outroom); } else { return 0; } }