Debian Codegraph @

Modules/cjkcodecs/_codecs_iso2022.c

raw
1 /*
2 * _codecs_iso2022.c: Codecs collection for ISO-2022 encodings.
3 *
4 * Written by Hye-Shik Chang <[email protected]>
5 */
6
7 #define USING_IMPORTED_MAPS
8 #define USING_BINARY_PAIR_SEARCH
9 #define EXTERN_JISX0213_PAIR
10 #define EMULATE_JISX0213_2000_ENCODE_INVALID MAP_UNMAPPABLE
11 #define EMULATE_JISX0213_2000_DECODE_INVALID MAP_UNMAPPABLE
12
13 #define CJK_MOD_SPECIFIC_STATE \
14 /* kr */ \
15 const encode_map *cp949_encmap; \
16 const decode_map *ksx1001_decmap; \
17 \
18 /* jp */ \
19 const encode_map *jisxcommon_encmap; \
20 const decode_map *jisx0208_decmap; \
21 const decode_map *jisx0212_decmap; \
22 const encode_map *jisx0213_bmp_encmap; \
23 const decode_map *jisx0213_1_bmp_decmap; \
24 const decode_map *jisx0213_2_bmp_decmap; \
25 const encode_map *jisx0213_emp_encmap; \
26 const decode_map *jisx0213_1_emp_decmap; \
27 const decode_map *jisx0213_2_emp_decmap; \
28 \
29 /* cn */ \
30 const encode_map *gbcommon_encmap; \
31 const decode_map *gb2312_decmap;
32
33
34 #include "cjkcodecs.h"
35 #include "alg_jisx0201.h"
36 #include "emu_jisx0213_2000.h"
37 #include "mappings_jisx0213_pair.h"
38
39 /* STATE
40
41 state->c[0-3]
42
43 00000000
44 ||^^^^^|
45 |+-----+---- G0-3 Character Set
46 +----------- Is G0-3 double byte?
47
48 state->c[4]
49
50 00000000
51 ||
52 |+---- Locked-Shift?
53 +----- ESC Throughout
54 */
55
56 #define ESC 0x1B
57 #define SO 0x0E
58 #define SI 0x0F
59 #define LF 0x0A
60
61 #define MAX_ESCSEQLEN 16
62
63 #define CHARSET_ISO8859_1 'A'
64 #define CHARSET_ASCII 'B'
65 #define CHARSET_ISO8859_7 'F'
66 #define CHARSET_JISX0201_K 'I'
67 #define CHARSET_JISX0201_R 'J'
68
69 #define CHARSET_GB2312 ('A'|CHARSET_DBCS)
70 #define CHARSET_JISX0208 ('B'|CHARSET_DBCS)
71 #define CHARSET_KSX1001 ('C'|CHARSET_DBCS)
72 #define CHARSET_JISX0212 ('D'|CHARSET_DBCS)
73 #define CHARSET_GB2312_8565 ('E'|CHARSET_DBCS)
74 #define CHARSET_CNS11643_1 ('G'|CHARSET_DBCS)
75 #define CHARSET_CNS11643_2 ('H'|CHARSET_DBCS)
76 #define CHARSET_JISX0213_2000_1 ('O'|CHARSET_DBCS)
77 #define CHARSET_JISX0213_2 ('P'|CHARSET_DBCS)
78 #define CHARSET_JISX0213_2004_1 ('Q'|CHARSET_DBCS)
79 #define CHARSET_JISX0208_O ('@'|CHARSET_DBCS)
80
81 #define CHARSET_DBCS 0x80
82 #define ESCMARK(mark) ((mark) & 0x7f)
83
84 #define IS_ESCEND(c) (((c) >= 'A' && (c) <= 'Z') || (c) == '@')
85 #define IS_ISO2022ESC(c2) \
86 ((c2) == '(' || (c2) == ')' || (c2) == '$' || \
87 (c2) == '.' || (c2) == '&')
88 /* this is not a complete list of ISO-2022 escape sequence headers.
89 * but, it's enough to implement CJK instances of iso-2022. */
90
91 #define MAP_UNMAPPABLE 0xFFFF
92 #define MAP_MULTIPLE_AVAIL 0xFFFE /* for JIS X 0213 */
93
94 #define F_SHIFTED 0x01
95 #define F_ESCTHROUGHOUT 0x02
96
97 #define STATE_SETG(dn, v) do { ((state)->c[dn]) = (v); } while (0)
98 #define STATE_GETG(dn) ((state)->c[dn])
99
100 #define STATE_G0 STATE_GETG(0)
101 #define STATE_G1 STATE_GETG(1)
102 #define STATE_G2 STATE_GETG(2)
103 #define STATE_G3 STATE_GETG(3)
104 #define STATE_SETG0(v) STATE_SETG(0, v)
105 #define STATE_SETG1(v) STATE_SETG(1, v)
106 #define STATE_SETG2(v) STATE_SETG(2, v)
107 #define STATE_SETG3(v) STATE_SETG(3, v)
108
109 #define STATE_SETFLAG(f) do { ((state)->c[4]) |= (f); } while (0)
110 #define STATE_GETFLAG(f) ((state)->c[4] & (f))
111 #define STATE_CLEARFLAG(f) do { ((state)->c[4]) &= ~(f); } while (0)
112 #define STATE_CLEARFLAGS() do { ((state)->c[4]) = 0; } while (0)
113
114 #define ISO2022_CONFIG ((const struct iso2022_config *)(codec->config))
115 #define CONFIG_ISSET(flag) (ISO2022_CONFIG->flags & (flag))
116 #define CONFIG_DESIGNATIONS (ISO2022_CONFIG->designations)
117
118 /* iso2022_config.flags */
119 #define NO_SHIFT 0x01
120 #define USE_G2 0x02
121 #define USE_JISX0208_EXT 0x04
122
123 /*-*- internal data structures -*-*/
124
125 typedef int (*iso2022_init_func)(const MultibyteCodec *codec);
126 typedef Py_UCS4 (*iso2022_decode_func)(const MultibyteCodec *codec,
127 const unsigned char *data);
128 typedef DBCHAR (*iso2022_encode_func)(const MultibyteCodec *codec,
129 const Py_UCS4 *data,
130 Py_ssize_t *length);
131
132 struct iso2022_designation {
133 unsigned char mark;
134 unsigned char plane;
135 unsigned char width;
136 iso2022_init_func initializer;
137 iso2022_decode_func decoder;
138 iso2022_encode_func encoder;
139 };
140
141 struct iso2022_config {
142 int flags;
143 const struct iso2022_designation *designations; /* non-ascii desigs */
144 };
145
146 /*-*- iso-2022 codec implementation -*-*/
147
148 CODEC_INIT(iso2022)
149 {
150 const struct iso2022_designation *desig;
151 for (desig = CONFIG_DESIGNATIONS; desig->mark; desig++) {
152 if (desig->initializer != NULL && desig->initializer(codec) != 0) {
153 return -1;
154 }
155 }
156 return 0;
157 }
158
159 ENCODER_INIT(iso2022)
160 {
161 STATE_CLEARFLAGS();
162 STATE_SETG0(CHARSET_ASCII);
163 STATE_SETG1(CHARSET_ASCII);
164 return 0;
165 }
166
167 ENCODER_RESET(iso2022)
168 {
169 if (STATE_GETFLAG(F_SHIFTED)) {
170 WRITEBYTE1(SI);
171 NEXT_OUT(1);
172 STATE_CLEARFLAG(F_SHIFTED);
173 }
174 if (STATE_G0 != CHARSET_ASCII) {
175 WRITEBYTE3(ESC, '(', 'B');
176 NEXT_OUT(3);
177 STATE_SETG0(CHARSET_ASCII);
178 }
179 return 0;
180 }
181
182 ENCODER(iso2022)
183 {
184 while (*inpos < inlen) {
185 const struct iso2022_designation *dsg;
186 DBCHAR encoded;
187 Py_UCS4 c = INCHAR1;
188 Py_ssize_t insize;
189
190 if (c < 0x80) {
191 if (STATE_G0 != CHARSET_ASCII) {
192 WRITEBYTE3(ESC, '(', 'B');
193 STATE_SETG0(CHARSET_ASCII);
194 NEXT_OUT(3);
195 }
196 if (STATE_GETFLAG(F_SHIFTED)) {
197 WRITEBYTE1(SI);
198 STATE_CLEARFLAG(F_SHIFTED);
199 NEXT_OUT(1);
200 }
201 WRITEBYTE1((unsigned char)c);
202 NEXT(1, 1);
203 continue;
204 }
205
206 insize = 1;
207
208 encoded = MAP_UNMAPPABLE;
209 for (dsg = CONFIG_DESIGNATIONS; dsg->mark; dsg++) {
210 Py_UCS4 buf[2] = {c, 0};
211 Py_ssize_t length = 1;
212 encoded = dsg->encoder(codec, buf, &length);
213 if (encoded == MAP_MULTIPLE_AVAIL) {
214 /* this implementation won't work for pair
215 * of non-bmp characters. */
216 if (inlen - *inpos < 2) {
217 if (!(flags & MBENC_FLUSH))
218 return MBERR_TOOFEW;
219 length = -1;
220 }
221 else {
222 buf[1] = INCHAR2;
223 length = 2;
224 }
225 encoded = dsg->encoder(codec, buf, &length);
226 if (encoded != MAP_UNMAPPABLE) {
227 insize = length;
228 break;
229 }
230 }
231 else if (encoded != MAP_UNMAPPABLE)
232 break;
233 }
234
235 if (!dsg->mark)
236 return 1;
237 assert(dsg->width == 1 || dsg->width == 2);
238
239 switch (dsg->plane) {
240 case 0: /* G0 */
241 if (STATE_GETFLAG(F_SHIFTED)) {
242 WRITEBYTE1(SI);
243 STATE_CLEARFLAG(F_SHIFTED);
244 NEXT_OUT(1);
245 }
246 if (STATE_G0 != dsg->mark) {
247 if (dsg->width == 1) {
248 WRITEBYTE3(ESC, '(', ESCMARK(dsg->mark));
249 STATE_SETG0(dsg->mark);
250 NEXT_OUT(3);
251 }
252 else if (dsg->mark == CHARSET_JISX0208) {
253 WRITEBYTE3(ESC, '$', ESCMARK(dsg->mark));
254 STATE_SETG0(dsg->mark);
255 NEXT_OUT(3);
256 }
257 else {
258 WRITEBYTE4(ESC, '$', '(',
259 ESCMARK(dsg->mark));
260 STATE_SETG0(dsg->mark);
261 NEXT_OUT(4);
262 }
263 }
264 break;
265 case 1: /* G1 */
266 if (STATE_G1 != dsg->mark) {
267 if (dsg->width == 1) {
268 WRITEBYTE3(ESC, ')', ESCMARK(dsg->mark));
269 STATE_SETG1(dsg->mark);
270 NEXT_OUT(3);
271 }
272 else {
273 WRITEBYTE4(ESC, '$', ')', ESCMARK(dsg->mark));
274 STATE_SETG1(dsg->mark);
275 NEXT_OUT(4);
276 }
277 }
278 if (!STATE_GETFLAG(F_SHIFTED)) {
279 WRITEBYTE1(SO);
280 STATE_SETFLAG(F_SHIFTED);
281 NEXT_OUT(1);
282 }
283 break;
284 default: /* G2 and G3 is not supported: no encoding in
285 * CJKCodecs are using them yet */
286 return MBERR_INTERNAL;
287 }
288
289 if (dsg->width == 1) {
290 WRITEBYTE1((unsigned char)encoded);
291 NEXT_OUT(1);
292 }
293 else {
294 WRITEBYTE2(encoded >> 8, encoded & 0xff);
295 NEXT_OUT(2);
296 }
297 NEXT_INCHAR(insize);
298 }
299
300 return 0;
301 }
302
303 DECODER_INIT(iso2022)
304 {
305 STATE_CLEARFLAGS();
306 STATE_SETG0(CHARSET_ASCII);
307 STATE_SETG1(CHARSET_ASCII);
308 STATE_SETG2(CHARSET_ASCII);
309 return 0;
310 }
311
312 DECODER_RESET(iso2022)
313 {
314 STATE_SETG0(CHARSET_ASCII);
315 STATE_CLEARFLAG(F_SHIFTED);
316 return 0;
317 }
318
319 static Py_ssize_t
320 iso2022processesc(const MultibyteCodec *codec, MultibyteCodec_State *state,
321 const unsigned char **inbuf, Py_ssize_t *inleft)
322 {
323 unsigned char charset, designation;
324 Py_ssize_t i, esclen = 0;
325
326 for (i = 1;i < MAX_ESCSEQLEN;i++) {
327 if (i >= *inleft)
328 return MBERR_TOOFEW;
329 if (IS_ESCEND((*inbuf)[i])) {
330 esclen = i + 1;
331 break;
332 }
333 else if (CONFIG_ISSET(USE_JISX0208_EXT) && i+1 < *inleft &&
334 (*inbuf)[i] == '&' && (*inbuf)[i+1] == '@') {
335 i += 2;
336 }
337 }
338
339 switch (esclen) {
340 case 0:
341 return 1; /* unterminated escape sequence */
342 case 3:
343 if (INBYTE2 == '$') {
344 charset = INBYTE3 | CHARSET_DBCS;
345 designation = 0;
346 }
347 else {
348 charset = INBYTE3;
349 if (INBYTE2 == '(')
350 designation = 0;
351 else if (INBYTE2 == ')')
352 designation = 1;
353 else if (CONFIG_ISSET(USE_G2) && INBYTE2 == '.')
354 designation = 2;
355 else
356 return 3;
357 }
358 break;
359 case 4:
360 if (INBYTE2 != '$')
361 return 4;
362
363 charset = INBYTE4 | CHARSET_DBCS;
364 if (INBYTE3 == '(')
365 designation = 0;
366 else if (INBYTE3 == ')')
367 designation = 1;
368 else
369 return 4;
370 break;
371 case 6: /* designation with prefix */
372 if (CONFIG_ISSET(USE_JISX0208_EXT) &&
373 (*inbuf)[3] == ESC && (*inbuf)[4] == '$' &&
374 (*inbuf)[5] == 'B') {
375 charset = 'B' | CHARSET_DBCS;
376 designation = 0;
377 }
378 else
379 return 6;
380 break;
381 default:
382 return esclen;
383 }
384
385 /* raise error when the charset is not designated for this encoding */
386 if (charset != CHARSET_ASCII) {
387 const struct iso2022_designation *dsg;
388
389 for (dsg = CONFIG_DESIGNATIONS; dsg->mark; dsg++) {
390 if (dsg->mark == charset)
391 break;
392 }
393 if (!dsg->mark)
394 return esclen;
395 }
396
397 STATE_SETG(designation, charset);
398 *inleft -= esclen;
399 (*inbuf) += esclen;
400 return 0;
401 }
402
403 #define ISO8859_7_DECODE(c, writer) \
404 if ((c) < 0xa0) { \
405 OUTCHAR(c); \
406 } else if ((c) < 0xc0 && (0x288f3bc9L & (1L << ((c)-0xa0)))) { \
407 OUTCHAR(c); \
408 } else if ((c) >= 0xb4 && (c) <= 0xfe && ((c) >= 0xd4 || \
409 (0xbffffd77L & (1L << ((c)-0xb4))))) { \
410 OUTCHAR(0x02d0 + (c)); \
411 } else if ((c) == 0xa1) { \
412 OUTCHAR(0x2018); \
413 } else if ((c) == 0xa2) { \
414 OUTCHAR(0x2019); \
415 } else if ((c) == 0xaf) { \
416 OUTCHAR(0x2015); \
417 }
418
419 static Py_ssize_t
420 iso2022processg2(const MultibyteCodec *codec, MultibyteCodec_State *state,
421 const unsigned char **inbuf, Py_ssize_t *inleft,
422 _PyUnicodeWriter *writer)
423 {
424 /* not written to use encoder, decoder functions because only few
425 * encodings use G2 designations in CJKCodecs */
426 if (STATE_G2 == CHARSET_ISO8859_1) {
427 if (INBYTE3 < 0x80)
428 OUTCHAR(INBYTE3 + 0x80);
429 else
430 return 3;
431 }
432 else if (STATE_G2 == CHARSET_ISO8859_7) {
433 ISO8859_7_DECODE(INBYTE3 ^ 0x80, writer)
434 else
435 return 3;
436 }
437 else if (STATE_G2 == CHARSET_ASCII) {
438 if (INBYTE3 & 0x80)
439 return 3;
440 else
441 OUTCHAR(INBYTE3);
442 }
443 else
444 return MBERR_INTERNAL;
445
446 (*inbuf) += 3;
447 *inleft -= 3;
448 return 0;
449 }
450
451 DECODER(iso2022)
452 {
453 const struct iso2022_designation *dsgcache = NULL;
454
455 while (inleft > 0) {
456 unsigned char c = INBYTE1;
457 Py_ssize_t err;
458
459 if (STATE_GETFLAG(F_ESCTHROUGHOUT)) {
460 /* ESC throughout mode:
461 * for non-iso2022 escape sequences */
462 OUTCHAR(c); /* assume as ISO-8859-1 */
463 NEXT_IN(1);
464 if (IS_ESCEND(c)) {
465 STATE_CLEARFLAG(F_ESCTHROUGHOUT);
466 }
467 continue;
468 }
469
470 switch (c) {
471 case ESC:
472 REQUIRE_INBUF(2);
473 if (IS_ISO2022ESC(INBYTE2)) {
474 err = iso2022processesc(codec, state,
475 inbuf, &inleft);
476 if (err != 0)
477 return err;
478 }
479 else if (CONFIG_ISSET(USE_G2) && INBYTE2 == 'N') {/* SS2 */
480 REQUIRE_INBUF(3);
481 err = iso2022processg2(codec, state,
482 inbuf, &inleft, writer);
483 if (err != 0)
484 return err;
485 }
486 else {
487 OUTCHAR(ESC);
488 STATE_SETFLAG(F_ESCTHROUGHOUT);
489 NEXT_IN(1);
490 }
491 break;
492 case SI:
493 if (CONFIG_ISSET(NO_SHIFT))
494 goto bypass;
495 STATE_CLEARFLAG(F_SHIFTED);
496 NEXT_IN(1);
497 break;
498 case SO:
499 if (CONFIG_ISSET(NO_SHIFT))
500 goto bypass;
501 STATE_SETFLAG(F_SHIFTED);
502 NEXT_IN(1);
503 break;
504 case LF:
505 STATE_CLEARFLAG(F_SHIFTED);
506 OUTCHAR(LF);
507 NEXT_IN(1);
508 break;
509 default:
510 if (c < 0x20) /* C0 */
511 goto bypass;
512 else if (c >= 0x80)
513 return 1;
514 else {
515 const struct iso2022_designation *dsg;
516 unsigned char charset;
517 Py_UCS4 decoded;
518
519 if (STATE_GETFLAG(F_SHIFTED))
520 charset = STATE_G1;
521 else
522 charset = STATE_G0;
523
524 if (charset == CHARSET_ASCII) {
525 bypass:
526 OUTCHAR(c);
527 NEXT_IN(1);
528 break;
529 }
530
531 if (dsgcache != NULL &&
532 dsgcache->mark == charset)
533 dsg = dsgcache;
534 else {
535 for (dsg = CONFIG_DESIGNATIONS;
536 dsg->mark != charset
537 #ifdef Py_DEBUG
538 && dsg->mark != '\0'
539 #endif
540 ; dsg++)
541 {
542 /* noop */
543 }
544 assert(dsg->mark != '\0');
545 dsgcache = dsg;
546 }
547
548 REQUIRE_INBUF(dsg->width);
549 decoded = dsg->decoder(codec, *inbuf);
550 if (decoded == MAP_UNMAPPABLE)
551 return dsg->width;
552
553 if (decoded < 0x10000) {
554 OUTCHAR(decoded);
555 }
556 else if (decoded < 0x30000) {
557 OUTCHAR(decoded);
558 }
559 else { /* JIS X 0213 pairs */
560 OUTCHAR2(decoded >> 16, decoded & 0xffff);
561 }
562 NEXT_IN(dsg->width);
563 }
564 break;
565 }
566 }
567 return 0;
568 }
569
570 /*-*- mapping access functions -*-*/
571
572 static int
573 ksx1001_init(const MultibyteCodec *codec)
574 {
575 cjkcodecs_module_state *st = codec->modstate;
576 if (IMPORT_MAP(kr, cp949, &st->cp949_encmap, NULL) ||
577 IMPORT_MAP(kr, ksx1001, NULL, &st->ksx1001_decmap))
578 {
579 return -1;
580 }
581 return 0;
582 }
583
584 static Py_UCS4
585 ksx1001_decoder(const MultibyteCodec *codec, const unsigned char *data)
586 {
587 Py_UCS4 u;
588 if (TRYMAP_DEC_ST(ksx1001, u, data[0], data[1]))
589 return u;
590 else
591 return MAP_UNMAPPABLE;
592 }
593
594 static DBCHAR
595 ksx1001_encoder(const MultibyteCodec *codec, const Py_UCS4 *data,
596 Py_ssize_t *length)
597 {
598 DBCHAR coded;
599 assert(*length == 1);
600 if (*data < 0x10000) {
601 if (TRYMAP_ENC_ST(cp949, coded, *data)) {
602 if (!(coded & 0x8000))
603 return coded;
604 }
605 }
606 return MAP_UNMAPPABLE;
607 }
608
609 static int
610 jisx0208_init(const MultibyteCodec *codec)
611 {
612 cjkcodecs_module_state *st = codec->modstate;
613 if (IMPORT_MAP(jp, jisxcommon, &st->jisxcommon_encmap, NULL) ||
614 IMPORT_MAP(jp, jisx0208, NULL, &st->jisx0208_decmap))
615 {
616 return -1;
617 }
618 return 0;
619 }
620
621 static Py_UCS4
622 jisx0208_decoder(const MultibyteCodec *codec, const unsigned char *data)
623 {
624 Py_UCS4 u;
625 if (data[0] == 0x21 && data[1] == 0x40) /* F/W REVERSE SOLIDUS */
626 return 0xff3c;
627 else if (TRYMAP_DEC_ST(jisx0208, u, data[0], data[1]))
628 return u;
629 else
630 return MAP_UNMAPPABLE;
631 }
632
633 static DBCHAR
634 jisx0208_encoder(const MultibyteCodec *codec, const Py_UCS4 *data,
635 Py_ssize_t *length)
636 {
637 DBCHAR coded;
638 assert(*length == 1);
639 if (*data < 0x10000) {
640 if (*data == 0xff3c) /* F/W REVERSE SOLIDUS */
641 return 0x2140;
642 else if (TRYMAP_ENC_ST(jisxcommon, coded, *data)) {
643 if (!(coded & 0x8000))
644 return coded;
645 }
646 }
647 return MAP_UNMAPPABLE;
648 }
649
650 static int
651 jisx0212_init(const MultibyteCodec *codec)
652 {
653 cjkcodecs_module_state *st = codec->modstate;
654 if (IMPORT_MAP(jp, jisxcommon, &st->jisxcommon_encmap, NULL) ||
655 IMPORT_MAP(jp, jisx0212, NULL, &st->jisx0212_decmap))
656 {
657 return -1;
658 }
659 return 0;
660 }
661
662 static Py_UCS4
663 jisx0212_decoder(const MultibyteCodec *codec, const unsigned char *data)
664 {
665 Py_UCS4 u;
666 if (TRYMAP_DEC_ST(jisx0212, u, data[0], data[1]))
667 return u;
668 else
669 return MAP_UNMAPPABLE;
670 }
671
672 static DBCHAR
673 jisx0212_encoder(const MultibyteCodec *codec, const Py_UCS4 *data,
674 Py_ssize_t *length)
675 {
676 DBCHAR coded;
677 assert(*length == 1);
678 if (*data < 0x10000) {
679 if (TRYMAP_ENC_ST(jisxcommon, coded, *data)) {
680 if (coded & 0x8000)
681 return coded & 0x7fff;
682 }
683 }
684 return MAP_UNMAPPABLE;
685 }
686
687 static int
688 jisx0213_init(const MultibyteCodec *codec)
689 {
690 cjkcodecs_module_state *st = codec->modstate;
691 if (jisx0208_init(codec) ||
692 IMPORT_MAP(jp, jisx0213_bmp, &st->jisx0213_bmp_encmap, NULL) ||
693 IMPORT_MAP(jp, jisx0213_1_bmp, NULL, &st->jisx0213_1_bmp_decmap) ||
694 IMPORT_MAP(jp, jisx0213_2_bmp, NULL, &st->jisx0213_2_bmp_decmap) ||
695 IMPORT_MAP(jp, jisx0213_emp, &st->jisx0213_emp_encmap, NULL) ||
696 IMPORT_MAP(jp, jisx0213_1_emp, NULL, &st->jisx0213_1_emp_decmap) ||
697 IMPORT_MAP(jp, jisx0213_2_emp, NULL, &st->jisx0213_2_emp_decmap) ||
698 IMPORT_MAP(jp, jisx0213_pair,
699 &jisx0213_pair_encmap, &jisx0213_pair_decmap))
700 {
701 return -1;
702 }
703 return 0;
704 }
705
706 #define config ((void *)2000)
707 static Py_UCS4
708 jisx0213_2000_1_decoder(const MultibyteCodec *codec, const unsigned char *data)
709 {
710 Py_UCS4 u;
711 EMULATE_JISX0213_2000_DECODE_PLANE1(config, u, data[0], data[1])
712 else if (data[0] == 0x21 && data[1] == 0x40) /* F/W REVERSE SOLIDUS */
713 return 0xff3c;
714 else if (TRYMAP_DEC_ST(jisx0208, u, data[0], data[1]))
715 ;
716 else if (TRYMAP_DEC_ST(jisx0213_1_bmp, u, data[0], data[1]))
717 ;
718 else if (TRYMAP_DEC_ST(jisx0213_1_emp, u, data[0], data[1]))
719 u |= 0x20000;
720 else if (TRYMAP_DEC(jisx0213_pair, u, data[0], data[1]))
721 ;
722 else
723 return MAP_UNMAPPABLE;
724 return u;
725 }
726
727 static Py_UCS4
728 jisx0213_2000_2_decoder(const MultibyteCodec *codec, const unsigned char *data)
729 {
730 Py_UCS4 u;
731 EMULATE_JISX0213_2000_DECODE_PLANE2_CHAR(config, u, data[0], data[1])
732 if (TRYMAP_DEC_ST(jisx0213_2_bmp, u, data[0], data[1]))
733 ;
734 else if (TRYMAP_DEC_ST(jisx0213_2_emp, u, data[0], data[1]))
735 u |= 0x20000;
736 else
737 return MAP_UNMAPPABLE;
738 return u;
739 }
740 #undef config
741
742 static Py_UCS4
743 jisx0213_2004_1_decoder(const MultibyteCodec *codec, const unsigned char *data)
744 {
745 Py_UCS4 u;
746 if (data[0] == 0x21 && data[1] == 0x40) /* F/W REVERSE SOLIDUS */
747 return 0xff3c;
748 else if (TRYMAP_DEC_ST(jisx0208, u, data[0], data[1]))
749 ;
750 else if (TRYMAP_DEC_ST(jisx0213_1_bmp, u, data[0], data[1]))
751 ;
752 else if (TRYMAP_DEC_ST(jisx0213_1_emp, u, data[0], data[1]))
753 u |= 0x20000;
754 else if (TRYMAP_DEC(jisx0213_pair, u, data[0], data[1]))
755 ;
756 else
757 return MAP_UNMAPPABLE;
758 return u;
759 }
760
761 static Py_UCS4
762 jisx0213_2004_2_decoder(const MultibyteCodec *codec, const unsigned char *data)
763 {
764 Py_UCS4 u;
765 if (TRYMAP_DEC_ST(jisx0213_2_bmp, u, data[0], data[1]))
766 ;
767 else if (TRYMAP_DEC_ST(jisx0213_2_emp, u, data[0], data[1]))
768 u |= 0x20000;
769 else
770 return MAP_UNMAPPABLE;
771 return u;
772 }
773
774 static DBCHAR
775 jisx0213_encoder(const MultibyteCodec *codec, const Py_UCS4 *data,
776 Py_ssize_t *length, const void *config)
777 {
778 DBCHAR coded;
779
780 switch (*length) {
781 case 1: /* first character */
782 if (*data >= 0x10000) {
783 if ((*data) >> 16 == 0x20000 >> 16) {
784 EMULATE_JISX0213_2000_ENCODE_EMP(config, coded, *data)
785 else if (TRYMAP_ENC_ST(jisx0213_emp, coded, (*data) & 0xffff))
786 return coded;
787 }
788 return MAP_UNMAPPABLE;
789 }
790
791 EMULATE_JISX0213_2000_ENCODE_BMP(config, coded, *data)
792 else if (TRYMAP_ENC_ST(jisx0213_bmp, coded, *data)) {
793 if (coded == MULTIC)
794 return MAP_MULTIPLE_AVAIL;
795 }
796 else if (TRYMAP_ENC_ST(jisxcommon, coded, *data)) {
797 if (coded & 0x8000)
798 return MAP_UNMAPPABLE;
799 }
800 else
801 return MAP_UNMAPPABLE;
802 return coded;
803
804 case 2: /* second character of unicode pair */
805 if (data[1] != 0) { /* Don't consume null char as part of pair */
806 coded = find_pairencmap((ucs2_t)data[0], (ucs2_t)data[1],
807 jisx0213_pair_encmap, JISX0213_ENCPAIRS);
808 if (coded != DBCINV) {
809 return coded;
810 }
811 }
812 _Py_FALLTHROUGH;
813
814 case -1: /* flush unterminated */
815 *length = 1;
816 coded = find_pairencmap((ucs2_t)data[0], 0,
817 jisx0213_pair_encmap, JISX0213_ENCPAIRS);
818 if (coded == DBCINV)
819 return MAP_UNMAPPABLE;
820 else
821 return coded;
822 break;
823
824 default:
825 return MAP_UNMAPPABLE;
826 }
827 }
828
829 static DBCHAR
830 jisx0213_2000_1_encoder(const MultibyteCodec *codec, const Py_UCS4 *data,
831 Py_ssize_t *length)
832 {
833 DBCHAR coded = jisx0213_encoder(codec, data, length, (void *)2000);
834 if (coded == MAP_UNMAPPABLE || coded == MAP_MULTIPLE_AVAIL)
835 return coded;
836 else if (coded & 0x8000)
837 return MAP_UNMAPPABLE;
838 else
839 return coded;
840 }
841
842 static DBCHAR
843 jisx0213_2000_1_encoder_paironly(const MultibyteCodec *codec,
844 const Py_UCS4 *data, Py_ssize_t *length)
845 {
846 DBCHAR coded;
847 Py_ssize_t ilength = *length;
848
849 coded = jisx0213_encoder(codec, data, length, (void *)2000);
850 switch (ilength) {
851 case 1:
852 if (coded == MAP_MULTIPLE_AVAIL)
853 return MAP_MULTIPLE_AVAIL;
854 else
855 return MAP_UNMAPPABLE;
856 case 2:
857 if (*length != 2)
858 return MAP_UNMAPPABLE;
859 else
860 return coded;
861 default:
862 return MAP_UNMAPPABLE;
863 }
864 }
865
866 static DBCHAR
867 jisx0213_2000_2_encoder(const MultibyteCodec *codec, const Py_UCS4 *data,
868 Py_ssize_t *length)
869 {
870 DBCHAR coded = jisx0213_encoder(codec, data, length, (void *)2000);
871 if (coded == MAP_UNMAPPABLE || coded == MAP_MULTIPLE_AVAIL)
872 return coded;
873 else if (coded & 0x8000)
874 return coded & 0x7fff;
875 else
876 return MAP_UNMAPPABLE;
877 }
878
879 static DBCHAR
880 jisx0213_2004_1_encoder(const MultibyteCodec *codec, const Py_UCS4 *data,
881 Py_ssize_t *length)
882 {
883 DBCHAR coded = jisx0213_encoder(codec, data, length, NULL);
884 if (coded == MAP_UNMAPPABLE || coded == MAP_MULTIPLE_AVAIL)
885 return coded;
886 else if (coded & 0x8000)
887 return MAP_UNMAPPABLE;
888 else
889 return coded;
890 }
891
892 static DBCHAR
893 jisx0213_2004_1_encoder_paironly(const MultibyteCodec *codec,
894 const Py_UCS4 *data, Py_ssize_t *length)
895 {
896 DBCHAR coded;
897 Py_ssize_t ilength = *length;
898
899 coded = jisx0213_encoder(codec, data, length, NULL);
900 switch (ilength) {
901 case 1:
902 if (coded == MAP_MULTIPLE_AVAIL)
903 return MAP_MULTIPLE_AVAIL;
904 else
905 return MAP_UNMAPPABLE;
906 case 2:
907 if (*length != 2)
908 return MAP_UNMAPPABLE;
909 else
910 return coded;
911 default:
912 return MAP_UNMAPPABLE;
913 }
914 }
915
916 static DBCHAR
917 jisx0213_2004_2_encoder(const MultibyteCodec *codec, const Py_UCS4 *data,
918 Py_ssize_t *length)
919 {
920 DBCHAR coded = jisx0213_encoder(codec, data, length, NULL);
921 if (coded == MAP_UNMAPPABLE || coded == MAP_MULTIPLE_AVAIL)
922 return coded;
923 else if (coded & 0x8000)
924 return coded & 0x7fff;
925 else
926 return MAP_UNMAPPABLE;
927 }
928
929 static Py_UCS4
930 jisx0201_r_decoder(const MultibyteCodec *codec, const unsigned char *data)
931 {
932 Py_UCS4 u;
933 JISX0201_R_DECODE_CHAR(*data, u)
934 else
935 return MAP_UNMAPPABLE;
936 return u;
937 }
938
939 static DBCHAR
940 jisx0201_r_encoder(const MultibyteCodec *codec, const Py_UCS4 *data,
941 Py_ssize_t *length)
942 {
943 DBCHAR coded;
944 JISX0201_R_ENCODE(*data, coded)
945 else
946 return MAP_UNMAPPABLE;
947 return coded;
948 }
949
950 static Py_UCS4
951 jisx0201_k_decoder(const MultibyteCodec *codec, const unsigned char *data)
952 {
953 Py_UCS4 u;
954 JISX0201_K_DECODE_CHAR(*data ^ 0x80, u)
955 else
956 return MAP_UNMAPPABLE;
957 return u;
958 }
959
960 static DBCHAR
961 jisx0201_k_encoder(const MultibyteCodec *codec, const Py_UCS4 *data,
962 Py_ssize_t *length)
963 {
964 DBCHAR coded;
965 JISX0201_K_ENCODE(*data, coded)
966 else
967 return MAP_UNMAPPABLE;
968 return coded - 0x80;
969 }
970
971 static int
972 gb2312_init(const MultibyteCodec *codec)
973 {
974 cjkcodecs_module_state *st = codec->modstate;
975 if (IMPORT_MAP(cn, gbcommon, &st->gbcommon_encmap, NULL) ||
976 IMPORT_MAP(cn, gb2312, NULL, &st->gb2312_decmap))
977 {
978 return -1;
979 }
980 return 0;
981 }
982
983 static Py_UCS4
984 gb2312_decoder(const MultibyteCodec *codec, const unsigned char *data)
985 {
986 Py_UCS4 u;
987 if (TRYMAP_DEC_ST(gb2312, u, data[0], data[1]))
988 return u;
989 else
990 return MAP_UNMAPPABLE;
991 }
992
993 static DBCHAR
994 gb2312_encoder(const MultibyteCodec *codec, const Py_UCS4 *data,
995 Py_ssize_t *length)
996 {
997 DBCHAR coded;
998 assert(*length == 1);
999 if (*data < 0x10000) {
1000 if (TRYMAP_ENC_ST(gbcommon, coded, *data)) {
1001 if (!(coded & 0x8000))
1002 return coded;
1003 }
1004 }
1005 return MAP_UNMAPPABLE;
1006 }
1007
1008
1009 static Py_UCS4
1010 dummy_decoder(const MultibyteCodec *codec, const unsigned char *data)
1011 {
1012 return MAP_UNMAPPABLE;
1013 }
1014
1015 static DBCHAR
1016 dummy_encoder(const MultibyteCodec *codec, const Py_UCS4 *data,
1017 Py_ssize_t *length)
1018 {
1019 return MAP_UNMAPPABLE;
1020 }
1021
1022 /*-*- registry tables -*-*/
1023
1024 #define REGISTRY_KSX1001_G0 { CHARSET_KSX1001, 0, 2, \
1025 ksx1001_init, \
1026 ksx1001_decoder, ksx1001_encoder }
1027 #define REGISTRY_KSX1001_G1 { CHARSET_KSX1001, 1, 2, \
1028 ksx1001_init, \
1029 ksx1001_decoder, ksx1001_encoder }
1030 #define REGISTRY_JISX0201_R { CHARSET_JISX0201_R, 0, 1, \
1031 NULL, \
1032 jisx0201_r_decoder, jisx0201_r_encoder }
1033 #define REGISTRY_JISX0201_K { CHARSET_JISX0201_K, 0, 1, \
1034 NULL, \
1035 jisx0201_k_decoder, jisx0201_k_encoder }
1036 #define REGISTRY_JISX0208 { CHARSET_JISX0208, 0, 2, \
1037 jisx0208_init, \
1038 jisx0208_decoder, jisx0208_encoder }
1039 #define REGISTRY_JISX0208_O { CHARSET_JISX0208_O, 0, 2, \
1040 jisx0208_init, \
1041 jisx0208_decoder, jisx0208_encoder }
1042 #define REGISTRY_JISX0212 { CHARSET_JISX0212, 0, 2, \
1043 jisx0212_init, \
1044 jisx0212_decoder, jisx0212_encoder }
1045 #define REGISTRY_JISX0213_2000_1 { CHARSET_JISX0213_2000_1, 0, 2, \
1046 jisx0213_init, \
1047 jisx0213_2000_1_decoder, \
1048 jisx0213_2000_1_encoder }
1049 #define REGISTRY_JISX0213_2000_1_PAIRONLY { CHARSET_JISX0213_2000_1, 0, 2, \
1050 jisx0213_init, \
1051 jisx0213_2000_1_decoder, \
1052 jisx0213_2000_1_encoder_paironly }
1053 #define REGISTRY_JISX0213_2000_2 { CHARSET_JISX0213_2, 0, 2, \
1054 jisx0213_init, \
1055 jisx0213_2000_2_decoder, \
1056 jisx0213_2000_2_encoder }
1057 #define REGISTRY_JISX0213_2004_1 { CHARSET_JISX0213_2004_1, 0, 2, \
1058 jisx0213_init, \
1059 jisx0213_2004_1_decoder, \
1060 jisx0213_2004_1_encoder }
1061 #define REGISTRY_JISX0213_2004_1_PAIRONLY { CHARSET_JISX0213_2004_1, 0, 2, \
1062 jisx0213_init, \
1063 jisx0213_2004_1_decoder, \
1064 jisx0213_2004_1_encoder_paironly }
1065 #define REGISTRY_JISX0213_2004_2 { CHARSET_JISX0213_2, 0, 2, \
1066 jisx0213_init, \
1067 jisx0213_2004_2_decoder, \
1068 jisx0213_2004_2_encoder }
1069 #define REGISTRY_GB2312 { CHARSET_GB2312, 0, 2, \
1070 gb2312_init, \
1071 gb2312_decoder, gb2312_encoder }
1072 #define REGISTRY_CNS11643_1 { CHARSET_CNS11643_1, 1, 2, \
1073 cns11643_init, \
1074 cns11643_1_decoder, cns11643_1_encoder }
1075 #define REGISTRY_CNS11643_2 { CHARSET_CNS11643_2, 2, 2, \
1076 cns11643_init, \
1077 cns11643_2_decoder, cns11643_2_encoder }
1078 #define REGISTRY_ISO8859_1 { CHARSET_ISO8859_1, 2, 1, \
1079 NULL, dummy_decoder, dummy_encoder }
1080 #define REGISTRY_ISO8859_7 { CHARSET_ISO8859_7, 2, 1, \
1081 NULL, dummy_decoder, dummy_encoder }
1082 #define REGISTRY_SENTINEL { 0, }
1083 #define CONFIGDEF(var, attrs) \
1084 static const struct iso2022_config iso2022_##var##_config = { \
1085 attrs, iso2022_##var##_designations \
1086 };
1087
1088 static const struct iso2022_designation iso2022_kr_designations[] = {
1089 REGISTRY_KSX1001_G1, REGISTRY_SENTINEL
1090 };
1091 CONFIGDEF(kr, 0)
1092
1093 static const struct iso2022_designation iso2022_jp_designations[] = {
1094 REGISTRY_JISX0208, REGISTRY_JISX0201_R, REGISTRY_JISX0208_O,
1095 REGISTRY_SENTINEL
1096 };
1097 CONFIGDEF(jp, NO_SHIFT | USE_JISX0208_EXT)
1098
1099 static const struct iso2022_designation iso2022_jp_1_designations[] = {
1100 REGISTRY_JISX0208, REGISTRY_JISX0212, REGISTRY_JISX0201_R,
1101 REGISTRY_JISX0208_O, REGISTRY_SENTINEL
1102 };
1103 CONFIGDEF(jp_1, NO_SHIFT | USE_JISX0208_EXT)
1104
1105 static const struct iso2022_designation iso2022_jp_2_designations[] = {
1106 REGISTRY_JISX0208, REGISTRY_JISX0212, REGISTRY_KSX1001_G0,
1107 REGISTRY_GB2312, REGISTRY_JISX0201_R, REGISTRY_JISX0208_O,
1108 REGISTRY_ISO8859_1, REGISTRY_ISO8859_7, REGISTRY_SENTINEL
1109 };
1110 CONFIGDEF(jp_2, NO_SHIFT | USE_G2 | USE_JISX0208_EXT)
1111
1112 static const struct iso2022_designation iso2022_jp_2004_designations[] = {
1113 REGISTRY_JISX0213_2004_1_PAIRONLY, REGISTRY_JISX0208,
1114 REGISTRY_JISX0213_2004_1, REGISTRY_JISX0213_2004_2, REGISTRY_SENTINEL
1115 };
1116 CONFIGDEF(jp_2004, NO_SHIFT | USE_JISX0208_EXT)
1117
1118 static const struct iso2022_designation iso2022_jp_3_designations[] = {
1119 REGISTRY_JISX0213_2000_1_PAIRONLY, REGISTRY_JISX0208,
1120 REGISTRY_JISX0213_2000_1, REGISTRY_JISX0213_2000_2, REGISTRY_SENTINEL
1121 };
1122 CONFIGDEF(jp_3, NO_SHIFT | USE_JISX0208_EXT)
1123
1124 static const struct iso2022_designation iso2022_jp_ext_designations[] = {
1125 REGISTRY_JISX0208, REGISTRY_JISX0212, REGISTRY_JISX0201_R,
1126 REGISTRY_JISX0201_K, REGISTRY_JISX0208_O, REGISTRY_SENTINEL
1127 };
1128 CONFIGDEF(jp_ext, NO_SHIFT | USE_JISX0208_EXT)
1129
1130
1131 BEGIN_MAPPINGS_LIST(0)
1132 /* no mapping table here */
1133 END_MAPPINGS_LIST
1134
1135 #define ISO2022_CODEC(variation) \
1136 NEXT_CODEC = (MultibyteCodec){ \
1137 "iso2022_" #variation, \
1138 &iso2022_##variation##_config, \
1139 iso2022_codec_init, \
1140 _STATEFUL_METHODS(iso2022) \
1141 };
1142
1143 BEGIN_CODECS_LIST(7)
1144 ISO2022_CODEC(kr)
1145 ISO2022_CODEC(jp)
1146 ISO2022_CODEC(jp_1)
1147 ISO2022_CODEC(jp_2)
1148 ISO2022_CODEC(jp_2004)
1149 ISO2022_CODEC(jp_3)
1150 ISO2022_CODEC(jp_ext)
1151 END_CODECS_LIST
1152
1153 I_AM_A_MODULE_FOR(iso2022)