Debian Codegraph @

Modules/cjkcodecs/cjkcodecs.h

raw
1 /*
2 * cjkcodecs.h: common header for cjkcodecs
3 *
4 * Written by Hye-Shik Chang <[email protected]>
5 */
6
7 #ifndef _CJKCODECS_H_
8 #define _CJKCODECS_H_
9
10 #ifndef Py_BUILD_CORE_BUILTIN
11 # define Py_BUILD_CORE_MODULE 1
12 #endif
13
14 #include "Python.h"
15 #include "multibytecodec.h"
16
17
18 /* a unicode "undefined" code point */
19 #define UNIINV 0xFFFE
20
21 /* internal-use DBCS code points which aren't used by any charsets */
22 #define NOCHAR 0xFFFF
23 #define MULTIC 0xFFFE
24 #define DBCINV 0xFFFD
25
26 /* shorter macros to save source size of mapping tables */
27 #define U UNIINV
28 #define N NOCHAR
29 #define M MULTIC
30 #define D DBCINV
31
32 struct dbcs_index {
33 const ucs2_t *map;
34 unsigned char bottom, top;
35 };
36 typedef struct dbcs_index decode_map;
37
38 struct widedbcs_index {
39 const Py_UCS4 *map;
40 unsigned char bottom, top;
41 };
42 typedef struct widedbcs_index widedecode_map;
43
44 struct unim_index {
45 const DBCHAR *map;
46 unsigned char bottom, top;
47 };
48 typedef struct unim_index encode_map;
49
50 struct unim_index_bytebased {
51 const unsigned char *map;
52 unsigned char bottom, top;
53 };
54
55 struct dbcs_map {
56 const char *charset;
57 const struct unim_index *encmap;
58 const struct dbcs_index *decmap;
59 };
60
61 struct pair_encodemap {
62 Py_UCS4 uniseq;
63 DBCHAR code;
64 };
65
66 #ifndef CJK_MOD_SPECIFIC_STATE
67 #define CJK_MOD_SPECIFIC_STATE
68 #endif
69
70 typedef struct _cjk_mod_state {
71 int num_mappings;
72 int num_codecs;
73 struct dbcs_map *mapping_list;
74 MultibyteCodec *codec_list;
75
76 CJK_MOD_SPECIFIC_STATE
77 } cjkcodecs_module_state;
78
79 static inline cjkcodecs_module_state *
80 get_module_state(PyObject *mod)
81 {
82 void *state = PyModule_GetState(mod);
83 assert(state != NULL);
84 return (cjkcodecs_module_state *)state;
85 }
86
87 #define CODEC_INIT(encoding) \
88 static int encoding##_codec_init(const MultibyteCodec *codec)
89
90 #define ENCODER_INIT(encoding) \
91 static int encoding##_encode_init( \
92 MultibyteCodec_State *state, const MultibyteCodec *codec)
93 #define ENCODER(encoding) \
94 static Py_ssize_t encoding##_encode( \
95 MultibyteCodec_State *state, const MultibyteCodec *codec, \
96 int kind, const void *data, \
97 Py_ssize_t *inpos, Py_ssize_t inlen, \
98 unsigned char **outbuf, Py_ssize_t outleft, int flags)
99 #define ENCODER_RESET(encoding) \
100 static Py_ssize_t encoding##_encode_reset( \
101 MultibyteCodec_State *state, const MultibyteCodec *codec, \
102 unsigned char **outbuf, Py_ssize_t outleft)
103
104 #define DECODER_INIT(encoding) \
105 static int encoding##_decode_init( \
106 MultibyteCodec_State *state, const MultibyteCodec *codec)
107 #define DECODER(encoding) \
108 static Py_ssize_t encoding##_decode( \
109 MultibyteCodec_State *state, const MultibyteCodec *codec, \
110 const unsigned char **inbuf, Py_ssize_t inleft, \
111 _PyUnicodeWriter *writer)
112 #define DECODER_RESET(encoding) \
113 static Py_ssize_t encoding##_decode_reset( \
114 MultibyteCodec_State *state, const MultibyteCodec *codec)
115
116 #define NEXT_IN(i) \
117 do { \
118 (*inbuf) += (i); \
119 (inleft) -= (i); \
120 } while (0)
121 #define NEXT_INCHAR(i) \
122 do { \
123 (*inpos) += (i); \
124 } while (0)
125 #define NEXT_OUT(o) \
126 do { \
127 (*outbuf) += (o); \
128 (outleft) -= (o); \
129 } while (0)
130 #define NEXT(i, o) \
131 do { \
132 NEXT_INCHAR(i); \
133 NEXT_OUT(o); \
134 } while (0)
135
136 #define REQUIRE_INBUF(n) \
137 do { \
138 if (inleft < (n)) \
139 return MBERR_TOOFEW; \
140 } while (0)
141
142 #define REQUIRE_OUTBUF(n) \
143 do { \
144 if (outleft < (n)) \
145 return MBERR_TOOSMALL; \
146 } while (0)
147
148 #define INBYTE1 ((*inbuf)[0])
149 #define INBYTE2 ((*inbuf)[1])
150 #define INBYTE3 ((*inbuf)[2])
151 #define INBYTE4 ((*inbuf)[3])
152
153 #define INCHAR1 (PyUnicode_READ(kind, data, *inpos))
154 #define INCHAR2 (PyUnicode_READ(kind, data, *inpos + 1))
155
156 #define OUTCHAR(c) \
157 do { \
158 if (_PyUnicodeWriter_WriteChar(writer, (c)) < 0) \
159 return MBERR_EXCEPTION; \
160 } while (0)
161
162 #define OUTCHAR2(c1, c2) \
163 do { \
164 Py_UCS4 _c1 = (c1); \
165 Py_UCS4 _c2 = (c2); \
166 if (_PyUnicodeWriter_Prepare(writer, 2, Py_MAX(_c1, c2)) < 0) \
167 return MBERR_EXCEPTION; \
168 PyUnicode_WRITE(writer->kind, writer->data, writer->pos, _c1); \
169 PyUnicode_WRITE(writer->kind, writer->data, writer->pos + 1, _c2); \
170 writer->pos += 2; \
171 } while (0)
172
173 #define OUTBYTEI(c, i) \
174 do { \
175 assert((unsigned char)(c) == (c)); \
176 ((*outbuf)[i]) = (c); \
177 } while (0)
178
179 #define OUTBYTE1(c) OUTBYTEI(c, 0)
180 #define OUTBYTE2(c) OUTBYTEI(c, 1)
181 #define OUTBYTE3(c) OUTBYTEI(c, 2)
182 #define OUTBYTE4(c) OUTBYTEI(c, 3)
183
184 #define WRITEBYTE1(c1) \
185 do { \
186 REQUIRE_OUTBUF(1); \
187 OUTBYTE1(c1); \
188 } while (0)
189 #define WRITEBYTE2(c1, c2) \
190 do { \
191 REQUIRE_OUTBUF(2); \
192 OUTBYTE1(c1); \
193 OUTBYTE2(c2); \
194 } while (0)
195 #define WRITEBYTE3(c1, c2, c3) \
196 do { \
197 REQUIRE_OUTBUF(3); \
198 OUTBYTE1(c1); \
199 OUTBYTE2(c2); \
200 OUTBYTE3(c3); \
201 } while (0)
202 #define WRITEBYTE4(c1, c2, c3, c4) \
203 do { \
204 REQUIRE_OUTBUF(4); \
205 OUTBYTE1(c1); \
206 OUTBYTE2(c2); \
207 OUTBYTE3(c3); \
208 OUTBYTE4(c4); \
209 } while (0)
210
211 #define _TRYMAP_ENC(m, assi, val) \
212 ((m)->map != NULL && (val) >= (m)->bottom && \
213 (val)<= (m)->top && ((assi) = (m)->map[(val) - \
214 (m)->bottom]) != NOCHAR)
215 #define TRYMAP_ENC(charset, assi, uni) \
216 _TRYMAP_ENC(&charset##_encmap[(uni) >> 8], assi, (uni) & 0xff)
217 #define TRYMAP_ENC_ST(charset, assi, uni) \
218 _TRYMAP_ENC(&(codec->modstate->charset##_encmap)[(uni) >> 8], \
219 assi, (uni) & 0xff)
220
221 #define _TRYMAP_DEC(m, assi, val) \
222 ((m)->map != NULL && \
223 (val) >= (m)->bottom && \
224 (val)<= (m)->top && \
225 ((assi) = (m)->map[(val) - (m)->bottom]) != UNIINV)
226 #define TRYMAP_DEC(charset, assi, c1, c2) \
227 _TRYMAP_DEC(&charset##_decmap[c1], assi, c2)
228 #define TRYMAP_DEC_ST(charset, assi, c1, c2) \
229 _TRYMAP_DEC(&(codec->modstate->charset##_decmap)[c1], assi, c2)
230
231 #define BEGIN_MAPPINGS_LIST(NUM) \
232 static int \
233 add_mappings(cjkcodecs_module_state *st) \
234 { \
235 int idx = 0; \
236 (void)idx; \
237 st->num_mappings = NUM; \
238 st->mapping_list = PyMem_Calloc(NUM, sizeof(struct dbcs_map)); \
239 if (st->mapping_list == NULL) { \
240 return -1; \
241 }
242
243 #define MAPPING_ENCONLY(enc) \
244 st->mapping_list[idx++] = (struct dbcs_map){#enc, (void*)enc##_encmap, NULL};
245 #define MAPPING_DECONLY(enc) \
246 st->mapping_list[idx++] = (struct dbcs_map){#enc, NULL, (void*)enc##_decmap};
247 #define MAPPING_ENCDEC(enc) \
248 st->mapping_list[idx++] = (struct dbcs_map){#enc, (void*)enc##_encmap, (void*)enc##_decmap};
249
250 #define END_MAPPINGS_LIST \
251 assert(st->num_mappings == idx); \
252 return 0; \
253 }
254
255 #define BEGIN_CODECS_LIST(NUM) \
256 static int \
257 add_codecs(cjkcodecs_module_state *st) \
258 { \
259 int idx = 0; \
260 (void)idx; \
261 st->num_codecs = NUM; \
262 st->codec_list = PyMem_Calloc(NUM, sizeof(MultibyteCodec)); \
263 if (st->codec_list == NULL) { \
264 return -1; \
265 }
266
267 #define _STATEFUL_METHODS(enc) \
268 enc##_encode, \
269 enc##_encode_init, \
270 enc##_encode_reset, \
271 enc##_decode, \
272 enc##_decode_init, \
273 enc##_decode_reset,
274 #define _STATELESS_METHODS(enc) \
275 enc##_encode, NULL, NULL, \
276 enc##_decode, NULL, NULL,
277
278 #define NEXT_CODEC \
279 st->codec_list[idx++]
280
281 #define CODEC_STATEFUL(enc) \
282 NEXT_CODEC = (MultibyteCodec){#enc, NULL, NULL, _STATEFUL_METHODS(enc)};
283 #define CODEC_STATELESS(enc) \
284 NEXT_CODEC = (MultibyteCodec){#enc, NULL, NULL, _STATELESS_METHODS(enc)};
285 #define CODEC_STATELESS_WINIT(enc) \
286 NEXT_CODEC = (MultibyteCodec){#enc, NULL, enc##_codec_init, _STATELESS_METHODS(enc)};
287
288 #define END_CODECS_LIST \
289 assert(st->num_codecs == idx); \
290 for (int i = 0; i < st->num_codecs; i++) { \
291 st->codec_list[i].modstate = st; \
292 } \
293 return 0; \
294 }
295
296
297
298 static PyObject *
299 getmultibytecodec(void)
300 {
301 return PyImport_ImportModuleAttrString("_multibytecodec", "__create_codec");
302 }
303
304 static void
305 destroy_codec_capsule(PyObject *capsule)
306 {
307 void *ptr = PyCapsule_GetPointer(capsule, CODEC_CAPSULE);
308 codec_capsule *data = (codec_capsule *)ptr;
309 Py_DECREF(data->cjk_module);
310 PyMem_Free(ptr);
311 }
312
313 static codec_capsule *
314 capsulate_codec(PyObject *mod, const MultibyteCodec *codec)
315 {
316 codec_capsule *data = PyMem_Malloc(sizeof(codec_capsule));
317 if (data == NULL) {
318 PyErr_NoMemory();
319 return NULL;
320 }
321 data->codec = codec;
322 data->cjk_module = Py_NewRef(mod);
323 return data;
324 }
325
326 static PyObject *
327 _getcodec(PyObject *self, const MultibyteCodec *codec)
328 {
329 PyObject *cofunc = getmultibytecodec();
330 if (cofunc == NULL) {
331 return NULL;
332 }
333
334 codec_capsule *data = capsulate_codec(self, codec);
335 if (data == NULL) {
336 Py_DECREF(cofunc);
337 return NULL;
338 }
339 PyObject *codecobj = PyCapsule_New(data, CODEC_CAPSULE,
340 destroy_codec_capsule);
341 if (codecobj == NULL) {
342 PyMem_Free(data);
343 Py_DECREF(cofunc);
344 return NULL;
345 }
346
347 PyObject *res = PyObject_CallOneArg(cofunc, codecobj);
348 Py_DECREF(codecobj);
349 Py_DECREF(cofunc);
350 return res;
351 }
352
353 static PyObject *
354 getcodec(PyObject *self, PyObject *encoding)
355 {
356 if (!PyUnicode_Check(encoding)) {
357 PyErr_SetString(PyExc_TypeError,
358 "encoding name must be a string.");
359 return NULL;
360 }
361 const char *enc = PyUnicode_AsUTF8(encoding);
362 if (enc == NULL) {
363 return NULL;
364 }
365
366 cjkcodecs_module_state *st = get_module_state(self);
367 for (int i = 0; i < st->num_codecs; i++) {
368 const MultibyteCodec *codec = &st->codec_list[i];
369 if (strcmp(codec->encoding, enc) == 0) {
370 return _getcodec(self, codec);
371 }
372 }
373
374 PyErr_SetString(PyExc_LookupError,
375 "no such codec is supported.");
376 return NULL;
377 }
378
379 static int add_mappings(cjkcodecs_module_state *);
380 static int add_codecs(cjkcodecs_module_state *);
381
382 static int
383 register_maps(PyObject *module)
384 {
385 // Init module state.
386 cjkcodecs_module_state *st = get_module_state(module);
387 if (add_mappings(st) < 0) {
388 return -1;
389 }
390 if (add_codecs(st) < 0) {
391 return -1;
392 }
393
394 for (int i = 0; i < st->num_mappings; i++) {
395 const struct dbcs_map *h = &st->mapping_list[i];
396 char mhname[256] = "__map_";
397 strcpy(mhname + sizeof("__map_") - 1, h->charset);
398
399 PyObject *capsule = PyCapsule_New((void *)h, MAP_CAPSULE, NULL);
400 if (PyModule_Add(module, mhname, capsule) < 0) {
401 return -1;
402 }
403 }
404 return 0;
405 }
406
407 #ifdef USING_BINARY_PAIR_SEARCH
408 static DBCHAR
409 find_pairencmap(ucs2_t body, ucs2_t modifier,
410 const struct pair_encodemap *haystack, int haystacksize)
411 {
412 int pos, min, max;
413 Py_UCS4 value = body << 16 | modifier;
414
415 min = 0;
416 max = haystacksize;
417
418 for (pos = haystacksize >> 1; min != max; pos = (min + max) >> 1) {
419 if (value < haystack[pos].uniseq) {
420 if (max != pos) {
421 max = pos;
422 continue;
423 }
424 }
425 else if (value > haystack[pos].uniseq) {
426 if (min != pos) {
427 min = pos;
428 continue;
429 }
430 }
431 break;
432 }
433
434 if (value == haystack[pos].uniseq) {
435 return haystack[pos].code;
436 }
437 return DBCINV;
438 }
439 #endif
440
441 #ifdef USING_IMPORTED_MAPS
442 #define IMPORT_MAP(locale, charset, encmap, decmap) \
443 importmap("_codecs_" #locale, "__map_" #charset, \
444 (const void**)encmap, (const void**)decmap)
445
446 static int
447 importmap(const char *modname, const char *symbol,
448 const void **encmap, const void **decmap)
449 {
450 PyObject *o, *mod;
451
452 mod = PyImport_ImportModule(modname);
453 if (mod == NULL)
454 return -1;
455
456 o = PyObject_GetAttrString(mod, symbol);
457 if (o == NULL)
458 goto errorexit;
459 else if (!PyCapsule_IsValid(o, MAP_CAPSULE)) {
460 PyErr_SetString(PyExc_ValueError,
461 "map data must be a Capsule.");
462 goto errorexit;
463 }
464 else {
465 struct dbcs_map *map;
466 map = PyCapsule_GetPointer(o, MAP_CAPSULE);
467 if (encmap != NULL)
468 *encmap = map->encmap;
469 if (decmap != NULL)
470 *decmap = map->decmap;
471 Py_DECREF(o);
472 }
473
474 Py_DECREF(mod);
475 return 0;
476
477 errorexit:
478 Py_DECREF(mod);
479 return -1;
480 }
481 #endif
482
483 static int
484 _cjk_exec(PyObject *module)
485 {
486 return register_maps(module);
487 }
488
489 static void
490 _cjk_free(void *mod)
491 {
492 cjkcodecs_module_state *st = get_module_state((PyObject *)mod);
493 PyMem_Free(st->mapping_list);
494 PyMem_Free(st->codec_list);
495 }
496
497 static struct PyMethodDef _cjk_methods[] = {
498 {"getcodec", getcodec, METH_O, ""},
499 {NULL, NULL},
500 };
501
502 static PyModuleDef_Slot _cjk_slots[] = {
503 {Py_mod_exec, _cjk_exec},
504 {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
505 {Py_mod_gil, Py_MOD_GIL_NOT_USED},
506 {0, NULL}
507 };
508
509 #define I_AM_A_MODULE_FOR(loc) \
510 static struct PyModuleDef _cjk_module = { \
511 PyModuleDef_HEAD_INIT, \
512 .m_name = "_codecs_"#loc, \
513 .m_size = sizeof(cjkcodecs_module_state), \
514 .m_methods = _cjk_methods, \
515 .m_slots = _cjk_slots, \
516 .m_free = _cjk_free, \
517 }; \
518 \
519 PyMODINIT_FUNC \
520 PyInit__codecs_##loc(void) \
521 { \
522 return PyModuleDef_Init(&_cjk_module); \
523 }
524
525 #endif