Debian Codegraph @

Modules/cjkcodecs/multibytecodec.c

raw
1 /*
2 * multibytecodec.c: Common Multibyte Codec Implementation
3 *
4 * Written by Hye-Shik Chang <[email protected]>
5 */
6
7 #ifndef Py_BUILD_CORE_BUILTIN
8 # define Py_BUILD_CORE_MODULE 1
9 #endif
10
11 #include "Python.h"
12
13 #include "multibytecodec.h"
14 #include "clinic/multibytecodec.c.h"
15
16 #include <stddef.h> // offsetof()
17
18 #define MODULE_NAME "_multibytecodec"
19
20 typedef struct {
21 PyTypeObject *encoder_type;
22 PyTypeObject *decoder_type;
23 PyTypeObject *reader_type;
24 PyTypeObject *writer_type;
25 PyTypeObject *multibytecodec_type;
26 PyObject *str_write;
27 } module_state;
28
29 static module_state *
30 get_module_state(PyObject *module)
31 {
32 module_state *state = PyModule_GetState(module);
33 assert(state != NULL);
34 return state;
35 }
36
37 static struct PyModuleDef _multibytecodecmodule;
38
39 static module_state *
40 find_state_by_def(PyTypeObject *type)
41 {
42 PyObject *module = PyType_GetModuleByDef(type, &_multibytecodecmodule);
43 assert(module != NULL);
44 return get_module_state(module);
45 }
46
47 #define clinic_get_state() find_state_by_def(type)
48 /*[clinic input]
49 module _multibytecodec
50 class _multibytecodec.MultibyteCodec "MultibyteCodecObject *" "clinic_get_state()->multibytecodec_type"
51 class _multibytecodec.MultibyteIncrementalEncoder "MultibyteIncrementalEncoderObject *" "clinic_get_state()->encoder_type"
52 class _multibytecodec.MultibyteIncrementalDecoder "MultibyteIncrementalDecoderObject *" "clinic_get_state()->decoder_type"
53 class _multibytecodec.MultibyteStreamReader "MultibyteStreamReaderObject *" "clinic_get_state()->reader_type"
54 class _multibytecodec.MultibyteStreamWriter "MultibyteStreamWriterObject *" "clinic_get_state()->writer_type"
55 [clinic start generated code]*/
56 /*[clinic end generated code: output=da39a3ee5e6b4b0d input=305a76dfdd24b99c]*/
57 #undef clinic_get_state
58
59 #define _MultibyteCodec_CAST(op) ((MultibyteCodec *)(op))
60 #define _MultibyteCodecObject_CAST(op) ((MultibyteCodecObject *)(op))
61
62 #define _MultibyteStatefulCodecContext_CAST(op) \
63 ((MultibyteStatefulCodecContext *)(op))
64
65 #define _MultibyteStatefulEncoderContext_CAST(op) \
66 ((MultibyteStatefulEncoderContext *)(op))
67 #define _MultibyteStatefulDecoderContext_CAST(op) \
68 ((MultibyteStatefulDecoderContext *)(op))
69
70 #define _MultibyteIncrementalEncoderObject_CAST(op) \
71 ((MultibyteIncrementalEncoderObject *)(op))
72 #define _MultibyteIncrementalDecoderObject_CAST(op) \
73 ((MultibyteIncrementalDecoderObject *)(op))
74
75 #define _MultibyteStreamReaderObject_CAST(op) \
76 ((MultibyteStreamReaderObject *)(op))
77 #define _MultibyteStreamWriterObject_CAST(op) \
78 ((MultibyteStreamWriterObject *)(op))
79
80 typedef struct {
81 PyObject *inobj;
82 Py_ssize_t inpos, inlen;
83 unsigned char *outbuf, *outbuf_end;
84 PyObject *excobj, *outobj;
85 } MultibyteEncodeBuffer;
86
87 typedef struct {
88 const unsigned char *inbuf, *inbuf_top, *inbuf_end;
89 PyObject *excobj;
90 _PyUnicodeWriter writer;
91 } MultibyteDecodeBuffer;
92
93 static char *incnewkwarglist[] = {"errors", NULL};
94 static char *streamkwarglist[] = {"stream", "errors", NULL};
95
96 static PyObject *multibytecodec_encode(const MultibyteCodec *,
97 MultibyteCodec_State *, PyObject *, Py_ssize_t *,
98 PyObject *, int);
99
100 #define MBENC_RESET MBENC_MAX<<1 /* reset after an encoding session */
101
102 static PyObject *
103 make_tuple(PyObject *object, Py_ssize_t len)
104 {
105 PyObject *v, *w;
106
107 if (object == NULL)
108 return NULL;
109
110 v = PyTuple_New(2);
111 if (v == NULL) {
112 Py_DECREF(object);
113 return NULL;
114 }
115 PyTuple_SET_ITEM(v, 0, object);
116
117 w = PyLong_FromSsize_t(len);
118 if (w == NULL) {
119 Py_DECREF(v);
120 return NULL;
121 }
122 PyTuple_SET_ITEM(v, 1, w);
123
124 return v;
125 }
126
127 static PyObject *
128 internal_error_callback(const char *errors)
129 {
130 if (errors == NULL || strcmp(errors, "strict") == 0)
131 return ERROR_STRICT;
132 else if (strcmp(errors, "ignore") == 0)
133 return ERROR_IGNORE;
134 else if (strcmp(errors, "replace") == 0)
135 return ERROR_REPLACE;
136 else
137 return PyUnicode_FromString(errors);
138 }
139
140 static PyObject *
141 call_error_callback(PyObject *errors, PyObject *exc)
142 {
143 PyObject *cb, *r;
144 const char *str;
145
146 assert(PyUnicode_Check(errors));
147 str = PyUnicode_AsUTF8(errors);
148 if (str == NULL)
149 return NULL;
150 cb = PyCodec_LookupError(str);
151 if (cb == NULL)
152 return NULL;
153
154 r = PyObject_CallOneArg(cb, exc);
155 Py_DECREF(cb);
156 return r;
157 }
158
159 static PyObject *
160 codecctx_errors_get(PyObject *op, void *Py_UNUSED(closure))
161 {
162 const char *errors;
163 MultibyteStatefulCodecContext *self = _MultibyteStatefulCodecContext_CAST(op);
164
165 if (self->errors == ERROR_STRICT)
166 errors = "strict";
167 else if (self->errors == ERROR_IGNORE)
168 errors = "ignore";
169 else if (self->errors == ERROR_REPLACE)
170 errors = "replace";
171 else {
172 return Py_NewRef(self->errors);
173 }
174
175 return PyUnicode_FromString(errors);
176 }
177
178 static int
179 codecctx_errors_set(PyObject *op, PyObject *value, void *Py_UNUSED(closure))
180 {
181 PyObject *cb;
182 const char *str;
183 MultibyteStatefulCodecContext *self = _MultibyteStatefulCodecContext_CAST(op);
184
185 if (value == NULL) {
186 PyErr_SetString(PyExc_AttributeError, "cannot delete attribute");
187 return -1;
188 }
189 if (!PyUnicode_Check(value)) {
190 PyErr_SetString(PyExc_TypeError, "errors must be a string");
191 return -1;
192 }
193
194 str = PyUnicode_AsUTF8(value);
195 if (str == NULL)
196 return -1;
197
198 cb = internal_error_callback(str);
199 if (cb == NULL)
200 return -1;
201
202 ERROR_DECREF(self->errors);
203 self->errors = cb;
204 return 0;
205 }
206
207 /* This getset handlers list is used by all the stateful codec objects */
208 static PyGetSetDef codecctx_getsets[] = {
209 {"errors", codecctx_errors_get, codecctx_errors_set,
210 PyDoc_STR("how to treat errors")},
211 {NULL,}
212 };
213
214 static int
215 expand_encodebuffer(MultibyteEncodeBuffer *buf, Py_ssize_t esize)
216 {
217 Py_ssize_t orgpos, orgsize, incsize;
218
219 orgpos = (Py_ssize_t)((char *)buf->outbuf -
220 PyBytes_AS_STRING(buf->outobj));
221 orgsize = PyBytes_GET_SIZE(buf->outobj);
222 incsize = (esize < (orgsize >> 1) ? (orgsize >> 1) | 1 : esize);
223
224 if (orgsize > PY_SSIZE_T_MAX - incsize) {
225 PyErr_NoMemory();
226 return -1;
227 }
228
229 if (_PyBytes_Resize(&buf->outobj, orgsize + incsize) == -1)
230 return -1;
231
232 buf->outbuf = (unsigned char *)PyBytes_AS_STRING(buf->outobj) +orgpos;
233 buf->outbuf_end = (unsigned char *)PyBytes_AS_STRING(buf->outobj)
234 + PyBytes_GET_SIZE(buf->outobj);
235
236 return 0;
237 }
238 #define REQUIRE_ENCODEBUFFER(buf, s) do { \
239 if ((s) < 0 || (s) > (buf)->outbuf_end - (buf)->outbuf) \
240 if (expand_encodebuffer(buf, s) == -1) \
241 goto errorexit; \
242 } while(0)
243
244
245 /**
246 * MultibyteCodec object
247 */
248
249 static int
250 multibytecodec_encerror(const MultibyteCodec *codec,
251 MultibyteCodec_State *state,
252 MultibyteEncodeBuffer *buf,
253 PyObject *errors, Py_ssize_t e)
254 {
255 PyObject *retobj = NULL, *retstr = NULL, *tobj;
256 Py_ssize_t retstrsize, newpos;
257 Py_ssize_t esize, start, end;
258 const char *reason;
259
260 if (e > 0) {
261 reason = "illegal multibyte sequence";
262 esize = e;
263 }
264 else {
265 switch (e) {
266 case MBERR_TOOSMALL:
267 REQUIRE_ENCODEBUFFER(buf, -1);
268 return 0; /* retry it */
269 case MBERR_TOOFEW:
270 reason = "incomplete multibyte sequence";
271 esize = (Py_ssize_t)buf->inpos;
272 break;
273 case MBERR_INTERNAL:
274 PyErr_SetString(PyExc_RuntimeError,
275 "internal codec error");
276 return -1;
277 default:
278 PyErr_SetString(PyExc_RuntimeError,
279 "unknown runtime error");
280 return -1;
281 }
282 }
283
284 if (errors == ERROR_REPLACE) {
285 PyObject *replchar;
286 Py_ssize_t r;
287 Py_ssize_t inpos;
288 int kind;
289 const void *data;
290
291 replchar = PyUnicode_FromOrdinal('?');
292 if (replchar == NULL)
293 goto errorexit;
294 kind = PyUnicode_KIND(replchar);
295 data = PyUnicode_DATA(replchar);
296
297 inpos = 0;
298 for (;;) {
299 Py_ssize_t outleft = (Py_ssize_t)(buf->outbuf_end - buf->outbuf);
300
301 r = codec->encode(state, codec,
302 kind, data, &inpos, 1,
303 &buf->outbuf, outleft, 0);
304 if (r == MBERR_TOOSMALL) {
305 REQUIRE_ENCODEBUFFER(buf, -1);
306 continue;
307 }
308 else
309 break;
310 }
311
312 Py_DECREF(replchar);
313
314 if (r != 0) {
315 REQUIRE_ENCODEBUFFER(buf, 1);
316 *buf->outbuf++ = '?';
317 }
318 }
319 if (errors == ERROR_IGNORE || errors == ERROR_REPLACE) {
320 buf->inpos += esize;
321 return 0;
322 }
323
324 start = (Py_ssize_t)buf->inpos;
325 end = start + esize;
326
327 /* use cached exception object if available */
328 if (buf->excobj == NULL) {
329 buf->excobj = PyObject_CallFunction(PyExc_UnicodeEncodeError,
330 "sOnns",
331 codec->encoding, buf->inobj,
332 start, end, reason);
333 if (buf->excobj == NULL)
334 goto errorexit;
335 }
336 else
337 if (PyUnicodeEncodeError_SetStart(buf->excobj, start) != 0 ||
338 PyUnicodeEncodeError_SetEnd(buf->excobj, end) != 0 ||
339 PyUnicodeEncodeError_SetReason(buf->excobj, reason) != 0)
340 goto errorexit;
341
342 if (errors == ERROR_STRICT) {
343 PyCodec_StrictErrors(buf->excobj);
344 goto errorexit;
345 }
346
347 retobj = call_error_callback(errors, buf->excobj);
348 if (retobj == NULL)
349 goto errorexit;
350
351 if (!PyTuple_Check(retobj) || PyTuple_GET_SIZE(retobj) != 2 ||
352 (!PyUnicode_Check((tobj = PyTuple_GET_ITEM(retobj, 0))) && !PyBytes_Check(tobj)) ||
353 !PyLong_Check(PyTuple_GET_ITEM(retobj, 1))) {
354 PyErr_SetString(PyExc_TypeError,
355 "encoding error handler must return "
356 "(str, int) tuple");
357 goto errorexit;
358 }
359
360 if (PyUnicode_Check(tobj)) {
361 Py_ssize_t inpos;
362
363 retstr = multibytecodec_encode(codec, state, tobj,
364 &inpos, ERROR_STRICT,
365 MBENC_FLUSH);
366 if (retstr == NULL)
367 goto errorexit;
368 }
369 else {
370 retstr = Py_NewRef(tobj);
371 }
372
373 assert(PyBytes_Check(retstr));
374 retstrsize = PyBytes_GET_SIZE(retstr);
375 if (retstrsize > 0) {
376 REQUIRE_ENCODEBUFFER(buf, retstrsize);
377 memcpy(buf->outbuf, PyBytes_AS_STRING(retstr), retstrsize);
378 buf->outbuf += retstrsize;
379 }
380
381 newpos = PyLong_AsSsize_t(PyTuple_GET_ITEM(retobj, 1));
382 if (newpos < 0 && !PyErr_Occurred())
383 newpos += (Py_ssize_t)buf->inlen;
384 if (newpos < 0 || newpos > buf->inlen) {
385 PyErr_Clear();
386 PyErr_Format(PyExc_IndexError,
387 "position %zd from error handler out of bounds",
388 newpos);
389 goto errorexit;
390 }
391 buf->inpos = newpos;
392
393 Py_DECREF(retobj);
394 Py_DECREF(retstr);
395 return 0;
396
397 errorexit:
398 Py_XDECREF(retobj);
399 Py_XDECREF(retstr);
400 return -1;
401 }
402
403 static int
404 multibytecodec_decerror(const MultibyteCodec *codec,
405 MultibyteCodec_State *state,
406 MultibyteDecodeBuffer *buf,
407 PyObject *errors, Py_ssize_t e)
408 {
409 PyObject *retobj = NULL, *retuni = NULL;
410 Py_ssize_t newpos;
411 const char *reason;
412 Py_ssize_t esize, start, end;
413
414 if (e > 0) {
415 reason = "illegal multibyte sequence";
416 esize = e;
417 }
418 else {
419 switch (e) {
420 case MBERR_TOOSMALL:
421 return 0; /* retry it */
422 case MBERR_TOOFEW:
423 reason = "incomplete multibyte sequence";
424 esize = (Py_ssize_t)(buf->inbuf_end - buf->inbuf);
425 break;
426 case MBERR_INTERNAL:
427 PyErr_SetString(PyExc_RuntimeError,
428 "internal codec error");
429 return -1;
430 case MBERR_EXCEPTION:
431 return -1;
432 default:
433 PyErr_SetString(PyExc_RuntimeError,
434 "unknown runtime error");
435 return -1;
436 }
437 }
438
439 if (errors == ERROR_REPLACE) {
440 if (_PyUnicodeWriter_WriteChar(&buf->writer,
441 Py_UNICODE_REPLACEMENT_CHARACTER) < 0)
442 goto errorexit;
443 }
444 if (errors == ERROR_IGNORE || errors == ERROR_REPLACE) {
445 buf->inbuf += esize;
446 return 0;
447 }
448
449 start = (Py_ssize_t)(buf->inbuf - buf->inbuf_top);
450 end = start + esize;
451
452 /* use cached exception object if available */
453 if (buf->excobj == NULL) {
454 buf->excobj = PyUnicodeDecodeError_Create(codec->encoding,
455 (const char *)buf->inbuf_top,
456 (Py_ssize_t)(buf->inbuf_end - buf->inbuf_top),
457 start, end, reason);
458 if (buf->excobj == NULL)
459 goto errorexit;
460 }
461 else
462 if (PyUnicodeDecodeError_SetStart(buf->excobj, start) ||
463 PyUnicodeDecodeError_SetEnd(buf->excobj, end) ||
464 PyUnicodeDecodeError_SetReason(buf->excobj, reason))
465 goto errorexit;
466
467 if (errors == ERROR_STRICT) {
468 PyCodec_StrictErrors(buf->excobj);
469 goto errorexit;
470 }
471
472 retobj = call_error_callback(errors, buf->excobj);
473 if (retobj == NULL)
474 goto errorexit;
475
476 if (!PyTuple_Check(retobj) || PyTuple_GET_SIZE(retobj) != 2 ||
477 !PyUnicode_Check((retuni = PyTuple_GET_ITEM(retobj, 0))) ||
478 !PyLong_Check(PyTuple_GET_ITEM(retobj, 1))) {
479 PyErr_SetString(PyExc_TypeError,
480 "decoding error handler must return "
481 "(str, int) tuple");
482 goto errorexit;
483 }
484
485 if (_PyUnicodeWriter_WriteStr(&buf->writer, retuni) < 0)
486 goto errorexit;
487
488 newpos = PyLong_AsSsize_t(PyTuple_GET_ITEM(retobj, 1));
489 if (newpos < 0 && !PyErr_Occurred())
490 newpos += (Py_ssize_t)(buf->inbuf_end - buf->inbuf_top);
491 if (newpos < 0 || buf->inbuf_top + newpos > buf->inbuf_end) {
492 PyErr_Clear();
493 PyErr_Format(PyExc_IndexError,
494 "position %zd from error handler out of bounds",
495 newpos);
496 goto errorexit;
497 }
498 buf->inbuf = buf->inbuf_top + newpos;
499 Py_DECREF(retobj);
500 return 0;
501
502 errorexit:
503 Py_XDECREF(retobj);
504 return -1;
505 }
506
507 static PyObject *
508 multibytecodec_encode(const MultibyteCodec *codec,
509 MultibyteCodec_State *state,
510 PyObject *text, Py_ssize_t *inpos_t,
511 PyObject *errors, int flags)
512 {
513 MultibyteEncodeBuffer buf;
514 Py_ssize_t finalsize, r = 0;
515 Py_ssize_t datalen;
516 int kind;
517 const void *data;
518
519 datalen = PyUnicode_GET_LENGTH(text);
520
521 if (datalen == 0 && !(flags & MBENC_RESET))
522 return PyBytes_FromStringAndSize(NULL, 0);
523
524 buf.excobj = NULL;
525 buf.outobj = NULL;
526 buf.inobj = text; /* borrowed reference */
527 buf.inpos = 0;
528 buf.inlen = datalen;
529 kind = PyUnicode_KIND(buf.inobj);
530 data = PyUnicode_DATA(buf.inobj);
531
532 if (datalen > (PY_SSIZE_T_MAX - 16) / 2) {
533 PyErr_NoMemory();
534 goto errorexit;
535 }
536
537 buf.outobj = PyBytes_FromStringAndSize(NULL, datalen * 2 + 16);
538 if (buf.outobj == NULL)
539 goto errorexit;
540 buf.outbuf = (unsigned char *)PyBytes_AS_STRING(buf.outobj);
541 buf.outbuf_end = buf.outbuf + PyBytes_GET_SIZE(buf.outobj);
542
543 while (buf.inpos < buf.inlen) {
544 /* we don't reuse inleft and outleft here.
545 * error callbacks can relocate the cursor anywhere on buffer*/
546 Py_ssize_t outleft = (Py_ssize_t)(buf.outbuf_end - buf.outbuf);
547
548 r = codec->encode(state, codec,
549 kind, data,
550 &buf.inpos, buf.inlen,
551 &buf.outbuf, outleft, flags);
552 if ((r == 0) || (r == MBERR_TOOFEW && !(flags & MBENC_FLUSH)))
553 break;
554 else if (multibytecodec_encerror(codec, state, &buf, errors,r))
555 goto errorexit;
556 else if (r == MBERR_TOOFEW)
557 break;
558 }
559
560 if (codec->encreset != NULL && (flags & MBENC_RESET))
561 for (;;) {
562 Py_ssize_t outleft;
563
564 outleft = (Py_ssize_t)(buf.outbuf_end - buf.outbuf);
565 r = codec->encreset(state, codec, &buf.outbuf,
566 outleft);
567 if (r == 0)
568 break;
569 else if (multibytecodec_encerror(codec, state,
570 &buf, errors, r))
571 goto errorexit;
572 }
573
574 finalsize = (Py_ssize_t)((char *)buf.outbuf -
575 PyBytes_AS_STRING(buf.outobj));
576
577 if (finalsize != PyBytes_GET_SIZE(buf.outobj))
578 if (_PyBytes_Resize(&buf.outobj, finalsize) == -1)
579 goto errorexit;
580
581 if (inpos_t)
582 *inpos_t = buf.inpos;
583 Py_XDECREF(buf.excobj);
584 return buf.outobj;
585
586 errorexit:
587 Py_XDECREF(buf.excobj);
588 Py_XDECREF(buf.outobj);
589 return NULL;
590 }
591
592 /*[clinic input]
593 _multibytecodec.MultibyteCodec.encode
594
595 input: object
596 errors: str(accept={str, NoneType}) = None
597
598 Return an encoded string version of 'input'.
599
600 'errors' may be given to set a different error handling scheme. Default is
601 'strict' meaning that encoding errors raise a UnicodeEncodeError. Other possible
602 values are 'ignore', 'replace' and 'xmlcharrefreplace' as well as any other name
603 registered with codecs.register_error that can handle UnicodeEncodeErrors.
604 [clinic start generated code]*/
605
606 static PyObject *
607 _multibytecodec_MultibyteCodec_encode_impl(MultibyteCodecObject *self,
608 PyObject *input,
609 const char *errors)
610 /*[clinic end generated code: output=7b26652045ba56a9 input=2841745b95ed338f]*/
611 {
612 MultibyteCodec_State state;
613 PyObject *errorcb, *r, *ucvt;
614 Py_ssize_t datalen;
615
616 if (PyUnicode_Check(input))
617 ucvt = NULL;
618 else {
619 input = ucvt = PyObject_Str(input);
620 if (input == NULL)
621 return NULL;
622 else if (!PyUnicode_Check(input)) {
623 PyErr_SetString(PyExc_TypeError,
624 "couldn't convert the object to unicode.");
625 Py_DECREF(ucvt);
626 return NULL;
627 }
628 }
629
630 datalen = PyUnicode_GET_LENGTH(input);
631
632 errorcb = internal_error_callback(errors);
633 if (errorcb == NULL) {
634 Py_XDECREF(ucvt);
635 return NULL;
636 }
637
638 if (self->codec->encinit != NULL &&
639 self->codec->encinit(&state, self->codec) != 0)
640 goto errorexit;
641 r = multibytecodec_encode(self->codec, &state,
642 input, NULL, errorcb,
643 MBENC_FLUSH | MBENC_RESET);
644 if (r == NULL)
645 goto errorexit;
646
647 ERROR_DECREF(errorcb);
648 Py_XDECREF(ucvt);
649 return make_tuple(r, datalen);
650
651 errorexit:
652 ERROR_DECREF(errorcb);
653 Py_XDECREF(ucvt);
654 return NULL;
655 }
656
657 /*[clinic input]
658 _multibytecodec.MultibyteCodec.decode
659
660 input: Py_buffer
661 errors: str(accept={str, NoneType}) = None
662
663 Decodes 'input'.
664
665 'errors' may be given to set a different error handling scheme. Default is
666 'strict' meaning that encoding errors raise a UnicodeDecodeError. Other possible
667 values are 'ignore' and 'replace' as well as any other name registered with
668 codecs.register_error that is able to handle UnicodeDecodeErrors."
669 [clinic start generated code]*/
670
671 static PyObject *
672 _multibytecodec_MultibyteCodec_decode_impl(MultibyteCodecObject *self,
673 Py_buffer *input,
674 const char *errors)
675 /*[clinic end generated code: output=ff419f65bad6cc77 input=e0c78fc7ab190def]*/
676 {
677 MultibyteCodec_State state;
678 MultibyteDecodeBuffer buf;
679 PyObject *errorcb, *res;
680 const char *data;
681 Py_ssize_t datalen;
682
683 data = input->buf;
684 datalen = input->len;
685
686 errorcb = internal_error_callback(errors);
687 if (errorcb == NULL) {
688 return NULL;
689 }
690
691 if (datalen == 0) {
692 ERROR_DECREF(errorcb);
693 return make_tuple(Py_GetConstant(Py_CONSTANT_EMPTY_STR), 0);
694 }
695
696 _PyUnicodeWriter_Init(&buf.writer);
697 buf.writer.min_length = datalen;
698 buf.excobj = NULL;
699 buf.inbuf = buf.inbuf_top = (unsigned char *)data;
700 buf.inbuf_end = buf.inbuf_top + datalen;
701
702 if (self->codec->decinit != NULL &&
703 self->codec->decinit(&state, self->codec) != 0)
704 goto errorexit;
705
706 while (buf.inbuf < buf.inbuf_end) {
707 Py_ssize_t inleft, r;
708
709 inleft = (Py_ssize_t)(buf.inbuf_end - buf.inbuf);
710
711 r = self->codec->decode(&state, self->codec,
712 &buf.inbuf, inleft, &buf.writer);
713 if (r == 0)
714 break;
715 else if (multibytecodec_decerror(self->codec, &state,
716 &buf, errorcb, r))
717 goto errorexit;
718 }
719
720 res = _PyUnicodeWriter_Finish(&buf.writer);
721 if (res == NULL)
722 goto errorexit;
723
724 Py_XDECREF(buf.excobj);
725 ERROR_DECREF(errorcb);
726 return make_tuple(res, datalen);
727
728 errorexit:
729 ERROR_DECREF(errorcb);
730 Py_XDECREF(buf.excobj);
731 _PyUnicodeWriter_Dealloc(&buf.writer);
732
733 return NULL;
734 }
735
736 static struct PyMethodDef multibytecodec_methods[] = {
737 _MULTIBYTECODEC_MULTIBYTECODEC_ENCODE_METHODDEF
738 _MULTIBYTECODEC_MULTIBYTECODEC_DECODE_METHODDEF
739 {NULL, NULL},
740 };
741
742 static int
743 multibytecodec_clear(PyObject *op)
744 {
745 MultibyteCodecObject *self = _MultibyteCodecObject_CAST(op);
746 Py_CLEAR(self->cjk_module);
747 return 0;
748 }
749
750 static int
751 multibytecodec_traverse(PyObject *op, visitproc visit, void *arg)
752 {
753 MultibyteCodecObject *self = _MultibyteCodecObject_CAST(op);
754 Py_VISIT(Py_TYPE(self));
755 Py_VISIT(self->cjk_module);
756 return 0;
757 }
758
759 static void
760 multibytecodec_dealloc(PyObject *self)
761 {
762 PyObject_GC_UnTrack(self);
763 PyTypeObject *tp = Py_TYPE(self);
764 (void)multibytecodec_clear(self);
765 tp->tp_free(self);
766 Py_DECREF(tp);
767 }
768
769 static PyType_Slot multibytecodec_slots[] = {
770 {Py_tp_dealloc, multibytecodec_dealloc},
771 {Py_tp_getattro, PyObject_GenericGetAttr},
772 {Py_tp_methods, multibytecodec_methods},
773 {Py_tp_traverse, multibytecodec_traverse},
774 {Py_tp_clear, multibytecodec_clear},
775 {0, NULL},
776 };
777
778 static PyType_Spec multibytecodec_spec = {
779 .name = MODULE_NAME ".MultibyteCodec",
780 .basicsize = sizeof(MultibyteCodecObject),
781 .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC |
782 Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_IMMUTABLETYPE),
783 .slots = multibytecodec_slots,
784 };
785
786
787 /**
788 * Utility functions for stateful codec mechanism
789 */
790
791 #define STATEFUL_DCTX(o) ((MultibyteStatefulDecoderContext *)(o))
792 #define STATEFUL_ECTX(o) ((MultibyteStatefulEncoderContext *)(o))
793
794 static PyObject *
795 encoder_encode_stateful(MultibyteStatefulEncoderContext *ctx,
796 PyObject *unistr, int final)
797 {
798 PyObject *ucvt, *r = NULL;
799 PyObject *inbuf = NULL;
800 Py_ssize_t inpos, datalen;
801 PyObject *origpending = NULL;
802
803 if (PyUnicode_Check(unistr))
804 ucvt = NULL;
805 else {
806 unistr = ucvt = PyObject_Str(unistr);
807 if (unistr == NULL)
808 return NULL;
809 else if (!PyUnicode_Check(unistr)) {
810 PyErr_SetString(PyExc_TypeError,
811 "couldn't convert the object to str.");
812 Py_DECREF(ucvt);
813 return NULL;
814 }
815 }
816
817 if (ctx->pending) {
818 PyObject *inbuf_tmp;
819
820 origpending = Py_NewRef(ctx->pending);
821
822 inbuf_tmp = Py_NewRef(ctx->pending);
823 PyUnicode_Append(&inbuf_tmp, unistr);
824 if (inbuf_tmp == NULL)
825 goto errorexit;
826 Py_CLEAR(ctx->pending);
827 inbuf = inbuf_tmp;
828 }
829 else {
830 origpending = NULL;
831
832 inbuf = Py_NewRef(unistr);
833 }
834 inpos = 0;
835 datalen = PyUnicode_GET_LENGTH(inbuf);
836
837 r = multibytecodec_encode(ctx->codec, &ctx->state,
838 inbuf, &inpos,
839 ctx->errors, final ? MBENC_FLUSH | MBENC_RESET : 0);
840 if (r == NULL) {
841 /* recover the original pending buffer */
842 Py_XSETREF(ctx->pending, origpending);
843 origpending = NULL;
844 goto errorexit;
845 }
846 Py_XDECREF(origpending);
847
848 if (inpos < datalen) {
849 if (datalen - inpos > MAXENCPENDING) {
850 /* normal codecs can't reach here */
851 PyObject *excobj = PyObject_CallFunction(PyExc_UnicodeEncodeError,
852 "sOnns",
853 ctx->codec->encoding,
854 inbuf,
855 inpos, datalen,
856 "pending buffer overflow");
857 if (excobj == NULL) goto errorexit;
858 PyErr_SetObject(PyExc_UnicodeEncodeError, excobj);
859 Py_DECREF(excobj);
860 goto errorexit;
861 }
862 ctx->pending = PyUnicode_Substring(inbuf, inpos, datalen);
863 if (ctx->pending == NULL) {
864 /* normal codecs can't reach here */
865 goto errorexit;
866 }
867 }
868
869 Py_DECREF(inbuf);
870 Py_XDECREF(ucvt);
871 return r;
872
873 errorexit:
874 Py_XDECREF(r);
875 Py_XDECREF(ucvt);
876 Py_XDECREF(origpending);
877 Py_XDECREF(inbuf);
878 return NULL;
879 }
880
881 static int
882 decoder_append_pending(MultibyteStatefulDecoderContext *ctx,
883 MultibyteDecodeBuffer *buf)
884 {
885 Py_ssize_t npendings;
886
887 npendings = (Py_ssize_t)(buf->inbuf_end - buf->inbuf);
888 if (npendings + ctx->pendingsize > MAXDECPENDING ||
889 npendings > PY_SSIZE_T_MAX - ctx->pendingsize) {
890 Py_ssize_t bufsize = (Py_ssize_t)(buf->inbuf_end - buf->inbuf_top);
891 PyObject *excobj = PyUnicodeDecodeError_Create(ctx->codec->encoding,
892 (const char *)buf->inbuf_top,
893 bufsize,
894 0,
895 bufsize,
896 "pending buffer overflow");
897 if (excobj == NULL) return -1;
898 PyErr_SetObject(PyExc_UnicodeDecodeError, excobj);
899 Py_DECREF(excobj);
900 return -1;
901 }
902 memcpy(ctx->pending + ctx->pendingsize, buf->inbuf, npendings);
903 ctx->pendingsize += npendings;
904 return 0;
905 }
906
907 static int
908 decoder_prepare_buffer(MultibyteDecodeBuffer *buf, const char *data,
909 Py_ssize_t size)
910 {
911 buf->inbuf = buf->inbuf_top = (const unsigned char *)data;
912 buf->inbuf_end = buf->inbuf_top + size;
913 buf->writer.min_length += size;
914 return 0;
915 }
916
917 static int
918 decoder_feed_buffer(MultibyteStatefulDecoderContext *ctx,
919 MultibyteDecodeBuffer *buf)
920 {
921 while (buf->inbuf < buf->inbuf_end) {
922 Py_ssize_t inleft;
923 Py_ssize_t r;
924
925 inleft = (Py_ssize_t)(buf->inbuf_end - buf->inbuf);
926
927 r = ctx->codec->decode(&ctx->state, ctx->codec,
928 &buf->inbuf, inleft, &buf->writer);
929 if (r == 0 || r == MBERR_TOOFEW)
930 break;
931 else if (multibytecodec_decerror(ctx->codec, &ctx->state,
932 buf, ctx->errors, r))
933 return -1;
934 }
935 return 0;
936 }
937
938
939 /*[clinic input]
940 _multibytecodec.MultibyteIncrementalEncoder.encode
941
942 input: object
943 final: bool = False
944 [clinic start generated code]*/
945
946 static PyObject *
947 _multibytecodec_MultibyteIncrementalEncoder_encode_impl(MultibyteIncrementalEncoderObject *self,
948 PyObject *input,
949 int final)
950 /*[clinic end generated code: output=123361b6c505e2c1 input=bd5f7d40d43e99b0]*/
951 {
952 return encoder_encode_stateful(STATEFUL_ECTX(self), input, final);
953 }
954
955 /*[clinic input]
956 _multibytecodec.MultibyteIncrementalEncoder.getstate
957 [clinic start generated code]*/
958
959 static PyObject *
960 _multibytecodec_MultibyteIncrementalEncoder_getstate_impl(MultibyteIncrementalEncoderObject *self)
961 /*[clinic end generated code: output=9794a5ace70d7048 input=4a2a82874ffa40bb]*/
962 {
963 /* state made up of 1 byte for buffer size, up to MAXENCPENDING*4 bytes
964 for UTF-8 encoded buffer (each character can use up to 4
965 bytes), and required bytes for MultibyteCodec_State.c. A byte
966 array is used to avoid different compilers generating different
967 values for the same state, e.g. as a result of struct padding.
968 */
969 unsigned char statebytes[1 + MAXENCPENDING*4 + sizeof(self->state.c)];
970 Py_ssize_t statesize;
971 const char *pendingbuffer = NULL;
972 Py_ssize_t pendingsize;
973
974 if (self->pending != NULL) {
975 pendingbuffer = PyUnicode_AsUTF8AndSize(self->pending, &pendingsize);
976 if (pendingbuffer == NULL) {
977 return NULL;
978 }
979 if (pendingsize > MAXENCPENDING*4) {
980 PyObject *excobj = PyObject_CallFunction(PyExc_UnicodeEncodeError,
981 "sOnns",
982 self->codec->encoding,
983 self->pending,
984 0, PyUnicode_GET_LENGTH(self->pending),
985 "pending buffer too large");
986 if (excobj == NULL) {
987 return NULL;
988 }
989 PyErr_SetObject(PyExc_UnicodeEncodeError, excobj);
990 Py_DECREF(excobj);
991 return NULL;
992 }
993 statebytes[0] = (unsigned char)pendingsize;
994 memcpy(statebytes + 1, pendingbuffer, pendingsize);
995 statesize = 1 + pendingsize;
996 } else {
997 statebytes[0] = 0;
998 statesize = 1;
999 }
1000 memcpy(statebytes+statesize, self->state.c,
1001 sizeof(self->state.c));
1002 statesize += sizeof(self->state.c);
1003
1004 return (PyObject *)_PyLong_FromByteArray(statebytes, statesize,
1005 1 /* little-endian */ ,
1006 0 /* unsigned */ );
1007 }
1008
1009 /*[clinic input]
1010 _multibytecodec.MultibyteIncrementalEncoder.setstate
1011 state as statelong: object(type='PyLongObject *', subclass_of='&PyLong_Type')
1012 /
1013 [clinic start generated code]*/
1014
1015 static PyObject *
1016 _multibytecodec_MultibyteIncrementalEncoder_setstate_impl(MultibyteIncrementalEncoderObject *self,
1017 PyLongObject *statelong)
1018 /*[clinic end generated code: output=4e5e98ac1f4039ca input=c80fb5830d4d2f76]*/
1019 {
1020 PyObject *pending = NULL;
1021 unsigned char statebytes[1 + MAXENCPENDING*4 + sizeof(self->state.c)];
1022
1023 if (_PyLong_AsByteArray(statelong, statebytes, sizeof(statebytes),
1024 1 /* little-endian */ ,
1025 0 /* unsigned */ ,
1026 1 /* with_exceptions */) < 0) {
1027 goto errorexit;
1028 }
1029
1030 if (statebytes[0] > MAXENCPENDING*4) {
1031 PyErr_SetString(PyExc_UnicodeError, "pending buffer too large");
1032 return NULL;
1033 }
1034
1035 pending = PyUnicode_DecodeUTF8((const char *)statebytes+1,
1036 statebytes[0], "strict");
1037 if (pending == NULL) {
1038 goto errorexit;
1039 }
1040
1041 Py_XSETREF(self->pending, pending);
1042 memcpy(self->state.c, statebytes+1+statebytes[0],
1043 sizeof(self->state.c));
1044
1045 Py_RETURN_NONE;
1046
1047 errorexit:
1048 Py_XDECREF(pending);
1049 return NULL;
1050 }
1051
1052 /*[clinic input]
1053 _multibytecodec.MultibyteIncrementalEncoder.reset
1054 [clinic start generated code]*/
1055
1056 static PyObject *
1057 _multibytecodec_MultibyteIncrementalEncoder_reset_impl(MultibyteIncrementalEncoderObject *self)
1058 /*[clinic end generated code: output=b4125d8f537a253f input=930f06760707b6ea]*/
1059 {
1060 /* Longest output: 4 bytes (b'\x0F\x1F(B') with ISO 2022 */
1061 unsigned char buffer[4], *outbuf;
1062 Py_ssize_t r;
1063 if (self->codec->encreset != NULL) {
1064 outbuf = buffer;
1065 r = self->codec->encreset(&self->state, self->codec,
1066 &outbuf, sizeof(buffer));
1067 if (r != 0)
1068 return NULL;
1069 }
1070 Py_CLEAR(self->pending);
1071 Py_RETURN_NONE;
1072 }
1073
1074 static struct PyMethodDef mbiencoder_methods[] = {
1075 _MULTIBYTECODEC_MULTIBYTEINCREMENTALENCODER_ENCODE_METHODDEF
1076 _MULTIBYTECODEC_MULTIBYTEINCREMENTALENCODER_GETSTATE_METHODDEF
1077 _MULTIBYTECODEC_MULTIBYTEINCREMENTALENCODER_SETSTATE_METHODDEF
1078 _MULTIBYTECODEC_MULTIBYTEINCREMENTALENCODER_RESET_METHODDEF
1079 {NULL, NULL},
1080 };
1081
1082 static PyObject *
1083 mbiencoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
1084 {
1085 MultibyteIncrementalEncoderObject *self;
1086 PyObject *codec = NULL;
1087 char *errors = NULL;
1088
1089 if (!PyArg_ParseTupleAndKeywords(args, kwds, "|s:IncrementalEncoder",
1090 incnewkwarglist, &errors))
1091 return NULL;
1092
1093 self = (MultibyteIncrementalEncoderObject *)type->tp_alloc(type, 0);
1094 if (self == NULL)
1095 return NULL;
1096
1097 codec = PyObject_GetAttrString((PyObject *)type, "codec");
1098 if (codec == NULL)
1099 goto errorexit;
1100
1101 module_state *state = find_state_by_def(type);
1102 if (!MultibyteCodec_Check(state, codec)) {
1103 PyErr_SetString(PyExc_TypeError, "codec is unexpected type");
1104 goto errorexit;
1105 }
1106
1107 self->codec = ((MultibyteCodecObject *)codec)->codec;
1108 self->pending = NULL;
1109 self->errors = internal_error_callback(errors);
1110 if (self->errors == NULL)
1111 goto errorexit;
1112 if (self->codec->encinit != NULL &&
1113 self->codec->encinit(&self->state, self->codec) != 0)
1114 goto errorexit;
1115
1116 Py_DECREF(codec);
1117 return (PyObject *)self;
1118
1119 errorexit:
1120 Py_XDECREF(self);
1121 Py_XDECREF(codec);
1122 return NULL;
1123 }
1124
1125 static int
1126 mbiencoder_init(PyObject *self, PyObject *args, PyObject *kwds)
1127 {
1128 return 0;
1129 }
1130
1131 static int
1132 mbiencoder_traverse(PyObject *op, visitproc visit, void *arg)
1133 {
1134 MultibyteIncrementalEncoderObject *self = _MultibyteIncrementalEncoderObject_CAST(op);
1135 if (ERROR_ISCUSTOM(self->errors))
1136 Py_VISIT(self->errors);
1137 return 0;
1138 }
1139
1140 static void
1141 mbiencoder_dealloc(PyObject *op)
1142 {
1143 MultibyteIncrementalEncoderObject *self = _MultibyteIncrementalEncoderObject_CAST(op);
1144 PyTypeObject *tp = Py_TYPE(self);
1145 PyObject_GC_UnTrack(self);
1146 ERROR_DECREF(self->errors);
1147 Py_CLEAR(self->pending);
1148 tp->tp_free(self);
1149 Py_DECREF(tp);
1150 }
1151
1152 static PyType_Slot encoder_slots[] = {
1153 {Py_tp_dealloc, mbiencoder_dealloc},
1154 {Py_tp_getattro, PyObject_GenericGetAttr},
1155 {Py_tp_traverse, mbiencoder_traverse},
1156 {Py_tp_methods, mbiencoder_methods},
1157 {Py_tp_getset, codecctx_getsets},
1158 {Py_tp_init, mbiencoder_init},
1159 {Py_tp_new, mbiencoder_new},
1160 {0, NULL},
1161 };
1162
1163 static PyType_Spec encoder_spec = {
1164 .name = MODULE_NAME ".MultibyteIncrementalEncoder",
1165 .basicsize = sizeof(MultibyteIncrementalEncoderObject),
1166 .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE |
1167 Py_TPFLAGS_IMMUTABLETYPE),
1168 .slots = encoder_slots,
1169 };
1170
1171
1172 /*[clinic input]
1173 _multibytecodec.MultibyteIncrementalDecoder.decode
1174
1175 input: Py_buffer
1176 final: bool = False
1177 [clinic start generated code]*/
1178
1179 static PyObject *
1180 _multibytecodec_MultibyteIncrementalDecoder_decode_impl(MultibyteIncrementalDecoderObject *self,
1181 Py_buffer *input,
1182 int final)
1183 /*[clinic end generated code: output=b9b9090e8a9ce2ba input=8795fbb20860027a]*/
1184 {
1185 MultibyteDecodeBuffer buf;
1186 char *data, *wdata = NULL;
1187 Py_ssize_t wsize, size, origpending;
1188 PyObject *res;
1189
1190 data = input->buf;
1191 size = input->len;
1192
1193 _PyUnicodeWriter_Init(&buf.writer);
1194 buf.excobj = NULL;
1195 origpending = self->pendingsize;
1196
1197 if (self->pendingsize == 0) {
1198 wsize = size;
1199 wdata = data;
1200 }
1201 else {
1202 if (size > PY_SSIZE_T_MAX - self->pendingsize) {
1203 PyErr_NoMemory();
1204 goto errorexit;
1205 }
1206 wsize = size + self->pendingsize;
1207 wdata = PyMem_Malloc(wsize);
1208 if (wdata == NULL) {
1209 PyErr_NoMemory();
1210 goto errorexit;
1211 }
1212 memcpy(wdata, self->pending, self->pendingsize);
1213 memcpy(wdata + self->pendingsize, data, size);
1214 self->pendingsize = 0;
1215 }
1216
1217 if (decoder_prepare_buffer(&buf, wdata, wsize) != 0)
1218 goto errorexit;
1219
1220 if (decoder_feed_buffer(STATEFUL_DCTX(self), &buf))
1221 goto errorexit;
1222
1223 if (final && buf.inbuf < buf.inbuf_end) {
1224 if (multibytecodec_decerror(self->codec, &self->state,
1225 &buf, self->errors, MBERR_TOOFEW)) {
1226 /* recover the original pending buffer */
1227 memcpy(self->pending, wdata, origpending);
1228 self->pendingsize = origpending;
1229 goto errorexit;
1230 }
1231 }
1232
1233 if (buf.inbuf < buf.inbuf_end) { /* pending sequence still exists */
1234 if (decoder_append_pending(STATEFUL_DCTX(self), &buf) != 0)
1235 goto errorexit;
1236 }
1237
1238 res = _PyUnicodeWriter_Finish(&buf.writer);
1239 if (res == NULL)
1240 goto errorexit;
1241
1242 if (wdata != data)
1243 PyMem_Free(wdata);
1244 Py_XDECREF(buf.excobj);
1245 return res;
1246
1247 errorexit:
1248 if (wdata != NULL && wdata != data)
1249 PyMem_Free(wdata);
1250 Py_XDECREF(buf.excobj);
1251 _PyUnicodeWriter_Dealloc(&buf.writer);
1252 return NULL;
1253 }
1254
1255 /*[clinic input]
1256 _multibytecodec.MultibyteIncrementalDecoder.getstate
1257 [clinic start generated code]*/
1258
1259 static PyObject *
1260 _multibytecodec_MultibyteIncrementalDecoder_getstate_impl(MultibyteIncrementalDecoderObject *self)
1261 /*[clinic end generated code: output=255009c4713b7f82 input=4006aa49bddbaa75]*/
1262 {
1263 PyObject *buffer;
1264 PyObject *statelong;
1265
1266 buffer = PyBytes_FromStringAndSize((const char *)self->pending,
1267 self->pendingsize);
1268 if (buffer == NULL) {
1269 return NULL;
1270 }
1271
1272 statelong = (PyObject *)_PyLong_FromByteArray(self->state.c,
1273 sizeof(self->state.c),
1274 1 /* little-endian */ ,
1275 0 /* unsigned */ );
1276 if (statelong == NULL) {
1277 Py_DECREF(buffer);
1278 return NULL;
1279 }
1280
1281 return Py_BuildValue("NN", buffer, statelong);
1282 }
1283
1284 /*[clinic input]
1285 _multibytecodec.MultibyteIncrementalDecoder.setstate
1286 state: object(subclass_of='&PyTuple_Type')
1287 /
1288 [clinic start generated code]*/
1289
1290 static PyObject *
1291 _multibytecodec_MultibyteIncrementalDecoder_setstate_impl(MultibyteIncrementalDecoderObject *self,
1292 PyObject *state)
1293 /*[clinic end generated code: output=106b2fbca3e2dcc2 input=e5d794e8baba1a47]*/
1294 {
1295 PyObject *buffer;
1296 PyLongObject *statelong;
1297 Py_ssize_t buffersize;
1298 const char *bufferstr;
1299 unsigned char statebytes[8];
1300
1301 if (!PyArg_ParseTuple(state, "SO!;setstate(): illegal state argument",
1302 &buffer, &PyLong_Type, &statelong))
1303 {
1304 return NULL;
1305 }
1306
1307 if (_PyLong_AsByteArray(statelong, statebytes, sizeof(statebytes),
1308 1 /* little-endian */ ,
1309 0 /* unsigned */ ,
1310 1 /* with_exceptions */) < 0) {
1311 return NULL;
1312 }
1313
1314 buffersize = PyBytes_Size(buffer);
1315 if (buffersize == -1) {
1316 return NULL;
1317 }
1318
1319 if (buffersize > MAXDECPENDING) {
1320 PyObject *excobj = PyUnicodeDecodeError_Create(self->codec->encoding,
1321 PyBytes_AS_STRING(buffer), buffersize,
1322 0, buffersize,
1323 "pending buffer too large");
1324 if (excobj == NULL) return NULL;
1325 PyErr_SetObject(PyExc_UnicodeDecodeError, excobj);
1326 Py_DECREF(excobj);
1327 return NULL;
1328 }
1329
1330 bufferstr = PyBytes_AsString(buffer);
1331 if (bufferstr == NULL) {
1332 return NULL;
1333 }
1334 self->pendingsize = buffersize;
1335 memcpy(self->pending, bufferstr, self->pendingsize);
1336 memcpy(self->state.c, statebytes, sizeof(statebytes));
1337
1338 Py_RETURN_NONE;
1339 }
1340
1341 /*[clinic input]
1342 _multibytecodec.MultibyteIncrementalDecoder.reset
1343 [clinic start generated code]*/
1344
1345 static PyObject *
1346 _multibytecodec_MultibyteIncrementalDecoder_reset_impl(MultibyteIncrementalDecoderObject *self)
1347 /*[clinic end generated code: output=da423b1782c23ed1 input=3b63b3be85b2fb45]*/
1348 {
1349 if (self->codec->decreset != NULL &&
1350 self->codec->decreset(&self->state, self->codec) != 0)
1351 return NULL;
1352 self->pendingsize = 0;
1353
1354 Py_RETURN_NONE;
1355 }
1356
1357 static struct PyMethodDef mbidecoder_methods[] = {
1358 _MULTIBYTECODEC_MULTIBYTEINCREMENTALDECODER_DECODE_METHODDEF
1359 _MULTIBYTECODEC_MULTIBYTEINCREMENTALDECODER_GETSTATE_METHODDEF
1360 _MULTIBYTECODEC_MULTIBYTEINCREMENTALDECODER_SETSTATE_METHODDEF
1361 _MULTIBYTECODEC_MULTIBYTEINCREMENTALDECODER_RESET_METHODDEF
1362 {NULL, NULL},
1363 };
1364
1365 static PyObject *
1366 mbidecoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
1367 {
1368 MultibyteIncrementalDecoderObject *self;
1369 PyObject *codec = NULL;
1370 char *errors = NULL;
1371
1372 if (!PyArg_ParseTupleAndKeywords(args, kwds, "|s:IncrementalDecoder",
1373 incnewkwarglist, &errors))
1374 return NULL;
1375
1376 self = (MultibyteIncrementalDecoderObject *)type->tp_alloc(type, 0);
1377 if (self == NULL)
1378 return NULL;
1379
1380 codec = PyObject_GetAttrString((PyObject *)type, "codec");
1381 if (codec == NULL)
1382 goto errorexit;
1383
1384 module_state *state = find_state_by_def(type);
1385 if (!MultibyteCodec_Check(state, codec)) {
1386 PyErr_SetString(PyExc_TypeError, "codec is unexpected type");
1387 goto errorexit;
1388 }
1389
1390 self->codec = ((MultibyteCodecObject *)codec)->codec;
1391 self->pendingsize = 0;
1392 self->errors = internal_error_callback(errors);
1393 if (self->errors == NULL)
1394 goto errorexit;
1395 if (self->codec->decinit != NULL &&
1396 self->codec->decinit(&self->state, self->codec) != 0)
1397 goto errorexit;
1398
1399 Py_DECREF(codec);
1400 return (PyObject *)self;
1401
1402 errorexit:
1403 Py_XDECREF(self);
1404 Py_XDECREF(codec);
1405 return NULL;
1406 }
1407
1408 static int
1409 mbidecoder_init(PyObject *self, PyObject *args, PyObject *kwds)
1410 {
1411 return 0;
1412 }
1413
1414 static int
1415 mbidecoder_traverse(PyObject *op, visitproc visit, void *arg)
1416 {
1417 MultibyteIncrementalDecoderObject *self = _MultibyteIncrementalDecoderObject_CAST(op);
1418 if (ERROR_ISCUSTOM(self->errors))
1419 Py_VISIT(self->errors);
1420 return 0;
1421 }
1422
1423 static void
1424 mbidecoder_dealloc(PyObject *op)
1425 {
1426 MultibyteIncrementalDecoderObject *self = _MultibyteIncrementalDecoderObject_CAST(op);
1427 PyTypeObject *tp = Py_TYPE(self);
1428 PyObject_GC_UnTrack(self);
1429 ERROR_DECREF(self->errors);
1430 tp->tp_free(self);
1431 Py_DECREF(tp);
1432 }
1433
1434 static PyType_Slot decoder_slots[] = {
1435 {Py_tp_dealloc, mbidecoder_dealloc},
1436 {Py_tp_getattro, PyObject_GenericGetAttr},
1437 {Py_tp_traverse, mbidecoder_traverse},
1438 {Py_tp_methods, mbidecoder_methods},
1439 {Py_tp_getset, codecctx_getsets},
1440 {Py_tp_init, mbidecoder_init},
1441 {Py_tp_new, mbidecoder_new},
1442 {0, NULL},
1443 };
1444
1445 static PyType_Spec decoder_spec = {
1446 .name = MODULE_NAME ".MultibyteIncrementalDecoder",
1447 .basicsize = sizeof(MultibyteIncrementalDecoderObject),
1448 .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE |
1449 Py_TPFLAGS_IMMUTABLETYPE),
1450 .slots = decoder_slots,
1451 };
1452
1453 static PyObject *
1454 mbstreamreader_iread(MultibyteStreamReaderObject *self,
1455 const char *method, Py_ssize_t sizehint)
1456 {
1457 MultibyteDecodeBuffer buf;
1458 PyObject *cres, *res;
1459 Py_ssize_t rsize;
1460
1461 if (sizehint == 0)
1462 return Py_GetConstant(Py_CONSTANT_EMPTY_STR);
1463
1464 _PyUnicodeWriter_Init(&buf.writer);
1465 buf.excobj = NULL;
1466 cres = NULL;
1467
1468 for (;;) {
1469 int endoffile;
1470
1471 if (sizehint < 0)
1472 cres = PyObject_CallMethod(self->stream,
1473 method, NULL);
1474 else
1475 cres = PyObject_CallMethod(self->stream,
1476 method, "i", sizehint);
1477 if (cres == NULL)
1478 goto errorexit;
1479
1480 if (!PyBytes_Check(cres)) {
1481 PyErr_Format(PyExc_TypeError,
1482 "stream function returned a "
1483 "non-bytes object (%.100s)",
1484 Py_TYPE(cres)->tp_name);
1485 goto errorexit;
1486 }
1487
1488 endoffile = (PyBytes_GET_SIZE(cres) == 0);
1489
1490 if (self->pendingsize > 0) {
1491 PyObject *ctr;
1492 char *ctrdata;
1493
1494 if (PyBytes_GET_SIZE(cres) > PY_SSIZE_T_MAX - self->pendingsize) {
1495 PyErr_NoMemory();
1496 goto errorexit;
1497 }
1498 rsize = PyBytes_GET_SIZE(cres) + self->pendingsize;
1499 ctr = PyBytes_FromStringAndSize(NULL, rsize);
1500 if (ctr == NULL)
1501 goto errorexit;
1502 ctrdata = PyBytes_AS_STRING(ctr);
1503 memcpy(ctrdata, self->pending, self->pendingsize);
1504 memcpy(ctrdata + self->pendingsize,
1505 PyBytes_AS_STRING(cres),
1506 PyBytes_GET_SIZE(cres));
1507 Py_SETREF(cres, ctr);
1508 self->pendingsize = 0;
1509 }
1510
1511 rsize = PyBytes_GET_SIZE(cres);
1512 if (decoder_prepare_buffer(&buf, PyBytes_AS_STRING(cres),
1513 rsize) != 0)
1514 goto errorexit;
1515
1516 if (rsize > 0 && decoder_feed_buffer(
1517 (MultibyteStatefulDecoderContext *)self, &buf))
1518 goto errorexit;
1519
1520 if (endoffile || sizehint < 0) {
1521 if (buf.inbuf < buf.inbuf_end &&
1522 multibytecodec_decerror(self->codec, &self->state,
1523 &buf, self->errors, MBERR_TOOFEW))
1524 goto errorexit;
1525 }
1526
1527 if (buf.inbuf < buf.inbuf_end) { /* pending sequence exists */
1528 if (decoder_append_pending(STATEFUL_DCTX(self),
1529 &buf) != 0)
1530 goto errorexit;
1531 }
1532
1533 Py_SETREF(cres, NULL);
1534
1535 if (sizehint < 0 || buf.writer.pos != 0 || rsize == 0)
1536 break;
1537
1538 sizehint = 1; /* read 1 more byte and retry */
1539 }
1540
1541 res = _PyUnicodeWriter_Finish(&buf.writer);
1542 if (res == NULL)
1543 goto errorexit;
1544
1545 Py_XDECREF(cres);
1546 Py_XDECREF(buf.excobj);
1547 return res;
1548
1549 errorexit:
1550 Py_XDECREF(cres);
1551 Py_XDECREF(buf.excobj);
1552 _PyUnicodeWriter_Dealloc(&buf.writer);
1553 return NULL;
1554 }
1555
1556 /*[clinic input]
1557 _multibytecodec.MultibyteStreamReader.read
1558
1559 sizeobj: object = None
1560 /
1561 [clinic start generated code]*/
1562
1563 static PyObject *
1564 _multibytecodec_MultibyteStreamReader_read_impl(MultibyteStreamReaderObject *self,
1565 PyObject *sizeobj)
1566 /*[clinic end generated code: output=35621eb75355d5b8 input=015b0d3ff2fca485]*/
1567 {
1568 Py_ssize_t size;
1569
1570 if (sizeobj == Py_None)
1571 size = -1;
1572 else if (PyLong_Check(sizeobj))
1573 size = PyLong_AsSsize_t(sizeobj);
1574 else {
1575 PyErr_SetString(PyExc_TypeError, "arg 1 must be an integer");
1576 return NULL;
1577 }
1578
1579 if (size == -1 && PyErr_Occurred())
1580 return NULL;
1581
1582 return mbstreamreader_iread(self, "read", size);
1583 }
1584
1585 /*[clinic input]
1586 _multibytecodec.MultibyteStreamReader.readline
1587
1588 sizeobj: object = None
1589 /
1590 [clinic start generated code]*/
1591
1592 static PyObject *
1593 _multibytecodec_MultibyteStreamReader_readline_impl(MultibyteStreamReaderObject *self,
1594 PyObject *sizeobj)
1595 /*[clinic end generated code: output=4fbfaae1ed457a11 input=41ccc64f9bb0cec3]*/
1596 {
1597 Py_ssize_t size;
1598
1599 if (sizeobj == Py_None)
1600 size = -1;
1601 else if (PyLong_Check(sizeobj))
1602 size = PyLong_AsSsize_t(sizeobj);
1603 else {
1604 PyErr_SetString(PyExc_TypeError, "arg 1 must be an integer");
1605 return NULL;
1606 }
1607
1608 if (size == -1 && PyErr_Occurred())
1609 return NULL;
1610
1611 return mbstreamreader_iread(self, "readline", size);
1612 }
1613
1614 /*[clinic input]
1615 _multibytecodec.MultibyteStreamReader.readlines
1616
1617 sizehintobj: object = None
1618 /
1619 [clinic start generated code]*/
1620
1621 static PyObject *
1622 _multibytecodec_MultibyteStreamReader_readlines_impl(MultibyteStreamReaderObject *self,
1623 PyObject *sizehintobj)
1624 /*[clinic end generated code: output=e7c4310768ed2ad4 input=54932f5d4d88e880]*/
1625 {
1626 PyObject *r, *sr;
1627 Py_ssize_t sizehint;
1628
1629 if (sizehintobj == Py_None)
1630 sizehint = -1;
1631 else if (PyLong_Check(sizehintobj))
1632 sizehint = PyLong_AsSsize_t(sizehintobj);
1633 else {
1634 PyErr_SetString(PyExc_TypeError, "arg 1 must be an integer");
1635 return NULL;
1636 }
1637
1638 if (sizehint == -1 && PyErr_Occurred())
1639 return NULL;
1640
1641 r = mbstreamreader_iread(self, "read", sizehint);
1642 if (r == NULL)
1643 return NULL;
1644
1645 sr = PyUnicode_Splitlines(r, 1);
1646 Py_DECREF(r);
1647 return sr;
1648 }
1649
1650 /*[clinic input]
1651 _multibytecodec.MultibyteStreamReader.reset
1652 [clinic start generated code]*/
1653
1654 static PyObject *
1655 _multibytecodec_MultibyteStreamReader_reset_impl(MultibyteStreamReaderObject *self)
1656 /*[clinic end generated code: output=138490370a680abc input=5d4140db84b5e1e2]*/
1657 {
1658 if (self->codec->decreset != NULL &&
1659 self->codec->decreset(&self->state, self->codec) != 0)
1660 return NULL;
1661 self->pendingsize = 0;
1662
1663 Py_RETURN_NONE;
1664 }
1665
1666 static struct PyMethodDef mbstreamreader_methods[] = {
1667 _MULTIBYTECODEC_MULTIBYTESTREAMREADER_READ_METHODDEF
1668 _MULTIBYTECODEC_MULTIBYTESTREAMREADER_READLINE_METHODDEF
1669 _MULTIBYTECODEC_MULTIBYTESTREAMREADER_READLINES_METHODDEF
1670 _MULTIBYTECODEC_MULTIBYTESTREAMREADER_RESET_METHODDEF
1671 {NULL, NULL},
1672 };
1673
1674 static PyMemberDef mbstreamreader_members[] = {
1675 {"stream", _Py_T_OBJECT,
1676 offsetof(MultibyteStreamReaderObject, stream),
1677 Py_READONLY, NULL},
1678 {NULL,}
1679 };
1680
1681 static PyObject *
1682 mbstreamreader_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
1683 {
1684 MultibyteStreamReaderObject *self;
1685 PyObject *stream, *codec = NULL;
1686 char *errors = NULL;
1687
1688 if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|s:StreamReader",
1689 streamkwarglist, &stream, &errors))
1690 return NULL;
1691
1692 self = (MultibyteStreamReaderObject *)type->tp_alloc(type, 0);
1693 if (self == NULL)
1694 return NULL;
1695
1696 codec = PyObject_GetAttrString((PyObject *)type, "codec");
1697 if (codec == NULL)
1698 goto errorexit;
1699
1700 module_state *state = find_state_by_def(type);
1701 if (!MultibyteCodec_Check(state, codec)) {
1702 PyErr_SetString(PyExc_TypeError, "codec is unexpected type");
1703 goto errorexit;
1704 }
1705
1706 self->codec = ((MultibyteCodecObject *)codec)->codec;
1707 self->stream = Py_NewRef(stream);
1708 self->pendingsize = 0;
1709 self->errors = internal_error_callback(errors);
1710 if (self->errors == NULL)
1711 goto errorexit;
1712 if (self->codec->decinit != NULL &&
1713 self->codec->decinit(&self->state, self->codec) != 0)
1714 goto errorexit;
1715
1716 Py_DECREF(codec);
1717 return (PyObject *)self;
1718
1719 errorexit:
1720 Py_XDECREF(self);
1721 Py_XDECREF(codec);
1722 return NULL;
1723 }
1724
1725 static int
1726 mbstreamreader_init(PyObject *self, PyObject *args, PyObject *kwds)
1727 {
1728 return 0;
1729 }
1730
1731 static int
1732 mbstreamreader_traverse(PyObject *op, visitproc visit, void *arg)
1733 {
1734 MultibyteStreamReaderObject *self = _MultibyteStreamReaderObject_CAST(op);
1735 if (ERROR_ISCUSTOM(self->errors))
1736 Py_VISIT(self->errors);
1737 Py_VISIT(self->stream);
1738 return 0;
1739 }
1740
1741 static void
1742 mbstreamreader_dealloc(PyObject *op)
1743 {
1744 MultibyteStreamReaderObject *self = _MultibyteStreamReaderObject_CAST(op);
1745 PyTypeObject *tp = Py_TYPE(self);
1746 PyObject_GC_UnTrack(self);
1747 ERROR_DECREF(self->errors);
1748 Py_XDECREF(self->stream);
1749 tp->tp_free(self);
1750 Py_DECREF(tp);
1751 }
1752
1753 static PyType_Slot reader_slots[] = {
1754 {Py_tp_dealloc, mbstreamreader_dealloc},
1755 {Py_tp_getattro, PyObject_GenericGetAttr},
1756 {Py_tp_traverse, mbstreamreader_traverse},
1757 {Py_tp_methods, mbstreamreader_methods},
1758 {Py_tp_members, mbstreamreader_members},
1759 {Py_tp_getset, codecctx_getsets},
1760 {Py_tp_init, mbstreamreader_init},
1761 {Py_tp_new, mbstreamreader_new},
1762 {0, NULL},
1763 };
1764
1765 static PyType_Spec reader_spec = {
1766 .name = MODULE_NAME ".MultibyteStreamReader",
1767 .basicsize = sizeof(MultibyteStreamReaderObject),
1768 .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE |
1769 Py_TPFLAGS_IMMUTABLETYPE),
1770 .slots = reader_slots,
1771 };
1772
1773 static int
1774 mbstreamwriter_iwrite(MultibyteStreamWriterObject *self,
1775 PyObject *unistr, PyObject *str_write)
1776 {
1777 PyObject *str, *wr;
1778
1779 str = encoder_encode_stateful(STATEFUL_ECTX(self), unistr, 0);
1780 if (str == NULL)
1781 return -1;
1782
1783 wr = PyObject_CallMethodOneArg(self->stream, str_write, str);
1784 Py_DECREF(str);
1785 if (wr == NULL)
1786 return -1;
1787
1788 Py_DECREF(wr);
1789 return 0;
1790 }
1791
1792 /*[clinic input]
1793 _multibytecodec.MultibyteStreamWriter.write
1794
1795 cls: defining_class
1796 strobj: object
1797 /
1798 [clinic start generated code]*/
1799
1800 static PyObject *
1801 _multibytecodec_MultibyteStreamWriter_write_impl(MultibyteStreamWriterObject *self,
1802 PyTypeObject *cls,
1803 PyObject *strobj)
1804 /*[clinic end generated code: output=68ade3aea26410ac input=199f26f68bd8425a]*/
1805 {
1806 module_state *state = PyType_GetModuleState(cls);
1807 assert(state != NULL);
1808 if (mbstreamwriter_iwrite(self, strobj, state->str_write)) {
1809 return NULL;
1810 }
1811 Py_RETURN_NONE;
1812 }
1813
1814 /*[clinic input]
1815 _multibytecodec.MultibyteStreamWriter.writelines
1816
1817 cls: defining_class
1818 lines: object
1819 /
1820 [clinic start generated code]*/
1821
1822 static PyObject *
1823 _multibytecodec_MultibyteStreamWriter_writelines_impl(MultibyteStreamWriterObject *self,
1824 PyTypeObject *cls,
1825 PyObject *lines)
1826 /*[clinic end generated code: output=b4c99d2cf23ffb88 input=a6d5fe7c74972a34]*/
1827 {
1828 PyObject *strobj;
1829 int i, r;
1830
1831 if (!PySequence_Check(lines)) {
1832 PyErr_SetString(PyExc_TypeError,
1833 "arg must be a sequence object");
1834 return NULL;
1835 }
1836
1837 module_state *state = PyType_GetModuleState(cls);
1838 assert(state != NULL);
1839 for (i = 0; i < PySequence_Length(lines); i++) {
1840 /* length can be changed even within this loop */
1841 strobj = PySequence_GetItem(lines, i);
1842 if (strobj == NULL)
1843 return NULL;
1844
1845 r = mbstreamwriter_iwrite(self, strobj, state->str_write);
1846 Py_DECREF(strobj);
1847 if (r == -1)
1848 return NULL;
1849 }
1850 /* PySequence_Length() can fail */
1851 if (PyErr_Occurred())
1852 return NULL;
1853
1854 Py_RETURN_NONE;
1855 }
1856
1857 /*[clinic input]
1858 _multibytecodec.MultibyteStreamWriter.reset
1859
1860 cls: defining_class
1861 /
1862
1863 [clinic start generated code]*/
1864
1865 static PyObject *
1866 _multibytecodec_MultibyteStreamWriter_reset_impl(MultibyteStreamWriterObject *self,
1867 PyTypeObject *cls)
1868 /*[clinic end generated code: output=32ef224c2a38aa3d input=28af6a9cd38d1979]*/
1869 {
1870 PyObject *pwrt;
1871
1872 if (!self->pending)
1873 Py_RETURN_NONE;
1874
1875 pwrt = multibytecodec_encode(self->codec, &self->state,
1876 self->pending, NULL, self->errors,
1877 MBENC_FLUSH | MBENC_RESET);
1878 /* some pending buffer can be truncated when UnicodeEncodeError is
1879 * raised on 'strict' mode. but, 'reset' method is designed to
1880 * reset the pending buffer or states so failed string sequence
1881 * ought to be missed */
1882 Py_CLEAR(self->pending);
1883 if (pwrt == NULL)
1884 return NULL;
1885
1886 assert(PyBytes_Check(pwrt));
1887
1888 module_state *state = PyType_GetModuleState(cls);
1889 assert(state != NULL);
1890
1891 if (PyBytes_Size(pwrt) > 0) {
1892 PyObject *wr;
1893
1894 wr = PyObject_CallMethodOneArg(self->stream, state->str_write, pwrt);
1895 if (wr == NULL) {
1896 Py_DECREF(pwrt);
1897 return NULL;
1898 }
1899 }
1900 Py_DECREF(pwrt);
1901
1902 Py_RETURN_NONE;
1903 }
1904
1905 static PyObject *
1906 mbstreamwriter_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
1907 {
1908 MultibyteStreamWriterObject *self;
1909 PyObject *stream, *codec = NULL;
1910 char *errors = NULL;
1911
1912 if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|s:StreamWriter",
1913 streamkwarglist, &stream, &errors))
1914 return NULL;
1915
1916 self = (MultibyteStreamWriterObject *)type->tp_alloc(type, 0);
1917 if (self == NULL)
1918 return NULL;
1919
1920 codec = PyObject_GetAttrString((PyObject *)type, "codec");
1921 if (codec == NULL)
1922 goto errorexit;
1923
1924 module_state *state = find_state_by_def(type);
1925 if (!MultibyteCodec_Check(state, codec)) {
1926 PyErr_SetString(PyExc_TypeError, "codec is unexpected type");
1927 goto errorexit;
1928 }
1929
1930 self->codec = ((MultibyteCodecObject *)codec)->codec;
1931 self->stream = Py_NewRef(stream);
1932 self->pending = NULL;
1933 self->errors = internal_error_callback(errors);
1934 if (self->errors == NULL)
1935 goto errorexit;
1936 if (self->codec->encinit != NULL &&
1937 self->codec->encinit(&self->state, self->codec) != 0)
1938 goto errorexit;
1939
1940 Py_DECREF(codec);
1941 return (PyObject *)self;
1942
1943 errorexit:
1944 Py_XDECREF(self);
1945 Py_XDECREF(codec);
1946 return NULL;
1947 }
1948
1949 static int
1950 mbstreamwriter_init(PyObject *self, PyObject *args, PyObject *kwds)
1951 {
1952 return 0;
1953 }
1954
1955 static int
1956 mbstreamwriter_traverse(PyObject *op, visitproc visit, void *arg)
1957 {
1958 MultibyteStreamWriterObject *self = _MultibyteStreamWriterObject_CAST(op);
1959 if (ERROR_ISCUSTOM(self->errors))
1960 Py_VISIT(self->errors);
1961 Py_VISIT(self->stream);
1962 return 0;
1963 }
1964
1965 static void
1966 mbstreamwriter_dealloc(PyObject *op)
1967 {
1968 MultibyteStreamWriterObject *self = _MultibyteStreamWriterObject_CAST(op);
1969 PyTypeObject *tp = Py_TYPE(self);
1970 PyObject_GC_UnTrack(self);
1971 ERROR_DECREF(self->errors);
1972 Py_XDECREF(self->stream);
1973 tp->tp_free(self);
1974 Py_DECREF(tp);
1975 }
1976
1977 static struct PyMethodDef mbstreamwriter_methods[] = {
1978 _MULTIBYTECODEC_MULTIBYTESTREAMWRITER_WRITE_METHODDEF
1979 _MULTIBYTECODEC_MULTIBYTESTREAMWRITER_WRITELINES_METHODDEF
1980 _MULTIBYTECODEC_MULTIBYTESTREAMWRITER_RESET_METHODDEF
1981 {NULL, NULL},
1982 };
1983
1984 static PyMemberDef mbstreamwriter_members[] = {
1985 {"stream", _Py_T_OBJECT,
1986 offsetof(MultibyteStreamWriterObject, stream),
1987 Py_READONLY, NULL},
1988 {NULL,}
1989 };
1990
1991 static PyType_Slot writer_slots[] = {
1992 {Py_tp_dealloc, mbstreamwriter_dealloc},
1993 {Py_tp_getattro, PyObject_GenericGetAttr},
1994 {Py_tp_traverse, mbstreamwriter_traverse},
1995 {Py_tp_methods, mbstreamwriter_methods},
1996 {Py_tp_members, mbstreamwriter_members},
1997 {Py_tp_getset, codecctx_getsets},
1998 {Py_tp_init, mbstreamwriter_init},
1999 {Py_tp_new, mbstreamwriter_new},
2000 {0, NULL},
2001 };
2002
2003 static PyType_Spec writer_spec = {
2004 .name = MODULE_NAME ".MultibyteStreamWriter",
2005 .basicsize = sizeof(MultibyteStreamWriterObject),
2006 .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE |
2007 Py_TPFLAGS_IMMUTABLETYPE),
2008 .slots = writer_slots,
2009 };
2010
2011
2012 /*[clinic input]
2013 _multibytecodec.__create_codec
2014
2015 arg: object
2016 /
2017 [clinic start generated code]*/
2018
2019 static PyObject *
2020 _multibytecodec___create_codec(PyObject *module, PyObject *arg)
2021 /*[clinic end generated code: output=cfa3dce8260e809d input=6840b2a6b183fcfa]*/
2022 {
2023 MultibyteCodecObject *self;
2024
2025 if (!PyCapsule_IsValid(arg, CODEC_CAPSULE)) {
2026 PyErr_SetString(PyExc_ValueError, "argument type invalid");
2027 return NULL;
2028 }
2029
2030 codec_capsule *data = PyCapsule_GetPointer(arg, CODEC_CAPSULE);
2031 const MultibyteCodec *codec = data->codec;
2032 if (codec->codecinit != NULL && codec->codecinit(codec) != 0)
2033 return NULL;
2034
2035 module_state *state = get_module_state(module);
2036 self = PyObject_GC_New(MultibyteCodecObject, state->multibytecodec_type);
2037 if (self == NULL)
2038 return NULL;
2039 self->codec = codec;
2040 self->cjk_module = Py_NewRef(data->cjk_module);
2041
2042 PyObject_GC_Track(self);
2043 return (PyObject *)self;
2044 }
2045
2046 static int
2047 _multibytecodec_traverse(PyObject *mod, visitproc visit, void *arg)
2048 {
2049 module_state *state = get_module_state(mod);
2050 Py_VISIT(state->multibytecodec_type);
2051 Py_VISIT(state->encoder_type);
2052 Py_VISIT(state->decoder_type);
2053 Py_VISIT(state->reader_type);
2054 Py_VISIT(state->writer_type);
2055 return 0;
2056 }
2057
2058 static int
2059 _multibytecodec_clear(PyObject *mod)
2060 {
2061 module_state *state = get_module_state(mod);
2062 Py_CLEAR(state->multibytecodec_type);
2063 Py_CLEAR(state->encoder_type);
2064 Py_CLEAR(state->decoder_type);
2065 Py_CLEAR(state->reader_type);
2066 Py_CLEAR(state->writer_type);
2067 Py_CLEAR(state->str_write);
2068 return 0;
2069 }
2070
2071 static void
2072 _multibytecodec_free(void *mod)
2073 {
2074 (void)_multibytecodec_clear((PyObject *)mod);
2075 }
2076
2077 #define CREATE_TYPE(module, type, spec) \
2078 do { \
2079 type = (PyTypeObject *)PyType_FromModuleAndSpec(module, spec, NULL); \
2080 if (!type) { \
2081 return -1; \
2082 } \
2083 } while (0)
2084
2085 #define ADD_TYPE(module, type) \
2086 do { \
2087 if (PyModule_AddType(module, type) < 0) { \
2088 return -1; \
2089 } \
2090 } while (0)
2091
2092 static int
2093 _multibytecodec_exec(PyObject *mod)
2094 {
2095 module_state *state = get_module_state(mod);
2096 state->str_write = PyUnicode_InternFromString("write");
2097 if (state->str_write == NULL) {
2098 return -1;
2099 }
2100 CREATE_TYPE(mod, state->multibytecodec_type, &multibytecodec_spec);
2101 CREATE_TYPE(mod, state->encoder_type, &encoder_spec);
2102 CREATE_TYPE(mod, state->decoder_type, &decoder_spec);
2103 CREATE_TYPE(mod, state->reader_type, &reader_spec);
2104 CREATE_TYPE(mod, state->writer_type, &writer_spec);
2105
2106 ADD_TYPE(mod, state->encoder_type);
2107 ADD_TYPE(mod, state->decoder_type);
2108 ADD_TYPE(mod, state->reader_type);
2109 ADD_TYPE(mod, state->writer_type);
2110 return 0;
2111 }
2112
2113 #undef CREATE_TYPE
2114 #undef ADD_TYPE
2115
2116 static struct PyMethodDef _multibytecodec_methods[] = {
2117 _MULTIBYTECODEC___CREATE_CODEC_METHODDEF
2118 {NULL, NULL},
2119 };
2120
2121 static PyModuleDef_Slot _multibytecodec_slots[] = {
2122 {Py_mod_exec, _multibytecodec_exec},
2123 {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
2124 {Py_mod_gil, Py_MOD_GIL_NOT_USED},
2125 {0, NULL}
2126 };
2127
2128 static struct PyModuleDef _multibytecodecmodule = {
2129 .m_base = PyModuleDef_HEAD_INIT,
2130 .m_name = "_multibytecodec",
2131 .m_size = sizeof(module_state),
2132 .m_methods = _multibytecodec_methods,
2133 .m_slots = _multibytecodec_slots,
2134 .m_traverse = _multibytecodec_traverse,
2135 .m_clear = _multibytecodec_clear,
2136 .m_free = _multibytecodec_free,
2137 };
2138
2139 PyMODINIT_FUNC
2140 PyInit__multibytecodec(void)
2141 {
2142 return PyModuleDef_Init(&_multibytecodecmodule);
2143 }