Debian Codegraph @

Modules/cjkcodecs/multibytecodec.c

raw
1 /*
2 * multibytecodec.c: Common Multibyte Codec Implementation
3 *
4 * Written by Hye-Shik Chang <[email protected]>
5 */
6
7 #ifndef Py_BUILD_CORE_BUILTIN
8 # define Py_BUILD_CORE_MODULE 1
9 #endif
10
11 #include "Python.h"
12
13 #include "multibytecodec.h"
14 #include "clinic/multibytecodec.c.h"
15
16 #include <stddef.h> // offsetof()
17
18 #define MODULE_NAME "_multibytecodec"
19
20 typedef struct {
21 PyTypeObject *encoder_type;
22 PyTypeObject *decoder_type;
23 PyTypeObject *reader_type;
24 PyTypeObject *writer_type;
25 PyTypeObject *multibytecodec_type;
26 PyObject *str_write;
27 } module_state;
28
29 static module_state *
30 get_module_state(PyObject *module)
31 {
32 module_state *state = PyModule_GetState(module);
33 assert(state != NULL);
34 return state;
35 }
36
37 static struct PyModuleDef _multibytecodecmodule;
38
39 static module_state *
40 find_state_by_def(PyTypeObject *type)
41 {
42 PyObject *module = PyType_GetModuleByDef(type, &_multibytecodecmodule);
43 assert(module != NULL);
44 return get_module_state(module);
45 }
46
47 #define clinic_get_state() find_state_by_def(type)
48 /*[clinic input]
49 module _multibytecodec
50 class _multibytecodec.MultibyteCodec "MultibyteCodecObject *" "clinic_get_state()->multibytecodec_type"
51 class _multibytecodec.MultibyteIncrementalEncoder "MultibyteIncrementalEncoderObject *" "clinic_get_state()->encoder_type"
52 class _multibytecodec.MultibyteIncrementalDecoder "MultibyteIncrementalDecoderObject *" "clinic_get_state()->decoder_type"
53 class _multibytecodec.MultibyteStreamReader "MultibyteStreamReaderObject *" "clinic_get_state()->reader_type"
54 class _multibytecodec.MultibyteStreamWriter "MultibyteStreamWriterObject *" "clinic_get_state()->writer_type"
55 [clinic start generated code]*/
56 /*[clinic end generated code: output=da39a3ee5e6b4b0d input=305a76dfdd24b99c]*/
57 #undef clinic_get_state
58
59 #define _MultibyteCodec_CAST(op) ((MultibyteCodec *)(op))
60 #define _MultibyteCodecObject_CAST(op) ((MultibyteCodecObject *)(op))
61
62 #define _MultibyteStatefulCodecContext_CAST(op) \
63 ((MultibyteStatefulCodecContext *)(op))
64
65 #define _MultibyteStatefulEncoderContext_CAST(op) \
66 ((MultibyteStatefulEncoderContext *)(op))
67 #define _MultibyteStatefulDecoderContext_CAST(op) \
68 ((MultibyteStatefulDecoderContext *)(op))
69
70 #define _MultibyteIncrementalEncoderObject_CAST(op) \
71 ((MultibyteIncrementalEncoderObject *)(op))
72 #define _MultibyteIncrementalDecoderObject_CAST(op) \
73 ((MultibyteIncrementalDecoderObject *)(op))
74
75 #define _MultibyteStreamReaderObject_CAST(op) \
76 ((MultibyteStreamReaderObject *)(op))
77 #define _MultibyteStreamWriterObject_CAST(op) \
78 ((MultibyteStreamWriterObject *)(op))
79
80 typedef struct {
81 PyObject *inobj;
82 Py_ssize_t inpos, inlen;
83 unsigned char *outbuf, *outbuf_end;
84 PyObject *excobj, *outobj;
85 } MultibyteEncodeBuffer;
86
87 typedef struct {
88 const unsigned char *inbuf, *inbuf_top, *inbuf_end;
89 PyObject *excobj;
90 _PyUnicodeWriter writer;
91 } MultibyteDecodeBuffer;
92
93 static char *incnewkwarglist[] = {"errors", NULL};
94 static char *streamkwarglist[] = {"stream", "errors", NULL};
95
96 static PyObject *multibytecodec_encode(const MultibyteCodec *,
97 MultibyteCodec_State *, PyObject *, Py_ssize_t *,
98 PyObject *, int);
99
100 #define MBENC_RESET MBENC_MAX<<1 /* reset after an encoding session */
101
102 static PyObject *
103 make_tuple(PyObject *object, Py_ssize_t len)
104 {
105 PyObject *v, *w;
106
107 if (object == NULL)
108 return NULL;
109
110 v = PyTuple_New(2);
111 if (v == NULL) {
112 Py_DECREF(object);
113 return NULL;
114 }
115 PyTuple_SET_ITEM(v, 0, object);
116
117 w = PyLong_FromSsize_t(len);
118 if (w == NULL) {
119 Py_DECREF(v);
120 return NULL;
121 }
122 PyTuple_SET_ITEM(v, 1, w);
123
124 return v;
125 }
126
127 static PyObject *
128 internal_error_callback(const char *errors)
129 {
130 if (errors == NULL || strcmp(errors, "strict") == 0)
131 return ERROR_STRICT;
132 else if (strcmp(errors, "ignore") == 0)
133 return ERROR_IGNORE;
134 else if (strcmp(errors, "replace") == 0)
135 return ERROR_REPLACE;
136 else
137 return PyUnicode_FromString(errors);
138 }
139
140 static PyObject *
141 call_error_callback(PyObject *errors, PyObject *exc)
142 {
143 PyObject *cb, *r;
144 const char *str;
145
146 assert(PyUnicode_Check(errors));
147 str = PyUnicode_AsUTF8(errors);
148 if (str == NULL)
149 return NULL;
150 cb = PyCodec_LookupError(str);
151 if (cb == NULL)
152 return NULL;
153
154 r = PyObject_CallOneArg(cb, exc);
155 Py_DECREF(cb);
156 return r;
157 }
158
159 static PyObject *
160 codecctx_errors_get(PyObject *op, void *Py_UNUSED(closure))
161 {
162 const char *errors;
163 MultibyteStatefulCodecContext *self = _MultibyteStatefulCodecContext_CAST(op);
164
165 if (self->errors == ERROR_STRICT)
166 errors = "strict";
167 else if (self->errors == ERROR_IGNORE)
168 errors = "ignore";
169 else if (self->errors == ERROR_REPLACE)
170 errors = "replace";
171 else {
172 return Py_NewRef(self->errors);
173 }
174
175 return PyUnicode_FromString(errors);
176 }
177
178 static int
179 codecctx_errors_set(PyObject *op, PyObject *value, void *Py_UNUSED(closure))
180 {
181 PyObject *cb;
182 const char *str;
183 MultibyteStatefulCodecContext *self = _MultibyteStatefulCodecContext_CAST(op);
184
185 if (value == NULL) {
186 PyErr_SetString(PyExc_AttributeError, "cannot delete attribute");
187 return -1;
188 }
189 if (!PyUnicode_Check(value)) {
190 PyErr_SetString(PyExc_TypeError, "errors must be a string");
191 return -1;
192 }
193
194 str = PyUnicode_AsUTF8(value);
195 if (str == NULL)
196 return -1;
197
198 cb = internal_error_callback(str);
199 if (cb == NULL)
200 return -1;
201
202 ERROR_DECREF(self->errors);
203 self->errors = cb;
204 return 0;
205 }
206
207 /* This getset handlers list is used by all the stateful codec objects */
208 static PyGetSetDef codecctx_getsets[] = {
209 {"errors", codecctx_errors_get, codecctx_errors_set,
210 PyDoc_STR("how to treat errors")},
211 {NULL,}
212 };
213
214 static int
215 expand_encodebuffer(MultibyteEncodeBuffer *buf, Py_ssize_t esize)
216 {
217 Py_ssize_t orgpos, orgsize, incsize;
218
219 orgpos = (Py_ssize_t)((char *)buf->outbuf -
220 PyBytes_AS_STRING(buf->outobj));
221 orgsize = PyBytes_GET_SIZE(buf->outobj);
222 incsize = (esize < (orgsize >> 1) ? (orgsize >> 1) | 1 : esize);
223
224 if (orgsize > PY_SSIZE_T_MAX - incsize) {
225 PyErr_NoMemory();
226 return -1;
227 }
228
229 if (_PyBytes_Resize(&buf->outobj, orgsize + incsize) == -1)
230 return -1;
231
232 buf->outbuf = (unsigned char *)PyBytes_AS_STRING(buf->outobj) +orgpos;
233 buf->outbuf_end = (unsigned char *)PyBytes_AS_STRING(buf->outobj)
234 + PyBytes_GET_SIZE(buf->outobj);
235
236 return 0;
237 }
238 #define REQUIRE_ENCODEBUFFER(buf, s) do { \
239 if ((s) < 0 || (s) > (buf)->outbuf_end - (buf)->outbuf) \
240 if (expand_encodebuffer(buf, s) == -1) \
241 goto errorexit; \
242 } while(0)
243
244
245 /**
246 * MultibyteCodec object
247 */
248
249 static int
250 multibytecodec_encerror(const MultibyteCodec *codec,
251 MultibyteCodec_State *state,
252 MultibyteEncodeBuffer *buf,
253 PyObject *errors, Py_ssize_t e)
254 {
255 PyObject *retobj = NULL, *retstr = NULL, *tobj;
256 Py_ssize_t retstrsize, newpos;
257 Py_ssize_t esize, start, end;
258 const char *reason;
259
260 if (e > 0) {
261 reason = "illegal multibyte sequence";
262 esize = e;
263 }
264 else {
265 switch (e) {
266 case MBERR_TOOSMALL:
267 REQUIRE_ENCODEBUFFER(buf, -1);
268 return 0; /* retry it */
269 case MBERR_TOOFEW:
270 reason = "incomplete multibyte sequence";
271 esize = (Py_ssize_t)buf->inpos;
272 break;
273 case MBERR_INTERNAL:
274 PyErr_SetString(PyExc_RuntimeError,
275 "internal codec error");
276 return -1;
277 default:
278 PyErr_SetString(PyExc_RuntimeError,
279 "unknown runtime error");
280 return -1;
281 }
282 }
283
284 if (errors == ERROR_REPLACE) {
285 PyObject *replchar;
286 Py_ssize_t r;
287 Py_ssize_t inpos;
288 int kind;
289 const void *data;
290
291 replchar = PyUnicode_FromOrdinal('?');
292 if (replchar == NULL)
293 goto errorexit;
294 kind = PyUnicode_KIND(replchar);
295 data = PyUnicode_DATA(replchar);
296
297 inpos = 0;
298 for (;;) {
299 Py_ssize_t outleft = (Py_ssize_t)(buf->outbuf_end - buf->outbuf);
300
301 r = codec->encode(state, codec,
302 kind, data, &inpos, 1,
303 &buf->outbuf, outleft, 0);
304 if (r == MBERR_TOOSMALL) {
305 REQUIRE_ENCODEBUFFER(buf, -1);
306 continue;
307 }
308 else
309 break;
310 }
311
312 Py_DECREF(replchar);
313
314 if (r != 0) {
315 REQUIRE_ENCODEBUFFER(buf, 1);
316 *buf->outbuf++ = '?';
317 }
318 }
319 if (errors == ERROR_IGNORE || errors == ERROR_REPLACE) {
320 buf->inpos += esize;
321 return 0;
322 }
323
324 start = (Py_ssize_t)buf->inpos;
325 end = start + esize;
326
327 /* use cached exception object if available */
328 if (buf->excobj == NULL) {
329 buf->excobj = PyObject_CallFunction(PyExc_UnicodeEncodeError,
330 "sOnns",
331 codec->encoding, buf->inobj,
332 start, end, reason);
333 if (buf->excobj == NULL)
334 goto errorexit;
335 }
336 else
337 if (PyUnicodeEncodeError_SetStart(buf->excobj, start) != 0 ||
338 PyUnicodeEncodeError_SetEnd(buf->excobj, end) != 0 ||
339 PyUnicodeEncodeError_SetReason(buf->excobj, reason) != 0)
340 goto errorexit;
341
342 if (errors == ERROR_STRICT) {
343 PyCodec_StrictErrors(buf->excobj);
344 goto errorexit;
345 }
346
347 retobj = call_error_callback(errors, buf->excobj);
348 if (retobj == NULL)
349 goto errorexit;
350
351 if (!PyTuple_Check(retobj) || PyTuple_GET_SIZE(retobj) != 2 ||
352 (!PyUnicode_Check((tobj = PyTuple_GET_ITEM(retobj, 0))) && !PyBytes_Check(tobj)) ||
353 !PyLong_Check(PyTuple_GET_ITEM(retobj, 1))) {
354 PyErr_SetString(PyExc_TypeError,
355 "encoding error handler must return "
356 "(str, int) tuple");
357 goto errorexit;
358 }
359
360 if (PyUnicode_Check(tobj)) {
361 Py_ssize_t inpos;
362
363 retstr = multibytecodec_encode(codec, state, tobj,
364 &inpos, ERROR_STRICT,
365 MBENC_FLUSH);
366 if (retstr == NULL)
367 goto errorexit;
368 }
369 else {
370 retstr = Py_NewRef(tobj);
371 }
372
373 assert(PyBytes_Check(retstr));
374 retstrsize = PyBytes_GET_SIZE(retstr);
375 if (retstrsize > 0) {
376 REQUIRE_ENCODEBUFFER(buf, retstrsize);
377 memcpy(buf->outbuf, PyBytes_AS_STRING(retstr), retstrsize);
378 buf->outbuf += retstrsize;
379 }
380
381 newpos = PyLong_AsSsize_t(PyTuple_GET_ITEM(retobj, 1));
382 if (newpos < 0 && !PyErr_Occurred())
383 newpos += (Py_ssize_t)buf->inlen;
384 if (newpos < 0 || newpos > buf->inlen) {
385 PyErr_Clear();
386 PyErr_Format(PyExc_IndexError,
387 "position %zd from error handler out of bounds",
388 newpos);
389 goto errorexit;
390 }
391 buf->inpos = newpos;
392
393 Py_DECREF(retobj);
394 Py_DECREF(retstr);
395 return 0;
396
397 errorexit:
398 Py_XDECREF(retobj);
399 Py_XDECREF(retstr);
400 return -1;
401 }
402
403 static int
404 multibytecodec_decerror(const MultibyteCodec *codec,
405 MultibyteCodec_State *state,
406 MultibyteDecodeBuffer *buf,
407 PyObject *errors, Py_ssize_t e)
408 {
409 PyObject *retobj = NULL, *retuni = NULL;
410 Py_ssize_t newpos;
411 const char *reason;
412 Py_ssize_t esize, start, end;
413
414 if (e > 0) {
415 reason = "illegal multibyte sequence";
416 esize = e;
417 }
418 else {
419 switch (e) {
420 case MBERR_TOOSMALL:
421 return 0; /* retry it */
422 case MBERR_TOOFEW:
423 reason = "incomplete multibyte sequence";
424 esize = (Py_ssize_t)(buf->inbuf_end - buf->inbuf);
425 break;
426 case MBERR_INTERNAL:
427 PyErr_SetString(PyExc_RuntimeError,
428 "internal codec error");
429 return -1;
430 case MBERR_EXCEPTION:
431 return -1;
432 default:
433 PyErr_SetString(PyExc_RuntimeError,
434 "unknown runtime error");
435 return -1;
436 }
437 }
438
439 if (errors == ERROR_REPLACE) {
440 if (_PyUnicodeWriter_WriteChar(&buf->writer,
441 Py_UNICODE_REPLACEMENT_CHARACTER) < 0)
442 goto errorexit;
443 }
444 if (errors == ERROR_IGNORE || errors == ERROR_REPLACE) {
445 buf->inbuf += esize;
446 return 0;
447 }
448
449 start = (Py_ssize_t)(buf->inbuf - buf->inbuf_top);
450 end = start + esize;
451
452 /* use cached exception object if available */
453 if (buf->excobj == NULL) {
454 buf->excobj = PyUnicodeDecodeError_Create(codec->encoding,
455 (const char *)buf->inbuf_top,
456 (Py_ssize_t)(buf->inbuf_end - buf->inbuf_top),
457 start, end, reason);
458 if (buf->excobj == NULL)
459 goto errorexit;
460 }
461 else
462 if (PyUnicodeDecodeError_SetStart(buf->excobj, start) ||
463 PyUnicodeDecodeError_SetEnd(buf->excobj, end) ||
464 PyUnicodeDecodeError_SetReason(buf->excobj, reason))
465 goto errorexit;
466
467 if (errors == ERROR_STRICT) {
468 PyCodec_StrictErrors(buf->excobj);
469 goto errorexit;
470 }
471
472 retobj = call_error_callback(errors, buf->excobj);
473 if (retobj == NULL)
474 goto errorexit;
475
476 if (!PyTuple_Check(retobj) || PyTuple_GET_SIZE(retobj) != 2 ||
477 !PyUnicode_Check((retuni = PyTuple_GET_ITEM(retobj, 0))) ||
478 !PyLong_Check(PyTuple_GET_ITEM(retobj, 1))) {
479 PyErr_SetString(PyExc_TypeError,
480 "decoding error handler must return "
481 "(str, int) tuple");
482 goto errorexit;
483 }
484
485 if (_PyUnicodeWriter_WriteStr(&buf->writer, retuni) < 0)
486 goto errorexit;
487
488 newpos = PyLong_AsSsize_t(PyTuple_GET_ITEM(retobj, 1));
489 if (newpos < 0 && !PyErr_Occurred())
490 newpos += (Py_ssize_t)(buf->inbuf_end - buf->inbuf_top);
491 if (newpos < 0 || buf->inbuf_top + newpos > buf->inbuf_end) {
492 PyErr_Clear();
493 PyErr_Format(PyExc_IndexError,
494 "position %zd from error handler out of bounds",
495 newpos);
496 goto errorexit;
497 }
498 buf->inbuf = buf->inbuf_top + newpos;
499 Py_DECREF(retobj);
500 return 0;
501
502 errorexit:
503 Py_XDECREF(retobj);
504 return -1;
505 }
506
507 static PyObject *
508 multibytecodec_encode(const MultibyteCodec *codec,
509 MultibyteCodec_State *state,
510 PyObject *text, Py_ssize_t *inpos_t,
511 PyObject *errors, int flags)
512 {
513 MultibyteEncodeBuffer buf;
514 Py_ssize_t finalsize, r = 0;
515 Py_ssize_t datalen;
516 int kind;
517 const void *data;
518
519 datalen = PyUnicode_GET_LENGTH(text);
520
521 if (datalen == 0 && !(flags & MBENC_RESET))
522 return PyBytes_FromStringAndSize(NULL, 0);
523
524 buf.excobj = NULL;
525 buf.outobj = NULL;
526 buf.inobj = text; /* borrowed reference */
527 buf.inpos = 0;
528 buf.inlen = datalen;
529 kind = PyUnicode_KIND(buf.inobj);
530 data = PyUnicode_DATA(buf.inobj);
531
532 if (datalen > (PY_SSIZE_T_MAX - 16) / 2) {
533 PyErr_NoMemory();
534 goto errorexit;
535 }
536
537 buf.outobj = PyBytes_FromStringAndSize(NULL, datalen * 2 + 16);
538 if (buf.outobj == NULL)
539 goto errorexit;
540 buf.outbuf = (unsigned char *)PyBytes_AS_STRING(buf.outobj);
541 buf.outbuf_end = buf.outbuf + PyBytes_GET_SIZE(buf.outobj);
542
543 while (buf.inpos < buf.inlen) {
544 /* we don't reuse inleft and outleft here.
545 * error callbacks can relocate the cursor anywhere on buffer*/
546 Py_ssize_t outleft = (Py_ssize_t)(buf.outbuf_end - buf.outbuf);
547
548 r = codec->encode(state, codec,
549 kind, data,
550 &buf.inpos, buf.inlen,
551 &buf.outbuf, outleft, flags);
552 if ((r == 0) || (r == MBERR_TOOFEW && !(flags & MBENC_FLUSH)))
553 break;
554 else if (multibytecodec_encerror(codec, state, &buf, errors,r))
555 goto errorexit;
556 else if (r == MBERR_TOOFEW)
557 break;
558 }
559
560 if (codec->encreset != NULL && (flags & MBENC_RESET))
561 for (;;) {
562 Py_ssize_t outleft;
563
564 outleft = (Py_ssize_t)(buf.outbuf_end - buf.outbuf);
565 r = codec->encreset(state, codec, &buf.outbuf,
566 outleft);
567 if (r == 0)
568 break;
569 else if (multibytecodec_encerror(codec, state,
570 &buf, errors, r))
571 goto errorexit;
572 }
573
574 finalsize = (Py_ssize_t)((char *)buf.outbuf -
575 PyBytes_AS_STRING(buf.outobj));
576
577 if (finalsize != PyBytes_GET_SIZE(buf.outobj))
578 if (_PyBytes_Resize(&buf.outobj, finalsize) == -1)
579 goto errorexit;
580
581 if (inpos_t)
582 *inpos_t = buf.inpos;
583 Py_XDECREF(buf.excobj);
584 return buf.outobj;
585
586 errorexit:
587 Py_XDECREF(buf.excobj);
588 Py_XDECREF(buf.outobj);
589 return NULL;
590 }
591
592 /*[clinic input]
593 _multibytecodec.MultibyteCodec.encode
594
595 input: object
596 errors: str(accept={str, NoneType}) = None
597
598 Return an encoded string version of 'input'.
599
600 'errors' may be given to set a different error handling scheme.
601 Default is 'strict' meaning that encoding errors raise
602 a UnicodeEncodeError. Other possible values are 'ignore', 'replace'
603 and 'xmlcharrefreplace' as well as any other name registered with
604 codecs.register_error that can handle UnicodeEncodeErrors.
605 [clinic start generated code]*/
606
607 static PyObject *
608 _multibytecodec_MultibyteCodec_encode_impl(MultibyteCodecObject *self,
609 PyObject *input,
610 const char *errors)
611 /*[clinic end generated code: output=7b26652045ba56a9 input=980002ed1447697b]*/
612 {
613 MultibyteCodec_State state;
614 PyObject *errorcb, *r, *ucvt;
615 Py_ssize_t datalen;
616
617 if (PyUnicode_Check(input))
618 ucvt = NULL;
619 else {
620 input = ucvt = PyObject_Str(input);
621 if (input == NULL)
622 return NULL;
623 else if (!PyUnicode_Check(input)) {
624 PyErr_SetString(PyExc_TypeError,
625 "couldn't convert the object to unicode.");
626 Py_DECREF(ucvt);
627 return NULL;
628 }
629 }
630
631 datalen = PyUnicode_GET_LENGTH(input);
632
633 errorcb = internal_error_callback(errors);
634 if (errorcb == NULL) {
635 Py_XDECREF(ucvt);
636 return NULL;
637 }
638
639 if (self->codec->encinit != NULL &&
640 self->codec->encinit(&state, self->codec) != 0)
641 goto errorexit;
642 r = multibytecodec_encode(self->codec, &state,
643 input, NULL, errorcb,
644 MBENC_FLUSH | MBENC_RESET);
645 if (r == NULL)
646 goto errorexit;
647
648 ERROR_DECREF(errorcb);
649 Py_XDECREF(ucvt);
650 return make_tuple(r, datalen);
651
652 errorexit:
653 ERROR_DECREF(errorcb);
654 Py_XDECREF(ucvt);
655 return NULL;
656 }
657
658 /*[clinic input]
659 _multibytecodec.MultibyteCodec.decode
660
661 input: Py_buffer
662 errors: str(accept={str, NoneType}) = None
663
664 Decodes 'input'.
665
666 'errors' may be given to set a different error handling scheme.
667 Default is 'strict' meaning that encoding errors raise
668 a UnicodeDecodeError. Other possible values are 'ignore' and
669 'replace' as well as any other name registered with
670 codecs.register_error that is able to handle UnicodeDecodeErrors."
671 [clinic start generated code]*/
672
673 static PyObject *
674 _multibytecodec_MultibyteCodec_decode_impl(MultibyteCodecObject *self,
675 Py_buffer *input,
676 const char *errors)
677 /*[clinic end generated code: output=ff419f65bad6cc77 input=dbf93d8bb98ca440]*/
678 {
679 MultibyteCodec_State state;
680 MultibyteDecodeBuffer buf;
681 PyObject *errorcb, *res;
682 const char *data;
683 Py_ssize_t datalen;
684
685 data = input->buf;
686 datalen = input->len;
687
688 errorcb = internal_error_callback(errors);
689 if (errorcb == NULL) {
690 return NULL;
691 }
692
693 if (datalen == 0) {
694 ERROR_DECREF(errorcb);
695 return make_tuple(Py_GetConstant(Py_CONSTANT_EMPTY_STR), 0);
696 }
697
698 _PyUnicodeWriter_Init(&buf.writer);
699 buf.writer.min_length = datalen;
700 buf.excobj = NULL;
701 buf.inbuf = buf.inbuf_top = (unsigned char *)data;
702 buf.inbuf_end = buf.inbuf_top + datalen;
703
704 if (self->codec->decinit != NULL &&
705 self->codec->decinit(&state, self->codec) != 0)
706 goto errorexit;
707
708 while (buf.inbuf < buf.inbuf_end) {
709 Py_ssize_t inleft, r;
710
711 inleft = (Py_ssize_t)(buf.inbuf_end - buf.inbuf);
712
713 r = self->codec->decode(&state, self->codec,
714 &buf.inbuf, inleft, &buf.writer);
715 if (r == 0)
716 break;
717 else if (multibytecodec_decerror(self->codec, &state,
718 &buf, errorcb, r))
719 goto errorexit;
720 }
721
722 res = _PyUnicodeWriter_Finish(&buf.writer);
723 if (res == NULL)
724 goto errorexit;
725
726 Py_XDECREF(buf.excobj);
727 ERROR_DECREF(errorcb);
728 return make_tuple(res, datalen);
729
730 errorexit:
731 ERROR_DECREF(errorcb);
732 Py_XDECREF(buf.excobj);
733 _PyUnicodeWriter_Dealloc(&buf.writer);
734
735 return NULL;
736 }
737
738 static struct PyMethodDef multibytecodec_methods[] = {
739 _MULTIBYTECODEC_MULTIBYTECODEC_ENCODE_METHODDEF
740 _MULTIBYTECODEC_MULTIBYTECODEC_DECODE_METHODDEF
741 {NULL, NULL},
742 };
743
744 static int
745 multibytecodec_clear(PyObject *op)
746 {
747 MultibyteCodecObject *self = _MultibyteCodecObject_CAST(op);
748 Py_CLEAR(self->cjk_module);
749 return 0;
750 }
751
752 static int
753 multibytecodec_traverse(PyObject *op, visitproc visit, void *arg)
754 {
755 MultibyteCodecObject *self = _MultibyteCodecObject_CAST(op);
756 Py_VISIT(Py_TYPE(self));
757 Py_VISIT(self->cjk_module);
758 return 0;
759 }
760
761 static void
762 multibytecodec_dealloc(PyObject *self)
763 {
764 PyObject_GC_UnTrack(self);
765 PyTypeObject *tp = Py_TYPE(self);
766 (void)multibytecodec_clear(self);
767 tp->tp_free(self);
768 Py_DECREF(tp);
769 }
770
771 static PyType_Slot multibytecodec_slots[] = {
772 {Py_tp_dealloc, multibytecodec_dealloc},
773 {Py_tp_getattro, PyObject_GenericGetAttr},
774 {Py_tp_methods, multibytecodec_methods},
775 {Py_tp_traverse, multibytecodec_traverse},
776 {Py_tp_clear, multibytecodec_clear},
777 {0, NULL},
778 };
779
780 static PyType_Spec multibytecodec_spec = {
781 .name = MODULE_NAME ".MultibyteCodec",
782 .basicsize = sizeof(MultibyteCodecObject),
783 .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC |
784 Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_IMMUTABLETYPE),
785 .slots = multibytecodec_slots,
786 };
787
788
789 /**
790 * Utility functions for stateful codec mechanism
791 */
792
793 #define STATEFUL_DCTX(o) ((MultibyteStatefulDecoderContext *)(o))
794 #define STATEFUL_ECTX(o) ((MultibyteStatefulEncoderContext *)(o))
795
796 static PyObject *
797 encoder_encode_stateful(MultibyteStatefulEncoderContext *ctx,
798 PyObject *unistr, int final)
799 {
800 PyObject *ucvt, *r = NULL;
801 PyObject *inbuf = NULL;
802 Py_ssize_t inpos, datalen;
803 PyObject *origpending = NULL;
804
805 if (PyUnicode_Check(unistr))
806 ucvt = NULL;
807 else {
808 unistr = ucvt = PyObject_Str(unistr);
809 if (unistr == NULL)
810 return NULL;
811 else if (!PyUnicode_Check(unistr)) {
812 PyErr_SetString(PyExc_TypeError,
813 "couldn't convert the object to str.");
814 Py_DECREF(ucvt);
815 return NULL;
816 }
817 }
818
819 if (ctx->pending) {
820 PyObject *inbuf_tmp;
821
822 origpending = Py_NewRef(ctx->pending);
823
824 inbuf_tmp = Py_NewRef(ctx->pending);
825 PyUnicode_Append(&inbuf_tmp, unistr);
826 if (inbuf_tmp == NULL)
827 goto errorexit;
828 Py_CLEAR(ctx->pending);
829 inbuf = inbuf_tmp;
830 }
831 else {
832 origpending = NULL;
833
834 inbuf = Py_NewRef(unistr);
835 }
836 inpos = 0;
837 datalen = PyUnicode_GET_LENGTH(inbuf);
838
839 r = multibytecodec_encode(ctx->codec, &ctx->state,
840 inbuf, &inpos,
841 ctx->errors, final ? MBENC_FLUSH | MBENC_RESET : 0);
842 if (r == NULL) {
843 /* recover the original pending buffer */
844 Py_XSETREF(ctx->pending, origpending);
845 origpending = NULL;
846 goto errorexit;
847 }
848 Py_XDECREF(origpending);
849
850 if (inpos < datalen) {
851 if (datalen - inpos > MAXENCPENDING) {
852 /* normal codecs can't reach here */
853 PyObject *excobj = PyObject_CallFunction(PyExc_UnicodeEncodeError,
854 "sOnns",
855 ctx->codec->encoding,
856 inbuf,
857 inpos, datalen,
858 "pending buffer overflow");
859 if (excobj == NULL) goto errorexit;
860 PyErr_SetObject(PyExc_UnicodeEncodeError, excobj);
861 Py_DECREF(excobj);
862 goto errorexit;
863 }
864 ctx->pending = PyUnicode_Substring(inbuf, inpos, datalen);
865 if (ctx->pending == NULL) {
866 /* normal codecs can't reach here */
867 goto errorexit;
868 }
869 }
870
871 Py_DECREF(inbuf);
872 Py_XDECREF(ucvt);
873 return r;
874
875 errorexit:
876 Py_XDECREF(r);
877 Py_XDECREF(ucvt);
878 Py_XDECREF(origpending);
879 Py_XDECREF(inbuf);
880 return NULL;
881 }
882
883 static int
884 decoder_append_pending(MultibyteStatefulDecoderContext *ctx,
885 MultibyteDecodeBuffer *buf)
886 {
887 Py_ssize_t npendings;
888
889 npendings = (Py_ssize_t)(buf->inbuf_end - buf->inbuf);
890 if (npendings + ctx->pendingsize > MAXDECPENDING ||
891 npendings > PY_SSIZE_T_MAX - ctx->pendingsize) {
892 Py_ssize_t bufsize = (Py_ssize_t)(buf->inbuf_end - buf->inbuf_top);
893 PyObject *excobj = PyUnicodeDecodeError_Create(ctx->codec->encoding,
894 (const char *)buf->inbuf_top,
895 bufsize,
896 0,
897 bufsize,
898 "pending buffer overflow");
899 if (excobj == NULL) return -1;
900 PyErr_SetObject(PyExc_UnicodeDecodeError, excobj);
901 Py_DECREF(excobj);
902 return -1;
903 }
904 memcpy(ctx->pending + ctx->pendingsize, buf->inbuf, npendings);
905 ctx->pendingsize += npendings;
906 return 0;
907 }
908
909 static int
910 decoder_prepare_buffer(MultibyteDecodeBuffer *buf, const char *data,
911 Py_ssize_t size)
912 {
913 buf->inbuf = buf->inbuf_top = (const unsigned char *)data;
914 buf->inbuf_end = buf->inbuf_top + size;
915 buf->writer.min_length += size;
916 return 0;
917 }
918
919 static int
920 decoder_feed_buffer(MultibyteStatefulDecoderContext *ctx,
921 MultibyteDecodeBuffer *buf)
922 {
923 while (buf->inbuf < buf->inbuf_end) {
924 Py_ssize_t inleft;
925 Py_ssize_t r;
926
927 inleft = (Py_ssize_t)(buf->inbuf_end - buf->inbuf);
928
929 r = ctx->codec->decode(&ctx->state, ctx->codec,
930 &buf->inbuf, inleft, &buf->writer);
931 if (r == 0 || r == MBERR_TOOFEW)
932 break;
933 else if (multibytecodec_decerror(ctx->codec, &ctx->state,
934 buf, ctx->errors, r))
935 return -1;
936 }
937 return 0;
938 }
939
940
941 /*[clinic input]
942 _multibytecodec.MultibyteIncrementalEncoder.encode
943
944 input: object
945 final: bool = False
946 [clinic start generated code]*/
947
948 static PyObject *
949 _multibytecodec_MultibyteIncrementalEncoder_encode_impl(MultibyteIncrementalEncoderObject *self,
950 PyObject *input,
951 int final)
952 /*[clinic end generated code: output=123361b6c505e2c1 input=bd5f7d40d43e99b0]*/
953 {
954 return encoder_encode_stateful(STATEFUL_ECTX(self), input, final);
955 }
956
957 /*[clinic input]
958 _multibytecodec.MultibyteIncrementalEncoder.getstate
959 [clinic start generated code]*/
960
961 static PyObject *
962 _multibytecodec_MultibyteIncrementalEncoder_getstate_impl(MultibyteIncrementalEncoderObject *self)
963 /*[clinic end generated code: output=9794a5ace70d7048 input=4a2a82874ffa40bb]*/
964 {
965 /* state made up of 1 byte for buffer size, up to MAXENCPENDING*4 bytes
966 for UTF-8 encoded buffer (each character can use up to 4
967 bytes), and required bytes for MultibyteCodec_State.c. A byte
968 array is used to avoid different compilers generating different
969 values for the same state, e.g. as a result of struct padding.
970 */
971 unsigned char statebytes[1 + MAXENCPENDING*4 + sizeof(self->state.c)];
972 Py_ssize_t statesize;
973 const char *pendingbuffer = NULL;
974 Py_ssize_t pendingsize;
975
976 if (self->pending != NULL) {
977 pendingbuffer = PyUnicode_AsUTF8AndSize(self->pending, &pendingsize);
978 if (pendingbuffer == NULL) {
979 return NULL;
980 }
981 if (pendingsize > MAXENCPENDING*4) {
982 PyObject *excobj = PyObject_CallFunction(PyExc_UnicodeEncodeError,
983 "sOnns",
984 self->codec->encoding,
985 self->pending,
986 0, PyUnicode_GET_LENGTH(self->pending),
987 "pending buffer too large");
988 if (excobj == NULL) {
989 return NULL;
990 }
991 PyErr_SetObject(PyExc_UnicodeEncodeError, excobj);
992 Py_DECREF(excobj);
993 return NULL;
994 }
995 statebytes[0] = (unsigned char)pendingsize;
996 memcpy(statebytes + 1, pendingbuffer, pendingsize);
997 statesize = 1 + pendingsize;
998 } else {
999 statebytes[0] = 0;
1000 statesize = 1;
1001 }
1002 memcpy(statebytes+statesize, self->state.c,
1003 sizeof(self->state.c));
1004 statesize += sizeof(self->state.c);
1005
1006 return (PyObject *)_PyLong_FromByteArray(statebytes, statesize,
1007 1 /* little-endian */ ,
1008 0 /* unsigned */ );
1009 }
1010
1011 /*[clinic input]
1012 _multibytecodec.MultibyteIncrementalEncoder.setstate
1013 state as statelong: object(type='PyLongObject *', subclass_of='&PyLong_Type')
1014 /
1015 [clinic start generated code]*/
1016
1017 static PyObject *
1018 _multibytecodec_MultibyteIncrementalEncoder_setstate_impl(MultibyteIncrementalEncoderObject *self,
1019 PyLongObject *statelong)
1020 /*[clinic end generated code: output=4e5e98ac1f4039ca input=c80fb5830d4d2f76]*/
1021 {
1022 PyObject *pending = NULL;
1023 unsigned char statebytes[1 + MAXENCPENDING*4 + sizeof(self->state.c)];
1024
1025 if (_PyLong_AsByteArray(statelong, statebytes, sizeof(statebytes),
1026 1 /* little-endian */ ,
1027 0 /* unsigned */ ,
1028 1 /* with_exceptions */) < 0) {
1029 goto errorexit;
1030 }
1031
1032 if (statebytes[0] > MAXENCPENDING*4) {
1033 PyErr_SetString(PyExc_UnicodeError, "pending buffer too large");
1034 return NULL;
1035 }
1036
1037 pending = PyUnicode_DecodeUTF8((const char *)statebytes+1,
1038 statebytes[0], "strict");
1039 if (pending == NULL) {
1040 goto errorexit;
1041 }
1042
1043 Py_XSETREF(self->pending, pending);
1044 memcpy(self->state.c, statebytes+1+statebytes[0],
1045 sizeof(self->state.c));
1046
1047 Py_RETURN_NONE;
1048
1049 errorexit:
1050 Py_XDECREF(pending);
1051 return NULL;
1052 }
1053
1054 /*[clinic input]
1055 _multibytecodec.MultibyteIncrementalEncoder.reset
1056 [clinic start generated code]*/
1057
1058 static PyObject *
1059 _multibytecodec_MultibyteIncrementalEncoder_reset_impl(MultibyteIncrementalEncoderObject *self)
1060 /*[clinic end generated code: output=b4125d8f537a253f input=930f06760707b6ea]*/
1061 {
1062 /* Longest output: 4 bytes (b'\x0F\x1F(B') with ISO 2022 */
1063 unsigned char buffer[4], *outbuf;
1064 Py_ssize_t r;
1065 if (self->codec->encreset != NULL) {
1066 outbuf = buffer;
1067 r = self->codec->encreset(&self->state, self->codec,
1068 &outbuf, sizeof(buffer));
1069 if (r != 0)
1070 return NULL;
1071 }
1072 Py_CLEAR(self->pending);
1073 Py_RETURN_NONE;
1074 }
1075
1076 static struct PyMethodDef mbiencoder_methods[] = {
1077 _MULTIBYTECODEC_MULTIBYTEINCREMENTALENCODER_ENCODE_METHODDEF
1078 _MULTIBYTECODEC_MULTIBYTEINCREMENTALENCODER_GETSTATE_METHODDEF
1079 _MULTIBYTECODEC_MULTIBYTEINCREMENTALENCODER_SETSTATE_METHODDEF
1080 _MULTIBYTECODEC_MULTIBYTEINCREMENTALENCODER_RESET_METHODDEF
1081 {NULL, NULL},
1082 };
1083
1084 static PyObject *
1085 mbiencoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
1086 {
1087 MultibyteIncrementalEncoderObject *self;
1088 PyObject *codec = NULL;
1089 char *errors = NULL;
1090
1091 if (!PyArg_ParseTupleAndKeywords(args, kwds, "|s:IncrementalEncoder",
1092 incnewkwarglist, &errors))
1093 return NULL;
1094
1095 self = (MultibyteIncrementalEncoderObject *)type->tp_alloc(type, 0);
1096 if (self == NULL)
1097 return NULL;
1098
1099 codec = PyObject_GetAttrString((PyObject *)type, "codec");
1100 if (codec == NULL)
1101 goto errorexit;
1102
1103 module_state *state = find_state_by_def(type);
1104 if (!MultibyteCodec_Check(state, codec)) {
1105 PyErr_SetString(PyExc_TypeError, "codec is unexpected type");
1106 goto errorexit;
1107 }
1108
1109 self->codec = ((MultibyteCodecObject *)codec)->codec;
1110 self->pending = NULL;
1111 self->errors = internal_error_callback(errors);
1112 if (self->errors == NULL)
1113 goto errorexit;
1114 if (self->codec->encinit != NULL &&
1115 self->codec->encinit(&self->state, self->codec) != 0)
1116 goto errorexit;
1117
1118 Py_DECREF(codec);
1119 return (PyObject *)self;
1120
1121 errorexit:
1122 Py_XDECREF(self);
1123 Py_XDECREF(codec);
1124 return NULL;
1125 }
1126
1127 static int
1128 mbiencoder_init(PyObject *self, PyObject *args, PyObject *kwds)
1129 {
1130 return 0;
1131 }
1132
1133 static int
1134 mbiencoder_traverse(PyObject *op, visitproc visit, void *arg)
1135 {
1136 MultibyteIncrementalEncoderObject *self = _MultibyteIncrementalEncoderObject_CAST(op);
1137 if (ERROR_ISCUSTOM(self->errors))
1138 Py_VISIT(self->errors);
1139 return 0;
1140 }
1141
1142 static void
1143 mbiencoder_dealloc(PyObject *op)
1144 {
1145 MultibyteIncrementalEncoderObject *self = _MultibyteIncrementalEncoderObject_CAST(op);
1146 PyTypeObject *tp = Py_TYPE(self);
1147 PyObject_GC_UnTrack(self);
1148 ERROR_DECREF(self->errors);
1149 Py_CLEAR(self->pending);
1150 tp->tp_free(self);
1151 Py_DECREF(tp);
1152 }
1153
1154 static PyType_Slot encoder_slots[] = {
1155 {Py_tp_dealloc, mbiencoder_dealloc},
1156 {Py_tp_getattro, PyObject_GenericGetAttr},
1157 {Py_tp_traverse, mbiencoder_traverse},
1158 {Py_tp_methods, mbiencoder_methods},
1159 {Py_tp_getset, codecctx_getsets},
1160 {Py_tp_init, mbiencoder_init},
1161 {Py_tp_new, mbiencoder_new},
1162 {0, NULL},
1163 };
1164
1165 static PyType_Spec encoder_spec = {
1166 .name = MODULE_NAME ".MultibyteIncrementalEncoder",
1167 .basicsize = sizeof(MultibyteIncrementalEncoderObject),
1168 .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE |
1169 Py_TPFLAGS_IMMUTABLETYPE),
1170 .slots = encoder_slots,
1171 };
1172
1173
1174 /*[clinic input]
1175 _multibytecodec.MultibyteIncrementalDecoder.decode
1176
1177 input: Py_buffer
1178 final: bool = False
1179 [clinic start generated code]*/
1180
1181 static PyObject *
1182 _multibytecodec_MultibyteIncrementalDecoder_decode_impl(MultibyteIncrementalDecoderObject *self,
1183 Py_buffer *input,
1184 int final)
1185 /*[clinic end generated code: output=b9b9090e8a9ce2ba input=8795fbb20860027a]*/
1186 {
1187 MultibyteDecodeBuffer buf;
1188 char *data, *wdata = NULL;
1189 Py_ssize_t wsize, size, origpending;
1190 PyObject *res;
1191
1192 data = input->buf;
1193 size = input->len;
1194
1195 _PyUnicodeWriter_Init(&buf.writer);
1196 buf.excobj = NULL;
1197 origpending = self->pendingsize;
1198
1199 if (self->pendingsize == 0) {
1200 wsize = size;
1201 wdata = data;
1202 }
1203 else {
1204 if (size > PY_SSIZE_T_MAX - self->pendingsize) {
1205 PyErr_NoMemory();
1206 goto errorexit;
1207 }
1208 wsize = size + self->pendingsize;
1209 wdata = PyMem_Malloc(wsize);
1210 if (wdata == NULL) {
1211 PyErr_NoMemory();
1212 goto errorexit;
1213 }
1214 memcpy(wdata, self->pending, self->pendingsize);
1215 memcpy(wdata + self->pendingsize, data, size);
1216 self->pendingsize = 0;
1217 }
1218
1219 if (decoder_prepare_buffer(&buf, wdata, wsize) != 0)
1220 goto errorexit;
1221
1222 if (decoder_feed_buffer(STATEFUL_DCTX(self), &buf))
1223 goto errorexit;
1224
1225 if (final && buf.inbuf < buf.inbuf_end) {
1226 if (multibytecodec_decerror(self->codec, &self->state,
1227 &buf, self->errors, MBERR_TOOFEW)) {
1228 /* recover the original pending buffer */
1229 memcpy(self->pending, wdata, origpending);
1230 self->pendingsize = origpending;
1231 goto errorexit;
1232 }
1233 }
1234
1235 if (buf.inbuf < buf.inbuf_end) { /* pending sequence still exists */
1236 if (decoder_append_pending(STATEFUL_DCTX(self), &buf) != 0)
1237 goto errorexit;
1238 }
1239
1240 res = _PyUnicodeWriter_Finish(&buf.writer);
1241 if (res == NULL)
1242 goto errorexit;
1243
1244 if (wdata != data)
1245 PyMem_Free(wdata);
1246 Py_XDECREF(buf.excobj);
1247 return res;
1248
1249 errorexit:
1250 if (wdata != NULL && wdata != data)
1251 PyMem_Free(wdata);
1252 Py_XDECREF(buf.excobj);
1253 _PyUnicodeWriter_Dealloc(&buf.writer);
1254 return NULL;
1255 }
1256
1257 /*[clinic input]
1258 _multibytecodec.MultibyteIncrementalDecoder.getstate
1259 [clinic start generated code]*/
1260
1261 static PyObject *
1262 _multibytecodec_MultibyteIncrementalDecoder_getstate_impl(MultibyteIncrementalDecoderObject *self)
1263 /*[clinic end generated code: output=255009c4713b7f82 input=4006aa49bddbaa75]*/
1264 {
1265 PyObject *buffer;
1266 PyObject *statelong;
1267
1268 buffer = PyBytes_FromStringAndSize((const char *)self->pending,
1269 self->pendingsize);
1270 if (buffer == NULL) {
1271 return NULL;
1272 }
1273
1274 statelong = (PyObject *)_PyLong_FromByteArray(self->state.c,
1275 sizeof(self->state.c),
1276 1 /* little-endian */ ,
1277 0 /* unsigned */ );
1278 if (statelong == NULL) {
1279 Py_DECREF(buffer);
1280 return NULL;
1281 }
1282
1283 return Py_BuildValue("NN", buffer, statelong);
1284 }
1285
1286 /*[clinic input]
1287 _multibytecodec.MultibyteIncrementalDecoder.setstate
1288 state: object(subclass_of='&PyTuple_Type')
1289 /
1290 [clinic start generated code]*/
1291
1292 static PyObject *
1293 _multibytecodec_MultibyteIncrementalDecoder_setstate_impl(MultibyteIncrementalDecoderObject *self,
1294 PyObject *state)
1295 /*[clinic end generated code: output=106b2fbca3e2dcc2 input=e5d794e8baba1a47]*/
1296 {
1297 PyObject *buffer;
1298 PyLongObject *statelong;
1299 Py_ssize_t buffersize;
1300 const char *bufferstr;
1301 unsigned char statebytes[8];
1302
1303 if (!PyArg_ParseTuple(state, "SO!;setstate(): illegal state argument",
1304 &buffer, &PyLong_Type, &statelong))
1305 {
1306 return NULL;
1307 }
1308
1309 if (_PyLong_AsByteArray(statelong, statebytes, sizeof(statebytes),
1310 1 /* little-endian */ ,
1311 0 /* unsigned */ ,
1312 1 /* with_exceptions */) < 0) {
1313 return NULL;
1314 }
1315
1316 buffersize = PyBytes_Size(buffer);
1317 if (buffersize == -1) {
1318 return NULL;
1319 }
1320
1321 if (buffersize > MAXDECPENDING) {
1322 PyObject *excobj = PyUnicodeDecodeError_Create(self->codec->encoding,
1323 PyBytes_AS_STRING(buffer), buffersize,
1324 0, buffersize,
1325 "pending buffer too large");
1326 if (excobj == NULL) return NULL;
1327 PyErr_SetObject(PyExc_UnicodeDecodeError, excobj);
1328 Py_DECREF(excobj);
1329 return NULL;
1330 }
1331
1332 bufferstr = PyBytes_AsString(buffer);
1333 if (bufferstr == NULL) {
1334 return NULL;
1335 }
1336 self->pendingsize = buffersize;
1337 memcpy(self->pending, bufferstr, self->pendingsize);
1338 memcpy(self->state.c, statebytes, sizeof(statebytes));
1339
1340 Py_RETURN_NONE;
1341 }
1342
1343 /*[clinic input]
1344 _multibytecodec.MultibyteIncrementalDecoder.reset
1345 [clinic start generated code]*/
1346
1347 static PyObject *
1348 _multibytecodec_MultibyteIncrementalDecoder_reset_impl(MultibyteIncrementalDecoderObject *self)
1349 /*[clinic end generated code: output=da423b1782c23ed1 input=3b63b3be85b2fb45]*/
1350 {
1351 if (self->codec->decreset != NULL &&
1352 self->codec->decreset(&self->state, self->codec) != 0)
1353 return NULL;
1354 self->pendingsize = 0;
1355
1356 Py_RETURN_NONE;
1357 }
1358
1359 static struct PyMethodDef mbidecoder_methods[] = {
1360 _MULTIBYTECODEC_MULTIBYTEINCREMENTALDECODER_DECODE_METHODDEF
1361 _MULTIBYTECODEC_MULTIBYTEINCREMENTALDECODER_GETSTATE_METHODDEF
1362 _MULTIBYTECODEC_MULTIBYTEINCREMENTALDECODER_SETSTATE_METHODDEF
1363 _MULTIBYTECODEC_MULTIBYTEINCREMENTALDECODER_RESET_METHODDEF
1364 {NULL, NULL},
1365 };
1366
1367 static PyObject *
1368 mbidecoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
1369 {
1370 MultibyteIncrementalDecoderObject *self;
1371 PyObject *codec = NULL;
1372 char *errors = NULL;
1373
1374 if (!PyArg_ParseTupleAndKeywords(args, kwds, "|s:IncrementalDecoder",
1375 incnewkwarglist, &errors))
1376 return NULL;
1377
1378 self = (MultibyteIncrementalDecoderObject *)type->tp_alloc(type, 0);
1379 if (self == NULL)
1380 return NULL;
1381
1382 codec = PyObject_GetAttrString((PyObject *)type, "codec");
1383 if (codec == NULL)
1384 goto errorexit;
1385
1386 module_state *state = find_state_by_def(type);
1387 if (!MultibyteCodec_Check(state, codec)) {
1388 PyErr_SetString(PyExc_TypeError, "codec is unexpected type");
1389 goto errorexit;
1390 }
1391
1392 self->codec = ((MultibyteCodecObject *)codec)->codec;
1393 self->pendingsize = 0;
1394 self->errors = internal_error_callback(errors);
1395 if (self->errors == NULL)
1396 goto errorexit;
1397 if (self->codec->decinit != NULL &&
1398 self->codec->decinit(&self->state, self->codec) != 0)
1399 goto errorexit;
1400
1401 Py_DECREF(codec);
1402 return (PyObject *)self;
1403
1404 errorexit:
1405 Py_XDECREF(self);
1406 Py_XDECREF(codec);
1407 return NULL;
1408 }
1409
1410 static int
1411 mbidecoder_init(PyObject *self, PyObject *args, PyObject *kwds)
1412 {
1413 return 0;
1414 }
1415
1416 static int
1417 mbidecoder_traverse(PyObject *op, visitproc visit, void *arg)
1418 {
1419 MultibyteIncrementalDecoderObject *self = _MultibyteIncrementalDecoderObject_CAST(op);
1420 if (ERROR_ISCUSTOM(self->errors))
1421 Py_VISIT(self->errors);
1422 return 0;
1423 }
1424
1425 static void
1426 mbidecoder_dealloc(PyObject *op)
1427 {
1428 MultibyteIncrementalDecoderObject *self = _MultibyteIncrementalDecoderObject_CAST(op);
1429 PyTypeObject *tp = Py_TYPE(self);
1430 PyObject_GC_UnTrack(self);
1431 ERROR_DECREF(self->errors);
1432 tp->tp_free(self);
1433 Py_DECREF(tp);
1434 }
1435
1436 static PyType_Slot decoder_slots[] = {
1437 {Py_tp_dealloc, mbidecoder_dealloc},
1438 {Py_tp_getattro, PyObject_GenericGetAttr},
1439 {Py_tp_traverse, mbidecoder_traverse},
1440 {Py_tp_methods, mbidecoder_methods},
1441 {Py_tp_getset, codecctx_getsets},
1442 {Py_tp_init, mbidecoder_init},
1443 {Py_tp_new, mbidecoder_new},
1444 {0, NULL},
1445 };
1446
1447 static PyType_Spec decoder_spec = {
1448 .name = MODULE_NAME ".MultibyteIncrementalDecoder",
1449 .basicsize = sizeof(MultibyteIncrementalDecoderObject),
1450 .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE |
1451 Py_TPFLAGS_IMMUTABLETYPE),
1452 .slots = decoder_slots,
1453 };
1454
1455 static PyObject *
1456 mbstreamreader_iread(MultibyteStreamReaderObject *self,
1457 const char *method, Py_ssize_t sizehint)
1458 {
1459 MultibyteDecodeBuffer buf;
1460 PyObject *cres, *res;
1461 Py_ssize_t rsize;
1462
1463 if (sizehint == 0)
1464 return Py_GetConstant(Py_CONSTANT_EMPTY_STR);
1465
1466 _PyUnicodeWriter_Init(&buf.writer);
1467 buf.excobj = NULL;
1468 cres = NULL;
1469
1470 for (;;) {
1471 int endoffile;
1472
1473 if (sizehint < 0)
1474 cres = PyObject_CallMethod(self->stream,
1475 method, NULL);
1476 else
1477 cres = PyObject_CallMethod(self->stream,
1478 method, "i", sizehint);
1479 if (cres == NULL)
1480 goto errorexit;
1481
1482 if (!PyBytes_Check(cres)) {
1483 PyErr_Format(PyExc_TypeError,
1484 "stream function returned a "
1485 "non-bytes object (%.100s)",
1486 Py_TYPE(cres)->tp_name);
1487 goto errorexit;
1488 }
1489
1490 endoffile = (PyBytes_GET_SIZE(cres) == 0);
1491
1492 if (self->pendingsize > 0) {
1493 PyObject *ctr;
1494 char *ctrdata;
1495
1496 if (PyBytes_GET_SIZE(cres) > PY_SSIZE_T_MAX - self->pendingsize) {
1497 PyErr_NoMemory();
1498 goto errorexit;
1499 }
1500 rsize = PyBytes_GET_SIZE(cres) + self->pendingsize;
1501 ctr = PyBytes_FromStringAndSize(NULL, rsize);
1502 if (ctr == NULL)
1503 goto errorexit;
1504 ctrdata = PyBytes_AS_STRING(ctr);
1505 memcpy(ctrdata, self->pending, self->pendingsize);
1506 memcpy(ctrdata + self->pendingsize,
1507 PyBytes_AS_STRING(cres),
1508 PyBytes_GET_SIZE(cres));
1509 Py_SETREF(cres, ctr);
1510 self->pendingsize = 0;
1511 }
1512
1513 rsize = PyBytes_GET_SIZE(cres);
1514 if (decoder_prepare_buffer(&buf, PyBytes_AS_STRING(cres),
1515 rsize) != 0)
1516 goto errorexit;
1517
1518 if (rsize > 0 && decoder_feed_buffer(
1519 (MultibyteStatefulDecoderContext *)self, &buf))
1520 goto errorexit;
1521
1522 if (endoffile || sizehint < 0) {
1523 if (buf.inbuf < buf.inbuf_end &&
1524 multibytecodec_decerror(self->codec, &self->state,
1525 &buf, self->errors, MBERR_TOOFEW))
1526 goto errorexit;
1527 }
1528
1529 if (buf.inbuf < buf.inbuf_end) { /* pending sequence exists */
1530 if (decoder_append_pending(STATEFUL_DCTX(self),
1531 &buf) != 0)
1532 goto errorexit;
1533 }
1534
1535 Py_SETREF(cres, NULL);
1536
1537 if (sizehint < 0 || buf.writer.pos != 0 || rsize == 0)
1538 break;
1539
1540 sizehint = 1; /* read 1 more byte and retry */
1541 }
1542
1543 res = _PyUnicodeWriter_Finish(&buf.writer);
1544 if (res == NULL)
1545 goto errorexit;
1546
1547 Py_XDECREF(cres);
1548 Py_XDECREF(buf.excobj);
1549 return res;
1550
1551 errorexit:
1552 Py_XDECREF(cres);
1553 Py_XDECREF(buf.excobj);
1554 _PyUnicodeWriter_Dealloc(&buf.writer);
1555 return NULL;
1556 }
1557
1558 /*[clinic input]
1559 _multibytecodec.MultibyteStreamReader.read
1560
1561 sizeobj: object = None
1562 /
1563 [clinic start generated code]*/
1564
1565 static PyObject *
1566 _multibytecodec_MultibyteStreamReader_read_impl(MultibyteStreamReaderObject *self,
1567 PyObject *sizeobj)
1568 /*[clinic end generated code: output=35621eb75355d5b8 input=015b0d3ff2fca485]*/
1569 {
1570 Py_ssize_t size;
1571
1572 if (sizeobj == Py_None)
1573 size = -1;
1574 else if (PyLong_Check(sizeobj))
1575 size = PyLong_AsSsize_t(sizeobj);
1576 else {
1577 PyErr_SetString(PyExc_TypeError, "arg 1 must be an integer");
1578 return NULL;
1579 }
1580
1581 if (size == -1 && PyErr_Occurred())
1582 return NULL;
1583
1584 return mbstreamreader_iread(self, "read", size);
1585 }
1586
1587 /*[clinic input]
1588 _multibytecodec.MultibyteStreamReader.readline
1589
1590 sizeobj: object = None
1591 /
1592 [clinic start generated code]*/
1593
1594 static PyObject *
1595 _multibytecodec_MultibyteStreamReader_readline_impl(MultibyteStreamReaderObject *self,
1596 PyObject *sizeobj)
1597 /*[clinic end generated code: output=4fbfaae1ed457a11 input=41ccc64f9bb0cec3]*/
1598 {
1599 Py_ssize_t size;
1600
1601 if (sizeobj == Py_None)
1602 size = -1;
1603 else if (PyLong_Check(sizeobj))
1604 size = PyLong_AsSsize_t(sizeobj);
1605 else {
1606 PyErr_SetString(PyExc_TypeError, "arg 1 must be an integer");
1607 return NULL;
1608 }
1609
1610 if (size == -1 && PyErr_Occurred())
1611 return NULL;
1612
1613 return mbstreamreader_iread(self, "readline", size);
1614 }
1615
1616 /*[clinic input]
1617 _multibytecodec.MultibyteStreamReader.readlines
1618
1619 sizehintobj: object = None
1620 /
1621 [clinic start generated code]*/
1622
1623 static PyObject *
1624 _multibytecodec_MultibyteStreamReader_readlines_impl(MultibyteStreamReaderObject *self,
1625 PyObject *sizehintobj)
1626 /*[clinic end generated code: output=e7c4310768ed2ad4 input=54932f5d4d88e880]*/
1627 {
1628 PyObject *r, *sr;
1629 Py_ssize_t sizehint;
1630
1631 if (sizehintobj == Py_None)
1632 sizehint = -1;
1633 else if (PyLong_Check(sizehintobj))
1634 sizehint = PyLong_AsSsize_t(sizehintobj);
1635 else {
1636 PyErr_SetString(PyExc_TypeError, "arg 1 must be an integer");
1637 return NULL;
1638 }
1639
1640 if (sizehint == -1 && PyErr_Occurred())
1641 return NULL;
1642
1643 r = mbstreamreader_iread(self, "read", sizehint);
1644 if (r == NULL)
1645 return NULL;
1646
1647 sr = PyUnicode_Splitlines(r, 1);
1648 Py_DECREF(r);
1649 return sr;
1650 }
1651
1652 /*[clinic input]
1653 _multibytecodec.MultibyteStreamReader.reset
1654 [clinic start generated code]*/
1655
1656 static PyObject *
1657 _multibytecodec_MultibyteStreamReader_reset_impl(MultibyteStreamReaderObject *self)
1658 /*[clinic end generated code: output=138490370a680abc input=5d4140db84b5e1e2]*/
1659 {
1660 if (self->codec->decreset != NULL &&
1661 self->codec->decreset(&self->state, self->codec) != 0)
1662 return NULL;
1663 self->pendingsize = 0;
1664
1665 Py_RETURN_NONE;
1666 }
1667
1668 static struct PyMethodDef mbstreamreader_methods[] = {
1669 _MULTIBYTECODEC_MULTIBYTESTREAMREADER_READ_METHODDEF
1670 _MULTIBYTECODEC_MULTIBYTESTREAMREADER_READLINE_METHODDEF
1671 _MULTIBYTECODEC_MULTIBYTESTREAMREADER_READLINES_METHODDEF
1672 _MULTIBYTECODEC_MULTIBYTESTREAMREADER_RESET_METHODDEF
1673 {NULL, NULL},
1674 };
1675
1676 static PyMemberDef mbstreamreader_members[] = {
1677 {"stream", _Py_T_OBJECT,
1678 offsetof(MultibyteStreamReaderObject, stream),
1679 Py_READONLY, NULL},
1680 {NULL,}
1681 };
1682
1683 static PyObject *
1684 mbstreamreader_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
1685 {
1686 MultibyteStreamReaderObject *self;
1687 PyObject *stream, *codec = NULL;
1688 char *errors = NULL;
1689
1690 if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|s:StreamReader",
1691 streamkwarglist, &stream, &errors))
1692 return NULL;
1693
1694 self = (MultibyteStreamReaderObject *)type->tp_alloc(type, 0);
1695 if (self == NULL)
1696 return NULL;
1697
1698 codec = PyObject_GetAttrString((PyObject *)type, "codec");
1699 if (codec == NULL)
1700 goto errorexit;
1701
1702 module_state *state = find_state_by_def(type);
1703 if (!MultibyteCodec_Check(state, codec)) {
1704 PyErr_SetString(PyExc_TypeError, "codec is unexpected type");
1705 goto errorexit;
1706 }
1707
1708 self->codec = ((MultibyteCodecObject *)codec)->codec;
1709 self->stream = Py_NewRef(stream);
1710 self->pendingsize = 0;
1711 self->errors = internal_error_callback(errors);
1712 if (self->errors == NULL)
1713 goto errorexit;
1714 if (self->codec->decinit != NULL &&
1715 self->codec->decinit(&self->state, self->codec) != 0)
1716 goto errorexit;
1717
1718 Py_DECREF(codec);
1719 return (PyObject *)self;
1720
1721 errorexit:
1722 Py_XDECREF(self);
1723 Py_XDECREF(codec);
1724 return NULL;
1725 }
1726
1727 static int
1728 mbstreamreader_init(PyObject *self, PyObject *args, PyObject *kwds)
1729 {
1730 return 0;
1731 }
1732
1733 static int
1734 mbstreamreader_traverse(PyObject *op, visitproc visit, void *arg)
1735 {
1736 MultibyteStreamReaderObject *self = _MultibyteStreamReaderObject_CAST(op);
1737 if (ERROR_ISCUSTOM(self->errors))
1738 Py_VISIT(self->errors);
1739 Py_VISIT(self->stream);
1740 return 0;
1741 }
1742
1743 static void
1744 mbstreamreader_dealloc(PyObject *op)
1745 {
1746 MultibyteStreamReaderObject *self = _MultibyteStreamReaderObject_CAST(op);
1747 PyTypeObject *tp = Py_TYPE(self);
1748 PyObject_GC_UnTrack(self);
1749 ERROR_DECREF(self->errors);
1750 Py_XDECREF(self->stream);
1751 tp->tp_free(self);
1752 Py_DECREF(tp);
1753 }
1754
1755 static PyType_Slot reader_slots[] = {
1756 {Py_tp_dealloc, mbstreamreader_dealloc},
1757 {Py_tp_getattro, PyObject_GenericGetAttr},
1758 {Py_tp_traverse, mbstreamreader_traverse},
1759 {Py_tp_methods, mbstreamreader_methods},
1760 {Py_tp_members, mbstreamreader_members},
1761 {Py_tp_getset, codecctx_getsets},
1762 {Py_tp_init, mbstreamreader_init},
1763 {Py_tp_new, mbstreamreader_new},
1764 {0, NULL},
1765 };
1766
1767 static PyType_Spec reader_spec = {
1768 .name = MODULE_NAME ".MultibyteStreamReader",
1769 .basicsize = sizeof(MultibyteStreamReaderObject),
1770 .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE |
1771 Py_TPFLAGS_IMMUTABLETYPE),
1772 .slots = reader_slots,
1773 };
1774
1775 static int
1776 mbstreamwriter_iwrite(MultibyteStreamWriterObject *self,
1777 PyObject *unistr, PyObject *str_write)
1778 {
1779 PyObject *str, *wr;
1780
1781 str = encoder_encode_stateful(STATEFUL_ECTX(self), unistr, 0);
1782 if (str == NULL)
1783 return -1;
1784
1785 wr = PyObject_CallMethodOneArg(self->stream, str_write, str);
1786 Py_DECREF(str);
1787 if (wr == NULL)
1788 return -1;
1789
1790 Py_DECREF(wr);
1791 return 0;
1792 }
1793
1794 /*[clinic input]
1795 _multibytecodec.MultibyteStreamWriter.write
1796
1797 cls: defining_class
1798 strobj: object
1799 /
1800 [clinic start generated code]*/
1801
1802 static PyObject *
1803 _multibytecodec_MultibyteStreamWriter_write_impl(MultibyteStreamWriterObject *self,
1804 PyTypeObject *cls,
1805 PyObject *strobj)
1806 /*[clinic end generated code: output=68ade3aea26410ac input=199f26f68bd8425a]*/
1807 {
1808 module_state *state = PyType_GetModuleState(cls);
1809 assert(state != NULL);
1810 if (mbstreamwriter_iwrite(self, strobj, state->str_write)) {
1811 return NULL;
1812 }
1813 Py_RETURN_NONE;
1814 }
1815
1816 /*[clinic input]
1817 _multibytecodec.MultibyteStreamWriter.writelines
1818
1819 cls: defining_class
1820 lines: object
1821 /
1822 [clinic start generated code]*/
1823
1824 static PyObject *
1825 _multibytecodec_MultibyteStreamWriter_writelines_impl(MultibyteStreamWriterObject *self,
1826 PyTypeObject *cls,
1827 PyObject *lines)
1828 /*[clinic end generated code: output=b4c99d2cf23ffb88 input=a6d5fe7c74972a34]*/
1829 {
1830 PyObject *strobj;
1831 int i, r;
1832
1833 if (!PySequence_Check(lines)) {
1834 PyErr_SetString(PyExc_TypeError,
1835 "arg must be a sequence object");
1836 return NULL;
1837 }
1838
1839 module_state *state = PyType_GetModuleState(cls);
1840 assert(state != NULL);
1841 for (i = 0; i < PySequence_Length(lines); i++) {
1842 /* length can be changed even within this loop */
1843 strobj = PySequence_GetItem(lines, i);
1844 if (strobj == NULL)
1845 return NULL;
1846
1847 r = mbstreamwriter_iwrite(self, strobj, state->str_write);
1848 Py_DECREF(strobj);
1849 if (r == -1)
1850 return NULL;
1851 }
1852 /* PySequence_Length() can fail */
1853 if (PyErr_Occurred())
1854 return NULL;
1855
1856 Py_RETURN_NONE;
1857 }
1858
1859 /*[clinic input]
1860 _multibytecodec.MultibyteStreamWriter.reset
1861
1862 cls: defining_class
1863 /
1864
1865 [clinic start generated code]*/
1866
1867 static PyObject *
1868 _multibytecodec_MultibyteStreamWriter_reset_impl(MultibyteStreamWriterObject *self,
1869 PyTypeObject *cls)
1870 /*[clinic end generated code: output=32ef224c2a38aa3d input=28af6a9cd38d1979]*/
1871 {
1872 PyObject *pwrt;
1873
1874 if (!self->pending)
1875 Py_RETURN_NONE;
1876
1877 pwrt = multibytecodec_encode(self->codec, &self->state,
1878 self->pending, NULL, self->errors,
1879 MBENC_FLUSH | MBENC_RESET);
1880 /* some pending buffer can be truncated when UnicodeEncodeError is
1881 * raised on 'strict' mode. but, 'reset' method is designed to
1882 * reset the pending buffer or states so failed string sequence
1883 * ought to be missed */
1884 Py_CLEAR(self->pending);
1885 if (pwrt == NULL)
1886 return NULL;
1887
1888 assert(PyBytes_Check(pwrt));
1889
1890 module_state *state = PyType_GetModuleState(cls);
1891 assert(state != NULL);
1892
1893 if (PyBytes_Size(pwrt) > 0) {
1894 PyObject *wr;
1895
1896 wr = PyObject_CallMethodOneArg(self->stream, state->str_write, pwrt);
1897 if (wr == NULL) {
1898 Py_DECREF(pwrt);
1899 return NULL;
1900 }
1901 }
1902 Py_DECREF(pwrt);
1903
1904 Py_RETURN_NONE;
1905 }
1906
1907 static PyObject *
1908 mbstreamwriter_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
1909 {
1910 MultibyteStreamWriterObject *self;
1911 PyObject *stream, *codec = NULL;
1912 char *errors = NULL;
1913
1914 if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|s:StreamWriter",
1915 streamkwarglist, &stream, &errors))
1916 return NULL;
1917
1918 self = (MultibyteStreamWriterObject *)type->tp_alloc(type, 0);
1919 if (self == NULL)
1920 return NULL;
1921
1922 codec = PyObject_GetAttrString((PyObject *)type, "codec");
1923 if (codec == NULL)
1924 goto errorexit;
1925
1926 module_state *state = find_state_by_def(type);
1927 if (!MultibyteCodec_Check(state, codec)) {
1928 PyErr_SetString(PyExc_TypeError, "codec is unexpected type");
1929 goto errorexit;
1930 }
1931
1932 self->codec = ((MultibyteCodecObject *)codec)->codec;
1933 self->stream = Py_NewRef(stream);
1934 self->pending = NULL;
1935 self->errors = internal_error_callback(errors);
1936 if (self->errors == NULL)
1937 goto errorexit;
1938 if (self->codec->encinit != NULL &&
1939 self->codec->encinit(&self->state, self->codec) != 0)
1940 goto errorexit;
1941
1942 Py_DECREF(codec);
1943 return (PyObject *)self;
1944
1945 errorexit:
1946 Py_XDECREF(self);
1947 Py_XDECREF(codec);
1948 return NULL;
1949 }
1950
1951 static int
1952 mbstreamwriter_init(PyObject *self, PyObject *args, PyObject *kwds)
1953 {
1954 return 0;
1955 }
1956
1957 static int
1958 mbstreamwriter_traverse(PyObject *op, visitproc visit, void *arg)
1959 {
1960 MultibyteStreamWriterObject *self = _MultibyteStreamWriterObject_CAST(op);
1961 if (ERROR_ISCUSTOM(self->errors))
1962 Py_VISIT(self->errors);
1963 Py_VISIT(self->stream);
1964 return 0;
1965 }
1966
1967 static void
1968 mbstreamwriter_dealloc(PyObject *op)
1969 {
1970 MultibyteStreamWriterObject *self = _MultibyteStreamWriterObject_CAST(op);
1971 PyTypeObject *tp = Py_TYPE(self);
1972 PyObject_GC_UnTrack(self);
1973 ERROR_DECREF(self->errors);
1974 Py_XDECREF(self->stream);
1975 tp->tp_free(self);
1976 Py_DECREF(tp);
1977 }
1978
1979 static struct PyMethodDef mbstreamwriter_methods[] = {
1980 _MULTIBYTECODEC_MULTIBYTESTREAMWRITER_WRITE_METHODDEF
1981 _MULTIBYTECODEC_MULTIBYTESTREAMWRITER_WRITELINES_METHODDEF
1982 _MULTIBYTECODEC_MULTIBYTESTREAMWRITER_RESET_METHODDEF
1983 {NULL, NULL},
1984 };
1985
1986 static PyMemberDef mbstreamwriter_members[] = {
1987 {"stream", _Py_T_OBJECT,
1988 offsetof(MultibyteStreamWriterObject, stream),
1989 Py_READONLY, NULL},
1990 {NULL,}
1991 };
1992
1993 static PyType_Slot writer_slots[] = {
1994 {Py_tp_dealloc, mbstreamwriter_dealloc},
1995 {Py_tp_getattro, PyObject_GenericGetAttr},
1996 {Py_tp_traverse, mbstreamwriter_traverse},
1997 {Py_tp_methods, mbstreamwriter_methods},
1998 {Py_tp_members, mbstreamwriter_members},
1999 {Py_tp_getset, codecctx_getsets},
2000 {Py_tp_init, mbstreamwriter_init},
2001 {Py_tp_new, mbstreamwriter_new},
2002 {0, NULL},
2003 };
2004
2005 static PyType_Spec writer_spec = {
2006 .name = MODULE_NAME ".MultibyteStreamWriter",
2007 .basicsize = sizeof(MultibyteStreamWriterObject),
2008 .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE |
2009 Py_TPFLAGS_IMMUTABLETYPE),
2010 .slots = writer_slots,
2011 };
2012
2013
2014 /*[clinic input]
2015 _multibytecodec.__create_codec
2016
2017 arg: object
2018 /
2019 [clinic start generated code]*/
2020
2021 static PyObject *
2022 _multibytecodec___create_codec(PyObject *module, PyObject *arg)
2023 /*[clinic end generated code: output=cfa3dce8260e809d input=6840b2a6b183fcfa]*/
2024 {
2025 MultibyteCodecObject *self;
2026
2027 if (!PyCapsule_IsValid(arg, CODEC_CAPSULE)) {
2028 PyErr_SetString(PyExc_ValueError, "argument type invalid");
2029 return NULL;
2030 }
2031
2032 codec_capsule *data = PyCapsule_GetPointer(arg, CODEC_CAPSULE);
2033 const MultibyteCodec *codec = data->codec;
2034 if (codec->codecinit != NULL && codec->codecinit(codec) != 0)
2035 return NULL;
2036
2037 module_state *state = get_module_state(module);
2038 self = PyObject_GC_New(MultibyteCodecObject, state->multibytecodec_type);
2039 if (self == NULL)
2040 return NULL;
2041 self->codec = codec;
2042 self->cjk_module = Py_NewRef(data->cjk_module);
2043
2044 PyObject_GC_Track(self);
2045 return (PyObject *)self;
2046 }
2047
2048 static int
2049 _multibytecodec_traverse(PyObject *mod, visitproc visit, void *arg)
2050 {
2051 module_state *state = get_module_state(mod);
2052 Py_VISIT(state->multibytecodec_type);
2053 Py_VISIT(state->encoder_type);
2054 Py_VISIT(state->decoder_type);
2055 Py_VISIT(state->reader_type);
2056 Py_VISIT(state->writer_type);
2057 return 0;
2058 }
2059
2060 static int
2061 _multibytecodec_clear(PyObject *mod)
2062 {
2063 module_state *state = get_module_state(mod);
2064 Py_CLEAR(state->multibytecodec_type);
2065 Py_CLEAR(state->encoder_type);
2066 Py_CLEAR(state->decoder_type);
2067 Py_CLEAR(state->reader_type);
2068 Py_CLEAR(state->writer_type);
2069 Py_CLEAR(state->str_write);
2070 return 0;
2071 }
2072
2073 static void
2074 _multibytecodec_free(void *mod)
2075 {
2076 (void)_multibytecodec_clear((PyObject *)mod);
2077 }
2078
2079 #define CREATE_TYPE(module, type, spec) \
2080 do { \
2081 type = (PyTypeObject *)PyType_FromModuleAndSpec(module, spec, NULL); \
2082 if (!type) { \
2083 return -1; \
2084 } \
2085 } while (0)
2086
2087 #define ADD_TYPE(module, type) \
2088 do { \
2089 if (PyModule_AddType(module, type) < 0) { \
2090 return -1; \
2091 } \
2092 } while (0)
2093
2094 static int
2095 _multibytecodec_exec(PyObject *mod)
2096 {
2097 module_state *state = get_module_state(mod);
2098 state->str_write = PyUnicode_InternFromString("write");
2099 if (state->str_write == NULL) {
2100 return -1;
2101 }
2102 CREATE_TYPE(mod, state->multibytecodec_type, &multibytecodec_spec);
2103 CREATE_TYPE(mod, state->encoder_type, &encoder_spec);
2104 CREATE_TYPE(mod, state->decoder_type, &decoder_spec);
2105 CREATE_TYPE(mod, state->reader_type, &reader_spec);
2106 CREATE_TYPE(mod, state->writer_type, &writer_spec);
2107
2108 ADD_TYPE(mod, state->encoder_type);
2109 ADD_TYPE(mod, state->decoder_type);
2110 ADD_TYPE(mod, state->reader_type);
2111 ADD_TYPE(mod, state->writer_type);
2112 return 0;
2113 }
2114
2115 #undef CREATE_TYPE
2116 #undef ADD_TYPE
2117
2118 static struct PyMethodDef _multibytecodec_methods[] = {
2119 _MULTIBYTECODEC___CREATE_CODEC_METHODDEF
2120 {NULL, NULL},
2121 };
2122
2123 static PyModuleDef_Slot _multibytecodec_slots[] = {
2124 {Py_mod_exec, _multibytecodec_exec},
2125 {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
2126 {Py_mod_gil, Py_MOD_GIL_NOT_USED},
2127 {0, NULL}
2128 };
2129
2130 static struct PyModuleDef _multibytecodecmodule = {
2131 .m_base = PyModuleDef_HEAD_INIT,
2132 .m_name = "_multibytecodec",
2133 .m_size = sizeof(module_state),
2134 .m_methods = _multibytecodec_methods,
2135 .m_slots = _multibytecodec_slots,
2136 .m_traverse = _multibytecodec_traverse,
2137 .m_clear = _multibytecodec_clear,
2138 .m_free = _multibytecodec_free,
2139 };
2140
2141 PyMODINIT_FUNC
2142 PyInit__multibytecodec(void)
2143 {
2144 return PyModuleDef_Init(&_multibytecodecmodule);
2145 }