Debian Codegraph @

Modules/_testcapi/unicode.c

raw
1 #include <stddef.h> // ptrdiff_t
2
3 #include "parts.h"
4 #include "util.h"
5
6 /* Test PyUnicode_New() */
7 static PyObject *
8 unicode_new(PyObject *self, PyObject *args)
9 {
10 Py_ssize_t size;
11 unsigned int maxchar;
12 PyObject *result;
13
14 if (!PyArg_ParseTuple(args, "nI", &size, &maxchar)) {
15 return NULL;
16 }
17
18 result = PyUnicode_New(size, (Py_UCS4)maxchar);
19 if (!result) {
20 return NULL;
21 }
22 if (size > 0 && maxchar <= 0x10ffff &&
23 PyUnicode_Fill(result, 0, size, (Py_UCS4)maxchar) < 0)
24 {
25 Py_DECREF(result);
26 return NULL;
27 }
28 return result;
29 }
30
31
32 static PyObject *
33 unicode_copy(PyObject *unicode)
34 {
35 PyObject *copy;
36
37 if (!unicode) {
38 return NULL;
39 }
40 if (!PyUnicode_Check(unicode)) {
41 Py_INCREF(unicode);
42 return unicode;
43 }
44
45 copy = PyUnicode_New(PyUnicode_GET_LENGTH(unicode),
46 PyUnicode_MAX_CHAR_VALUE(unicode));
47 if (!copy) {
48 return NULL;
49 }
50 if (PyUnicode_CopyCharacters(copy, 0, unicode,
51 0, PyUnicode_GET_LENGTH(unicode)) < 0)
52 {
53 Py_DECREF(copy);
54 return NULL;
55 }
56 return copy;
57 }
58
59
60 /* Test PyUnicode_Fill() */
61 static PyObject *
62 unicode_fill(PyObject *self, PyObject *args)
63 {
64 PyObject *to, *to_copy;
65 Py_ssize_t start, length, filled;
66 unsigned int fill_char;
67
68 if (!PyArg_ParseTuple(args, "OnnI", &to, &start, &length, &fill_char)) {
69 return NULL;
70 }
71
72 NULLABLE(to);
73 if (!(to_copy = unicode_copy(to)) && to) {
74 return NULL;
75 }
76
77 filled = PyUnicode_Fill(to_copy, start, length, (Py_UCS4)fill_char);
78 if (filled == -1 && PyErr_Occurred()) {
79 Py_DECREF(to_copy);
80 return NULL;
81 }
82 return Py_BuildValue("(Nn)", to_copy, filled);
83 }
84
85
86 /* Test PyUnicode_FromKindAndData() */
87 static PyObject *
88 unicode_fromkindanddata(PyObject *self, PyObject *args)
89 {
90 int kind;
91 void *buffer;
92 Py_ssize_t bsize;
93 Py_ssize_t size = -100;
94
95 if (!PyArg_ParseTuple(args, "iz#|n", &kind, &buffer, &bsize, &size)) {
96 return NULL;
97 }
98
99 if (size == -100) {
100 size = bsize;
101 }
102 if (kind && size % kind) {
103 PyErr_SetString(PyExc_AssertionError,
104 "invalid size in unicode_fromkindanddata()");
105 return NULL;
106 }
107 return PyUnicode_FromKindAndData(kind, buffer, kind ? size / kind : 0);
108 }
109
110
111 // Test PyUnicode_AsUCS4().
112 // Part of the limited C API, but the test needs PyUnicode_FromKindAndData().
113 static PyObject *
114 unicode_asucs4(PyObject *self, PyObject *args)
115 {
116 PyObject *unicode, *result;
117 Py_UCS4 *buffer;
118 int copy_null;
119 Py_ssize_t str_len, buf_len;
120
121 if (!PyArg_ParseTuple(args, "Onp:unicode_asucs4", &unicode, &str_len, &copy_null)) {
122 return NULL;
123 }
124
125 NULLABLE(unicode);
126 buf_len = str_len + 1;
127 buffer = PyMem_NEW(Py_UCS4, buf_len);
128 if (buffer == NULL) {
129 return PyErr_NoMemory();
130 }
131 memset(buffer, 0, sizeof(Py_UCS4)*buf_len);
132 buffer[str_len] = 0xffffU;
133
134 if (!PyUnicode_AsUCS4(unicode, buffer, buf_len, copy_null)) {
135 PyMem_Free(buffer);
136 return NULL;
137 }
138
139 result = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, buffer, buf_len);
140 PyMem_Free(buffer);
141 return result;
142 }
143
144
145 // Test PyUnicode_AsUCS4Copy().
146 // Part of the limited C API, but the test needs PyUnicode_FromKindAndData().
147 static PyObject *
148 unicode_asucs4copy(PyObject *self, PyObject *args)
149 {
150 PyObject *unicode;
151 Py_UCS4 *buffer;
152 PyObject *result;
153
154 if (!PyArg_ParseTuple(args, "O", &unicode)) {
155 return NULL;
156 }
157
158 NULLABLE(unicode);
159 buffer = PyUnicode_AsUCS4Copy(unicode);
160 if (buffer == NULL) {
161 return NULL;
162 }
163 result = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND,
164 buffer,
165 PyUnicode_GET_LENGTH(unicode) + 1);
166 PyMem_FREE(buffer);
167 return result;
168 }
169
170
171 /* Test PyUnicode_AsUTF8() */
172 static PyObject *
173 unicode_asutf8(PyObject *self, PyObject *args)
174 {
175 PyObject *unicode;
176 Py_ssize_t buflen;
177 const char *s;
178
179 if (!PyArg_ParseTuple(args, "On", &unicode, &buflen))
180 return NULL;
181
182 NULLABLE(unicode);
183 s = PyUnicode_AsUTF8(unicode);
184 if (s == NULL)
185 return NULL;
186
187 return PyBytes_FromStringAndSize(s, buflen);
188 }
189
190
191 /* Test PyUnicode_CopyCharacters() */
192 static PyObject *
193 unicode_copycharacters(PyObject *self, PyObject *args)
194 {
195 PyObject *from, *to, *to_copy;
196 Py_ssize_t from_start, to_start, how_many, copied;
197
198 if (!PyArg_ParseTuple(args, "UnOnn", &to, &to_start,
199 &from, &from_start, &how_many)) {
200 return NULL;
201 }
202
203 NULLABLE(from);
204 if (!(to_copy = PyUnicode_New(PyUnicode_GET_LENGTH(to),
205 PyUnicode_MAX_CHAR_VALUE(to)))) {
206 return NULL;
207 }
208 if (PyUnicode_Fill(to_copy, 0, PyUnicode_GET_LENGTH(to_copy), 0U) < 0) {
209 Py_DECREF(to_copy);
210 return NULL;
211 }
212
213 copied = PyUnicode_CopyCharacters(to_copy, to_start, from,
214 from_start, how_many);
215 if (copied == -1 && PyErr_Occurred()) {
216 Py_DECREF(to_copy);
217 return NULL;
218 }
219
220 return Py_BuildValue("(Nn)", to_copy, copied);
221 }
222
223
224 // --- PyUnicodeWriter type -------------------------------------------------
225
226 typedef struct {
227 PyObject_HEAD
228 PyUnicodeWriter *writer;
229 } WriterObject;
230
231
232 static PyObject *
233 writer_new(PyTypeObject *type, PyObject *args, PyObject *kwargs)
234 {
235 WriterObject *self = (WriterObject *)type->tp_alloc(type, 0);
236 if (!self) {
237 return NULL;
238 }
239 self->writer = NULL;
240 return (PyObject*)self;
241 }
242
243
244 static int
245 writer_init(PyObject *self_raw, PyObject *args, PyObject *kwargs)
246 {
247 WriterObject *self = (WriterObject *)self_raw;
248
249 Py_ssize_t size;
250 if (!PyArg_ParseTuple(args, "n", &size)) {
251 return -1;
252 }
253
254 if (self->writer) {
255 PyUnicodeWriter_Discard(self->writer);
256 }
257
258 self->writer = PyUnicodeWriter_Create(size);
259 if (self->writer == NULL) {
260 return -1;
261 }
262 return 0;
263 }
264
265
266 static void
267 writer_dealloc(PyObject *self_raw)
268 {
269 WriterObject *self = (WriterObject *)self_raw;
270 PyTypeObject *tp = Py_TYPE(self);
271 if (self->writer) {
272 PyUnicodeWriter_Discard(self->writer);
273 }
274 tp->tp_free(self);
275 Py_DECREF(tp);
276 }
277
278
279 static inline int
280 writer_check(WriterObject *self)
281 {
282 if (self->writer == NULL) {
283 PyErr_SetString(PyExc_ValueError, "operation on finished writer");
284 return -1;
285 }
286 return 0;
287 }
288
289
290 static PyObject*
291 writer_write_char(PyObject *self_raw, PyObject *args)
292 {
293 WriterObject *self = (WriterObject *)self_raw;
294 if (writer_check(self) < 0) {
295 return NULL;
296 }
297
298 unsigned int ch;
299 if (!PyArg_ParseTuple(args, "I", &ch)) {
300 return NULL;
301 }
302
303 if (PyUnicodeWriter_WriteChar(self->writer, (Py_UCS4)ch) < 0) {
304 return NULL;
305 }
306 Py_RETURN_NONE;
307 }
308
309
310 static PyObject*
311 writer_write_utf8(PyObject *self_raw, PyObject *args)
312 {
313 WriterObject *self = (WriterObject *)self_raw;
314 if (writer_check(self) < 0) {
315 return NULL;
316 }
317
318 const char *str;
319 Py_ssize_t bsize, size;
320 if (!PyArg_ParseTuple(args, "z#n", &str, &bsize, &size)) {
321 return NULL;
322 }
323
324 if (PyUnicodeWriter_WriteUTF8(self->writer, str, size) < 0) {
325 return NULL;
326 }
327 Py_RETURN_NONE;
328 }
329
330
331 static PyObject*
332 writer_write_ascii(PyObject *self_raw, PyObject *args)
333 {
334 WriterObject *self = (WriterObject *)self_raw;
335 if (writer_check(self) < 0) {
336 return NULL;
337 }
338
339 const char *str;
340 Py_ssize_t bsize, size;
341 if (!PyArg_ParseTuple(args, "z#n", &str, &bsize, &size)) {
342 return NULL;
343 }
344
345 if (PyUnicodeWriter_WriteASCII(self->writer, str, size) < 0) {
346 return NULL;
347 }
348 Py_RETURN_NONE;
349 }
350
351
352 static PyObject*
353 writer_write_widechar(PyObject *self_raw, PyObject *args)
354 {
355 WriterObject *self = (WriterObject *)self_raw;
356 if (writer_check(self) < 0) {
357 return NULL;
358 }
359
360 const char *s;
361 Py_ssize_t bsize;
362 Py_ssize_t size = -100;
363
364 if (!PyArg_ParseTuple(args, "z#|n", &s, &bsize, &size)) {
365 return NULL;
366 }
367 if (size == -100) {
368 if (bsize % SIZEOF_WCHAR_T) {
369 PyErr_SetString(PyExc_AssertionError,
370 "invalid size in writer.write_widechar()");
371 return NULL;
372 }
373 size = bsize / SIZEOF_WCHAR_T;
374 }
375
376 int res = PyUnicodeWriter_WriteWideChar(self->writer, (const wchar_t *)s, size);
377 if (res < 0) {
378 return NULL;
379 }
380 Py_RETURN_NONE;
381 }
382
383
384 static PyObject*
385 writer_write_ucs4(PyObject *self_raw, PyObject *args)
386 {
387 WriterObject *self = (WriterObject *)self_raw;
388 if (writer_check(self) < 0) {
389 return NULL;
390 }
391
392 const char *s;
393 Py_ssize_t bsize;
394 Py_ssize_t size = -100;
395
396 if (!PyArg_ParseTuple(args, "z#|n", &s, &bsize, &size)) {
397 return NULL;
398 }
399 if (size == -100) {
400 if (bsize % sizeof(Py_UCS4)) {
401 PyErr_SetString(PyExc_AssertionError,
402 "invalid size in writer.write_ucs4()");
403 return NULL;
404 }
405 size = bsize / sizeof(Py_UCS4);
406 }
407
408 int res = PyUnicodeWriter_WriteUCS4(self->writer, (Py_UCS4 *)s, size);
409 if (res < 0) {
410 return NULL;
411 }
412 Py_RETURN_NONE;
413 }
414
415
416 static PyObject*
417 writer_write_str(PyObject *self_raw, PyObject *obj)
418 {
419 WriterObject *self = (WriterObject *)self_raw;
420 if (writer_check(self) < 0) {
421 return NULL;
422 }
423
424 NULLABLE(obj);
425 if (PyUnicodeWriter_WriteStr(self->writer, obj) < 0) {
426 return NULL;
427 }
428 Py_RETURN_NONE;
429 }
430
431
432 static PyObject*
433 writer_write_repr(PyObject *self_raw, PyObject *obj)
434 {
435 WriterObject *self = (WriterObject *)self_raw;
436 if (writer_check(self) < 0) {
437 return NULL;
438 }
439
440 NULLABLE(obj);
441 if (PyUnicodeWriter_WriteRepr(self->writer, obj) < 0) {
442 return NULL;
443 }
444 Py_RETURN_NONE;
445 }
446
447
448 static PyObject*
449 writer_write_substring(PyObject *self_raw, PyObject *args)
450 {
451 WriterObject *self = (WriterObject *)self_raw;
452 if (writer_check(self) < 0) {
453 return NULL;
454 }
455
456 PyObject *str;
457 Py_ssize_t start, end;
458 if (!PyArg_ParseTuple(args, "Onn", &str, &start, &end)) {
459 return NULL;
460 }
461 NULLABLE(str);
462
463 if (PyUnicodeWriter_WriteSubstring(self->writer, str, start, end) < 0) {
464 return NULL;
465 }
466 Py_RETURN_NONE;
467 }
468
469
470 static PyObject*
471 writer_decodeutf8stateful(PyObject *self_raw, PyObject *args)
472 {
473 WriterObject *self = (WriterObject *)self_raw;
474 if (writer_check(self) < 0) {
475 return NULL;
476 }
477
478 const char *str;
479 Py_ssize_t bsize, len;
480 const char *errors;
481 int use_consumed = 0;
482 if (!PyArg_ParseTuple(args, "z#nz#|p", &str, &bsize, &len, &errors, &bsize, &use_consumed)) {
483 return NULL;
484 }
485
486 Py_ssize_t consumed = 12345;
487 Py_ssize_t *pconsumed = use_consumed ? &consumed : NULL;
488 if (PyUnicodeWriter_DecodeUTF8Stateful(self->writer, str, len,
489 errors, pconsumed) < 0) {
490 if (use_consumed) {
491 assert(consumed == 0);
492 }
493 return NULL;
494 }
495
496 if (use_consumed) {
497 return PyLong_FromSsize_t(consumed);
498 }
499 Py_RETURN_NONE;
500 }
501
502
503 static PyObject*
504 writer_get_pointer(PyObject *self_raw, PyObject *args)
505 {
506 WriterObject *self = (WriterObject *)self_raw;
507 if (writer_check(self) < 0) {
508 return NULL;
509 }
510
511 return PyLong_FromVoidPtr(self->writer);
512 }
513
514
515 static PyObject*
516 writer_finish(PyObject *self_raw, PyObject *Py_UNUSED(args))
517 {
518 WriterObject *self = (WriterObject *)self_raw;
519 if (writer_check(self) < 0) {
520 return NULL;
521 }
522
523 PyObject *str = PyUnicodeWriter_Finish(self->writer);
524 self->writer = NULL;
525 return str;
526 }
527
528
529 static PyMethodDef writer_methods[] = {
530 {"write_char", _PyCFunction_CAST(writer_write_char), METH_VARARGS},
531 {"write_utf8", _PyCFunction_CAST(writer_write_utf8), METH_VARARGS},
532 {"write_ascii", _PyCFunction_CAST(writer_write_ascii), METH_VARARGS},
533 {"write_widechar", _PyCFunction_CAST(writer_write_widechar), METH_VARARGS},
534 {"write_ucs4", _PyCFunction_CAST(writer_write_ucs4), METH_VARARGS},
535 {"write_str", _PyCFunction_CAST(writer_write_str), METH_O},
536 {"write_repr", _PyCFunction_CAST(writer_write_repr), METH_O},
537 {"write_substring", _PyCFunction_CAST(writer_write_substring), METH_VARARGS},
538 {"decodeutf8stateful", _PyCFunction_CAST(writer_decodeutf8stateful), METH_VARARGS},
539 {"get_pointer", _PyCFunction_CAST(writer_get_pointer), METH_VARARGS},
540 {"finish", _PyCFunction_CAST(writer_finish), METH_NOARGS},
541 {NULL, NULL} /* sentinel */
542 };
543
544 static PyType_Slot Writer_Type_slots[] = {
545 {Py_tp_new, writer_new},
546 {Py_tp_init, writer_init},
547 {Py_tp_dealloc, writer_dealloc},
548 {Py_tp_methods, writer_methods},
549 {0, 0}, /* sentinel */
550 };
551
552 static PyType_Spec Writer_spec = {
553 .name = "_testcapi.PyUnicodeWriter",
554 .basicsize = sizeof(WriterObject),
555 .flags = Py_TPFLAGS_DEFAULT,
556 .slots = Writer_Type_slots,
557 };
558
559
560 static PyMethodDef TestMethods[] = {
561 {"unicode_new", unicode_new, METH_VARARGS},
562 {"unicode_fill", unicode_fill, METH_VARARGS},
563 {"unicode_fromkindanddata", unicode_fromkindanddata, METH_VARARGS},
564 {"unicode_asucs4", unicode_asucs4, METH_VARARGS},
565 {"unicode_asucs4copy", unicode_asucs4copy, METH_VARARGS},
566 {"unicode_asutf8", unicode_asutf8, METH_VARARGS},
567 {"unicode_copycharacters", unicode_copycharacters, METH_VARARGS},
568 {NULL},
569 };
570
571 int
572 _PyTestCapi_Init_Unicode(PyObject *m) {
573 if (PyModule_AddFunctions(m, TestMethods) < 0) {
574 return -1;
575 }
576
577 PyTypeObject *writer_type = (PyTypeObject *)PyType_FromSpec(&Writer_spec);
578 if (writer_type == NULL) {
579 return -1;
580 }
581 if (PyModule_AddType(m, writer_type) < 0) {
582 Py_DECREF(writer_type);
583 return -1;
584 }
585 Py_DECREF(writer_type);
586
587 return 0;
588 }