Debian Codegraph @

Modules/_io/bytesio.c

raw
1 #include "Python.h"
2 #include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION()
3 #include "pycore_object.h"
4 #include "pycore_sysmodule.h" // _PySys_GetSizeOf()
5 #include "pycore_weakref.h" // FT_CLEAR_WEAKREFS()
6 #include "pycore_pyatomic_ft_wrappers.h"
7
8 #include <stddef.h> // offsetof()
9 #include "_iomodule.h"
10
11 /*[clinic input]
12 module _io
13 class _io.BytesIO "bytesio *" "clinic_state()->PyBytesIO_Type"
14 [clinic start generated code]*/
15 /*[clinic end generated code: output=da39a3ee5e6b4b0d input=48ede2f330f847c3]*/
16
17 typedef struct {
18 PyObject_HEAD
19 PyObject *buf;
20 Py_ssize_t pos;
21 Py_ssize_t string_size;
22 PyObject *dict;
23 PyObject *weakreflist;
24 Py_ssize_t exports;
25 } bytesio;
26
27 #define bytesio_CAST(op) ((bytesio *)(op))
28
29 typedef struct {
30 PyObject_HEAD
31 bytesio *source;
32 } bytesiobuf;
33
34 #define bytesiobuf_CAST(op) ((bytesiobuf *)(op))
35
36 /* The bytesio object can be in three states:
37 * Py_REFCNT(buf) == 1, exports == 0.
38 * Py_REFCNT(buf) > 1. exports == 0,
39 first modification or export causes the internal buffer copying.
40 * exports > 0. Py_REFCNT(buf) == 1, any modifications are forbidden.
41 */
42
43 static int
44 check_closed(bytesio *self)
45 {
46 if (self->buf == NULL) {
47 PyErr_SetString(PyExc_ValueError, "I/O operation on closed file.");
48 return 1;
49 }
50 return 0;
51 }
52
53 static int
54 check_exports(bytesio *self)
55 {
56 if (FT_ATOMIC_LOAD_SSIZE_RELAXED(self->exports) > 0) {
57 PyErr_SetString(PyExc_BufferError,
58 "Existing exports of data: object cannot be re-sized");
59 return 1;
60 }
61 return 0;
62 }
63
64 #define CHECK_CLOSED(self) \
65 if (check_closed(self)) { \
66 return NULL; \
67 }
68
69 #define CHECK_EXPORTS(self) \
70 if (check_exports(self)) { \
71 return NULL; \
72 }
73
74 #define SHARED_BUF(self) (!_PyObject_IsUniquelyReferenced((self)->buf))
75
76
77 /* Internal routine to get a line from the buffer of a BytesIO
78 object. Returns the length between the current position to the
79 next newline character. */
80 static Py_ssize_t
81 scan_eol_lock_held(bytesio *self, Py_ssize_t len)
82 {
83 _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self);
84
85 const char *start, *n;
86 Py_ssize_t maxlen;
87
88 assert(self->buf != NULL);
89 assert(self->pos >= 0);
90
91 if (self->pos >= self->string_size)
92 return 0;
93
94 /* Move to the end of the line, up to the end of the string, s. */
95 maxlen = self->string_size - self->pos;
96 if (len < 0 || len > maxlen)
97 len = maxlen;
98
99 if (len) {
100 start = PyBytes_AS_STRING(self->buf) + self->pos;
101 n = memchr(start, '\n', len);
102 if (n)
103 /* Get the length from the current position to the end of
104 the line. */
105 len = n - start + 1;
106 }
107 assert(len >= 0);
108 assert(self->pos < PY_SSIZE_T_MAX - len);
109
110 return len;
111 }
112
113 /* Internal routine for detaching the shared buffer of BytesIO objects.
114 The caller should ensure that the 'size' argument is non-negative and
115 not lesser than self->string_size. Returns 0 on success, -1 otherwise. */
116 static int
117 unshare_buffer_lock_held(bytesio *self, size_t size)
118 {
119 _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self);
120
121 PyObject *new_buf;
122 assert(SHARED_BUF(self));
123 assert(FT_ATOMIC_LOAD_SSIZE_RELAXED(self->exports) == 0);
124 assert(size >= (size_t)self->string_size);
125 new_buf = PyBytes_FromStringAndSize(NULL, size);
126 if (new_buf == NULL)
127 return -1;
128 memcpy(PyBytes_AS_STRING(new_buf), PyBytes_AS_STRING(self->buf),
129 self->string_size);
130 Py_SETREF(self->buf, new_buf);
131 return 0;
132 }
133
134 /* Internal routine for changing the size of the buffer of BytesIO objects.
135 The caller should ensure that the 'size' argument is non-negative. Returns
136 0 on success, -1 otherwise. */
137 static int
138 resize_buffer_lock_held(bytesio *self, size_t size)
139 {
140 _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self);
141
142 assert(self->buf != NULL);
143 assert(FT_ATOMIC_LOAD_SSIZE_RELAXED(self->exports) == 0);
144
145 /* Here, unsigned types are used to avoid dealing with signed integer
146 overflow, which is undefined in C. */
147 size_t alloc = PyBytes_GET_SIZE(self->buf);
148
149 /* For simplicity, stay in the range of the signed type. Anyway, Python
150 doesn't allow strings to be longer than this. */
151 if (size > PY_SSIZE_T_MAX)
152 goto overflow;
153
154 if (size < alloc / 2) {
155 /* Major downsize; resize down to exact size. */
156 alloc = size + 1;
157 }
158 else if (size < alloc) {
159 /* Within allocated size; quick exit */
160 return 0;
161 }
162 else if (size <= alloc * 1.125) {
163 /* Moderate upsize; overallocate similar to list_resize() */
164 alloc = size + (size >> 3) + (size < 9 ? 3 : 6);
165 }
166 else {
167 /* Major upsize; resize up to exact size */
168 alloc = size + 1;
169 }
170
171 if (SHARED_BUF(self)) {
172 if (unshare_buffer_lock_held(self, alloc) < 0)
173 return -1;
174 }
175 else {
176 if (_PyBytes_Resize(&self->buf, alloc) < 0)
177 return -1;
178 }
179
180 return 0;
181
182 overflow:
183 PyErr_SetString(PyExc_OverflowError,
184 "new buffer size too large");
185 return -1;
186 }
187
188 /* Internal routine for writing a string of bytes to the buffer of a BytesIO
189 object. Returns the number of bytes written, or -1 on error.
190 Inlining is disabled because it's significantly decreases performance
191 of writelines() in PGO build. */
192 Py_NO_INLINE static Py_ssize_t
193 write_bytes_lock_held(bytesio *self, PyObject *b)
194 {
195 _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self);
196
197 Py_buffer buf;
198 Py_ssize_t len;
199 if (PyObject_GetBuffer(b, &buf, PyBUF_CONTIG_RO) < 0) {
200 return -1;
201 }
202
203 if (check_closed(self) || check_exports(self)) {
204 len = -1;
205 goto done;
206 }
207
208 len = buf.len;
209 if (len == 0) {
210 goto done;
211 }
212
213 assert(self->pos >= 0);
214 size_t endpos = (size_t)self->pos + len;
215 if (endpos > (size_t)PyBytes_GET_SIZE(self->buf)) {
216 if (resize_buffer_lock_held(self, endpos) < 0) {
217 len = -1;
218 goto done;
219 }
220 }
221 else if (SHARED_BUF(self)) {
222 if (unshare_buffer_lock_held(self, Py_MAX(endpos, (size_t)self->string_size)) < 0) {
223 len = -1;
224 goto done;
225 }
226 }
227
228 if (self->pos > self->string_size) {
229 /* In case of overseek, pad with null bytes the buffer region between
230 the end of stream and the current position.
231
232 0 lo string_size hi
233 | |<---used--->|<----------available----------->|
234 | | <--to pad-->|<---to write---> |
235 0 buf position
236 */
237 memset(PyBytes_AS_STRING(self->buf) + self->string_size, '\0',
238 (self->pos - self->string_size) * sizeof(char));
239 }
240
241 /* Copy the data to the internal buffer, overwriting some of the existing
242 data if self->pos < self->string_size. */
243 memcpy(PyBytes_AS_STRING(self->buf) + self->pos, buf.buf, len);
244 self->pos = endpos;
245
246 /* Set the new length of the internal string if it has changed. */
247 if ((size_t)self->string_size < endpos) {
248 self->string_size = endpos;
249 }
250
251 done:
252 PyBuffer_Release(&buf);
253 return len;
254 }
255
256 static PyObject *
257 bytesio_get_closed(PyObject *op, void *Py_UNUSED(closure))
258 {
259 PyObject *ret;
260 bytesio *self = bytesio_CAST(op);
261 Py_BEGIN_CRITICAL_SECTION(self);
262 if (self->buf == NULL) {
263 ret = Py_True;
264 }
265 else {
266 ret = Py_False;
267 }
268 Py_END_CRITICAL_SECTION();
269 return ret;
270 }
271
272 /*[clinic input]
273 @critical_section
274 _io.BytesIO.readable
275
276 Returns True if the IO object can be read.
277 [clinic start generated code]*/
278
279 static PyObject *
280 _io_BytesIO_readable_impl(bytesio *self)
281 /*[clinic end generated code: output=4e93822ad5b62263 input=ab7816facef48bfd]*/
282 {
283 CHECK_CLOSED(self);
284 Py_RETURN_TRUE;
285 }
286
287 /*[clinic input]
288 @critical_section
289 _io.BytesIO.writable
290
291 Returns True if the IO object can be written.
292 [clinic start generated code]*/
293
294 static PyObject *
295 _io_BytesIO_writable_impl(bytesio *self)
296 /*[clinic end generated code: output=64ff6a254b1150b8 input=4f35d49d26dab024]*/
297 {
298 CHECK_CLOSED(self);
299 Py_RETURN_TRUE;
300 }
301
302 /*[clinic input]
303 @critical_section
304 _io.BytesIO.seekable
305
306 Returns True if the IO object can be seeked.
307 [clinic start generated code]*/
308
309 static PyObject *
310 _io_BytesIO_seekable_impl(bytesio *self)
311 /*[clinic end generated code: output=6b417f46dcc09b56 input=9cc78d15aa1deaa3]*/
312 {
313 CHECK_CLOSED(self);
314 Py_RETURN_TRUE;
315 }
316
317 /*[clinic input]
318 @critical_section
319 _io.BytesIO.flush
320
321 Does nothing.
322 [clinic start generated code]*/
323
324 static PyObject *
325 _io_BytesIO_flush_impl(bytesio *self)
326 /*[clinic end generated code: output=187e3d781ca134a0 input=c60842743910b381]*/
327 {
328 CHECK_CLOSED(self);
329 Py_RETURN_NONE;
330 }
331
332 /*[clinic input]
333 @critical_section
334 _io.BytesIO.getbuffer
335
336 cls: defining_class
337 /
338
339 Get a read-write view over the contents of the BytesIO object.
340 [clinic start generated code]*/
341
342 static PyObject *
343 _io_BytesIO_getbuffer_impl(bytesio *self, PyTypeObject *cls)
344 /*[clinic end generated code: output=045091d7ce87fe4e input=8295764061be77fd]*/
345 {
346 _PyIO_State *state = get_io_state_by_cls(cls);
347 PyTypeObject *type = state->PyBytesIOBuffer_Type;
348 bytesiobuf *buf;
349 PyObject *view;
350
351 CHECK_CLOSED(self);
352
353 buf = (bytesiobuf *) type->tp_alloc(type, 0);
354 if (buf == NULL)
355 return NULL;
356 buf->source = (bytesio*)Py_NewRef(self);
357 view = PyMemoryView_FromObject((PyObject *) buf);
358 Py_DECREF(buf);
359 return view;
360 }
361
362 /*[clinic input]
363 @critical_section
364 _io.BytesIO.getvalue
365
366 Retrieve the entire contents of the BytesIO object.
367 [clinic start generated code]*/
368
369 static PyObject *
370 _io_BytesIO_getvalue_impl(bytesio *self)
371 /*[clinic end generated code: output=b3f6a3233c8fd628 input=c91bff398df0c352]*/
372 {
373 CHECK_CLOSED(self);
374 if (self->string_size <= 1 || FT_ATOMIC_LOAD_SSIZE_RELAXED(self->exports) > 0)
375 return PyBytes_FromStringAndSize(PyBytes_AS_STRING(self->buf),
376 self->string_size);
377
378 if (self->string_size != PyBytes_GET_SIZE(self->buf)) {
379 if (SHARED_BUF(self)) {
380 if (unshare_buffer_lock_held(self, self->string_size) < 0)
381 return NULL;
382 }
383 else {
384 if (_PyBytes_Resize(&self->buf, self->string_size) < 0)
385 return NULL;
386 }
387 }
388 return Py_NewRef(self->buf);
389 }
390
391 /*[clinic input]
392 @critical_section
393 _io.BytesIO.isatty
394
395 Always returns False.
396
397 BytesIO objects are not connected to a TTY-like device.
398 [clinic start generated code]*/
399
400 static PyObject *
401 _io_BytesIO_isatty_impl(bytesio *self)
402 /*[clinic end generated code: output=df67712e669f6c8f input=50487b74dc5ae8a9]*/
403 {
404 CHECK_CLOSED(self);
405 Py_RETURN_FALSE;
406 }
407
408 /*[clinic input]
409 @critical_section
410 _io.BytesIO.tell
411
412 Current file position, an integer.
413 [clinic start generated code]*/
414
415 static PyObject *
416 _io_BytesIO_tell_impl(bytesio *self)
417 /*[clinic end generated code: output=b54b0f93cd0e5e1d input=2c7b0e8f82e05c4d]*/
418 {
419 CHECK_CLOSED(self);
420 return PyLong_FromSsize_t(self->pos);
421 }
422
423 static PyObject *
424 read_bytes_lock_held(bytesio *self, Py_ssize_t size)
425 {
426 _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self);
427
428 const char *output;
429
430 assert(self->buf != NULL);
431 assert(size <= self->string_size);
432 if (size > 1 &&
433 self->pos == 0 && size == PyBytes_GET_SIZE(self->buf) &&
434 FT_ATOMIC_LOAD_SSIZE_RELAXED(self->exports) == 0) {
435 self->pos += size;
436 return Py_NewRef(self->buf);
437 }
438
439 /* gh-141311: Avoid undefined behavior when self->pos (limit PY_SSIZE_T_MAX)
440 is beyond the size of self->buf. Assert above validates size is always in
441 bounds. When self->pos is out of bounds calling code sets size to 0. */
442 if (size == 0) {
443 return PyBytes_FromStringAndSize(NULL, 0);
444 }
445
446 output = PyBytes_AS_STRING(self->buf) + self->pos;
447 self->pos += size;
448 return PyBytes_FromStringAndSize(output, size);
449 }
450
451 /*[clinic input]
452 @critical_section
453 _io.BytesIO.read
454 size: Py_ssize_t(accept={int, NoneType}) = -1
455 /
456
457 Read at most size bytes, returned as a bytes object.
458
459 If the size argument is negative, read until EOF is reached.
460 Return an empty bytes object at EOF.
461 [clinic start generated code]*/
462
463 static PyObject *
464 _io_BytesIO_read_impl(bytesio *self, Py_ssize_t size)
465 /*[clinic end generated code: output=9cc025f21c75bdd2 input=9e2f7ff3075fdd39]*/
466 {
467 Py_ssize_t n;
468
469 CHECK_CLOSED(self);
470
471 /* adjust invalid sizes */
472 n = self->string_size - self->pos;
473 if (size < 0 || size > n) {
474 size = n;
475 if (size < 0)
476 size = 0;
477 }
478
479 return read_bytes_lock_held(self, size);
480 }
481
482
483 /*[clinic input]
484 @critical_section
485 _io.BytesIO.read1
486 size: Py_ssize_t(accept={int, NoneType}) = -1
487 /
488
489 Read at most size bytes, returned as a bytes object.
490
491 If the size argument is negative or omitted, read until EOF is
492 reached. Return an empty bytes object at EOF.
493 [clinic start generated code]*/
494
495 static PyObject *
496 _io_BytesIO_read1_impl(bytesio *self, Py_ssize_t size)
497 /*[clinic end generated code: output=d0f843285aa95f1c input=796ff4e0efccc4d9]*/
498 {
499 return _io_BytesIO_read_impl(self, size);
500 }
501
502 /*[clinic input]
503 @critical_section
504 _io.BytesIO.readline
505 size: Py_ssize_t(accept={int, NoneType}) = -1
506 /
507
508 Next line from the file, as a bytes object.
509
510 Retain newline. A non-negative size argument limits the maximum
511 number of bytes to return (an incomplete line may be returned then).
512 Return an empty bytes object at EOF.
513 [clinic start generated code]*/
514
515 static PyObject *
516 _io_BytesIO_readline_impl(bytesio *self, Py_ssize_t size)
517 /*[clinic end generated code: output=4bff3c251df8ffcd input=db09d47e23cf2c9e]*/
518 {
519 Py_ssize_t n;
520
521 CHECK_CLOSED(self);
522
523 n = scan_eol_lock_held(self, size);
524
525 return read_bytes_lock_held(self, n);
526 }
527
528 /*[clinic input]
529 @critical_section
530 _io.BytesIO.readlines
531 size as arg: object = None
532 /
533
534 List of bytes objects, each a line from the file.
535
536 Call readline() repeatedly and return a list of the lines so read.
537 The optional size argument, if given, is an approximate bound on the
538 total number of bytes in the lines returned.
539 [clinic start generated code]*/
540
541 static PyObject *
542 _io_BytesIO_readlines_impl(bytesio *self, PyObject *arg)
543 /*[clinic end generated code: output=09b8e34c880808ff input=5c57d7d78e409985]*/
544 {
545 Py_ssize_t maxsize, size, n;
546 PyObject *result, *line;
547 const char *output;
548
549 CHECK_CLOSED(self);
550
551 if (PyLong_Check(arg)) {
552 maxsize = PyLong_AsSsize_t(arg);
553 if (maxsize == -1 && PyErr_Occurred())
554 return NULL;
555 }
556 else if (arg == Py_None) {
557 /* No size limit, by default. */
558 maxsize = -1;
559 }
560 else {
561 PyErr_Format(PyExc_TypeError, "integer argument expected, got '%s'",
562 Py_TYPE(arg)->tp_name);
563 return NULL;
564 }
565
566 size = 0;
567 result = PyList_New(0);
568 if (!result)
569 return NULL;
570
571 output = PyBytes_AS_STRING(self->buf) + self->pos;
572 while ((n = scan_eol_lock_held(self, -1)) != 0) {
573 self->pos += n;
574 line = PyBytes_FromStringAndSize(output, n);
575 if (!line)
576 goto on_error;
577 if (PyList_Append(result, line) == -1) {
578 Py_DECREF(line);
579 goto on_error;
580 }
581 Py_DECREF(line);
582 size += n;
583 if (maxsize > 0 && size >= maxsize)
584 break;
585 output += n;
586 }
587 return result;
588
589 on_error:
590 Py_DECREF(result);
591 return NULL;
592 }
593
594 /*[clinic input]
595 @critical_section
596 _io.BytesIO.readinto
597 buffer: Py_buffer(accept={rwbuffer})
598 /
599
600 Read bytes into buffer.
601
602 Returns number of bytes read (0 for EOF), or None if the object
603 is set not to block and has no data to read.
604 [clinic start generated code]*/
605
606 static PyObject *
607 _io_BytesIO_readinto_impl(bytesio *self, Py_buffer *buffer)
608 /*[clinic end generated code: output=a5d407217dcf0639 input=093a8d330de3fcd1]*/
609 {
610 Py_ssize_t len, n;
611
612 CHECK_CLOSED(self);
613
614 /* adjust invalid sizes */
615 len = buffer->len;
616 n = self->string_size - self->pos;
617 if (len > n) {
618 len = n;
619 if (len < 0) {
620 /* gh-141311: Avoid undefined behavior when self->pos (limit
621 PY_SSIZE_T_MAX) points beyond the size of self->buf. */
622 return PyLong_FromSsize_t(0);
623 }
624 }
625
626 assert(self->pos + len <= PY_SSIZE_T_MAX);
627 assert(len >= 0);
628 memcpy(buffer->buf, PyBytes_AS_STRING(self->buf) + self->pos, len);
629 self->pos += len;
630
631 return PyLong_FromSsize_t(len);
632 }
633
634 /*[clinic input]
635 @critical_section
636 _io.BytesIO.truncate
637 size: object = None
638 /
639
640 Truncate the file to at most size bytes.
641
642 Size defaults to the current file position, as returned by tell().
643 The current file position is unchanged. Returns the new size.
644 [clinic start generated code]*/
645
646 static PyObject *
647 _io_BytesIO_truncate_impl(bytesio *self, PyObject *size)
648 /*[clinic end generated code: output=ab42491b4824f384 input=b4acb5f80481c053]*/
649 {
650 CHECK_CLOSED(self);
651 CHECK_EXPORTS(self);
652
653 Py_ssize_t new_size;
654
655 if (size == Py_None) {
656 new_size = self->pos;
657 }
658 else {
659 new_size = PyLong_AsLong(size);
660 if (new_size == -1 && PyErr_Occurred()) {
661 return NULL;
662 }
663 if (new_size < 0) {
664 PyErr_Format(PyExc_ValueError,
665 "negative size value %zd", new_size);
666 return NULL;
667 }
668 }
669
670 if (new_size < self->string_size) {
671 self->string_size = new_size;
672 if (resize_buffer_lock_held(self, new_size) < 0)
673 return NULL;
674 }
675
676 return PyLong_FromSsize_t(new_size);
677 }
678
679 static PyObject *
680 bytesio_iternext_lock_held(PyObject *op)
681 {
682 _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op);
683
684 Py_ssize_t n;
685 bytesio *self = bytesio_CAST(op);
686
687 CHECK_CLOSED(self);
688
689 n = scan_eol_lock_held(self, -1);
690
691 if (n == 0)
692 return NULL;
693
694 return read_bytes_lock_held(self, n);
695 }
696
697 static PyObject *
698 bytesio_iternext(PyObject *op)
699 {
700 PyObject *ret;
701 Py_BEGIN_CRITICAL_SECTION(op);
702 ret = bytesio_iternext_lock_held(op);
703 Py_END_CRITICAL_SECTION();
704 return ret;
705 }
706
707 /*[clinic input]
708 @critical_section
709 _io.BytesIO.seek
710 pos: Py_ssize_t
711 whence: int = 0
712 /
713
714 Change stream position.
715
716 Seek to byte offset pos relative to position indicated by whence:
717 0 Start of stream (the default). pos should be >= 0;
718 1 Current position - pos may be negative;
719 2 End of stream - pos usually negative.
720 Returns the new absolute position.
721 [clinic start generated code]*/
722
723 static PyObject *
724 _io_BytesIO_seek_impl(bytesio *self, Py_ssize_t pos, int whence)
725 /*[clinic end generated code: output=c26204a68e9190e4 input=20f05ddf659255df]*/
726 {
727 CHECK_CLOSED(self);
728
729 if (pos < 0 && whence == 0) {
730 PyErr_Format(PyExc_ValueError,
731 "negative seek value %zd", pos);
732 return NULL;
733 }
734
735 /* whence = 0: offset relative to beginning of the string.
736 whence = 1: offset relative to current position.
737 whence = 2: offset relative the end of the string. */
738 if (whence == 1) {
739 if (pos > PY_SSIZE_T_MAX - self->pos) {
740 PyErr_SetString(PyExc_OverflowError,
741 "new position too large");
742 return NULL;
743 }
744 pos += self->pos;
745 }
746 else if (whence == 2) {
747 if (pos > PY_SSIZE_T_MAX - self->string_size) {
748 PyErr_SetString(PyExc_OverflowError,
749 "new position too large");
750 return NULL;
751 }
752 pos += self->string_size;
753 }
754 else if (whence != 0) {
755 PyErr_Format(PyExc_ValueError,
756 "invalid whence (%i, should be 0, 1 or 2)", whence);
757 return NULL;
758 }
759
760 if (pos < 0)
761 pos = 0;
762 self->pos = pos;
763
764 return PyLong_FromSsize_t(self->pos);
765 }
766
767 /*[clinic input]
768 @critical_section
769 _io.BytesIO.write
770 b: object
771 /
772
773 Write bytes to file.
774
775 Return the number of bytes written.
776 [clinic start generated code]*/
777
778 static PyObject *
779 _io_BytesIO_write_impl(bytesio *self, PyObject *b)
780 /*[clinic end generated code: output=d3e46bcec8d9e21c input=46c0c17eac7474a4]*/
781 {
782 Py_ssize_t n = write_bytes_lock_held(self, b);
783 return n >= 0 ? PyLong_FromSsize_t(n) : NULL;
784 }
785
786 /*[clinic input]
787 @critical_section
788 _io.BytesIO.writelines
789 lines: object
790 /
791
792 Write lines to the file.
793
794 Note that newlines are not added. lines can be any iterable object
795 producing bytes-like objects. This is equivalent to calling write()
796 for each element.
797 [clinic start generated code]*/
798
799 static PyObject *
800 _io_BytesIO_writelines_impl(bytesio *self, PyObject *lines)
801 /*[clinic end generated code: output=03a43a75773bc397 input=d265f76533b058e7]*/
802 {
803 PyObject *it, *item;
804
805 CHECK_CLOSED(self);
806
807 it = PyObject_GetIter(lines);
808 if (it == NULL)
809 return NULL;
810
811 while ((item = PyIter_Next(it)) != NULL) {
812 Py_ssize_t ret = write_bytes_lock_held(self, item);
813 Py_DECREF(item);
814 if (ret < 0) {
815 Py_DECREF(it);
816 return NULL;
817 }
818 }
819 Py_DECREF(it);
820
821 /* See if PyIter_Next failed */
822 if (PyErr_Occurred())
823 return NULL;
824
825 Py_RETURN_NONE;
826 }
827
828 /*[clinic input]
829 @critical_section
830 _io.BytesIO.close
831
832 Disable all I/O operations.
833 [clinic start generated code]*/
834
835 static PyObject *
836 _io_BytesIO_close_impl(bytesio *self)
837 /*[clinic end generated code: output=1471bb9411af84a0 input=34ce76d8bd17a23b]*/
838 {
839 CHECK_EXPORTS(self);
840 Py_CLEAR(self->buf);
841 Py_RETURN_NONE;
842 }
843
844 /* Pickling support.
845
846 Note that only pickle protocol 2 and onward are supported since we use
847 extended __reduce__ API of PEP 307 to make BytesIO instances picklable.
848
849 Providing support for protocol < 2 would require the __reduce_ex__ method
850 which is notably long-winded when defined properly.
851
852 For BytesIO, the implementation would similar to one coded for
853 object.__reduce_ex__, but slightly less general. To be more specific, we
854 could call bytesio_getstate directly and avoid checking for the presence of
855 a fallback __reduce__ method. However, we would still need a __newobj__
856 function to use the efficient instance representation of PEP 307.
857 */
858
859 static PyObject *
860 bytesio_getstate_lock_held(PyObject *op)
861 {
862 _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op);
863
864 bytesio *self = bytesio_CAST(op);
865 PyObject *initvalue = _io_BytesIO_getvalue_impl(self);
866 PyObject *dict;
867 PyObject *state;
868
869 if (initvalue == NULL)
870 return NULL;
871 if (self->dict == NULL) {
872 dict = Py_NewRef(Py_None);
873 }
874 else {
875 dict = PyDict_Copy(self->dict);
876 if (dict == NULL) {
877 Py_DECREF(initvalue);
878 return NULL;
879 }
880 }
881
882 state = Py_BuildValue("(OnN)", initvalue, self->pos, dict);
883 Py_DECREF(initvalue);
884 return state;
885 }
886
887 static PyObject *
888 bytesio_getstate(PyObject *op, PyObject *Py_UNUSED(dummy))
889 {
890 PyObject *ret;
891 Py_BEGIN_CRITICAL_SECTION(op);
892 ret = bytesio_getstate_lock_held(op);
893 Py_END_CRITICAL_SECTION();
894 return ret;
895 }
896
897 static PyObject *
898 bytesio_setstate_lock_held(PyObject *op, PyObject *state)
899 {
900 _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op);
901
902 PyObject *result;
903 PyObject *position_obj;
904 PyObject *dict;
905 Py_ssize_t pos;
906 bytesio *self = bytesio_CAST(op);
907
908 assert(state != NULL);
909
910 /* We allow the state tuple to be longer than 3, because we may need
911 someday to extend the object's state without breaking
912 backward-compatibility. */
913 if (!PyTuple_Check(state) || PyTuple_GET_SIZE(state) < 3) {
914 PyErr_Format(PyExc_TypeError,
915 "%.200s.__setstate__ argument should be 3-tuple, got %.200s",
916 Py_TYPE(self)->tp_name, Py_TYPE(state)->tp_name);
917 return NULL;
918 }
919 CHECK_EXPORTS(self);
920 /* Reset the object to its default state. This is only needed to handle
921 the case of repeated calls to __setstate__. */
922 self->string_size = 0;
923 self->pos = 0;
924
925 /* Set the value of the internal buffer. If state[0] does not support the
926 buffer protocol, bytesio_write will raise the appropriate TypeError. */
927 result = _io_BytesIO_write_impl(self, PyTuple_GET_ITEM(state, 0));
928 if (result == NULL)
929 return NULL;
930 Py_DECREF(result);
931
932 /* Set carefully the position value. Alternatively, we could use the seek
933 method instead of modifying self->pos directly to better protect the
934 object internal state against erroneous (or malicious) inputs. */
935 position_obj = PyTuple_GET_ITEM(state, 1);
936 if (!PyLong_Check(position_obj)) {
937 PyErr_Format(PyExc_TypeError,
938 "second item of state must be an integer, not %.200s",
939 Py_TYPE(position_obj)->tp_name);
940 return NULL;
941 }
942 pos = PyLong_AsSsize_t(position_obj);
943 if (pos == -1 && PyErr_Occurred())
944 return NULL;
945 if (pos < 0) {
946 PyErr_SetString(PyExc_ValueError,
947 "position value cannot be negative");
948 return NULL;
949 }
950 self->pos = pos;
951
952 /* Set the dictionary of the instance variables. */
953 dict = PyTuple_GET_ITEM(state, 2);
954 if (dict != Py_None) {
955 if (!PyDict_Check(dict)) {
956 PyErr_Format(PyExc_TypeError,
957 "third item of state should be a dict, got a %.200s",
958 Py_TYPE(dict)->tp_name);
959 return NULL;
960 }
961 if (self->dict) {
962 /* Alternatively, we could replace the internal dictionary
963 completely. However, it seems more practical to just update it. */
964 if (PyDict_Update(self->dict, dict) < 0)
965 return NULL;
966 }
967 else {
968 self->dict = Py_NewRef(dict);
969 }
970 }
971
972 Py_RETURN_NONE;
973 }
974
975 static PyObject *
976 bytesio_setstate(PyObject *op, PyObject *state)
977 {
978 PyObject *ret;
979 Py_BEGIN_CRITICAL_SECTION(op);
980 ret = bytesio_setstate_lock_held(op, state);
981 Py_END_CRITICAL_SECTION();
982 return ret;
983 }
984
985 static void
986 bytesio_dealloc(PyObject *op)
987 {
988 bytesio *self = bytesio_CAST(op);
989 PyTypeObject *tp = Py_TYPE(self);
990 _PyObject_GC_UNTRACK(self);
991 if (FT_ATOMIC_LOAD_SSIZE_RELAXED(self->exports) > 0) {
992 PyErr_SetString(PyExc_SystemError,
993 "deallocated BytesIO object has exported buffers");
994 PyErr_Print();
995 }
996 Py_CLEAR(self->buf);
997 Py_CLEAR(self->dict);
998 FT_CLEAR_WEAKREFS(op, self->weakreflist);
999 tp->tp_free(self);
1000 Py_DECREF(tp);
1001 }
1002
1003 static PyObject *
1004 bytesio_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
1005 {
1006 bytesio *self;
1007
1008 assert(type != NULL && type->tp_alloc != NULL);
1009 self = (bytesio *)type->tp_alloc(type, 0);
1010 if (self == NULL)
1011 return NULL;
1012
1013 /* tp_alloc initializes all the fields to zero. So we don't have to
1014 initialize them here. */
1015
1016 self->buf = PyBytes_FromStringAndSize(NULL, 0);
1017 if (self->buf == NULL) {
1018 Py_DECREF(self);
1019 return PyErr_NoMemory();
1020 }
1021
1022 return (PyObject *)self;
1023 }
1024
1025 /*[clinic input]
1026 @critical_section
1027 _io.BytesIO.__init__
1028 initial_bytes as initvalue: object(c_default="NULL") = b''
1029
1030 Buffered I/O implementation using an in-memory bytes buffer.
1031 [clinic start generated code]*/
1032
1033 static int
1034 _io_BytesIO___init___impl(bytesio *self, PyObject *initvalue)
1035 /*[clinic end generated code: output=65c0c51e24c5b621 input=3da5a74ee4c4f1ac]*/
1036 {
1037 /* In case, __init__ is called multiple times. */
1038 self->string_size = 0;
1039 self->pos = 0;
1040
1041 if (FT_ATOMIC_LOAD_SSIZE_RELAXED(self->exports) > 0) {
1042 PyErr_SetString(PyExc_BufferError,
1043 "Existing exports of data: object cannot be re-sized");
1044 return -1;
1045 }
1046 if (initvalue && initvalue != Py_None) {
1047 if (PyBytes_CheckExact(initvalue)) {
1048 Py_XSETREF(self->buf, Py_NewRef(initvalue));
1049 self->string_size = PyBytes_GET_SIZE(initvalue);
1050 }
1051 else {
1052 PyObject *res;
1053 res = _io_BytesIO_write_impl(self, initvalue);
1054 if (res == NULL)
1055 return -1;
1056 Py_DECREF(res);
1057 self->pos = 0;
1058 }
1059 }
1060
1061 return 0;
1062 }
1063
1064 static PyObject *
1065 bytesio_sizeof_lock_held(PyObject *op)
1066 {
1067 _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op);
1068
1069 bytesio *self = bytesio_CAST(op);
1070 size_t res = _PyObject_SIZE(Py_TYPE(self));
1071 if (self->buf && !SHARED_BUF(self)) {
1072 size_t s = _PySys_GetSizeOf(self->buf);
1073 if (s == (size_t)-1) {
1074 return NULL;
1075 }
1076 res += s;
1077 }
1078 return PyLong_FromSize_t(res);
1079 }
1080
1081 static PyObject *
1082 bytesio_sizeof(PyObject *op, PyObject *Py_UNUSED(dummy))
1083 {
1084 PyObject *ret;
1085 Py_BEGIN_CRITICAL_SECTION(op);
1086 ret = bytesio_sizeof_lock_held(op);
1087 Py_END_CRITICAL_SECTION();
1088 return ret;
1089 }
1090
1091 static int
1092 bytesio_traverse(PyObject *op, visitproc visit, void *arg)
1093 {
1094 bytesio *self = bytesio_CAST(op);
1095 Py_VISIT(Py_TYPE(self));
1096 Py_VISIT(self->dict);
1097 Py_VISIT(self->buf);
1098 return 0;
1099 }
1100
1101 static int
1102 bytesio_clear(PyObject *op)
1103 {
1104 bytesio *self = bytesio_CAST(op);
1105 Py_CLEAR(self->dict);
1106 if (FT_ATOMIC_LOAD_SSIZE_RELAXED(self->exports) == 0) {
1107 Py_CLEAR(self->buf);
1108 }
1109 return 0;
1110 }
1111
1112
1113 #define clinic_state() (find_io_state_by_def(Py_TYPE(self)))
1114 #include "clinic/bytesio.c.h"
1115 #undef clinic_state
1116
1117 static PyGetSetDef bytesio_getsetlist[] = {
1118 {"closed", bytesio_get_closed, NULL,
1119 "True if the file is closed."},
1120 {NULL}, /* sentinel */
1121 };
1122
1123 static struct PyMethodDef bytesio_methods[] = {
1124 _IO_BYTESIO_READABLE_METHODDEF
1125 _IO_BYTESIO_SEEKABLE_METHODDEF
1126 _IO_BYTESIO_WRITABLE_METHODDEF
1127 _IO_BYTESIO_CLOSE_METHODDEF
1128 _IO_BYTESIO_FLUSH_METHODDEF
1129 _IO_BYTESIO_ISATTY_METHODDEF
1130 _IO_BYTESIO_TELL_METHODDEF
1131 _IO_BYTESIO_WRITE_METHODDEF
1132 _IO_BYTESIO_WRITELINES_METHODDEF
1133 _IO_BYTESIO_READ1_METHODDEF
1134 _IO_BYTESIO_READINTO_METHODDEF
1135 _IO_BYTESIO_READLINE_METHODDEF
1136 _IO_BYTESIO_READLINES_METHODDEF
1137 _IO_BYTESIO_READ_METHODDEF
1138 _IO_BYTESIO_GETBUFFER_METHODDEF
1139 _IO_BYTESIO_GETVALUE_METHODDEF
1140 _IO_BYTESIO_SEEK_METHODDEF
1141 _IO_BYTESIO_TRUNCATE_METHODDEF
1142 {"__getstate__", bytesio_getstate, METH_NOARGS, NULL},
1143 {"__setstate__", bytesio_setstate, METH_O, NULL},
1144 {"__sizeof__", bytesio_sizeof, METH_NOARGS, NULL},
1145 {NULL, NULL} /* sentinel */
1146 };
1147
1148 static PyMemberDef bytesio_members[] = {
1149 {"__weaklistoffset__", Py_T_PYSSIZET, offsetof(bytesio, weakreflist), Py_READONLY},
1150 {"__dictoffset__", Py_T_PYSSIZET, offsetof(bytesio, dict), Py_READONLY},
1151 {NULL}
1152 };
1153
1154 static PyType_Slot bytesio_slots[] = {
1155 {Py_tp_dealloc, bytesio_dealloc},
1156 {Py_tp_doc, (void *)_io_BytesIO___init____doc__},
1157 {Py_tp_traverse, bytesio_traverse},
1158 {Py_tp_clear, bytesio_clear},
1159 {Py_tp_iter, PyObject_SelfIter},
1160 {Py_tp_iternext, bytesio_iternext},
1161 {Py_tp_methods, bytesio_methods},
1162 {Py_tp_members, bytesio_members},
1163 {Py_tp_getset, bytesio_getsetlist},
1164 {Py_tp_init, _io_BytesIO___init__},
1165 {Py_tp_new, bytesio_new},
1166 {0, NULL},
1167 };
1168
1169 PyType_Spec bytesio_spec = {
1170 .name = "_io.BytesIO",
1171 .basicsize = sizeof(bytesio),
1172 .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC |
1173 Py_TPFLAGS_IMMUTABLETYPE),
1174 .slots = bytesio_slots,
1175 };
1176
1177 /*
1178 * Implementation of the small intermediate object used by getbuffer().
1179 * getbuffer() returns a memoryview over this object, which should make it
1180 * invisible from Python code.
1181 */
1182
1183 static int
1184 bytesiobuf_getbuffer_lock_held(PyObject *op, Py_buffer *view, int flags)
1185 {
1186 bytesiobuf *obj = bytesiobuf_CAST(op);
1187 bytesio *b = bytesio_CAST(obj->source);
1188
1189 _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(b);
1190
1191 if (FT_ATOMIC_LOAD_SSIZE_RELAXED(b->exports) == 0 && SHARED_BUF(b)) {
1192 if (unshare_buffer_lock_held(b, b->string_size) < 0)
1193 return -1;
1194 }
1195
1196 /* cannot fail if view != NULL and readonly == 0 */
1197 (void)PyBuffer_FillInfo(view, op,
1198 PyBytes_AS_STRING(b->buf), b->string_size,
1199 0, flags);
1200 FT_ATOMIC_ADD_SSIZE(b->exports, 1);
1201 return 0;
1202 }
1203
1204 static int
1205 bytesiobuf_getbuffer(PyObject *op, Py_buffer *view, int flags)
1206 {
1207 if (view == NULL) {
1208 PyErr_SetString(PyExc_BufferError,
1209 "bytesiobuf_getbuffer: view==NULL argument is obsolete");
1210 return -1;
1211 }
1212
1213 int ret;
1214 Py_BEGIN_CRITICAL_SECTION(bytesiobuf_CAST(op)->source);
1215 ret = bytesiobuf_getbuffer_lock_held(op, view, flags);
1216 Py_END_CRITICAL_SECTION();
1217 return ret;
1218 }
1219
1220 static void
1221 bytesiobuf_releasebuffer(PyObject *op, Py_buffer *Py_UNUSED(view))
1222 {
1223 bytesiobuf *obj = bytesiobuf_CAST(op);
1224 bytesio *b = bytesio_CAST(obj->source);
1225 FT_ATOMIC_ADD_SSIZE(b->exports, -1);
1226 }
1227
1228 static int
1229 bytesiobuf_traverse(PyObject *op, visitproc visit, void *arg)
1230 {
1231 bytesiobuf *self = bytesiobuf_CAST(op);
1232 Py_VISIT(Py_TYPE(self));
1233 Py_VISIT(self->source);
1234 return 0;
1235 }
1236
1237 static void
1238 bytesiobuf_dealloc(PyObject *op)
1239 {
1240 bytesiobuf *self = bytesiobuf_CAST(op);
1241 PyTypeObject *tp = Py_TYPE(self);
1242 /* bpo-31095: UnTrack is needed before calling any callbacks */
1243 PyObject_GC_UnTrack(op);
1244 Py_CLEAR(self->source);
1245 tp->tp_free(self);
1246 Py_DECREF(tp);
1247 }
1248
1249 static PyType_Slot bytesiobuf_slots[] = {
1250 {Py_tp_dealloc, bytesiobuf_dealloc},
1251 {Py_tp_traverse, bytesiobuf_traverse},
1252
1253 // Buffer protocol
1254 {Py_bf_getbuffer, bytesiobuf_getbuffer},
1255 {Py_bf_releasebuffer, bytesiobuf_releasebuffer},
1256 {0, NULL},
1257 };
1258
1259 PyType_Spec bytesiobuf_spec = {
1260 .name = "_io._BytesIOBuffer",
1261 .basicsize = sizeof(bytesiobuf),
1262 .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC |
1263 Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_DISALLOW_INSTANTIATION),
1264 .slots = bytesiobuf_slots,
1265 };