Debian Codegraph @

Modules/_io/fileio.c

raw
1 /* Author: Daniel Stutzbach */
2
3 #include "Python.h"
4 #include "pycore_fileutils.h" // _Py_BEGIN_SUPPRESS_IPH
5 #include "pycore_object.h" // _PyObject_GC_UNTRACK()
6 #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1()
7 #include "pycore_weakref.h" // FT_CLEAR_WEAKREFS()
8
9 #include <stdbool.h> // bool
10 #ifdef HAVE_UNISTD_H
11 # include <unistd.h> // lseek()
12 #endif
13 #ifdef HAVE_SYS_TYPES_H
14 # include <sys/types.h>
15 #endif
16 #ifdef HAVE_IO_H
17 # include <io.h>
18 #endif
19 #ifdef HAVE_FCNTL_H
20 # include <fcntl.h> // open()
21 #endif
22
23 #include "_iomodule.h"
24
25 /*
26 * Known likely problems:
27 *
28 * - Files larger then 2**32-1
29 * - Files with unicode filenames
30 * - Passing numbers greater than 2**32-1 when an integer is expected
31 * - Making it work on Windows and other oddball platforms
32 *
33 * To Do:
34 *
35 * - autoconfify header file inclusion
36 */
37
38 #ifdef MS_WINDOWS
39 // can simulate truncate with Win32 API functions; see file_truncate
40 # define HAVE_FTRUNCATE
41 # ifndef WIN32_LEAN_AND_MEAN
42 # define WIN32_LEAN_AND_MEAN
43 # endif
44 # include <windows.h>
45 #endif
46
47 #if BUFSIZ < (8*1024)
48 # define SMALLCHUNK (8*1024)
49 #elif (BUFSIZ >= (2 << 25))
50 # error "unreasonable BUFSIZ > 64 MiB defined"
51 #else
52 # define SMALLCHUNK BUFSIZ
53 #endif
54
55 /* Size at which a buffer is considered "large" and behavior should change to
56 avoid excessive memory allocation */
57 #define LARGE_BUFFER_CUTOFF_SIZE 65536
58
59 /*[clinic input]
60 module _io
61 class _io.FileIO "fileio *" "clinic_state()->PyFileIO_Type"
62 [clinic start generated code]*/
63 /*[clinic end generated code: output=da39a3ee5e6b4b0d input=ac25ec278f4d6703]*/
64
65 typedef struct {
66 PyObject_HEAD
67 int fd;
68 unsigned int created : 1;
69 unsigned int readable : 1;
70 unsigned int writable : 1;
71 unsigned int appending : 1;
72 signed int seekable : 2; /* -1 means unknown */
73 unsigned int closefd : 1;
74 char finalizing;
75 /* Stat result which was grabbed at file open, useful for optimizing common
76 File I/O patterns to be more efficient. This is only guidance / an
77 estimate, as it is subject to Time-Of-Check to Time-Of-Use (TOCTOU)
78 issues / bugs. Both the underlying file descriptor and file may be
79 modified outside of the fileio object / Python (ex. gh-90102, GH-121941,
80 gh-109523). */
81 struct _Py_stat_struct *stat_atopen;
82 PyObject *weakreflist;
83 PyObject *dict;
84 } fileio;
85
86 #define PyFileIO_Check(state, op) (PyObject_TypeCheck((op), state->PyFileIO_Type))
87 #define PyFileIO_CAST(op) ((fileio *)(op))
88
89 /* Forward declarations */
90 static PyObject* portable_lseek(fileio *self, PyObject *posobj, int whence, bool suppress_pipe_error);
91
92 int
93 _PyFileIO_closed(PyObject *self)
94 {
95 return (PyFileIO_CAST(self)->fd < 0);
96 }
97
98 /* Because this can call arbitrary code, it shouldn't be called when
99 the refcount is 0 (that is, not directly from tp_dealloc unless
100 the refcount has been temporarily re-incremented). */
101 static PyObject *
102 fileio_dealloc_warn(PyObject *op, PyObject *source)
103 {
104 fileio *self = PyFileIO_CAST(op);
105 if (self->fd >= 0 && self->closefd) {
106 PyObject *exc = PyErr_GetRaisedException();
107 if (PyErr_ResourceWarning(source, 1, "unclosed file %R", source)) {
108 /* Spurious errors can appear at shutdown */
109 if (PyErr_ExceptionMatches(PyExc_Warning)) {
110 PyErr_FormatUnraisable("Exception ignored "
111 "while finalizing file %R", self);
112 }
113 }
114 PyErr_SetRaisedException(exc);
115 }
116 Py_RETURN_NONE;
117 }
118
119 /* Returns 0 on success, -1 with exception set on failure. */
120 static int
121 internal_close(fileio *self)
122 {
123 int err = 0;
124 int save_errno = 0;
125 if (self->fd >= 0) {
126 int fd = self->fd;
127 self->fd = -1;
128 /* fd is accessible and someone else may have closed it */
129 Py_BEGIN_ALLOW_THREADS
130 _Py_BEGIN_SUPPRESS_IPH
131 err = close(fd);
132 if (err < 0)
133 save_errno = errno;
134 _Py_END_SUPPRESS_IPH
135 Py_END_ALLOW_THREADS
136 }
137 PyMem_Free(self->stat_atopen);
138 self->stat_atopen = NULL;
139 if (err < 0) {
140 errno = save_errno;
141 PyErr_SetFromErrno(PyExc_OSError);
142 return -1;
143 }
144 return 0;
145 }
146
147 /*[clinic input]
148 _io.FileIO.close
149
150 cls: defining_class
151 /
152
153 Close the file.
154
155 A closed file cannot be used for further I/O operations. close()
156 may be called more than once without error.
157 [clinic start generated code]*/
158
159 static PyObject *
160 _io_FileIO_close_impl(fileio *self, PyTypeObject *cls)
161 /*[clinic end generated code: output=c30cbe9d1f23ca58 input=b405751dc4163da3]*/
162 {
163 PyObject *res;
164 int rc;
165 _PyIO_State *state = get_io_state_by_cls(cls);
166 res = PyObject_CallMethodOneArg((PyObject*)state->PyRawIOBase_Type,
167 &_Py_ID(close), (PyObject *)self);
168 if (!self->closefd) {
169 self->fd = -1;
170 return res;
171 }
172
173 PyObject *exc = NULL;
174 if (res == NULL) {
175 exc = PyErr_GetRaisedException();
176 }
177 if (self->finalizing) {
178 PyObject *r = fileio_dealloc_warn((PyObject*)self, (PyObject *) self);
179 if (r) {
180 Py_DECREF(r);
181 }
182 else {
183 PyErr_Clear();
184 }
185 }
186 rc = internal_close(self);
187 if (res == NULL) {
188 _PyErr_ChainExceptions1(exc);
189 }
190 if (rc < 0) {
191 Py_CLEAR(res);
192 }
193 return res;
194 }
195
196 static PyObject *
197 fileio_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
198 {
199 assert(type != NULL && type->tp_alloc != NULL);
200
201 fileio *self = (fileio *) type->tp_alloc(type, 0);
202 if (self == NULL) {
203 return NULL;
204 }
205
206 self->fd = -1;
207 self->created = 0;
208 self->readable = 0;
209 self->writable = 0;
210 self->appending = 0;
211 self->seekable = -1;
212 self->stat_atopen = NULL;
213 self->closefd = 1;
214 self->weakreflist = NULL;
215 return (PyObject *) self;
216 }
217
218 #ifdef O_CLOEXEC
219 extern int _Py_open_cloexec_works;
220 #endif
221
222 /*[clinic input]
223 _io.FileIO.__init__
224 file as nameobj: object
225 mode: str = "r"
226 closefd: bool = True
227 opener: object = None
228
229 Open a file.
230
231 The mode can be 'r' (default), 'w', 'x' or 'a' for reading,
232 writing, exclusive creation or appending. The file will be created
233 if it doesn't exist when opened for writing or appending; it will be
234 truncated when opened for writing. A FileExistsError will be raised
235 if it already exists when opened for creating. Opening a file for
236 creating implies writing so this mode behaves in a similar way to
237 'w'. Add a '+' to the mode to allow simultaneous reading and
238 writing.
239
240 A custom opener can be used by passing a callable as *opener*.
241 The underlying file descriptor for the file object is then obtained
242 by calling opener with (*name*, *flags*). *opener* must return
243 an open file descriptor (passing os.open as *opener* results in
244 functionality similar to passing None).
245 [clinic start generated code]*/
246
247 static int
248 _io_FileIO___init___impl(fileio *self, PyObject *nameobj, const char *mode,
249 int closefd, PyObject *opener)
250 /*[clinic end generated code: output=23413f68e6484bbd input=bac4efcd8f930bf3]*/
251 {
252 #ifdef MS_WINDOWS
253 wchar_t *widename = NULL;
254 #else
255 const char *name = NULL;
256 #endif
257 PyObject *stringobj = NULL;
258 const char *s;
259 int ret = 0;
260 int rwa = 0, plus = 0;
261 int flags = 0;
262 int fd = -1;
263 int fd_is_own = 0;
264 #ifdef O_CLOEXEC
265 int *atomic_flag_works = &_Py_open_cloexec_works;
266 #elif !defined(MS_WINDOWS)
267 int *atomic_flag_works = NULL;
268 #endif
269 int fstat_result;
270 int async_err = 0;
271
272 #ifdef Py_DEBUG
273 _PyIO_State *state = find_io_state_by_def(Py_TYPE(self));
274 assert(PyFileIO_Check(state, self));
275 #endif
276 if (self->fd >= 0) {
277 if (self->closefd) {
278 /* Have to close the existing file first. */
279 if (internal_close(self) < 0) {
280 return -1;
281 }
282 }
283 else
284 self->fd = -1;
285 }
286
287 if (PyBool_Check(nameobj)) {
288 if (PyErr_WarnEx(PyExc_RuntimeWarning,
289 "bool is used as a file descriptor", 1))
290 {
291 return -1;
292 }
293 }
294 fd = PyLong_AsInt(nameobj);
295 if (fd < 0) {
296 if (!PyErr_Occurred()) {
297 PyErr_SetString(PyExc_ValueError,
298 "negative file descriptor");
299 return -1;
300 }
301 PyErr_Clear();
302 }
303
304 if (fd < 0) {
305 #ifdef MS_WINDOWS
306 if (!PyUnicode_FSDecoder(nameobj, &stringobj)) {
307 return -1;
308 }
309 widename = PyUnicode_AsWideCharString(stringobj, NULL);
310 if (widename == NULL)
311 return -1;
312 #else
313 if (!PyUnicode_FSConverter(nameobj, &stringobj)) {
314 return -1;
315 }
316 name = PyBytes_AS_STRING(stringobj);
317 #endif
318 }
319
320 s = mode;
321 while (*s) {
322 switch (*s++) {
323 case 'x':
324 if (rwa) {
325 bad_mode:
326 PyErr_SetString(PyExc_ValueError,
327 "Must have exactly one of create/read/write/append "
328 "mode and at most one plus");
329 goto error;
330 }
331 rwa = 1;
332 self->created = 1;
333 self->writable = 1;
334 flags |= O_EXCL | O_CREAT;
335 break;
336 case 'r':
337 if (rwa)
338 goto bad_mode;
339 rwa = 1;
340 self->readable = 1;
341 break;
342 case 'w':
343 if (rwa)
344 goto bad_mode;
345 rwa = 1;
346 self->writable = 1;
347 flags |= O_CREAT | O_TRUNC;
348 break;
349 case 'a':
350 if (rwa)
351 goto bad_mode;
352 rwa = 1;
353 self->writable = 1;
354 self->appending = 1;
355 flags |= O_APPEND | O_CREAT;
356 break;
357 case 'b':
358 break;
359 case '+':
360 if (plus)
361 goto bad_mode;
362 self->readable = self->writable = 1;
363 plus = 1;
364 break;
365 default:
366 PyErr_Format(PyExc_ValueError,
367 "invalid mode: %.200s", mode);
368 goto error;
369 }
370 }
371
372 if (!rwa)
373 goto bad_mode;
374
375 if (self->readable && self->writable)
376 flags |= O_RDWR;
377 else if (self->readable)
378 flags |= O_RDONLY;
379 else
380 flags |= O_WRONLY;
381
382 #ifdef O_BINARY
383 flags |= O_BINARY;
384 #endif
385
386 #ifdef MS_WINDOWS
387 flags |= O_NOINHERIT;
388 #elif defined(O_CLOEXEC)
389 flags |= O_CLOEXEC;
390 #endif
391
392 if (PySys_Audit("open", "Osi", nameobj, mode, flags) < 0) {
393 goto error;
394 }
395
396 if (fd >= 0) {
397 self->fd = fd;
398 self->closefd = closefd;
399 }
400 else {
401 self->closefd = 1;
402 if (!closefd) {
403 PyErr_SetString(PyExc_ValueError,
404 "Cannot use closefd=False with file name");
405 goto error;
406 }
407
408 errno = 0;
409 if (opener == Py_None) {
410 do {
411 Py_BEGIN_ALLOW_THREADS
412 #ifdef MS_WINDOWS
413 self->fd = _wopen(widename, flags, 0666);
414 #else
415 self->fd = open(name, flags, 0666);
416 #endif
417 Py_END_ALLOW_THREADS
418 } while (self->fd < 0 && errno == EINTR &&
419 !(async_err = PyErr_CheckSignals()));
420
421 if (async_err)
422 goto error;
423
424 if (self->fd < 0) {
425 PyErr_SetFromErrnoWithFilenameObject(PyExc_OSError, nameobj);
426 goto error;
427 }
428 }
429 else {
430 PyObject *fdobj;
431
432 #ifndef MS_WINDOWS
433 /* the opener may clear the atomic flag */
434 atomic_flag_works = NULL;
435 #endif
436
437 fdobj = PyObject_CallFunction(opener, "Oi", nameobj, flags);
438 if (fdobj == NULL)
439 goto error;
440 if (!PyLong_Check(fdobj)) {
441 Py_DECREF(fdobj);
442 PyErr_SetString(PyExc_TypeError,
443 "expected integer from opener");
444 goto error;
445 }
446
447 self->fd = PyLong_AsInt(fdobj);
448 Py_DECREF(fdobj);
449 if (self->fd < 0) {
450 if (!PyErr_Occurred()) {
451 /* The opener returned a negative but didn't set an
452 exception. See issue #27066 */
453 PyErr_Format(PyExc_ValueError,
454 "opener returned %d", self->fd);
455 }
456 goto error;
457 }
458 }
459 fd_is_own = 1;
460
461 #ifndef MS_WINDOWS
462 if (_Py_set_inheritable(self->fd, 0, atomic_flag_works) < 0)
463 goto error;
464 #endif
465 }
466
467 PyMem_Free(self->stat_atopen);
468 self->stat_atopen = PyMem_New(struct _Py_stat_struct, 1);
469 if (self->stat_atopen == NULL) {
470 PyErr_NoMemory();
471 goto error;
472 }
473 Py_BEGIN_ALLOW_THREADS
474 fstat_result = _Py_fstat_noraise(self->fd, self->stat_atopen);
475 Py_END_ALLOW_THREADS
476 if (fstat_result < 0) {
477 /* Tolerate fstat() errors other than EBADF. See Issue #25717, where
478 an anonymous file on a Virtual Box shared folder filesystem would
479 raise ENOENT. */
480 #ifdef MS_WINDOWS
481 if (GetLastError() == ERROR_INVALID_HANDLE) {
482 PyErr_SetFromWindowsErr(0);
483 #else
484 if (errno == EBADF) {
485 PyErr_SetFromErrno(PyExc_OSError);
486 #endif
487 goto error;
488 }
489
490 PyMem_Free(self->stat_atopen);
491 self->stat_atopen = NULL;
492 }
493 else {
494 #if defined(S_ISDIR) && defined(EISDIR)
495 /* On Unix, open will succeed for directories.
496 In Python, there should be no file objects referring to
497 directories, so we need a check. */
498 if (S_ISDIR(self->stat_atopen->st_mode)) {
499 errno = EISDIR;
500 PyErr_SetFromErrnoWithFilenameObject(PyExc_OSError, nameobj);
501 goto error;
502 }
503 #endif /* defined(S_ISDIR) */
504 }
505
506 #if defined(MS_WINDOWS) || defined(__CYGWIN__)
507 /* don't translate newlines (\r\n <=> \n) */
508 _setmode(self->fd, O_BINARY);
509 #endif
510
511 if (PyObject_SetAttr((PyObject *)self, &_Py_ID(name), nameobj) < 0)
512 goto error;
513
514 if (self->appending) {
515 /* For consistent behaviour, we explicitly seek to the
516 end of file (otherwise, it might be done only on the
517 first write()). */
518 PyObject *pos = portable_lseek(self, NULL, 2, true);
519 if (pos == NULL)
520 goto error;
521 Py_DECREF(pos);
522 }
523
524 goto done;
525
526 error:
527 ret = -1;
528 if (!fd_is_own)
529 self->fd = -1;
530 if (self->fd >= 0) {
531 PyObject *exc = PyErr_GetRaisedException();
532 internal_close(self);
533 _PyErr_ChainExceptions1(exc);
534 }
535 PyMem_Free(self->stat_atopen);
536 self->stat_atopen = NULL;
537
538 done:
539 #ifdef MS_WINDOWS
540 PyMem_Free(widename);
541 #endif
542 Py_CLEAR(stringobj);
543 return ret;
544 }
545
546 static int
547 fileio_traverse(PyObject *op, visitproc visit, void *arg)
548 {
549 fileio *self = PyFileIO_CAST(op);
550 Py_VISIT(Py_TYPE(self));
551 Py_VISIT(self->dict);
552 return 0;
553 }
554
555 static int
556 fileio_clear(PyObject *op)
557 {
558 fileio *self = PyFileIO_CAST(op);
559 Py_CLEAR(self->dict);
560 return 0;
561 }
562
563 static void
564 fileio_dealloc(PyObject *op)
565 {
566 fileio *self = PyFileIO_CAST(op);
567 self->finalizing = 1;
568 if (_PyIOBase_finalize(op) < 0) {
569 return;
570 }
571
572 _PyObject_GC_UNTRACK(self);
573 if (self->stat_atopen != NULL) {
574 PyMem_Free(self->stat_atopen);
575 self->stat_atopen = NULL;
576 }
577 FT_CLEAR_WEAKREFS(op, self->weakreflist);
578 (void)fileio_clear(op);
579
580 PyTypeObject *tp = Py_TYPE(op);
581 tp->tp_free(op);
582 Py_DECREF(tp);
583 }
584
585 static PyObject *
586 err_closed(void)
587 {
588 PyErr_SetString(PyExc_ValueError, "I/O operation on closed file");
589 return NULL;
590 }
591
592 static PyObject *
593 err_mode(_PyIO_State *state, const char *action)
594 {
595 return PyErr_Format(state->unsupported_operation,
596 "File not open for %s", action);
597 }
598
599 /*[clinic input]
600 _io.FileIO.fileno
601
602 Return the underlying file descriptor (an integer).
603 [clinic start generated code]*/
604
605 static PyObject *
606 _io_FileIO_fileno_impl(fileio *self)
607 /*[clinic end generated code: output=a9626ce5398ece90 input=0b9b2de67335ada3]*/
608 {
609 if (self->fd < 0)
610 return err_closed();
611 return PyLong_FromLong((long) self->fd);
612 }
613
614 /*[clinic input]
615 _io.FileIO.readable
616
617 True if file was opened in a read mode.
618 [clinic start generated code]*/
619
620 static PyObject *
621 _io_FileIO_readable_impl(fileio *self)
622 /*[clinic end generated code: output=640744a6150fe9ba input=a3fdfed6eea721c5]*/
623 {
624 if (self->fd < 0)
625 return err_closed();
626 return PyBool_FromLong((long) self->readable);
627 }
628
629 /*[clinic input]
630 _io.FileIO.writable
631
632 True if file was opened in a write mode.
633 [clinic start generated code]*/
634
635 static PyObject *
636 _io_FileIO_writable_impl(fileio *self)
637 /*[clinic end generated code: output=96cefc5446e89977 input=c204a808ca2e1748]*/
638 {
639 if (self->fd < 0)
640 return err_closed();
641 return PyBool_FromLong((long) self->writable);
642 }
643
644 /*[clinic input]
645 _io.FileIO.seekable
646
647 True if file supports random-access.
648 [clinic start generated code]*/
649
650 static PyObject *
651 _io_FileIO_seekable_impl(fileio *self)
652 /*[clinic end generated code: output=47909ca0a42e9287 input=c8e5554d2fd63c7f]*/
653 {
654 if (self->fd < 0)
655 return err_closed();
656 if (self->seekable < 0) {
657 /* portable_lseek() sets the seekable attribute */
658 PyObject *pos = portable_lseek(self, NULL, SEEK_CUR, false);
659 assert(self->seekable >= 0);
660 if (pos == NULL) {
661 PyErr_Clear();
662 }
663 else {
664 Py_DECREF(pos);
665 }
666 }
667 return PyBool_FromLong((long) self->seekable);
668 }
669
670 /*[clinic input]
671 _io.FileIO.readinto
672 cls: defining_class
673 buffer: Py_buffer(accept={rwbuffer})
674 /
675
676 Same as RawIOBase.readinto().
677 [clinic start generated code]*/
678
679 static PyObject *
680 _io_FileIO_readinto_impl(fileio *self, PyTypeObject *cls, Py_buffer *buffer)
681 /*[clinic end generated code: output=97f0f3d69534db34 input=fd20323e18ce1ec8]*/
682 {
683 Py_ssize_t n;
684 int err;
685
686 if (self->fd < 0)
687 return err_closed();
688 if (!self->readable) {
689 _PyIO_State *state = get_io_state_by_cls(cls);
690 return err_mode(state, "reading");
691 }
692
693 n = _Py_read(self->fd, buffer->buf, buffer->len);
694 /* copy errno because PyBuffer_Release() can indirectly modify it */
695 err = errno;
696
697 if (n == -1) {
698 if (err == EAGAIN) {
699 PyErr_Clear();
700 Py_RETURN_NONE;
701 }
702 return NULL;
703 }
704
705 return PyLong_FromSsize_t(n);
706 }
707
708 static size_t
709 new_buffersize(fileio *self, size_t currentsize)
710 {
711 size_t addend;
712
713 /* Expand the buffer by an amount proportional to the current size,
714 giving us amortized linear-time behavior. For bigger sizes, use a
715 less-than-double growth factor to avoid excessive allocation. */
716 assert(currentsize <= PY_SSIZE_T_MAX);
717 if (currentsize > LARGE_BUFFER_CUTOFF_SIZE)
718 addend = currentsize >> 3;
719 else
720 addend = 256 + currentsize;
721 if (addend < SMALLCHUNK)
722 /* Avoid tiny read() calls. */
723 addend = SMALLCHUNK;
724 return addend + currentsize;
725 }
726
727 /*[clinic input]
728 _io.FileIO.readall
729
730 Read all data from the file, returned as bytes.
731
732 Reads until either there is an error or read() returns size 0
733 (indicates EOF). If the file is already at EOF, returns an empty
734 bytes object.
735
736 In non-blocking mode, returns as much data as could be read before
737 EAGAIN. If no data is available (EAGAIN is returned before bytes
738 are read) returns None.
739 [clinic start generated code]*/
740
741 static PyObject *
742 _io_FileIO_readall_impl(fileio *self)
743 /*[clinic end generated code: output=faa0292b213b4022 input=393194434f380889]*/
744 {
745 Py_off_t pos, end;
746 PyObject *result;
747 Py_ssize_t bytes_read = 0;
748 Py_ssize_t n;
749 size_t bufsize;
750
751 if (self->fd < 0) {
752 return err_closed();
753 }
754
755 if (self->stat_atopen != NULL && self->stat_atopen->st_size < _PY_READ_MAX) {
756 end = (Py_off_t)self->stat_atopen->st_size;
757 }
758 else {
759 end = -1;
760 }
761 if (end <= 0) {
762 /* Use a default size and resize as needed. */
763 bufsize = SMALLCHUNK;
764 }
765 else {
766 /* This is probably a real file. */
767 if (end > _PY_READ_MAX - 1) {
768 bufsize = _PY_READ_MAX;
769 }
770 else {
771 /* In order to detect end of file, need a read() of at
772 least 1 byte which returns size 0. Oversize the buffer
773 by 1 byte so the I/O can be completed with two read()
774 calls (one for all data, one for EOF) without needing
775 to resize the buffer. */
776 bufsize = (size_t)end + 1;
777 }
778
779 /* While a lot of code does open().read() to get the whole contents
780 of a file it is possible a caller seeks/reads a ways into the file
781 then calls readall() to get the rest, which would result in allocating
782 more than required. Guard against that for larger files where we expect
783 the I/O time to dominate anyways while keeping small files fast. */
784 if (bufsize > LARGE_BUFFER_CUTOFF_SIZE) {
785 Py_BEGIN_ALLOW_THREADS
786 _Py_BEGIN_SUPPRESS_IPH
787 #ifdef MS_WINDOWS
788 pos = _lseeki64(self->fd, 0L, SEEK_CUR);
789 #else
790 pos = lseek(self->fd, 0L, SEEK_CUR);
791 #endif
792 _Py_END_SUPPRESS_IPH
793 Py_END_ALLOW_THREADS
794
795 if (end >= pos && pos >= 0 && (end - pos) < (_PY_READ_MAX - 1)) {
796 bufsize = (size_t)(end - pos) + 1;
797 }
798 }
799 }
800
801
802 result = PyBytes_FromStringAndSize(NULL, bufsize);
803 if (result == NULL)
804 return NULL;
805
806 while (1) {
807 if (bytes_read >= (Py_ssize_t)bufsize) {
808 bufsize = new_buffersize(self, bytes_read);
809 if (bufsize > PY_SSIZE_T_MAX || bufsize <= 0) {
810 PyErr_SetString(PyExc_OverflowError,
811 "unbounded read returned more bytes "
812 "than a Python bytes object can hold");
813 Py_DECREF(result);
814 return NULL;
815 }
816
817 if (PyBytes_GET_SIZE(result) < (Py_ssize_t)bufsize) {
818 if (_PyBytes_Resize(&result, bufsize) < 0)
819 return NULL;
820 }
821 }
822
823 n = _Py_read(self->fd,
824 PyBytes_AS_STRING(result) + bytes_read,
825 bufsize - bytes_read);
826
827 if (n == 0)
828 break;
829 if (n == -1) {
830 if (errno == EAGAIN) {
831 PyErr_Clear();
832 if (bytes_read > 0)
833 break;
834 Py_DECREF(result);
835 Py_RETURN_NONE;
836 }
837 Py_DECREF(result);
838 return NULL;
839 }
840 bytes_read += n;
841 }
842
843 if (PyBytes_GET_SIZE(result) > bytes_read) {
844 if (_PyBytes_Resize(&result, bytes_read) < 0)
845 return NULL;
846 }
847 return result;
848 }
849
850 /*[clinic input]
851 _io.FileIO.read
852 cls: defining_class
853 size: Py_ssize_t(accept={int, NoneType}) = -1
854 /
855
856 Read at most size bytes, returned as bytes.
857
858 If size is less than 0, read all bytes in the file making multiple
859 read calls. See ``FileIO.readall``.
860
861 Attempts to make only one system call, retrying only per PEP 475
862 (EINTR). This means less data may be returned than requested.
863
864 In non-blocking mode, returns None if no data is available. Return
865 an empty bytes object at EOF.
866 [clinic start generated code]*/
867
868 static PyObject *
869 _io_FileIO_read_impl(fileio *self, PyTypeObject *cls, Py_ssize_t size)
870 /*[clinic end generated code: output=bbd749c7c224143e input=c7baa3b440af9337]*/
871 {
872 char *ptr;
873 Py_ssize_t n;
874 PyObject *bytes;
875
876 if (self->fd < 0)
877 return err_closed();
878 if (!self->readable) {
879 _PyIO_State *state = get_io_state_by_cls(cls);
880 return err_mode(state, "reading");
881 }
882
883 if (size < 0)
884 return _io_FileIO_readall_impl(self);
885
886 if (size > _PY_READ_MAX) {
887 size = _PY_READ_MAX;
888 }
889
890 bytes = PyBytes_FromStringAndSize(NULL, size);
891 if (bytes == NULL)
892 return NULL;
893 ptr = PyBytes_AS_STRING(bytes);
894
895 n = _Py_read(self->fd, ptr, size);
896 if (n == -1) {
897 /* copy errno because Py_DECREF() can indirectly modify it */
898 int err = errno;
899 Py_DECREF(bytes);
900 if (err == EAGAIN) {
901 PyErr_Clear();
902 Py_RETURN_NONE;
903 }
904 return NULL;
905 }
906
907 if (n != size) {
908 if (_PyBytes_Resize(&bytes, n) < 0) {
909 Py_CLEAR(bytes);
910 return NULL;
911 }
912 }
913
914 return (PyObject *) bytes;
915 }
916
917 /*[clinic input]
918 _io.FileIO.write
919 cls: defining_class
920 b: Py_buffer
921 /
922
923 Write buffer b to file, return number of bytes written.
924
925 Only makes one system call, so not all of the data may be written.
926 The number of bytes actually written is returned. In non-blocking
927 mode, returns None if the write would block.
928 [clinic start generated code]*/
929
930 static PyObject *
931 _io_FileIO_write_impl(fileio *self, PyTypeObject *cls, Py_buffer *b)
932 /*[clinic end generated code: output=927e25be80f3b77b input=233f1f70f9e8b09e]*/
933 {
934 Py_ssize_t n;
935 int err;
936
937 if (self->fd < 0)
938 return err_closed();
939 if (!self->writable) {
940 _PyIO_State *state = get_io_state_by_cls(cls);
941 return err_mode(state, "writing");
942 }
943
944 n = _Py_write(self->fd, b->buf, b->len);
945 /* copy errno because PyBuffer_Release() can indirectly modify it */
946 err = errno;
947
948 if (n < 0) {
949 if (err == EAGAIN) {
950 PyErr_Clear();
951 Py_RETURN_NONE;
952 }
953 return NULL;
954 }
955
956 return PyLong_FromSsize_t(n);
957 }
958
959 /* XXX Windows support below is likely incomplete */
960
961 /* Cribbed from posix_lseek() */
962 static PyObject *
963 portable_lseek(fileio *self, PyObject *posobj, int whence, bool suppress_pipe_error)
964 {
965 Py_off_t pos, res;
966 int fd = self->fd;
967
968 #ifdef SEEK_SET
969 /* Turn 0, 1, 2 into SEEK_{SET,CUR,END} */
970 switch (whence) {
971 #if SEEK_SET != 0
972 case 0: whence = SEEK_SET; break;
973 #endif
974 #if SEEK_CUR != 1
975 case 1: whence = SEEK_CUR; break;
976 #endif
977 #if SEEK_END != 2
978 case 2: whence = SEEK_END; break;
979 #endif
980 }
981 #endif /* SEEK_SET */
982
983 if (posobj == NULL) {
984 pos = 0;
985 }
986 else {
987 #if defined(HAVE_LARGEFILE_SUPPORT)
988 pos = PyLong_AsLongLong(posobj);
989 #else
990 pos = PyLong_AsLong(posobj);
991 #endif
992 if (PyErr_Occurred())
993 return NULL;
994 }
995
996 Py_BEGIN_ALLOW_THREADS
997 _Py_BEGIN_SUPPRESS_IPH
998 #ifdef MS_WINDOWS
999 res = _lseeki64(fd, pos, whence);
1000 #else
1001 res = lseek(fd, pos, whence);
1002 #endif
1003 _Py_END_SUPPRESS_IPH
1004 Py_END_ALLOW_THREADS
1005
1006 if (self->seekable < 0) {
1007 self->seekable = (res >= 0);
1008 }
1009
1010 if (res < 0) {
1011 if (suppress_pipe_error && errno == ESPIPE) {
1012 res = 0;
1013 } else {
1014 return PyErr_SetFromErrno(PyExc_OSError);
1015 }
1016 }
1017
1018 #if defined(HAVE_LARGEFILE_SUPPORT)
1019 return PyLong_FromLongLong(res);
1020 #else
1021 return PyLong_FromLong(res);
1022 #endif
1023 }
1024
1025 /*[clinic input]
1026 _io.FileIO.seek
1027 pos: object
1028 whence: int = 0
1029 /
1030
1031 Move to new file position and return the file position.
1032
1033 Argument offset is a byte count. Optional argument whence defaults
1034 to SEEK_SET or 0 (offset from start of file, offset should be >= 0);
1035 other values are SEEK_CUR or 1 (move relative to current position,
1036 positive or negative), and SEEK_END or 2 (move relative to end of
1037 file, usually negative, although many platforms allow seeking beyond
1038 the end of a file).
1039
1040 Note that not all file objects are seekable.
1041 [clinic start generated code]*/
1042
1043 static PyObject *
1044 _io_FileIO_seek_impl(fileio *self, PyObject *pos, int whence)
1045 /*[clinic end generated code: output=c976acdf054e6655 input=f165a1b4f5d494ad]*/
1046 {
1047 if (self->fd < 0)
1048 return err_closed();
1049
1050 return portable_lseek(self, pos, whence, false);
1051 }
1052
1053 /*[clinic input]
1054 _io.FileIO.tell
1055
1056 Current file position.
1057
1058 Can raise OSError for non seekable files.
1059 [clinic start generated code]*/
1060
1061 static PyObject *
1062 _io_FileIO_tell_impl(fileio *self)
1063 /*[clinic end generated code: output=ffe2147058809d0b input=807e24ead4cec2f9]*/
1064 {
1065 if (self->fd < 0)
1066 return err_closed();
1067
1068 return portable_lseek(self, NULL, 1, false);
1069 }
1070
1071 #ifdef HAVE_FTRUNCATE
1072 /*[clinic input]
1073 _io.FileIO.truncate
1074 cls: defining_class
1075 size as posobj: object = None
1076 /
1077
1078 Truncate the file to at most size bytes and return the truncated size.
1079
1080 Size defaults to the current file position, as returned by tell().
1081 The current file position is changed to the value of size.
1082 [clinic start generated code]*/
1083
1084 static PyObject *
1085 _io_FileIO_truncate_impl(fileio *self, PyTypeObject *cls, PyObject *posobj)
1086 /*[clinic end generated code: output=d936732a49e8d5a2 input=c367fb45d6bb2c18]*/
1087 {
1088 Py_off_t pos;
1089 int ret;
1090 int fd;
1091
1092 fd = self->fd;
1093 if (fd < 0)
1094 return err_closed();
1095 if (!self->writable) {
1096 _PyIO_State *state = get_io_state_by_cls(cls);
1097 return err_mode(state, "writing");
1098 }
1099
1100 if (posobj == Py_None) {
1101 /* Get the current position. */
1102 posobj = portable_lseek(self, NULL, 1, false);
1103 if (posobj == NULL)
1104 return NULL;
1105 }
1106 else {
1107 Py_INCREF(posobj);
1108 }
1109
1110 #if defined(HAVE_LARGEFILE_SUPPORT)
1111 pos = PyLong_AsLongLong(posobj);
1112 #else
1113 pos = PyLong_AsLong(posobj);
1114 #endif
1115 if (PyErr_Occurred()){
1116 Py_DECREF(posobj);
1117 return NULL;
1118 }
1119
1120 Py_BEGIN_ALLOW_THREADS
1121 _Py_BEGIN_SUPPRESS_IPH
1122 errno = 0;
1123 #ifdef MS_WINDOWS
1124 ret = _chsize_s(fd, pos);
1125 #else
1126 ret = ftruncate(fd, pos);
1127 #endif
1128 _Py_END_SUPPRESS_IPH
1129 Py_END_ALLOW_THREADS
1130
1131 if (ret != 0) {
1132 PyErr_SetFromErrno(PyExc_OSError);
1133 Py_DECREF(posobj);
1134 return NULL;
1135 }
1136
1137 /* Since the file was truncated, its size at open is no longer accurate
1138 as an estimate. Clear out the stat result, and rely on dynamic resize
1139 code if a readall is requested. */
1140 if (self->stat_atopen != NULL) {
1141 PyMem_Free(self->stat_atopen);
1142 self->stat_atopen = NULL;
1143 }
1144
1145 return posobj;
1146 }
1147 #endif /* HAVE_FTRUNCATE */
1148
1149 static const char *
1150 mode_string(fileio *self)
1151 {
1152 if (self->created) {
1153 if (self->readable)
1154 return "xb+";
1155 else
1156 return "xb";
1157 }
1158 if (self->appending) {
1159 if (self->readable)
1160 return "ab+";
1161 else
1162 return "ab";
1163 }
1164 else if (self->readable) {
1165 if (self->writable)
1166 return "rb+";
1167 else
1168 return "rb";
1169 }
1170 else
1171 return "wb";
1172 }
1173
1174 static PyObject *
1175 fileio_repr(PyObject *op)
1176 {
1177 fileio *self = PyFileIO_CAST(op);
1178 const char *type_name = Py_TYPE(self)->tp_name;
1179
1180 if (self->fd < 0) {
1181 return PyUnicode_FromFormat("<%.100s [closed]>", type_name);
1182 }
1183
1184 PyObject *nameobj;
1185 if (PyObject_GetOptionalAttr((PyObject *) self, &_Py_ID(name), &nameobj) < 0) {
1186 return NULL;
1187 }
1188 PyObject *res;
1189 if (nameobj == NULL) {
1190 res = PyUnicode_FromFormat(
1191 "<%.100s fd=%d mode='%s' closefd=%s>",
1192 type_name, self->fd, mode_string(self), self->closefd ? "True" : "False");
1193 }
1194 else {
1195 int status = Py_ReprEnter((PyObject *)self);
1196 res = NULL;
1197 if (status == 0) {
1198 res = PyUnicode_FromFormat(
1199 "<%.100s name=%R mode='%s' closefd=%s>",
1200 type_name, nameobj, mode_string(self), self->closefd ? "True" : "False");
1201 Py_ReprLeave((PyObject *)self);
1202 }
1203 else if (status > 0) {
1204 PyErr_Format(PyExc_RuntimeError,
1205 "reentrant call inside %.100s.__repr__", type_name);
1206 }
1207 Py_DECREF(nameobj);
1208 }
1209 return res;
1210 }
1211
1212 /*[clinic input]
1213 _io.FileIO.isatty
1214
1215 True if the file is connected to a TTY device.
1216 [clinic start generated code]*/
1217
1218 static PyObject *
1219 _io_FileIO_isatty_impl(fileio *self)
1220 /*[clinic end generated code: output=932c39924e9a8070 input=cd94ca1f5e95e843]*/
1221 {
1222 long res;
1223
1224 if (self->fd < 0)
1225 return err_closed();
1226 Py_BEGIN_ALLOW_THREADS
1227 _Py_BEGIN_SUPPRESS_IPH
1228 res = isatty(self->fd);
1229 _Py_END_SUPPRESS_IPH
1230 Py_END_ALLOW_THREADS
1231 return PyBool_FromLong(res);
1232 }
1233
1234 /* Checks whether the file is a TTY using an open-only optimization.
1235
1236 TTYs are always character devices. If the interpreter knows a file is
1237 not a character device when it would call ``isatty``, can skip that
1238 call. Inside ``open()`` there is a fresh stat result that contains that
1239 information. Use the stat result to skip a system call. Outside of that
1240 context TOCTOU issues (the fd could be arbitrarily modified by
1241 surrounding code). */
1242 static PyObject *
1243 _io_FileIO_isatty_open_only(PyObject *op, PyObject *Py_UNUSED(dummy))
1244 {
1245 fileio *self = PyFileIO_CAST(op);
1246 if (self->stat_atopen != NULL && !S_ISCHR(self->stat_atopen->st_mode)) {
1247 Py_RETURN_FALSE;
1248 }
1249 return _io_FileIO_isatty_impl(self);
1250 }
1251
1252 #include "clinic/fileio.c.h"
1253
1254 static PyMethodDef fileio_methods[] = {
1255 _IO_FILEIO_READ_METHODDEF
1256 _IO_FILEIO_READALL_METHODDEF
1257 _IO_FILEIO_READINTO_METHODDEF
1258 _IO_FILEIO_WRITE_METHODDEF
1259 _IO_FILEIO_SEEK_METHODDEF
1260 _IO_FILEIO_TELL_METHODDEF
1261 _IO_FILEIO_TRUNCATE_METHODDEF
1262 _IO_FILEIO_CLOSE_METHODDEF
1263 _IO_FILEIO_SEEKABLE_METHODDEF
1264 _IO_FILEIO_READABLE_METHODDEF
1265 _IO_FILEIO_WRITABLE_METHODDEF
1266 _IO_FILEIO_FILENO_METHODDEF
1267 _IO_FILEIO_ISATTY_METHODDEF
1268 {"_isatty_open_only", _io_FileIO_isatty_open_only, METH_NOARGS},
1269 {"_dealloc_warn", fileio_dealloc_warn, METH_O, NULL},
1270 {"__getstate__", _PyIOBase_cannot_pickle, METH_NOARGS},
1271 {NULL, NULL} /* sentinel */
1272 };
1273
1274 /* 'closed' and 'mode' are attributes for backwards compatibility reasons. */
1275
1276 static PyObject *
1277 fileio_get_closed(PyObject *op, void *closure)
1278 {
1279 fileio *self = PyFileIO_CAST(op);
1280 return PyBool_FromLong((long)(self->fd < 0));
1281 }
1282
1283 static PyObject *
1284 fileio_get_closefd(PyObject *op, void *closure)
1285 {
1286 fileio *self = PyFileIO_CAST(op);
1287 return PyBool_FromLong((long)(self->closefd));
1288 }
1289
1290 static PyObject *
1291 fileio_get_mode(PyObject *op, void *closure)
1292 {
1293 fileio *self = PyFileIO_CAST(op);
1294 return PyUnicode_FromString(mode_string(self));
1295 }
1296
1297 static PyObject *
1298 fileio_get_blksize(PyObject *op, void *closure)
1299 {
1300 #ifdef HAVE_STRUCT_STAT_ST_BLKSIZE
1301 fileio *self = PyFileIO_CAST(op);
1302 if (self->stat_atopen != NULL && self->stat_atopen->st_blksize > 1) {
1303 return PyLong_FromLong(self->stat_atopen->st_blksize);
1304 }
1305 #endif /* HAVE_STRUCT_STAT_ST_BLKSIZE */
1306 return PyLong_FromLong(DEFAULT_BUFFER_SIZE);
1307 }
1308
1309 static PyGetSetDef fileio_getsetlist[] = {
1310 {"closed", fileio_get_closed, NULL, "True if the file is closed"},
1311 {"closefd", fileio_get_closefd, NULL,
1312 "True if the file descriptor will be closed by close()."},
1313 {"mode", fileio_get_mode, NULL, "String giving the file mode"},
1314 {"_blksize", fileio_get_blksize, NULL, "Stat st_blksize if available"},
1315 {NULL},
1316 };
1317
1318 static PyMemberDef fileio_members[] = {
1319 {"_finalizing", Py_T_BOOL, offsetof(fileio, finalizing), 0},
1320 {"__weaklistoffset__", Py_T_PYSSIZET, offsetof(fileio, weakreflist), Py_READONLY},
1321 {"__dictoffset__", Py_T_PYSSIZET, offsetof(fileio, dict), Py_READONLY},
1322 {NULL}
1323 };
1324
1325 static PyType_Slot fileio_slots[] = {
1326 {Py_tp_dealloc, fileio_dealloc},
1327 {Py_tp_repr, fileio_repr},
1328 {Py_tp_doc, (void *)_io_FileIO___init____doc__},
1329 {Py_tp_traverse, fileio_traverse},
1330 {Py_tp_clear, fileio_clear},
1331 {Py_tp_methods, fileio_methods},
1332 {Py_tp_members, fileio_members},
1333 {Py_tp_getset, fileio_getsetlist},
1334 {Py_tp_init, _io_FileIO___init__},
1335 {Py_tp_new, fileio_new},
1336 {0, NULL},
1337 };
1338
1339 PyType_Spec fileio_spec = {
1340 .name = "_io.FileIO",
1341 .basicsize = sizeof(fileio),
1342 .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC |
1343 Py_TPFLAGS_IMMUTABLETYPE),
1344 .slots = fileio_slots,
1345 };