Debian Codegraph @

Objects/stringlib/join.h

raw
1 /* stringlib: bytes joining implementation */
2
3 #if STRINGLIB_IS_UNICODE
4 #error join.h only compatible with byte-wise strings
5 #endif
6
7 Py_LOCAL_INLINE(PyObject *)
8 STRINGLIB(bytes_join)(PyObject *sep, PyObject *iterable)
9 {
10 const char *sepstr = STRINGLIB_STR(sep);
11 Py_ssize_t seplen = STRINGLIB_LEN(sep);
12 PyObject *res = NULL;
13 char *p;
14 Py_ssize_t seqlen = 0;
15 Py_ssize_t sz = 0;
16 Py_ssize_t i, nbufs;
17 PyObject *seq, *item;
18 Py_buffer *buffers = NULL;
19 #define NB_STATIC_BUFFERS 10
20 Py_buffer static_buffers[NB_STATIC_BUFFERS];
21 #define GIL_THRESHOLD 1048576
22 int drop_gil = 1;
23 PyThreadState *save = NULL;
24
25 seq = PySequence_Fast(iterable, "can only join an iterable");
26 if (seq == NULL) {
27 return NULL;
28 }
29
30 seqlen = PySequence_Fast_GET_SIZE(seq);
31 if (seqlen == 0) {
32 Py_DECREF(seq);
33 return STRINGLIB_NEW(NULL, 0);
34 }
35 #if !STRINGLIB_MUTABLE
36 if (seqlen == 1) {
37 item = PySequence_Fast_GET_ITEM(seq, 0);
38 if (STRINGLIB_CHECK_EXACT(item)) {
39 Py_INCREF(item);
40 Py_DECREF(seq);
41 return item;
42 }
43 }
44 #endif
45 if (seqlen > NB_STATIC_BUFFERS) {
46 buffers = PyMem_NEW(Py_buffer, seqlen);
47 if (buffers == NULL) {
48 Py_DECREF(seq);
49 PyErr_NoMemory();
50 return NULL;
51 }
52 }
53 else {
54 buffers = static_buffers;
55 }
56
57 /* Here is the general case. Do a pre-pass to figure out the total
58 * amount of space we'll need (sz), and see whether all arguments are
59 * bytes-like.
60 */
61 for (i = 0, nbufs = 0; i < seqlen; i++) {
62 Py_ssize_t itemlen;
63 item = PySequence_Fast_GET_ITEM(seq, i);
64 if (PyBytes_CheckExact(item)) {
65 /* Fast path. */
66 buffers[i].obj = Py_NewRef(item);
67 buffers[i].buf = PyBytes_AS_STRING(item);
68 buffers[i].len = PyBytes_GET_SIZE(item);
69 }
70 else {
71 /* item is only borrowed; its __buffer__() may run Python that
72 drops the sequence's last reference to it. */
73 Py_INCREF(item);
74 if (PyObject_GetBuffer(item, &buffers[i], PyBUF_SIMPLE) != 0) {
75 Py_DECREF(item);
76 PyErr_Format(PyExc_TypeError,
77 "sequence item %zd: expected a bytes-like object, "
78 "%.80s found",
79 i, Py_TYPE(item)->tp_name);
80 goto error;
81 }
82 Py_DECREF(item);
83 /* If the backing objects are mutable, then dropping the GIL
84 * opens up race conditions where another thread tries to modify
85 * the object which we hold a buffer on it. Such code has data
86 * races anyway, but this is a conservative approach that avoids
87 * changing the behaviour of that data race.
88 */
89 drop_gil = 0;
90 }
91 nbufs = i + 1; /* for error cleanup */
92 itemlen = buffers[i].len;
93 if (itemlen > PY_SSIZE_T_MAX - sz) {
94 PyErr_SetString(PyExc_OverflowError,
95 "join() result is too long");
96 goto error;
97 }
98 sz += itemlen;
99 if (i != 0) {
100 if (seplen > PY_SSIZE_T_MAX - sz) {
101 PyErr_SetString(PyExc_OverflowError,
102 "join() result is too long");
103 goto error;
104 }
105 sz += seplen;
106 }
107 if (seqlen != PySequence_Fast_GET_SIZE(seq)) {
108 PyErr_SetString(PyExc_RuntimeError,
109 "sequence changed size during iteration");
110 goto error;
111 }
112 }
113
114 /* Allocate result space. */
115 res = STRINGLIB_NEW(NULL, sz);
116 if (res == NULL)
117 goto error;
118
119 /* Catenate everything. */
120 p = STRINGLIB_STR(res);
121 if (sz < GIL_THRESHOLD) {
122 drop_gil = 0; /* Benefits are likely outweighed by the overheads */
123 }
124 if (drop_gil) {
125 save = PyEval_SaveThread();
126 }
127 if (!seplen) {
128 /* fast path */
129 for (i = 0; i < nbufs; i++) {
130 Py_ssize_t n = buffers[i].len;
131 char *q = buffers[i].buf;
132 memcpy(p, q, n);
133 p += n;
134 }
135 }
136 else {
137 for (i = 0; i < nbufs; i++) {
138 Py_ssize_t n;
139 char *q;
140 if (i) {
141 memcpy(p, sepstr, seplen);
142 p += seplen;
143 }
144 n = buffers[i].len;
145 q = buffers[i].buf;
146 memcpy(p, q, n);
147 p += n;
148 }
149 }
150 if (drop_gil) {
151 PyEval_RestoreThread(save);
152 }
153 goto done;
154
155 error:
156 res = NULL;
157 done:
158 Py_DECREF(seq);
159 for (i = 0; i < nbufs; i++)
160 PyBuffer_Release(&buffers[i]);
161 if (buffers != static_buffers)
162 PyMem_Free(buffers);
163 return res;
164 }
165
166 #undef NB_STATIC_BUFFERS
167 #undef GIL_THRESHOLD