Debian Codegraph @

Include/internal/pycore_unicodeobject.h

raw
1 #ifndef Py_INTERNAL_UNICODEOBJECT_H
2 #define Py_INTERNAL_UNICODEOBJECT_H
3 #ifdef __cplusplus
4 extern "C" {
5 #endif
6
7 #ifndef Py_BUILD_CORE
8 # error "this header requires Py_BUILD_CORE define"
9 #endif
10
11 #include "pycore_fileutils.h" // _Py_error_handler
12 #include "pycore_ucnhash.h" // _PyUnicode_Name_CAPI
13
14 /* --- Characters Type APIs ----------------------------------------------- */
15
16 extern int _PyUnicode_IsXidStart(Py_UCS4 ch);
17 extern int _PyUnicode_IsXidContinue(Py_UCS4 ch);
18 extern int _PyUnicode_ToLowerFull(Py_UCS4 ch, Py_UCS4 *res);
19 extern int _PyUnicode_ToTitleFull(Py_UCS4 ch, Py_UCS4 *res);
20 extern int _PyUnicode_ToUpperFull(Py_UCS4 ch, Py_UCS4 *res);
21 extern int _PyUnicode_ToFoldedFull(Py_UCS4 ch, Py_UCS4 *res);
22 extern int _PyUnicode_IsCaseIgnorable(Py_UCS4 ch);
23 extern int _PyUnicode_IsCased(Py_UCS4 ch);
24
25 /* --- Unicode API -------------------------------------------------------- */
26
27 // Export for '_json' shared extension
28 PyAPI_FUNC(int) _PyUnicode_CheckConsistency(
29 PyObject *op,
30 int check_content);
31
32 PyAPI_FUNC(void) _PyUnicode_ExactDealloc(PyObject *op);
33 extern Py_ssize_t _PyUnicode_InternedSize(void);
34 extern Py_ssize_t _PyUnicode_InternedSize_Immortal(void);
35
36 // Get a copy of a Unicode string.
37 // Export for '_datetime' shared extension.
38 PyAPI_FUNC(PyObject*) _PyUnicode_Copy(
39 PyObject *unicode);
40
41 /* Unsafe version of PyUnicode_Fill(): don't check arguments and so may crash
42 if parameters are invalid (e.g. if length is longer than the string). */
43 extern void _PyUnicode_FastFill(
44 PyObject *unicode,
45 Py_ssize_t start,
46 Py_ssize_t length,
47 Py_UCS4 fill_char
48 );
49
50 /* Unsafe version of PyUnicode_CopyCharacters(): don't check arguments and so
51 may crash if parameters are invalid (e.g. if the output string
52 is too short). */
53 extern void _PyUnicode_FastCopyCharacters(
54 PyObject *to,
55 Py_ssize_t to_start,
56 PyObject *from,
57 Py_ssize_t from_start,
58 Py_ssize_t how_many
59 );
60
61 /* Create a new string from a buffer of ASCII characters.
62 WARNING: Don't check if the string contains any non-ASCII character. */
63 extern PyObject* _PyUnicode_FromASCII(
64 const char *buffer,
65 Py_ssize_t size);
66
67 /* Compute the maximum character of the substring unicode[start:end].
68 Return 127 for an empty string. */
69 extern Py_UCS4 _PyUnicode_FindMaxChar (
70 PyObject *unicode,
71 Py_ssize_t start,
72 Py_ssize_t end);
73
74 /* --- _PyUnicodeWriter API ----------------------------------------------- */
75
76 /* Format the object based on the format_spec, as defined in PEP 3101
77 (Advanced String Formatting). */
78 extern int _PyUnicode_FormatAdvancedWriter(
79 _PyUnicodeWriter *writer,
80 PyObject *obj,
81 PyObject *format_spec,
82 Py_ssize_t start,
83 Py_ssize_t end);
84
85 /* --- UTF-7 Codecs ------------------------------------------------------- */
86
87 extern PyObject* _PyUnicode_EncodeUTF7(
88 PyObject *unicode, /* Unicode object */
89 int base64SetO, /* Encode RFC2152 Set O characters in base64 */
90 int base64WhiteSpace, /* Encode whitespace (sp, ht, nl, cr) in base64 */
91 const char *errors); /* error handling */
92
93 /* --- UTF-8 Codecs ------------------------------------------------------- */
94
95 // Export for '_tkinter' shared extension.
96 PyAPI_FUNC(PyObject*) _PyUnicode_AsUTF8String(
97 PyObject *unicode,
98 const char *errors);
99
100 /* --- UTF-32 Codecs ------------------------------------------------------ */
101
102 // Export for '_tkinter' shared extension
103 PyAPI_FUNC(PyObject*) _PyUnicode_EncodeUTF32(
104 PyObject *object, /* Unicode object */
105 const char *errors, /* error handling */
106 int byteorder); /* byteorder to use 0=BOM+native;-1=LE,1=BE */
107
108 /* --- UTF-16 Codecs ------------------------------------------------------ */
109
110 // Returns a Python string object holding the UTF-16 encoded value of
111 // the Unicode data.
112 //
113 // If byteorder is not 0, output is written according to the following
114 // byte order:
115 //
116 // byteorder == -1: little endian
117 // byteorder == 0: native byte order (writes a BOM mark)
118 // byteorder == 1: big endian
119 //
120 // If byteorder is 0, the output string will always start with the
121 // Unicode BOM mark (U+FEFF). In the other two modes, no BOM mark is
122 // prepended.
123 //
124 // Export for '_tkinter' shared extension
125 PyAPI_FUNC(PyObject*) _PyUnicode_EncodeUTF16(
126 PyObject* unicode, /* Unicode object */
127 const char *errors, /* error handling */
128 int byteorder); /* byteorder to use 0=BOM+native;-1=LE,1=BE */
129
130 /* --- Unicode-Escape Codecs ---------------------------------------------- */
131
132 /* Variant of PyUnicode_DecodeUnicodeEscape that supports partial decoding. */
133 extern PyObject* _PyUnicode_DecodeUnicodeEscapeStateful(
134 const char *string, /* Unicode-Escape encoded string */
135 Py_ssize_t length, /* size of string */
136 const char *errors, /* error handling */
137 Py_ssize_t *consumed); /* bytes consumed */
138
139 // Helper for PyUnicode_DecodeUnicodeEscape that detects invalid escape
140 // chars.
141 // Export for test_peg_generator.
142 PyAPI_FUNC(PyObject*) _PyUnicode_DecodeUnicodeEscapeInternal2(
143 const char *string, /* Unicode-Escape encoded string */
144 Py_ssize_t length, /* size of string */
145 const char *errors, /* error handling */
146 Py_ssize_t *consumed, /* bytes consumed */
147 int *first_invalid_escape_char, /* on return, if not -1, contain the first
148 invalid escaped char (<= 0xff) or invalid
149 octal escape (> 0xff) in string. */
150 const char **first_invalid_escape_ptr); /* on return, if not NULL, may
151 point to the first invalid escaped
152 char in string.
153 May be NULL if errors is not NULL. */
154
155 /* --- Raw-Unicode-Escape Codecs ---------------------------------------------- */
156
157 /* Variant of PyUnicode_DecodeRawUnicodeEscape that supports partial decoding. */
158 extern PyObject* _PyUnicode_DecodeRawUnicodeEscapeStateful(
159 const char *string, /* Unicode-Escape encoded string */
160 Py_ssize_t length, /* size of string */
161 const char *errors, /* error handling */
162 Py_ssize_t *consumed); /* bytes consumed */
163
164 /* --- Latin-1 Codecs ----------------------------------------------------- */
165
166 extern PyObject* _PyUnicode_AsLatin1String(
167 PyObject* unicode,
168 const char* errors);
169
170 /* --- ASCII Codecs ------------------------------------------------------- */
171
172 extern PyObject* _PyUnicode_AsASCIIString(
173 PyObject* unicode,
174 const char* errors);
175
176 /* --- Character Map Codecs ----------------------------------------------- */
177
178 /* Translate an Unicode object by applying a character mapping table to
179 it and return the resulting Unicode object.
180
181 The mapping table must map Unicode ordinal integers to Unicode strings,
182 Unicode ordinal integers or None (causing deletion of the character).
183
184 Mapping tables may be dictionaries or sequences. Unmapped character
185 ordinals (ones which cause a LookupError) are left untouched and
186 are copied as-is.
187 */
188 extern PyObject* _PyUnicode_EncodeCharmap(
189 PyObject *unicode, /* Unicode object */
190 PyObject *mapping, /* encoding mapping */
191 const char *errors); /* error handling */
192
193 /* --- Decimal Encoder ---------------------------------------------------- */
194
195 // Converts a Unicode object holding a decimal value to an ASCII string
196 // for using in int, float and complex parsers.
197 // Transforms code points that have decimal digit property to the
198 // corresponding ASCII digit code points. Transforms spaces to ASCII.
199 // Transforms code points starting from the first non-ASCII code point that
200 // is neither a decimal digit nor a space to the end into '?'.
201 //
202 // Export for '_testinternalcapi' shared extension.
203 PyAPI_FUNC(PyObject*) _PyUnicode_TransformDecimalAndSpaceToASCII(
204 PyObject *unicode); /* Unicode object */
205
206 /* --- Methods & Slots ---------------------------------------------------- */
207
208 PyAPI_FUNC(PyObject*) _PyUnicode_JoinArray(
209 PyObject *separator,
210 PyObject *const *items,
211 Py_ssize_t seqlen
212 );
213
214 /* Test whether a unicode is equal to ASCII identifier. Return 1 if true,
215 0 otherwise. The right argument must be ASCII identifier.
216 Any error occurs inside will be cleared before return. */
217 extern int _PyUnicode_EqualToASCIIId(
218 PyObject *left, /* Left string */
219 _Py_Identifier *right /* Right identifier */
220 );
221
222 // Test whether a unicode is equal to ASCII string. Return 1 if true,
223 // 0 otherwise. The right argument must be ASCII-encoded string.
224 // Any error occurs inside will be cleared before return.
225 // Export for '_ctypes' shared extension
226 PyAPI_FUNC(int) _PyUnicode_EqualToASCIIString(
227 PyObject *left,
228 const char *right /* ASCII-encoded string */
229 );
230
231 /* Externally visible for str.strip(unicode) */
232 extern PyObject* _PyUnicode_XStrip(
233 PyObject *self,
234 int striptype,
235 PyObject *sepobj
236 );
237
238
239 /* Using explicit passed-in values, insert the thousands grouping
240 into the string pointed to by buffer. For the argument descriptions,
241 see Objects/stringlib/localeutil.h */
242 extern Py_ssize_t _PyUnicode_InsertThousandsGrouping(
243 _PyUnicodeWriter *writer,
244 Py_ssize_t n_buffer,
245 PyObject *digits,
246 Py_ssize_t d_pos,
247 Py_ssize_t n_digits,
248 Py_ssize_t min_width,
249 const char *grouping,
250 PyObject *thousands_sep,
251 Py_UCS4 *maxchar,
252 int forward);
253
254 /* Dedent a string.
255 Behaviour is expected to be an exact match of `textwrap.dedent`.
256 Return a new reference on success, NULL with exception set on error.
257 */
258 extern PyObject* _PyUnicode_Dedent(PyObject *unicode);
259
260 /* --- Misc functions ----------------------------------------------------- */
261
262 extern PyObject* _PyUnicode_FormatLong(PyObject *, int, int, int);
263
264 // Fast equality check when the inputs are known to be exact unicode types.
265 // Export for '_pickle' shared extension.
266 PyAPI_FUNC(int) _PyUnicode_Equal(PyObject *, PyObject *);
267
268 extern int _PyUnicode_WideCharString_Converter(PyObject *, void *);
269 extern int _PyUnicode_WideCharString_Opt_Converter(PyObject *, void *);
270
271 // Export for test_peg_generator
272 PyAPI_FUNC(Py_ssize_t) _PyUnicode_ScanIdentifier(PyObject *);
273
274 /* --- Runtime lifecycle -------------------------------------------------- */
275
276 extern void _PyUnicode_InitState(PyInterpreterState *);
277 extern PyStatus _PyUnicode_InitGlobalObjects(PyInterpreterState *);
278 extern PyStatus _PyUnicode_InitTypes(PyInterpreterState *);
279 extern void _PyUnicode_Fini(PyInterpreterState *);
280 extern void _PyUnicode_FiniTypes(PyInterpreterState *);
281
282 extern PyTypeObject _PyUnicodeASCIIIter_Type;
283
284 /* --- Interning ---------------------------------------------------------- */
285
286 // All these are "ref-neutral", like the public PyUnicode_InternInPlace.
287
288 // Explicit interning routines:
289 PyAPI_FUNC(void) _PyUnicode_InternMortal(PyInterpreterState *interp, PyObject **);
290 PyAPI_FUNC(void) _PyUnicode_InternImmortal(PyInterpreterState *interp, PyObject **);
291 // Left here to help backporting:
292 PyAPI_FUNC(void) _PyUnicode_InternInPlace(PyInterpreterState *interp, PyObject **p);
293 // Only for singletons in the _PyRuntime struct:
294 extern void _PyUnicode_InternStatic(PyInterpreterState *interp, PyObject **);
295
296 /* --- Other API ---------------------------------------------------------- */
297
298 extern void _PyUnicode_ClearInterned(PyInterpreterState *interp);
299
300 // Like PyUnicode_AsUTF8(), but check for embedded null characters.
301 // Export for '_sqlite3' shared extension.
302 PyAPI_FUNC(const char *) _PyUnicode_AsUTF8NoNUL(PyObject *);
303
304
305 #ifdef __cplusplus
306 }
307 #endif
308 #endif /* !Py_INTERNAL_UNICODEOBJECT_H */