Debian Codegraph @

Tools/clinic/libclinic/converter.py

raw
1 from __future__ import annotations
2 import builtins as bltns
3 import functools
4 from typing import Any, TypeVar, Literal, TYPE_CHECKING, cast
5 from collections.abc import Callable
6
7 import libclinic
8 from libclinic import fail
9 from libclinic import Sentinels, unspecified, unknown, NULL
10 from libclinic.codegen import CRenderData, Include, TemplateDict
11 from libclinic.function import Function, Parameter
12
13
14 CConverterClassT = TypeVar("CConverterClassT", bound=type["CConverter"])
15
16
17 type_checks = {
18 '&PyLong_Type': ('PyLong_Check', 'int'),
19 '&PyTuple_Type': ('PyTuple_Check', 'tuple'),
20 '&PyList_Type': ('PyList_Check', 'list'),
21 '&PySet_Type': ('PySet_Check', 'set'),
22 '&PyFrozenSet_Type': ('PyFrozenSet_Check', 'frozenset'),
23 '&PyDict_Type': ('PyDict_Check', 'dict'),
24 '&PyUnicode_Type': ('PyUnicode_Check', 'str'),
25 '&PyBytes_Type': ('PyBytes_Check', 'bytes'),
26 '&PyByteArray_Type': ('PyByteArray_Check', 'bytearray'),
27 }
28
29
30 def add_c_converter(
31 f: CConverterClassT,
32 name: str | None = None
33 ) -> CConverterClassT:
34 if not name:
35 name = f.__name__
36 if not name.endswith('_converter'):
37 return f
38 name = name.removesuffix('_converter')
39 converters[name] = f
40 return f
41
42
43 def add_default_legacy_c_converter(cls: CConverterClassT) -> CConverterClassT:
44 # automatically add converter for default format unit
45 # (but without stomping on the existing one if it's already
46 # set, in case you subclass)
47 if ((cls.format_unit not in ('O&', '')) and
48 (cls.format_unit not in legacy_converters)):
49 legacy_converters[cls.format_unit] = cls
50 return cls
51
52
53 class CConverterAutoRegister(type):
54 def __init__(
55 cls, name: str, bases: tuple[type[object], ...], classdict: dict[str, Any]
56 ) -> None:
57 converter_cls = cast(type["CConverter"], cls)
58 add_c_converter(converter_cls)
59 add_default_legacy_c_converter(converter_cls)
60
61 class CConverter(metaclass=CConverterAutoRegister):
62 """
63 For the init function, self, name, function, and default
64 must be keyword-or-positional parameters. All other
65 parameters must be keyword-only.
66 """
67
68 # The C name to use for this variable.
69 name: str
70
71 # The Python name to use for this variable.
72 py_name: str
73
74 # The C type to use for this variable.
75 # 'type' should be a Python string specifying the type, e.g. "int".
76 # If this is a pointer type, the type string should end with ' *'.
77 type: str | None = None
78
79 # The Python default value for this parameter, as a Python value.
80 # Or the magic value "unspecified" if there is no default.
81 # Or the magic value "unknown" if this value is a cannot be evaluated
82 # at Argument-Clinic-preprocessing time (but is presumed to be valid
83 # at runtime).
84 default: object = unspecified
85
86 # default must be isinstance() of this type.
87 # (You can also specify a tuple of types.)
88 default_type: bltns.type[object] | tuple[bltns.type[object], ...] = object
89
90 # "default" converted into a C value, as a string.
91 # Or None if there is no default.
92 c_default: str | None = None
93
94 # "default" converted into a Python value, as a string.
95 # Or None if there is no default.
96 py_default: str | None = None
97
98 # The default value used to initialize the C variable when
99 # there is no default.
100 #
101 # Every non-abstract subclass with non-trivial cleanup() should supply
102 # a valid value.
103 c_init_default: str = ''
104
105 # The default value used to initialize the C variable when
106 # there is no default, but not specifying a default may
107 # result in an "uninitialized variable" warning. This can
108 # easily happen when using option groups--although
109 # properly-written code won't actually use the variable,
110 # the variable does get passed in to the _impl. (Ah, if
111 # only dataflow analysis could inline the static function!)
112 #
113 # This value is specified as a string.
114 # Every non-abstract subclass should supply a valid value.
115 c_ignored_default: str = ''
116
117 # If true, wrap with Py_UNUSED.
118 unused = False
119
120 # The C converter *function* to be used, if any.
121 # (If this is not None, format_unit must be 'O&'.)
122 converter: str | None = None
123
124 # Should Argument Clinic add a '&' before the name of
125 # the variable when passing it into the _impl function?
126 impl_by_reference = False
127
128 # Should Argument Clinic add a '&' before the name of
129 # the variable when passing it into PyArg_ParseTuple (AndKeywords)?
130 parse_by_reference = True
131
132 #############################################################
133 #############################################################
134 ## You shouldn't need to read anything below this point to ##
135 ## write your own converter functions. ##
136 #############################################################
137 #############################################################
138
139 # The "format unit" to specify for this variable when
140 # parsing arguments using PyArg_ParseTuple (AndKeywords).
141 # Custom converters should always use the default value of 'O&'.
142 format_unit = 'O&'
143
144 # What encoding do we want for this variable? Only used
145 # by format units starting with 'e'.
146 encoding: str | None = None
147
148 # Should this object be required to be a subclass of a specific type?
149 # If not None, should be a string representing a pointer to a
150 # PyTypeObject (e.g. "&PyUnicode_Type").
151 # Only used by the 'O!' format unit (and the "object" converter).
152 subclass_of: str | None = None
153
154 # See also the 'length_name' property.
155 # Only used by format units ending with '#'.
156 length = False
157
158 # Should we show this parameter in the generated
159 # __text_signature__? This is *almost* always True.
160 # (It's only False for __new__, __init__, and METH_STATIC functions.)
161 show_in_signature = True
162
163 # Overrides the name used in a text signature.
164 # The name used for a "self" parameter must be one of
165 # self, type, or module; however users can set their own.
166 # This lets the self_converter overrule the user-settable
167 # name, *just* for the text signature.
168 # Only set by self_converter.
169 signature_name: str | None = None
170
171 broken_limited_capi: bool = False
172
173 # keep in sync with self_converter.__init__!
174 def __init__(self,
175 # Positional args:
176 name: str,
177 py_name: str,
178 function: Function,
179 default: object = unspecified,
180 *, # Keyword only args:
181 c_default: str | None = None,
182 py_default: str | None = None,
183 annotation: str | Literal[Sentinels.unspecified] = unspecified,
184 unused: bool = False,
185 **kwargs: Any
186 ) -> None:
187 self.name = libclinic.ensure_legal_c_identifier(name)
188 self.py_name = py_name
189 self.unused = unused
190 self._includes: list[Include] = []
191
192 if c_default:
193 self.c_default = c_default
194 if py_default:
195 self.py_default = py_default
196
197 if annotation is not unspecified:
198 fail("The 'annotation' parameter is not currently permitted.")
199
200 # Make sure not to set self.function until after converter_init() has been called.
201 # This prevents you from caching information
202 # about the function in converter_init().
203 # (That breaks if we get cloned.)
204 self.converter_init(**kwargs)
205
206 if default is not unspecified:
207 if self.default_type == ():
208 conv_name = self.__class__.__name__.removesuffix('_converter')
209 fail(f"A '{conv_name}' parameter cannot be marked optional.")
210 if (default is not unknown
211 and not isinstance(default, self.default_type)
212 ):
213 if isinstance(self.default_type, type):
214 types_str = self.default_type.__name__
215 else:
216 names = [cls.__name__ for cls in self.default_type]
217 types_str = ', '.join(names)
218 cls_name = self.__class__.__name__
219 fail(f"{cls_name}: default value {default!r} for field "
220 f"{name!r} is not of type {types_str!r}")
221 self.default = default
222
223 if not self.c_default:
224 if default is unspecified:
225 if self.c_init_default:
226 self.c_default = self.c_init_default
227 elif default is NULL:
228 self.c_default = self.c_ignored_default or self.c_init_default
229 if not self.c_default:
230 cls_name = self.__class__.__name__
231 fail(f"{cls_name}: c_default is required for "
232 f"default value NULL")
233 else:
234 assert default is not unknown
235 self.c_default_init()
236 if not self.c_default:
237 if default is None:
238 self.c_default = self.c_init_default
239 if not self.c_default:
240 cls_name = self.__class__.__name__
241 fail(f"{cls_name}: c_default is required for "
242 f"default value None")
243 elif isinstance(default, str):
244 self.c_default = libclinic.c_str_repr(default)
245 elif isinstance(default, bytes):
246 self.c_default = libclinic.c_bytes_repr(default)
247 elif isinstance(default, (int, float)):
248 self.c_default = repr(default)
249 else:
250 cls_name = self.__class__.__name__
251 fail(f"{cls_name}: c_default is required for "
252 f"default value {default!r}")
253 fail(f"Unsupported default value {default!r}.")
254
255 self.function = function
256
257 # Add a custom __getattr__ method to improve the error message
258 # if somebody tries to access self.function in converter_init().
259 #
260 # mypy will assume arbitrary access is okay for a class with a __getattr__ method,
261 # and that's not what we want,
262 # so put it inside an `if not TYPE_CHECKING` block
263 if not TYPE_CHECKING:
264 def __getattr__(self, attr):
265 if attr == "function":
266 fail(
267 f"{self.__class__.__name__!r} object has no attribute 'function'.\n"
268 f"Note: accessing self.function inside converter_init is disallowed!"
269 )
270 return super().__getattr__(attr)
271 # this branch is just here for coverage reporting
272 else: # pragma: no cover
273 pass
274
275 def converter_init(self) -> None:
276 pass
277
278 def c_default_init(self) -> None:
279 return
280
281 def is_optional(self) -> bool:
282 return (self.default is not unspecified)
283
284 def _render_self(self, parameter: Parameter, data: CRenderData) -> None:
285 self.parameter = parameter
286 name = self.parser_name
287
288 # impl_arguments
289 s = ("&" if self.impl_by_reference else "") + name
290 data.impl_arguments.append(s)
291 if self.length:
292 data.impl_arguments.append(self.length_name)
293
294 # impl_parameters
295 data.impl_parameters.append(self.simple_declaration(by_reference=self.impl_by_reference))
296 if self.length:
297 data.impl_parameters.append(f"Py_ssize_t {self.length_name}")
298
299 def _render_non_self(
300 self,
301 parameter: Parameter,
302 data: CRenderData
303 ) -> None:
304 self.parameter = parameter
305 name = self.name
306
307 # declarations
308 d = self.declaration(in_parser=True)
309 data.declarations.append(d)
310
311 # initializers
312 initializers = self.initialize()
313 if initializers:
314 data.initializers.append('/* initializers for ' + name + ' */\n' + initializers.rstrip())
315
316 # modifications
317 modifications = self.modify()
318 if modifications:
319 data.modifications.append('/* modifications for ' + name + ' */\n' + modifications.rstrip())
320
321 # keywords
322 if parameter.is_vararg():
323 pass
324 elif parameter.is_positional_only():
325 data.keywords.append('')
326 else:
327 data.keywords.append(parameter.name)
328
329 # format_units
330 if self.is_optional() and '|' not in data.format_units:
331 data.format_units.append('|')
332 if parameter.is_keyword_only() and '$' not in data.format_units:
333 data.format_units.append('$')
334 data.format_units.append(self.format_unit)
335
336 # parse_arguments
337 self.parse_argument(data.parse_arguments)
338
339 # post_parsing
340 if post_parsing := self.post_parsing():
341 data.post_parsing.append('/* Post parse cleanup for ' + name + ' */\n' + post_parsing.rstrip() + '\n')
342
343 # cleanup
344 cleanup = self.cleanup()
345 if cleanup:
346 data.cleanup.append('/* Cleanup for ' + name + ' */\n' + cleanup.rstrip() + "\n")
347
348 def render(self, parameter: Parameter, data: CRenderData) -> None:
349 """
350 parameter is a clinic.Parameter instance.
351 data is a CRenderData instance.
352 """
353 self._render_self(parameter, data)
354 self._render_non_self(parameter, data)
355
356 @functools.cached_property
357 def length_name(self) -> str:
358 """Computes the name of the associated "length" variable."""
359 assert self.length is not None
360 return self.name + "_length"
361
362 # Why is this one broken out separately?
363 # For "positional-only" function parsing,
364 # which generates a bunch of PyArg_ParseTuple calls.
365 def parse_argument(self, args: list[str]) -> None:
366 assert not (self.converter and self.encoding)
367 if self.format_unit == 'O&':
368 assert self.converter
369 args.append(self.converter)
370
371 if self.encoding:
372 args.append(libclinic.c_str_repr(self.encoding))
373 elif self.subclass_of:
374 args.append(self.subclass_of)
375
376 s = ("&" if self.parse_by_reference else "") + self.parser_name
377 args.append(s)
378
379 if self.length:
380 args.append(f"&{self.length_name}")
381
382 #
383 # All the functions after here are intended as extension points.
384 #
385
386 def simple_declaration(
387 self,
388 by_reference: bool = False,
389 *,
390 in_parser: bool = False
391 ) -> str:
392 """
393 Computes the basic declaration of the variable.
394 Used in computing the prototype declaration and the
395 variable declaration.
396 """
397 assert isinstance(self.type, str)
398 prototype = [self.type]
399 if by_reference or not self.type.endswith('*'):
400 prototype.append(" ")
401 if by_reference:
402 prototype.append('*')
403 if in_parser:
404 name = self.parser_name
405 else:
406 name = self.name
407 if self.unused:
408 name = f"Py_UNUSED({name})"
409 prototype.append(name)
410 return "".join(prototype)
411
412 def declaration(self, *, in_parser: bool = False) -> str:
413 """
414 The C statement to declare this variable.
415 """
416 declaration = [self.simple_declaration(in_parser=True)]
417 default = self.c_default
418 if not default and self.parameter.group:
419 default = self.c_ignored_default or self.c_init_default
420 if default:
421 declaration.append(" = ")
422 declaration.append(default)
423 declaration.append(";")
424 if self.length:
425 declaration.append('\n')
426 declaration.append(f"Py_ssize_t {self.length_name};")
427 return "".join(declaration)
428
429 def initialize(self) -> str:
430 """
431 The C statements required to set up this variable before parsing.
432 Returns a string containing this code indented at column 0.
433 If no initialization is necessary, returns an empty string.
434 """
435 return ""
436
437 def modify(self) -> str:
438 """
439 The C statements required to modify this variable after parsing.
440 Returns a string containing this code indented at column 0.
441 If no modification is necessary, returns an empty string.
442 """
443 return ""
444
445 def post_parsing(self) -> str:
446 """
447 The C statements required to do some operations after the end of parsing but before cleaning up.
448 Return a string containing this code indented at column 0.
449 If no operation is necessary, return an empty string.
450 """
451 return ""
452
453 def cleanup(self) -> str:
454 """
455 The C statements required to clean up after this variable.
456 Returns a string containing this code indented at column 0.
457 If no cleanup is necessary, returns an empty string.
458 """
459 return ""
460
461 def pre_render(self) -> None:
462 """
463 A second initialization function, like converter_init,
464 called just before rendering.
465 You are permitted to examine self.function here.
466 """
467 pass
468
469 def bad_argument(self, displayname: str, expected: str, *, limited_capi: bool, expected_literal: bool = True) -> str:
470 assert '"' not in expected
471 if limited_capi:
472 if expected_literal:
473 return (f'PyErr_Format(PyExc_TypeError, '
474 f'"{{{{name}}}}() {displayname} must be {expected}, not %T", '
475 f'{{argname}});')
476 else:
477 return (f'PyErr_Format(PyExc_TypeError, '
478 f'"{{{{name}}}}() {displayname} must be %s, not %T", '
479 f'"{expected}", {{argname}});')
480 else:
481 if expected_literal:
482 expected = f'"{expected}"'
483 self.add_include('pycore_modsupport.h', '_PyArg_BadArgument()')
484 return f'_PyArg_BadArgument("{{{{name}}}}", "{displayname}", {expected}, {{argname}});'
485
486 def format_code(self, fmt: str, *,
487 argname: str,
488 bad_argument: str | None = None,
489 bad_argument2: str | None = None,
490 **kwargs: Any) -> str:
491 if '{bad_argument}' in fmt:
492 if not bad_argument:
493 raise TypeError("required 'bad_argument' argument")
494 fmt = fmt.replace('{bad_argument}', bad_argument)
495 if '{bad_argument2}' in fmt:
496 if not bad_argument2:
497 raise TypeError("required 'bad_argument2' argument")
498 fmt = fmt.replace('{bad_argument2}', bad_argument2)
499 return fmt.format(argname=argname, paramname=self.parser_name, **kwargs)
500
501 def use_converter(self) -> None:
502 """Method called when self.converter is used to parse an argument."""
503 pass
504
505 def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None:
506 if self.format_unit == 'O&':
507 self.use_converter()
508 return self.format_code("""
509 if (!{converter}({argname}, &{paramname})) {{{{
510 goto exit;
511 }}}}
512 """,
513 argname=argname,
514 converter=self.converter)
515 if self.format_unit == 'O!':
516 cast = '(%s)' % self.type if self.type != 'PyObject *' else ''
517 if self.subclass_of in type_checks:
518 typecheck, typename = type_checks[self.subclass_of]
519 return self.format_code("""
520 if (!{typecheck}({argname})) {{{{
521 {bad_argument}
522 goto exit;
523 }}}}
524 {paramname} = {cast}{argname};
525 """,
526 argname=argname,
527 bad_argument=self.bad_argument(displayname, typename, limited_capi=limited_capi),
528 typecheck=typecheck, typename=typename, cast=cast)
529 return self.format_code("""
530 if (!PyObject_TypeCheck({argname}, {subclass_of})) {{{{
531 {bad_argument}
532 goto exit;
533 }}}}
534 {paramname} = {cast}{argname};
535 """,
536 argname=argname,
537 bad_argument=self.bad_argument(displayname, '({subclass_of})->tp_name',
538 expected_literal=False, limited_capi=limited_capi),
539 subclass_of=self.subclass_of, cast=cast)
540 if self.format_unit == 'O':
541 cast = '(%s)' % self.type if self.type != 'PyObject *' else ''
542 return self.format_code("""
543 {paramname} = {cast}{argname};
544 """,
545 argname=argname, cast=cast)
546 return None
547
548 def set_template_dict(self, template_dict: TemplateDict) -> None:
549 pass
550
551 @property
552 def parser_name(self) -> str:
553 if self.name in libclinic.CLINIC_PREFIXED_ARGS: # bpo-39741
554 return libclinic.CLINIC_PREFIX + self.name
555 else:
556 return self.name
557
558 def add_include(self, name: str, reason: str,
559 *, condition: str | None = None) -> None:
560 include = Include(name, reason, condition)
561 self._includes.append(include)
562
563 def get_includes(self) -> list[Include]:
564 return self._includes
565
566
567 ConverterType = Callable[..., CConverter]
568 ConverterDict = dict[str, ConverterType]
569
570 # maps strings to callables.
571 # these callables must be of the form:
572 # def foo(name, default, *, ...)
573 # The callable may have any number of keyword-only parameters.
574 # The callable must return a CConverter object.
575 # The callable should not call builtins.print.
576 converters: ConverterDict = {}
577
578 # maps strings to callables.
579 # these callables follow the same rules as those for "converters" above.
580 # note however that they will never be called with keyword-only parameters.
581 legacy_converters: ConverterDict = {}
582
583
584 def add_legacy_c_converter(
585 format_unit: str,
586 **kwargs: Any
587 ) -> Callable[[CConverterClassT], CConverterClassT]:
588 def closure(f: CConverterClassT) -> CConverterClassT:
589 added_f: Callable[..., CConverter]
590 if not kwargs:
591 added_f = f
592 else:
593 added_f = functools.partial(f, **kwargs)
594 if format_unit:
595 legacy_converters[format_unit] = added_f
596 return f
597 return closure