| 1 |
from __future__ import annotations |
| 2 |
import ast |
| 3 |
import enum |
| 4 |
import inspect |
| 5 |
import pprint |
| 6 |
import re |
| 7 |
import shlex |
| 8 |
import sys |
| 9 |
from collections.abc import Callable |
| 10 |
from types import FunctionType |
| 11 |
from typing import TYPE_CHECKING, Any, NamedTuple |
| 12 |
|
| 13 |
import libclinic |
| 14 |
from libclinic import ( |
| 15 |
ClinicError, VersionTuple, |
| 16 |
fail, warn, unspecified, unknown, NULL) |
| 17 |
from libclinic.function import ( |
| 18 |
Module, Class, Function, Parameter, |
| 19 |
FunctionKind, |
| 20 |
CALLABLE, STATIC_METHOD, CLASS_METHOD, METHOD_INIT, METHOD_NEW, |
| 21 |
GETTER, SETTER) |
| 22 |
from libclinic.converter import ( |
| 23 |
converters, legacy_converters) |
| 24 |
from libclinic.converters import ( |
| 25 |
self_converter, defining_class_converter, |
| 26 |
correct_name_for_self) |
| 27 |
from libclinic.return_converters import ( |
| 28 |
CReturnConverter, return_converters, |
| 29 |
int_return_converter) |
| 30 |
from libclinic.parser import create_parser_namespace |
| 31 |
if TYPE_CHECKING: |
| 32 |
from libclinic.block_parser import Block |
| 33 |
from libclinic.app import Clinic |
| 34 |
|
| 35 |
|
| 36 |
unsupported_special_methods: set[str] = set(""" |
| 37 |
|
| 38 |
__abs__ |
| 39 |
__add__ |
| 40 |
__and__ |
| 41 |
__call__ |
| 42 |
__delitem__ |
| 43 |
__divmod__ |
| 44 |
__eq__ |
| 45 |
__float__ |
| 46 |
__floordiv__ |
| 47 |
__ge__ |
| 48 |
__getattr__ |
| 49 |
__getattribute__ |
| 50 |
__getitem__ |
| 51 |
__gt__ |
| 52 |
__hash__ |
| 53 |
__iadd__ |
| 54 |
__iand__ |
| 55 |
__ifloordiv__ |
| 56 |
__ilshift__ |
| 57 |
__imatmul__ |
| 58 |
__imod__ |
| 59 |
__imul__ |
| 60 |
__index__ |
| 61 |
__int__ |
| 62 |
__invert__ |
| 63 |
__ior__ |
| 64 |
__ipow__ |
| 65 |
__irshift__ |
| 66 |
__isub__ |
| 67 |
__iter__ |
| 68 |
__itruediv__ |
| 69 |
__ixor__ |
| 70 |
__le__ |
| 71 |
__len__ |
| 72 |
__lshift__ |
| 73 |
__lt__ |
| 74 |
__matmul__ |
| 75 |
__mod__ |
| 76 |
__mul__ |
| 77 |
__neg__ |
| 78 |
__next__ |
| 79 |
__or__ |
| 80 |
__pos__ |
| 81 |
__pow__ |
| 82 |
__radd__ |
| 83 |
__rand__ |
| 84 |
__rdivmod__ |
| 85 |
__repr__ |
| 86 |
__rfloordiv__ |
| 87 |
__rlshift__ |
| 88 |
__rmatmul__ |
| 89 |
__rmod__ |
| 90 |
__rmul__ |
| 91 |
__ror__ |
| 92 |
__rpow__ |
| 93 |
__rrshift__ |
| 94 |
__rshift__ |
| 95 |
__rsub__ |
| 96 |
__rtruediv__ |
| 97 |
__rxor__ |
| 98 |
__setattr__ |
| 99 |
__setitem__ |
| 100 |
__str__ |
| 101 |
__sub__ |
| 102 |
__truediv__ |
| 103 |
__xor__ |
| 104 |
|
| 105 |
""".strip().split()) |
| 106 |
|
| 107 |
|
| 108 |
StateKeeper = Callable[[str], None] |
| 109 |
ConverterArgs = dict[str, Any] |
| 110 |
|
| 111 |
|
| 112 |
class ParamState(enum.IntEnum): |
| 113 |
"""Parameter parsing state. |
| 114 |
|
| 115 |
[ [ a, b, ] c, ] d, e, f=3, [ g, h, [ i ] ] <- line |
| 116 |
01 2 3 4 5 6 <- state transitions |
| 117 |
""" |
| 118 |
# Before we've seen anything. |
| 119 |
# Legal transitions: to LEFT_SQUARE_BEFORE or REQUIRED |
| 120 |
START = 0 |
| 121 |
|
| 122 |
# Left square brackets before required params. |
| 123 |
LEFT_SQUARE_BEFORE = 1 |
| 124 |
|
| 125 |
# In a group, before required params. |
| 126 |
GROUP_BEFORE = 2 |
| 127 |
|
| 128 |
# Required params, positional-or-keyword or positional-only (we |
| 129 |
# don't know yet). Renumber left groups! |
| 130 |
REQUIRED = 3 |
| 131 |
|
| 132 |
# Positional-or-keyword or positional-only params that now must have |
| 133 |
# default values. |
| 134 |
OPTIONAL = 4 |
| 135 |
|
| 136 |
# In a group, after required params. |
| 137 |
GROUP_AFTER = 5 |
| 138 |
|
| 139 |
# Right square brackets after required params. |
| 140 |
RIGHT_SQUARE_AFTER = 6 |
| 141 |
|
| 142 |
|
| 143 |
class FunctionNames(NamedTuple): |
| 144 |
full_name: str |
| 145 |
c_basename: str |
| 146 |
|
| 147 |
|
| 148 |
def eval_ast_expr( |
| 149 |
node: ast.expr, |
| 150 |
*, |
| 151 |
filename: str = '-' |
| 152 |
) -> Any: |
| 153 |
""" |
| 154 |
Takes an ast.Expr node. Compiles it into a function object, |
| 155 |
then calls the function object with 0 arguments. |
| 156 |
Returns the result of that function call. |
| 157 |
|
| 158 |
globals represents the globals dict the expression |
| 159 |
should see. (There's no equivalent for "locals" here.) |
| 160 |
""" |
| 161 |
|
| 162 |
if isinstance(node, ast.Expr): |
| 163 |
node = node.value |
| 164 |
|
| 165 |
expr = ast.Expression(node) |
| 166 |
namespace = create_parser_namespace() |
| 167 |
co = compile(expr, filename, 'eval') |
| 168 |
fn = FunctionType(co, namespace) |
| 169 |
return fn() |
| 170 |
|
| 171 |
|
| 172 |
class IndentStack: |
| 173 |
def __init__(self) -> None: |
| 174 |
self.indents: list[int] = [] |
| 175 |
self.margin: str | None = None |
| 176 |
|
| 177 |
def _ensure(self) -> None: |
| 178 |
if not self.indents: |
| 179 |
fail('IndentStack expected indents, but none are defined.') |
| 180 |
|
| 181 |
def measure(self, line: str) -> int: |
| 182 |
""" |
| 183 |
Returns the length of the line's margin. |
| 184 |
""" |
| 185 |
if '\t' in line: |
| 186 |
fail('Tab characters are illegal in the Argument Clinic DSL.') |
| 187 |
stripped = line.lstrip() |
| 188 |
if not len(stripped): |
| 189 |
# we can't tell anything from an empty line |
| 190 |
# so just pretend it's indented like our current indent |
| 191 |
self._ensure() |
| 192 |
return self.indents[-1] |
| 193 |
return len(line) - len(stripped) |
| 194 |
|
| 195 |
def infer(self, line: str) -> int: |
| 196 |
""" |
| 197 |
Infer what is now the current margin based on this line. |
| 198 |
Returns: |
| 199 |
1 if we have indented (or this is the first margin) |
| 200 |
0 if the margin has not changed |
| 201 |
-N if we have dedented N times |
| 202 |
""" |
| 203 |
indent = self.measure(line) |
| 204 |
margin = ' ' * indent |
| 205 |
if not self.indents: |
| 206 |
self.indents.append(indent) |
| 207 |
self.margin = margin |
| 208 |
return 1 |
| 209 |
current = self.indents[-1] |
| 210 |
if indent == current: |
| 211 |
return 0 |
| 212 |
if indent > current: |
| 213 |
self.indents.append(indent) |
| 214 |
self.margin = margin |
| 215 |
return 1 |
| 216 |
# indent < current |
| 217 |
if indent not in self.indents: |
| 218 |
fail("Illegal outdent.") |
| 219 |
outdent_count = 0 |
| 220 |
while indent != current: |
| 221 |
self.indents.pop() |
| 222 |
current = self.indents[-1] |
| 223 |
outdent_count -= 1 |
| 224 |
self.margin = margin |
| 225 |
return outdent_count |
| 226 |
|
| 227 |
@property |
| 228 |
def depth(self) -> int: |
| 229 |
""" |
| 230 |
Returns how many margins are currently defined. |
| 231 |
""" |
| 232 |
return len(self.indents) |
| 233 |
|
| 234 |
def dedent(self, line: str) -> str: |
| 235 |
""" |
| 236 |
Dedents a line by the currently defined margin. |
| 237 |
""" |
| 238 |
assert self.margin is not None, "Cannot call .dedent() before calling .infer()" |
| 239 |
margin = self.margin |
| 240 |
indent = self.indents[-1] |
| 241 |
if not line.startswith(margin): |
| 242 |
fail('Cannot dedent; line does not start with the previous margin.') |
| 243 |
return line[indent:] |
| 244 |
|
| 245 |
|
| 246 |
class DSLParser: |
| 247 |
function: Function | None |
| 248 |
state: StateKeeper |
| 249 |
keyword_only: bool |
| 250 |
positional_only: bool |
| 251 |
deprecated_positional: VersionTuple | None |
| 252 |
deprecated_keyword: VersionTuple | None |
| 253 |
group: int |
| 254 |
parameter_state: ParamState |
| 255 |
indent: IndentStack |
| 256 |
kind: FunctionKind |
| 257 |
coexist: bool |
| 258 |
forced_text_signature: str | None |
| 259 |
parameter_continuation: str |
| 260 |
preserve_output: bool |
| 261 |
critical_section: bool |
| 262 |
target_critical_section: list[str] |
| 263 |
disable_fastcall: bool |
| 264 |
from_version_re = re.compile(r'([*/]) +\[from +(.+)\]') |
| 265 |
|
| 266 |
def __init__(self, clinic: Clinic) -> None: |
| 267 |
self.clinic = clinic |
| 268 |
|
| 269 |
self.directives = {} |
| 270 |
for name in dir(self): |
| 271 |
# functions that start with directive_ are added to directives |
| 272 |
_, s, key = name.partition("directive_") |
| 273 |
if s: |
| 274 |
self.directives[key] = getattr(self, name) |
| 275 |
|
| 276 |
# functions that start with at_ are too, with an @ in front |
| 277 |
_, s, key = name.partition("at_") |
| 278 |
if s: |
| 279 |
self.directives['@' + key] = getattr(self, name) |
| 280 |
|
| 281 |
self.reset() |
| 282 |
|
| 283 |
def reset(self) -> None: |
| 284 |
self.function = None |
| 285 |
self.state = self.state_dsl_start |
| 286 |
self.keyword_only = False |
| 287 |
self.positional_only = False |
| 288 |
self.deprecated_positional = None |
| 289 |
self.deprecated_keyword = None |
| 290 |
self.group = 0 |
| 291 |
self.parameter_state: ParamState = ParamState.START |
| 292 |
self.indent = IndentStack() |
| 293 |
self.kind = CALLABLE |
| 294 |
self.coexist = False |
| 295 |
self.forced_text_signature = None |
| 296 |
self.parameter_continuation = '' |
| 297 |
self.preserve_output = False |
| 298 |
self.critical_section = False |
| 299 |
self.target_critical_section = [] |
| 300 |
self.disable_fastcall = False |
| 301 |
|
| 302 |
def directive_module(self, name: str) -> None: |
| 303 |
fields = name.split('.')[:-1] |
| 304 |
module, cls = self.clinic._module_and_class(fields) |
| 305 |
if cls: |
| 306 |
fail("Can't nest a module inside a class!") |
| 307 |
|
| 308 |
if name in module.modules: |
| 309 |
fail(f"Already defined module {name!r}!") |
| 310 |
|
| 311 |
m = Module(name, module) |
| 312 |
module.modules[name] = m |
| 313 |
self.block.signatures.append(m) |
| 314 |
|
| 315 |
def directive_class( |
| 316 |
self, |
| 317 |
name: str, |
| 318 |
typedef: str, |
| 319 |
type_object: str |
| 320 |
) -> None: |
| 321 |
fields = name.split('.') |
| 322 |
name = fields.pop() |
| 323 |
module, cls = self.clinic._module_and_class(fields) |
| 324 |
|
| 325 |
parent = cls or module |
| 326 |
if name in parent.classes: |
| 327 |
fail(f"Already defined class {name!r}!") |
| 328 |
|
| 329 |
c = Class(name, module, cls, typedef, type_object) |
| 330 |
parent.classes[name] = c |
| 331 |
self.block.signatures.append(c) |
| 332 |
|
| 333 |
def directive_set(self, name: str, value: str) -> None: |
| 334 |
if name not in ("line_prefix", "line_suffix"): |
| 335 |
fail(f"unknown variable {name!r}") |
| 336 |
|
| 337 |
value = value.format_map({ |
| 338 |
'block comment start': '/*', |
| 339 |
'block comment end': '*/', |
| 340 |
}) |
| 341 |
|
| 342 |
self.clinic.__dict__[name] = value |
| 343 |
|
| 344 |
def directive_destination( |
| 345 |
self, |
| 346 |
name: str, |
| 347 |
command: str, |
| 348 |
*args: str |
| 349 |
) -> None: |
| 350 |
match command: |
| 351 |
case "new": |
| 352 |
self.clinic.add_destination(name, *args) |
| 353 |
case "clear": |
| 354 |
self.clinic.get_destination(name).clear() |
| 355 |
case _: |
| 356 |
fail(f"unknown destination command {command!r}") |
| 357 |
|
| 358 |
|
| 359 |
def directive_output( |
| 360 |
self, |
| 361 |
command_or_name: str, |
| 362 |
destination: str = '' |
| 363 |
) -> None: |
| 364 |
fd = self.clinic.destination_buffers |
| 365 |
|
| 366 |
if command_or_name == "preset": |
| 367 |
preset = self.clinic.presets.get(destination) |
| 368 |
if not preset: |
| 369 |
fail(f"Unknown preset {destination!r}!") |
| 370 |
fd.update(preset) |
| 371 |
return |
| 372 |
|
| 373 |
if command_or_name == "push": |
| 374 |
self.clinic.destination_buffers_stack.append(fd.copy()) |
| 375 |
return |
| 376 |
|
| 377 |
if command_or_name == "pop": |
| 378 |
if not self.clinic.destination_buffers_stack: |
| 379 |
fail("Can't 'output pop', stack is empty!") |
| 380 |
previous_fd = self.clinic.destination_buffers_stack.pop() |
| 381 |
fd.update(previous_fd) |
| 382 |
return |
| 383 |
|
| 384 |
# secret command for debugging! |
| 385 |
if command_or_name == "print": |
| 386 |
self.block.output.append(pprint.pformat(fd)) |
| 387 |
self.block.output.append('\n') |
| 388 |
return |
| 389 |
|
| 390 |
d = self.clinic.get_destination_buffer(destination) |
| 391 |
|
| 392 |
if command_or_name == "everything": |
| 393 |
for name in list(fd): |
| 394 |
fd[name] = d |
| 395 |
return |
| 396 |
|
| 397 |
if command_or_name not in fd: |
| 398 |
allowed = ["preset", "push", "pop", "print", "everything"] |
| 399 |
allowed.extend(fd) |
| 400 |
fail(f"Invalid command or destination name {command_or_name!r}. " |
| 401 |
"Must be one of:\n -", |
| 402 |
"\n - ".join([repr(word) for word in allowed])) |
| 403 |
fd[command_or_name] = d |
| 404 |
|
| 405 |
def directive_dump(self, name: str) -> None: |
| 406 |
self.block.output.append(self.clinic.get_destination(name).dump()) |
| 407 |
|
| 408 |
def directive_printout(self, *args: str) -> None: |
| 409 |
self.block.output.append(' '.join(args)) |
| 410 |
self.block.output.append('\n') |
| 411 |
|
| 412 |
def directive_preserve(self) -> None: |
| 413 |
if self.preserve_output: |
| 414 |
fail("Can't have 'preserve' twice in one block!") |
| 415 |
self.preserve_output = True |
| 416 |
|
| 417 |
def at_classmethod(self) -> None: |
| 418 |
if self.kind is not CALLABLE: |
| 419 |
fail("Can't set @classmethod, function is not a normal callable") |
| 420 |
self.kind = CLASS_METHOD |
| 421 |
|
| 422 |
def at_critical_section(self, *args: str) -> None: |
| 423 |
if len(args) > 2: |
| 424 |
fail("Up to 2 critical section variables are supported") |
| 425 |
self.target_critical_section.extend(args) |
| 426 |
self.critical_section = True |
| 427 |
|
| 428 |
def at_disable(self, *args: str) -> None: |
| 429 |
if self.kind is not CALLABLE: |
| 430 |
fail("Can't set @disable, function is not a normal callable") |
| 431 |
if not args: |
| 432 |
fail("@disable expects at least one argument") |
| 433 |
features = list(args) |
| 434 |
if 'fastcall' in features: |
| 435 |
features.remove('fastcall') |
| 436 |
self.disable_fastcall = True |
| 437 |
if features: |
| 438 |
fail("invalid argument for @disable:", features[0]) |
| 439 |
|
| 440 |
def at_getter(self) -> None: |
| 441 |
match self.kind: |
| 442 |
case FunctionKind.GETTER: |
| 443 |
fail("Cannot apply @getter twice to the same function!") |
| 444 |
case FunctionKind.SETTER: |
| 445 |
fail("Cannot apply both @getter and @setter to the same function!") |
| 446 |
case _: |
| 447 |
self.kind = FunctionKind.GETTER |
| 448 |
|
| 449 |
def at_setter(self) -> None: |
| 450 |
match self.kind: |
| 451 |
case FunctionKind.SETTER: |
| 452 |
fail("Cannot apply @setter twice to the same function!") |
| 453 |
case FunctionKind.GETTER: |
| 454 |
fail("Cannot apply both @getter and @setter to the same function!") |
| 455 |
case _: |
| 456 |
self.kind = FunctionKind.SETTER |
| 457 |
|
| 458 |
def at_staticmethod(self) -> None: |
| 459 |
if self.kind is not CALLABLE: |
| 460 |
fail("Can't set @staticmethod, function is not a normal callable") |
| 461 |
self.kind = STATIC_METHOD |
| 462 |
|
| 463 |
def at_coexist(self) -> None: |
| 464 |
if self.coexist: |
| 465 |
fail("Called @coexist twice!") |
| 466 |
self.coexist = True |
| 467 |
|
| 468 |
def at_text_signature(self, text_signature: str) -> None: |
| 469 |
if self.forced_text_signature: |
| 470 |
fail("Called @text_signature twice!") |
| 471 |
self.forced_text_signature = text_signature |
| 472 |
|
| 473 |
def parse(self, block: Block) -> None: |
| 474 |
self.reset() |
| 475 |
self.block = block |
| 476 |
self.saved_output = self.block.output |
| 477 |
block.output = [] |
| 478 |
block_start = self.clinic.block_parser.line_number |
| 479 |
lines = block.input.split('\n') |
| 480 |
for line_number, line in enumerate(lines, self.clinic.block_parser.block_start_line_number): |
| 481 |
if '\t' in line: |
| 482 |
fail(f'Tab characters are illegal in the Clinic DSL: {line!r}', |
| 483 |
line_number=block_start) |
| 484 |
try: |
| 485 |
self.state(line) |
| 486 |
except ClinicError as exc: |
| 487 |
exc.lineno = line_number |
| 488 |
exc.filename = self.clinic.filename |
| 489 |
raise |
| 490 |
|
| 491 |
self.do_post_block_processing_cleanup(line_number) |
| 492 |
block.output.extend(self.clinic.language.render(self.clinic, block.signatures)) |
| 493 |
|
| 494 |
if self.preserve_output: |
| 495 |
if block.output: |
| 496 |
fail("'preserve' only works for blocks that don't produce any output!", |
| 497 |
line_number=line_number) |
| 498 |
block.output = self.saved_output |
| 499 |
|
| 500 |
def in_docstring(self) -> bool: |
| 501 |
"""Return true if we are processing a docstring.""" |
| 502 |
return self.state in { |
| 503 |
self.state_parameter_docstring, |
| 504 |
self.state_function_docstring, |
| 505 |
} |
| 506 |
|
| 507 |
def valid_line(self, line: str) -> bool: |
| 508 |
# ignore comment-only lines |
| 509 |
if line.lstrip().startswith('#'): |
| 510 |
return False |
| 511 |
|
| 512 |
# Ignore empty lines too |
| 513 |
# (but not in docstring sections!) |
| 514 |
if not self.in_docstring() and not line.strip(): |
| 515 |
return False |
| 516 |
|
| 517 |
return True |
| 518 |
|
| 519 |
def next( |
| 520 |
self, |
| 521 |
state: StateKeeper, |
| 522 |
line: str | None = None |
| 523 |
) -> None: |
| 524 |
self.state = state |
| 525 |
if line is not None: |
| 526 |
self.state(line) |
| 527 |
|
| 528 |
def state_dsl_start(self, line: str) -> None: |
| 529 |
if not self.valid_line(line): |
| 530 |
return |
| 531 |
|
| 532 |
# is it a directive? |
| 533 |
fields = shlex.split(line) |
| 534 |
directive_name = fields[0] |
| 535 |
directive = self.directives.get(directive_name, None) |
| 536 |
if directive: |
| 537 |
try: |
| 538 |
directive(*fields[1:]) |
| 539 |
except TypeError as e: |
| 540 |
fail(str(e)) |
| 541 |
return |
| 542 |
|
| 543 |
self.next(self.state_modulename_name, line) |
| 544 |
|
| 545 |
def parse_function_names(self, line: str) -> FunctionNames: |
| 546 |
left, as_, right = line.partition(' as ') |
| 547 |
full_name = left.strip() |
| 548 |
c_basename = right.strip() |
| 549 |
if as_ and not c_basename: |
| 550 |
fail("No C basename provided after 'as' keyword") |
| 551 |
if not c_basename: |
| 552 |
fields = full_name.split(".") |
| 553 |
if fields[-1] == '__new__': |
| 554 |
fields.pop() |
| 555 |
c_basename = "_".join(fields) |
| 556 |
if not libclinic.is_legal_py_identifier(full_name): |
| 557 |
fail(f"Illegal function name: {full_name!r}") |
| 558 |
if not libclinic.is_legal_c_identifier(c_basename): |
| 559 |
fail(f"Illegal C basename: {c_basename!r}") |
| 560 |
names = FunctionNames(full_name=full_name, c_basename=c_basename) |
| 561 |
self.normalize_function_kind(names.full_name) |
| 562 |
return names |
| 563 |
|
| 564 |
def normalize_function_kind(self, fullname: str) -> None: |
| 565 |
# Fetch the method name and possibly class. |
| 566 |
fields = fullname.split('.') |
| 567 |
name = fields.pop() |
| 568 |
_, cls = self.clinic._module_and_class(fields) |
| 569 |
|
| 570 |
# Check special method requirements. |
| 571 |
if name in unsupported_special_methods: |
| 572 |
fail(f"{name!r} is a special method and cannot be converted to Argument Clinic!") |
| 573 |
if name == '__init__' and (self.kind is not CALLABLE or not cls): |
| 574 |
fail(f"{name!r} must be a normal method; got '{self.kind}'!") |
| 575 |
if name == '__new__' and (self.kind is not CLASS_METHOD or not cls): |
| 576 |
fail("'__new__' must be a class method!") |
| 577 |
if self.kind in {GETTER, SETTER} and not cls: |
| 578 |
fail("@getter and @setter must be methods") |
| 579 |
|
| 580 |
# Normalise self.kind. |
| 581 |
if name == '__new__': |
| 582 |
self.kind = METHOD_NEW |
| 583 |
elif name == '__init__': |
| 584 |
self.kind = METHOD_INIT |
| 585 |
|
| 586 |
def resolve_return_converter( |
| 587 |
self, full_name: str, forced_converter: str |
| 588 |
) -> CReturnConverter: |
| 589 |
if forced_converter: |
| 590 |
if self.kind in {GETTER, SETTER}: |
| 591 |
fail(f"@{self.kind.name.lower()} method cannot define a return type") |
| 592 |
if self.kind is METHOD_INIT: |
| 593 |
fail("__init__ methods cannot define a return type") |
| 594 |
ast_input = f"def x() -> {forced_converter}: pass" |
| 595 |
try: |
| 596 |
module_node = ast.parse(ast_input) |
| 597 |
except SyntaxError: |
| 598 |
fail(f"Badly formed annotation for {full_name!r}: {forced_converter!r}") |
| 599 |
function_node = module_node.body[0] |
| 600 |
assert isinstance(function_node, ast.FunctionDef) |
| 601 |
try: |
| 602 |
name, legacy, kwargs = self.parse_converter(function_node.returns) |
| 603 |
if legacy: |
| 604 |
fail(f"Legacy converter {name!r} not allowed as a return converter") |
| 605 |
if name not in return_converters: |
| 606 |
fail(f"No available return converter called {name!r}") |
| 607 |
return return_converters[name](**kwargs) |
| 608 |
except ValueError: |
| 609 |
fail(f"Badly formed annotation for {full_name!r}: {forced_converter!r}") |
| 610 |
|
| 611 |
if self.kind in {METHOD_INIT, SETTER}: |
| 612 |
return int_return_converter() |
| 613 |
return CReturnConverter() |
| 614 |
|
| 615 |
def parse_cloned_function(self, names: FunctionNames, existing: str) -> None: |
| 616 |
full_name, c_basename = names |
| 617 |
fields = [x.strip() for x in existing.split('.')] |
| 618 |
function_name = fields.pop() |
| 619 |
module, cls = self.clinic._module_and_class(fields) |
| 620 |
parent = cls or module |
| 621 |
|
| 622 |
for existing_function in parent.functions: |
| 623 |
if existing_function.name == function_name: |
| 624 |
break |
| 625 |
else: |
| 626 |
print(f"{cls=}, {module=}, {existing=}", file=sys.stderr) |
| 627 |
print(f"{(cls or module).functions=}", file=sys.stderr) |
| 628 |
fail(f"Couldn't find existing function {existing!r}!") |
| 629 |
|
| 630 |
fields = [x.strip() for x in full_name.split('.')] |
| 631 |
function_name = fields.pop() |
| 632 |
module, cls = self.clinic._module_and_class(fields) |
| 633 |
|
| 634 |
overrides: dict[str, Any] = { |
| 635 |
"name": function_name, |
| 636 |
"full_name": full_name, |
| 637 |
"module": module, |
| 638 |
"cls": cls, |
| 639 |
"c_basename": c_basename, |
| 640 |
"docstring": "", |
| 641 |
} |
| 642 |
if not (existing_function.kind is self.kind and |
| 643 |
existing_function.coexist == self.coexist): |
| 644 |
# Allow __new__ or __init__ methods. |
| 645 |
if existing_function.kind.new_or_init: |
| 646 |
overrides["kind"] = self.kind |
| 647 |
# Future enhancement: allow custom return converters |
| 648 |
overrides["return_converter"] = CReturnConverter() |
| 649 |
else: |
| 650 |
fail("'kind' of function and cloned function don't match! " |
| 651 |
"(@classmethod/@staticmethod/@coexist)") |
| 652 |
function = existing_function.copy(**overrides) |
| 653 |
self.function = function |
| 654 |
self.block.signatures.append(function) |
| 655 |
(cls or module).functions.append(function) |
| 656 |
self.next(self.state_function_docstring) |
| 657 |
|
| 658 |
def state_modulename_name(self, line: str) -> None: |
| 659 |
# looking for declaration, which establishes the leftmost column |
| 660 |
# line should be |
| 661 |
# modulename.fnname [as c_basename] [-> return annotation] |
| 662 |
# square brackets denote optional syntax. |
| 663 |
# |
| 664 |
# alternatively: |
| 665 |
# modulename.fnname [as c_basename] = modulename.existing_fn_name |
| 666 |
# clones the parameters and return converter from that |
| 667 |
# function. you can't modify them. you must enter a |
| 668 |
# new docstring. |
| 669 |
# |
| 670 |
# (but we might find a directive first!) |
| 671 |
# |
| 672 |
# this line is permitted to start with whitespace. |
| 673 |
# we'll call this number of spaces F (for "function"). |
| 674 |
|
| 675 |
assert self.valid_line(line) |
| 676 |
self.indent.infer(line) |
| 677 |
|
| 678 |
# are we cloning? |
| 679 |
before, equals, existing = line.rpartition('=') |
| 680 |
if equals: |
| 681 |
existing = existing.strip() |
| 682 |
if libclinic.is_legal_py_identifier(existing): |
| 683 |
if self.forced_text_signature: |
| 684 |
fail("Cannot use @text_signature when cloning a function") |
| 685 |
# we're cloning! |
| 686 |
names = self.parse_function_names(before) |
| 687 |
return self.parse_cloned_function(names, existing) |
| 688 |
|
| 689 |
line, _, returns = line.partition('->') |
| 690 |
returns = returns.strip() |
| 691 |
full_name, c_basename = self.parse_function_names(line) |
| 692 |
return_converter = self.resolve_return_converter(full_name, returns) |
| 693 |
|
| 694 |
fields = [x.strip() for x in full_name.split('.')] |
| 695 |
function_name = fields.pop() |
| 696 |
module, cls = self.clinic._module_and_class(fields) |
| 697 |
|
| 698 |
func = Function( |
| 699 |
name=function_name, |
| 700 |
full_name=full_name, |
| 701 |
module=module, |
| 702 |
cls=cls, |
| 703 |
c_basename=c_basename, |
| 704 |
return_converter=return_converter, |
| 705 |
kind=self.kind, |
| 706 |
coexist=self.coexist, |
| 707 |
critical_section=self.critical_section, |
| 708 |
disable_fastcall=self.disable_fastcall, |
| 709 |
target_critical_section=self.target_critical_section, |
| 710 |
forced_text_signature=self.forced_text_signature |
| 711 |
) |
| 712 |
self.add_function(func) |
| 713 |
|
| 714 |
self.next(self.state_parameters_start) |
| 715 |
|
| 716 |
def add_function(self, func: Function) -> None: |
| 717 |
# Insert a self converter automatically. |
| 718 |
tp, name = correct_name_for_self(func) |
| 719 |
if func.cls and tp == "PyObject *": |
| 720 |
func.self_converter = self_converter(name, name, func, |
| 721 |
type=func.cls.typedef) |
| 722 |
else: |
| 723 |
func.self_converter = self_converter(name, name, func) |
| 724 |
func.parameters[name] = Parameter( |
| 725 |
name, |
| 726 |
inspect.Parameter.POSITIONAL_ONLY, |
| 727 |
function=func, |
| 728 |
converter=func.self_converter |
| 729 |
) |
| 730 |
|
| 731 |
self.block.signatures.append(func) |
| 732 |
self.function = func |
| 733 |
(func.cls or func.module).functions.append(func) |
| 734 |
|
| 735 |
# Now entering the parameters section. The rules, formally stated: |
| 736 |
# |
| 737 |
# * All lines must be indented with spaces only. |
| 738 |
# * The first line must be a parameter declaration. |
| 739 |
# * The first line must be indented. |
| 740 |
# * This first line establishes the indent for parameters. |
| 741 |
# * We'll call this number of spaces P (for "parameter"). |
| 742 |
# * Thenceforth: |
| 743 |
# * Lines indented with P spaces specify a parameter. |
| 744 |
# * Lines indented with > P spaces are docstrings for the previous |
| 745 |
# parameter. |
| 746 |
# * We'll call this number of spaces D (for "docstring"). |
| 747 |
# * All subsequent lines indented with >= D spaces are stored as |
| 748 |
# part of the per-parameter docstring. |
| 749 |
# * All lines will have the first D spaces of the indent stripped |
| 750 |
# before they are stored. |
| 751 |
# * It's illegal to have a line starting with a number of spaces X |
| 752 |
# such that P < X < D. |
| 753 |
# * A line with < P spaces is the first line of the function |
| 754 |
# docstring, which ends processing for parameters and per-parameter |
| 755 |
# docstrings. |
| 756 |
# * The first line of the function docstring must be at the same |
| 757 |
# indent as the function declaration. |
| 758 |
# * It's illegal to have any line in the parameters section starting |
| 759 |
# with X spaces such that F < X < P. (As before, F is the indent |
| 760 |
# of the function declaration.) |
| 761 |
# |
| 762 |
# Also, currently Argument Clinic places the following restrictions on groups: |
| 763 |
# * Each group must contain at least one parameter. |
| 764 |
# * Each group may contain at most one group, which must be the furthest |
| 765 |
# thing in the group from the required parameters. (The nested group |
| 766 |
# must be the first in the group when it's before the required |
| 767 |
# parameters, and the last thing in the group when after the required |
| 768 |
# parameters.) |
| 769 |
# * There may be at most one (top-level) group to the left or right of |
| 770 |
# the required parameters. |
| 771 |
# * You must specify a slash, and it must be after all parameters. |
| 772 |
# (In other words: either all parameters are positional-only, |
| 773 |
# or none are.) |
| 774 |
# |
| 775 |
# Said another way: |
| 776 |
# * Each group must contain at least one parameter. |
| 777 |
# * All left square brackets before the required parameters must be |
| 778 |
# consecutive. (You can't have a left square bracket followed |
| 779 |
# by a parameter, then another left square bracket. You can't |
| 780 |
# have a left square bracket, a parameter, a right square bracket, |
| 781 |
# and then a left square bracket.) |
| 782 |
# * All right square brackets after the required parameters must be |
| 783 |
# consecutive. |
| 784 |
# |
| 785 |
# These rules are enforced with a single state variable: |
| 786 |
# "parameter_state". (Previously the code was a miasma of ifs and |
| 787 |
# separate boolean state variables.) The states are defined in the |
| 788 |
# ParamState class. |
| 789 |
|
| 790 |
def state_parameters_start(self, line: str) -> None: |
| 791 |
if not self.valid_line(line): |
| 792 |
return |
| 793 |
|
| 794 |
# if this line is not indented, we have no parameters |
| 795 |
if not self.indent.infer(line): |
| 796 |
return self.next(self.state_function_docstring, line) |
| 797 |
|
| 798 |
assert self.function is not None |
| 799 |
if self.function.kind in {GETTER, SETTER}: |
| 800 |
getset = self.function.kind.name.lower() |
| 801 |
fail(f"@{getset} methods cannot define parameters") |
| 802 |
|
| 803 |
self.parameter_continuation = '' |
| 804 |
return self.next(self.state_parameter, line) |
| 805 |
|
| 806 |
|
| 807 |
def to_required(self) -> None: |
| 808 |
""" |
| 809 |
Transition to the "required" parameter state. |
| 810 |
""" |
| 811 |
if self.parameter_state is not ParamState.REQUIRED: |
| 812 |
self.parameter_state = ParamState.REQUIRED |
| 813 |
assert self.function is not None |
| 814 |
for p in self.function.parameters.values(): |
| 815 |
p.group = -p.group |
| 816 |
|
| 817 |
def state_parameter(self, line: str) -> None: |
| 818 |
assert isinstance(self.function, Function) |
| 819 |
|
| 820 |
if not self.valid_line(line): |
| 821 |
return |
| 822 |
|
| 823 |
if self.parameter_continuation: |
| 824 |
line = self.parameter_continuation + ' ' + line.lstrip() |
| 825 |
self.parameter_continuation = '' |
| 826 |
|
| 827 |
assert self.indent.depth == 2 |
| 828 |
indent = self.indent.infer(line) |
| 829 |
if indent == -1: |
| 830 |
# we outdented, must be to definition column |
| 831 |
return self.next(self.state_function_docstring, line) |
| 832 |
|
| 833 |
if indent == 1: |
| 834 |
# we indented, must be to new parameter docstring column |
| 835 |
return self.next(self.state_parameter_docstring_start, line) |
| 836 |
|
| 837 |
line = line.rstrip() |
| 838 |
if line.endswith('\\'): |
| 839 |
self.parameter_continuation = line[:-1] |
| 840 |
return |
| 841 |
|
| 842 |
line = line.lstrip() |
| 843 |
version: VersionTuple | None = None |
| 844 |
match = self.from_version_re.fullmatch(line) |
| 845 |
if match: |
| 846 |
line = match[1] |
| 847 |
version = self.parse_version(match[2]) |
| 848 |
|
| 849 |
func = self.function |
| 850 |
match line: |
| 851 |
case '*': |
| 852 |
self.parse_star(func, version) |
| 853 |
case '[': |
| 854 |
self.parse_opening_square_bracket(func) |
| 855 |
case ']': |
| 856 |
self.parse_closing_square_bracket(func) |
| 857 |
case '/': |
| 858 |
self.parse_slash(func, version) |
| 859 |
case param: |
| 860 |
self.parse_parameter(param) |
| 861 |
|
| 862 |
def parse_parameter(self, line: str) -> None: |
| 863 |
assert self.function is not None |
| 864 |
|
| 865 |
match self.parameter_state: |
| 866 |
case ParamState.START | ParamState.REQUIRED: |
| 867 |
self.to_required() |
| 868 |
case ParamState.LEFT_SQUARE_BEFORE: |
| 869 |
self.parameter_state = ParamState.GROUP_BEFORE |
| 870 |
case ParamState.GROUP_BEFORE: |
| 871 |
if not self.group: |
| 872 |
self.to_required() |
| 873 |
case ParamState.GROUP_AFTER | ParamState.OPTIONAL: |
| 874 |
pass |
| 875 |
case st: |
| 876 |
fail(f"Function {self.function.name} has an unsupported group configuration. (Unexpected state {st}.a)") |
| 877 |
|
| 878 |
# handle "as" for parameters too |
| 879 |
c_name = None |
| 880 |
m = re.match(r'(?:\* *)?\w+( +as +(\w+))', line) |
| 881 |
if m: |
| 882 |
c_name = m[2] |
| 883 |
line = line[:m.start(1)] + line[m.end(1):] |
| 884 |
|
| 885 |
try: |
| 886 |
ast_input = f"def x({line}\n): pass" |
| 887 |
module = ast.parse(ast_input) |
| 888 |
except SyntaxError: |
| 889 |
fail(f"Function {self.function.name!r} has an invalid parameter declaration: {line!r}") |
| 890 |
|
| 891 |
function = module.body[0] |
| 892 |
assert isinstance(function, ast.FunctionDef) |
| 893 |
function_args = function.args |
| 894 |
|
| 895 |
if len(function_args.args) > 1: |
| 896 |
fail(f"Function {self.function.name!r} has an " |
| 897 |
f"invalid parameter declaration (comma?): {line!r}") |
| 898 |
if function_args.kwarg: |
| 899 |
fail(f"Function {self.function.name!r} has an " |
| 900 |
f"invalid parameter declaration (**kwargs?): {line!r}") |
| 901 |
|
| 902 |
if function_args.vararg: |
| 903 |
self.check_previous_star() |
| 904 |
self.check_remaining_star() |
| 905 |
is_vararg = True |
| 906 |
parameter = function_args.vararg |
| 907 |
else: |
| 908 |
is_vararg = False |
| 909 |
parameter = function_args.args[0] |
| 910 |
|
| 911 |
parameter_name = parameter.arg |
| 912 |
name, legacy, kwargs = self.parse_converter(parameter.annotation) |
| 913 |
if is_vararg: |
| 914 |
name = 'varpos_' + name |
| 915 |
|
| 916 |
value: object |
| 917 |
has_c_default = 'c_default' in kwargs |
| 918 |
if not function_args.defaults: |
| 919 |
value = unspecified |
| 920 |
if (not is_vararg |
| 921 |
and self.parameter_state is ParamState.OPTIONAL): |
| 922 |
fail(f"Can't have a parameter without a default ({parameter_name!r}) " |
| 923 |
"after a parameter with a default!") |
| 924 |
if 'py_default' in kwargs: |
| 925 |
fail("You can't specify py_default without specifying a default value!") |
| 926 |
if has_c_default: |
| 927 |
fail("You can't specify c_default without specifying a default value!") |
| 928 |
else: |
| 929 |
expr = function_args.defaults[0] |
| 930 |
default = ast_input[expr.col_offset: expr.end_col_offset].strip() |
| 931 |
|
| 932 |
if self.parameter_state is ParamState.REQUIRED: |
| 933 |
self.parameter_state = ParamState.OPTIONAL |
| 934 |
bad = False |
| 935 |
try: |
| 936 |
if not has_c_default: |
| 937 |
# we can only represent very simple data values in C. |
| 938 |
# detect whether default is okay, via a denylist |
| 939 |
# of disallowed ast nodes. |
| 940 |
class DetectBadNodes(ast.NodeVisitor): |
| 941 |
bad = False |
| 942 |
def bad_node(self, node: ast.AST) -> None: |
| 943 |
self.bad = True |
| 944 |
|
| 945 |
# inline function call |
| 946 |
visit_Call = bad_node |
| 947 |
# inline if statement ("x = 3 if y else z") |
| 948 |
visit_IfExp = bad_node |
| 949 |
|
| 950 |
# comprehensions and generator expressions |
| 951 |
visit_ListComp = visit_SetComp = bad_node |
| 952 |
visit_DictComp = visit_GeneratorExp = bad_node |
| 953 |
|
| 954 |
# literals for advanced types |
| 955 |
visit_Dict = visit_Set = bad_node |
| 956 |
visit_List = visit_Tuple = bad_node |
| 957 |
|
| 958 |
# "starred": "a = [1, 2, 3]; *a" |
| 959 |
visit_Starred = bad_node |
| 960 |
|
| 961 |
denylist = DetectBadNodes() |
| 962 |
denylist.visit(expr) |
| 963 |
bad = denylist.bad |
| 964 |
else: |
| 965 |
# if they specify a c_default, we can be more lenient about the default value. |
| 966 |
# but at least make an attempt at ensuring it's a valid expression. |
| 967 |
code = compile(ast.Expression(expr), '<expr>', 'eval') |
| 968 |
try: |
| 969 |
value = eval(code) |
| 970 |
except NameError: |
| 971 |
pass # probably a named constant |
| 972 |
except Exception as e: |
| 973 |
fail("Malformed expression given as default value " |
| 974 |
f"{default!r} caused {e!r}") |
| 975 |
else: |
| 976 |
if value is unspecified: |
| 977 |
fail("'unspecified' is not a legal default value!") |
| 978 |
if bad: |
| 979 |
fail(f"Unsupported expression as default value: {default!r}") |
| 980 |
|
| 981 |
# mild hack: explicitly support NULL as a default value |
| 982 |
if isinstance(expr, ast.Name) and expr.id == 'NULL': |
| 983 |
value = NULL |
| 984 |
py_default = '<unrepresentable>' |
| 985 |
elif (isinstance(expr, ast.BinOp) or |
| 986 |
(isinstance(expr, ast.UnaryOp) and |
| 987 |
not (isinstance(expr.operand, ast.Constant) and |
| 988 |
type(expr.operand.value) in {int, float, complex}) |
| 989 |
)): |
| 990 |
if not has_c_default: |
| 991 |
fail(f"When you specify an expression ({default!r}) " |
| 992 |
f"as your default value, " |
| 993 |
f"you MUST specify a valid c_default.", |
| 994 |
ast.dump(expr)) |
| 995 |
py_default = default |
| 996 |
value = unknown |
| 997 |
elif isinstance(expr, ast.Attribute): |
| 998 |
a = [] |
| 999 |
n: ast.expr | ast.Attribute = expr |
| 1000 |
while isinstance(n, ast.Attribute): |
| 1001 |
a.append(n.attr) |
| 1002 |
n = n.value |
| 1003 |
if not isinstance(n, ast.Name): |
| 1004 |
fail(f"Unsupported default value {default!r} " |
| 1005 |
"(looked like a Python constant)") |
| 1006 |
a.append(n.id) |
| 1007 |
py_default = ".".join(reversed(a)) |
| 1008 |
|
| 1009 |
if not has_c_default: |
| 1010 |
fail(f"When you specify a named constant ({py_default!r}) " |
| 1011 |
"as your default value, " |
| 1012 |
"you MUST specify a valid c_default.") |
| 1013 |
|
| 1014 |
try: |
| 1015 |
value = eval(py_default) |
| 1016 |
except NameError: |
| 1017 |
value = unknown |
| 1018 |
else: |
| 1019 |
value = ast.literal_eval(expr) |
| 1020 |
py_default = repr(value) |
| 1021 |
|
| 1022 |
except (ValueError, AttributeError): |
| 1023 |
value = unknown |
| 1024 |
py_default = default |
| 1025 |
if not has_c_default: |
| 1026 |
fail("When you specify a named constant " |
| 1027 |
f"({py_default!r}) as your default value, " |
| 1028 |
"you MUST specify a valid c_default.") |
| 1029 |
|
| 1030 |
kwargs.setdefault('py_default', py_default) |
| 1031 |
|
| 1032 |
dict = legacy_converters if legacy else converters |
| 1033 |
legacy_str = "legacy " if legacy else "" |
| 1034 |
if name not in dict: |
| 1035 |
fail(f'{name!r} is not a valid {legacy_str}converter') |
| 1036 |
# if you use a c_name for the parameter, we just give that name to the converter |
| 1037 |
# but the parameter object gets the python name |
| 1038 |
converter = dict[name](c_name or parameter_name, parameter_name, self.function, value, **kwargs) |
| 1039 |
|
| 1040 |
kind: inspect._ParameterKind |
| 1041 |
if is_vararg: |
| 1042 |
kind = inspect.Parameter.VAR_POSITIONAL |
| 1043 |
elif self.keyword_only: |
| 1044 |
kind = inspect.Parameter.KEYWORD_ONLY |
| 1045 |
else: |
| 1046 |
kind = inspect.Parameter.POSITIONAL_OR_KEYWORD |
| 1047 |
|
| 1048 |
if isinstance(converter, self_converter): |
| 1049 |
if len(self.function.parameters) == 1: |
| 1050 |
if self.group: |
| 1051 |
fail("A 'self' parameter cannot be in an optional group.") |
| 1052 |
assert self.parameter_state is ParamState.REQUIRED |
| 1053 |
assert value is unspecified |
| 1054 |
kind = inspect.Parameter.POSITIONAL_ONLY |
| 1055 |
self.parameter_state = ParamState.START |
| 1056 |
self.function.parameters.clear() |
| 1057 |
else: |
| 1058 |
fail("A 'self' parameter, if specified, must be the " |
| 1059 |
"very first thing in the parameter block.") |
| 1060 |
|
| 1061 |
if isinstance(converter, defining_class_converter): |
| 1062 |
_lp = len(self.function.parameters) |
| 1063 |
if _lp == 1: |
| 1064 |
if self.group: |
| 1065 |
fail("A 'defining_class' parameter cannot be in an optional group.") |
| 1066 |
if self.function.cls is None: |
| 1067 |
fail("A 'defining_class' parameter cannot be defined at module level.") |
| 1068 |
assert self.parameter_state is ParamState.REQUIRED |
| 1069 |
assert value is unspecified |
| 1070 |
kind = inspect.Parameter.POSITIONAL_ONLY |
| 1071 |
else: |
| 1072 |
fail("A 'defining_class' parameter, if specified, must either " |
| 1073 |
"be the first thing in the parameter block, or come just " |
| 1074 |
"after 'self'.") |
| 1075 |
|
| 1076 |
|
| 1077 |
p = Parameter(parameter_name, kind, function=self.function, |
| 1078 |
converter=converter, default=value, group=self.group, |
| 1079 |
deprecated_positional=self.deprecated_positional) |
| 1080 |
|
| 1081 |
names = [k.name for k in self.function.parameters.values()] |
| 1082 |
if parameter_name in names[1:]: |
| 1083 |
fail(f"You can't have two parameters named {parameter_name!r}!") |
| 1084 |
elif names and parameter_name == names[0] and c_name is None: |
| 1085 |
fail(f"Parameter {parameter_name!r} requires a custom C name") |
| 1086 |
|
| 1087 |
key = f"{parameter_name}_as_{c_name}" if c_name else parameter_name |
| 1088 |
self.function.parameters[key] = p |
| 1089 |
|
| 1090 |
if is_vararg: |
| 1091 |
self.keyword_only = True |
| 1092 |
|
| 1093 |
@staticmethod |
| 1094 |
def parse_converter( |
| 1095 |
annotation: ast.expr | None |
| 1096 |
) -> tuple[str, bool, ConverterArgs]: |
| 1097 |
match annotation: |
| 1098 |
case ast.Constant(value=str() as value): |
| 1099 |
return value, True, {} |
| 1100 |
case ast.Name(name): |
| 1101 |
return name, False, {} |
| 1102 |
case ast.Call(func=ast.Name(name)): |
| 1103 |
kwargs: ConverterArgs = {} |
| 1104 |
for node in annotation.keywords: |
| 1105 |
if not isinstance(node.arg, str): |
| 1106 |
fail("Cannot use a kwarg splat in a function-call annotation") |
| 1107 |
kwargs[node.arg] = eval_ast_expr(node.value) |
| 1108 |
return name, False, kwargs |
| 1109 |
case _: |
| 1110 |
fail( |
| 1111 |
"Annotations must be either a name, a function call, or a string." |
| 1112 |
) |
| 1113 |
|
| 1114 |
def parse_version(self, thenceforth: str) -> VersionTuple: |
| 1115 |
"""Parse Python version in `[from ...]` marker.""" |
| 1116 |
assert isinstance(self.function, Function) |
| 1117 |
|
| 1118 |
try: |
| 1119 |
major, minor = thenceforth.split(".") |
| 1120 |
return int(major), int(minor) |
| 1121 |
except ValueError: |
| 1122 |
fail( |
| 1123 |
f"Function {self.function.name!r}: expected format '[from major.minor]' " |
| 1124 |
f"where 'major' and 'minor' are integers; got {thenceforth!r}" |
| 1125 |
) |
| 1126 |
|
| 1127 |
def parse_star(self, function: Function, version: VersionTuple | None) -> None: |
| 1128 |
"""Parse keyword-only parameter marker '*'. |
| 1129 |
|
| 1130 |
The 'version' parameter signifies the future version from which |
| 1131 |
the marker will take effect (None means it is already in effect). |
| 1132 |
""" |
| 1133 |
if version is None: |
| 1134 |
self.check_previous_star() |
| 1135 |
self.check_remaining_star() |
| 1136 |
self.keyword_only = True |
| 1137 |
else: |
| 1138 |
if self.keyword_only: |
| 1139 |
fail(f"Function {function.name!r}: '* [from ...]' must precede '*'") |
| 1140 |
if self.deprecated_positional: |
| 1141 |
if self.deprecated_positional == version: |
| 1142 |
fail(f"Function {function.name!r} uses '* [from " |
| 1143 |
f"{version[0]}.{version[1]}]' more than once.") |
| 1144 |
if self.deprecated_positional < version: |
| 1145 |
fail(f"Function {function.name!r}: '* [from " |
| 1146 |
f"{version[0]}.{version[1]}]' must precede '* [from " |
| 1147 |
f"{self.deprecated_positional[0]}.{self.deprecated_positional[1]}]'") |
| 1148 |
self.deprecated_positional = version |
| 1149 |
|
| 1150 |
def parse_opening_square_bracket(self, function: Function) -> None: |
| 1151 |
"""Parse opening parameter group symbol '['.""" |
| 1152 |
match self.parameter_state: |
| 1153 |
case ParamState.START | ParamState.LEFT_SQUARE_BEFORE: |
| 1154 |
self.parameter_state = ParamState.LEFT_SQUARE_BEFORE |
| 1155 |
case ParamState.REQUIRED | ParamState.GROUP_AFTER: |
| 1156 |
self.parameter_state = ParamState.GROUP_AFTER |
| 1157 |
case st: |
| 1158 |
fail(f"Function {function.name!r} " |
| 1159 |
f"has an unsupported group configuration. " |
| 1160 |
f"(Unexpected state {st}.b)") |
| 1161 |
self.group += 1 |
| 1162 |
function.docstring_only = True |
| 1163 |
|
| 1164 |
def parse_closing_square_bracket(self, function: Function) -> None: |
| 1165 |
"""Parse closing parameter group symbol ']'.""" |
| 1166 |
if not self.group: |
| 1167 |
fail(f"Function {function.name!r} has a ']' without a matching '['.") |
| 1168 |
if not any(p.group == self.group for p in function.parameters.values()): |
| 1169 |
fail(f"Function {function.name!r} has an empty group. " |
| 1170 |
"All groups must contain at least one parameter.") |
| 1171 |
self.group -= 1 |
| 1172 |
match self.parameter_state: |
| 1173 |
case ParamState.LEFT_SQUARE_BEFORE | ParamState.GROUP_BEFORE: |
| 1174 |
self.parameter_state = ParamState.GROUP_BEFORE |
| 1175 |
case ParamState.GROUP_AFTER | ParamState.RIGHT_SQUARE_AFTER: |
| 1176 |
self.parameter_state = ParamState.RIGHT_SQUARE_AFTER |
| 1177 |
case st: |
| 1178 |
fail(f"Function {function.name!r} " |
| 1179 |
f"has an unsupported group configuration. " |
| 1180 |
f"(Unexpected state {st}.c)") |
| 1181 |
|
| 1182 |
def parse_slash(self, function: Function, version: VersionTuple | None) -> None: |
| 1183 |
"""Parse positional-only parameter marker '/'. |
| 1184 |
|
| 1185 |
The 'version' parameter signifies the future version from which |
| 1186 |
the marker will take effect (None means it is already in effect). |
| 1187 |
""" |
| 1188 |
if version is None: |
| 1189 |
if self.deprecated_keyword: |
| 1190 |
fail(f"Function {function.name!r}: '/' must precede '/ [from ...]'") |
| 1191 |
if self.deprecated_positional: |
| 1192 |
fail(f"Function {function.name!r}: '/' must precede '* [from ...]'") |
| 1193 |
if self.keyword_only: |
| 1194 |
fail(f"Function {function.name!r}: '/' must precede '*'") |
| 1195 |
if self.positional_only: |
| 1196 |
fail(f"Function {function.name!r} uses '/' more than once.") |
| 1197 |
else: |
| 1198 |
if self.deprecated_keyword: |
| 1199 |
if self.deprecated_keyword == version: |
| 1200 |
fail(f"Function {function.name!r} uses '/ [from " |
| 1201 |
f"{version[0]}.{version[1]}]' more than once.") |
| 1202 |
if self.deprecated_keyword > version: |
| 1203 |
fail(f"Function {function.name!r}: '/ [from " |
| 1204 |
f"{version[0]}.{version[1]}]' must precede '/ [from " |
| 1205 |
f"{self.deprecated_keyword[0]}.{self.deprecated_keyword[1]}]'") |
| 1206 |
if self.deprecated_positional: |
| 1207 |
fail(f"Function {function.name!r}: '/ [from ...]' must precede '* [from ...]'") |
| 1208 |
if self.keyword_only: |
| 1209 |
fail(f"Function {function.name!r}: '/ [from ...]' must precede '*'") |
| 1210 |
self.positional_only = True |
| 1211 |
self.deprecated_keyword = version |
| 1212 |
if version is not None: |
| 1213 |
found = False |
| 1214 |
for p in reversed(function.parameters.values()): |
| 1215 |
found = p.kind is inspect.Parameter.POSITIONAL_OR_KEYWORD |
| 1216 |
break |
| 1217 |
if not found: |
| 1218 |
fail(f"Function {function.name!r} specifies '/ [from ...]' " |
| 1219 |
f"without preceding parameters.") |
| 1220 |
# REQUIRED and OPTIONAL are allowed here, that allows positional-only |
| 1221 |
# without option groups to work (and have default values!) |
| 1222 |
allowed = { |
| 1223 |
ParamState.REQUIRED, |
| 1224 |
ParamState.OPTIONAL, |
| 1225 |
ParamState.RIGHT_SQUARE_AFTER, |
| 1226 |
ParamState.GROUP_BEFORE, |
| 1227 |
} |
| 1228 |
if (self.parameter_state not in allowed) or self.group: |
| 1229 |
fail(f"Function {function.name!r} has an unsupported group configuration. " |
| 1230 |
f"(Unexpected state {self.parameter_state}.d)") |
| 1231 |
# fixup preceding parameters |
| 1232 |
for p in function.parameters.values(): |
| 1233 |
if p.kind is inspect.Parameter.POSITIONAL_OR_KEYWORD: |
| 1234 |
if version is None: |
| 1235 |
p.kind = inspect.Parameter.POSITIONAL_ONLY |
| 1236 |
elif p.deprecated_keyword is None: |
| 1237 |
p.deprecated_keyword = version |
| 1238 |
|
| 1239 |
def state_parameter_docstring_start(self, line: str) -> None: |
| 1240 |
assert self.indent.margin is not None, "self.margin.infer() has not yet been called to set the margin" |
| 1241 |
self.parameter_docstring_indent = len(self.indent.margin) |
| 1242 |
assert self.indent.depth == 3 |
| 1243 |
return self.next(self.state_parameter_docstring, line) |
| 1244 |
|
| 1245 |
def docstring_append(self, obj: Function | Parameter, line: str) -> None: |
| 1246 |
"""Add a rstripped line to the current docstring.""" |
| 1247 |
# gh-80282: We filter out non-ASCII characters from the docstring, |
| 1248 |
# since historically, some compilers may balk on non-ASCII input. |
| 1249 |
# If you're using Argument Clinic in an external project, |
| 1250 |
# you may not need to support the same array of platforms as CPython, |
| 1251 |
# so you may be able to remove this restriction. |
| 1252 |
matches = re.finditer(r'[^\x00-\x7F]', line) |
| 1253 |
if offending := ", ".join([repr(m[0]) for m in matches]): |
| 1254 |
warn("Non-ascii characters are not allowed in docstrings:", |
| 1255 |
offending) |
| 1256 |
|
| 1257 |
docstring = obj.docstring |
| 1258 |
if docstring: |
| 1259 |
docstring += "\n" |
| 1260 |
if stripped := line.rstrip(): |
| 1261 |
docstring += self.indent.dedent(stripped) |
| 1262 |
obj.docstring = docstring |
| 1263 |
|
| 1264 |
# every line of the docstring must start with at least F spaces, |
| 1265 |
# where F > P. |
| 1266 |
# these F spaces will be stripped. |
| 1267 |
def state_parameter_docstring(self, line: str) -> None: |
| 1268 |
if not self.valid_line(line): |
| 1269 |
return |
| 1270 |
|
| 1271 |
indent = self.indent.measure(line) |
| 1272 |
if indent < self.parameter_docstring_indent: |
| 1273 |
self.indent.infer(line) |
| 1274 |
assert self.indent.depth < 3 |
| 1275 |
if self.indent.depth == 2: |
| 1276 |
# back to a parameter |
| 1277 |
return self.next(self.state_parameter, line) |
| 1278 |
assert self.indent.depth == 1 |
| 1279 |
return self.next(self.state_function_docstring, line) |
| 1280 |
|
| 1281 |
assert self.function and self.function.parameters |
| 1282 |
last_param = next(reversed(self.function.parameters.values())) |
| 1283 |
self.docstring_append(last_param, line) |
| 1284 |
|
| 1285 |
# the final stanza of the DSL is the docstring. |
| 1286 |
def state_function_docstring(self, line: str) -> None: |
| 1287 |
assert self.function is not None |
| 1288 |
|
| 1289 |
if self.group: |
| 1290 |
fail(f"Function {self.function.name!r} has a ']' without a matching '['.") |
| 1291 |
|
| 1292 |
if not self.valid_line(line): |
| 1293 |
return |
| 1294 |
|
| 1295 |
self.docstring_append(self.function, line) |
| 1296 |
|
| 1297 |
@staticmethod |
| 1298 |
def format_docstring_signature( |
| 1299 |
f: Function, parameters: list[Parameter] |
| 1300 |
) -> str: |
| 1301 |
lines = [] |
| 1302 |
lines.append(f.displayname) |
| 1303 |
if f.forced_text_signature: |
| 1304 |
lines.append(f.forced_text_signature) |
| 1305 |
elif f.kind in {GETTER, SETTER}: |
| 1306 |
# @getter and @setter do not need signatures like a method or a function. |
| 1307 |
return '' |
| 1308 |
else: |
| 1309 |
lines.append('(') |
| 1310 |
|
| 1311 |
# populate "right_bracket_count" field for every parameter |
| 1312 |
assert parameters, "We should always have a self parameter. " + repr(f) |
| 1313 |
assert isinstance(parameters[0].converter, self_converter) |
| 1314 |
# self is always positional-only. |
| 1315 |
assert parameters[0].is_positional_only() |
| 1316 |
assert parameters[0].right_bracket_count == 0 |
| 1317 |
positional_only = True |
| 1318 |
for p in parameters[1:]: |
| 1319 |
if not p.is_positional_only(): |
| 1320 |
positional_only = False |
| 1321 |
else: |
| 1322 |
assert positional_only |
| 1323 |
if positional_only: |
| 1324 |
p.right_bracket_count = abs(p.group) |
| 1325 |
else: |
| 1326 |
# don't put any right brackets around non-positional-only parameters, ever. |
| 1327 |
p.right_bracket_count = 0 |
| 1328 |
|
| 1329 |
right_bracket_count = 0 |
| 1330 |
|
| 1331 |
def fix_right_bracket_count(desired: int) -> str: |
| 1332 |
nonlocal right_bracket_count |
| 1333 |
s = '' |
| 1334 |
while right_bracket_count < desired: |
| 1335 |
s += '[' |
| 1336 |
right_bracket_count += 1 |
| 1337 |
while right_bracket_count > desired: |
| 1338 |
s += ']' |
| 1339 |
right_bracket_count -= 1 |
| 1340 |
return s |
| 1341 |
|
| 1342 |
need_slash = False |
| 1343 |
added_slash = False |
| 1344 |
need_a_trailing_slash = False |
| 1345 |
|
| 1346 |
# we only need a trailing slash: |
| 1347 |
# * if this is not a "docstring_only" signature |
| 1348 |
# * and if the last *shown* parameter is |
| 1349 |
# positional only |
| 1350 |
if not f.docstring_only: |
| 1351 |
for p in reversed(parameters): |
| 1352 |
if not p.converter.show_in_signature: |
| 1353 |
continue |
| 1354 |
if p.is_positional_only(): |
| 1355 |
need_a_trailing_slash = True |
| 1356 |
break |
| 1357 |
|
| 1358 |
|
| 1359 |
added_star = False |
| 1360 |
|
| 1361 |
first_parameter = True |
| 1362 |
last_p = parameters[-1] |
| 1363 |
line_length = len(''.join(lines)) |
| 1364 |
indent = " " * line_length |
| 1365 |
def add_parameter(text: str) -> None: |
| 1366 |
nonlocal line_length |
| 1367 |
nonlocal first_parameter |
| 1368 |
if first_parameter: |
| 1369 |
s = text |
| 1370 |
first_parameter = False |
| 1371 |
else: |
| 1372 |
s = ' ' + text |
| 1373 |
if line_length + len(s) >= 72: |
| 1374 |
lines.extend(["\n", indent]) |
| 1375 |
line_length = len(indent) |
| 1376 |
s = text |
| 1377 |
line_length += len(s) |
| 1378 |
lines.append(s) |
| 1379 |
|
| 1380 |
for p in parameters: |
| 1381 |
if not p.converter.show_in_signature: |
| 1382 |
continue |
| 1383 |
assert p.name |
| 1384 |
|
| 1385 |
is_self = isinstance(p.converter, self_converter) |
| 1386 |
if is_self and f.docstring_only: |
| 1387 |
# this isn't a real machine-parsable signature, |
| 1388 |
# so let's not print the "self" parameter |
| 1389 |
continue |
| 1390 |
|
| 1391 |
if p.is_positional_only(): |
| 1392 |
need_slash = not f.docstring_only |
| 1393 |
elif need_slash and not (added_slash or p.is_positional_only()): |
| 1394 |
added_slash = True |
| 1395 |
add_parameter('/,') |
| 1396 |
|
| 1397 |
if p.is_keyword_only() and not added_star: |
| 1398 |
added_star = True |
| 1399 |
add_parameter('*,') |
| 1400 |
|
| 1401 |
p_lines = [fix_right_bracket_count(p.right_bracket_count)] |
| 1402 |
|
| 1403 |
if isinstance(p.converter, self_converter): |
| 1404 |
# annotate first parameter as being a "self". |
| 1405 |
# |
| 1406 |
# if inspect.Signature gets this function, |
| 1407 |
# and it's already bound, the self parameter |
| 1408 |
# will be stripped off. |
| 1409 |
# |
| 1410 |
# if it's not bound, it should be marked |
| 1411 |
# as positional-only. |
| 1412 |
# |
| 1413 |
# note: we don't print "self" for __init__, |
| 1414 |
# because this isn't actually the signature |
| 1415 |
# for __init__. (it can't be, __init__ doesn't |
| 1416 |
# have a docstring.) if this is an __init__ |
| 1417 |
# (or __new__), then this signature is for |
| 1418 |
# calling the class to construct a new instance. |
| 1419 |
p_lines.append('$') |
| 1420 |
|
| 1421 |
if p.is_vararg(): |
| 1422 |
p_lines.append("*") |
| 1423 |
added_star = True |
| 1424 |
|
| 1425 |
name = p.converter.signature_name or p.name |
| 1426 |
p_lines.append(name) |
| 1427 |
|
| 1428 |
if not p.is_vararg() and p.converter.is_optional(): |
| 1429 |
p_lines.append('=') |
| 1430 |
value = p.converter.py_default |
| 1431 |
if not value: |
| 1432 |
value = repr(p.converter.default) |
| 1433 |
p_lines.append(value) |
| 1434 |
|
| 1435 |
if (p != last_p) or need_a_trailing_slash: |
| 1436 |
p_lines.append(',') |
| 1437 |
|
| 1438 |
p_output = "".join(p_lines) |
| 1439 |
add_parameter(p_output) |
| 1440 |
|
| 1441 |
lines.append(fix_right_bracket_count(0)) |
| 1442 |
if need_a_trailing_slash: |
| 1443 |
add_parameter('/') |
| 1444 |
lines.append(')') |
| 1445 |
|
| 1446 |
# PEP 8 says: |
| 1447 |
# |
| 1448 |
# The Python standard library will not use function annotations |
| 1449 |
# as that would result in a premature commitment to a particular |
| 1450 |
# annotation style. Instead, the annotations are left for users |
| 1451 |
# to discover and experiment with useful annotation styles. |
| 1452 |
# |
| 1453 |
# therefore this is commented out: |
| 1454 |
# |
| 1455 |
# if f.return_converter.py_default: |
| 1456 |
# lines.append(' -> ') |
| 1457 |
# lines.append(f.return_converter.py_default) |
| 1458 |
|
| 1459 |
if not f.docstring_only: |
| 1460 |
lines.append("\n" + libclinic.SIG_END_MARKER + "\n") |
| 1461 |
|
| 1462 |
signature_line = "".join(lines) |
| 1463 |
|
| 1464 |
# now fix up the places where the brackets look wrong |
| 1465 |
return signature_line.replace(', ]', ',] ') |
| 1466 |
|
| 1467 |
@staticmethod |
| 1468 |
def format_docstring_parameters(params: list[Parameter]) -> str: |
| 1469 |
"""Create substitution text for {parameters}""" |
| 1470 |
return "".join(p.render_docstring() + "\n" for p in params if p.docstring) |
| 1471 |
|
| 1472 |
def format_docstring(self) -> str: |
| 1473 |
assert self.function is not None |
| 1474 |
f = self.function |
| 1475 |
# For the following special cases, it does not make sense to render a docstring. |
| 1476 |
if f.kind in {METHOD_INIT, METHOD_NEW, GETTER, SETTER} and not f.docstring: |
| 1477 |
return f.docstring |
| 1478 |
|
| 1479 |
# Enforce the summary line! |
| 1480 |
# The first line of a docstring should be a summary of the function. |
| 1481 |
# It should fit on one line (80 columns? 79 maybe?) and be a paragraph |
| 1482 |
# by itself. |
| 1483 |
# |
| 1484 |
# Argument Clinic enforces the following rule: |
| 1485 |
# * either the docstring is empty, |
| 1486 |
# * or it must have a summary line. |
| 1487 |
# |
| 1488 |
# Guido said Clinic should enforce this: |
| 1489 |
# http://mail.python.org/pipermail/python-dev/2013-June/127110.html |
| 1490 |
|
| 1491 |
lines = f.docstring.split('\n') |
| 1492 |
if len(lines) >= 2: |
| 1493 |
if lines[1]: |
| 1494 |
fail(f"Docstring for {f.full_name!r} does not have a summary line!\n" |
| 1495 |
"Every non-blank function docstring must start with " |
| 1496 |
"a single line summary followed by an empty line.") |
| 1497 |
elif len(lines) == 1: |
| 1498 |
# the docstring is only one line right now--the summary line. |
| 1499 |
# add an empty line after the summary line so we have space |
| 1500 |
# between it and the {parameters} we're about to add. |
| 1501 |
lines.append('') |
| 1502 |
|
| 1503 |
parameters_marker_count = len(f.docstring.split('{parameters}')) - 1 |
| 1504 |
if parameters_marker_count > 1: |
| 1505 |
fail('You may not specify {parameters} more than once in a docstring!') |
| 1506 |
|
| 1507 |
# insert signature at front and params after the summary line |
| 1508 |
if not parameters_marker_count: |
| 1509 |
lines.insert(2, '{parameters}') |
| 1510 |
lines.insert(0, '{signature}') |
| 1511 |
|
| 1512 |
# finalize docstring |
| 1513 |
params = f.render_parameters |
| 1514 |
parameters = self.format_docstring_parameters(params) |
| 1515 |
signature = self.format_docstring_signature(f, params) |
| 1516 |
docstring = "\n".join(lines) |
| 1517 |
return libclinic.linear_format(docstring, |
| 1518 |
signature=signature, |
| 1519 |
parameters=parameters).rstrip() |
| 1520 |
|
| 1521 |
def check_remaining_star(self, lineno: int | None = None) -> None: |
| 1522 |
assert isinstance(self.function, Function) |
| 1523 |
|
| 1524 |
if self.keyword_only: |
| 1525 |
symbol = '*' |
| 1526 |
elif self.deprecated_positional: |
| 1527 |
symbol = '* [from ...]' |
| 1528 |
else: |
| 1529 |
return |
| 1530 |
|
| 1531 |
for p in reversed(self.function.parameters.values()): |
| 1532 |
if self.keyword_only: |
| 1533 |
if (p.kind == inspect.Parameter.KEYWORD_ONLY or |
| 1534 |
p.kind == inspect.Parameter.VAR_POSITIONAL): |
| 1535 |
return |
| 1536 |
elif self.deprecated_positional: |
| 1537 |
if p.deprecated_positional == self.deprecated_positional: |
| 1538 |
return |
| 1539 |
break |
| 1540 |
|
| 1541 |
fail(f"Function {self.function.name!r} specifies {symbol!r} " |
| 1542 |
f"without following parameters.", line_number=lineno) |
| 1543 |
|
| 1544 |
def check_previous_star(self) -> None: |
| 1545 |
assert isinstance(self.function, Function) |
| 1546 |
|
| 1547 |
if self.keyword_only: |
| 1548 |
fail(f"Function {self.function.name!r} uses '*' more than once.") |
| 1549 |
|
| 1550 |
|
| 1551 |
def do_post_block_processing_cleanup(self, lineno: int) -> None: |
| 1552 |
""" |
| 1553 |
Called when processing the block is done. |
| 1554 |
""" |
| 1555 |
if not self.function: |
| 1556 |
return |
| 1557 |
|
| 1558 |
self.check_remaining_star(lineno) |
| 1559 |
try: |
| 1560 |
self.function.docstring = self.format_docstring() |
| 1561 |
except ClinicError as exc: |
| 1562 |
exc.lineno = lineno |
| 1563 |
exc.filename = self.clinic.filename |
| 1564 |
raise |