Debian Codegraph @

Doc/tools/extensions/c_annotations.py

raw
1 """Support annotations for C API elements.
2
3 * Reference count annotations for C API functions.
4 * Stable ABI annotations
5 * Limited API annotations
6 * Thread safety annotations for C API functions.
7
8 Configuration:
9 * Set ``refcount_file`` to the path to the reference count data file.
10 * Set ``stable_abi_file`` to the path to stable ABI list.
11 * Set ``threadsafety_file`` to the path to the thread safety data file.
12 """
13
14 from __future__ import annotations
15
16 import csv
17 import dataclasses
18 from pathlib import Path
19 from typing import TYPE_CHECKING
20
21 from docutils import nodes
22 from docutils.statemachine import StringList
23 from sphinx import addnodes
24 from sphinx.locale import _ as sphinx_gettext
25 from sphinx.util.docutils import SphinxDirective
26
27 if TYPE_CHECKING:
28 from sphinx.application import Sphinx
29 from sphinx.util.typing import ExtensionMetadata
30
31 ROLE_TO_OBJECT_TYPE = {
32 "func": "function",
33 "macro": "macro",
34 "member": "member",
35 "type": "type",
36 "data": "var",
37 }
38
39
40 @dataclasses.dataclass(slots=True)
41 class RefCountEntry:
42 # Name of the function.
43 name: str
44 # List of (argument name, type, refcount effect) tuples.
45 # (Currently not used. If it was, a dataclass might work better.)
46 args: list = dataclasses.field(default_factory=list)
47 # Return type of the function.
48 result_type: str = ""
49 # Reference count effect for the return value.
50 result_refs: int | None = None
51
52
53 @dataclasses.dataclass(frozen=True, slots=True)
54 class ThreadSafetyEntry:
55 # Name of the function.
56 name: str
57 # Thread safety level.
58 # One of: 'incompatible', 'compatible', 'safe'.
59 level: str
60
61
62 @dataclasses.dataclass(frozen=True, slots=True)
63 class StableABIEntry:
64 # Role of the object.
65 # Source: Each [item_kind] in stable_abi.toml is mapped to a C Domain role.
66 role: str
67 # Name of the object.
68 # Source: [<item_kind>.*] in stable_abi.toml.
69 name: str
70 # Version when the object was added to the stable ABI.
71 # (Source: [<item_kind>.*.added] in stable_abi.toml.
72 added: str
73 # An explananatory blurb for the ifdef.
74 # Source: ``feature_macro.*.doc`` in stable_abi.toml.
75 ifdef_note: str
76 # Defines how much of the struct is exposed. Only relevant for structs.
77 # Source: [<item_kind>.*.struct_abi_kind] in stable_abi.toml.
78 struct_abi_kind: str
79
80
81 def read_refcount_data(refcount_filename: Path) -> dict[str, RefCountEntry]:
82 refcount_data = {}
83 refcounts = refcount_filename.read_text(encoding="utf8")
84 for line in refcounts.splitlines():
85 line = line.strip()
86 if not line or line.startswith("#"):
87 # blank lines and comments
88 continue
89
90 # Each line is of the form
91 # function ':' type ':' [param name] ':' [refcount effect] ':' [comment]
92 parts = line.split(":", 4)
93 if len(parts) != 5:
94 raise ValueError(f"Wrong field count in {line!r}")
95 function, type, arg, refcount, _comment = parts
96
97 # Get the entry, creating it if needed:
98 try:
99 entry = refcount_data[function]
100 except KeyError:
101 entry = refcount_data[function] = RefCountEntry(function)
102 if not refcount or refcount == "null":
103 refcount = None
104 else:
105 refcount = int(refcount)
106 # Update the entry with the new parameter
107 # or the result information.
108 if arg:
109 entry.args.append((arg, type, refcount))
110 else:
111 entry.result_type = type
112 entry.result_refs = refcount
113
114 return refcount_data
115
116
117 def read_stable_abi_data(stable_abi_file: Path) -> dict[str, StableABIEntry]:
118 stable_abi_data = {}
119 with open(stable_abi_file, encoding="utf8") as fp:
120 for record in csv.DictReader(fp):
121 name = record["name"]
122 stable_abi_data[name] = StableABIEntry(**record)
123
124 return stable_abi_data
125
126
127 _VALID_THREADSAFETY_LEVELS = frozenset({
128 "incompatible",
129 "compatible",
130 "distinct",
131 "shared",
132 "atomic",
133 })
134
135
136 def read_threadsafety_data(
137 threadsafety_filename: Path,
138 ) -> dict[str, ThreadSafetyEntry]:
139 threadsafety_data = {}
140 for line in threadsafety_filename.read_text(encoding="utf8").splitlines():
141 line = line.strip()
142 if not line or line.startswith("#"):
143 continue
144 # Each line is of the form: function_name : level : [comment]
145 parts = line.split(":", 2)
146 if len(parts) < 2:
147 raise ValueError(f"Wrong field count in {line!r}")
148 name, level = parts[0].strip(), parts[1].strip()
149 if level not in _VALID_THREADSAFETY_LEVELS:
150 raise ValueError(
151 f"Unknown thread safety level {level!r} for {name!r}. "
152 f"Valid levels: {sorted(_VALID_THREADSAFETY_LEVELS)}"
153 )
154 threadsafety_data[name] = ThreadSafetyEntry(name=name, level=level)
155 return threadsafety_data
156
157
158 def add_annotations(app: Sphinx, doctree: nodes.document) -> None:
159 state = app.env.domaindata["c_annotations"]
160 refcount_data = state["refcount_data"]
161 stable_abi_data = state["stable_abi_data"]
162 threadsafety_data = state["threadsafety_data"]
163 for node in doctree.findall(addnodes.desc_content):
164 par = node.parent
165 if par["domain"] != "c":
166 continue
167 if not par[0].get("ids", None):
168 continue
169 name = par[0]["ids"][0].removeprefix("c.")
170 objtype = par["objtype"]
171
172 # Thread safety annotation — inserted first so it appears last (bottom-most)
173 # among all annotations.
174 if entry := threadsafety_data.get(name):
175 annotation = _threadsafety_annotation(entry.level)
176 node.insert(0, annotation)
177
178 # Stable ABI annotation.
179 if record := stable_abi_data.get(name):
180 if ROLE_TO_OBJECT_TYPE[record.role] != objtype:
181 msg = (
182 f"Object type mismatch in limited API annotation for {name}: "
183 f"{ROLE_TO_OBJECT_TYPE[record.role]!r} != {objtype!r}"
184 )
185 raise ValueError(msg)
186 annotation = _stable_abi_annotation(record)
187 node.insert(0, annotation)
188
189 # Unstable API annotation.
190 if name.startswith("PyUnstable"):
191 annotation = _unstable_api_annotation()
192 node.insert(0, annotation)
193
194 # Return value annotation
195 if objtype != "function":
196 continue
197 if name not in refcount_data:
198 continue
199 entry = refcount_data[name]
200 if not entry.result_type.endswith("Object*"):
201 continue
202 annotation = _return_value_annotation(entry.result_refs)
203 node.insert(0, annotation)
204
205
206 def _stable_abi_annotation(
207 record: StableABIEntry,
208 is_corresponding_slot: bool = False,
209 ) -> nodes.emphasis:
210 """Create the Stable ABI annotation.
211
212 These have two forms:
213 Part of the `Stable ABI <link>`_.
214 Part of the `Stable ABI <link>`_ since version X.Y.
215 For structs, there's some more info in the message:
216 Part of the `Limited API <link>`_ (as an opaque struct).
217 Part of the `Stable ABI <link>`_ (including all members).
218 Part of the `Limited API <link>`_ (Only some members are part
219 of the stable ABI.).
220 ... all of which can have "since version X.Y" appended.
221 """
222 stable_added = record.added
223 emph_node = nodes.emphasis('', '', classes=["stableabi"])
224 if is_corresponding_slot:
225 # See "Type slot annotations" in add_annotations
226 ref_node = addnodes.pending_xref(
227 "slot ID",
228 refdomain="c",
229 reftarget="PyType_Slot",
230 reftype="type",
231 refexplicit="True",
232 )
233 ref_node += nodes.Text(sphinx_gettext("slot ID"))
234
235 message = sphinx_gettext("The corresponding")
236 emph_node += nodes.Text(" " + message + " ")
237 emph_node += ref_node
238 emph_node += nodes.Text(" ")
239 emph_node += nodes.literal(record.name, record.name)
240 message = sphinx_gettext("is part of the")
241 emph_node += nodes.Text(" " + message + " ")
242 else:
243 message = sphinx_gettext("Part of the")
244 emph_node += nodes.Text(" " + message + " ")
245 ref_node = addnodes.pending_xref(
246 "Stable ABI",
247 refdomain="std",
248 reftarget="stable",
249 reftype="ref",
250 refexplicit="False",
251 )
252 struct_abi_kind = record.struct_abi_kind
253 if struct_abi_kind in {"opaque", "members"}:
254 ref_node += nodes.Text(sphinx_gettext("Limited API"))
255 else:
256 ref_node += nodes.Text(sphinx_gettext("Stable ABI"))
257 emph_node += ref_node
258 if struct_abi_kind == "opaque":
259 emph_node += nodes.Text(" " + sphinx_gettext("(as an opaque struct)"))
260 elif struct_abi_kind == "full-abi":
261 emph_node += nodes.Text(
262 " " + sphinx_gettext("(including all members)")
263 )
264 if record.ifdef_note:
265 emph_node += nodes.Text(f" {record.ifdef_note}")
266 if stable_added == "3.2":
267 # Stable ABI was introduced in 3.2.
268 pass
269 else:
270 emph_node += nodes.Text(
271 " " + sphinx_gettext("since version %s") % stable_added
272 )
273 emph_node += nodes.Text(".")
274 if struct_abi_kind == "members":
275 msg = " " + sphinx_gettext(
276 "(Only some members are part of the stable ABI.)"
277 )
278 emph_node += nodes.Text(msg)
279 return emph_node
280
281
282 def _unstable_api_annotation() -> nodes.admonition:
283 ref_node = addnodes.pending_xref(
284 "Unstable API",
285 nodes.Text(sphinx_gettext("Unstable API")),
286 refdomain="std",
287 reftarget="unstable-c-api",
288 reftype="ref",
289 refexplicit="False",
290 )
291 emph_node = nodes.emphasis(
292 "This is ",
293 sphinx_gettext("This is") + " ",
294 ref_node,
295 nodes.Text(
296 sphinx_gettext(
297 ". It may change without warning in minor releases."
298 )
299 ),
300 )
301 return nodes.admonition(
302 "",
303 emph_node,
304 classes=["unstable-c-api", "warning"],
305 )
306
307
308 def _threadsafety_annotation(level: str) -> nodes.emphasis:
309 match level:
310 case "incompatible":
311 display = sphinx_gettext("Not safe to call from multiple threads")
312 reftarget = "threadsafety-level-incompatible"
313 case "compatible":
314 display = sphinx_gettext(
315 "Safe to call from multiple threads"
316 " with external synchronization only"
317 )
318 reftarget = "threadsafety-level-compatible"
319 case "distinct":
320 display = sphinx_gettext(
321 "Safe to call without external synchronization"
322 " on distinct objects"
323 )
324 reftarget = "threadsafety-level-distinct"
325 case "shared":
326 display = sphinx_gettext(
327 "Safe for concurrent use on the same object"
328 )
329 reftarget = "threadsafety-level-shared"
330 case "atomic":
331 display = sphinx_gettext("Atomic")
332 reftarget = "threadsafety-level-atomic"
333 case _:
334 raise AssertionError(f"Unknown thread safety level {level!r}")
335 ref_node = addnodes.pending_xref(
336 display,
337 nodes.Text(display),
338 refdomain="std",
339 reftarget=reftarget,
340 reftype="ref",
341 refexplicit="True",
342 )
343 prefix = " " + sphinx_gettext("Thread safety:") + " "
344 classes = ["threadsafety", f"threadsafety-{level}"]
345 return nodes.emphasis(
346 "", prefix, ref_node, nodes.Text("."), classes=classes
347 )
348
349
350 def _return_value_annotation(result_refs: int | None) -> nodes.emphasis:
351 classes = ["refcount"]
352 if result_refs is None:
353 rc = sphinx_gettext("Return value: Always NULL.")
354 classes.append("return_null")
355 elif result_refs:
356 rc = sphinx_gettext("Return value: New reference.")
357 classes.append("return_new_ref")
358 else:
359 rc = sphinx_gettext("Return value: Borrowed reference.")
360 classes.append("return_borrowed_ref")
361 return nodes.emphasis(rc, rc, classes=classes)
362
363
364 class LimitedAPIList(SphinxDirective):
365 has_content = False
366 required_arguments = 0
367 optional_arguments = 0
368 final_argument_whitespace = True
369
370 def run(self) -> list[nodes.Node]:
371 state = self.env.domaindata["c_annotations"]
372 content = [
373 f"* :c:{record.role}:`{record.name}`"
374 for record in state["stable_abi_data"].values()
375 ]
376 node = nodes.paragraph()
377 self.state.nested_parse(StringList(content), 0, node)
378 return [node]
379
380
381 class CorrespondingTypeSlot(SphinxDirective):
382 """Type slot annotations
383
384 Docs for these are with the corresponding field, for example,
385 "Py_tp_repr" is documented under "PyTypeObject.tp_repr", with
386 only a stable ABI note mentioning "Py_tp_repr" (and linking to
387 docs on how this works).
388
389 If there is no corresponding field, these should be documented as normal
390 macros.
391 """
392
393 has_content = False
394
395 required_arguments = 1
396 optional_arguments = 0
397
398 def run(self) -> list[nodes.Node]:
399 name = self.arguments[0]
400 state = self.env.domaindata["c_annotations"]
401 stable_abi_data = state["stable_abi_data"]
402
403 try:
404 record = stable_abi_data[name]
405 except LookupError as err:
406 raise LookupError(
407 f"{name} is not part of stable ABI. "
408 + "Document it as `c:macro::` rather than "
409 + "`corresponding-type-slot::`."
410 ) from err
411
412 annotation = _stable_abi_annotation(record, is_corresponding_slot=True)
413
414 node = nodes.paragraph()
415 content = [
416 ".. c:namespace:: NULL",
417 "",
418 ".. c:macro:: " + name,
419 " :no-typesetting:",
420 ]
421 self.state.nested_parse(StringList(content), 0, node)
422 node.insert(0, annotation)
423 return [node]
424
425
426 def init_annotations(app: Sphinx) -> None:
427 # Using domaindata is a bit hack-ish,
428 # but allows storing state without a global variable or closure.
429 app.env.domaindata["c_annotations"] = state = {}
430 state["refcount_data"] = read_refcount_data(
431 Path(app.srcdir, app.config.refcount_file)
432 )
433 state["stable_abi_data"] = read_stable_abi_data(
434 Path(app.srcdir, app.config.stable_abi_file)
435 )
436 state["threadsafety_data"] = read_threadsafety_data(
437 Path(app.srcdir, app.config.threadsafety_file)
438 )
439
440
441 def setup(app: Sphinx) -> ExtensionMetadata:
442 app.add_config_value("refcount_file", "", "env", types={str})
443 app.add_config_value("stable_abi_file", "", "env", types={str})
444 app.add_config_value("threadsafety_file", "", "env", types={str})
445 app.add_directive("limited-api-list", LimitedAPIList)
446 app.add_directive("corresponding-type-slot", CorrespondingTypeSlot)
447 app.connect("builder-inited", init_annotations)
448 app.connect("doctree-read", add_annotations)
449
450 return {
451 "version": "1.0",
452 "parallel_read_safe": True,
453 "parallel_write_safe": True,
454 }