| 1 |
import re |
| 2 |
from pathlib import Path |
| 3 |
import sys |
| 4 |
import _colorize |
| 5 |
import textwrap |
| 6 |
|
| 7 |
SIMPLE_FUNCTION_REGEX = re.compile(r"PyAPI_FUNC(.+) (\w+)\(") |
| 8 |
SIMPLE_MACRO_REGEX = re.compile(r"# *define *(\w+)(\(.+\))? ") |
| 9 |
SIMPLE_INLINE_REGEX = re.compile(r"static inline .+( |\n)(\w+)") |
| 10 |
SIMPLE_DATA_REGEX = re.compile(r"PyAPI_DATA\(.+\) (\w+)") |
| 11 |
API_NAME_REGEX = re.compile(r'\bP[yY][a-zA-Z0-9_]+') |
| 12 |
|
| 13 |
CPYTHON = Path(__file__).parent.parent.parent |
| 14 |
INCLUDE = CPYTHON / "Include" |
| 15 |
C_API_DOCS = CPYTHON / "Doc" / "c-api" |
| 16 |
IGNORED = ( |
| 17 |
(CPYTHON / "Tools" / "check-c-api-docs" / "ignored_c_api.txt") |
| 18 |
.read_text() |
| 19 |
.split("\n") |
| 20 |
) |
| 21 |
|
| 22 |
for index, line in enumerate(IGNORED): |
| 23 |
if line.startswith("#"): |
| 24 |
IGNORED.pop(index) |
| 25 |
|
| 26 |
MISTAKE = """ |
| 27 |
If this is a mistake and this script should not be failing, create an |
| 28 |
issue and tag Peter (@ZeroIntensity) on it.\ |
| 29 |
""" |
| 30 |
|
| 31 |
|
| 32 |
def found_undocumented(singular: bool) -> str: |
| 33 |
some = "an" if singular else "some" |
| 34 |
s = "" if singular else "s" |
| 35 |
these = "this" if singular else "these" |
| 36 |
them = "it" if singular else "them" |
| 37 |
were = "was" if singular else "were" |
| 38 |
|
| 39 |
return ( |
| 40 |
textwrap.dedent( |
| 41 |
f""" |
| 42 |
Found {some} undocumented C API{s}! |
| 43 |
|
| 44 |
Python requires documentation on all public C API symbols, macros, and types. |
| 45 |
If {these} API{s} {were} not meant to be public, prefix {them} with a |
| 46 |
leading underscore (_PySomething_API) or move {them} to the internal C API |
| 47 |
(pycore_*.h files). |
| 48 |
|
| 49 |
In exceptional cases, certain APIs can be ignored by adding them to |
| 50 |
Tools/check-c-api-docs/ignored_c_api.txt |
| 51 |
""" |
| 52 |
) |
| 53 |
+ MISTAKE |
| 54 |
) |
| 55 |
|
| 56 |
|
| 57 |
def found_ignored_documented(singular: bool) -> str: |
| 58 |
some = "a" if singular else "some" |
| 59 |
s = "" if singular else "s" |
| 60 |
them = "it" if singular else "them" |
| 61 |
were = "was" if singular else "were" |
| 62 |
they = "it" if singular else "they" |
| 63 |
|
| 64 |
return ( |
| 65 |
textwrap.dedent( |
| 66 |
f""" |
| 67 |
Found {some} C API{s} listed in Tools/c-api-docs-check/ignored_c_api.txt, but |
| 68 |
{they} {were} found in the documentation. To fix this, remove {them} from |
| 69 |
ignored_c_api.txt. |
| 70 |
""" |
| 71 |
) |
| 72 |
+ MISTAKE |
| 73 |
) |
| 74 |
|
| 75 |
|
| 76 |
def scan_file_for_docs( |
| 77 |
filename: str, |
| 78 |
text: str, |
| 79 |
names: set[str]) -> tuple[list[str], list[str]]: |
| 80 |
""" |
| 81 |
Scan a header file for C API functions. |
| 82 |
""" |
| 83 |
undocumented: list[str] = [] |
| 84 |
documented_ignored: list[str] = [] |
| 85 |
colors = _colorize.get_colors() |
| 86 |
|
| 87 |
def check_for_name(name: str) -> None: |
| 88 |
documented = name in names |
| 89 |
if documented and (name in IGNORED): |
| 90 |
documented_ignored.append(name) |
| 91 |
elif not documented and (name not in IGNORED): |
| 92 |
undocumented.append(name) |
| 93 |
|
| 94 |
for function in SIMPLE_FUNCTION_REGEX.finditer(text): |
| 95 |
name = function.group(2) |
| 96 |
if not API_NAME_REGEX.fullmatch(name): |
| 97 |
continue |
| 98 |
|
| 99 |
check_for_name(name) |
| 100 |
|
| 101 |
for macro in SIMPLE_MACRO_REGEX.finditer(text): |
| 102 |
name = macro.group(1) |
| 103 |
if not API_NAME_REGEX.fullmatch(name): |
| 104 |
continue |
| 105 |
|
| 106 |
if "(" in name: |
| 107 |
name = name[: name.index("(")] |
| 108 |
|
| 109 |
check_for_name(name) |
| 110 |
|
| 111 |
for inline in SIMPLE_INLINE_REGEX.finditer(text): |
| 112 |
name = inline.group(2) |
| 113 |
if not API_NAME_REGEX.fullmatch(name): |
| 114 |
continue |
| 115 |
|
| 116 |
check_for_name(name) |
| 117 |
|
| 118 |
for data in SIMPLE_DATA_REGEX.finditer(text): |
| 119 |
name = data.group(1) |
| 120 |
if not API_NAME_REGEX.fullmatch(name): |
| 121 |
continue |
| 122 |
|
| 123 |
check_for_name(name) |
| 124 |
|
| 125 |
# Remove duplicates and sort alphabetically to keep the output deterministic |
| 126 |
undocumented = list(set(undocumented)) |
| 127 |
undocumented.sort() |
| 128 |
|
| 129 |
if undocumented or documented_ignored: |
| 130 |
print(f"{filename} {colors.RED}BAD{colors.RESET}") |
| 131 |
for name in undocumented: |
| 132 |
print(f"{colors.BOLD_RED}UNDOCUMENTED:{colors.RESET} {name}") |
| 133 |
for name in documented_ignored: |
| 134 |
print(f"{colors.BOLD_YELLOW}DOCUMENTED BUT IGNORED:{colors.RESET} {name}") |
| 135 |
else: |
| 136 |
print(f"{filename} {colors.GREEN}OK{colors.RESET}") |
| 137 |
|
| 138 |
return undocumented, documented_ignored |
| 139 |
|
| 140 |
|
| 141 |
def main() -> None: |
| 142 |
print("Gathering C API names from docs...") |
| 143 |
names = set() |
| 144 |
for path in C_API_DOCS.glob('**/*.rst'): |
| 145 |
text = path.read_text(encoding="utf-8") |
| 146 |
for name in API_NAME_REGEX.findall(text): |
| 147 |
names.add(name) |
| 148 |
print(f"Got {len(names)} names!") |
| 149 |
|
| 150 |
print("Scanning for undocumented C API functions...") |
| 151 |
files = [*INCLUDE.iterdir(), *(INCLUDE / "cpython").iterdir()] |
| 152 |
all_missing: list[str] = [] |
| 153 |
all_found_ignored: list[str] = [] |
| 154 |
|
| 155 |
for file in files: |
| 156 |
if file.is_dir(): |
| 157 |
continue |
| 158 |
assert file.exists() |
| 159 |
text = file.read_text(encoding="utf-8") |
| 160 |
missing, ignored = scan_file_for_docs(str(file.relative_to(INCLUDE)), text, names) |
| 161 |
all_found_ignored += ignored |
| 162 |
all_missing += missing |
| 163 |
|
| 164 |
fail = False |
| 165 |
to_check = [ |
| 166 |
(all_missing, "missing", found_undocumented(len(all_missing) == 1)), |
| 167 |
( |
| 168 |
all_found_ignored, |
| 169 |
"documented but ignored", |
| 170 |
found_ignored_documented(len(all_found_ignored) == 1), |
| 171 |
), |
| 172 |
] |
| 173 |
for name_list, what, message in to_check: |
| 174 |
if not name_list: |
| 175 |
continue |
| 176 |
|
| 177 |
s = "s" if len(name_list) != 1 else "" |
| 178 |
print(f"-- {len(name_list)} {what} C API{s} --") |
| 179 |
for name in name_list: |
| 180 |
print(f" - {name}") |
| 181 |
print(message) |
| 182 |
fail = True |
| 183 |
|
| 184 |
sys.exit(1 if fail else 0) |
| 185 |
|
| 186 |
|
| 187 |
if __name__ == "__main__": |
| 188 |
main() |