Debian Codegraph @

Tools/build/check_warnings.py

raw
1 """
2 Parses compiler output from Clang or GCC and checks that warnings
3 exist only in files that are expected to have warnings.
4 """
5
6 import argparse
7 import re
8 import sys
9 from collections import defaultdict
10 from pathlib import Path
11 from typing import NamedTuple, TypedDict
12
13
14 class IgnoreRule(NamedTuple):
15 file_path: str
16 count: int # type: ignore[assignment]
17 ignore_all: bool = False
18 is_directory: bool = False
19
20
21 class CompileWarning(TypedDict):
22 file: str
23 line: str
24 column: str
25 message: str
26 option: str
27
28
29 def parse_warning_ignore_file(file_path: str) -> set[IgnoreRule]:
30 """
31 Parses the warning ignore file and returns a set of IgnoreRules
32 """
33 files_with_expected_warnings: set[IgnoreRule] = set()
34 with Path(file_path).open(encoding="UTF-8") as ignore_rules_file:
35 files_with_expected_warnings = set()
36 for i, line in enumerate(ignore_rules_file):
37 line = line.strip()
38 if line and not line.startswith("#"):
39 line_parts = line.split()
40 if len(line_parts) >= 2:
41 file_name = line_parts[0]
42 count = line_parts[1]
43 ignore_all = count == "*"
44 is_directory = file_name.endswith("/")
45
46 # Directories must have a wildcard count
47 if is_directory and count != "*":
48 print(
49 f"Error parsing ignore file: {file_path} "
50 f"at line: {i}"
51 )
52 print(
53 f"Directory {file_name} must have count set to *"
54 )
55 sys.exit(1)
56 if ignore_all:
57 count = "0"
58
59 files_with_expected_warnings.add(
60 IgnoreRule(
61 file_name, int(count), ignore_all, is_directory
62 )
63 )
64
65 return files_with_expected_warnings
66
67
68 def extract_warnings_from_compiler_output(
69 compiler_output: str,
70 compiler_output_type: str,
71 path_prefix: str = "",
72 ) -> list[CompileWarning]:
73 """
74 Extracts warnings from the compiler output based on compiler
75 output type. Removes path prefix from file paths if provided.
76 Compatible with GCC and Clang compiler output.
77 """
78 # Choose pattern and compile regex for particular compiler output
79 if compiler_output_type == "gcc":
80 regex_pattern = (
81 r"(?P<file>.*):(?P<line>\d+):(?P<column>\d+): warning: "
82 r"(?P<message>.*?)(?: (?P<option>\[-[^\]]+\]))?$"
83 )
84 elif compiler_output_type == "clang":
85 regex_pattern = (
86 r"(?P<file>.*):(?P<line>\d+):(?P<column>\d+): warning: "
87 r"(?P<message>.*) (?P<option>\[-[^\]]+\])$"
88 )
89 else:
90 raise RuntimeError(
91 f"Unsupported compiler output type: {compiler_output_type}",
92 )
93 compiled_regex = re.compile(regex_pattern)
94 compiler_warnings: list[CompileWarning] = []
95 for i, line in enumerate(compiler_output.splitlines(), start=1):
96 if match := compiled_regex.match(line):
97 try:
98 compiler_warnings.append({
99 "file": match.group("file").removeprefix(path_prefix),
100 "line": match.group("line"),
101 "column": match.group("column"),
102 "message": match.group("message"),
103 "option": match.group("option").lstrip("[").rstrip("]"),
104 })
105 except AttributeError:
106 print(
107 f"Error parsing compiler output. "
108 f"Unable to extract warning on line {i}:\n{line}"
109 )
110 sys.exit(1)
111
112 return compiler_warnings
113
114
115 def get_warnings_by_file(
116 warnings: list[CompileWarning],
117 ) -> dict[str, list[CompileWarning]]:
118 """
119 Returns a dictionary where the key is the file and the data is the
120 warnings in that file. Does not include duplicate warnings for a
121 file from list of provided warnings.
122 """
123 warnings_by_file = defaultdict(list)
124 warnings_added = set()
125 for warning in warnings:
126 warning_key = (
127 f"{warning['file']}-{warning['line']}-"
128 f"{warning['column']}-{warning['option']}"
129 )
130 if warning_key not in warnings_added:
131 warnings_added.add(warning_key)
132 warnings_by_file[warning["file"]].append(warning)
133
134 return warnings_by_file
135
136
137 def is_file_ignored(
138 file_path: str, ignore_rules: set[IgnoreRule]
139 ) -> IgnoreRule | None:
140 """Return the IgnoreRule object for the file path.
141
142 Return ``None`` if there is no related rule for that path.
143 """
144 for rule in ignore_rules:
145 if rule.is_directory:
146 if file_path.startswith(rule.file_path):
147 return rule
148 elif file_path == rule.file_path:
149 return rule
150 return None
151
152
153 def get_unexpected_warnings(
154 ignore_rules: set[IgnoreRule],
155 files_with_warnings: dict[str, list[CompileWarning]],
156 ) -> int:
157 """
158 Returns failure status if warnings discovered in list of warnings
159 are associated with a file that is not found in the list of files
160 with expected warnings
161 """
162 unexpected_warnings = {}
163 for file in files_with_warnings.keys():
164 rule = is_file_ignored(file, ignore_rules)
165
166 if rule:
167 if rule.ignore_all:
168 continue
169
170 if len(files_with_warnings[file]) > rule.count:
171 unexpected_warnings[file] = (
172 files_with_warnings[file],
173 rule.count,
174 )
175 continue
176 elif rule is None:
177 # If the file is not in the ignore list, then it is unexpected
178 unexpected_warnings[file] = (files_with_warnings[file], 0)
179
180 if unexpected_warnings:
181 print("Unexpected warnings:")
182 for file in unexpected_warnings:
183 print(
184 f"{file} expected {unexpected_warnings[file][1]} warnings,"
185 f" found {len(unexpected_warnings[file][0])}"
186 )
187 for warning in unexpected_warnings[file][0]:
188 print(warning)
189
190 return 1
191
192 return 0
193
194
195 def get_unexpected_improvements(
196 ignore_rules: set[IgnoreRule],
197 files_with_warnings: dict[str, list[CompileWarning]],
198 ) -> int:
199 """
200 Returns failure status if the number of warnings for a file is greater
201 than the expected number of warnings for that file based on the ignore
202 rules
203 """
204 unexpected_improvements = []
205 for rule in ignore_rules:
206 if (
207 not rule.ignore_all
208 and rule.file_path not in files_with_warnings.keys()
209 ):
210 if rule.file_path not in files_with_warnings.keys():
211 unexpected_improvements.append((rule.file_path, rule.count, 0))
212 elif len(files_with_warnings[rule.file_path]) < rule.count:
213 unexpected_improvements.append((
214 rule.file_path,
215 rule.count,
216 len(files_with_warnings[rule.file_path]),
217 ))
218
219 if unexpected_improvements:
220 print("Unexpected improvements:")
221 for file in unexpected_improvements:
222 print(f"{file[0]} expected {file[1]} warnings, found {file[2]}")
223 return 1
224
225 return 0
226
227
228 def main(argv: list[str] | None = None) -> int:
229 parser = argparse.ArgumentParser()
230 parser.add_argument(
231 "-c",
232 "--compiler-output-file-path",
233 type=str,
234 required=True,
235 help="Path to the compiler output file",
236 )
237 parser.add_argument(
238 "-i",
239 "--warning-ignore-file-path",
240 type=str,
241 help="Path to the warning ignore file",
242 )
243 parser.add_argument(
244 "-x",
245 "--fail-on-regression",
246 action="store_true",
247 default=False,
248 help="Flag to fail if new warnings are found",
249 )
250 parser.add_argument(
251 "-X",
252 "--fail-on-improvement",
253 action="store_true",
254 default=False,
255 help="Flag to fail if files that were expected "
256 "to have warnings have no warnings",
257 )
258 parser.add_argument(
259 "-t",
260 "--compiler-output-type",
261 type=str,
262 required=True,
263 choices=["gcc", "clang"],
264 help="Type of compiler output file (GCC or Clang)",
265 )
266 parser.add_argument(
267 "-p",
268 "--path-prefix",
269 type=str,
270 help="Path prefix to remove from the start of file paths"
271 " in compiler output",
272 )
273
274 args = parser.parse_args(argv)
275
276 exit_code = 0
277
278 # Check that the compiler output file is a valid path
279 if not Path(args.compiler_output_file_path).is_file():
280 print(
281 f"Compiler output file does not exist:"
282 f" {args.compiler_output_file_path}"
283 )
284 return 1
285
286 # Check that a warning ignore file was specified and if so is a valid path
287 if not args.warning_ignore_file_path:
288 print(
289 "Warning ignore file not specified."
290 " Continuing without it (no warnings ignored)."
291 )
292 ignore_rules = set()
293 else:
294 if not Path(args.warning_ignore_file_path).is_file():
295 print(
296 f"Warning ignore file does not exist:"
297 f" {args.warning_ignore_file_path}"
298 )
299 return 1
300 ignore_rules = parse_warning_ignore_file(args.warning_ignore_file_path)
301
302 with Path(args.compiler_output_file_path).open(encoding="UTF-8") as f:
303 compiler_output_file_contents = f.read()
304
305 warnings = extract_warnings_from_compiler_output(
306 compiler_output_file_contents,
307 args.compiler_output_type,
308 args.path_prefix,
309 )
310
311 files_with_warnings = get_warnings_by_file(warnings)
312
313 status = get_unexpected_warnings(ignore_rules, files_with_warnings)
314 if args.fail_on_regression:
315 exit_code |= status
316
317 status = get_unexpected_improvements(ignore_rules, files_with_warnings)
318 if args.fail_on_improvement:
319 exit_code |= status
320
321 print(
322 "For information about this tool and its configuration"
323 " visit https://devguide.python.org/development-tools/warnings/"
324 )
325
326 return exit_code
327
328
329 if __name__ == "__main__":
330 sys.exit(main())