Debian Codegraph @

Tools/build/compute-changes.py

raw
1 """Determine which GitHub Actions workflows to run.
2
3 Called by ``.github/workflows/reusable-context.yml``.
4 We only want to run tests on PRs when related files are changed,
5 or when someone triggers a manual workflow run.
6 This improves developer experience by not doing (slow)
7 unnecessary work in GHA, and saves CI resources.
8 """
9
10 from __future__ import annotations
11
12 import os
13 import subprocess
14 from dataclasses import dataclass, fields
15 from pathlib import Path
16
17 TYPE_CHECKING = False
18 if TYPE_CHECKING:
19 from collections.abc import Set
20
21 GITHUB_DEFAULT_BRANCH = os.environ["GITHUB_DEFAULT_BRANCH"]
22 GITHUB_WORKFLOWS_PATH = Path(".github/workflows")
23
24 RUN_TESTS_IGNORE = frozenset({
25 Path("Tools/check-c-api-docs/ignored_c_api.txt"),
26 Path(".github/CODEOWNERS"),
27 Path(".pre-commit-config.yaml"),
28 Path(".ruff.toml"),
29 Path("mypy.ini"),
30 })
31
32 UNIX_BUILD_SYSTEM_FILE_NAMES = frozenset({
33 Path("aclocal.m4"),
34 Path("config.guess"),
35 Path("config.sub"),
36 Path("configure"),
37 Path("configure.ac"),
38 Path("install-sh"),
39 Path("Makefile.pre.in"),
40 Path("Modules/makesetup"),
41 Path("Modules/Setup"),
42 Path("Modules/Setup.bootstrap.in"),
43 Path("Modules/Setup.stdlib.in"),
44 Path("Tools/build/regen-configure.sh"),
45 })
46
47 SUFFIXES_C_OR_CPP = frozenset({".c", ".h", ".cpp"})
48 SUFFIXES_DOCUMENTATION = frozenset({".rst", ".md"})
49
50 ANDROID_DIRS = frozenset({"Android"})
51 MACOS_DIRS = frozenset({"Mac"})
52 WASI_DIRS = frozenset({Path("Tools", "wasm")})
53
54 LIBRARY_FUZZER_PATHS = frozenset({
55 # All C/CPP fuzzers.
56 Path("configure"),
57 Path(".github/workflows/reusable-cifuzz.yml"),
58 # ast
59 Path("Lib/ast.py"),
60 Path("Python/ast.c"),
61 # configparser
62 Path("Lib/configparser.py"),
63 # csv
64 Path("Lib/csv.py"),
65 Path("Modules/_csv.c"),
66 # decode
67 Path("Lib/encodings/"),
68 Path("Modules/_codecsmodule.c"),
69 Path("Modules/cjkcodecs/"),
70 Path("Modules/unicodedata*"),
71 # difflib
72 Path("Lib/difflib.py"),
73 # email
74 Path("Lib/email/"),
75 # html
76 Path("Lib/html/"),
77 Path("Lib/_markupbase.py"),
78 # http.client
79 Path("Lib/http/client.py"),
80 # json
81 Path("Lib/json/"),
82 Path("Modules/_json.c"),
83 # plist
84 Path("Lib/plistlib.py"),
85 # re
86 Path("Lib/re/"),
87 Path("Modules/_sre/"),
88 # tarfile
89 Path("Lib/tarfile.py"),
90 # tomllib
91 Path("Modules/tomllib/"),
92 # xml
93 Path("Lib/xml/"),
94 Path("Lib/_markupbase.py"),
95 Path("Modules/expat/"),
96 Path("Modules/pyexpat.c"),
97 # zipfile
98 Path("Lib/zipfile/"),
99 # zoneinfo
100 Path("Lib/zoneinfo/"),
101 Path("Modules/_zoneinfo.c"),
102 })
103
104
105 @dataclass(kw_only=True, slots=True)
106 class Outputs:
107 run_android: bool = False
108 run_ci_fuzz: bool = False
109 run_ci_fuzz_stdlib: bool = False
110 run_docs: bool = False
111 run_macos: bool = False
112 run_tests: bool = False
113 run_ubuntu: bool = False
114 run_wasi: bool = False
115 run_windows_msi: bool = False
116 run_windows_tests: bool = False
117
118
119 def compute_changes() -> None:
120 target_branch, head_ref = git_refs()
121 if os.environ.get("GITHUB_EVENT_NAME", "") == "pull_request":
122 # Getting changed files only makes sense on a pull request
123 files = get_changed_files(target_branch, head_ref)
124 outputs = process_changed_files(files)
125 else:
126 # Otherwise, just run the tests
127 outputs = Outputs(
128 run_android=True,
129 run_macos=True,
130 run_tests=True,
131 run_ubuntu=True,
132 run_wasi=True,
133 run_windows_tests=True,
134 )
135 outputs = process_target_branch(outputs, target_branch)
136
137 if outputs.run_tests:
138 print("Run tests")
139 if outputs.run_windows_tests:
140 print("Run Windows tests")
141
142 if outputs.run_ci_fuzz:
143 print("Run CIFuzz tests")
144 else:
145 print("Branch too old for CIFuzz tests; or no C files were changed")
146
147 if outputs.run_ci_fuzz_stdlib:
148 print("Run CIFuzz tests for stdlib")
149 else:
150 print("Branch too old for CIFuzz tests; or no stdlib files were changed")
151
152 if outputs.run_docs:
153 print("Build documentation")
154
155 if outputs.run_windows_msi:
156 print("Build Windows MSI")
157
158 print(outputs)
159
160 write_github_output(outputs)
161
162
163 def git_refs() -> tuple[str, str]:
164 target_ref = os.environ.get("CCF_TARGET_REF", "")
165 target_ref = target_ref.removeprefix("refs/heads/")
166 print(f"target ref: {target_ref!r}")
167
168 head_ref = os.environ.get("CCF_HEAD_REF", "")
169 head_ref = head_ref.removeprefix("refs/heads/")
170 print(f"head ref: {head_ref!r}")
171 return f"origin/{target_ref}", head_ref
172
173
174 def get_changed_files(
175 ref_a: str = GITHUB_DEFAULT_BRANCH, ref_b: str = "HEAD"
176 ) -> Set[Path]:
177 """List the files changed between two Git refs, filtered by change type."""
178 args = ("git", "diff", "--name-only", f"{ref_a}...{ref_b}", "--")
179 print(*args)
180 changed_files_result = subprocess.run(
181 args, stdout=subprocess.PIPE, check=True, encoding="utf-8"
182 )
183 changed_files = changed_files_result.stdout.strip().splitlines()
184 return frozenset(map(Path, filter(None, map(str.strip, changed_files))))
185
186
187 def get_file_platform(file: Path) -> str | None:
188 if not file.parts:
189 return None
190 first_part = file.parts[0]
191 if first_part in MACOS_DIRS:
192 return "macos"
193 if first_part in ANDROID_DIRS:
194 return "android"
195 if len(file.parts) >= 2 and Path(*file.parts[:2]) in WASI_DIRS: # Tools/wasm/
196 return "wasi"
197 return None
198
199
200 def is_fuzzable_library_file(file: Path) -> bool:
201 return any(
202 (file.is_relative_to(needs_fuzz) and needs_fuzz.is_dir())
203 or (file == needs_fuzz and file.is_file())
204 for needs_fuzz in LIBRARY_FUZZER_PATHS
205 )
206
207
208 def process_changed_files(changed_files: Set[Path]) -> Outputs:
209 run_tests = False
210 run_ci_fuzz = False
211 run_ci_fuzz_stdlib = False
212 run_docs = False
213 run_windows_tests = False
214 run_windows_msi = False
215
216 platforms_changed = set()
217 has_platform_specific_change = True
218
219 for file in changed_files:
220 # Documentation files
221 doc_or_misc = file.parts[0] in {"Doc", "Misc"}
222 doc_file = file.suffix in SUFFIXES_DOCUMENTATION or doc_or_misc
223
224 if file.parent == GITHUB_WORKFLOWS_PATH:
225 if file.name in ("build.yml", "reusable-cifuzz.yml"):
226 run_tests = run_ci_fuzz = run_ci_fuzz_stdlib = run_windows_tests = True
227 has_platform_specific_change = False
228 continue
229 if file.name == "reusable-docs.yml":
230 run_docs = True
231 continue
232 if file.name == "reusable-windows.yml":
233 run_tests = True
234 run_windows_tests = True
235 continue
236 if file.name == "reusable-windows-msi.yml":
237 run_windows_msi = True
238 continue
239 if file.name == "reusable-macos.yml":
240 run_tests = True
241 platforms_changed.add("macos")
242 continue
243 if file.name == "reusable-wasi.yml":
244 run_tests = True
245 platforms_changed.add("wasi")
246 continue
247
248 if not doc_file and file not in RUN_TESTS_IGNORE:
249 run_tests = True
250
251 platform = get_file_platform(file)
252 if platform is not None:
253 platforms_changed.add(platform)
254 else:
255 has_platform_specific_change = False
256 if file not in UNIX_BUILD_SYSTEM_FILE_NAMES:
257 run_windows_tests = True
258
259 # The fuzz tests are pretty slow so they are executed only for PRs
260 # changing relevant files.
261 if file.suffix in SUFFIXES_C_OR_CPP:
262 run_ci_fuzz = True
263 if file.parts[:2] in {
264 ("configure",),
265 ("Modules", "_xxtestfuzz"),
266 }:
267 run_ci_fuzz = True
268 if not run_ci_fuzz_stdlib and is_fuzzable_library_file(file):
269 run_ci_fuzz_stdlib = True
270
271 # Check for changed documentation-related files
272 if doc_file:
273 run_docs = True
274
275 # Check for changed MSI installer-related files
276 if file.parts[:2] == ("Tools", "msi"):
277 run_windows_msi = True
278
279 # Check which platform specific tests to run
280 if run_tests:
281 if not has_platform_specific_change or not platforms_changed:
282 run_android = True
283 run_macos = True
284 run_ubuntu = True
285 run_wasi = True
286 else:
287 run_android = "android" in platforms_changed
288 run_macos = "macos" in platforms_changed
289 run_ubuntu = False
290 run_wasi = "wasi" in platforms_changed
291 else:
292 run_android = False
293 run_macos = False
294 run_ubuntu = False
295 run_wasi = False
296
297 return Outputs(
298 run_android=run_android,
299 run_ci_fuzz=run_ci_fuzz,
300 run_ci_fuzz_stdlib=run_ci_fuzz_stdlib,
301 run_docs=run_docs,
302 run_macos=run_macos,
303 run_tests=run_tests,
304 run_ubuntu=run_ubuntu,
305 run_wasi=run_wasi,
306 run_windows_msi=run_windows_msi,
307 run_windows_tests=run_windows_tests,
308 )
309
310
311 def process_target_branch(outputs: Outputs, git_branch: str) -> Outputs:
312 if not git_branch:
313 outputs.run_tests = True
314
315 # CIFuzz / OSS-Fuzz compatibility with older branches may be broken.
316 if git_branch != GITHUB_DEFAULT_BRANCH:
317 outputs.run_ci_fuzz = False
318
319 if os.environ.get("GITHUB_EVENT_NAME", "").lower() == "workflow_dispatch":
320 outputs.run_docs = True
321 outputs.run_windows_msi = True
322
323 return outputs
324
325
326 def write_github_output(outputs: Outputs) -> None:
327 # https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/store-information-in-variables#default-environment-variables
328 # https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions#setting-an-output-parameter
329 if "GITHUB_OUTPUT" not in os.environ:
330 print("GITHUB_OUTPUT not defined!")
331 return
332
333 with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as f:
334 for field in fields(outputs):
335 name = field.name.replace("_", "-")
336 val = bool_lower(getattr(outputs, field.name))
337 f.write(f"{name}={val}\n")
338
339
340 def bool_lower(value: bool, /) -> str:
341 return "true" if value else "false"
342
343
344 if __name__ == "__main__":
345 compute_changes()