Debian Codegraph @

Tools/build/compute-changes.py

raw
1 """Determine which GitHub Actions workflows to run.
2
3 Called by ``.github/workflows/reusable-context.yml``.
4 We only want to run tests on PRs when related files are changed,
5 or when someone triggers a manual workflow run.
6 This improves developer experience by not doing (slow)
7 unnecessary work in GHA, and saves CI resources.
8 """
9
10 from __future__ import annotations
11
12 import os
13 import subprocess
14 from dataclasses import dataclass, fields
15 from pathlib import Path
16
17 TYPE_CHECKING = False
18 if TYPE_CHECKING:
19 from collections.abc import Set
20
21 GITHUB_DEFAULT_BRANCH = os.environ["GITHUB_DEFAULT_BRANCH"]
22 GITHUB_WORKFLOWS_PATH = Path(".github/workflows")
23
24 RUN_TESTS_IGNORE = frozenset({
25 Path("Tools/check-c-api-docs/ignored_c_api.txt"),
26 Path(".github/CODEOWNERS"),
27 Path(".pre-commit-config.yaml"),
28 Path(".ruff.toml"),
29 Path("mypy.ini"),
30 })
31
32 UNIX_BUILD_SYSTEM_FILE_NAMES = frozenset({
33 Path("aclocal.m4"),
34 Path("config.guess"),
35 Path("config.sub"),
36 Path("configure"),
37 Path("configure.ac"),
38 Path("install-sh"),
39 Path("Makefile.pre.in"),
40 Path("Modules/makesetup"),
41 Path("Modules/Setup"),
42 Path("Modules/Setup.bootstrap.in"),
43 Path("Modules/Setup.stdlib.in"),
44 Path("Tools/build/regen-configure.sh"),
45 })
46
47 SUFFIXES_C_OR_CPP = frozenset({".c", ".h", ".cpp"})
48 SUFFIXES_DOCUMENTATION = frozenset({".rst", ".md"})
49
50 ANDROID_DIRS = frozenset({"Android"})
51 EMSCRIPTEN_DIRS = frozenset({Path("Platforms", "emscripten")})
52 IOS_DIRS = frozenset({"Apple", "iOS"})
53 MACOS_DIRS = frozenset({"Mac"})
54 WASI_DIRS = frozenset({Path("Tools", "wasm")})
55
56 LIBRARY_FUZZER_PATHS = frozenset({
57 # All C/CPP fuzzers.
58 Path("configure"),
59 Path(".github/workflows/reusable-cifuzz.yml"),
60 # ast
61 Path("Lib/ast.py"),
62 Path("Python/ast.c"),
63 # configparser
64 Path("Lib/configparser.py"),
65 # csv
66 Path("Lib/csv.py"),
67 Path("Modules/_csv.c"),
68 # decode
69 Path("Lib/encodings/"),
70 Path("Modules/_codecsmodule.c"),
71 Path("Modules/cjkcodecs/"),
72 Path("Modules/unicodedata*"),
73 # difflib
74 Path("Lib/difflib.py"),
75 # email
76 Path("Lib/email/"),
77 # html
78 Path("Lib/html/"),
79 Path("Lib/_markupbase.py"),
80 # http.client
81 Path("Lib/http/client.py"),
82 # json
83 Path("Lib/json/"),
84 Path("Modules/_json.c"),
85 # plist
86 Path("Lib/plistlib.py"),
87 # re
88 Path("Lib/re/"),
89 Path("Modules/_sre/"),
90 # tarfile
91 Path("Lib/tarfile.py"),
92 # tomllib
93 Path("Modules/tomllib/"),
94 # xml
95 Path("Lib/xml/"),
96 Path("Lib/_markupbase.py"),
97 Path("Modules/expat/"),
98 Path("Modules/pyexpat.c"),
99 # zipfile
100 Path("Lib/zipfile/"),
101 # zoneinfo
102 Path("Lib/zoneinfo/"),
103 Path("Modules/_zoneinfo.c"),
104 })
105
106
107 @dataclass(kw_only=True, slots=True)
108 class Outputs:
109 run_android: bool = False
110 run_ci_fuzz: bool = False
111 run_ci_fuzz_stdlib: bool = False
112 run_docs: bool = False
113 run_emscripten: bool = False
114 run_ios: bool = False
115 run_macos: bool = False
116 run_tests: bool = False
117 run_ubuntu: bool = False
118 run_wasi: bool = False
119 run_windows_msi: bool = False
120 run_windows_tests: bool = False
121
122
123 def compute_changes() -> None:
124 target_branch, head_ref = git_refs()
125 if os.environ.get("GITHUB_EVENT_NAME", "") == "pull_request":
126 # Getting changed files only makes sense on a pull request
127 files = get_changed_files(target_branch, head_ref)
128 outputs = process_changed_files(files)
129 else:
130 # Otherwise, just run the tests
131 outputs = Outputs(
132 run_android=True,
133 run_emscripten=True,
134 run_ios=True,
135 run_macos=True,
136 run_tests=True,
137 run_ubuntu=True,
138 run_wasi=True,
139 run_windows_tests=True,
140 )
141 outputs = process_target_branch(outputs, target_branch)
142
143 if outputs.run_tests:
144 print("Run tests")
145 if outputs.run_windows_tests:
146 print("Run Windows tests")
147
148 if outputs.run_ci_fuzz:
149 print("Run CIFuzz tests")
150 else:
151 print("Branch too old for CIFuzz tests; or no C files were changed")
152
153 if outputs.run_ci_fuzz_stdlib:
154 print("Run CIFuzz tests for stdlib")
155 else:
156 print("Branch too old for CIFuzz tests; or no stdlib files were changed")
157
158 if outputs.run_docs:
159 print("Build documentation")
160
161 if outputs.run_windows_msi:
162 print("Build Windows MSI")
163
164 print(outputs)
165
166 write_github_output(outputs)
167
168
169 def git_refs() -> tuple[str, str]:
170 target_ref = os.environ.get("CCF_TARGET_REF", "")
171 target_ref = target_ref.removeprefix("refs/heads/")
172 print(f"target ref: {target_ref!r}")
173
174 head_ref = os.environ.get("CCF_HEAD_REF", "")
175 head_ref = head_ref.removeprefix("refs/heads/")
176 print(f"head ref: {head_ref!r}")
177 return f"origin/{target_ref}", head_ref
178
179
180 def get_changed_files(
181 ref_a: str = GITHUB_DEFAULT_BRANCH, ref_b: str = "HEAD"
182 ) -> Set[Path]:
183 """List the files changed between two Git refs, filtered by change type."""
184 args = ("git", "diff", "--name-only", f"{ref_a}...{ref_b}", "--")
185 print(*args)
186 changed_files_result = subprocess.run(
187 args, stdout=subprocess.PIPE, check=True, encoding="utf-8"
188 )
189 changed_files = changed_files_result.stdout.strip().splitlines()
190 return frozenset(map(Path, filter(None, map(str.strip, changed_files))))
191
192
193 def get_file_platform(file: Path) -> str | None:
194 if not file.parts:
195 return None
196 first_part = file.parts[0]
197 if first_part in MACOS_DIRS:
198 return "macos"
199 if first_part in IOS_DIRS:
200 return "ios"
201 if first_part in ANDROID_DIRS:
202 return "android"
203 if len(file.parts) >= 2 and Path(*file.parts[:2]) in EMSCRIPTEN_DIRS:
204 return "emscripten"
205 if len(file.parts) >= 2 and Path(*file.parts[:2]) in WASI_DIRS: # Tools/wasm/
206 return "wasi"
207 return None
208
209
210 def is_fuzzable_library_file(file: Path) -> bool:
211 return any(
212 (file.is_relative_to(needs_fuzz) and needs_fuzz.is_dir())
213 or (file == needs_fuzz and file.is_file())
214 for needs_fuzz in LIBRARY_FUZZER_PATHS
215 )
216
217
218 def process_changed_files(changed_files: Set[Path]) -> Outputs:
219 run_tests = False
220 run_ci_fuzz = False
221 run_ci_fuzz_stdlib = False
222 run_docs = False
223 run_windows_tests = False
224 run_windows_msi = False
225
226 platforms_changed = set()
227 has_platform_specific_change = True
228
229 for file in changed_files:
230 # Documentation files
231 doc_or_misc = file.parts[0] in {"Doc", "Misc"}
232 doc_file = file.suffix in SUFFIXES_DOCUMENTATION or doc_or_misc
233
234 if file.parent == GITHUB_WORKFLOWS_PATH:
235 if file.name in ("build.yml", "reusable-cifuzz.yml"):
236 run_tests = run_ci_fuzz = run_ci_fuzz_stdlib = run_windows_tests = True
237 has_platform_specific_change = False
238 continue
239 if file.name == "reusable-docs.yml":
240 run_docs = True
241 continue
242 if file.name == "reusable-windows.yml":
243 run_tests = True
244 run_windows_tests = True
245 continue
246 if file.name == "reusable-windows-msi.yml":
247 run_windows_msi = True
248 continue
249 if file.name == "reusable-macos.yml":
250 run_tests = True
251 platforms_changed.add("macos")
252 continue
253 if file.name == "reusable-emscripten.yml":
254 run_tests = True
255 platforms_changed.add("emscripten")
256 continue
257 if file.name == "reusable-wasi.yml":
258 run_tests = True
259 platforms_changed.add("wasi")
260 continue
261
262 if not doc_file and file not in RUN_TESTS_IGNORE:
263 run_tests = True
264
265 platform = get_file_platform(file)
266 if platform is not None:
267 platforms_changed.add(platform)
268 else:
269 has_platform_specific_change = False
270 if file not in UNIX_BUILD_SYSTEM_FILE_NAMES:
271 run_windows_tests = True
272
273 # The fuzz tests are pretty slow so they are executed only for PRs
274 # changing relevant files.
275 if file.suffix in SUFFIXES_C_OR_CPP:
276 run_ci_fuzz = True
277 if file.parts[:2] in {
278 ("configure",),
279 ("Modules", "_xxtestfuzz"),
280 }:
281 run_ci_fuzz = True
282 if not run_ci_fuzz_stdlib and is_fuzzable_library_file(file):
283 run_ci_fuzz_stdlib = True
284
285 # Check for changed documentation-related files
286 if doc_file:
287 run_docs = True
288
289 # Check for changed MSI installer-related files
290 if file.parts[:2] == ("Tools", "msi"):
291 run_windows_msi = True
292
293 # Check which platform specific tests to run
294 if run_tests:
295 if not has_platform_specific_change or not platforms_changed:
296 run_android = True
297 run_emscripten = True
298 run_ios = True
299 run_macos = True
300 run_ubuntu = True
301 run_wasi = True
302 else:
303 run_android = "android" in platforms_changed
304 run_emscripten = "emscripten" in platforms_changed
305 run_ios = "ios" in platforms_changed
306 run_macos = "macos" in platforms_changed
307 run_ubuntu = False
308 run_wasi = "wasi" in platforms_changed
309 else:
310 run_android = False
311 run_emscripten = False
312 run_ios = False
313 run_macos = False
314 run_ubuntu = False
315 run_wasi = False
316
317 return Outputs(
318 run_android=run_android,
319 run_ci_fuzz=run_ci_fuzz,
320 run_ci_fuzz_stdlib=run_ci_fuzz_stdlib,
321 run_docs=run_docs,
322 run_emscripten=run_emscripten,
323 run_ios=run_ios,
324 run_macos=run_macos,
325 run_tests=run_tests,
326 run_ubuntu=run_ubuntu,
327 run_wasi=run_wasi,
328 run_windows_msi=run_windows_msi,
329 run_windows_tests=run_windows_tests,
330 )
331
332
333 def process_target_branch(outputs: Outputs, git_branch: str) -> Outputs:
334 if not git_branch:
335 outputs.run_tests = True
336
337 # CIFuzz / OSS-Fuzz compatibility with older branches may be broken.
338 if git_branch != GITHUB_DEFAULT_BRANCH:
339 outputs.run_ci_fuzz = False
340
341 if os.environ.get("GITHUB_EVENT_NAME", "").lower() == "workflow_dispatch":
342 outputs.run_docs = True
343 outputs.run_windows_msi = True
344
345 return outputs
346
347
348 def write_github_output(outputs: Outputs) -> None:
349 # https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/store-information-in-variables#default-environment-variables
350 # https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions#setting-an-output-parameter
351 if "GITHUB_OUTPUT" not in os.environ:
352 print("GITHUB_OUTPUT not defined!")
353 return
354
355 with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as f:
356 for field in fields(outputs):
357 name = field.name.replace("_", "-")
358 val = bool_lower(getattr(outputs, field.name))
359 f.write(f"{name}={val}\n")
360
361
362 def bool_lower(value: bool, /) -> str:
363 return "true" if value else "false"
364
365
366 if __name__ == "__main__":
367 compute_changes()