Debian Codegraph @

Tools/build/generate_sbom.py

raw
1 """Tool for generating Software Bill of Materials (SBOM) for Python's dependencies"""
2
3 import glob
4 import hashlib
5 import json
6 import os
7 import random
8 import re
9 import subprocess
10 import sys
11 import time
12 import typing
13 import urllib.error
14 import urllib.request
15 from pathlib import Path, PurePosixPath, PureWindowsPath
16
17 CPYTHON_ROOT_DIR = Path(__file__).parent.parent.parent
18
19 # Before adding a new entry to this list, double check that
20 # the license expression is a valid SPDX license expression:
21 # See: https://spdx.org/licenses
22 ALLOWED_LICENSE_EXPRESSIONS = {
23 "Apache-2.0",
24 "Apache-2.0 OR BSD-2-Clause",
25 "BSD-2-Clause",
26 "BSD-3-Clause",
27 "CC0-1.0",
28 "ISC",
29 "LGPL-2.1-only",
30 "MIT",
31 "MPL-2.0",
32 "Python-2.0.1",
33 }
34
35 # Properties which are required for our purposes.
36 REQUIRED_PROPERTIES_PACKAGE = frozenset([
37 "SPDXID",
38 "name",
39 "versionInfo",
40 "downloadLocation",
41 "checksums",
42 "licenseConcluded",
43 "externalRefs",
44 "primaryPackagePurpose",
45 ])
46
47
48 class PackageFiles(typing.NamedTuple):
49 """Structure for describing the files of a package"""
50 include: list[str] | None
51 exclude: list[str] | None = None
52
53
54 # SBOMS don't have a method to specify the sources of files
55 # so we need to do that external to the SBOM itself. Add new
56 # values to 'exclude' if we create new files within tracked
57 # directories that aren't sourced from third-party packages.
58 PACKAGE_TO_FILES = {
59 "mpdecimal": PackageFiles(
60 include=["Modules/_decimal/libmpdec/**"]
61 ),
62 "expat": PackageFiles(
63 include=["Modules/expat/**"],
64 exclude=[
65 "Modules/expat/expat_config.h",
66 "Modules/expat/pyexpatns.h",
67 "Modules/expat/refresh.sh",
68 ]
69 ),
70 "macholib": PackageFiles(
71 include=["Lib/ctypes/macholib/**"],
72 exclude=[
73 "Lib/ctypes/macholib/README.ctypes",
74 "Lib/ctypes/macholib/fetch_macholib",
75 "Lib/ctypes/macholib/fetch_macholib.bat",
76 ],
77 ),
78 "libb2": PackageFiles(
79 include=["Modules/_blake2/impl/**"]
80 ),
81 "hacl-star": PackageFiles(
82 include=["Modules/_hacl/**"],
83 exclude=[
84 "Modules/_hacl/refresh.sh",
85 "Modules/_hacl/README.md",
86 "Modules/_hacl/python_hacl_namespace.h",
87 ]
88 ),
89 }
90
91
92 def spdx_id(value: str) -> str:
93 """Encode a value into characters that are valid in an SPDX ID"""
94 return re.sub(r"[^a-zA-Z0-9.\-]+", "-", value)
95
96
97 def error_if(value: bool, error_message: str) -> None:
98 """Prints an error if a comparison fails along with a link to the devguide"""
99 if value:
100 print(error_message)
101 print("See 'https://devguide.python.org/developer-workflow/sbom' for more information.")
102 sys.exit(1)
103
104
105 def is_root_directory_git_index() -> bool:
106 """Checks if the root directory is a git index"""
107 try:
108 subprocess.check_call(
109 ["git", "-C", str(CPYTHON_ROOT_DIR), "rev-parse"],
110 stdout=subprocess.DEVNULL,
111 stderr=subprocess.DEVNULL,
112 )
113 except subprocess.CalledProcessError:
114 return False
115 return True
116
117
118 def filter_gitignored_paths(paths: list[str]) -> list[str]:
119 """
120 Filter out paths excluded by the gitignore file.
121 The output of 'git check-ignore --non-matching --verbose' looks
122 like this for non-matching (included) files:
123
124 '::<whitespace><path>'
125
126 And looks like this for matching (excluded) files:
127
128 '.gitignore:9:*.a Tools/lib.a'
129 """
130 # No paths means no filtering to be done.
131 if not paths:
132 return []
133
134 # Filter out files in gitignore.
135 # Non-matching files show up as '::<whitespace><path>'
136 git_check_ignore_proc = subprocess.run(
137 ["git", "check-ignore", "--verbose", "--non-matching", *paths],
138 cwd=CPYTHON_ROOT_DIR,
139 check=False,
140 stdout=subprocess.PIPE,
141 )
142 # 1 means matches, 0 means no matches.
143 assert git_check_ignore_proc.returncode in (0, 1)
144
145 # Paths may or may not be quoted, Windows quotes paths.
146 git_check_ignore_re = re.compile(r"^::\s+(\"([^\"]+)\"|(.+))\Z")
147
148 # Return the list of paths sorted
149 git_check_ignore_lines = git_check_ignore_proc.stdout.decode().splitlines()
150 git_check_not_ignored = []
151 for line in git_check_ignore_lines:
152 if match := git_check_ignore_re.fullmatch(line):
153 git_check_not_ignored.append(match.group(2) or match.group(3))
154 return sorted(git_check_not_ignored)
155
156
157 def get_externals() -> list[str]:
158 """
159 Parses 'PCbuild/get_externals.bat' for external libraries.
160 Returns a list of (git tag, name, version) tuples.
161 """
162 get_externals_bat_path = CPYTHON_ROOT_DIR / "PCbuild/get_externals.bat"
163 externals = re.findall(
164 r"set\s+libraries\s*=\s*%libraries%\s+([a-zA-Z0-9.-]+)\s",
165 get_externals_bat_path.read_text()
166 )
167 return externals
168
169
170 def download_with_retries(download_location: str,
171 max_retries: int = 7,
172 base_delay: float = 2.25,
173 max_jitter: float = 1.0) -> typing.Any:
174 """Download a file with exponential backoff retry."""
175 for attempt in range(max_retries + 1):
176 try:
177 resp = urllib.request.urlopen(download_location)
178 except (urllib.error.URLError, ConnectionError) as ex:
179 if attempt == max_retries:
180 msg = f"Download from {download_location} failed."
181 raise OSError(msg) from ex
182 time.sleep(base_delay**attempt + random.uniform(0, max_jitter))
183 else:
184 return resp
185
186
187 def check_sbom_packages(sbom_data: dict[str, typing.Any]) -> None:
188 """Make a bunch of assertions about the SBOM package data to ensure it's consistent."""
189
190 for package in sbom_data["packages"]:
191 # Properties and ID must be properly formed.
192 error_if(
193 "name" not in package,
194 "Package is missing the 'name' field"
195 )
196
197 # Verify that the checksum matches the expected value
198 # and that the download URL is valid.
199 if "checksums" not in package or "CI" in os.environ:
200 download_location = package["downloadLocation"]
201 resp = download_with_retries(download_location)
202 error_if(resp.status != 200, f"Couldn't access URL: {download_location}'")
203
204 package["checksums"] = [{
205 "algorithm": "SHA256",
206 "checksumValue": hashlib.sha256(resp.read()).hexdigest()
207 }]
208
209 missing_required_keys = REQUIRED_PROPERTIES_PACKAGE - set(package.keys())
210 error_if(
211 bool(missing_required_keys),
212 f"Package '{package['name']}' is missing required fields: {missing_required_keys}",
213 )
214 error_if(
215 package["SPDXID"] != spdx_id(f"SPDXRef-PACKAGE-{package['name']}"),
216 f"Package '{package['name']}' has a malformed SPDXID",
217 )
218
219 # Version must be in the download and external references.
220 version = package["versionInfo"]
221 error_if(
222 version not in package["downloadLocation"],
223 f"Version '{version}' for package '{package['name']} not in 'downloadLocation' field",
224 )
225 error_if(
226 any(version not in ref["referenceLocator"] for ref in package["externalRefs"]),
227 (
228 f"Version '{version}' for package '{package['name']} not in "
229 f"all 'externalRefs[].referenceLocator' fields"
230 ),
231 )
232
233 # HACL* specifies its expected rev in a refresh script.
234 if package["name"] == "hacl-star":
235 hacl_refresh_sh = (CPYTHON_ROOT_DIR / "Modules/_hacl/refresh.sh").read_text()
236 hacl_expected_rev_match = re.search(
237 r"expected_hacl_star_rev=([0-9a-f]{40})",
238 hacl_refresh_sh
239 )
240 hacl_expected_rev = hacl_expected_rev_match and hacl_expected_rev_match.group(1)
241
242 error_if(
243 hacl_expected_rev != version,
244 "HACL* SBOM version doesn't match value in 'Modules/_hacl/refresh.sh'"
245 )
246
247 # libexpat specifies its expected rev in a refresh script.
248 if package["name"] == "expat":
249 libexpat_refresh_sh = (CPYTHON_ROOT_DIR / "Modules/expat/refresh.sh").read_text()
250 libexpat_expected_version_match = re.search(
251 r"expected_libexpat_version=\"([0-9]+\.[0-9]+\.[0-9]+)\"",
252 libexpat_refresh_sh
253 )
254 libexpat_expected_sha256_match = re.search(
255 r"expected_libexpat_sha256=\"([a-f0-9]{64})\"",
256 libexpat_refresh_sh
257 )
258 libexpat_expected_version = libexpat_expected_version_match and libexpat_expected_version_match.group(1)
259 libexpat_expected_sha256 = libexpat_expected_sha256_match and libexpat_expected_sha256_match.group(1)
260
261 error_if(
262 libexpat_expected_version != version,
263 "libexpat SBOM version doesn't match value in 'Modules/expat/refresh.sh'"
264 )
265 error_if(
266 package["checksums"] != [{
267 "algorithm": "SHA256",
268 "checksumValue": libexpat_expected_sha256
269 }],
270 "libexpat SBOM checksum doesn't match value in 'Modules/expat/refresh.sh'"
271 )
272
273 # License must be on the approved list for SPDX.
274 license_concluded = package["licenseConcluded"]
275 error_if(
276 license_concluded != "NOASSERTION",
277 "License identifier must be 'NOASSERTION'"
278 )
279
280
281 def create_source_sbom() -> None:
282 sbom_path = CPYTHON_ROOT_DIR / "Misc/sbom.spdx.json"
283 sbom_data = json.loads(sbom_path.read_bytes())
284
285 # We regenerate all of this information. Package information
286 # should be preserved though since that is edited by humans.
287 sbom_data["files"] = []
288 sbom_data["relationships"] = []
289
290 # Ensure all packages in this tool are represented also in the SBOM file.
291 actual_names = {package["name"] for package in sbom_data["packages"]}
292 expected_names = set(PACKAGE_TO_FILES)
293 error_if(
294 actual_names != expected_names,
295 f"Packages defined in SBOM tool don't match those defined in SBOM file: {actual_names}, {expected_names}",
296 )
297
298 check_sbom_packages(sbom_data)
299
300 # We call 'sorted()' here a lot to avoid filesystem scan order issues.
301 for name, files in sorted(PACKAGE_TO_FILES.items()):
302 package_spdx_id = spdx_id(f"SPDXRef-PACKAGE-{name}")
303 exclude = files.exclude or ()
304 for include in sorted(files.include or ()):
305 # Find all the paths and then filter them through .gitignore.
306 paths = glob.glob(include, root_dir=CPYTHON_ROOT_DIR, recursive=True)
307 paths = filter_gitignored_paths(paths)
308 error_if(
309 len(paths) == 0,
310 f"No valid paths found at path '{include}' for package '{name}",
311 )
312
313 for path in paths:
314
315 # Normalize the filename from any combination of slashes.
316 path = str(PurePosixPath(PureWindowsPath(path)))
317
318 # Skip directories and excluded files
319 if not (CPYTHON_ROOT_DIR / path).is_file() or path in exclude:
320 continue
321
322 # SPDX requires SHA1 to be used for files, but we provide SHA256 too.
323 data = (CPYTHON_ROOT_DIR / path).read_bytes()
324 # We normalize line-endings for consistent checksums.
325 # This is a rudimentary check for binary files.
326 if b"\x00" not in data:
327 data = data.replace(b"\r\n", b"\n")
328 checksum_sha1 = hashlib.sha1(data).hexdigest()
329 checksum_sha256 = hashlib.sha256(data).hexdigest()
330
331 file_spdx_id = spdx_id(f"SPDXRef-FILE-{path}")
332 sbom_data["files"].append({
333 "SPDXID": file_spdx_id,
334 "fileName": path,
335 "checksums": [
336 {"algorithm": "SHA1", "checksumValue": checksum_sha1},
337 {"algorithm": "SHA256", "checksumValue": checksum_sha256},
338 ],
339 })
340
341 # Tie each file back to its respective package.
342 sbom_data["relationships"].append({
343 "spdxElementId": package_spdx_id,
344 "relatedSpdxElement": file_spdx_id,
345 "relationshipType": "CONTAINS",
346 })
347
348 # Update the SBOM on disk
349 sbom_path.write_text(json.dumps(sbom_data, indent=2, sort_keys=True))
350
351
352 def create_externals_sbom() -> None:
353 sbom_path = CPYTHON_ROOT_DIR / "Misc/externals.spdx.json"
354 sbom_data = json.loads(sbom_path.read_bytes())
355
356 externals = get_externals()
357 externals_name_to_version = {}
358 externals_name_to_git_tag = {}
359 for git_tag in externals:
360 name, _, version = git_tag.rpartition("-")
361 externals_name_to_version[name] = version
362 externals_name_to_git_tag[name] = git_tag
363
364 # Ensure all packages in this tool are represented also in the SBOM file.
365 actual_names = {package["name"] for package in sbom_data["packages"]}
366 expected_names = set(externals_name_to_version)
367 error_if(
368 actual_names != expected_names,
369 f"Packages defined in SBOM tool don't match those defined in SBOM file: {actual_names}, {expected_names}",
370 )
371
372 # Set the versionInfo and downloadLocation fields for all packages.
373 for package in sbom_data["packages"]:
374 package_version = externals_name_to_version[package["name"]]
375
376 # Update the version information in all the locations.
377 package["versionInfo"] = package_version
378 for external_ref in package["externalRefs"]:
379 if external_ref["referenceType"] != "cpe23Type":
380 continue
381 # Version is the fifth field of a CPE.
382 cpe23ref = external_ref["referenceLocator"]
383 external_ref["referenceLocator"] = re.sub(
384 r"\A(cpe(?::[^:]+){4}):[^:]+:",
385 fr"\1:{package_version}:",
386 cpe23ref
387 )
388
389 download_location = (
390 f"https://github.com/python/cpython-source-deps/archive/refs/tags/{externals_name_to_git_tag[package['name']]}.tar.gz"
391 )
392 download_location_changed = download_location != package["downloadLocation"]
393 package["downloadLocation"] = download_location
394
395 # If the download URL has changed we want one to get recalulated.
396 if download_location_changed:
397 package.pop("checksums", None)
398
399 check_sbom_packages(sbom_data)
400
401 # Update the SBOM on disk
402 sbom_path.write_text(json.dumps(sbom_data, indent=2, sort_keys=True))
403
404
405 def main() -> None:
406 # Don't regenerate the SBOM if we're not a git repository.
407 if not is_root_directory_git_index():
408 print("Skipping SBOM generation due to not being a git repository")
409 return
410
411 create_source_sbom()
412 create_externals_sbom()
413
414
415 if __name__ == "__main__":
416 main()