Debian Codegraph @

Tools/build/generate_sbom.py

raw
1 """Tool for generating Software Bill of Materials (SBOM) for Python's dependencies"""
2
3 import glob
4 import hashlib
5 import json
6 import os
7 import random
8 import re
9 import subprocess
10 import sys
11 import time
12 import typing
13 import urllib.error
14 import urllib.request
15 from pathlib import Path, PurePosixPath, PureWindowsPath
16
17 CPYTHON_ROOT_DIR = Path(__file__).parent.parent.parent
18
19 # Before adding a new entry to this list, double check that
20 # the license expression is a valid SPDX license expression:
21 # See: https://spdx.org/licenses
22 ALLOWED_LICENSE_EXPRESSIONS = {
23 "Apache-2.0",
24 "Apache-2.0 OR BSD-2-Clause",
25 "BSD-2-Clause",
26 "BSD-3-Clause",
27 "CC0-1.0",
28 "ISC",
29 "LGPL-2.1-only",
30 "MIT",
31 "MPL-2.0",
32 "Python-2.0.1",
33 }
34
35 # Properties which are required for our purposes.
36 REQUIRED_PROPERTIES_PACKAGE = frozenset([
37 "SPDXID",
38 "name",
39 "versionInfo",
40 "downloadLocation",
41 "checksums",
42 "licenseConcluded",
43 "externalRefs",
44 "primaryPackagePurpose",
45 ])
46
47
48 class PackageFiles(typing.NamedTuple):
49 """Structure for describing the files of a package"""
50 include: list[str] | None
51 exclude: list[str] | None = None
52
53
54 # SBOMS don't have a method to specify the sources of files
55 # so we need to do that external to the SBOM itself. Add new
56 # values to 'exclude' if we create new files within tracked
57 # directories that aren't sourced from third-party packages.
58 PACKAGE_TO_FILES = {
59 "mpdecimal": PackageFiles(
60 include=["Modules/_decimal/libmpdec/**"]
61 ),
62 "expat": PackageFiles(
63 include=["Modules/expat/**"],
64 exclude=[
65 "Modules/expat/expat_config.h",
66 "Modules/expat/pyexpatns.h",
67 "Modules/expat/refresh.sh",
68 ]
69 ),
70 "macholib": PackageFiles(
71 include=["Lib/ctypes/macholib/**"],
72 exclude=[
73 "Lib/ctypes/macholib/README.ctypes",
74 "Lib/ctypes/macholib/fetch_macholib",
75 "Lib/ctypes/macholib/fetch_macholib.bat",
76 ],
77 ),
78 "hacl-star": PackageFiles(
79 include=["Modules/_hacl/**"],
80 exclude=[
81 "Modules/_hacl/refresh.sh",
82 "Modules/_hacl/README.md",
83 "Modules/_hacl/python_hacl_namespace.h",
84 ]
85 ),
86 }
87
88
89 def spdx_id(value: str) -> str:
90 """Encode a value into characters that are valid in an SPDX ID"""
91 return re.sub(r"[^a-zA-Z0-9.\-]+", "-", value)
92
93
94 def error_if(value: bool, error_message: str) -> None:
95 """Prints an error if a comparison fails along with a link to the devguide"""
96 if value:
97 print(error_message)
98 print("See 'https://devguide.python.org/developer-workflow/sbom' for more information.")
99 sys.exit(1)
100
101
102 def is_root_directory_git_index() -> bool:
103 """Checks if the root directory is a git index"""
104 try:
105 subprocess.check_call(
106 ["git", "-C", str(CPYTHON_ROOT_DIR), "rev-parse"],
107 stdout=subprocess.DEVNULL,
108 stderr=subprocess.DEVNULL,
109 )
110 except subprocess.CalledProcessError:
111 return False
112 return True
113
114
115 def filter_gitignored_paths(paths: list[str]) -> list[str]:
116 """
117 Filter out paths excluded by the gitignore file.
118 The output of 'git check-ignore --non-matching --verbose' looks
119 like this for non-matching (included) files:
120
121 '::<whitespace><path>'
122
123 And looks like this for matching (excluded) files:
124
125 '.gitignore:9:*.a Tools/lib.a'
126 """
127 # No paths means no filtering to be done.
128 if not paths:
129 return []
130
131 # Filter out files in gitignore.
132 # Non-matching files show up as '::<whitespace><path>'
133 git_check_ignore_proc = subprocess.run(
134 ["git", "check-ignore", "--verbose", "--non-matching", *paths],
135 cwd=CPYTHON_ROOT_DIR,
136 check=False,
137 stdout=subprocess.PIPE,
138 )
139 # 1 means matches, 0 means no matches.
140 assert git_check_ignore_proc.returncode in (0, 1)
141
142 # Paths may or may not be quoted, Windows quotes paths.
143 git_check_ignore_re = re.compile(r"^::\s+(\"([^\"]+)\"|(.+))\Z")
144
145 # Return the list of paths sorted
146 git_check_ignore_lines = git_check_ignore_proc.stdout.decode().splitlines()
147 git_check_not_ignored = []
148 for line in git_check_ignore_lines:
149 if match := git_check_ignore_re.fullmatch(line):
150 git_check_not_ignored.append(match.group(2) or match.group(3))
151 return sorted(git_check_not_ignored)
152
153
154 def get_externals() -> list[str]:
155 """
156 Parses 'PCbuild/get_externals.bat' for external libraries.
157 Returns a list of (git tag, name, version) tuples.
158 """
159 get_externals_bat_path = CPYTHON_ROOT_DIR / "PCbuild/get_externals.bat"
160 externals = re.findall(
161 r"set\s+libraries\s*=\s*%libraries%\s+([a-zA-Z0-9.-]+)\s",
162 get_externals_bat_path.read_text()
163 )
164 return externals
165
166
167 def download_with_retries(download_location: str,
168 max_retries: int = 7,
169 base_delay: float = 2.25,
170 max_jitter: float = 1.0) -> typing.Any:
171 """Download a file with exponential backoff retry."""
172 for attempt in range(max_retries + 1):
173 try:
174 resp = urllib.request.urlopen(download_location)
175 except (urllib.error.URLError, ConnectionError) as ex:
176 if attempt == max_retries:
177 msg = f"Download from {download_location} failed."
178 raise OSError(msg) from ex
179 time.sleep(base_delay**attempt + random.uniform(0, max_jitter))
180 else:
181 return resp
182
183
184 def check_sbom_packages(sbom_data: dict[str, typing.Any]) -> None:
185 """Make a bunch of assertions about the SBOM package data to ensure it's consistent."""
186
187 for package in sbom_data["packages"]:
188 # Properties and ID must be properly formed.
189 error_if(
190 "name" not in package,
191 "Package is missing the 'name' field"
192 )
193
194 # Verify that the checksum matches the expected value
195 # and that the download URL is valid.
196 if "checksums" not in package or "CI" in os.environ:
197 download_location = package["downloadLocation"]
198 resp = download_with_retries(download_location)
199 error_if(resp.status != 200, f"Couldn't access URL: {download_location}'")
200
201 package["checksums"] = [{
202 "algorithm": "SHA256",
203 "checksumValue": hashlib.sha256(resp.read()).hexdigest()
204 }]
205
206 missing_required_keys = REQUIRED_PROPERTIES_PACKAGE - set(package.keys())
207 error_if(
208 bool(missing_required_keys),
209 f"Package '{package['name']}' is missing required fields: {missing_required_keys}",
210 )
211 error_if(
212 package["SPDXID"] != spdx_id(f"SPDXRef-PACKAGE-{package['name']}"),
213 f"Package '{package['name']}' has a malformed SPDXID",
214 )
215
216 # Version must be in the download and external references.
217 version = package["versionInfo"]
218 error_if(
219 version not in package["downloadLocation"],
220 f"Version '{version}' for package '{package['name']} not in 'downloadLocation' field",
221 )
222 error_if(
223 any(version not in ref["referenceLocator"] for ref in package["externalRefs"]),
224 (
225 f"Version '{version}' for package '{package['name']} not in "
226 f"all 'externalRefs[].referenceLocator' fields"
227 ),
228 )
229
230 # HACL* specifies its expected rev in a refresh script.
231 if package["name"] == "hacl-star":
232 hacl_refresh_sh = (CPYTHON_ROOT_DIR / "Modules/_hacl/refresh.sh").read_text()
233 hacl_expected_rev_match = re.search(
234 r"expected_hacl_star_rev=([0-9a-f]{40})",
235 hacl_refresh_sh
236 )
237 hacl_expected_rev = hacl_expected_rev_match and hacl_expected_rev_match.group(1)
238
239 error_if(
240 hacl_expected_rev != version,
241 "HACL* SBOM version doesn't match value in 'Modules/_hacl/refresh.sh'"
242 )
243
244 # libexpat specifies its expected rev in a refresh script.
245 if package["name"] == "expat":
246 libexpat_refresh_sh = (CPYTHON_ROOT_DIR / "Modules/expat/refresh.sh").read_text()
247 libexpat_expected_version_match = re.search(
248 r"expected_libexpat_version=\"([0-9]+\.[0-9]+\.[0-9]+)\"",
249 libexpat_refresh_sh
250 )
251 libexpat_expected_sha256_match = re.search(
252 r"expected_libexpat_sha256=\"([a-f0-9]{64})\"",
253 libexpat_refresh_sh
254 )
255 libexpat_expected_version = libexpat_expected_version_match and libexpat_expected_version_match.group(1)
256 libexpat_expected_sha256 = libexpat_expected_sha256_match and libexpat_expected_sha256_match.group(1)
257
258 error_if(
259 libexpat_expected_version != version,
260 "libexpat SBOM version doesn't match value in 'Modules/expat/refresh.sh'"
261 )
262 error_if(
263 package["checksums"] != [{
264 "algorithm": "SHA256",
265 "checksumValue": libexpat_expected_sha256
266 }],
267 "libexpat SBOM checksum doesn't match value in 'Modules/expat/refresh.sh'"
268 )
269
270 # License must be on the approved list for SPDX.
271 license_concluded = package["licenseConcluded"]
272 error_if(
273 license_concluded != "NOASSERTION",
274 "License identifier must be 'NOASSERTION'"
275 )
276
277
278 def create_source_sbom() -> None:
279 sbom_path = CPYTHON_ROOT_DIR / "Misc/sbom.spdx.json"
280 sbom_data = json.loads(sbom_path.read_bytes())
281
282 # We regenerate all of this information. Package information
283 # should be preserved though since that is edited by humans.
284 sbom_data["files"] = []
285 sbom_data["relationships"] = []
286
287 # Ensure all packages in this tool are represented also in the SBOM file.
288 actual_names = {package["name"] for package in sbom_data["packages"]}
289 expected_names = set(PACKAGE_TO_FILES)
290 error_if(
291 actual_names != expected_names,
292 f"Packages defined in SBOM tool don't match those defined in SBOM file: {actual_names}, {expected_names}",
293 )
294
295 check_sbom_packages(sbom_data)
296
297 # We call 'sorted()' here a lot to avoid filesystem scan order issues.
298 for name, files in sorted(PACKAGE_TO_FILES.items()):
299 package_spdx_id = spdx_id(f"SPDXRef-PACKAGE-{name}")
300 exclude = files.exclude or ()
301 for include in sorted(files.include or ()):
302 # Find all the paths and then filter them through .gitignore.
303 paths = glob.glob(include, root_dir=CPYTHON_ROOT_DIR, recursive=True)
304 paths = filter_gitignored_paths(paths)
305 error_if(
306 len(paths) == 0,
307 f"No valid paths found at path '{include}' for package '{name}",
308 )
309
310 for path in paths:
311
312 # Normalize the filename from any combination of slashes.
313 path = str(PurePosixPath(PureWindowsPath(path)))
314
315 # Skip directories and excluded files
316 if not (CPYTHON_ROOT_DIR / path).is_file() or path in exclude:
317 continue
318
319 # SPDX requires SHA1 to be used for files, but we provide SHA256 too.
320 data = (CPYTHON_ROOT_DIR / path).read_bytes()
321 # We normalize line-endings for consistent checksums.
322 # This is a rudimentary check for binary files.
323 if b"\x00" not in data:
324 data = data.replace(b"\r\n", b"\n")
325 checksum_sha1 = hashlib.sha1(data).hexdigest()
326 checksum_sha256 = hashlib.sha256(data).hexdigest()
327
328 file_spdx_id = spdx_id(f"SPDXRef-FILE-{path}")
329 sbom_data["files"].append({
330 "SPDXID": file_spdx_id,
331 "fileName": path,
332 "checksums": [
333 {"algorithm": "SHA1", "checksumValue": checksum_sha1},
334 {"algorithm": "SHA256", "checksumValue": checksum_sha256},
335 ],
336 })
337
338 # Tie each file back to its respective package.
339 sbom_data["relationships"].append({
340 "spdxElementId": package_spdx_id,
341 "relatedSpdxElement": file_spdx_id,
342 "relationshipType": "CONTAINS",
343 })
344
345 # Update the SBOM on disk
346 sbom_path.write_text(json.dumps(sbom_data, indent=2, sort_keys=True))
347
348
349 def create_externals_sbom() -> None:
350 sbom_path = CPYTHON_ROOT_DIR / "Misc/externals.spdx.json"
351 sbom_data = json.loads(sbom_path.read_bytes())
352
353 externals = get_externals()
354 externals_name_to_version = {}
355 externals_name_to_git_tag = {}
356 for git_tag in externals:
357 name, _, version = git_tag.rpartition("-")
358 externals_name_to_version[name] = version
359 externals_name_to_git_tag[name] = git_tag
360
361 # Ensure all packages in this tool are represented also in the SBOM file.
362 actual_names = {package["name"] for package in sbom_data["packages"]}
363 expected_names = set(externals_name_to_version)
364 error_if(
365 actual_names != expected_names,
366 f"Packages defined in SBOM tool don't match those defined in SBOM file: {actual_names}, {expected_names}",
367 )
368
369 # Set the versionInfo and downloadLocation fields for all packages.
370 for package in sbom_data["packages"]:
371 package_version = externals_name_to_version[package["name"]]
372
373 # Update the version information in all the locations.
374 package["versionInfo"] = package_version
375 for external_ref in package["externalRefs"]:
376 if external_ref["referenceType"] != "cpe23Type":
377 continue
378 # Version is the fifth field of a CPE.
379 cpe23ref = external_ref["referenceLocator"]
380 external_ref["referenceLocator"] = re.sub(
381 r"\A(cpe(?::[^:]+){4}):[^:]+:",
382 fr"\1:{package_version}:",
383 cpe23ref
384 )
385
386 download_location = (
387 f"https://github.com/python/cpython-source-deps/archive/refs/tags/{externals_name_to_git_tag[package['name']]}.tar.gz"
388 )
389 download_location_changed = download_location != package["downloadLocation"]
390 package["downloadLocation"] = download_location
391
392 # If the download URL has changed we want one to get recalulated.
393 if download_location_changed:
394 package.pop("checksums", None)
395
396 check_sbom_packages(sbom_data)
397
398 # Update the SBOM on disk
399 sbom_path.write_text(json.dumps(sbom_data, indent=2, sort_keys=True))
400
401
402 def main() -> None:
403 # Don't regenerate the SBOM if we're not a git repository.
404 if not is_root_directory_git_index():
405 print("Skipping SBOM generation due to not being a git repository")
406 return
407
408 create_source_sbom()
409 create_externals_sbom()
410
411
412 if __name__ == "__main__":
413 main()