Debian Codegraph @

setup/unix-ci.py

raw
1 #!/usr/bin/env python
2 # License: GPLv3 Copyright: 2017, Kovid Goyal <kovid at kovidgoyal.net>
3
4
5 import glob
6 import io
7 import lzma
8 import os
9 import shlex
10 import subprocess
11 import sys
12 import tarfile
13 import time
14 from tempfile import NamedTemporaryFile
15 from urllib.request import Request, urlopen
16
17 _plat = sys.platform.lower()
18 ismacos = 'darwin' in _plat
19 iswindows = 'win32' in _plat or 'win64' in _plat
20
21
22 def setenv(key, val):
23 os.environ[key] = os.path.expandvars(val)
24
25
26 def download_with_retry(url: str | Request, count: int = 5) -> bytes:
27 for i in range(count):
28 try:
29 print('Downloading', getattr(url, 'full_url', url), flush=True)
30 with urlopen(url) as f:
31 ans: bytes = f.read()
32 return ans
33 except Exception as err:
34 if getattr(err, 'code', -1) == 403:
35 raise
36 if i >= count - 1:
37 raise
38 print(f'Download failed with error {err} retrying...', file=sys.stderr)
39 time.sleep(1)
40 return b''
41
42
43 if ismacos:
44 SWBASE = '/Users/Shared/calibre-build/sw'
45 SW = SWBASE + '/sw'
46
47 def install_env():
48 setenv('SWBASE', SWBASE)
49 setenv('SW', SW)
50 setenv('PATH', '$SW/bin:$SW/qt/bin:$SW/python/Python.framework/Versions/2.7/bin:$PWD/node_modules/.bin:$PATH')
51 setenv('CFLAGS', '-I$SW/include')
52 setenv('LDFLAGS', '-L$SW/lib')
53 setenv('QMAKE', '$SW/qt/bin/qmake')
54 setenv('QTWEBENGINE_DISABLE_SANDBOX', '1')
55 setenv('QT_PLUGIN_PATH', '$SW/qt/plugins')
56 old = os.environ.get('DYLD_FALLBACK_LIBRARY_PATH', '')
57 if old:
58 old += ':'
59 setenv('DYLD_FALLBACK_LIBRARY_PATH', old + '$SW/lib')
60 setenv('CALIBRE_ESPEAK_DATA_DIR', '$SW/share/espeak-ng-data')
61
62 else:
63 SWBASE = '/sw'
64 SW = SWBASE + '/sw'
65
66 def install_env():
67 setenv('SW', SW)
68 setenv('PATH', '$SW/bin:$PATH')
69 setenv('CFLAGS', '-I$SW/include')
70 setenv('LDFLAGS', '-L$SW/lib')
71 setenv('LD_LIBRARY_PATH', '$SW/qt/lib:$SW/ffmpeg/lib:$SW/lib')
72 setenv('PKG_CONFIG_PATH', '$SW/lib/pkgconfig')
73 setenv('QMAKE', '$SW/qt/bin/qmake')
74 setenv('CALIBRE_QT_PREFIX', '$SW/qt')
75 setenv('CALIBRE_ESPEAK_DATA_DIR', '$SW/share/espeak-ng-data')
76
77
78 def do_print_crash_reports() -> None:
79 print('Printing available crash reports...', flush=True)
80 if ismacos:
81 end_time = time.monotonic() + 90
82 while time.monotonic() < end_time:
83 time.sleep(1)
84 items = glob.glob(os.path.join(os.path.expanduser('~/Library/Logs/DiagnosticReports'), 'Python-*.ips'))
85 if items:
86 break
87 if items:
88 time.sleep(1)
89 print(os.path.basename(items[0]), flush=True)
90 sdir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
91 subprocess.check_call([sys.executable, os.path.join(sdir, '.github', 'workflows', 'macos_crash_report.py'), items[0]])
92 else:
93 run('sh -c "echo bt | coredumpctl debug"')
94 print(flush=True)
95
96
97 def run(*args, timeout=1200, print_crash_reports: bool = False):
98 if len(args) == 1:
99 args = shlex.split(args[0])
100 print(' '.join(args), flush=True)
101 p = subprocess.Popen(args)
102 try:
103 ret = p.wait(timeout=timeout)
104 except subprocess.TimeoutExpired as err:
105 ret = 1
106 print(err, file=sys.stderr, flush=True)
107 print('Timed out running:', ' '.join(args), flush=True, file=sys.stderr)
108 p.kill()
109
110 if ret != 0:
111 if print_crash_reports:
112 do_print_crash_reports()
113 raise SystemExit(ret)
114
115
116 def decompress(path, dest, compression):
117 run('tar', 'x' + compression + 'f', path, '-C', dest)
118
119
120 def download_and_decompress(url, dest, compression=None):
121 if compression is None:
122 compression = 'j' if url.endswith('.bz2') else 'J'
123 for i in range(5):
124 print('Downloading', url, '...')
125 with NamedTemporaryFile() as f:
126 ret = subprocess.Popen(['curl', '-fSL', url], stdout=f).wait()
127 if ret == 0:
128 decompress(f.name, dest, compression)
129 sys.stdout.flush(), sys.stderr.flush()
130 return
131 time.sleep(1)
132 raise SystemExit('Failed to download ' + url)
133
134
135 def install_qt_source_code():
136 dest = os.path.expanduser('~/qt-base')
137 os.mkdir(dest)
138 download_and_decompress('https://download.calibre-ebook.com/qtbase-everywhere-src-6.4.2.tar.xz', dest, 'J')
139 qdir = glob.glob(dest + '/*')[0]
140 os.environ['QT_SRC'] = qdir
141
142
143 def run_python(*args, timeout=1200, print_crash_reports: bool = True):
144 python = os.path.expandvars('$SW/bin/python')
145 if len(args) == 1:
146 args = shlex.split(args[0])
147 args = [python] + list(args)
148 run(*args, timeout=timeout, print_crash_reports=print_crash_reports)
149
150
151 def runtime_env():
152 """The environment needed to run calibre code, as opposed to merely build it."""
153 os.environ['CI'] = 'true'
154 os.environ['OPENSSL_MODULES'] = os.path.join(SW, 'lib', 'ossl-modules')
155 os.environ['PIPER_TTS_DIR'] = os.path.join(SW, 'piper')
156 # The bundled OpenSSL is deliberately built with an ssl dir that does not exist
157 # on the CI runners, so without this, any HTTPS connection made from calibre code
158 # fails with CERTIFICATE_VERIFY_FAILED. The certs are generated by the cacerts
159 # sub-command of bootstrap, which is run before this.
160 cadir = os.path.abspath('resources/mozilla-ca-certs')
161 if not os.path.isdir(cadir):
162 raise SystemExit(f'The Mozilla CA certs are missing from: {cadir}')
163 os.environ['SSL_CERT_DIR'] = cadir
164 if ismacos:
165 # needed to ensure correct libxml2 is loaded
166 os.environ['DYLD_INSERT_LIBRARIES'] = ':'.join(os.path.join(SW, 'lib', x) for x in 'libxml2.dylib libxslt.dylib libexslt.dylib'.split())
167 os.environ['OPENSSL_ENGINES'] = os.path.join(SW, 'lib', 'engines-3')
168 install_env()
169
170
171 def install_linux_deps():
172 run('sudo', 'apt-get', 'update', '-y')
173 # run('sudo', 'apt-get', 'upgrade', '-y')
174 run(
175 'sudo',
176 'apt-get',
177 'install',
178 '-y',
179 'gettext',
180 'libgl1-mesa-dev',
181 'libxkbcommon-dev',
182 'libxkbcommon-x11-dev',
183 'libfreetype-dev',
184 'pulseaudio',
185 'libasound2t64',
186 'libflite1',
187 'libspeechd2',
188 # needed by the Camoufox browser, which is a fork of Firefox
189 'libdbus-glib-1-2',
190 'libgtk-3-0t64',
191 )
192
193
194 def get_tx():
195 url = 'https://github.com/transifex/cli/releases/latest/download/tx-linux-amd64.tar.gz'
196 print('Downloading:', url)
197 raw = download_with_retry(url)
198 with tarfile.open(fileobj=io.BytesIO(raw), mode='r') as tf:
199 tf.extract('tx', filter='fully_trusted')
200
201
202 def install_grype(exe: str = '/tmp/grype') -> str:
203 raw = download_with_retry('https://download.calibre-ebook.com/ci/grype.xz')
204 raw = lzma.decompress(raw)
205 with open(exe, 'wb') as f:
206 f.write(raw)
207 os.fchmod(f.fileno(), 0o755)
208 subprocess.check_call([exe, 'db', 'update'])
209 return exe
210
211
212 IGNORED_DEPENDENCY_CVES = [
213 # python stdlib all these are erroneously marked as fixed in python 3.15
214 # when it hasnt even been released. Sigh.
215 'CVE-2026-1299',
216 'CVE-2026-0865',
217 'CVE-2026-0672',
218 'CVE-2025-15282',
219 'CVE-2025-15366',
220 'CVE-2025-15367',
221 'CVE-2025-12781',
222 'CVE-2025-11468',
223 'CVE-2026-2297',
224 'CVE-2026-3644',
225 'CVE-2026-1502',
226 'CVE-2026-4224', # expat parser unused
227 'CVE-2026-4519', # webbrowser() unused
228 'CVE-2026-7210', # DoS in unused XML parser
229 'CVE-2026-3276', # DoS in unicodedata.normalize()
230 'CVE-2026-7774', # tarfile.data_filter path traversal bypass
231 'CVE-2026-12003', # bug in release builds irrelevant to us
232 'CVE-2026-15308', # bug in stdlib html parser unused by us
233 'CVE-2026-11940', # tarfile, unused
234 'CVE-2026-11972', # tarfile, unused
235 'CVE-2026-0864', # configparser unused
236 'CVE-2026-17084', # idna encoding, unused
237 'CVE-2026-15806', # HTTPPasswordMgr unused
238 'CVE-2026-15310', # DoS in unzip
239 # libtiff
240 'CVE-2025-8851', # this is erroneously marked as fixed in the database but no release of libtiff has been made with the fix
241 # hyphen
242 'CVE-2017-1000376', # false match in the database
243 # espeak
244 'CVE-2023-4990', # false match because we currently build with a specific commit pending release of espeak 1.53
245 'CVE-2026-2673', # openssl fix not released
246 'CVE-2026-14456', # openssl fix not released
247 ]
248 IGNORED_DEPENDENCY_PACKAGES = [
249 'nodejs', # only used at build time
250 'ffmpeg', # cannot be updated till we update Qt
251 ]
252
253
254 LINUX_BUNDLE = 'linux-64'
255 MACOS_BUNDLE = 'macos-64'
256 WINDOWS_BUNDLE = 'windows-64'
257
258
259 def install_bundle(dest=SW, which=''):
260 run('sudo', 'mkdir', '-p', dest)
261 run('sudo', 'chown', '-R', os.environ['USER'], SWBASE)
262 tball = which or (MACOS_BUNDLE if ismacos else LINUX_BUNDLE)
263 download_and_decompress(f'https://download.calibre-ebook.com/ci/calibre7/{tball}.tar.xz', dest)
264
265
266 def check_dependencies() -> None:
267 dest = os.path.join(SW, LINUX_BUNDLE)
268 install_bundle(dest, os.path.basename(dest))
269 dest = os.path.join(SW, MACOS_BUNDLE)
270 install_bundle(dest, os.path.basename(dest))
271 dest = os.path.join(SW, WINDOWS_BUNDLE)
272 install_bundle(dest, os.path.basename(dest))
273 grype = install_grype()
274 with open((gc := os.path.expanduser('~/.grype.yml')), 'w') as f:
275 print('ignore:', file=f)
276 for x in IGNORED_DEPENDENCY_CVES:
277 print(' - vulnerability:', x, file=f)
278 for x in IGNORED_DEPENDENCY_PACKAGES:
279 print(' - package:', file=f)
280 print(f' name: {x}', file=f)
281 cmdline = [grype, '--by-cve', '--config', gc, '--fail-on', 'medium', '--only-fixed', '--add-cpes-if-none']
282 # disable testing against dir as it raises false positives on sqlite
283 # embedded in dependencies we dont use at runtime
284 # print('Testing against the bundle directories', flush=True)
285 # if (cp := subprocess.run(cmdline + ['dir:' + SW])).returncode != 0:
286 # raise SystemExit(cp.returncode)
287 # Test against the SBOM
288 print('Testing against the SBOM', flush=True)
289 import runpy
290
291 orig = sys.argv, sys.stdout
292 sys.argv = ['bypy', 'sbom', 'kovidgoyal/calibre', '1.0.0']
293 buf = io.StringIO()
294 sys.stdout = buf
295 runpy.run_path('bypy-src')
296 sys.argv, sys.stdout = orig
297 print(buf.getvalue())
298 if (cp := subprocess.run(cmdline, input=buf.getvalue().encode())).returncode != 0:
299 raise SystemExit(cp.returncode)
300
301
302 def main():
303 action = sys.argv[1]
304
305 if action == 'install':
306 # WebEngine is flaky in macOS CI so install rapydscript so bootstrap wont fail
307 import sysconfig
308
309 run(sys.executable, '-m', 'pip', 'install', 'rapydscript-ng')
310 with open(os.environ['GITHUB_PATH'], 'a') as f:
311 print(sysconfig.get_path('scripts'), file=f)
312
313 if iswindows:
314 import runpy
315
316 m = runpy.run_path('setup/win-ci.py')
317 return m['main']()
318
319 if action == 'install':
320 install_bundle()
321 if not ismacos:
322 install_linux_deps()
323
324 elif action == 'bootstrap':
325 run('rapydscript', '--version')
326 install_env()
327 run_python('setup.py bootstrap --ephemeral')
328
329 elif action == 'check-dependencies':
330 check_dependencies()
331
332 elif action == 'pot':
333 transifexrc = '''\
334 [https://www.transifex.com]
335 api_hostname = https://api.transifex.com
336 rest_hostname = https://rest.api.transifex.com
337 hostname = https://www.transifex.com
338 password = PASSWORD
339 token = PASSWORD
340 username = api
341 '''.replace('PASSWORD', os.environ['tx'])
342 with open(os.path.expanduser('~/.transifexrc'), 'w') as f:
343 f.write(transifexrc)
344 install_qt_source_code()
345 install_env()
346 get_tx()
347 os.environ['TX'] = os.path.abspath('tx')
348 run(sys.executable, 'setup.py', 'pot', timeout=30 * 60)
349 elif action == 'browser-deps':
350 runtime_env()
351 # Hundreds of megabytes of browser, so allow plenty of time for it
352 run_python('setup.py browser_deps', timeout=1800, print_crash_reports=False)
353
354 elif action == 'test':
355 runtime_env()
356 run_python('setup.py test')
357 if not ismacos: # webengine is flaky on macOS
358 run_python('setup.py test_rs')
359 else:
360 raise SystemExit(f'Unknown action: {action}')
361
362
363 if __name__ == '__main__':
364 main()