Debian Codegraph @

Tools/ssl/multissltests.py

raw
1 #!./python
2 """Run Python tests against multiple installations of OpenSSL and LibreSSL
3
4 The script
5
6 (1) downloads OpenSSL / LibreSSL tar bundle
7 (2) extracts it to ./src
8 (3) compiles OpenSSL / LibreSSL
9 (4) installs OpenSSL / LibreSSL into ../multissl/$LIB/$VERSION/
10 (5) forces a recompilation of Python modules using the
11 header and library files from ../multissl/$LIB/$VERSION/
12 (6) runs Python's test suite
13
14 The script must be run with Python's build directory as current working
15 directory.
16
17 The script uses LD_RUN_PATH, LD_LIBRARY_PATH, CPPFLAGS and LDFLAGS to bend
18 search paths for header files and shared libraries. It's known to work on
19 Linux with GCC and clang.
20
21 Please keep this script compatible with Python 2.7, and 3.4 to 3.7.
22
23 (c) 2013-2017 Christian Heimes <[email protected]>
24 """
25 from __future__ import print_function
26
27 import argparse
28 from datetime import datetime
29 import logging
30 import os
31 try:
32 from urllib.request import urlopen
33 from urllib.error import HTTPError
34 except ImportError:
35 from urllib2 import urlopen, HTTPError
36 import re
37 import shutil
38 import subprocess
39 import sys
40 import tarfile
41
42
43 log = logging.getLogger("multissl")
44
45 OPENSSL_OLD_VERSIONS = [
46 "1.1.1w",
47 "3.1.8",
48 "3.2.6",
49 ]
50
51 OPENSSL_RECENT_VERSIONS = [
52 "3.0.20",
53 "3.3.7",
54 "3.4.5",
55 "3.5.6",
56 "3.6.2",
57 # See make_ssl_data.py for notes on adding a new version.
58 ]
59
60 LIBRESSL_OLD_VERSIONS = [
61 ]
62
63 LIBRESSL_RECENT_VERSIONS = [
64 ]
65
66 # store files in ../multissl
67 HERE = os.path.dirname(os.path.abspath(__file__))
68 PYTHONROOT = os.path.abspath(os.path.join(HERE, '..', '..'))
69 MULTISSL_DIR = os.path.abspath(os.path.join(PYTHONROOT, '..', 'multissl'))
70
71
72 parser = argparse.ArgumentParser(
73 prog='multissl',
74 description=(
75 "Run CPython tests with multiple cryptography libraries/versions."
76 ),
77 )
78 parser.add_argument(
79 '--debug',
80 action='store_true',
81 help="Enable debug logging",
82 )
83 parser.add_argument(
84 '--disable-ancient',
85 action='store_true',
86 help="Don't test OpenSSL and LibreSSL versions without upstream support",
87 )
88 parser.add_argument(
89 '--openssl',
90 nargs='+',
91 default=(),
92 help=(
93 "OpenSSL versions, defaults to '{}' (ancient: '{}') if no "
94 "OpenSSL and LibreSSL versions are given."
95 ).format(OPENSSL_RECENT_VERSIONS, OPENSSL_OLD_VERSIONS)
96 )
97 parser.add_argument(
98 '--libressl',
99 nargs='+',
100 default=(),
101 help=(
102 "LibreSSL versions, defaults to '{}' (ancient: '{}') if no "
103 "OpenSSL and LibreSSL versions are given."
104 ).format(LIBRESSL_RECENT_VERSIONS, LIBRESSL_OLD_VERSIONS)
105 )
106 parser.add_argument(
107 '--tests',
108 nargs='*',
109 default=(),
110 help="Python tests to run, defaults to all SSL related tests.",
111 )
112 parser.add_argument(
113 '--base-directory',
114 default=MULTISSL_DIR,
115 help="Base directory for OpenSSL / LibreSSL sources and builds."
116 )
117 parser.add_argument(
118 '--no-network',
119 action='store_false',
120 dest='network',
121 help="Disable network tests."
122 )
123 parser.add_argument(
124 '--steps',
125 choices=['library', 'modules', 'tests'],
126 default='tests',
127 help=(
128 "Which steps to perform. 'library' downloads and compiles OpenSSL "
129 "or LibreSSL. 'module' also compiles Python modules. 'tests' builds "
130 "all and runs the test suite."
131 )
132 )
133 parser.add_argument(
134 '--system',
135 default='',
136 help="Override the automatic system type detection."
137 )
138 parser.add_argument(
139 '--force',
140 action='store_true',
141 dest='force',
142 help="Force build and installation."
143 )
144 parser.add_argument(
145 '--keep-sources',
146 action='store_true',
147 dest='keep_sources',
148 help="Keep original sources for debugging."
149 )
150
151
152 class AbstractBuilder(object):
153 library = None
154 url_templates = None
155 src_template = None
156 build_template = None
157 depend_target = None
158 install_target = 'install'
159 if hasattr(os, 'process_cpu_count'):
160 jobs = os.process_cpu_count()
161 else:
162 jobs = os.cpu_count()
163
164 module_files = (
165 os.path.join(PYTHONROOT, "Modules/_ssl.c"),
166 os.path.join(PYTHONROOT, "Modules/_hashopenssl.c"),
167 )
168 module_libs = ("_ssl", "_hashlib")
169
170 def __init__(self, version, args):
171 self.version = version
172 self.args = args
173 # installation directory
174 self.install_dir = os.path.join(
175 os.path.join(args.base_directory, self.library.lower()), version
176 )
177 # source file
178 self.src_dir = os.path.join(args.base_directory, 'src')
179 self.src_file = os.path.join(
180 self.src_dir, self.src_template.format(version))
181 # build directory (removed after install)
182 self.build_dir = os.path.join(
183 self.src_dir, self.build_template.format(version))
184 self.system = args.system
185
186 def __str__(self):
187 return "<{0.__class__.__name__} for {0.version}>".format(self)
188
189 def __eq__(self, other):
190 if not isinstance(other, AbstractBuilder):
191 return NotImplemented
192 return (
193 self.library == other.library
194 and self.version == other.version
195 )
196
197 def __hash__(self):
198 return hash((self.library, self.version))
199
200 @property
201 def short_version(self):
202 """Short version for OpenSSL download URL"""
203 return None
204
205 @property
206 def openssl_cli(self):
207 """openssl CLI binary"""
208 return os.path.join(self.install_dir, "bin", "openssl")
209
210 @property
211 def openssl_version(self):
212 """output of 'bin/openssl version'"""
213 cmd = [self.openssl_cli, "version"]
214 return self._subprocess_output(cmd)
215
216 @property
217 def pyssl_version(self):
218 """Value of ssl.OPENSSL_VERSION"""
219 cmd = [
220 sys.executable,
221 '-c', 'import ssl; print(ssl.OPENSSL_VERSION)'
222 ]
223 return self._subprocess_output(cmd)
224
225 @property
226 def include_dir(self):
227 return os.path.join(self.install_dir, "include")
228
229 @property
230 def lib_dir(self):
231 return os.path.join(self.install_dir, "lib")
232
233 @property
234 def has_openssl(self):
235 return os.path.isfile(self.openssl_cli)
236
237 @property
238 def has_src(self):
239 return os.path.isfile(self.src_file)
240
241 def _subprocess_call(self, cmd, env=None, **kwargs):
242 log.debug("Call '{}'".format(" ".join(cmd)))
243 return subprocess.check_call(cmd, env=env, **kwargs)
244
245 def _subprocess_output(self, cmd, env=None, **kwargs):
246 log.debug("Call '{}'".format(" ".join(cmd)))
247 if env is None:
248 env = os.environ.copy()
249 env["LD_LIBRARY_PATH"] = self.lib_dir
250 out = subprocess.check_output(cmd, env=env, **kwargs)
251 return out.strip().decode("utf-8")
252
253 def _download_src(self):
254 """Download sources"""
255 src_dir = os.path.dirname(self.src_file)
256 if not os.path.isdir(src_dir):
257 os.makedirs(src_dir)
258 data = None
259 for url_template in self.url_templates:
260 url = url_template.format(v=self.version, s=self.short_version)
261 log.info("Downloading from {}".format(url))
262 try:
263 req = urlopen(url)
264 # KISS, read all, write all
265 data = req.read()
266 except HTTPError as e:
267 log.error(
268 "Download from {} has from failed: {}".format(url, e)
269 )
270 else:
271 log.info("Successfully downloaded from {}".format(url))
272 break
273 if data is None:
274 raise ValueError("All download URLs have failed")
275 log.info("Storing {}".format(self.src_file))
276 with open(self.src_file, "wb") as f:
277 f.write(data)
278
279 def _unpack_src(self):
280 """Unpack tar.gz bundle"""
281 # cleanup
282 if os.path.isdir(self.build_dir):
283 shutil.rmtree(self.build_dir)
284 os.makedirs(self.build_dir)
285
286 tf = tarfile.open(self.src_file)
287 name = self.build_template.format(self.version)
288 base = name + '/'
289 # force extraction into build dir
290 members = tf.getmembers()
291 for member in list(members):
292 if member.name == name:
293 members.remove(member)
294 elif not member.name.startswith(base):
295 raise ValueError(member.name, base)
296 member.name = member.name[len(base):].lstrip('/')
297 log.info("Unpacking files to {}".format(self.build_dir))
298 tf.extractall(self.build_dir, members, filter='data')
299
300 def _build_src(self, config_args=()):
301 """Now build openssl"""
302 log.info("Running build in {}".format(self.build_dir))
303 cwd = self.build_dir
304 cmd = [
305 "./config", *config_args,
306 "shared", "--debug",
307 "--prefix={}".format(self.install_dir)
308 ]
309 # cmd.extend(["no-deprecated", "--api=1.1.0"])
310 env = os.environ.copy()
311 # set rpath
312 env["LD_RUN_PATH"] = self.lib_dir
313 if self.system:
314 env['SYSTEM'] = self.system
315 self._subprocess_call(cmd, cwd=cwd, env=env)
316 if self.depend_target:
317 self._subprocess_call(
318 ["make", "-j1", self.depend_target], cwd=cwd, env=env
319 )
320 self._subprocess_call(["make", f"-j{self.jobs}"], cwd=cwd, env=env)
321
322 def _make_install(self):
323 self._subprocess_call(
324 ["make", "-j1", self.install_target],
325 cwd=self.build_dir
326 )
327 self._post_install()
328 if not self.args.keep_sources:
329 shutil.rmtree(self.build_dir)
330
331 def _post_install(self):
332 pass
333
334 def install(self):
335 log.info(self.openssl_cli)
336 if not self.has_openssl or self.args.force:
337 if not self.has_src:
338 self._download_src()
339 else:
340 log.debug("Already has src {}".format(self.src_file))
341 self._unpack_src()
342 self._build_src()
343 self._make_install()
344 else:
345 log.info("Already has installation {}".format(self.install_dir))
346 # validate installation
347 version = self.openssl_version
348 if self.version not in version:
349 raise ValueError(version)
350
351 def recompile_pymods(self):
352 log.warning("Using build from {}".format(self.build_dir))
353 # force a rebuild of all modules that use OpenSSL APIs
354 for fname in self.module_files:
355 os.utime(fname, None)
356 # remove all build artefacts
357 for root, dirs, files in os.walk('build'):
358 for filename in files:
359 if filename.startswith(self.module_libs):
360 os.unlink(os.path.join(root, filename))
361
362 # overwrite header and library search paths
363 env = os.environ.copy()
364 env["CPPFLAGS"] = "-I{}".format(self.include_dir)
365 env["LDFLAGS"] = "-L{}".format(self.lib_dir)
366 # set rpath
367 env["LD_RUN_PATH"] = self.lib_dir
368
369 log.info("Rebuilding Python modules")
370 cmd = ["make", "sharedmods", "checksharedmods"]
371 self._subprocess_call(cmd, env=env)
372 self.check_imports()
373
374 def check_imports(self):
375 cmd = [sys.executable, "-c", "import _ssl; import _hashlib"]
376 self._subprocess_call(cmd)
377
378 def check_pyssl(self):
379 version = self.pyssl_version
380 if self.version not in version:
381 raise ValueError(version)
382
383 def run_python_tests(self, tests, network=True):
384 if not tests:
385 cmd = [
386 sys.executable,
387 os.path.join(PYTHONROOT, 'Lib/test/ssltests.py'),
388 '-j0'
389 ]
390 elif sys.version_info < (3, 3):
391 cmd = [sys.executable, '-m', 'test.regrtest']
392 else:
393 cmd = [sys.executable, '-m', 'test', '-j0']
394 if network:
395 cmd.extend(['-u', 'network', '-u', 'urlfetch'])
396 cmd.extend(['-w', '-r'])
397 cmd.extend(tests)
398 self._subprocess_call(cmd, stdout=None)
399
400
401 class BuildOpenSSL(AbstractBuilder):
402 library = "OpenSSL"
403 url_templates = (
404 "https://github.com/openssl/openssl/releases/download/openssl-{v}/openssl-{v}.tar.gz",
405 "https://www.openssl.org/source/openssl-{v}.tar.gz",
406 "https://www.openssl.org/source/old/{s}/openssl-{v}.tar.gz"
407 )
408 src_template = "openssl-{}.tar.gz"
409 build_template = "openssl-{}"
410 # only install software, skip docs
411 install_target = 'install_sw'
412 depend_target = 'depend'
413
414 def _post_install(self):
415 if self.version.startswith("3."):
416 self._post_install_3xx()
417
418 def _build_src(self, config_args=()):
419 if self.version.startswith("3."):
420 config_args += ("enable-fips",)
421 super()._build_src(config_args)
422
423 def _post_install_3xx(self):
424 # create ssl/ subdir with example configs
425 # Install FIPS module
426 self._subprocess_call(
427 ["make", "-j1", "install_ssldirs", "install_fips"],
428 cwd=self.build_dir
429 )
430 if not os.path.isdir(self.lib_dir):
431 # 3.0.0-beta2 uses lib64 on 64 bit platforms
432 lib64 = self.lib_dir + "64"
433 os.symlink(lib64, self.lib_dir)
434
435 @property
436 def short_version(self):
437 """Short version for OpenSSL download URL"""
438 mo = re.search(r"^(\d+)\.(\d+)\.(\d+)", self.version)
439 parsed = tuple(int(m) for m in mo.groups())
440 if parsed < (1, 0, 0):
441 return "0.9.x"
442 if parsed >= (3, 0, 0):
443 # OpenSSL 3.0.0 -> /old/3.0/
444 parsed = parsed[:2]
445 return ".".join(str(i) for i in parsed)
446
447
448 class BuildLibreSSL(AbstractBuilder):
449 library = "LibreSSL"
450 url_templates = (
451 "https://ftp.openbsd.org/pub/OpenBSD/LibreSSL/libressl-{v}.tar.gz",
452 )
453 src_template = "libressl-{}.tar.gz"
454 build_template = "libressl-{}"
455
456
457 def configure_make():
458 if not os.path.isfile('Makefile'):
459 log.info('Running ./configure')
460 subprocess.check_call([
461 './configure', '--config-cache', '--quiet',
462 '--with-pydebug'
463 ])
464
465 log.info('Running make')
466 subprocess.check_call(['make', '--quiet'])
467
468
469 def main():
470 args = parser.parse_args()
471 if not args.openssl and not args.libressl:
472 args.openssl = list(OPENSSL_RECENT_VERSIONS)
473 args.libressl = list(LIBRESSL_RECENT_VERSIONS)
474 if not args.disable_ancient:
475 args.openssl.extend(OPENSSL_OLD_VERSIONS)
476 args.libressl.extend(LIBRESSL_OLD_VERSIONS)
477
478 logging.basicConfig(
479 level=logging.DEBUG if args.debug else logging.INFO,
480 format="*** %(levelname)s %(message)s"
481 )
482
483 start = datetime.now()
484
485 if args.steps in {'modules', 'tests'}:
486 for name in ['Makefile.pre.in', 'Modules/_ssl.c']:
487 if not os.path.isfile(os.path.join(PYTHONROOT, name)):
488 parser.error(
489 "Must be executed from CPython build dir"
490 )
491 if not os.path.samefile('python', sys.executable):
492 parser.error(
493 "Must be executed with ./python from CPython build dir"
494 )
495 # check for configure and run make
496 configure_make()
497
498 # download and register builder
499 builds = []
500
501 for version in args.openssl:
502 build = BuildOpenSSL(
503 version,
504 args
505 )
506 build.install()
507 builds.append(build)
508
509 for version in args.libressl:
510 build = BuildLibreSSL(
511 version,
512 args
513 )
514 build.install()
515 builds.append(build)
516
517 if args.steps in {'modules', 'tests'}:
518 for build in builds:
519 try:
520 build.recompile_pymods()
521 build.check_pyssl()
522 if args.steps == 'tests':
523 build.run_python_tests(
524 tests=args.tests,
525 network=args.network,
526 )
527 except Exception as e:
528 log.exception("%s failed", build)
529 print("{} failed: {}".format(build, e), file=sys.stderr)
530 sys.exit(2)
531
532 log.info("\n{} finished in {}".format(
533 args.steps.capitalize(),
534 datetime.now() - start
535 ))
536 print('Python: ', sys.version)
537 if args.steps == 'tests':
538 if args.tests:
539 print('Executed Tests:', ' '.join(args.tests))
540 else:
541 print('Executed all SSL tests.')
542
543 print('OpenSSL / LibreSSL versions:')
544 for build in builds:
545 print(" * {0.library} {0.version}".format(build))
546
547
548 if __name__ == "__main__":
549 main()