Debian Codegraph @

external/boringssl/util/generate_build_files.py

raw
1 # Copyright (c) 2015, Google Inc.
2 #
3 # Permission to use, copy, modify, and/or distribute this software for any
4 # purpose with or without fee is hereby granted, provided that the above
5 # copyright notice and this permission notice appear in all copies.
6 #
7 # THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
8 # WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
9 # MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
10 # SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
11 # WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
12 # OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
13 # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
14
15 """Enumerates source files for consumption by various build systems."""
16
17 import optparse
18 import os
19 import subprocess
20 import sys
21 import json
22
23
24 # OS_ARCH_COMBOS maps from OS and platform to the OpenSSL assembly "style" for
25 # that platform and the extension used by asm files.
26 OS_ARCH_COMBOS = [
27 ('linux', 'arm', 'linux32', [], 'S'),
28 ('linux', 'aarch64', 'linux64', [], 'S'),
29 ('linux', 'x86', 'elf', ['-fPIC', '-DOPENSSL_IA32_SSE2'], 'S'),
30 ('linux', 'x86_64', 'elf', [], 'S'),
31 ('mac', 'x86', 'macosx', ['-fPIC', '-DOPENSSL_IA32_SSE2'], 'S'),
32 ('mac', 'x86_64', 'macosx', [], 'S'),
33 ('win', 'x86', 'win32n', ['-DOPENSSL_IA32_SSE2'], 'asm'),
34 ('win', 'x86_64', 'nasm', [], 'asm'),
35 ]
36
37 # NON_PERL_FILES enumerates assembly files that are not processed by the
38 # perlasm system.
39 NON_PERL_FILES = {
40 ('linux', 'arm'): [
41 'src/crypto/curve25519/asm/x25519-asm-arm.S',
42 'src/crypto/poly1305/poly1305_arm_asm.S',
43 ],
44 ('linux', 'x86_64'): [
45 'src/crypto/curve25519/asm/x25519-asm-x86_64.S',
46 ],
47 ('mac', 'x86_64'): [
48 'src/crypto/curve25519/asm/x25519-asm-x86_64.S',
49 ],
50 }
51
52 PREFIX = None
53
54
55 def PathOf(x):
56 return x if not PREFIX else os.path.join(PREFIX, x)
57
58
59 class Android(object):
60
61 def __init__(self):
62 self.header = \
63 """# Copyright (C) 2015 The Android Open Source Project
64 #
65 # Licensed under the Apache License, Version 2.0 (the "License");
66 # you may not use this file except in compliance with the License.
67 # You may obtain a copy of the License at
68 #
69 # http://www.apache.org/licenses/LICENSE-2.0
70 #
71 # Unless required by applicable law or agreed to in writing, software
72 # distributed under the License is distributed on an "AS IS" BASIS,
73 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
74 # See the License for the specific language governing permissions and
75 # limitations under the License.
76
77 """
78
79 def ExtraFiles(self):
80 return ['android_compat_hacks.c', 'android_compat_keywrap.c']
81
82 def PrintVariableSection(self, out, name, files):
83 out.write('%s := \\\n' % name)
84 for f in sorted(files):
85 out.write(' %s\\\n' % f)
86 out.write('\n')
87
88 def WriteFiles(self, files, asm_outputs):
89 with open('sources.mk', 'w+') as makefile:
90 makefile.write(self.header)
91
92 crypto_files = files['crypto'] + self.ExtraFiles()
93 self.PrintVariableSection(makefile, 'crypto_sources', crypto_files)
94 self.PrintVariableSection(makefile, 'ssl_sources', files['ssl'])
95 self.PrintVariableSection(makefile, 'tool_sources', files['tool'])
96
97 for ((osname, arch), asm_files) in asm_outputs:
98 self.PrintVariableSection(
99 makefile, '%s_%s_sources' % (osname, arch), asm_files)
100
101
102 class AndroidStandalone(Android):
103 """AndroidStandalone is for Android builds outside of the Android-system, i.e.
104
105 for applications that wish wish to ship BoringSSL.
106 """
107
108 def ExtraFiles(self):
109 return []
110
111
112 class Bazel(object):
113 """Bazel outputs files suitable for including in Bazel files."""
114
115 def __init__(self):
116 self.firstSection = True
117 self.header = \
118 """# This file is created by generate_build_files.py. Do not edit manually.
119
120 """
121
122 def PrintVariableSection(self, out, name, files):
123 if not self.firstSection:
124 out.write('\n')
125 self.firstSection = False
126
127 out.write('%s = [\n' % name)
128 for f in sorted(files):
129 out.write(' "%s",\n' % PathOf(f))
130 out.write(']\n')
131
132 def WriteFiles(self, files, asm_outputs):
133 with open('BUILD.generated.bzl', 'w+') as out:
134 out.write(self.header)
135
136 self.PrintVariableSection(out, 'ssl_headers', files['ssl_headers'])
137 self.PrintVariableSection(
138 out, 'ssl_internal_headers', files['ssl_internal_headers'])
139 self.PrintVariableSection(out, 'ssl_sources', files['ssl'])
140 self.PrintVariableSection(out, 'crypto_headers', files['crypto_headers'])
141 self.PrintVariableSection(
142 out, 'crypto_internal_headers', files['crypto_internal_headers'])
143 self.PrintVariableSection(out, 'crypto_sources', files['crypto'])
144 self.PrintVariableSection(out, 'tool_sources', files['tool'])
145
146 for ((osname, arch), asm_files) in asm_outputs:
147 self.PrintVariableSection(
148 out, 'crypto_sources_%s_%s' % (osname, arch), asm_files)
149
150 with open('BUILD.generated_tests.bzl', 'w+') as out:
151 out.write(self.header)
152
153 out.write('test_support_sources = [\n')
154 for filename in (files['test_support'] +
155 files['crypto_internal_headers'] +
156 files['ssl_internal_headers']):
157 if os.path.basename(filename) == 'malloc.cc':
158 continue
159 out.write(' "%s",\n' % PathOf(filename))
160
161 out.write(']\n\n')
162
163 out.write('def create_tests(copts, crypto, ssl):\n')
164 name_counts = {}
165 for test in files['tests']:
166 name = os.path.basename(test[0])
167 name_counts[name] = name_counts.get(name, 0) + 1
168
169 first = True
170 for test in files['tests']:
171 name = os.path.basename(test[0])
172 if name_counts[name] > 1:
173 if '/' in test[1]:
174 name += '_' + os.path.splitext(os.path.basename(test[1]))[0]
175 else:
176 name += '_' + test[1].replace('-', '_')
177
178 if not first:
179 out.write('\n')
180 first = False
181
182 src_prefix = 'src/' + test[0]
183 for src in files['test']:
184 if src.startswith(src_prefix):
185 src = src
186 break
187 else:
188 raise ValueError("Can't find source for %s" % test[0])
189
190 out.write(' native.cc_test(\n')
191 out.write(' name = "%s",\n' % name)
192 out.write(' size = "small",\n')
193 out.write(' srcs = ["%s"] + test_support_sources,\n' %
194 PathOf(src))
195
196 data_files = []
197 if len(test) > 1:
198
199 out.write(' args = [\n')
200 for arg in test[1:]:
201 if '/' in arg:
202 out.write(' "$(location %s)",\n' %
203 PathOf(os.path.join('src', arg)))
204 data_files.append('src/%s' % arg)
205 else:
206 out.write(' "%s",\n' % arg)
207 out.write(' ],\n')
208
209 out.write(' copts = copts,\n')
210
211 if len(data_files) > 0:
212 out.write(' data = [\n')
213 for filename in data_files:
214 out.write(' "%s",\n' % PathOf(filename))
215 out.write(' ],\n')
216
217 if 'ssl/' in test[0]:
218 out.write(' deps = [\n')
219 out.write(' crypto,\n')
220 out.write(' ssl,\n')
221 out.write(' ],\n')
222 else:
223 out.write(' deps = [crypto],\n')
224 out.write(' )\n')
225
226
227 class GN(object):
228
229 def __init__(self):
230 self.firstSection = True
231 self.header = \
232 """# Copyright (c) 2016 The Chromium Authors. All rights reserved.
233 # Use of this source code is governed by a BSD-style license that can be
234 # found in the LICENSE file.
235
236 # This file is created by generate_build_files.py. Do not edit manually.
237
238 """
239
240 def PrintVariableSection(self, out, name, files):
241 if not self.firstSection:
242 out.write('\n')
243 self.firstSection = False
244
245 out.write('%s = [\n' % name)
246 for f in sorted(files):
247 out.write(' "%s",\n' % f)
248 out.write(']\n')
249
250 def WriteFiles(self, files, asm_outputs):
251 with open('BUILD.generated.gni', 'w+') as out:
252 out.write(self.header)
253
254 self.PrintVariableSection(out, 'crypto_sources', files['crypto'])
255 self.PrintVariableSection(out, 'ssl_sources', files['ssl'])
256
257 for ((osname, arch), asm_files) in asm_outputs:
258 self.PrintVariableSection(
259 out, 'crypto_sources_%s_%s' % (osname, arch), asm_files)
260
261 fuzzers = [os.path.splitext(os.path.basename(fuzzer))[0]
262 for fuzzer in files['fuzz']]
263 self.PrintVariableSection(out, 'fuzzers', fuzzers)
264
265 with open('BUILD.generated_tests.gni', 'w+') as out:
266 self.firstSection = True
267 out.write(self.header)
268
269 self.PrintVariableSection(out, '_test_support_sources',
270 files['test_support'])
271 out.write('\n')
272
273 out.write('template("create_tests") {\n')
274
275 all_tests = []
276 for test in sorted(files['test']):
277 test_name = 'boringssl_%s' % os.path.splitext(os.path.basename(test))[0]
278 all_tests.append(test_name)
279
280 out.write(' executable("%s") {\n' % test_name)
281 out.write(' sources = [\n')
282 out.write(' "%s",\n' % test)
283 out.write(' ]\n')
284 out.write(' sources += _test_support_sources\n')
285 out.write(' if (defined(invoker.configs_exclude)) {\n')
286 out.write(' configs -= invoker.configs_exclude\n')
287 out.write(' }\n')
288 out.write(' configs += invoker.configs\n')
289 out.write(' deps = invoker.deps\n')
290 out.write(' }\n')
291 out.write('\n')
292
293 out.write(' group(target_name) {\n')
294 out.write(' deps = [\n')
295 for test_name in sorted(all_tests):
296 out.write(' ":%s",\n' % test_name)
297 out.write(' ]\n')
298 out.write(' }\n')
299 out.write('}\n')
300
301
302 class GYP(object):
303
304 def __init__(self):
305 self.header = \
306 """# Copyright (c) 2016 The Chromium Authors. All rights reserved.
307 # Use of this source code is governed by a BSD-style license that can be
308 # found in the LICENSE file.
309
310 # This file is created by generate_build_files.py. Do not edit manually.
311
312 """
313
314 def PrintVariableSection(self, out, name, files):
315 out.write(' \'%s\': [\n' % name)
316 for f in sorted(files):
317 out.write(' \'%s\',\n' % f)
318 out.write(' ],\n')
319
320 def WriteFiles(self, files, asm_outputs):
321 with open('boringssl.gypi', 'w+') as gypi:
322 gypi.write(self.header + '{\n \'variables\': {\n')
323
324 self.PrintVariableSection(
325 gypi, 'boringssl_ssl_sources', files['ssl'])
326 self.PrintVariableSection(
327 gypi, 'boringssl_crypto_sources', files['crypto'])
328
329 for ((osname, arch), asm_files) in asm_outputs:
330 self.PrintVariableSection(gypi, 'boringssl_%s_%s_sources' %
331 (osname, arch), asm_files)
332
333 gypi.write(' }\n}\n')
334
335 with open('boringssl_tests.gypi', 'w+') as test_gypi:
336 test_gypi.write(self.header + '{\n \'targets\': [\n')
337
338 test_names = []
339 for test in sorted(files['test']):
340 test_name = 'boringssl_%s' % os.path.splitext(os.path.basename(test))[0]
341 test_gypi.write(""" {
342 'target_name': '%s',
343 'type': 'executable',
344 'dependencies': [
345 'boringssl.gyp:boringssl',
346 ],
347 'sources': [
348 '%s',
349 '<@(boringssl_test_support_sources)',
350 ],
351 # TODO(davidben): Fix size_t truncations in BoringSSL.
352 # https://crbug.com/429039
353 'msvs_disabled_warnings': [ 4267, ],
354 },\n""" % (test_name, test))
355 test_names.append(test_name)
356
357 test_names.sort()
358
359 test_gypi.write(' ],\n \'variables\': {\n')
360
361 self.PrintVariableSection(
362 test_gypi, 'boringssl_test_support_sources', files['test_support'])
363
364 test_gypi.write(' \'boringssl_test_targets\': [\n')
365
366 for test in sorted(test_names):
367 test_gypi.write(""" '%s',\n""" % test)
368
369 test_gypi.write(' ],\n }\n}\n')
370
371
372 def FindCMakeFiles(directory):
373 """Returns list of all CMakeLists.txt files recursively in directory."""
374 cmakefiles = []
375
376 for (path, _, filenames) in os.walk(directory):
377 for filename in filenames:
378 if filename == 'CMakeLists.txt':
379 cmakefiles.append(os.path.join(path, filename))
380
381 return cmakefiles
382
383
384 def NoTests(dent, is_dir):
385 """Filter function that can be passed to FindCFiles in order to remove test
386 sources."""
387 if is_dir:
388 return dent != 'test'
389 return 'test.' not in dent and not dent.startswith('example_')
390
391
392 def OnlyTests(dent, is_dir):
393 """Filter function that can be passed to FindCFiles in order to remove
394 non-test sources."""
395 if is_dir:
396 return dent != 'test'
397 return '_test.' in dent or dent.startswith('example_')
398
399
400 def AllFiles(dent, is_dir):
401 """Filter function that can be passed to FindCFiles in order to include all
402 sources."""
403 return True
404
405
406 def SSLHeaderFiles(dent, is_dir):
407 return dent in ['ssl.h', 'tls1.h', 'ssl23.h', 'ssl3.h', 'dtls1.h']
408
409
410 def FindCFiles(directory, filter_func):
411 """Recurses through directory and returns a list of paths to all the C source
412 files that pass filter_func."""
413 cfiles = []
414
415 for (path, dirnames, filenames) in os.walk(directory):
416 for filename in filenames:
417 if not filename.endswith('.c') and not filename.endswith('.cc'):
418 continue
419 if not filter_func(filename, False):
420 continue
421 cfiles.append(os.path.join(path, filename))
422
423 for (i, dirname) in enumerate(dirnames):
424 if not filter_func(dirname, True):
425 del dirnames[i]
426
427 return cfiles
428
429
430 def FindHeaderFiles(directory, filter_func):
431 """Recurses through directory and returns a list of paths to all the header files that pass filter_func."""
432 hfiles = []
433
434 for (path, dirnames, filenames) in os.walk(directory):
435 for filename in filenames:
436 if not filename.endswith('.h'):
437 continue
438 if not filter_func(filename, False):
439 continue
440 hfiles.append(os.path.join(path, filename))
441
442 for (i, dirname) in enumerate(dirnames):
443 if not filter_func(dirname, True):
444 del dirnames[i]
445
446 return hfiles
447
448
449 def ExtractPerlAsmFromCMakeFile(cmakefile):
450 """Parses the contents of the CMakeLists.txt file passed as an argument and
451 returns a list of all the perlasm() directives found in the file."""
452 perlasms = []
453 with open(cmakefile) as f:
454 for line in f:
455 line = line.strip()
456 if not line.startswith('perlasm('):
457 continue
458 if not line.endswith(')'):
459 raise ValueError('Bad perlasm line in %s' % cmakefile)
460 # Remove "perlasm(" from start and ")" from end
461 params = line[8:-1].split()
462 if len(params) < 2:
463 raise ValueError('Bad perlasm line in %s' % cmakefile)
464 perlasms.append({
465 'extra_args': params[2:],
466 'input': os.path.join(os.path.dirname(cmakefile), params[1]),
467 'output': os.path.join(os.path.dirname(cmakefile), params[0]),
468 })
469
470 return perlasms
471
472
473 def ReadPerlAsmOperations():
474 """Returns a list of all perlasm() directives found in CMake config files in
475 src/."""
476 perlasms = []
477 cmakefiles = FindCMakeFiles('src')
478
479 for cmakefile in cmakefiles:
480 perlasms.extend(ExtractPerlAsmFromCMakeFile(cmakefile))
481
482 return perlasms
483
484
485 def PerlAsm(output_filename, input_filename, perlasm_style, extra_args):
486 """Runs the a perlasm script and puts the output into output_filename."""
487 base_dir = os.path.dirname(output_filename)
488 if not os.path.isdir(base_dir):
489 os.makedirs(base_dir)
490 output = subprocess.check_output(
491 ['perl', input_filename, perlasm_style] + extra_args)
492 with open(output_filename, 'w+') as out_file:
493 out_file.write(output)
494
495
496 def ArchForAsmFilename(filename):
497 """Returns the architectures that a given asm file should be compiled for
498 based on substrings in the filename."""
499
500 if 'x86_64' in filename or 'avx2' in filename:
501 return ['x86_64']
502 elif ('x86' in filename and 'x86_64' not in filename) or '586' in filename:
503 return ['x86']
504 elif 'armx' in filename:
505 return ['arm', 'aarch64']
506 elif 'armv8' in filename:
507 return ['aarch64']
508 elif 'arm' in filename:
509 return ['arm']
510 else:
511 raise ValueError('Unknown arch for asm filename: ' + filename)
512
513
514 def WriteAsmFiles(perlasms):
515 """Generates asm files from perlasm directives for each supported OS x
516 platform combination."""
517 asmfiles = {}
518
519 for osarch in OS_ARCH_COMBOS:
520 (osname, arch, perlasm_style, extra_args, asm_ext) = osarch
521 key = (osname, arch)
522 outDir = '%s-%s' % key
523
524 for perlasm in perlasms:
525 filename = os.path.basename(perlasm['input'])
526 output = perlasm['output']
527 if not output.startswith('src'):
528 raise ValueError('output missing src: %s' % output)
529 output = os.path.join(outDir, output[4:])
530 if output.endswith('-armx.${ASM_EXT}'):
531 output = output.replace('-armx',
532 '-armx64' if arch == 'aarch64' else '-armx32')
533 output = output.replace('${ASM_EXT}', asm_ext)
534
535 if arch in ArchForAsmFilename(filename):
536 PerlAsm(output, perlasm['input'], perlasm_style,
537 perlasm['extra_args'] + extra_args)
538 asmfiles.setdefault(key, []).append(output)
539
540 for (key, non_perl_asm_files) in NON_PERL_FILES.iteritems():
541 asmfiles.setdefault(key, []).extend(non_perl_asm_files)
542
543 return asmfiles
544
545
546 def main(platforms):
547 crypto_c_files = FindCFiles(os.path.join('src', 'crypto'), NoTests)
548 ssl_c_files = FindCFiles(os.path.join('src', 'ssl'), NoTests)
549 tool_c_files = FindCFiles(os.path.join('src', 'tool'), NoTests)
550
551 # Generate err_data.c
552 with open('err_data.c', 'w+') as err_data:
553 subprocess.check_call(['go', 'run', 'err_data_generate.go'],
554 cwd=os.path.join('src', 'crypto', 'err'),
555 stdout=err_data)
556 crypto_c_files.append('err_data.c')
557
558 test_support_c_files = FindCFiles(os.path.join('src', 'crypto', 'test'),
559 AllFiles)
560 test_support_h_files = (
561 FindHeaderFiles(os.path.join('src', 'crypto', 'test'), AllFiles) +
562 FindHeaderFiles(os.path.join('src', 'ssl', 'test'), AllFiles))
563
564 test_c_files = FindCFiles(os.path.join('src', 'crypto'), OnlyTests)
565 test_c_files += FindCFiles(os.path.join('src', 'ssl'), OnlyTests)
566
567 fuzz_c_files = FindCFiles(os.path.join('src', 'fuzz'), NoTests)
568
569 ssl_h_files = (
570 FindHeaderFiles(
571 os.path.join('src', 'include', 'openssl'),
572 SSLHeaderFiles))
573
574 def NotSSLHeaderFiles(filename, is_dir):
575 return not SSLHeaderFiles(filename, is_dir)
576 crypto_h_files = (
577 FindHeaderFiles(
578 os.path.join('src', 'include', 'openssl'),
579 NotSSLHeaderFiles))
580
581 ssl_internal_h_files = FindHeaderFiles(os.path.join('src', 'ssl'), NoTests)
582 crypto_internal_h_files = FindHeaderFiles(
583 os.path.join('src', 'crypto'), NoTests)
584
585 with open('src/util/all_tests.json', 'r') as f:
586 tests = json.load(f)
587 # Skip tests for libdecrepit. Consumers import that manually.
588 tests = [test for test in tests if not test[0].startswith("decrepit/")]
589 test_binaries = set([test[0] for test in tests])
590 test_sources = set([
591 test.replace('.cc', '').replace('.c', '').replace(
592 'src/',
593 '')
594 for test in test_c_files])
595 if test_binaries != test_sources:
596 print 'Test sources and configured tests do not match'
597 a = test_binaries.difference(test_sources)
598 if len(a) > 0:
599 print 'These tests are configured without sources: ' + str(a)
600 b = test_sources.difference(test_binaries)
601 if len(b) > 0:
602 print 'These test sources are not configured: ' + str(b)
603
604 files = {
605 'crypto': crypto_c_files,
606 'crypto_headers': crypto_h_files,
607 'crypto_internal_headers': crypto_internal_h_files,
608 'fuzz': fuzz_c_files,
609 'ssl': ssl_c_files,
610 'ssl_headers': ssl_h_files,
611 'ssl_internal_headers': ssl_internal_h_files,
612 'tool': tool_c_files,
613 'test': test_c_files,
614 'test_support': test_support_h_files + test_support_c_files,
615 'tests': tests,
616 }
617
618 asm_outputs = sorted(WriteAsmFiles(ReadPerlAsmOperations()).iteritems())
619
620 for platform in platforms:
621 platform.WriteFiles(files, asm_outputs)
622
623 return 0
624
625
626 if __name__ == '__main__':
627 parser = optparse.OptionParser(usage='Usage: %prog [--prefix=<path>]'
628 ' [android|android-standalone|bazel|gn|gyp]')
629 parser.add_option('--prefix', dest='prefix',
630 help='For Bazel, prepend argument to all source files')
631 options, args = parser.parse_args(sys.argv[1:])
632 PREFIX = options.prefix
633
634 if not args:
635 parser.print_help()
636 sys.exit(1)
637
638 platforms = []
639 for s in args:
640 if s == 'android':
641 platforms.append(Android())
642 elif s == 'android-standalone':
643 platforms.append(AndroidStandalone())
644 elif s == 'bazel':
645 platforms.append(Bazel())
646 elif s == 'gn':
647 platforms.append(GN())
648 elif s == 'gyp':
649 platforms.append(GYP())
650 else:
651 parser.print_help()
652 sys.exit(1)
653
654 sys.exit(main(platforms))