Debian Codegraph @

Mac/BuildScript/build-installer.py

raw
1 #!/usr/bin/env python
2 """
3 This script is used to build "official" universal installers on macOS.
4
5 NEW for 3.10 and backports:
6 - support universal2 variant with arm64 and x86_64 archs
7 - enable clang optimizations when building on 10.15+
8
9 NEW for 3.9.0 and backports:
10 - 2.7 end-of-life issues:
11 - Python 3 installs now update the Current version link
12 in /Library/Frameworks/Python.framework/Versions
13 - fully support running under Python 3 as well as 2.7
14 - support building on newer macOS systems with SIP
15 - fully support building on macOS 10.9+
16 - support 10.6+ on best effort
17 - support bypassing docs build by supplying a prebuilt
18 docs html tarball in the third-party source library,
19 in the format and filename conventional of those
20 downloadable from python.org:
21 python-3.x.y-docs-html.tar.bz2
22
23 NEW for 3.7.0:
24 - support Intel 64-bit-only () and 32-bit-only installer builds
25 - build and use internal Tcl/Tk 8.6 for 10.6+ builds
26 - deprecate use of explicit SDK (--sdk-path=) since all but the oldest
27 versions of Xcode support implicit setting of an SDK via environment
28 variables (SDKROOT and friends, see the xcrun man page for more info).
29 The SDK stuff was primarily needed for building universal installers
30 for 10.4; so as of 3.7.0, building installers for 10.4 is no longer
31 supported with build-installer.
32 - use generic "gcc" as compiler (CC env var) rather than "gcc-4.2"
33
34 TODO:
35 - test building with SDKROOT and DEVELOPER_DIR xcrun env variables
36
37 Usage: see USAGE variable in the script.
38 """
39 import platform, os, sys, getopt, textwrap, shutil, stat, time, pwd, grp
40 try:
41 import urllib2 as urllib_request
42 except ImportError:
43 import urllib.request as urllib_request
44
45 STAT_0o755 = ( stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR
46 | stat.S_IRGRP | stat.S_IXGRP
47 | stat.S_IROTH | stat.S_IXOTH )
48
49 STAT_0o775 = ( stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR
50 | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP
51 | stat.S_IROTH | stat.S_IXOTH )
52
53 INCLUDE_TIMESTAMP = 1
54 VERBOSE = 1
55
56 RUNNING_ON_PYTHON2 = sys.version_info.major == 2
57
58 if RUNNING_ON_PYTHON2:
59 from plistlib import writePlist
60 else:
61 from plistlib import dump
62 def writePlist(path, plist):
63 with open(plist, 'wb') as fp:
64 dump(path, fp)
65
66 def shellQuote(value):
67 """
68 Return the string value in a form that can safely be inserted into
69 a shell command.
70 """
71 return "'%s'"%(value.replace("'", "'\"'\"'"))
72
73 def grepValue(fn, variable):
74 """
75 Return the unquoted value of a variable from a file..
76 QUOTED_VALUE='quotes' -> str('quotes')
77 UNQUOTED_VALUE=noquotes -> str('noquotes')
78 """
79 variable = variable + '='
80 for ln in open(fn, 'r'):
81 if ln.startswith(variable):
82 value = ln[len(variable):].strip()
83 return value.strip("\"'")
84 raise RuntimeError("Cannot find variable %s" % variable[:-1])
85
86 _cache_getVersion = None
87
88 def getVersion():
89 global _cache_getVersion
90 if _cache_getVersion is None:
91 _cache_getVersion = grepValue(
92 os.path.join(SRCDIR, 'configure'), 'PACKAGE_VERSION')
93 return _cache_getVersion
94
95 def getVersionMajorMinor():
96 return tuple([int(n) for n in getVersion().split('.', 2)])
97
98 _cache_getFullVersion = None
99
100 def getFullVersion():
101 global _cache_getFullVersion
102 if _cache_getFullVersion is not None:
103 return _cache_getFullVersion
104 fn = os.path.join(SRCDIR, 'Include', 'patchlevel.h')
105 for ln in open(fn):
106 if 'PY_VERSION' in ln:
107 _cache_getFullVersion = ln.split()[-1][1:-1]
108 return _cache_getFullVersion
109 raise RuntimeError("Cannot find full version??")
110
111 FW_PREFIX = ["Library", "Frameworks", "Python.framework"]
112 FW_VERSION_PREFIX = "--undefined--" # initialized in parseOptions
113 FW_SSL_DIRECTORY = "--undefined--" # initialized in parseOptions
114
115 # The directory we'll use to create the build (will be erased and recreated)
116 WORKDIR = "/tmp/_py"
117
118 # The directory we'll use to store third-party sources. Set this to something
119 # else if you don't want to re-fetch required libraries every time.
120 DEPSRC = os.path.join(WORKDIR, 'third-party')
121 DEPSRC = os.path.expanduser('~/Universal/other-sources')
122
123 universal_opts_map = { 'universal2': ('arm64', 'x86_64'),
124 '32-bit': ('i386', 'ppc',),
125 '64-bit': ('x86_64', 'ppc64',),
126 'intel': ('i386', 'x86_64'),
127 'intel-32': ('i386',),
128 'intel-64': ('x86_64',),
129 '3-way': ('ppc', 'i386', 'x86_64'),
130 'all': ('i386', 'ppc', 'x86_64', 'ppc64',) }
131 default_target_map = {
132 'universal2': '10.9',
133 '64-bit': '10.5',
134 '3-way': '10.5',
135 'intel': '10.5',
136 'intel-32': '10.4',
137 'intel-64': '10.5',
138 'all': '10.5',
139 }
140
141 UNIVERSALOPTS = tuple(universal_opts_map.keys())
142
143 UNIVERSALARCHS = '32-bit'
144
145 ARCHLIST = universal_opts_map[UNIVERSALARCHS]
146
147 # Source directory (assume we're in Mac/BuildScript)
148 SRCDIR = os.path.dirname(
149 os.path.dirname(
150 os.path.dirname(
151 os.path.abspath(__file__
152 ))))
153
154 # $MACOSX_DEPLOYMENT_TARGET -> minimum OS X level
155 DEPTARGET = '10.5'
156
157 def getDeptargetTuple():
158 return tuple([int(n) for n in DEPTARGET.split('.')[0:2]])
159
160 def getBuildTuple():
161 return tuple([int(n) for n in platform.mac_ver()[0].split('.')[0:2]])
162
163 def getTargetCompilers():
164 target_cc_map = {
165 '10.4': ('gcc-4.0', 'g++-4.0'),
166 '10.5': ('gcc', 'g++'),
167 '10.6': ('gcc', 'g++'),
168 '10.7': ('gcc', 'g++'),
169 '10.8': ('gcc', 'g++'),
170 }
171 return target_cc_map.get(DEPTARGET, ('clang', 'clang++') )
172
173 CC, CXX = getTargetCompilers()
174
175 PYTHON_3 = getVersionMajorMinor() >= (3, 0)
176
177 USAGE = textwrap.dedent("""\
178 Usage: build_python [options]
179
180 Options:
181 -? or -h: Show this message
182 -b DIR
183 --build-dir=DIR: Create build here (default: %(WORKDIR)r)
184 --third-party=DIR: Store third-party sources here (default: %(DEPSRC)r)
185 --sdk-path=DIR: Location of the SDK (deprecated, use SDKROOT env variable)
186 --src-dir=DIR: Location of the Python sources (default: %(SRCDIR)r)
187 --dep-target=10.n macOS deployment target (default: %(DEPTARGET)r)
188 --universal-archs=x universal architectures (options: %(UNIVERSALOPTS)r, default: %(UNIVERSALARCHS)r)
189 """)% globals()
190
191 # Dict of object file names with shared library names to check after building.
192 # This is to ensure that we ended up dynamically linking with the shared
193 # library paths and versions we expected. For example:
194 # EXPECTED_SHARED_LIBS['_tkinter.so'] = [
195 # '/Library/Frameworks/Tcl.framework/Versions/8.5/Tcl',
196 # '/Library/Frameworks/Tk.framework/Versions/8.5/Tk']
197 EXPECTED_SHARED_LIBS = {}
198
199 # Are we building and linking with our own copy of Tcl/TK?
200 # For now, do so if deployment target is 10.6+.
201 def internalTk():
202 return getDeptargetTuple() >= (10, 6)
203
204 # Do we use 8.6.8 when building our own copy
205 # of Tcl/Tk or a modern version.
206 # We use the old version when building on
207 # old versions of macOS due to build issues.
208 def useOldTk():
209 return getBuildTuple() < (10, 15)
210
211
212 def tweak_tcl_build(basedir, archList):
213 with open("Makefile", "r") as fp:
214 contents = fp.readlines()
215
216 # For reasons I don't understand the tcl configure script
217 # decides that some stdlib symbols aren't present, before
218 # deciding that strtod is broken.
219 new_contents = []
220 for line in contents:
221 if line.startswith("COMPAT_OBJS"):
222 # note: the space before strtod.o is intentional,
223 # the detection of a broken strtod results in
224 # "fixstrod.o" on this line.
225 for nm in ("strstr.o", "strtoul.o", " strtod.o"):
226 line = line.replace(nm, "")
227 new_contents.append(line)
228
229 with open("Makefile", "w") as fp:
230 fp.writelines(new_contents)
231
232 # List of names of third party software built with this installer.
233 # The names will be inserted into the rtf version of the License.
234 THIRD_PARTY_LIBS = []
235
236 # Instructions for building libraries that are necessary for building a
237 # batteries included python.
238 # [The recipes are defined here for convenience but instantiated later after
239 # command line options have been processed.]
240 def library_recipes():
241 result = []
242
243 # Since Apple removed the header files for the deprecated system
244 # OpenSSL as of the Xcode 7 release (for OS X 10.10+), we do not
245 # have much choice but to build our own copy here, too.
246
247 result.extend([
248 dict(
249 name="OpenSSL 3.5.7",
250 url="https://github.com/openssl/openssl/releases/download/openssl-3.5.7/openssl-3.5.7.tar.gz",
251 checksum="a8c0d28a529ca480f9f36cf5792e2cd21984552a3c8e4aa11a24aa31aeac98e8",
252 buildrecipe=build_universal_openssl,
253 configure=None,
254 install=None,
255 ),
256 ])
257
258 if internalTk():
259 if useOldTk():
260 tcl_tk_ver='8.6.8'
261 tcl_checksum='81656d3367af032e0ae6157eff134f89'
262
263 tk_checksum='5e0faecba458ee1386078fb228d008ba'
264 tk_patches = ['backport_gh71383_fix.patch', 'tk868_on_10_8_10_9.patch', 'backport_gh110950_fix.patch']
265
266 else:
267 tcl_tk_ver='9.0.3'
268 tcl_checksum='2537ba0c86112c8c953f7c09d33f134dd45c0fb3a71f2d7f7691fd301d2c33a6'
269
270 tk_checksum='bf344efadb618babb7933f69275620f72454d1c8220130da93e3f7feb0efbf9b'
271 tk_patches = []
272
273
274 base_url = "https://prdownloads.sourceforge.net/tcl/{what}{version}-src.tar.gz"
275 result.extend([
276 dict(
277 name="Tcl %s"%(tcl_tk_ver,),
278 url=base_url.format(what="tcl", version=tcl_tk_ver),
279 checksum=tcl_checksum,
280 buildDir="unix",
281 configure_pre=[
282 '--enable-shared',
283 '--enable-threads',
284 '--libdir=/Library/Frameworks/Python.framework/Versions/%s/lib'%(getVersion(),),
285 ],
286 useLDFlags=False,
287 buildrecipe=tweak_tcl_build,
288 install='make TCL_LIBRARY=%(TCL_LIBRARY)s && make install TCL_LIBRARY=%(TCL_LIBRARY)s DESTDIR=%(DESTDIR)s'%{
289 "DESTDIR": shellQuote(os.path.join(WORKDIR, 'libraries')),
290 "TCL_LIBRARY": shellQuote('/Library/Frameworks/Python.framework/Versions/%s/lib/tcl8.6'%(getVersion())),
291 },
292 ),
293 dict(
294 name="Tk %s"%(tcl_tk_ver,),
295 url=base_url.format(what="tk", version=tcl_tk_ver),
296 checksum=tk_checksum,
297 patches=tk_patches,
298 buildDir="unix",
299 configure_pre=[
300 '--enable-aqua',
301 '--enable-shared',
302 '--enable-threads',
303 '--libdir=/Library/Frameworks/Python.framework/Versions/%s/lib'%(getVersion(),),
304 ],
305 useLDFlags=False,
306 install='make TCL_LIBRARY=%(TCL_LIBRARY)s TK_LIBRARY=%(TK_LIBRARY)s && make install TCL_LIBRARY=%(TCL_LIBRARY)s TK_LIBRARY=%(TK_LIBRARY)s DESTDIR=%(DESTDIR)s'%{
307 "DESTDIR": shellQuote(os.path.join(WORKDIR, 'libraries')),
308 "TCL_LIBRARY": shellQuote('/Library/Frameworks/Python.framework/Versions/%s/lib/tcl8.6'%(getVersion())),
309 "TK_LIBRARY": shellQuote('/Library/Frameworks/Python.framework/Versions/%s/lib/tk8.6'%(getVersion())),
310 },
311 ),
312 ])
313
314 if PYTHON_3:
315 result.extend([
316 dict(
317 name="XZ 5.2.3",
318 url="http://tukaani.org/xz/xz-5.2.3.tar.gz",
319 checksum='ef68674fb47a8b8e741b34e429d86e9d',
320 configure_pre=[
321 '--disable-dependency-tracking',
322 ]
323 ),
324 ])
325
326 result.extend([
327 dict(
328 name="NCurses 6.5",
329 url="https://ftp.gnu.org/gnu/ncurses/ncurses-6.5.tar.gz",
330 checksum="136d91bc269a9a5785e5f9e980bc76ab57428f604ce3e5a5a90cebc767971cc6",
331 configure_pre=[
332 "--datadir=/usr/share",
333 "--disable-lib-suffixes",
334 "--disable-db-install",
335 "--disable-mixed-case",
336 "--enable-overwrite",
337 "--enable-widec",
338 f"--libdir=/Library/Frameworks/Python.framework/Versions/{getVersion()}/lib",
339 "--sharedstatedir=/usr/com",
340 "--sysconfdir=/etc",
341 "--with-default-terminfo-dir=/usr/share/terminfo",
342 "--with-shared",
343 "--with-terminfo-dirs=/usr/share/terminfo",
344 "--without-ada",
345 "--without-cxx",
346 "--without-cxx-binding",
347 "--without-cxx-shared",
348 "--without-debug",
349 "--without-manpages",
350 "--without-normal",
351 "--without-progs",
352 "--without-tests",
353 ],
354 useLDFlags=False,
355 install='make && make install DESTDIR=%s && cd %s/usr/local/lib && ln -fs ../../../Library/Frameworks/Python.framework/Versions/%s/lib/lib* .'%(
356 shellQuote(os.path.join(WORKDIR, 'libraries')),
357 shellQuote(os.path.join(WORKDIR, 'libraries')),
358 getVersion(),
359 ),
360 ),
361 dict(
362 name="SQLite 3.50.4",
363 url="https://www.sqlite.org/2025/sqlite-autoconf-3500400.tar.gz",
364 checksum="a3db587a1b92ee5ddac2f66b3edb41b26f9c867275782d46c3a088977d6a5b18",
365 extra_cflags=('-Os '
366 '-DSQLITE_ENABLE_FTS5 '
367 '-DSQLITE_ENABLE_FTS4 '
368 '-DSQLITE_ENABLE_FTS3_PARENTHESIS '
369 '-DSQLITE_ENABLE_RTREE '
370 '-DSQLITE_OMIT_AUTOINIT '
371 '-DSQLITE_TCL=0 '
372 ),
373 configure_pre=[
374 '--enable-threadsafe',
375 '--disable-readline',
376 '--disable-dependency-tracking',
377 ],
378 install=f"make && ranlib libsqlite3.a && make install DESTDIR={shellQuote(os.path.join(WORKDIR, 'libraries'))}",
379 ),
380 dict(
381 name="libmpdec 4.0.1",
382 url="https://www.bytereef.org/software/mpdecimal/releases/mpdecimal-4.0.1.tar.gz",
383 checksum="96d33abb4bb0070c7be0fed4246cd38416188325f820468214471938545b1ac8",
384 configure_pre=[
385 "--disable-cxx",
386 "MACHINE=universal",
387 ]
388 ),
389 ])
390
391 if not PYTHON_3:
392 result.extend([
393 dict(
394 name="Sleepycat DB 4.7.25",
395 url="http://download.oracle.com/berkeley-db/db-4.7.25.tar.gz",
396 checksum='ec2b87e833779681a0c3a814aa71359e',
397 buildDir="build_unix",
398 configure="../dist/configure",
399 configure_pre=[
400 '--includedir=/usr/local/include/db4',
401 ]
402 ),
403 ])
404
405 return result
406
407 def compilerCanOptimize():
408 """
409 Return True iff the default Xcode version can use PGO and LTO
410 """
411 # bpo-42235: The version check is pretty conservative, can be
412 # adjusted after testing
413 mac_ver = tuple(map(int, platform.mac_ver()[0].split('.')))
414 return mac_ver >= (10, 15)
415
416 # Instructions for building packages inside the .mpkg.
417 def pkg_recipes():
418 unselected_for_python3 = ('selected', 'unselected')[PYTHON_3]
419 result = [
420 dict(
421 name="PythonFramework",
422 long_name="Python Framework",
423 source="/Library/Frameworks/Python.framework",
424 readme="""\
425 This package installs Python.framework, that is the python
426 interpreter and the standard library.
427 """,
428 postflight="scripts/postflight.framework",
429 selected='selected',
430 ),
431 dict(
432 name="PythonApplications",
433 long_name="GUI Applications",
434 source="/Applications/Python %(VER)s",
435 readme="""\
436 This package installs IDLE (an interactive Python IDE),
437 Python Launcher and Build Applet (create application bundles
438 from python scripts).
439
440 It also installs a number of examples and demos.
441 """,
442 required=False,
443 selected='selected',
444 ),
445 dict(
446 name="PythonUnixTools",
447 long_name="UNIX command-line tools",
448 source="/usr/local/bin",
449 readme="""\
450 This package installs the unix tools in /usr/local/bin for
451 compatibility with older releases of Python. This package
452 is not necessary to use Python.
453 """,
454 required=False,
455 selected='selected',
456 ),
457 dict(
458 name="PythonDocumentation",
459 long_name="Python Documentation",
460 topdir="/Library/Frameworks/Python.framework/Versions/%(VER)s/Resources/English.lproj/Documentation",
461 source="/pydocs",
462 readme="""\
463 This package installs the python documentation at a location
464 that is usable for pydoc and IDLE.
465 """,
466 postflight="scripts/postflight.documentation",
467 required=False,
468 selected='selected',
469 ),
470 dict(
471 name="PythonProfileChanges",
472 long_name="Shell profile updater",
473 readme="""\
474 This packages updates your shell profile to make sure that
475 the Python tools are found by your shell in preference of
476 the system provided Python tools.
477
478 If you don't install this package you'll have to add
479 "/Library/Frameworks/Python.framework/Versions/%(VER)s/bin"
480 to your PATH by hand.
481 """,
482 postflight="scripts/postflight.patch-profile",
483 topdir="/Library/Frameworks/Python.framework",
484 source="/empty-dir",
485 required=False,
486 selected='selected',
487 ),
488 dict(
489 name="PythonInstallPip",
490 long_name="Install or upgrade pip",
491 readme="""\
492 This package installs (or upgrades from an earlier version)
493 pip, a tool for installing and managing Python packages.
494 """,
495 postflight="scripts/postflight.ensurepip",
496 topdir="/Library/Frameworks/Python.framework",
497 source="/empty-dir",
498 required=False,
499 selected='selected',
500 ),
501 ]
502
503 return result
504
505 def fatal(msg):
506 """
507 A fatal error, bail out.
508 """
509 sys.stderr.write('FATAL: ')
510 sys.stderr.write(msg)
511 sys.stderr.write('\n')
512 sys.exit(1)
513
514 def fileContents(fn):
515 """
516 Return the contents of the named file
517 """
518 return open(fn, 'r').read()
519
520 def runCommand(commandline):
521 """
522 Run a command and raise RuntimeError if it fails. Output is suppressed
523 unless the command fails.
524 """
525 fd = os.popen(commandline, 'r')
526 data = fd.read()
527 xit = fd.close()
528 if xit is not None:
529 sys.stdout.write(data)
530 raise RuntimeError("command failed: %s"%(commandline,))
531
532 if VERBOSE:
533 sys.stdout.write(data); sys.stdout.flush()
534
535 def captureCommand(commandline):
536 fd = os.popen(commandline, 'r')
537 data = fd.read()
538 xit = fd.close()
539 if xit is not None:
540 sys.stdout.write(data)
541 raise RuntimeError("command failed: %s"%(commandline,))
542
543 return data
544
545 def getTclTkVersion(configfile, versionline):
546 """
547 search Tcl or Tk configuration file for version line
548 """
549 try:
550 f = open(configfile, "r")
551 except OSError:
552 fatal("Framework configuration file not found: %s" % configfile)
553
554 for l in f:
555 if l.startswith(versionline):
556 f.close()
557 return l
558
559 fatal("Version variable %s not found in framework configuration file: %s"
560 % (versionline, configfile))
561
562 def checkEnvironment():
563 """
564 Check that we're running on a supported system.
565 """
566
567 if sys.version_info[0:2] < (2, 7):
568 fatal("This script must be run with Python 2.7 (or later)")
569
570 if platform.system() != 'Darwin':
571 fatal("This script should be run on a macOS 10.5 (or later) system")
572
573 if int(platform.release().split('.')[0]) < 8:
574 fatal("This script should be run on a macOS 10.5 (or later) system")
575
576 # Because we only support dynamic load of only one major/minor version of
577 # Tcl/Tk, if we are not using building and using our own private copy of
578 # Tcl/Tk, ensure:
579 # 1. there is a user-installed framework (usually ActiveTcl) in (or linked
580 # in) SDKROOT/Library/Frameworks. As of Python 3.7.0, we no longer
581 # enforce that the version of the user-installed framework also
582 # exists in the system-supplied Tcl/Tk frameworks. Time to support
583 # Tcl/Tk 8.6 even if Apple does not.
584 if not internalTk():
585 frameworks = {}
586 for framework in ['Tcl', 'Tk']:
587 fwpth = 'Library/Frameworks/%s.framework/Versions/Current' % framework
588 libfw = os.path.join('/', fwpth)
589 usrfw = os.path.join(os.getenv('HOME'), fwpth)
590 frameworks[framework] = os.readlink(libfw)
591 if not os.path.exists(libfw):
592 fatal("Please install a link to a current %s %s as %s so "
593 "the user can override the system framework."
594 % (framework, frameworks[framework], libfw))
595 if os.path.exists(usrfw):
596 fatal("Please rename %s to avoid possible dynamic load issues."
597 % usrfw)
598
599 if frameworks['Tcl'] != frameworks['Tk']:
600 fatal("The Tcl and Tk frameworks are not the same version.")
601
602 print(" -- Building with external Tcl/Tk %s frameworks"
603 % frameworks['Tk'])
604
605 # add files to check after build
606 EXPECTED_SHARED_LIBS['_tkinter.so'] = [
607 "/Library/Frameworks/Tcl.framework/Versions/%s/Tcl"
608 % frameworks['Tcl'],
609 "/Library/Frameworks/Tk.framework/Versions/%s/Tk"
610 % frameworks['Tk'],
611 ]
612 else:
613 print(" -- Building private copy of Tcl/Tk")
614 print("")
615
616 # Remove inherited environment variables which might influence build
617 environ_var_prefixes = ['CPATH', 'C_INCLUDE_', 'DYLD_', 'LANG', 'LC_',
618 'LD_', 'LIBRARY_', 'PATH', 'PYTHON']
619 for ev in list(os.environ):
620 for prefix in environ_var_prefixes:
621 if ev.startswith(prefix) :
622 print("INFO: deleting environment variable %s=%s" % (
623 ev, os.environ[ev]))
624 del os.environ[ev]
625
626 base_path = '/bin:/sbin:/usr/bin:/usr/sbin'
627 if 'SDK_TOOLS_BIN' in os.environ:
628 base_path = os.environ['SDK_TOOLS_BIN'] + ':' + base_path
629 # Xcode 2.5 on OS X 10.4 does not include SetFile in its usr/bin;
630 # add its fixed location here if it exists
631 OLD_DEVELOPER_TOOLS = '/Developer/Tools'
632 if os.path.isdir(OLD_DEVELOPER_TOOLS):
633 base_path = base_path + ':' + OLD_DEVELOPER_TOOLS
634 os.environ['PATH'] = base_path
635 print("Setting default PATH: %s"%(os.environ['PATH']))
636
637 def parseOptions(args=None):
638 """
639 Parse arguments and update global settings.
640 """
641 global WORKDIR, DEPSRC, SRCDIR, DEPTARGET
642 global UNIVERSALOPTS, UNIVERSALARCHS, ARCHLIST, CC, CXX
643 global FW_VERSION_PREFIX
644 global FW_SSL_DIRECTORY
645
646 if args is None:
647 args = sys.argv[1:]
648
649 try:
650 options, args = getopt.getopt(args, '?hb',
651 [ 'build-dir=', 'third-party=', 'sdk-path=' , 'src-dir=',
652 'dep-target=', 'universal-archs=', 'help' ])
653 except getopt.GetoptError:
654 print(sys.exc_info()[1])
655 sys.exit(1)
656
657 if args:
658 print("Additional arguments")
659 sys.exit(1)
660
661 deptarget = None
662 for k, v in options:
663 if k in ('-h', '-?', '--help'):
664 print(USAGE)
665 sys.exit(0)
666
667 elif k in ('-d', '--build-dir'):
668 WORKDIR=v
669
670 elif k in ('--third-party',):
671 DEPSRC=v
672
673 elif k in ('--sdk-path',):
674 print(" WARNING: --sdk-path is no longer supported")
675
676 elif k in ('--src-dir',):
677 SRCDIR=v
678
679 elif k in ('--dep-target', ):
680 DEPTARGET=v
681 deptarget=v
682
683 elif k in ('--universal-archs', ):
684 if v in UNIVERSALOPTS:
685 UNIVERSALARCHS = v
686 ARCHLIST = universal_opts_map[UNIVERSALARCHS]
687 if deptarget is None:
688 # Select alternate default deployment
689 # target
690 DEPTARGET = default_target_map.get(v, '10.5')
691 else:
692 raise NotImplementedError(v)
693
694 else:
695 raise NotImplementedError(k)
696
697 SRCDIR=os.path.abspath(SRCDIR)
698 WORKDIR=os.path.abspath(WORKDIR)
699 DEPSRC=os.path.abspath(DEPSRC)
700
701 CC, CXX = getTargetCompilers()
702
703 FW_VERSION_PREFIX = FW_PREFIX[:] + ["Versions", getVersion()]
704 FW_SSL_DIRECTORY = FW_VERSION_PREFIX[:] + ["etc", "openssl"]
705
706 print("-- Settings:")
707 print(" * Source directory: %s" % SRCDIR)
708 print(" * Build directory: %s" % WORKDIR)
709 print(" * Third-party source: %s" % DEPSRC)
710 print(" * Deployment target: %s" % DEPTARGET)
711 print(" * Universal archs: %s" % str(ARCHLIST))
712 print(" * C compiler: %s" % CC)
713 print(" * C++ compiler: %s" % CXX)
714 print("")
715 print(" -- Building a Python %s framework at patch level %s"
716 % (getVersion(), getFullVersion()))
717 print("")
718
719 def extractArchive(builddir, archiveName):
720 """
721 Extract a source archive into 'builddir'. Returns the path of the
722 extracted archive.
723
724 XXX: This function assumes that archives contain a toplevel directory
725 that is has the same name as the basename of the archive. This is
726 safe enough for almost anything we use. Unfortunately, it does not
727 work for current Tcl and Tk source releases where the basename of
728 the archive ends with "-src" but the uncompressed directory does not.
729 For now, just special case Tcl and Tk tar.gz downloads.
730 """
731 curdir = os.getcwd()
732 try:
733 os.chdir(builddir)
734 if archiveName.endswith('.tar.gz'):
735 retval = os.path.basename(archiveName[:-7])
736 if ((retval.startswith('tcl') or retval.startswith('tk'))
737 and retval.endswith('-src')):
738 retval = retval[:-4]
739 # Strip rcxx suffix from Tcl/Tk release candidates
740 retval_rc = retval.find('rc')
741 if retval_rc > 0:
742 retval = retval[:retval_rc]
743 if os.path.exists(retval):
744 shutil.rmtree(retval)
745 fp = os.popen("tar zxf %s 2>&1"%(shellQuote(archiveName),), 'r')
746
747 elif archiveName.endswith('.tar.bz2'):
748 retval = os.path.basename(archiveName[:-8])
749 if os.path.exists(retval):
750 shutil.rmtree(retval)
751 fp = os.popen("tar jxf %s 2>&1"%(shellQuote(archiveName),), 'r')
752
753 elif archiveName.endswith('.tar'):
754 retval = os.path.basename(archiveName[:-4])
755 if os.path.exists(retval):
756 shutil.rmtree(retval)
757 fp = os.popen("tar xf %s 2>&1"%(shellQuote(archiveName),), 'r')
758
759 elif archiveName.endswith('.zip'):
760 retval = os.path.basename(archiveName[:-4])
761 if os.path.exists(retval):
762 shutil.rmtree(retval)
763 fp = os.popen("unzip %s 2>&1"%(shellQuote(archiveName),), 'r')
764
765 data = fp.read()
766 xit = fp.close()
767 if xit is not None:
768 sys.stdout.write(data)
769 raise RuntimeError("Cannot extract %s"%(archiveName,))
770
771 return os.path.join(builddir, retval)
772
773 finally:
774 os.chdir(curdir)
775
776 def downloadURL(url, fname):
777 """
778 Download the contents of the url into the file.
779 """
780 fpIn = urllib_request.urlopen(url)
781 fpOut = open(fname, 'wb')
782 block = fpIn.read(10240)
783 try:
784 while block:
785 fpOut.write(block)
786 block = fpIn.read(10240)
787 fpIn.close()
788 fpOut.close()
789 except:
790 try:
791 os.unlink(fname)
792 except OSError:
793 pass
794
795 def verifyThirdPartyFile(url, checksum, fname):
796 """
797 Download file from url to filename fname if it does not already exist.
798 Abort if file contents does not match supplied md5 checksum.
799 """
800 name = os.path.basename(fname)
801 if os.path.exists(fname):
802 print("Using local copy of %s"%(name,))
803 else:
804 print("Did not find local copy of %s"%(name,))
805 print("Downloading %s"%(name,))
806 downloadURL(url, fname)
807 print("Archive for %s stored as %s"%(name, fname))
808 if len(checksum) == 32:
809 algo = 'md5'
810 elif len(checksum) == 64:
811 algo = 'sha256'
812 else:
813 raise ValueError(checksum)
814 if os.system(
815 'CHECKSUM=$(openssl %s %s) ; test "${CHECKSUM##*= }" = "%s"'
816 % (algo, shellQuote(fname), checksum) ):
817 fatal('%s checksum mismatch for file %s' % (algo, fname))
818
819 def build_universal_openssl(basedir, archList):
820 """
821 Special case build recipe for universal build of openssl.
822
823 The upstream OpenSSL build system does not directly support
824 OS X universal builds. We need to build each architecture
825 separately then lipo them together into fat libraries.
826 """
827
828 # OpenSSL fails to build with Xcode 2.5 (on OS X 10.4).
829 # If we are building on a 10.4.x or earlier system,
830 # unilaterally disable assembly code building to avoid the problem.
831 no_asm = int(platform.release().split(".")[0]) < 9
832
833 def build_openssl_arch(archbase, arch):
834 "Build one architecture of openssl"
835 arch_opts = {
836 "i386": ["darwin-i386-cc"],
837 "x86_64": ["darwin64-x86_64-cc", "enable-ec_nistp_64_gcc_128"],
838 "arm64": ["darwin64-arm64-cc"],
839 "ppc": ["darwin-ppc-cc"],
840 "ppc64": ["darwin64-ppc-cc"],
841 }
842
843 # Somewhere between OpenSSL 1.1.0j and 1.1.1c, changes cause the
844 # "enable-ec_nistp_64_gcc_128" option to get compile errors when
845 # building on our 10.6 gcc-4.2 environment. There have been other
846 # reports of projects running into this when using older compilers.
847 # So, for now, do not try to use "enable-ec_nistp_64_gcc_128" when
848 # building for 10.6.
849 if getDeptargetTuple() == (10, 6):
850 arch_opts['x86_64'].remove('enable-ec_nistp_64_gcc_128')
851
852 configure_opts = [
853 "no-idea",
854 "no-mdc2",
855 "no-rc5",
856 "no-zlib",
857 "no-ssl3",
858 # "enable-unit-test",
859 "shared",
860 "--prefix=%s"%os.path.join("/", *FW_VERSION_PREFIX),
861 "--openssldir=%s"%os.path.join("/", *FW_SSL_DIRECTORY),
862 ]
863 if no_asm:
864 configure_opts.append("no-asm")
865 runCommand(" ".join(["perl", "Configure"]
866 + arch_opts[arch] + configure_opts))
867 runCommand("make depend")
868 runCommand("make all")
869 runCommand("make install_sw DESTDIR=%s"%shellQuote(archbase))
870 # runCommand("make test")
871 return
872
873 srcdir = os.getcwd()
874 universalbase = os.path.join(srcdir, "..",
875 os.path.basename(srcdir) + "-universal")
876 os.mkdir(universalbase)
877 archbasefws = []
878 for arch in archList:
879 # fresh copy of the source tree
880 archsrc = os.path.join(universalbase, arch, "src")
881 shutil.copytree(srcdir, archsrc, symlinks=True)
882 # install base for this arch
883 archbase = os.path.join(universalbase, arch, "root")
884 os.mkdir(archbase)
885 # Python framework base within install_prefix:
886 # the build will install into this framework..
887 # This is to ensure that the resulting shared libs have
888 # the desired real install paths built into them.
889 archbasefw = os.path.join(archbase, *FW_VERSION_PREFIX)
890
891 # build one architecture
892 os.chdir(archsrc)
893 build_openssl_arch(archbase, arch)
894 os.chdir(srcdir)
895 archbasefws.append(archbasefw)
896
897 # copy arch-independent files from last build into the basedir framework
898 basefw = os.path.join(basedir, *FW_VERSION_PREFIX)
899 shutil.copytree(
900 os.path.join(archbasefw, "include", "openssl"),
901 os.path.join(basefw, "include", "openssl")
902 )
903
904 shlib_version_number = grepValue(os.path.join(archsrc, "Makefile"),
905 "SHLIB_VERSION_NUMBER")
906 # e.g. -> "1.0.0"
907 libcrypto = "libcrypto.dylib"
908 libcrypto_versioned = libcrypto.replace(".", "."+shlib_version_number+".")
909 # e.g. -> "libcrypto.1.0.0.dylib"
910 libssl = "libssl.dylib"
911 libssl_versioned = libssl.replace(".", "."+shlib_version_number+".")
912 # e.g. -> "libssl.1.0.0.dylib"
913
914 try:
915 os.mkdir(os.path.join(basefw, "lib"))
916 except OSError:
917 pass
918
919 # merge the individual arch-dependent shared libs into a fat shared lib
920 archbasefws.insert(0, basefw)
921 for (lib_unversioned, lib_versioned) in [
922 (libcrypto, libcrypto_versioned),
923 (libssl, libssl_versioned)
924 ]:
925 runCommand("lipo -create -output " +
926 " ".join(shellQuote(
927 os.path.join(fw, "lib", lib_versioned))
928 for fw in archbasefws))
929 # and create an unversioned symlink of it
930 os.symlink(lib_versioned, os.path.join(basefw, "lib", lib_unversioned))
931
932 # Create links in the temp include and lib dirs that will be injected
933 # into the Python build so that setup.py can find them while building
934 # and the versioned links so that the setup.py post-build import test
935 # does not fail.
936 relative_path = os.path.join("..", "..", "..", *FW_VERSION_PREFIX)
937 for fn in [
938 ["include", "openssl"],
939 ["lib", libcrypto],
940 ["lib", libssl],
941 ["lib", libcrypto_versioned],
942 ["lib", libssl_versioned],
943 ]:
944 os.symlink(
945 os.path.join(relative_path, *fn),
946 os.path.join(basedir, "usr", "local", *fn)
947 )
948
949 return
950
951 def buildRecipe(recipe, basedir, archList):
952 """
953 Build software using a recipe. This function does the
954 'configure;make;make install' dance for C software, with a possibility
955 to customize this process, basically a poor-mans DarwinPorts.
956 """
957 curdir = os.getcwd()
958
959 name = recipe['name']
960 THIRD_PARTY_LIBS.append(name)
961 url = recipe['url']
962 configure = recipe.get('configure', './configure')
963 buildrecipe = recipe.get('buildrecipe', None)
964 install = recipe.get('install', 'make && make install DESTDIR=%s'%(
965 shellQuote(basedir)))
966
967 archiveName = os.path.split(url)[-1]
968 sourceArchive = os.path.join(DEPSRC, archiveName)
969
970 if not os.path.exists(DEPSRC):
971 os.mkdir(DEPSRC)
972
973 verifyThirdPartyFile(url, recipe['checksum'], sourceArchive)
974 print("Extracting archive for %s"%(name,))
975 buildDir=os.path.join(WORKDIR, '_bld')
976 if not os.path.exists(buildDir):
977 os.mkdir(buildDir)
978
979 workDir = extractArchive(buildDir, sourceArchive)
980 os.chdir(workDir)
981
982 for patch in recipe.get('patches', ()):
983 if isinstance(patch, tuple):
984 url, checksum = patch
985 fn = os.path.join(DEPSRC, os.path.basename(url))
986 verifyThirdPartyFile(url, checksum, fn)
987 else:
988 # patch is a file in the source directory
989 fn = os.path.join(curdir, patch)
990 runCommand('patch -p%s < %s'%(recipe.get('patchlevel', 1),
991 shellQuote(fn),))
992
993 for patchscript in recipe.get('patchscripts', ()):
994 if isinstance(patchscript, tuple):
995 url, checksum = patchscript
996 fn = os.path.join(DEPSRC, os.path.basename(url))
997 verifyThirdPartyFile(url, checksum, fn)
998 else:
999 # patch is a file in the source directory
1000 fn = os.path.join(curdir, patchscript)
1001 if fn.endswith('.bz2'):
1002 runCommand('bunzip2 -fk %s' % shellQuote(fn))
1003 fn = fn[:-4]
1004 runCommand('sh %s' % shellQuote(fn))
1005 os.unlink(fn)
1006
1007 if 'buildDir' in recipe:
1008 os.chdir(recipe['buildDir'])
1009
1010 if configure is not None:
1011 configure_args = [
1012 "--prefix=/usr/local",
1013 "--enable-static",
1014 "--disable-shared",
1015 #"CPP=gcc -arch %s -E"%(' -arch '.join(archList,),),
1016 ]
1017
1018 if 'configure_pre' in recipe:
1019 args = list(recipe['configure_pre'])
1020 if '--disable-static' in args:
1021 configure_args.remove('--enable-static')
1022 if '--enable-shared' in args:
1023 configure_args.remove('--disable-shared')
1024 configure_args.extend(args)
1025
1026 if recipe.get('useLDFlags', 1):
1027 configure_args.extend([
1028 "CFLAGS=%s-mmacosx-version-min=%s -arch %s "
1029 "-I%s/usr/local/include"%(
1030 recipe.get('extra_cflags', ''),
1031 DEPTARGET,
1032 ' -arch '.join(archList),
1033 shellQuote(basedir)[1:-1],),
1034 "LDFLAGS=-mmacosx-version-min=%s -L%s/usr/local/lib -arch %s"%(
1035 DEPTARGET,
1036 shellQuote(basedir)[1:-1],
1037 ' -arch '.join(archList)),
1038 ])
1039 else:
1040 configure_args.extend([
1041 "CFLAGS=%s-mmacosx-version-min=%s -arch %s "
1042 "-I%s/usr/local/include"%(
1043 recipe.get('extra_cflags', ''),
1044 DEPTARGET,
1045 ' -arch '.join(archList),
1046 shellQuote(basedir)[1:-1],),
1047 ])
1048
1049 if 'configure_post' in recipe:
1050 configure_args = configure_args + list(recipe['configure_post'])
1051
1052 configure_args.insert(0, configure)
1053 configure_args = [ shellQuote(a) for a in configure_args ]
1054
1055 print("Running configure for %s"%(name,))
1056 runCommand(' '.join(configure_args) + ' 2>&1')
1057
1058 if buildrecipe is not None:
1059 # call special-case build recipe, e.g. for openssl
1060 buildrecipe(basedir, archList)
1061
1062 if install is not None:
1063 print("Running install for %s"%(name,))
1064 runCommand('{ ' + install + ' ;} 2>&1')
1065
1066 print("Done %s"%(name,))
1067 print("")
1068
1069 os.chdir(curdir)
1070
1071 def buildLibraries():
1072 """
1073 Build our dependencies into $WORKDIR/libraries/usr/local
1074 """
1075 print("")
1076 print("Building required libraries")
1077 print("")
1078 universal = os.path.join(WORKDIR, 'libraries')
1079 os.mkdir(universal)
1080 os.makedirs(os.path.join(universal, 'usr', 'local', 'lib'))
1081 os.makedirs(os.path.join(universal, 'usr', 'local', 'include'))
1082
1083 for recipe in library_recipes():
1084 buildRecipe(recipe, universal, ARCHLIST)
1085
1086
1087
1088 def buildPythonDocs():
1089 # This stores the documentation as Resources/English.lproj/Documentation
1090 # inside the framework. pydoc and IDLE will pick it up there.
1091 print("Install python documentation")
1092 rootDir = os.path.join(WORKDIR, '_root')
1093 buildDir = os.path.join('../../Doc')
1094 docdir = os.path.join(rootDir, 'pydocs')
1095 curDir = os.getcwd()
1096 os.chdir(buildDir)
1097 runCommand('make clean')
1098
1099 # Search third-party source directory for a pre-built version of the docs.
1100 # Use the naming convention of the docs.python.org html downloads:
1101 # python-3.9.0b1-docs-html.tar.bz2
1102 doctarfiles = [ f for f in os.listdir(DEPSRC)
1103 if f.startswith('python-'+getFullVersion())
1104 if f.endswith('-docs-html.tar.bz2') ]
1105 if doctarfiles:
1106 doctarfile = doctarfiles[0]
1107 if not os.path.exists('build'):
1108 os.mkdir('build')
1109 # if build directory existed, it was emptied by make clean, above
1110 os.chdir('build')
1111 # Extract the first archive found for this version into build
1112 runCommand('tar xjf %s'%shellQuote(os.path.join(DEPSRC, doctarfile)))
1113 # see if tar extracted a directory ending in -docs-html
1114 archivefiles = [ f for f in os.listdir('.')
1115 if f.endswith('-docs-html')
1116 if os.path.isdir(f) ]
1117 if archivefiles:
1118 archivefile = archivefiles[0]
1119 # make it our 'Docs/build/html' directory
1120 print(' -- using pre-built python documentation from %s'%archivefile)
1121 os.rename(archivefile, 'html')
1122 os.chdir(buildDir)
1123
1124 htmlDir = os.path.join('build', 'html')
1125 if not os.path.exists(htmlDir):
1126 # Create virtual environment for docs builds with blurb and sphinx
1127 runCommand('make venv')
1128 runCommand('make html PYTHON=venv/bin/python')
1129 os.rename(htmlDir, docdir)
1130 os.chdir(curDir)
1131
1132
1133 def buildPython():
1134 print("Building a universal python for %s architectures" % UNIVERSALARCHS)
1135
1136 buildDir = os.path.join(WORKDIR, '_bld', 'python')
1137 rootDir = os.path.join(WORKDIR, '_root')
1138
1139 if os.path.exists(buildDir):
1140 shutil.rmtree(buildDir)
1141 if os.path.exists(rootDir):
1142 shutil.rmtree(rootDir)
1143 os.makedirs(buildDir)
1144 os.makedirs(rootDir)
1145 os.makedirs(os.path.join(rootDir, 'empty-dir'))
1146 curdir = os.getcwd()
1147 os.chdir(buildDir)
1148
1149 # Extract the version from the configure file, needed to calculate
1150 # several paths.
1151 version = getVersion()
1152
1153 # Since the extra libs are not in their installed framework location
1154 # during the build, augment the library path so that the interpreter
1155 # will find them during its extension import sanity checks.
1156
1157 print("Running configure...")
1158 print(" NOTE: --with-mimalloc=no pending resolution of weak linking issues")
1159 runCommand("%s -C --enable-framework --enable-universalsdk=/ "
1160 "--with-mimalloc=no "
1161 "--with-system-libmpdec "
1162 "--with-universal-archs=%s "
1163 "%s "
1164 "%s "
1165 "%s "
1166 "%s "
1167 "%s "
1168 "%s "
1169 "LDFLAGS='-g -L%s/libraries/usr/local/lib' "
1170 "CFLAGS='-g -I%s/libraries/usr/local/include' 2>&1"%(
1171 shellQuote(os.path.join(SRCDIR, 'configure')),
1172 UNIVERSALARCHS,
1173 (' ', '--with-computed-gotos ')[PYTHON_3],
1174 (' ', '--without-ensurepip ')[PYTHON_3],
1175 (' ', "--with-openssl='%s/libraries/usr/local'"%(
1176 shellQuote(WORKDIR)[1:-1],))[PYTHON_3],
1177 (' ', "--enable-optimizations --with-lto")[compilerCanOptimize()],
1178 (' ', "TCLTK_CFLAGS='-I%s/libraries/usr/local/include'"%(
1179 shellQuote(WORKDIR)[1:-1],))[internalTk()],
1180 (' ', "TCLTK_LIBS='-L%s/libraries/usr/local/lib -ltcl8.6 -ltk8.6'"%(
1181 shellQuote(WORKDIR)[1:-1],))[internalTk()],
1182 shellQuote(WORKDIR)[1:-1],
1183 shellQuote(WORKDIR)[1:-1]))
1184
1185 # As of macOS 10.11 with SYSTEM INTEGRITY PROTECTION, DYLD_*
1186 # environment variables are no longer automatically inherited
1187 # by child processes from their parents. We used to just set
1188 # DYLD_LIBRARY_PATH, pointing to the third-party libs,
1189 # in build-installer.py's process environment and it was
1190 # passed through the make utility into the environment of
1191 # setup.py. Instead, we now append DYLD_LIBRARY_PATH to
1192 # the existing RUNSHARED configuration value when we call
1193 # make for extension module builds.
1194
1195 runshared_for_make = "".join([
1196 " RUNSHARED=",
1197 "'",
1198 grepValue("Makefile", "RUNSHARED"),
1199 ' DYLD_LIBRARY_PATH=',
1200 os.path.join(WORKDIR, 'libraries', 'usr', 'local', 'lib'),
1201 "'" ])
1202
1203 # Look for environment value BUILDINSTALLER_BUILDPYTHON_MAKE_EXTRAS
1204 # and, if defined, append its value to the make command. This allows
1205 # us to pass in version control tags, like GITTAG, to a build from a
1206 # tarball rather than from a vcs checkout, thus eliminating the need
1207 # to have a working copy of the vcs program on the build machine.
1208 #
1209 # A typical use might be:
1210 # export BUILDINSTALLER_BUILDPYTHON_MAKE_EXTRAS=" \
1211 # GITVERSION='echo 123456789a' \
1212 # GITTAG='echo v3.6.0' \
1213 # GITBRANCH='echo 3.6'"
1214
1215 make_extras = os.getenv("BUILDINSTALLER_BUILDPYTHON_MAKE_EXTRAS")
1216 if make_extras:
1217 make_cmd = "make " + make_extras + runshared_for_make
1218 else:
1219 make_cmd = "make" + runshared_for_make
1220 print("Running " + make_cmd)
1221 runCommand(make_cmd)
1222
1223 make_cmd = "make install DESTDIR=%s %s"%(
1224 shellQuote(rootDir),
1225 runshared_for_make)
1226 print("Running " + make_cmd)
1227 runCommand(make_cmd)
1228
1229 make_cmd = "make frameworkinstallextras DESTDIR=%s %s"%(
1230 shellQuote(rootDir),
1231 runshared_for_make)
1232 print("Running " + make_cmd)
1233 runCommand(make_cmd)
1234
1235 print("Copying required shared libraries")
1236 if os.path.exists(os.path.join(WORKDIR, 'libraries', 'Library')):
1237 build_lib_dir = os.path.join(
1238 WORKDIR, 'libraries', 'Library', 'Frameworks',
1239 'Python.framework', 'Versions', getVersion(), 'lib')
1240 fw_lib_dir = os.path.join(
1241 WORKDIR, '_root', 'Library', 'Frameworks',
1242 'Python.framework', 'Versions', getVersion(), 'lib')
1243 if internalTk():
1244 # move Tcl and Tk pkgconfig files
1245 runCommand("mv %s/pkgconfig/* %s/pkgconfig"%(
1246 shellQuote(build_lib_dir),
1247 shellQuote(fw_lib_dir) ))
1248 runCommand("rm -r %s/pkgconfig"%(
1249 shellQuote(build_lib_dir), ))
1250 runCommand("mv %s/* %s"%(
1251 shellQuote(build_lib_dir),
1252 shellQuote(fw_lib_dir) ))
1253
1254 frmDir = os.path.join(rootDir, 'Library', 'Frameworks', 'Python.framework')
1255 frmDirVersioned = os.path.join(frmDir, 'Versions', version)
1256 path_to_lib = os.path.join(frmDirVersioned, 'lib', 'python%s'%(version,))
1257 # create directory for OpenSSL certificates
1258 sslDir = os.path.join(frmDirVersioned, 'etc', 'openssl')
1259 os.makedirs(sslDir)
1260
1261 print("Fix file modes")
1262 gid = grp.getgrnam('admin').gr_gid
1263
1264 shared_lib_error = False
1265 for dirpath, dirnames, filenames in os.walk(frmDir):
1266 for dn in dirnames:
1267 os.chmod(os.path.join(dirpath, dn), STAT_0o775)
1268 os.chown(os.path.join(dirpath, dn), -1, gid)
1269
1270 for fn in filenames:
1271 if os.path.islink(fn):
1272 continue
1273
1274 # "chmod g+w $fn"
1275 p = os.path.join(dirpath, fn)
1276 st = os.stat(p)
1277 os.chmod(p, stat.S_IMODE(st.st_mode) | stat.S_IWGRP)
1278 os.chown(p, -1, gid)
1279
1280 if fn in EXPECTED_SHARED_LIBS:
1281 # check to see that this file was linked with the
1282 # expected library path and version
1283 data = captureCommand("otool -L %s" % shellQuote(p))
1284 for sl in EXPECTED_SHARED_LIBS[fn]:
1285 if ("\t%s " % sl) not in data:
1286 print("Expected shared lib %s was not linked with %s"
1287 % (sl, p))
1288 shared_lib_error = True
1289
1290 if shared_lib_error:
1291 fatal("Unexpected shared library errors.")
1292
1293 if PYTHON_3:
1294 LDVERSION=None
1295 VERSION=None
1296 ABIFLAGS=None
1297
1298 fp = open(os.path.join(buildDir, 'Makefile'), 'r')
1299 for ln in fp:
1300 if ln.startswith('VERSION='):
1301 VERSION=ln.split()[1]
1302 if ln.startswith('ABIFLAGS='):
1303 ABIFLAGS=ln.split()
1304 ABIFLAGS=ABIFLAGS[1] if len(ABIFLAGS) > 1 else ''
1305 if ln.startswith('LDVERSION='):
1306 LDVERSION=ln.split()[1]
1307 fp.close()
1308
1309 LDVERSION = LDVERSION.replace('$(VERSION)', VERSION)
1310 LDVERSION = LDVERSION.replace('$(ABIFLAGS)', ABIFLAGS)
1311 config_suffix = '-' + LDVERSION
1312 if getVersionMajorMinor() >= (3, 6):
1313 config_suffix = config_suffix + '-darwin'
1314 else:
1315 config_suffix = '' # Python 2.x
1316
1317 # We added some directories to the search path during the configure
1318 # phase. Remove those because those directories won't be there on
1319 # the end-users system. Also remove the directories from _sysconfigdata.py
1320 # (added in 3.3) if it exists.
1321
1322 include_path = '-I%s/libraries/usr/local/include' % (WORKDIR,)
1323 lib_path = '-L%s/libraries/usr/local/lib' % (WORKDIR,)
1324
1325 # fix Makefile
1326 path = os.path.join(path_to_lib, 'config' + config_suffix, 'Makefile')
1327 fp = open(path, 'r')
1328 data = fp.read()
1329 fp.close()
1330
1331 for p in (include_path, lib_path):
1332 data = data.replace(" " + p, '')
1333 data = data.replace(p + " ", '')
1334
1335 fp = open(path, 'w')
1336 fp.write(data)
1337 fp.close()
1338
1339 # fix _sysconfigdata
1340 #
1341 # TODO: make this more robust! test_sysconfig_module of
1342 # distutils.tests.test_sysconfig.SysconfigTestCase tests that
1343 # the output from get_config_var in both sysconfig and
1344 # distutils.sysconfig is exactly the same for both CFLAGS and
1345 # LDFLAGS. The fixing up is now complicated by the pretty
1346 # printing in _sysconfigdata.py. Also, we are using the
1347 # pprint from the Python running the installer build which
1348 # may not cosmetically format the same as the pprint in the Python
1349 # being built (and which is used to originally generate
1350 # _sysconfigdata.py).
1351
1352 import pprint
1353 if getVersionMajorMinor() >= (3, 6):
1354 # XXX this is extra-fragile
1355 path = os.path.join(path_to_lib,
1356 '_sysconfigdata_%s_darwin_darwin.py' % (ABIFLAGS,))
1357 else:
1358 path = os.path.join(path_to_lib, '_sysconfigdata.py')
1359 fp = open(path, 'r')
1360 data = fp.read()
1361 fp.close()
1362 # create build_time_vars dict
1363 if RUNNING_ON_PYTHON2:
1364 exec(data)
1365 else:
1366 g_dict = {}
1367 l_dict = {}
1368 exec(data, g_dict, l_dict)
1369 build_time_vars = l_dict['build_time_vars']
1370 vars = {}
1371 for k, v in build_time_vars.items():
1372 if isinstance(v, str):
1373 for p in (include_path, lib_path):
1374 v = v.replace(' ' + p, '')
1375 v = v.replace(p + ' ', '')
1376 vars[k] = v
1377
1378 fp = open(path, 'w')
1379 # duplicated from sysconfig._generate_posix_vars()
1380 fp.write('# system configuration generated and used by'
1381 ' the sysconfig module\n')
1382 fp.write('build_time_vars = ')
1383 pprint.pprint(vars, stream=fp)
1384 fp.close()
1385
1386 # Add symlinks in /usr/local/bin, using relative links
1387 usr_local_bin = os.path.join(rootDir, 'usr', 'local', 'bin')
1388 to_framework = os.path.join('..', '..', '..', 'Library', 'Frameworks',
1389 'Python.framework', 'Versions', version, 'bin')
1390 if os.path.exists(usr_local_bin):
1391 shutil.rmtree(usr_local_bin)
1392 os.makedirs(usr_local_bin)
1393 for fn in os.listdir(
1394 os.path.join(frmDir, 'Versions', version, 'bin')):
1395 os.symlink(os.path.join(to_framework, fn),
1396 os.path.join(usr_local_bin, fn))
1397
1398 os.chdir(curdir)
1399
1400 def patchFile(inPath, outPath):
1401 data = fileContents(inPath)
1402 data = data.replace('$FULL_VERSION', getFullVersion())
1403 data = data.replace('$VERSION', getVersion())
1404 data = data.replace('$MACOSX_DEPLOYMENT_TARGET', ''.join((DEPTARGET, ' or later')))
1405 data = data.replace('$ARCHITECTURES', ", ".join(universal_opts_map[UNIVERSALARCHS]))
1406 data = data.replace('$INSTALL_SIZE', installSize())
1407 data = data.replace('$THIRD_PARTY_LIBS', "\\\n".join(THIRD_PARTY_LIBS))
1408
1409 # This one is not handy as a template variable
1410 data = data.replace('$PYTHONFRAMEWORKINSTALLDIR', '/Library/Frameworks/Python.framework')
1411 fp = open(outPath, 'w')
1412 fp.write(data)
1413 fp.close()
1414
1415 def patchScript(inPath, outPath):
1416 major, minor = getVersionMajorMinor()
1417 data = fileContents(inPath)
1418 data = data.replace('@PYMAJOR@', str(major))
1419 data = data.replace('@PYVER@', getVersion())
1420 fp = open(outPath, 'w')
1421 fp.write(data)
1422 fp.close()
1423 os.chmod(outPath, STAT_0o755)
1424
1425
1426
1427 def packageFromRecipe(targetDir, recipe):
1428 curdir = os.getcwd()
1429 try:
1430 # The major version (such as 2.5) is included in the package name
1431 # because having two version of python installed at the same time is
1432 # common.
1433 pkgname = '%s-%s'%(recipe['name'], getVersion())
1434 srcdir = recipe.get('source')
1435 pkgroot = recipe.get('topdir', srcdir)
1436 postflight = recipe.get('postflight')
1437 readme = textwrap.dedent(recipe['readme'])
1438 isRequired = recipe.get('required', True)
1439
1440 print("- building package %s"%(pkgname,))
1441
1442 # Substitute some variables
1443 textvars = dict(
1444 VER=getVersion(),
1445 FULLVER=getFullVersion(),
1446 )
1447 readme = readme % textvars
1448
1449 if pkgroot is not None:
1450 pkgroot = pkgroot % textvars
1451 else:
1452 pkgroot = '/'
1453
1454 if srcdir is not None:
1455 srcdir = os.path.join(WORKDIR, '_root', srcdir[1:])
1456 srcdir = srcdir % textvars
1457
1458 if postflight is not None:
1459 postflight = os.path.abspath(postflight)
1460
1461 packageContents = os.path.join(targetDir, pkgname + '.pkg', 'Contents')
1462 os.makedirs(packageContents)
1463
1464 if srcdir is not None:
1465 os.chdir(srcdir)
1466 runCommand("pax -wf %s . 2>&1"%(shellQuote(os.path.join(packageContents, 'Archive.pax')),))
1467 runCommand("gzip -9 %s 2>&1"%(shellQuote(os.path.join(packageContents, 'Archive.pax')),))
1468 runCommand("mkbom . %s 2>&1"%(shellQuote(os.path.join(packageContents, 'Archive.bom')),))
1469
1470 fn = os.path.join(packageContents, 'PkgInfo')
1471 fp = open(fn, 'w')
1472 fp.write('pmkrpkg1')
1473 fp.close()
1474
1475 rsrcDir = os.path.join(packageContents, "Resources")
1476 os.mkdir(rsrcDir)
1477 fp = open(os.path.join(rsrcDir, 'ReadMe.txt'), 'w')
1478 fp.write(readme)
1479 fp.close()
1480
1481 if postflight is not None:
1482 patchScript(postflight, os.path.join(rsrcDir, 'postflight'))
1483
1484 vers = getFullVersion()
1485 major, minor = getVersionMajorMinor()
1486 pl = dict(
1487 CFBundleGetInfoString="Python.%s %s"%(pkgname, vers,),
1488 CFBundleIdentifier='org.python.Python.%s'%(pkgname,),
1489 CFBundleName='Python.%s'%(pkgname,),
1490 CFBundleShortVersionString=vers,
1491 IFMajorVersion=major,
1492 IFMinorVersion=minor,
1493 IFPkgFormatVersion=0.10000000149011612,
1494 IFPkgFlagAllowBackRev=False,
1495 IFPkgFlagAuthorizationAction="RootAuthorization",
1496 IFPkgFlagDefaultLocation=pkgroot,
1497 IFPkgFlagFollowLinks=True,
1498 IFPkgFlagInstallFat=True,
1499 IFPkgFlagIsRequired=isRequired,
1500 IFPkgFlagOverwritePermissions=False,
1501 IFPkgFlagRelocatable=False,
1502 IFPkgFlagRestartAction="NoRestart",
1503 IFPkgFlagRootVolumeOnly=True,
1504 IFPkgFlagUpdateInstalledLanguages=False,
1505 )
1506 writePlist(pl, os.path.join(packageContents, 'Info.plist'))
1507
1508 pl = dict(
1509 IFPkgDescriptionDescription=readme,
1510 IFPkgDescriptionTitle=recipe.get('long_name', "Python.%s"%(pkgname,)),
1511 IFPkgDescriptionVersion=vers,
1512 )
1513 writePlist(pl, os.path.join(packageContents, 'Resources', 'Description.plist'))
1514
1515 finally:
1516 os.chdir(curdir)
1517
1518
1519 def makeMpkgPlist(path):
1520
1521 vers = getFullVersion()
1522 major, minor = getVersionMajorMinor()
1523
1524 pl = dict(
1525 CFBundleGetInfoString="Python %s"%(vers,),
1526 CFBundleIdentifier='org.python.Python',
1527 CFBundleName='Python',
1528 CFBundleShortVersionString=vers,
1529 IFMajorVersion=major,
1530 IFMinorVersion=minor,
1531 IFPkgFlagComponentDirectory="Contents/Packages",
1532 IFPkgFlagPackageList=[
1533 dict(
1534 IFPkgFlagPackageLocation='%s-%s.pkg'%(item['name'], getVersion()),
1535 IFPkgFlagPackageSelection=item.get('selected', 'selected'),
1536 )
1537 for item in pkg_recipes()
1538 ],
1539 IFPkgFormatVersion=0.10000000149011612,
1540 IFPkgFlagBackgroundScaling="proportional",
1541 IFPkgFlagBackgroundAlignment="left",
1542 IFPkgFlagAuthorizationAction="RootAuthorization",
1543 )
1544
1545 writePlist(pl, path)
1546
1547
1548 def buildInstaller():
1549
1550 # Zap all compiled files
1551 for dirpath, _, filenames in os.walk(os.path.join(WORKDIR, '_root')):
1552 for fn in filenames:
1553 if fn.endswith('.pyc') or fn.endswith('.pyo'):
1554 os.unlink(os.path.join(dirpath, fn))
1555
1556 outdir = os.path.join(WORKDIR, 'installer')
1557 if os.path.exists(outdir):
1558 shutil.rmtree(outdir)
1559 os.mkdir(outdir)
1560
1561 pkgroot = os.path.join(outdir, 'Python.mpkg', 'Contents')
1562 pkgcontents = os.path.join(pkgroot, 'Packages')
1563 os.makedirs(pkgcontents)
1564 for recipe in pkg_recipes():
1565 packageFromRecipe(pkgcontents, recipe)
1566
1567 rsrcDir = os.path.join(pkgroot, 'Resources')
1568
1569 fn = os.path.join(pkgroot, 'PkgInfo')
1570 fp = open(fn, 'w')
1571 fp.write('pmkrpkg1')
1572 fp.close()
1573
1574 os.mkdir(rsrcDir)
1575
1576 makeMpkgPlist(os.path.join(pkgroot, 'Info.plist'))
1577 pl = dict(
1578 IFPkgDescriptionTitle="Python",
1579 IFPkgDescriptionVersion=getVersion(),
1580 )
1581
1582 writePlist(pl, os.path.join(pkgroot, 'Resources', 'Description.plist'))
1583 for fn in os.listdir('resources'):
1584 if fn == '.svn': continue
1585 if fn.endswith('.jpg'):
1586 shutil.copy(os.path.join('resources', fn), os.path.join(rsrcDir, fn))
1587 else:
1588 patchFile(os.path.join('resources', fn), os.path.join(rsrcDir, fn))
1589
1590
1591 def installSize(clear=False, _saved=[]):
1592 if clear:
1593 del _saved[:]
1594 if not _saved:
1595 data = captureCommand("du -ks %s"%(
1596 shellQuote(os.path.join(WORKDIR, '_root'))))
1597 _saved.append("%d"%((0.5 + (int(data.split()[0]) / 1024.0)),))
1598 return _saved[0]
1599
1600
1601 def buildDMG():
1602 """
1603 Create DMG containing the rootDir.
1604 """
1605 outdir = os.path.join(WORKDIR, 'diskimage')
1606 if os.path.exists(outdir):
1607 shutil.rmtree(outdir)
1608
1609 # We used to use the deployment target as the last characters of the
1610 # installer file name. With the introduction of weaklinked installer
1611 # variants, we may have two variants with the same file name, i.e.
1612 # both ending in '10.9'. To avoid this, we now use the major/minor
1613 # version numbers of the macOS version we are building on.
1614 # Also, as of macOS 11, operating system version numbering has
1615 # changed from three components to two, i.e.
1616 # 10.14.1, 10.14.2, ...
1617 # 10.15.1, 10.15.2, ...
1618 # 11.1, 11.2, ...
1619 # 12.1, 12.2, ...
1620 # (A further twist is that, when running on macOS 11, binaries built
1621 # on older systems may be shown an operating system version of 10.16
1622 # instead of 11. We should not run into that situation here.)
1623 # Also we should use "macos" instead of "macosx" going forward.
1624 #
1625 # To maintain compatibility for legacy variants, the file name for
1626 # builds on macOS 10.15 and earlier remains:
1627 # python-3.x.y-macosx10.z.{dmg->pkg}
1628 # e.g. python-3.9.4-macosx10.9.{dmg->pkg}
1629 # and for builds on macOS 11+:
1630 # python-3.x.y-macosz.{dmg->pkg}
1631 # e.g. python-3.9.4-macos11.{dmg->pkg}
1632
1633 build_tuple = getBuildTuple()
1634 if build_tuple[0] < 11:
1635 os_name = 'macosx'
1636 build_system_version = '%s.%s' % build_tuple
1637 else:
1638 os_name = 'macos'
1639 build_system_version = str(build_tuple[0])
1640 imagepath = os.path.join(outdir,
1641 'python-%s-%s%s'%(getFullVersion(),os_name,build_system_version))
1642 if INCLUDE_TIMESTAMP:
1643 imagepath = imagepath + '-%04d-%02d-%02d'%(time.localtime()[:3])
1644 imagepath = imagepath + '.dmg'
1645
1646 os.mkdir(outdir)
1647
1648 # Try to mitigate race condition in certain versions of macOS, e.g. 10.9,
1649 # when hdiutil create fails with "Resource busy". For now, just retry
1650 # the create a few times and hope that it eventually works.
1651
1652 volname='Python %s'%(getFullVersion())
1653 cmd = ("hdiutil create -format UDRW -volname %s -srcfolder %s -size 100m %s"%(
1654 shellQuote(volname),
1655 shellQuote(os.path.join(WORKDIR, 'installer')),
1656 shellQuote(imagepath + ".tmp.dmg" )))
1657 for i in range(5):
1658 fd = os.popen(cmd, 'r')
1659 data = fd.read()
1660 xit = fd.close()
1661 if not xit:
1662 break
1663 sys.stdout.write(data)
1664 print(" -- retrying hdiutil create")
1665 time.sleep(5)
1666 else:
1667 raise RuntimeError("command failed: %s"%(cmd,))
1668
1669 if not os.path.exists(os.path.join(WORKDIR, "mnt")):
1670 os.mkdir(os.path.join(WORKDIR, "mnt"))
1671 runCommand("hdiutil attach %s -mountroot %s"%(
1672 shellQuote(imagepath + ".tmp.dmg"), shellQuote(os.path.join(WORKDIR, "mnt"))))
1673
1674 # Custom icon for the DMG, shown when the DMG is mounted.
1675 shutil.copy("../Icons/Disk Image.icns",
1676 os.path.join(WORKDIR, "mnt", volname, ".VolumeIcon.icns"))
1677 runCommand("SetFile -a C %s/"%(
1678 shellQuote(os.path.join(WORKDIR, "mnt", volname)),))
1679
1680 runCommand("hdiutil detach %s"%(shellQuote(os.path.join(WORKDIR, "mnt", volname))))
1681
1682 setIcon(imagepath + ".tmp.dmg", "../Icons/Disk Image.icns")
1683 runCommand("hdiutil convert %s -format UDZO -o %s"%(
1684 shellQuote(imagepath + ".tmp.dmg"), shellQuote(imagepath)))
1685 setIcon(imagepath, "../Icons/Disk Image.icns")
1686
1687 os.unlink(imagepath + ".tmp.dmg")
1688
1689 return imagepath
1690
1691
1692 def setIcon(filePath, icnsPath):
1693 """
1694 Set the custom icon for the specified file or directory.
1695 """
1696
1697 dirPath = os.path.normpath(os.path.dirname(__file__))
1698 toolPath = os.path.join(dirPath, "seticon.app/Contents/MacOS/seticon")
1699 if not os.path.exists(toolPath) or os.stat(toolPath).st_mtime < os.stat(dirPath + '/seticon.m').st_mtime:
1700 # NOTE: The tool is created inside an .app bundle, otherwise it won't work due
1701 # to connections to the window server.
1702 appPath = os.path.join(dirPath, "seticon.app/Contents/MacOS")
1703 if not os.path.exists(appPath):
1704 os.makedirs(appPath)
1705 runCommand("cc -o %s %s/seticon.m -framework Cocoa"%(
1706 shellQuote(toolPath), shellQuote(dirPath)))
1707
1708 runCommand("%s %s %s"%(shellQuote(os.path.abspath(toolPath)), shellQuote(icnsPath),
1709 shellQuote(filePath)))
1710
1711 def main():
1712 # First parse options and check if we can perform our work
1713 parseOptions()
1714 checkEnvironment()
1715
1716 os.environ['MACOSX_DEPLOYMENT_TARGET'] = DEPTARGET
1717 os.environ['CC'] = CC
1718 os.environ['CXX'] = CXX
1719
1720 if os.path.exists(WORKDIR):
1721 shutil.rmtree(WORKDIR)
1722 os.mkdir(WORKDIR)
1723
1724 os.environ['LC_ALL'] = 'C'
1725
1726 # Then build third-party libraries such as sleepycat DB4.
1727 buildLibraries()
1728
1729 # Now build python itself
1730 buildPython()
1731
1732 # And then build the documentation
1733 # Remove the Deployment Target from the shell
1734 # environment, it's no longer needed and
1735 # an unexpected build target can cause problems
1736 # when Sphinx and its dependencies need to
1737 # be (re-)installed.
1738 del os.environ['MACOSX_DEPLOYMENT_TARGET']
1739 buildPythonDocs()
1740
1741
1742 # Prepare the applications folder
1743 folder = os.path.join(WORKDIR, "_root", "Applications", "Python %s"%(
1744 getVersion(),))
1745 fn = os.path.join(folder, "License.rtf")
1746 patchFile("resources/License.rtf", fn)
1747 fn = os.path.join(folder, "ReadMe.rtf")
1748 patchFile("resources/ReadMe.rtf", fn)
1749 fn = os.path.join(folder, "Update Shell Profile.command")
1750 patchScript("resources/update_shell_profile.command", fn)
1751 fn = os.path.join(folder, "Install Certificates.command")
1752 patchScript("resources/install_certificates.command", fn)
1753 os.chmod(folder, STAT_0o755)
1754 setIcon(folder, "../Icons/Python Folder.icns")
1755
1756 # Create the installer
1757 buildInstaller()
1758
1759 # And copy the readme into the directory containing the installer
1760 patchFile('resources/ReadMe.rtf',
1761 os.path.join(WORKDIR, 'installer', 'ReadMe.rtf'))
1762
1763 # Ditto for the license file.
1764 patchFile('resources/License.rtf',
1765 os.path.join(WORKDIR, 'installer', 'License.rtf'))
1766
1767 fp = open(os.path.join(WORKDIR, 'installer', 'Build.txt'), 'w')
1768 fp.write("# BUILD INFO\n")
1769 fp.write("# Date: %s\n" % time.ctime())
1770 fp.write("# By: %s\n" % pwd.getpwuid(os.getuid()).pw_gecos)
1771 fp.close()
1772
1773 # And copy it to a DMG
1774 buildDMG()
1775
1776 if __name__ == "__main__":
1777 main()