Debian Codegraph @

bypy/macos/__main__.py

raw
1 #!/usr/bin/env python
2 # vim:fileencoding=utf-8
3 # License: GPLv3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
4
5 import errno
6 import glob
7 import json
8 import operator
9 import os
10 import plistlib
11 import runpy
12 import shutil
13 import stat
14 import subprocess
15 import sys
16 import tempfile
17 import time
18 import zipfile
19 from functools import partial, reduce
20 from itertools import repeat
21
22 import icnsutil
23
24 from bypy.constants import OUTPUT_DIR, PREFIX, PYTHON, python_major_minor_version
25 from bypy.constants import SRC as CALIBRE_DIR
26 from bypy.freeze import extract_extension_modules, fix_pycryptodome, freeze_python, is_package_dir, path_to_freeze_dir
27 from bypy.utils import current_dir, get_arches_in_binary, mkdtemp, py_compile, timeit, walk
28
29 abspath, join, basename, dirname = os.path.abspath, os.path.join, os.path.basename, os.path.dirname
30 iv = globals()['init_env']
31 calibre_constants = iv['calibre_constants']
32 QT_DLLS, QT_PLUGINS, PYQT_MODULES = iv['QT_DLLS'], iv['QT_PLUGINS'], iv['PYQT_MODULES']
33 QT_MAJOR = iv['QT_MAJOR']
34 py_ver = '.'.join(map(str, python_major_minor_version()))
35 sign_app = runpy.run_path(join(dirname(abspath(__file__)), 'sign.py'))['sign_app']
36
37 QT_PREFIX = join(PREFIX, 'qt')
38 QT_FRAMEWORKS = [x.replace(f'{QT_MAJOR}', '') for x in QT_DLLS]
39
40 ENV = dict(
41 FONTCONFIG_PATH='@executable_path/../Resources/fonts',
42 FONTCONFIG_FILE='@executable_path/../Resources/fonts/fonts.conf',
43 SSL_CERT_DIR='@executable_path/../Resources/resources/mozilla-ca-certs',
44 OPENSSL_ENGINES='@executable_path/../Frameworks/engines-3',
45 OPENSSL_MODULES='@executable_path/../Frameworks/ossl-modules',
46 )
47 APPNAME, VERSION = calibre_constants['appname'], calibre_constants['version']
48 basenames, main_modules, main_functions = calibre_constants['basenames'], calibre_constants['modules'], calibre_constants['functions']
49 ARCH_FLAGS = '-arch x86_64 -arch arm64'.split()
50 EXPECTED_ARCHES = {'x86_64', 'arm64'}
51 MINIMUM_SYSTEM_VERSION = '14.0.0'
52
53
54 def generate_icns(light_iconset: str, dark_iconset: str, output_path: str) -> None:
55 def add_iconset(icns, path):
56 for img in sorted(os.listdir(path)):
57 if img.endswith('.png'):
58 icns.add_media(file=os.path.join(path, img))
59 dark_icns = icnsutil.IcnsFile()
60 add_iconset(dark_icns, dark_iconset)
61 dark_icns.write(output_path)
62 light_icns = icnsutil.IcnsFile()
63 add_iconset(light_icns, light_iconset)
64 light_icns.add_media(icnsutil.IcnsType.key_from_readable('dark'), file=output_path)
65 light_icns.write(output_path)
66
67
68 def compile_launcher_lib(contents_dir, base, pyver, inc_dir):
69 print('\tCompiling calibre_launcher.dylib')
70 env, env_vals = [], []
71 for key, val in ENV.items():
72 env.append(f'"{key}"'), env_vals.append(f'"{val}"')
73 env = ','.join(env)
74 env_vals = ','.join(env_vals)
75
76 dest = join(contents_dir, 'Frameworks', 'calibre-launcher.dylib')
77 src = join(base, 'util.c')
78 cmd = gcc + ARCH_FLAGS + CFLAGS + '-Wall -dynamiclib -std=gnu99'.split() + [src] + \
79 ['-I' + base] + '-DPY_VERSION_MAJOR={} -DPY_VERSION_MINOR={}'.format(*pyver.split('.')).split() + \
80 [f'-I{path_to_freeze_dir()}', f'-I{inc_dir}'] + \
81 [f'-DENV_VARS={env}', f'-DENV_VAR_VALS={env_vals}'] + \
82 ['-I%s/python/Python.framework/Versions/Current/Headers' % PREFIX] + \
83 '-current_version 1.0 -compatibility_version 1.0'.split() + \
84 '-fvisibility=hidden -o'.split() + [dest]
85 # We need libxml2.dylib linked because Apple's system frameworks link
86 # against a system version. And libxml2 uses global variables so we get
87 # crashes if the system libxml2 is loaded. Loading plugins like the Qt
88 # ones or usbobserver causes it to be loaded. So pre-load our libxml2
89 # to avoid it.
90 cmd += ['-L', f'{PREFIX}/lib', '-l', 'xml2', '-l', 'xslt',] + \
91 ['-install_name',
92 '@executable_path/../Frameworks/' + os.path.basename(dest)] + \
93 [('-F%s/python' % PREFIX), '-framework', 'Python', '-framework', 'CoreFoundation', '-headerpad_max_install_names']
94 # print('\t'+' '.join(cmd))
95 sys.stdout.flush()
96 subprocess.check_call(cmd)
97 return dest
98
99
100 gcc = os.environ.get('CC', 'clang').split()
101 CFLAGS = os.environ.get('CFLAGS', '-Os').split()
102
103
104 def compile_launchers(contents_dir, inc_dir, xprograms, pyver):
105 base = dirname(abspath(__file__))
106 lib = compile_launcher_lib(contents_dir, base, pyver, inc_dir)
107 src = join(base, 'launcher.c')
108 programs = [lib]
109 for program, x in xprograms.items():
110 module, func, ptype = x
111 print('\tCompiling', program)
112 out = join(contents_dir, 'MacOS', program)
113 programs.append(out)
114 is_gui = 'true' if ptype == 'gui' else 'false'
115 cmd = gcc + ARCH_FLAGS + CFLAGS + [
116 '-Wall', f'-DPROGRAM=L"{program}"', f'-DMODULE=L"{module}"', f'-DFUNCTION=L"{func}"', f'-DIS_GUI={is_gui}',
117 '-I' + base, src, lib, '-o', out, '-headerpad_max_install_names',
118 ]
119 # print('\t'+' '.join(cmd))
120 sys.stdout.flush()
121 subprocess.check_call(cmd)
122 return programs
123
124
125 def flipwritable(fn, mode=None):
126 """
127 Flip the writability of a file and return the old mode. Returns None
128 if the file is already writable.
129 """
130 if os.access(fn, os.W_OK):
131 return None
132 old_mode = os.stat(fn).st_mode
133 os.chmod(fn, stat.S_IWRITE | old_mode)
134 return old_mode
135
136
137 def check_universal(path):
138 arches = get_arches_in_binary(path)
139 if arches != EXPECTED_ARCHES:
140 raise SystemExit(f'The file {path} is not a universal binary, it only has arches: {", ".join(arches)}')
141
142
143 STRIPCMD = ['/usr/bin/strip', '-x', '-S', '-']
144
145
146 def strip_files(files, argv_max=(256 * 1024)):
147 """
148 Strip a list of files
149 """
150 tostrip = [(fn, flipwritable(fn)) for fn in files if os.path.exists(fn)]
151 while tostrip:
152 cmd = list(STRIPCMD)
153 flips = []
154 pathlen = reduce(operator.add, [len(s) + 1 for s in cmd])
155 while pathlen < argv_max:
156 if not tostrip:
157 break
158 added, flip = tostrip.pop()
159 pathlen += len(added) + 1
160 cmd.append(added)
161 flips.append((added, flip))
162 else:
163 cmd.pop()
164 tostrip.append(flips.pop())
165 os.spawnv(os.P_WAIT, cmd[0], cmd)
166 for args in flips:
167 flipwritable(*args)
168
169
170 def flush(func):
171 def ff(*args, **kwargs):
172 sys.stdout.flush()
173 sys.stderr.flush()
174 ret = func(*args, **kwargs)
175 sys.stdout.flush()
176 sys.stderr.flush()
177 return ret
178 return ff
179
180
181 class Freeze:
182
183 FID = '@executable_path/../Frameworks'
184
185 def __init__(self, build_dir, ext_dir, inc_dir, test_runner, test_launchers=False, dont_strip=False, sign_installers=False, notarize=False):
186 self.build_dir = os.path.realpath(build_dir)
187 self.inc_dir = os.path.realpath(inc_dir)
188 self.sign_installers = sign_installers
189 self.notarize = notarize
190 self.ext_dir = os.path.realpath(ext_dir)
191 self.test_runner = test_runner
192 self.dont_strip = dont_strip
193 self.contents_dir = join(self.build_dir, 'Contents')
194 self.resources_dir = join(self.contents_dir, 'Resources')
195 self.frameworks_dir = join(self.contents_dir, 'Frameworks')
196 self.exe_dir = join(self.contents_dir, 'MacOS')
197 self.helpers_dir = join(self.contents_dir, 'utils.app', 'Contents', 'MacOS')
198 self.site_packages = join(self.resources_dir, 'Python', 'site-packages')
199 self.to_strip = []
200 self.warnings = []
201
202 self.run(test_launchers)
203
204 def run(self, test_launchers):
205 ret = 0
206 self.ext_map = {}
207 if not test_launchers:
208 if os.path.exists(self.build_dir):
209 shutil.rmtree(self.build_dir)
210 os.makedirs(self.build_dir)
211 self.create_skeleton()
212 self.create_plist()
213
214 self.add_python_framework()
215 self.add_site_packages()
216 self.add_stdlib()
217 self.add_qt_frameworks()
218 self.add_calibre_plugins()
219 self.add_podofo()
220 self.add_poppler()
221 self.add_imaging_libs()
222 self.add_fontconfig()
223 self.add_misc_libraries()
224
225 self.add_resources()
226 self.copy_site()
227 self.compile_py_modules()
228
229 self.create_exe()
230 if not test_launchers and not self.dont_strip:
231 self.strip_files()
232 if not test_launchers:
233 self.create_gui_apps()
234
235 self.run_tests()
236 ret = self.makedmg(self.build_dir, APPNAME + '-' + VERSION)
237
238 return ret
239
240 @flush
241 def run_tests(self):
242 print('Running tests...', flush=True)
243 self.test_runner(join(self.contents_dir, 'MacOS', 'calibre-debug'), self.contents_dir)
244
245 @flush
246 def add_resources(self):
247 shutil.copytree('resources', join(self.resources_dir,
248 'resources'))
249
250 @flush
251 def strip_files(self):
252 print('\nStripping files...')
253 strip_files(self.to_strip)
254
255 @flush
256 def create_exe(self):
257 print('\nCreating launchers')
258 programs = {}
259 progs = []
260 for x in ('console', 'gui'):
261 progs += list(zip(basenames[x], main_modules[x], main_functions[x], repeat(x)))
262 for program, module, func, ptype in progs:
263 programs[program] = (module, func, ptype)
264 programs = compile_launchers(self.contents_dir, self.inc_dir, programs, py_ver)
265 for out in programs:
266 self.fix_dependencies_in_lib(out)
267
268 @flush
269 def set_id(self, path_to_lib, new_id):
270 old_mode = flipwritable(path_to_lib)
271 subprocess.check_call(['install_name_tool', '-id', new_id, path_to_lib])
272 if old_mode is not None:
273 flipwritable(path_to_lib, old_mode)
274
275 @flush
276 def get_dependencies(self, path_to_lib):
277 install_name = subprocess.check_output(['otool', '-D', path_to_lib]).splitlines()[-1].strip()
278 raw = subprocess.check_output(['otool', '-L', path_to_lib]).decode('utf-8')
279 for line in raw.splitlines():
280 if 'compatibility' not in line or line.strip().endswith(':'):
281 continue
282 idx = line.find('(')
283 path = line[:idx].strip()
284 yield path, path == install_name
285
286 @flush
287 def get_local_dependencies(self, path_to_lib):
288 for x, is_id in self.get_dependencies(path_to_lib):
289 if x in ('libunrar.dylib', 'libstemmer.0.dylib', 'libstemmer.dylib', 'libjbig.2.1.dylib') and not is_id:
290 yield x, x, is_id
291 else:
292 for y in ('@rpath/', PREFIX + '/lib/', PREFIX + '/python/Python.framework/', PREFIX + '/ffmpeg/lib/'):
293 if x.startswith(y):
294 if y == PREFIX + '/python/Python.framework/':
295 y = PREFIX + '/python/'
296 yield x, x[len(y):], is_id
297 break
298
299 @flush
300 def change_dep(self, old_dep, new_dep, is_id, path_to_lib):
301 cmd = ['-id', new_dep] if is_id else ['-change', old_dep, new_dep]
302 subprocess.check_call(['install_name_tool'] + cmd + [path_to_lib])
303
304 @flush
305 def fix_dependencies_in_lib(self, path_to_lib):
306 check_universal(path_to_lib)
307 self.to_strip.append(path_to_lib)
308 old_mode = flipwritable(path_to_lib)
309 for dep, bname, is_id in self.get_local_dependencies(path_to_lib):
310 ndep = self.FID + '/' + bname
311 self.change_dep(dep, ndep, is_id, path_to_lib)
312 ldeps = list(self.get_local_dependencies(path_to_lib))
313 if ldeps:
314 print('\nFailed to fix dependencies in', path_to_lib)
315 print('Remaining local dependencies:', ldeps)
316 raise SystemExit(1)
317 if old_mode is not None:
318 flipwritable(path_to_lib, old_mode)
319
320 @flush
321 def add_python_framework(self):
322 print('\nAdding Python framework')
323 src = join(PREFIX + '/python', 'Python.framework')
324 x = join(self.frameworks_dir, 'Python.framework')
325 curr = os.path.realpath(join(src, 'Versions', 'Current'))
326 currd = join(x, 'Versions', basename(curr))
327 rd = join(currd, 'Resources')
328 os.makedirs(rd)
329 shutil.copy2(join(curr, 'Resources', 'Info.plist'), rd)
330 shutil.copy2(join(curr, 'Python'), currd)
331 self.set_id(join(currd, 'Python'),
332 self.FID + '/Python.framework/Versions/%s/Python' % basename(curr))
333 # The following is needed for codesign in OS X >= 10.9.5
334 with current_dir(x):
335 os.symlink(basename(curr), 'Versions/Current')
336 for y in ('Python', 'Resources'):
337 os.symlink('Versions/Current/%s' % y, y)
338
339 @flush
340 def add_qt_frameworks(self):
341 print('\nAdding Qt Frameworks')
342 # First add FFMPEG
343 for x in os.listdir(join(PREFIX, 'ffmpeg', 'lib')):
344 if x.endswith('.dylib') and x.count('.') == 2:
345 src = join(PREFIX, 'ffmpeg', 'lib', x)
346 shutil.copy2(src, self.frameworks_dir)
347 dest = join(self.frameworks_dir, os.path.basename(src))
348 self.set_id(dest, self.FID + '/' + os.path.basename(src))
349 self.fix_dependencies_in_lib(dest)
350 for f in QT_FRAMEWORKS:
351 self.add_qt_framework(f)
352 pdir = join(QT_PREFIX, 'plugins')
353 ddir = join(self.contents_dir, 'PlugIns')
354 os.mkdir(ddir)
355 for x in QT_PLUGINS:
356 shutil.copytree(join(pdir, x), join(ddir, x))
357 for l in glob.glob(join(ddir, '*/*.dylib')):
358 self.fix_dependencies_in_lib(l)
359 x = os.path.relpath(l, ddir)
360 self.set_id(l, '@executable_path/' + x)
361 webengine_process = os.path.realpath(join(
362 self.frameworks_dir, 'QtWebEngineCore.framework/Versions/Current/Helpers/QtWebEngineProcess.app/Contents/MacOS/QtWebEngineProcess'))
363 self.fix_dependencies_in_lib(webengine_process)
364 cdir = dirname(dirname(webengine_process))
365 dest = join(cdir, 'Frameworks')
366 os.symlink(os.path.relpath(self.frameworks_dir, cdir), dest)
367
368 def add_qt_framework(self, f):
369 libname = f
370 f = f + '.framework'
371 src = join(PREFIX, 'qt', 'lib', f)
372 ignore = shutil.ignore_patterns('Headers', '*.h', 'Headers/*')
373 dest = join(self.frameworks_dir, f)
374 shutil.copytree(src, dest, symlinks=True,
375 ignore=ignore)
376 lib = os.path.realpath(join(dest, libname))
377 rpath = os.path.relpath(lib, self.frameworks_dir)
378 self.set_id(lib, self.FID + '/' + rpath)
379 self.fix_dependencies_in_lib(lib)
380 # The following is needed for codesign in OS X >= 10.9.5
381 # The presence of the .prl file in the root of the framework causes
382 # codesign to fail.
383 with current_dir(dest):
384 for x in os.listdir('.'):
385 if x != 'Versions' and not os.path.islink(x):
386 os.remove(x)
387
388 @flush
389 def create_skeleton(self):
390 print('Creating skeleton')
391 c = join(self.build_dir, 'Contents')
392 for x in ('Frameworks', 'MacOS', 'Resources'):
393 os.makedirs(join(c, x))
394 icns_dir = join(CALIBRE_DIR, 'icons', 'icns')
395 shutil.copy(join(icns_dir, 'Assets.car'), self.resources_dir)
396 for x in glob.glob(join(icns_dir, '*.icns')):
397 shutil.copy(x, self.resources_dir)
398 for helpers in (self.helpers_dir,):
399 os.makedirs(helpers)
400 cdir = dirname(helpers)
401 dest = join(cdir, 'Frameworks')
402 src = self.frameworks_dir
403 os.symlink(os.path.relpath(src, cdir), dest)
404 dest = join(cdir, 'Resources')
405 src = self.resources_dir
406 os.symlink(os.path.relpath(src, cdir), dest)
407 pl = dict(
408 CFBundleDevelopmentRegion='English',
409 CFBundleDisplayName=APPNAME + ' - utils',
410 CFBundleName=APPNAME + '-utils',
411 CFBundleIdentifier='com.calibre-ebook.utils',
412 LSBackgroundOnly='1',
413 CFBundleVersion=VERSION,
414 CFBundleShortVersionString=VERSION,
415 CFBundlePackageType='APPL',
416 CFBundleSignature='????',
417 CFBundleExecutable='pdftohtml',
418 LSMinimumSystemVersion=MINIMUM_SYSTEM_VERSION,
419 LSRequiresNativeExecution=True,
420 NSAppleScriptEnabled=False,
421 CFBundleIconFile='book.icns',
422 CFBundleIconName='book',
423 )
424 with open(join(cdir, 'Info.plist'), 'wb') as p:
425 plistlib.dump(pl, p)
426
427 @flush
428 def add_calibre_plugins(self):
429 dest = join(self.frameworks_dir, 'plugins')
430 os.mkdir(dest)
431 print('Extracting extension modules from:', self.ext_dir, 'to', dest)
432 self.ext_map = extract_extension_modules(self.ext_dir, dest)
433 plugins = glob.glob(dest + '/*.so')
434 if not plugins:
435 raise SystemExit('No calibre plugins found in: ' + self.ext_dir)
436 for f in plugins:
437 self.fix_dependencies_in_lib(f)
438
439 @flush
440 def create_plist(self):
441 BOOK_EXTENSIONS = calibre_constants['book_extensions']
442 env = dict(**ENV)
443 env['CALIBRE_LAUNCHED_FROM_BUNDLE'] = '1'
444 docs = [{
445 'CFBundleTypeName': 'E-book',
446 'CFBundleTypeExtensions': list(BOOK_EXTENSIONS),
447 'CFBundleTypeIconFile': 'book.icns',
448 'CFBundleTypeRole': 'Viewer',
449 }]
450 url_handlers = [dict(
451 CFBundleTypeRole='Viewer',
452 CFBundleURLIconFile='calibre',
453 CFBundleURLName='com.calibre-ebook.calibre-url',
454 CFBundleURLSchemes=['calibre']
455 )]
456
457 pl = dict(
458 CFBundleDevelopmentRegion='English',
459 CFBundleDisplayName=APPNAME,
460 CFBundleName=APPNAME,
461 CFBundleIdentifier='net.kovidgoyal.calibre',
462 CFBundleVersion=VERSION,
463 CFBundleShortVersionString=VERSION,
464 CFBundlePackageType='APPL',
465 CFBundleSignature='????',
466 CFBundleExecutable='calibre',
467 CFBundleDocumentTypes=docs,
468 CFBundleURLTypes=url_handlers,
469 LSMinimumSystemVersion=MINIMUM_SYSTEM_VERSION,
470 LSRequiresNativeExecution=True,
471 NSAppleScriptEnabled=False,
472 NSSupportsAutomaticGraphicsSwitching=True,
473 NSHumanReadableCopyright=time.strftime('Copyright %Y, Kovid Goyal'),
474 CFBundleGetInfoString=('calibre, an E-book management '
475 'application. Visit https://calibre-ebook.com for details.'),
476 CFBundleIconName='calibre',
477 CFBundleIconFile='calibre.icns',
478 NSHighResolutionCapable=True,
479 LSApplicationCategoryType='public.app-category.productivity',
480 LSEnvironment=env
481 )
482 with open(join(self.contents_dir, 'Info.plist'), 'wb') as p:
483 plistlib.dump(pl, p)
484
485 @flush
486 def install_dylib(self, path, set_id=True, dest=None):
487 dest = dest or self.frameworks_dir
488 os.makedirs(dest, exist_ok=True)
489 shutil.copy2(path, dest)
490 if set_id:
491 self.set_id(join(dest, basename(path)),
492 self.FID + '/' + basename(path))
493 self.fix_dependencies_in_lib(join(dest, basename(path)))
494
495 @flush
496 def add_podofo(self):
497 print('\nAdding PoDoFo')
498 pdf = join(PREFIX, 'lib', 'libpodofo.4.dylib')
499 self.install_dylib(pdf)
500
501 @flush
502 def add_poppler(self):
503 print('\nAdding poppler')
504 for x in ('libopenjp2.7.dylib', 'libpoppler.154.dylib', 'liblcms2.2.dylib',):
505 self.install_dylib(join(PREFIX, 'lib', x))
506 for x in ('pdftohtml', 'pdftoppm', 'pdfinfo', 'pdftotext'):
507 self.install_dylib(
508 join(PREFIX, 'bin', x), set_id=False, dest=self.helpers_dir)
509
510 @flush
511 def add_imaging_libs(self):
512 print('\nAdding libjpeg, libpng, libwebp, optipng and mozjpeg')
513 for x in ('jpeg.8', 'png16.16', 'webp.7', 'webpmux.3', 'webpdemux.2', 'sharpyuv.0'):
514 self.install_dylib(join(PREFIX, 'lib', 'lib%s.dylib' % x))
515 for x in 'optipng', 'JxrDecApp', 'cwebp':
516 self.install_dylib(join(PREFIX, 'bin', x), set_id=False, dest=self.helpers_dir)
517 for x in ('jpegtran', 'cjpeg'):
518 self.install_dylib(
519 join(PREFIX, 'private', 'mozjpeg', 'bin', x), set_id=False, dest=self.helpers_dir)
520
521 @flush
522 def add_fontconfig(self):
523 print('\nAdding fontconfig')
524 for x in ('fontconfig.1', 'freetype.6', 'expat.1'):
525 src = join(PREFIX, 'lib', 'lib' + x + '.dylib')
526 self.install_dylib(src)
527 dst = join(self.resources_dir, 'fonts')
528 if os.path.exists(dst):
529 shutil.rmtree(dst)
530 src = join(PREFIX, 'etc', 'fonts')
531 shutil.copytree(src, dst, symlinks=False)
532 fc = join(dst, 'fonts.conf')
533 with open(fc, 'rb') as f:
534 raw = f.read().decode('utf-8')
535 raw = raw.replace('<dir>/usr/share/fonts</dir>', '''\
536 <dir>/Library/Fonts</dir>
537 <dir>/System/Library/Fonts</dir>
538 <dir>/usr/X11R6/lib/X11/fonts</dir>
539 <dir>/usr/share/fonts</dir>
540 <dir>/var/root/Library/Fonts</dir>
541 <dir>/usr/share/fonts</dir>
542 ''')
543 open(fc, 'wb').write(raw.encode('utf-8'))
544
545 @flush
546 def add_misc_libraries(self):
547
548 def add_lib(src):
549 x = os.path.basename(src)
550 print('\nAdding', x)
551 shutil.copy2(src, self.frameworks_dir)
552 dest = join(self.frameworks_dir, x)
553 self.set_id(dest, self.FID + '/' + x)
554 self.fix_dependencies_in_lib(dest)
555
556 for x in (
557 'usb-1.0.0', 'mtp.9', 'chm.0', 'sqlite3', 'hunspell-1.7.0',
558 'icudata.78', 'icui18n.78', 'icuio.78', 'icuuc.78', 'hyphen.0', 'uchardet.0',
559 'stemmer.0', 'xslt.1', 'exslt.0', 'xml2.16', 'z.1', 'unrar', 'lzma.5',
560 'brotlicommon.1', 'brotlidec.1', 'brotlienc.1', 'zstd.1', 'jbig.2.1', 'tiff.6',
561 'crypto.3', 'ssl.3', 'iconv.2', 'espeak-ng.1', 'onnxruntime.1.23.2', # 'ltdl.7'
562 ):
563 x = 'lib%s.dylib' % x
564 src = join(PREFIX, 'lib', x)
565 add_lib(src)
566
567 # OpenSSL modules and engines
568 for x in ('ossl-modules', 'engines-3'):
569 dest = join(self.frameworks_dir, x)
570 shutil.copytree(join(PREFIX, 'lib', x), dest)
571 for dylib in os.listdir(dest):
572 if dylib.endswith('.dylib'):
573 dylib = join(dest, dylib)
574 self.set_id(dylib, self.FID + '/' + x + '/' + os.path.basename(dylib))
575 self.fix_dependencies_in_lib(dylib)
576 # espeak voices used for piper phonemization
577 shutil.copytree(join(PREFIX, 'share', 'espeak-ng-data'), join(self.resources_dir, 'espeak-ng-data'))
578
579 @flush
580 def add_site_packages(self):
581 print('\nAdding site-packages')
582 os.makedirs(self.site_packages)
583 sys_path = json.loads(subprocess.check_output([
584 PYTHON, '-c', 'import sys, json; json.dump(sys.path, sys.stdout)']))
585 paths = reversed(tuple(map(abspath, [x for x in sys_path if x.startswith('/') and not x.startswith('/Library/')])))
586 upaths = []
587 for x in paths:
588 if x not in upaths and (x.endswith('.egg') or x.endswith('/site-packages')):
589 upaths.append(x)
590 upaths.append(join(CALIBRE_DIR, 'src'))
591 for x in upaths:
592 print('\t', x)
593 tdir = None
594 try:
595 if not os.path.isdir(x):
596 zf = zipfile.ZipFile(x)
597 tdir = tempfile.mkdtemp()
598 zf.extractall(tdir)
599 x = tdir
600 self.add_modules_from_dir(x)
601 self.add_packages_from_dir(x)
602 finally:
603 if tdir is not None:
604 shutil.rmtree(tdir)
605 fix_pycryptodome(self.site_packages)
606 try:
607 shutil.rmtree(join(self.site_packages, 'calibre', 'plugins'))
608 except OSError as err:
609 if err.errno != errno.ENOENT:
610 raise
611 sp = join(self.resources_dir, 'Python', 'site-packages')
612 self.remove_bytecode(sp)
613
614 @flush
615 def add_modules_from_dir(self, src):
616 for x in glob.glob(join(src, '*.py')) + glob.glob(join(src, '*.so')):
617 dest = join(self.site_packages, os.path.basename(x))
618 shutil.copy2(x, dest)
619 if x.endswith('.so'):
620 self.fix_dependencies_in_lib(dest)
621
622 @flush
623 def add_packages_from_dir(self, src):
624 for x in os.listdir(src):
625 x = join(src, x)
626 if os.path.isdir(x) and is_package_dir(x):
627 if self.filter_package(basename(x)):
628 continue
629 self.add_package_dir(x)
630
631 @flush
632 def add_package_dir(self, x, dest=None):
633 is_kakasi = 'pykakasi' in x
634 allowed_exts = ('', '.py', '.so')
635 if is_kakasi:
636 allowed_exts += ('.db',)
637 def ignore(root, files):
638 ans = []
639 for y in files:
640 ext = os.path.splitext(y)[1]
641 if ext not in allowed_exts or (not ext and not os.path.isdir(join(root, y))):
642 ans.append(y)
643
644 return ans
645 if dest is None:
646 dest = self.site_packages
647 dest = join(dest, basename(x))
648 shutil.copytree(x, dest, symlinks=True, ignore=ignore)
649 self.postprocess_package(x, dest)
650 for x in os.walk(dest):
651 for f in x[-1]:
652 if f.endswith('.so'):
653 f = join(x[0], f)
654 self.fix_dependencies_in_lib(f)
655
656 @flush
657 def filter_package(self, name):
658 return name in ('Cython', 'modulegraph', 'macholib', 'py2app',
659 'bdist_mpkg', 'altgraph')
660
661 @flush
662 def postprocess_package(self, src_path, dest_path):
663 pass
664
665 @flush
666 def add_stdlib(self):
667 print('\nAdding python stdlib')
668 src = PREFIX + '/python/Python.framework/Versions/Current/lib/python'
669 src += py_ver
670 dest = join(self.resources_dir, 'Python', 'lib', 'python')
671 dest += py_ver
672 os.makedirs(dest)
673 for x in os.listdir(src):
674 if x in ('site-packages', 'config', 'test', 'lib2to3', 'lib-tk',
675 'lib-old', 'idlelib', 'plat-mac', 'plat-darwin', 'site.py'):
676 continue
677 x = join(src, x)
678 if os.path.isdir(x):
679 self.add_package_dir(x, dest)
680 elif os.path.splitext(x)[1] in ('.so', '.py'):
681 shutil.copy2(x, dest)
682 dest2 = join(dest, basename(x))
683 if dest2.endswith('.so'):
684 self.fix_dependencies_in_lib(dest2)
685
686 target = join(self.resources_dir, 'Python', 'lib')
687 self.remove_bytecode(target)
688 for path in walk(target):
689 if path.endswith('.so'):
690 self.fix_dependencies_in_lib(path)
691
692 @flush
693 def remove_bytecode(self, dest):
694 for x in os.walk(dest):
695 root = x[0]
696 for f in x[-1]:
697 if os.path.splitext(f) in ('.pyc', '.pyo'):
698 os.remove(join(root, f))
699
700 @flush
701 def compile_py_modules(self):
702 print('\nCompiling Python modules')
703 base = join(self.resources_dir, 'Python')
704 pydir = join(base, f'lib/python{py_ver}')
705 src = join(pydir, 'lib-dynload')
706 dest = join(self.frameworks_dir, 'plugins')
707 print('Extracting extension modules from:', src, 'to', dest)
708 self.ext_map.update(extract_extension_modules(src, dest))
709 os.rmdir(src)
710 src = join(base, 'site-packages')
711 print('Extracting extension modules from:', src, 'to', dest)
712 self.ext_map.update(extract_extension_modules(src, dest))
713 for x in os.listdir(src):
714 os.rename(join(src, x), join(pydir, x))
715 os.rmdir(src)
716 py_compile(pydir)
717 freeze_python(
718 pydir, dest, self.inc_dir, self.ext_map,
719 develop_mode_env_var='CALIBRE_DEVELOP_FROM',
720 path_to_user_env_vars='~/Library/Preferences/calibre/macos-env.txt'
721 )
722 shutil.rmtree(pydir)
723
724 def create_app_clone(self, name, specialise_plist, remove_doc_types=False, base_dir=None):
725 print('\nCreating ' + name)
726 base_dir = base_dir or self.contents_dir
727 cc_dir = join(base_dir, name, 'Contents')
728 exe_dir = join(cc_dir, 'MacOS')
729 rel_path = os.path.relpath(join(self.contents_dir, 'MacOS'), exe_dir)
730 os.makedirs(exe_dir)
731 for x in os.listdir(self.contents_dir):
732 if x.endswith('.app'):
733 continue
734 if x == 'Info.plist':
735 with open(join(self.contents_dir, x), 'rb') as r:
736 plist = plistlib.load(r)
737 specialise_plist(plist)
738 if remove_doc_types:
739 plist.pop('CFBundleDocumentTypes')
740 exe = plist['CFBundleExecutable']
741 # We cannot symlink the bundle executable as if we do,
742 # codesigning fails
743 plist['CFBundleExecutable'] = exe + '-placeholder-for-codesigning'
744 nexe = join(exe_dir, plist['CFBundleExecutable'])
745 base = os.path.dirname(abspath(__file__))
746 cmd = gcc + ARCH_FLAGS + CFLAGS + [
747 '-Wall', '-Werror', '-DEXE_NAME="%s"' % exe, '-DREL_PATH="%s"' % rel_path,
748 join(base, 'placeholder.c'), '-o', nexe, '-headerpad_max_install_names'
749 ]
750 subprocess.check_call(cmd)
751 with open(join(cc_dir, x), 'wb') as p:
752 plistlib.dump(plist, p)
753 elif x == 'MacOS':
754 for item in os.listdir(join(self.contents_dir, 'MacOS')):
755 src = join(self.contents_dir, x, item)
756 os.symlink(os.path.relpath(src, exe_dir), join(exe_dir, item))
757 else:
758 src = join(self.contents_dir, x)
759 os.symlink(os.path.relpath(src, cc_dir), join(cc_dir, x))
760
761 @flush
762 def create_gui_apps(self):
763
764 def get_data(cmd):
765 return json.loads(subprocess.check_output([join(self.contents_dir, 'MacOS', 'calibre-debug'), '-c', cmd]))
766
767 data = get_data(
768 'from calibre.customize.ui import all_input_formats; import sys, json; from calibre.ebooks.oeb.polish.main import SUPPORTED;'
769 'sys.stdout.write(json.dumps({"i": tuple(all_input_formats()), "e": tuple(SUPPORTED)}))'
770 )
771 input_formats = sorted(set(data['i']))
772 edit_formats = sorted(set(data['e']))
773
774 def specialise_plist(launcher, formats, plist):
775 plist['CFBundleDisplayName'] = plist['CFBundleName'] = {
776 'ebook-viewer': 'E-book Viewer', 'ebook-edit': 'Edit Book',
777 }[launcher]
778 plist['CFBundleExecutable'] = launcher
779 plist['CFBundleIdentifier'] = 'com.calibre-ebook.' + launcher
780 plist['CFBundleIconName'] = launcher
781 plist['CFBundleIconFile'] = launcher + '.icns'
782 e = plist['CFBundleDocumentTypes'][0]
783 e['CFBundleTypeExtensions'] = [x.lower() for x in formats]
784
785 def headless_plist(plist):
786 plist['CFBundleDisplayName'] = 'calibre worker process'
787 plist['CFBundleExecutable'] = 'calibre-parallel'
788 plist['CFBundleIdentifier'] = 'com.calibre-ebook.calibre-parallel'
789 plist['LSBackgroundOnly'] = '1'
790 plist.pop('CFBundleDocumentTypes')
791
792 self.create_app_clone('ebook-viewer.app', partial(specialise_plist, 'ebook-viewer', input_formats))
793 self.create_app_clone('ebook-edit.app', partial(specialise_plist, 'ebook-edit', edit_formats),
794 base_dir=join(self.contents_dir, 'ebook-viewer.app', 'Contents'))
795 self.create_app_clone('headless.app', headless_plist,
796 base_dir=join(self.contents_dir, 'ebook-viewer.app', 'Contents', 'ebook-edit.app', 'Contents'))
797 # We need to move the webengine resources into the deepest sub-app
798 # because the sandbox gets set to the nearest enclosing app which
799 # means that WebEngine will fail to access its resources when running
800 # in the sub-apps unless they are present inside the sub app bundle
801 # somewhere
802 base_dest = join(self.contents_dir, 'ebook-viewer.app', 'Contents', 'ebook-edit.app', 'Contents', 'headless.app', 'Contents', 'SharedSupport')
803 os.mkdir(base_dest)
804 base_src = os.path.realpath(join(self.frameworks_dir, 'QtWebEngineCore.framework/Resources'))
805 items = [join(base_src, 'qtwebengine_locales')] + glob.glob(join(base_src, '*.pak')) + glob.glob(join(base_src, '*.dat'))
806 for src in items:
807 dest = join(base_dest, os.path.basename(src))
808 os.rename(src, dest)
809 os.symlink(os.path.relpath(dest, base_src), src)
810
811 @flush
812 def copy_site(self):
813 base = os.path.dirname(abspath(__file__))
814 shutil.copy2(join(base, 'site.py'), join(self.resources_dir, 'Python',
815 'lib', 'python' + py_ver))
816
817 @flush
818 def makedmg(self, d, volname):
819 ''' Copy a directory d into a dmg named volname '''
820 print('\nSigning...')
821 sys.stdout.flush()
822 destdir = OUTPUT_DIR
823 try:
824 shutil.rmtree(destdir)
825 except EnvironmentError as err:
826 if err.errno != errno.ENOENT:
827 raise
828 os.mkdir(destdir)
829 dmg = join(destdir, volname + '.dmg')
830 if os.path.exists(dmg):
831 os.unlink(dmg)
832 tdir = tempfile.mkdtemp()
833 appdir = join(tdir, os.path.basename(d))
834 shutil.copytree(d, appdir, symlinks=True)
835 if self.sign_installers or self.notarize:
836 with timeit() as times:
837 sign_app(appdir, self.notarize)
838 print('Signing completed in %d minutes %d seconds' % tuple(times))
839 os.symlink('/Applications', join(tdir, 'Applications'))
840 size_in_mb = int(subprocess.check_output(['du', '-s', '-k', tdir]).decode('utf-8').split()[0]) / 1024.
841 # ULMO (10.15+) gives the best compression, better than ULFO and better+faster than UDBZ
842 cmd = ['/usr/bin/hdiutil', 'create', '-srcfolder', tdir, '-volname', volname, '-format', 'ULMO']
843 if 190 < size_in_mb < 250:
844 # We need -size 255m because of a bug in hdiutil. When the size of
845 # srcfolder is close to 200MB hdiutil fails with
846 # diskimages-helper: resize request is above maximum size allowed.
847 cmd += ['-size', '255m']
848 print('\nCreating dmg...')
849 with timeit() as times:
850 subprocess.check_call(cmd + [dmg])
851 print('dmg created in %d minutes and %d seconds' % tuple(times))
852 shutil.rmtree(tdir)
853 size = os.stat(dmg).st_size / (1024 * 1024.)
854 print('\nInstaller size: %.2fMB\n' % size)
855 return dmg
856
857
858 def main(args, ext_dir, test_runner):
859 build_dir = abspath(join(mkdtemp('frozen-'), APPNAME + '.app'))
860 inc_dir = abspath(mkdtemp('include'))
861 if args.skip_tests:
862 def test_runner(*a):
863 return None
864 Freeze(build_dir, ext_dir, inc_dir, test_runner, dont_strip=args.dont_strip, sign_installers=args.sign_installers, notarize=args.notarize)
865
866
867 if __name__ == '__main__':
868 args = globals()['args']
869 ext_dir = globals()['ext_dir']
870 run_tests = iv['run_tests']
871 main(args, ext_dir, run_tests)