Debian Codegraph @

bypy/linux/__main__.py

raw
1 #!/usr/bin/env python
2 # vim:fileencoding=utf-8
3 # License: GPLv3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
4
5 import errno
6 import glob
7 import os
8 import shutil
9 import stat
10 import subprocess
11 import tarfile
12 import time
13 from functools import partial
14
15 from bypy.constants import LIBDIR, OUTPUT_DIR, PREFIX, python_major_minor_version
16 from bypy.constants import SRC as CALIBRE_DIR
17 from bypy.freeze import extract_extension_modules, fix_pycryptodome, freeze_python, is_package_dir, path_to_freeze_dir
18 from bypy.utils import create_job, get_dll_path, mkdtemp, parallel_build, py_compile, run, walk
19
20 j = os.path.join
21 self_dir = os.path.dirname(os.path.abspath(__file__))
22 machine = (os.uname()[4] or '').lower()
23 py_ver = '.'.join(map(str, python_major_minor_version()))
24 QT_PREFIX = os.path.join(PREFIX, 'qt')
25 FFMPEG_PREFIX = os.path.join(PREFIX, 'ffmpeg', 'lib')
26 iv = globals()['init_env']
27 calibre_constants = iv['calibre_constants']
28 QT_DLLS, QT_PLUGINS, PYQT_MODULES = iv['QT_DLLS'], iv['QT_PLUGINS'], iv['PYQT_MODULES']
29 qt_get_dll_path = partial(get_dll_path, loc=os.path.join(QT_PREFIX, 'lib'))
30 ffmpeg_get_dll_path = partial(get_dll_path, loc=FFMPEG_PREFIX)
31
32
33 def binary_includes():
34 ffmpeg_dlls = tuple(os.path.basename(x).partition('.')[0][3:] for x in glob.glob(os.path.join(FFMPEG_PREFIX, '*.so')))
35 return [
36 j(PREFIX, 'bin', x) for x in ('pdftohtml', 'pdfinfo', 'pdftoppm', 'pdftotext', 'optipng', 'cwebp', 'JxrDecApp')] + [
37
38 j(PREFIX, 'private', 'mozjpeg', 'bin', x) for x in ('jpegtran', 'cjpeg')] + [
39 ] + list(map(
40 get_dll_path,
41 ('usb-1.0 mtp expat ffi z lzma openjp2 poppler dbus-1 iconv xml2 xslt jpeg png16'
42 ' webp webpmux webpdemux sharpyuv exslt ncursesw readline chm hunspell-1.7 hyphen'
43 ' icudata icui18n icuuc icuio stemmer gcrypt gpg-error uchardet graphite2 espeak-ng'
44 ' brotlicommon brotlidec brotlienc zstd podofo ssl crypto deflate tiff onnxruntime'
45 ' gobject-2.0 glib-2.0 gthread-2.0 gmodule-2.0 gio-2.0 dbus-glib-1 lcms2').split()
46 )) + [
47 # debian/ubuntu for for some typical stupid reason use libpcre.so.3
48 # instead of libpcre.so.0 like other distros. And Qt's idiotic build
49 # system links against this pcre library despite being told to use
50 # the bundled pcre. Since libpcre doesn't depend on anything other
51 # than libc and libpthread we bundle the Ubuntu one here
52 glob.glob('/usr/lib/*/libpcre.so.3')[0],
53
54 get_dll_path('bz2', 2), j(PREFIX, 'lib', 'libunrar.so'), get_dll_path('sqlite3', 0),
55 get_dll_path('python' + py_ver, 2), get_dll_path('jbig', 2),
56
57 # We don't include libstdc++.so as the OpenGL dlls on the target
58 # computer fail to load in the QPA xcb plugin if they were compiled
59 # with a newer version of gcc than the one on the build computer.
60 # libstdc++, like glibc is forward compatible and I don't think any
61 # distros do not have libstdc++.so.6, so it should be safe to leave it out.
62 # https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html (The current
63 # debian stable libstdc++ is libstdc++.so.6.0.17)
64 ] + list(map(qt_get_dll_path, QT_DLLS)) + list(map(ffmpeg_get_dll_path, ffmpeg_dlls))
65
66
67 class Env:
68
69 def __init__(self):
70 self.src_root = CALIBRE_DIR
71 self.base = mkdtemp('frozen-')
72 self.lib_dir = j(self.base, 'lib')
73 self.py_dir = j(self.lib_dir, 'python' + py_ver)
74 os.makedirs(self.py_dir)
75 self.bin_dir = j(self.base, 'bin')
76 os.mkdir(self.bin_dir)
77 self.SRC = j(self.src_root, 'src')
78 self.obj_dir = mkdtemp('launchers-')
79
80
81 def ignore_in_lib(base, items, ignored_dirs=None):
82 ans = []
83 if ignored_dirs is None:
84 ignored_dirs = {'.svn', '.bzr', '.git', 'test', 'tests', 'testing'}
85 for name in items:
86 path = j(base, name)
87 is_kakasi = 'pykakasi' in path
88 if os.path.isdir(path):
89 if name != 'plugins' and (name in ignored_dirs or not is_package_dir(path)) and not (is_kakasi and name == 'data'):
90 ans.append(name)
91 else:
92 if name.rpartition('.')[-1] not in ('so', 'py') and not (is_kakasi and name.endswith('.db')):
93 ans.append(name)
94 return ans
95
96
97 def import_site_packages(srcdir, dest):
98 if not os.path.exists(dest):
99 os.mkdir(dest)
100 for x in os.listdir(srcdir):
101 ext = x.rpartition('.')[-1]
102 f = j(srcdir, x)
103 if ext in ('py', 'so'):
104 shutil.copy2(f, dest)
105 elif ext == 'pth' and x != 'setuptools.pth':
106 for line in open(f, 'rb').read().decode('utf-8').splitlines():
107 src = os.path.abspath(j(srcdir, line))
108 if os.path.exists(src) and os.path.isdir(src):
109 import_site_packages(src, dest)
110 elif is_package_dir(f):
111 shutil.copytree(f, j(dest, x), ignore=ignore_in_lib)
112
113
114 def copy_libs(env):
115 print('Copying libs...')
116
117 for x in binary_includes():
118 dest = env.bin_dir if '/bin/' in x else env.lib_dir
119 shutil.copy2(x, dest)
120 os.chmod(j(
121 dest, os.path.basename(x)),
122 stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
123 for x in ('ossl-modules',):
124 shutil.copytree(os.path.join(LIBDIR, x), os.path.join(env.lib_dir, x))
125
126 base = j(QT_PREFIX, 'plugins')
127 dest = j(env.lib_dir, '..', 'plugins')
128 os.mkdir(dest)
129 for x in QT_PLUGINS:
130 if x not in ('audio', 'printsupport'):
131 shutil.copytree(j(base, x), j(dest, x))
132 dest = j(env.lib_dir, '..', 'libexec')
133 os.mkdir(dest)
134 shutil.copy2(os.path.join(QT_PREFIX, 'libexec', 'QtWebEngineProcess'), dest)
135 dest = j(env.lib_dir, '..', 'share')
136 os.mkdir(dest)
137 shutil.copytree(os.path.join(PREFIX, 'share/espeak-ng-data'), os.path.join(dest, 'espeak-ng-data'))
138
139
140 def copy_python(env, ext_dir):
141 print('Copying python...')
142 srcdir = j(PREFIX, 'lib/python' + py_ver)
143
144 for x in os.listdir(srcdir):
145 y = j(srcdir, x)
146 ext = os.path.splitext(x)[1]
147 if os.path.isdir(y) and x not in ('test', 'hotshot',
148 'site-packages', 'idlelib', 'dist-packages'):
149 shutil.copytree(y, j(env.py_dir, x), ignore=ignore_in_lib)
150 if os.path.isfile(y) and ext in ('.py', '.so'):
151 shutil.copy2(y, env.py_dir)
152
153 srcdir = j(srcdir, 'site-packages')
154 dest = j(env.py_dir, 'site-packages')
155 import_site_packages(srcdir, dest)
156
157 for x in os.listdir(env.SRC):
158 c = j(env.SRC, x)
159 if os.path.exists(j(c, '__init__.py')):
160 shutil.copytree(c, j(dest, x), ignore=partial(ignore_in_lib, ignored_dirs={}))
161 elif os.path.isfile(c):
162 shutil.copy2(c, j(dest, x))
163 shutil.copytree(j(env.src_root, 'resources'), j(env.base, 'resources'))
164 for pak in glob.glob(j(QT_PREFIX, 'resources', '*')):
165 shutil.copy2(pak, j(env.base, 'resources'))
166 os.mkdir(j(env.base, 'translations'))
167 shutil.copytree(j(QT_PREFIX, 'translations', 'qtwebengine_locales'), j(env.base, 'translations', 'qtwebengine_locales'))
168 sitepy = j(self_dir, 'site.py')
169 shutil.copy2(sitepy, j(env.py_dir, 'site.py'))
170
171 pdir = j(env.lib_dir, 'calibre-extensions')
172 if not os.path.exists(pdir):
173 os.mkdir(pdir)
174 fix_pycryptodome(j(env.py_dir, 'site-packages'))
175 for x in os.listdir(j(env.py_dir, 'site-packages')):
176 os.rename(j(env.py_dir, 'site-packages', x), j(env.py_dir, x))
177 os.rmdir(j(env.py_dir, 'site-packages'))
178 print('Extracting extension modules from', ext_dir, 'to', pdir)
179 ext_map = extract_extension_modules(ext_dir, pdir)
180 shutil.rmtree(j(env.py_dir, 'calibre', 'plugins'))
181 print('Extracting extension modules from', env.py_dir, 'to', pdir)
182 ext_map.update(extract_extension_modules(env.py_dir, pdir))
183 py_compile(env.py_dir)
184 freeze_python(env.py_dir, pdir, env.obj_dir, ext_map, develop_mode_env_var='CALIBRE_DEVELOP_FROM')
185 shutil.rmtree(env.py_dir)
186
187
188 def build_launchers(env):
189 base = self_dir
190 sources = [j(base, x) for x in ['util.c']]
191 objects = [j(env.obj_dir, os.path.basename(x) + '.o') for x in sources]
192 cflags = '-fno-strict-aliasing -W -Wall -c -O2 -pipe -DPY_VERSION_MAJOR={} -DPY_VERSION_MINOR={}'.format(*py_ver.split('.'))
193 cflags = cflags.split() + ['-I%s/include/python%s' % (PREFIX, py_ver)]
194 cflags += [f'-I{path_to_freeze_dir()}', f'-I{env.obj_dir}']
195 for src, obj in zip(sources, objects):
196 cmd = ['gcc'] + cflags + ['-fPIC', '-o', obj, src]
197 run(*cmd)
198
199 dll = j(env.lib_dir, 'libcalibre-launcher.so')
200 cmd = ['gcc', '-O2', '-Wl,--rpath=$ORIGIN/../lib', '-fPIC', '-o', dll, '-shared'] + objects + \
201 ['-L%s/lib' % PREFIX, '-lpython' + py_ver]
202 run(*cmd)
203
204 src = j(base, 'main.c')
205
206 modules, basenames, functions = calibre_constants['modules'].copy(), calibre_constants['basenames'].copy(), calibre_constants['functions'].copy()
207 modules['console'].append('calibre.linux')
208 basenames['console'].append('calibre_postinstall')
209 functions['console'].append('main')
210 c_launcher = '/tmp/calibre-c-launcher'
211 lsrc = os.path.join(base, 'launcher.c')
212 cmd = ['gcc', '-O2', '-o', c_launcher, lsrc, ]
213 run(*cmd)
214
215 jobs = []
216 for typ in ('console', 'gui', ):
217 for mod, bname, func in zip(modules[typ], basenames[typ], functions[typ]):
218 xflags = list(cflags)
219 xflags.remove('-c')
220 xflags += ['-DGUI_APP=' + ('1' if typ == 'gui' else '0')]
221 xflags += ['-DMODULE=L"%s"' % mod, '-DBASENAME=L"%s"' % bname,
222 '-DFUNCTION=L"%s"' % func]
223
224 exe = j(env.bin_dir, bname)
225 cmd = ['gcc'] + xflags + [src, '-o', exe, '-L' + env.lib_dir, '-lcalibre-launcher']
226 jobs.append(create_job(cmd))
227 sh = j(env.base, bname)
228 shutil.copy2(c_launcher, sh)
229 os.chmod(sh,
230 stat.S_IREAD | stat.S_IEXEC | stat.S_IWRITE | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
231 if jobs:
232 if not parallel_build(jobs, verbose=False):
233 raise SystemExit(1)
234
235
236 def is_elf(path):
237 with open(path, 'rb') as f:
238 return f.read(4) == b'\x7fELF'
239
240
241 STRIPCMD = ['strip']
242
243
244 def strip_files(files, argv_max=(256 * 1024)):
245 """ Strip a list of files """
246 while files:
247 cmd = list(STRIPCMD)
248 pathlen = sum(len(s) + 1 for s in cmd)
249 while pathlen < argv_max and files:
250 f = files.pop()
251 cmd.append(f)
252 pathlen += len(f) + 1
253 if len(cmd) > len(STRIPCMD):
254 all_files = cmd[len(STRIPCMD):]
255 unwritable_files = tuple(filter(None, (None if os.access(x, os.W_OK) else (x, os.stat(x).st_mode) for x in all_files)))
256 [os.chmod(x, stat.S_IWRITE | old_mode) for x, old_mode in unwritable_files]
257 try:
258 subprocess.check_call(cmd)
259 except subprocess.CalledProcessError:
260 # Sometimes get file is busy errors
261 time.sleep(1)
262 subprocess.check_call(cmd)
263 [os.chmod(x, old_mode) for x, old_mode in unwritable_files]
264
265
266 def strip_binaries(env):
267 files = {j(env.bin_dir, x) for x in os.listdir(env.bin_dir)} | {
268 x for x in {
269 j(os.path.dirname(env.bin_dir), x) for x in os.listdir(env.bin_dir)} if os.path.exists(x)}
270 for x in walk(env.lib_dir):
271 x = os.path.realpath(x)
272 if x not in files and is_elf(x):
273 files.add(x)
274 files.add(j(env.lib_dir, '..', 'libexec', 'QtWebEngineProcess'))
275 print('Stripping %d files...' % len(files))
276 before = sum(os.path.getsize(x) for x in files)
277 strip_files(files)
278 after = sum(os.path.getsize(x) for x in files)
279 print('Stripped %.1f MB' % ((before - after) / (1024 * 1024.)))
280
281
282 def create_tarfile(env, compression_level='9'):
283 print('Creating archive...')
284 base = OUTPUT_DIR
285 arch = 'arm64' if 'arm64' in os.environ['BYPY_ARCH'] else ('i686' if 'i386' in os.environ['BYPY_ARCH'] else 'x86_64')
286 try:
287 shutil.rmtree(base)
288 except EnvironmentError as err:
289 if err.errno not in (errno.ENOENT, errno.EBUSY):
290 raise
291 os.makedirs(base, exist_ok=True) # when base is a mount point deleting it fails with EBUSY
292 dist = os.path.join(base, '%s-%s-%s.tar' % (calibre_constants['appname'], calibre_constants['version'], arch))
293 with tarfile.open(dist, mode='w', format=tarfile.PAX_FORMAT) as tf:
294 cwd = os.getcwd()
295 os.chdir(env.base)
296 try:
297 for x in os.listdir('.'):
298 tf.add(x)
299 finally:
300 os.chdir(cwd)
301 print('Compressing archive...')
302 ans = dist.rpartition('.')[0] + '.txz'
303 start_time = time.time()
304 subprocess.check_call(['xz', '--verbose', '--threads=0', '-f', '-' + compression_level, dist])
305 secs = time.time() - start_time
306 print('Compressed in %d minutes %d seconds' % (secs // 60, secs % 60))
307 os.rename(dist + '.xz', ans)
308 print('Archive %s created: %.2f MB' % (
309 os.path.basename(ans), os.stat(ans).st_size / (1024.**2)))
310
311
312 def main():
313 args = globals()['args']
314 ext_dir = globals()['ext_dir']
315 run_tests = iv['run_tests']
316 env = Env()
317 copy_libs(env)
318 copy_python(env, ext_dir)
319 build_launchers(env)
320 if not args.skip_tests:
321 run_tests(j(env.base, 'calibre-debug'), env.base)
322 if not args.dont_strip:
323 strip_binaries(env)
324 create_tarfile(env, args.compression_level)
325
326
327 if __name__ == '__main__':
328 main()