Debian Codegraph @

setup/installers.py

raw
1 #!/usr/bin/env python
2 # License: GPLv3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
3
4
5 import binascii
6 import json
7 import os
8 import subprocess
9 import sys
10
11 from setup import Command
12
13 d = os.path.dirname
14
15
16 def get_paths():
17 base = d(d(os.path.abspath(__file__)))
18 bypy = os.path.join(d(base), 'bypy')
19 bypy = os.environ.get('BYPY_LOCATION', bypy)
20 if not os.path.isdir(bypy):
21 raise SystemExit('Cannot find the bypy code. Set the environment variable BYPY_LOCATION to point to it')
22 return base, bypy
23
24
25 def get_exe():
26 return 'python3' if sys.version_info.major == 2 else sys.executable
27
28
29 def get_cmd(exe, bypy, which, bitness, sign_installers=False, notarize=True, compression_level='9', action='program', dont_strip=False):
30 cmd = [exe, bypy, which]
31 if bitness and bitness != '64':
32 cmd += ['--arch', bitness]
33 cmd.append(action)
34 if action == 'program':
35 if sign_installers or notarize:
36 cmd.append('--sign-installers')
37 if notarize:
38 cmd.append('--notarize')
39 if dont_strip:
40 cmd.append('--dont-strip')
41 cmd.append('--compression-level=' + compression_level)
42 return cmd
43
44
45 def get_dist(base, which, bitness):
46 dist = os.path.join(base, 'bypy', 'b', which)
47 if bitness:
48 dist = os.path.join(dist, bitness)
49 for q in 'dist sw/dist'.split():
50 if os.path.exists(os.path.join(dist, q)):
51 dist = os.path.join(dist, q)
52 break
53 return dist
54
55
56 def shutdown_allowed(which, bitness):
57 # The ARM64 VM is extremely flakey often booting up to a non-functional
58 # state so don't shut it down as it seems to be more stable once boot-up is
59 # done.
60 return bitness != 'arm64'
61
62
63 def build_only(which, bitness, spec, shutdown=False):
64 base, bypy = get_paths()
65 exe = get_exe()
66 cmd = get_cmd(exe, bypy, which, bitness, False)
67 cmd.extend(['--build-only', spec])
68 ret = subprocess.Popen(cmd).wait()
69 if ret != 0:
70 raise SystemExit(ret)
71 dist = get_dist(base, which, bitness)
72 dist = os.path.join(dist, 'c-extensions')
73 if shutdown and shutdown_allowed(which, bitness):
74 cmd = get_cmd(exe, bypy, which, bitness, action='shutdown')
75 subprocess.Popen(cmd).wait()
76 return dist
77
78
79 def build_single(which='windows', bitness='64', shutdown=True, sign_installers=True, notarize=True, compression_level='9', dont_strip=False):
80 base, bypy = get_paths()
81 exe = get_exe()
82 cmd = get_cmd(exe, bypy, which, bitness, sign_installers, notarize, compression_level=compression_level, dont_strip=dont_strip)
83 ret = subprocess.Popen(cmd).wait()
84 if ret != 0:
85 raise SystemExit(ret)
86 dist = get_dist(base, which, bitness)
87 for x in os.listdir(dist):
88 src = os.path.join(dist, x)
89 if os.path.isdir(src):
90 continue
91 print(x)
92 dest = os.path.join(base, 'dist', x)
93 try:
94 os.remove(dest)
95 except OSError:
96 pass
97 os.link(src, dest)
98 if shutdown and shutdown_allowed(which, bitness):
99 cmd = get_cmd(exe, bypy, which, bitness, action='shutdown')
100 subprocess.Popen(cmd).wait()
101
102
103 def build_dep(args):
104 base, bypy = get_paths()
105 exe = get_exe()
106 pl = args[0]
107 args.insert(1, 'dependencies')
108 if '-' in pl:
109 args[0:1] = pl.split('-')[0], f'--arch={pl.split("-")[-1]}'
110 cmd = [exe, bypy] + list(args)
111 ret = subprocess.Popen(cmd).wait()
112 if ret != 0:
113 raise SystemExit(ret)
114
115
116 class BuildInstaller(Command):
117 OS = ''
118 BITNESS = ''
119
120 def add_options(self, parser):
121 parser.add_option('--dont-shutdown', default=False, action='store_true', help='Do not shutdown the VM after building')
122 parser.add_option('--dont-sign', default=False, action='store_true', help='Do not sign the installers')
123 parser.add_option('--dont-notarize', default=False, action='store_true', help='Do not notarize the installers')
124 parser.add_option('--compression-level', default='9', choices=list('123456789'), help='Do not notarize the installers')
125 parser.add_option('--dont-strip', default=False, action='store_true', help='Do not strip the binaries')
126
127 def run(self, opts):
128 build_single(
129 self.OS,
130 self.BITNESS,
131 not opts.dont_shutdown,
132 not opts.dont_sign,
133 not opts.dont_notarize,
134 compression_level=opts.compression_level,
135 dont_strip=opts.dont_strip,
136 )
137
138
139 class BuildInstallers(BuildInstaller):
140 OS = ''
141 ALL_ARCHES = ('64',)
142
143 def run(self, opts):
144 for bitness in self.ALL_ARCHES:
145 shutdown = bitness is self.ALL_ARCHES[-1] and not opts.dont_shutdown
146 build_single(self.OS, bitness, shutdown)
147
148
149 class Linux64(BuildInstaller):
150 OS = 'linux'
151 BITNESS = '64'
152 description = 'Build the 64-bit Linux calibre installer'
153
154
155 class LinuxArm64(BuildInstaller):
156 OS = 'linux'
157 BITNESS = 'arm64'
158 description = 'Build the 64-bit ARM Linux calibre installer'
159
160
161 class Win64(BuildInstaller):
162 OS = 'windows'
163 BITNESS = '64'
164 description = 'Build the 64-bit windows calibre installer'
165
166
167 class OSX(BuildInstaller):
168 OS = 'macos'
169
170
171 class Linux(BuildInstallers):
172 OS = 'linux'
173 ALL_ARCHES = '64', 'arm64'
174
175
176 class Win(BuildInstallers):
177 OS = 'windows'
178
179
180 class BuildDep(Command):
181 description = (
182 'Build a calibre dependency. For example, build_dep windows expat.'
183 ' Without arguments builds all deps for specified platform. Use linux-arm64 for Linux ARM.'
184 ' Use build_dep all somedep to build a dep for all platforms.'
185 )
186
187 def run(self, opts):
188 args = opts.cli_args
189
190 if args and args[0] == 'all':
191 for x in ('linux', 'linux-arm64', 'macos', 'windows'):
192 build_dep(x.split() + list(args)[1:])
193 else:
194 build_dep(args)
195
196
197 class ExportPackages(Command):
198 description = 'Export built deps to a server for CI testing'
199
200 def run(self, opts):
201 base, bypy = get_paths()
202 exe = get_exe()
203 for which in ('linux', 'macos', 'windows'):
204 cmd = [exe, bypy, 'export'] + ['download.calibre-ebook.com:/srv/download/ci/calibre7'] + [which]
205 ret = subprocess.Popen(cmd).wait()
206 if ret != 0:
207 raise SystemExit(ret)
208
209
210 class ExtDev(Command):
211 description = 'Develop a single native extension conveniently'
212
213 def add_options(self, parser):
214 parser.add_option(
215 '-L',
216 '--libdir',
217 default=[],
218 action='append',
219 help='Path to directory from which to load extra libraries, relative to pkg directory of cross compile target.',
220 )
221
222 def run(self, opts):
223 which, ext = opts.cli_args[:2]
224 cmd = opts.cli_args[2:] or ['calibre-debug', '--test-build']
225 if which == 'windows':
226 os.environ['CROSS_LIBDIRS'] = os.pathsep.join(f'bypy/b/windows/64/pkg/{x}' for x in opts.libdir)
227 cp = subprocess.run([sys.executable, 'setup.py', 'build', '--cross-compile-extensions=windows', f'--only={ext}'])
228 if cp.returncode != 0:
229 raise SystemExit(cp.returncode)
230 src = f'src/calibre/plugins/{ext}.cross-windows-x64.pyd'
231 host = 'win'
232 path = '/cygdrive/c/Program Files/Calibre2/app/bin/{}.pyd'
233 bin_dir = '/cygdrive/c/Program Files/Calibre2'
234 elif which == 'macos':
235 os.environ['CROSS_LIBDIRS'] = os.pathsep.join(f'bypy/b/macos/pkg/{x}' for x in opts.libdir)
236 ext_dir = build_only(which, '', ext)
237 src = os.path.join(ext_dir, f'{ext}.so')
238 print(
239 '\n\n\x1b[33;1mWARNING: This does not work on macOS, unless you use un-signed builds with ',
240 ' ./update-on-ox develop\x1b[m',
241 file=sys.stderr,
242 end='\n\n\n',
243 )
244 host = 'ox'
245 path = '/Applications/calibre.app/Contents/Frameworks/plugins/{}.so'
246 bin_dir = '/Applications/calibre.app/Contents/MacOS'
247 else:
248 raise SystemExit(f'Unknown OS {which}')
249 control_path = os.path.expanduser('~/.ssh/extdev-master-%C')
250 if (
251 subprocess.Popen([
252 'ssh',
253 '-o',
254 'ControlMaster=auto',
255 '-o',
256 'ControlPath=' + control_path,
257 '-o',
258 'ControlPersist=yes',
259 host,
260 'echo',
261 'ssh master running',
262 ]).wait()
263 != 0
264 ):
265 raise SystemExit(1)
266 try:
267 path = path.format(ext)
268 subprocess.call(['ssh', '-S', control_path, host, 'chmod', '+wx', f'"{path}"'])
269 with open(src, 'rb') as f:
270 p = subprocess.Popen(['ssh', '-S', control_path, host, f'cat - > "{path}"'], stdin=subprocess.PIPE)
271 p.communicate(f.read())
272 if p.wait() != 0:
273 raise SystemExit(1)
274 subprocess.check_call(['ssh', '-S', control_path, host, 'bin/update-calibre'])
275 enc = json.dumps(cmd)
276 if not isinstance(enc, bytes):
277 enc = enc.encode('utf-8')
278 enc = binascii.hexlify(enc).decode('ascii')
279 wcmd = [
280 '"{}"'.format(os.path.join(bin_dir, 'calibre-debug')),
281 '-c',
282 (
283 '"'
284 'import sys, json, binascii, os, subprocess; cmd = json.loads(binascii.unhexlify(sys.argv[-1]));'
285 'env = os.environ.copy();'
286 '''env[str('CALIBRE_DEVELOP_FROM')] = str(os.path.abspath('calibre-src/src'));'''
287 'from calibre.debug import get_debug_executable; exe_dir = os.path.dirname(get_debug_executable()[0]);'
288 'cmd[0] = os.path.join(exe_dir, cmd[0]); ret = subprocess.Popen(cmd, env=env).wait();'
289 'sys.stdout.flush(); sys.stderr.flush(); sys.exit(ret)'
290 '"'
291 ),
292 enc,
293 ]
294 ret = subprocess.Popen(['ssh', '-S', control_path, host] + wcmd).wait()
295 if ret != 0:
296 raise SystemExit('The test command "{}" failed with exit code: {}'.format(' '.join(cmd), ret))
297 finally:
298 subprocess.Popen(['ssh', '-O', 'exit', '-S', control_path, host])