| 1 |
#!/usr/bin/env python |
| 2 |
# License: GPLv3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net> |
| 3 |
|
| 4 |
|
| 5 |
import binascii |
| 6 |
import json |
| 7 |
import os |
| 8 |
import subprocess |
| 9 |
import sys |
| 10 |
|
| 11 |
from setup import Command |
| 12 |
|
| 13 |
d = os.path.dirname |
| 14 |
|
| 15 |
|
| 16 |
def get_paths(): |
| 17 |
base = d(d(os.path.abspath(__file__))) |
| 18 |
bypy = os.path.join(d(base), 'bypy') |
| 19 |
bypy = os.environ.get('BYPY_LOCATION', bypy) |
| 20 |
if not os.path.isdir(bypy): |
| 21 |
raise SystemExit('Cannot find the bypy code. Set the environment variable BYPY_LOCATION to point to it') |
| 22 |
return base, bypy |
| 23 |
|
| 24 |
|
| 25 |
def get_exe(): |
| 26 |
return 'python3' if sys.version_info.major == 2 else sys.executable |
| 27 |
|
| 28 |
|
| 29 |
def get_cmd(exe, bypy, which, bitness, sign_installers=False, notarize=True, compression_level='9', action='program', dont_strip=False): |
| 30 |
cmd = [exe, bypy, which] |
| 31 |
if bitness and bitness != '64': |
| 32 |
cmd += ['--arch', bitness] |
| 33 |
cmd.append(action) |
| 34 |
if action == 'program': |
| 35 |
if sign_installers or notarize: |
| 36 |
cmd.append('--sign-installers') |
| 37 |
if notarize: |
| 38 |
cmd.append('--notarize') |
| 39 |
if dont_strip: |
| 40 |
cmd.append('--dont-strip') |
| 41 |
cmd.append('--compression-level=' + compression_level) |
| 42 |
return cmd |
| 43 |
|
| 44 |
|
| 45 |
def get_dist(base, which, bitness): |
| 46 |
dist = os.path.join(base, 'bypy', 'b', which) |
| 47 |
if bitness: |
| 48 |
dist = os.path.join(dist, bitness) |
| 49 |
for q in 'dist sw/dist'.split(): |
| 50 |
if os.path.exists(os.path.join(dist, q)): |
| 51 |
dist = os.path.join(dist, q) |
| 52 |
break |
| 53 |
return dist |
| 54 |
|
| 55 |
|
| 56 |
def shutdown_allowed(which, bitness): |
| 57 |
# The ARM64 VM is extremely flakey often booting up to a non-functional |
| 58 |
# state so don't shut it down as it seems to be more stable once boot-up is |
| 59 |
# done. |
| 60 |
return bitness != 'arm64' |
| 61 |
|
| 62 |
|
| 63 |
def build_only(which, bitness, spec, shutdown=False): |
| 64 |
base, bypy = get_paths() |
| 65 |
exe = get_exe() |
| 66 |
cmd = get_cmd(exe, bypy, which, bitness, False) |
| 67 |
cmd.extend(['--build-only', spec]) |
| 68 |
ret = subprocess.Popen(cmd).wait() |
| 69 |
if ret != 0: |
| 70 |
raise SystemExit(ret) |
| 71 |
dist = get_dist(base, which, bitness) |
| 72 |
dist = os.path.join(dist, 'c-extensions') |
| 73 |
if shutdown and shutdown_allowed(which, bitness): |
| 74 |
cmd = get_cmd(exe, bypy, which, bitness, action='shutdown') |
| 75 |
subprocess.Popen(cmd).wait() |
| 76 |
return dist |
| 77 |
|
| 78 |
|
| 79 |
def build_single(which='windows', bitness='64', shutdown=True, sign_installers=True, notarize=True, compression_level='9', dont_strip=False): |
| 80 |
base, bypy = get_paths() |
| 81 |
exe = get_exe() |
| 82 |
cmd = get_cmd(exe, bypy, which, bitness, sign_installers, notarize, compression_level=compression_level, dont_strip=dont_strip) |
| 83 |
ret = subprocess.Popen(cmd).wait() |
| 84 |
if ret != 0: |
| 85 |
raise SystemExit(ret) |
| 86 |
dist = get_dist(base, which, bitness) |
| 87 |
for x in os.listdir(dist): |
| 88 |
src = os.path.join(dist, x) |
| 89 |
if os.path.isdir(src): |
| 90 |
continue |
| 91 |
print(x) |
| 92 |
dest = os.path.join(base, 'dist', x) |
| 93 |
try: |
| 94 |
os.remove(dest) |
| 95 |
except OSError: |
| 96 |
pass |
| 97 |
os.link(src, dest) |
| 98 |
if shutdown and shutdown_allowed(which, bitness): |
| 99 |
cmd = get_cmd(exe, bypy, which, bitness, action='shutdown') |
| 100 |
subprocess.Popen(cmd).wait() |
| 101 |
|
| 102 |
|
| 103 |
def build_dep(args): |
| 104 |
base, bypy = get_paths() |
| 105 |
exe = get_exe() |
| 106 |
pl = args[0] |
| 107 |
args.insert(1, 'dependencies') |
| 108 |
if '-' in pl: |
| 109 |
args[0:1] = pl.split('-')[0], f'--arch={pl.split("-")[-1]}' |
| 110 |
cmd = [exe, bypy] + list(args) |
| 111 |
ret = subprocess.Popen(cmd).wait() |
| 112 |
if ret != 0: |
| 113 |
raise SystemExit(ret) |
| 114 |
|
| 115 |
|
| 116 |
class BuildInstaller(Command): |
| 117 |
OS = '' |
| 118 |
BITNESS = '' |
| 119 |
|
| 120 |
def add_options(self, parser): |
| 121 |
parser.add_option('--dont-shutdown', default=False, action='store_true', help='Do not shutdown the VM after building') |
| 122 |
parser.add_option('--dont-sign', default=False, action='store_true', help='Do not sign the installers') |
| 123 |
parser.add_option('--dont-notarize', default=False, action='store_true', help='Do not notarize the installers') |
| 124 |
parser.add_option('--compression-level', default='9', choices=list('123456789'), help='Do not notarize the installers') |
| 125 |
parser.add_option('--dont-strip', default=False, action='store_true', help='Do not strip the binaries') |
| 126 |
|
| 127 |
def run(self, opts): |
| 128 |
build_single( |
| 129 |
self.OS, |
| 130 |
self.BITNESS, |
| 131 |
not opts.dont_shutdown, |
| 132 |
not opts.dont_sign, |
| 133 |
not opts.dont_notarize, |
| 134 |
compression_level=opts.compression_level, |
| 135 |
dont_strip=opts.dont_strip, |
| 136 |
) |
| 137 |
|
| 138 |
|
| 139 |
class BuildInstallers(BuildInstaller): |
| 140 |
OS = '' |
| 141 |
ALL_ARCHES = ('64',) |
| 142 |
|
| 143 |
def run(self, opts): |
| 144 |
for bitness in self.ALL_ARCHES: |
| 145 |
shutdown = bitness is self.ALL_ARCHES[-1] and not opts.dont_shutdown |
| 146 |
build_single(self.OS, bitness, shutdown) |
| 147 |
|
| 148 |
|
| 149 |
class Linux64(BuildInstaller): |
| 150 |
OS = 'linux' |
| 151 |
BITNESS = '64' |
| 152 |
description = 'Build the 64-bit Linux calibre installer' |
| 153 |
|
| 154 |
|
| 155 |
class LinuxArm64(BuildInstaller): |
| 156 |
OS = 'linux' |
| 157 |
BITNESS = 'arm64' |
| 158 |
description = 'Build the 64-bit ARM Linux calibre installer' |
| 159 |
|
| 160 |
|
| 161 |
class Win64(BuildInstaller): |
| 162 |
OS = 'windows' |
| 163 |
BITNESS = '64' |
| 164 |
description = 'Build the 64-bit windows calibre installer' |
| 165 |
|
| 166 |
|
| 167 |
class OSX(BuildInstaller): |
| 168 |
OS = 'macos' |
| 169 |
|
| 170 |
|
| 171 |
class Linux(BuildInstallers): |
| 172 |
OS = 'linux' |
| 173 |
ALL_ARCHES = '64', 'arm64' |
| 174 |
|
| 175 |
|
| 176 |
class Win(BuildInstallers): |
| 177 |
OS = 'windows' |
| 178 |
|
| 179 |
|
| 180 |
class BuildDep(Command): |
| 181 |
description = ( |
| 182 |
'Build a calibre dependency. For example, build_dep windows expat.' |
| 183 |
' Without arguments builds all deps for specified platform. Use linux-arm64 for Linux ARM.' |
| 184 |
' Use build_dep all somedep to build a dep for all platforms.' |
| 185 |
) |
| 186 |
|
| 187 |
def run(self, opts): |
| 188 |
args = opts.cli_args |
| 189 |
|
| 190 |
if args and args[0] == 'all': |
| 191 |
for x in ('linux', 'linux-arm64', 'macos', 'windows'): |
| 192 |
build_dep(x.split() + list(args)[1:]) |
| 193 |
else: |
| 194 |
build_dep(args) |
| 195 |
|
| 196 |
|
| 197 |
class ExportPackages(Command): |
| 198 |
description = 'Export built deps to a server for CI testing' |
| 199 |
|
| 200 |
def run(self, opts): |
| 201 |
base, bypy = get_paths() |
| 202 |
exe = get_exe() |
| 203 |
for which in ('linux', 'macos', 'windows'): |
| 204 |
cmd = [exe, bypy, 'export'] + ['download.calibre-ebook.com:/srv/download/ci/calibre7'] + [which] |
| 205 |
ret = subprocess.Popen(cmd).wait() |
| 206 |
if ret != 0: |
| 207 |
raise SystemExit(ret) |
| 208 |
|
| 209 |
|
| 210 |
class ExtDev(Command): |
| 211 |
description = 'Develop a single native extension conveniently' |
| 212 |
|
| 213 |
def add_options(self, parser): |
| 214 |
parser.add_option( |
| 215 |
'-L', |
| 216 |
'--libdir', |
| 217 |
default=[], |
| 218 |
action='append', |
| 219 |
help='Path to directory from which to load extra libraries, relative to pkg directory of cross compile target.', |
| 220 |
) |
| 221 |
|
| 222 |
def run(self, opts): |
| 223 |
which, ext = opts.cli_args[:2] |
| 224 |
cmd = opts.cli_args[2:] or ['calibre-debug', '--test-build'] |
| 225 |
if which == 'windows': |
| 226 |
os.environ['CROSS_LIBDIRS'] = os.pathsep.join(f'bypy/b/windows/64/pkg/{x}' for x in opts.libdir) |
| 227 |
cp = subprocess.run([sys.executable, 'setup.py', 'build', '--cross-compile-extensions=windows', f'--only={ext}']) |
| 228 |
if cp.returncode != 0: |
| 229 |
raise SystemExit(cp.returncode) |
| 230 |
src = f'src/calibre/plugins/{ext}.cross-windows-x64.pyd' |
| 231 |
host = 'win' |
| 232 |
path = '/cygdrive/c/Program Files/Calibre2/app/bin/{}.pyd' |
| 233 |
bin_dir = '/cygdrive/c/Program Files/Calibre2' |
| 234 |
elif which == 'macos': |
| 235 |
os.environ['CROSS_LIBDIRS'] = os.pathsep.join(f'bypy/b/macos/pkg/{x}' for x in opts.libdir) |
| 236 |
ext_dir = build_only(which, '', ext) |
| 237 |
src = os.path.join(ext_dir, f'{ext}.so') |
| 238 |
print( |
| 239 |
'\n\n\x1b[33;1mWARNING: This does not work on macOS, unless you use un-signed builds with ', |
| 240 |
' ./update-on-ox develop\x1b[m', |
| 241 |
file=sys.stderr, |
| 242 |
end='\n\n\n', |
| 243 |
) |
| 244 |
host = 'ox' |
| 245 |
path = '/Applications/calibre.app/Contents/Frameworks/plugins/{}.so' |
| 246 |
bin_dir = '/Applications/calibre.app/Contents/MacOS' |
| 247 |
else: |
| 248 |
raise SystemExit(f'Unknown OS {which}') |
| 249 |
control_path = os.path.expanduser('~/.ssh/extdev-master-%C') |
| 250 |
if ( |
| 251 |
subprocess.Popen([ |
| 252 |
'ssh', |
| 253 |
'-o', |
| 254 |
'ControlMaster=auto', |
| 255 |
'-o', |
| 256 |
'ControlPath=' + control_path, |
| 257 |
'-o', |
| 258 |
'ControlPersist=yes', |
| 259 |
host, |
| 260 |
'echo', |
| 261 |
'ssh master running', |
| 262 |
]).wait() |
| 263 |
!= 0 |
| 264 |
): |
| 265 |
raise SystemExit(1) |
| 266 |
try: |
| 267 |
path = path.format(ext) |
| 268 |
subprocess.call(['ssh', '-S', control_path, host, 'chmod', '+wx', f'"{path}"']) |
| 269 |
with open(src, 'rb') as f: |
| 270 |
p = subprocess.Popen(['ssh', '-S', control_path, host, f'cat - > "{path}"'], stdin=subprocess.PIPE) |
| 271 |
p.communicate(f.read()) |
| 272 |
if p.wait() != 0: |
| 273 |
raise SystemExit(1) |
| 274 |
subprocess.check_call(['ssh', '-S', control_path, host, 'bin/update-calibre']) |
| 275 |
enc = json.dumps(cmd) |
| 276 |
if not isinstance(enc, bytes): |
| 277 |
enc = enc.encode('utf-8') |
| 278 |
enc = binascii.hexlify(enc).decode('ascii') |
| 279 |
wcmd = [ |
| 280 |
'"{}"'.format(os.path.join(bin_dir, 'calibre-debug')), |
| 281 |
'-c', |
| 282 |
( |
| 283 |
'"' |
| 284 |
'import sys, json, binascii, os, subprocess; cmd = json.loads(binascii.unhexlify(sys.argv[-1]));' |
| 285 |
'env = os.environ.copy();' |
| 286 |
'''env[str('CALIBRE_DEVELOP_FROM')] = str(os.path.abspath('calibre-src/src'));''' |
| 287 |
'from calibre.debug import get_debug_executable; exe_dir = os.path.dirname(get_debug_executable()[0]);' |
| 288 |
'cmd[0] = os.path.join(exe_dir, cmd[0]); ret = subprocess.Popen(cmd, env=env).wait();' |
| 289 |
'sys.stdout.flush(); sys.stderr.flush(); sys.exit(ret)' |
| 290 |
'"' |
| 291 |
), |
| 292 |
enc, |
| 293 |
] |
| 294 |
ret = subprocess.Popen(['ssh', '-S', control_path, host] + wcmd).wait() |
| 295 |
if ret != 0: |
| 296 |
raise SystemExit('The test command "{}" failed with exit code: {}'.format(' '.join(cmd), ret)) |
| 297 |
finally: |
| 298 |
subprocess.Popen(['ssh', '-O', 'exit', '-S', control_path, host]) |