| 1 |
#!/usr/bin/env python |
| 2 |
|
| 3 |
|
| 4 |
__license__ = 'GPL v3' |
| 5 |
__copyright__ = '2009, Kovid Goyal <[email protected]>' |
| 6 |
__docformat__ = 'restructuredtext en' |
| 7 |
|
| 8 |
import glob |
| 9 |
import hashlib |
| 10 |
import json |
| 11 |
import os |
| 12 |
import shutil |
| 13 |
import stat |
| 14 |
import subprocess |
| 15 |
import sys |
| 16 |
import time |
| 17 |
from subprocess import check_call |
| 18 |
from tempfile import NamedTemporaryFile, mkdtemp |
| 19 |
from urllib.request import Request, urlopen |
| 20 |
from zipfile import ZipFile |
| 21 |
|
| 22 |
if __name__ == '__main__': |
| 23 |
d = os.path.dirname |
| 24 |
sys.path.insert(0, d(d(os.path.abspath(__file__)))) |
| 25 |
|
| 26 |
from setup import Command, __appname__, __version__, installer_names, manual_build_dir |
| 27 |
|
| 28 |
DOWNLOADS = '/srv/main/downloads' |
| 29 |
HTML2LRF = 'calibre/ebooks/lrf/html/demo' |
| 30 |
TXT2LRF = 'src/calibre/ebooks/lrf/txt/demo' |
| 31 |
STAGING_HOST = 'download.calibre-ebook.com' |
| 32 |
BACKUP_HOST = 'code.calibre-ebook.com' |
| 33 |
STAGING_USER = BACKUP_USER = 'root' |
| 34 |
STAGING_DIR = '/root/staging' |
| 35 |
BACKUP_DIR = '/binaries' |
| 36 |
|
| 37 |
|
| 38 |
def installer_description(fname): |
| 39 |
if fname.endswith('.tar.xz'): |
| 40 |
return 'Source code' |
| 41 |
if fname.endswith('.txz'): |
| 42 |
return ('ARM' if 'arm64' in fname else 'AMD') + ' 64-bit Linux binary' |
| 43 |
if fname.endswith('.msi'): |
| 44 |
return 'Windows installer' |
| 45 |
if fname.endswith('.dmg'): |
| 46 |
return 'macOS dmg' |
| 47 |
if fname.endswith('.exe'): |
| 48 |
return 'Calibre Portable' |
| 49 |
return 'Unknown file' |
| 50 |
|
| 51 |
|
| 52 |
def upload_signatures(): |
| 53 |
tdir = mkdtemp() |
| 54 |
scp = ['scp'] |
| 55 |
try: |
| 56 |
for installer in installer_names(): |
| 57 |
if not os.path.exists(installer): |
| 58 |
continue |
| 59 |
sig = os.path.join(tdir, os.path.basename(installer + '.sig')) |
| 60 |
scp.append(sig) |
| 61 |
check_call([os.environ['PENV'] + '/gpg-as-kovid', '--output', sig, '--detach-sig', installer]) |
| 62 |
with open(installer, 'rb') as f: |
| 63 |
raw = f.read() |
| 64 |
fingerprint = hashlib.sha512(raw).hexdigest() |
| 65 |
sha512 = os.path.join(tdir, os.path.basename(installer + '.sha512')) |
| 66 |
with open(sha512, 'w') as f: |
| 67 |
f.write(fingerprint) |
| 68 |
scp.append(sha512) |
| 69 |
for srv in 'code main'.split(): |
| 70 |
check_call(scp + [f'{srv}:/srv/{srv}/signatures/']) |
| 71 |
check_call(['ssh', srv, 'chown', '-R', 'http:http', f'/srv/{srv}/signatures']) |
| 72 |
finally: |
| 73 |
shutil.rmtree(tdir) |
| 74 |
|
| 75 |
|
| 76 |
class ReUpload(Command): # {{{ |
| 77 |
description = 'Re-upload any installers present in dist/' |
| 78 |
|
| 79 |
sub_commands = ['upload_installers'] |
| 80 |
|
| 81 |
def pre_sub_commands(self, opts): |
| 82 |
opts.replace = True |
| 83 |
exists = {x for x in installer_names() if os.path.exists(x)} |
| 84 |
if not exists: |
| 85 |
print('There appear to be no installers!') |
| 86 |
raise SystemExit(1) |
| 87 |
|
| 88 |
def run(self, opts): |
| 89 |
for x in installer_names(): |
| 90 |
if os.path.exists(x): |
| 91 |
os.remove(x) |
| 92 |
|
| 93 |
|
| 94 |
# }}} |
| 95 |
|
| 96 |
|
| 97 |
# Data {{{ |
| 98 |
def get_github_data(): |
| 99 |
with open(os.environ['PENV'] + '/github-token', 'rb') as f: |
| 100 |
un, pw = f.read().decode('utf-8').strip().split(':') |
| 101 |
return {'username': un, 'password': pw} |
| 102 |
|
| 103 |
|
| 104 |
def get_sourceforge_data(): |
| 105 |
return {'username': 'kovidgoyal', 'project': 'calibre'} |
| 106 |
|
| 107 |
|
| 108 |
def get_fosshub_data(): |
| 109 |
with open(os.environ['PENV'] + '/fosshub', 'rb') as f: |
| 110 |
return f.read().decode('utf-8').strip() |
| 111 |
|
| 112 |
|
| 113 |
def send_data(loc): |
| 114 |
subprocess.check_call([ |
| 115 |
'rsync', |
| 116 |
'--inplace', |
| 117 |
'--delete', |
| 118 |
'-r', |
| 119 |
'-zz', |
| 120 |
'-h', |
| 121 |
'--info=progress2', |
| 122 |
'-e', |
| 123 |
'ssh -x', |
| 124 |
loc + '/', |
| 125 |
f'{STAGING_USER}@{STAGING_HOST}:{STAGING_DIR}', |
| 126 |
]) |
| 127 |
|
| 128 |
|
| 129 |
def send_to_backup(loc): |
| 130 |
host = f'{BACKUP_USER}@{BACKUP_HOST}' |
| 131 |
dest = f'{BACKUP_DIR}/{__version__}' |
| 132 |
subprocess.check_call(['ssh', '-x', host, 'mkdir', '-p', dest]) |
| 133 |
subprocess.check_call(['rsync', '--inplace', '--delete', '-r', '-zz', '-h', '--info=progress2', '-e', 'ssh -x', loc + '/', f'{host}:{dest}/']) |
| 134 |
|
| 135 |
|
| 136 |
def gh_cmdline(ver, data): |
| 137 |
safe = [ |
| 138 |
__appname__, |
| 139 |
ver, |
| 140 |
'fmap', |
| 141 |
'github', |
| 142 |
__appname__, |
| 143 |
data['username'], |
| 144 |
] |
| 145 |
return safe + [data['password']], safe + ['PASSWORD_REDACTED'] |
| 146 |
|
| 147 |
|
| 148 |
def sf_cmdline(ver, sdata): |
| 149 |
return [__appname__, ver, 'fmap', 'sourceforge', sdata['project'], sdata['username']] |
| 150 |
|
| 151 |
|
| 152 |
def calibre_cmdline(ver): |
| 153 |
return [__appname__, ver, 'fmap', 'calibre'] |
| 154 |
|
| 155 |
|
| 156 |
def run_remote_upload(args, safe=None): |
| 157 |
print('Running remotely:', ' '.join(safe or args)) |
| 158 |
subprocess.check_call(['ssh', '-x', f'{STAGING_USER}@{STAGING_HOST}', 'cd', STAGING_DIR, '&&', 'python', 'hosting.py'] + args) |
| 159 |
|
| 160 |
|
| 161 |
# }}} |
| 162 |
|
| 163 |
|
| 164 |
def upload_to_fosshub(): |
| 165 |
# https://devzone.fosshub.com/dashboard/restApi |
| 166 |
# fosshub has no API to do partial uploads, so we always upload all files. |
| 167 |
api_key = get_fosshub_data() |
| 168 |
|
| 169 |
def request(path, data=None): |
| 170 |
r = Request( |
| 171 |
'https://api.fosshub.com/rest/' + path.lstrip('/'), headers={'Content-Type': 'application/json', 'X-auth-key': api_key, 'User-Agent': 'calibre'} |
| 172 |
) |
| 173 |
res = urlopen(r, data=data) |
| 174 |
ans = json.loads(res.read()) |
| 175 |
if ans.get('error'): |
| 176 |
raise SystemExit(ans['error']) |
| 177 |
if res.getcode() != 200: |
| 178 |
raise SystemExit(f'Request to {path} failed with response code: {res.getcode()}') |
| 179 |
# from pprint import pprint |
| 180 |
# pprint(ans) |
| 181 |
return ans['status'] if 'status' in ans else ans['data'] |
| 182 |
|
| 183 |
print('Sending upload request to fosshub...') |
| 184 |
project_id = None |
| 185 |
|
| 186 |
for project in request('projects'): |
| 187 |
if project['name'].lower() == 'calibre': |
| 188 |
project_id = project['id'] |
| 189 |
break |
| 190 |
else: |
| 191 |
raise SystemExit('No calibre project found') |
| 192 |
|
| 193 |
files = set(installer_names()) |
| 194 |
entries = [] |
| 195 |
for fname in files: |
| 196 |
desc = installer_description(fname) |
| 197 |
url = f'https://download.calibre-ebook.com/{__version__}/{os.path.basename(fname)}' |
| 198 |
entries.append({ |
| 199 |
'fileUrl': url, |
| 200 |
'type': desc, |
| 201 |
'version': __version__, |
| 202 |
}) |
| 203 |
jq = { |
| 204 |
'version': __version__, |
| 205 |
'files': entries, |
| 206 |
'publish': True, |
| 207 |
'isOldRelease': False, |
| 208 |
} |
| 209 |
data = json.dumps(jq) |
| 210 |
# print(data) |
| 211 |
data = data.encode('utf-8') |
| 212 |
if not request(f'projects/{project_id}/releases/', data=data): |
| 213 |
raise SystemExit('Failed to queue publish job with fosshub') |
| 214 |
|
| 215 |
|
| 216 |
class UploadInstallers(Command): # {{{ |
| 217 |
def add_options(self, parser): |
| 218 |
parser.add_option('--replace', default=False, action='store_true', help='Replace existing installers') |
| 219 |
|
| 220 |
def run(self, opts): |
| 221 |
# return upload_to_fosshub() |
| 222 |
all_possible = set(installer_names()) |
| 223 |
available = set(glob.glob('dist/*')) |
| 224 |
files = {x: installer_description(x) for x in all_possible.intersection(available)} |
| 225 |
for x in files: |
| 226 |
os.chmod(x, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH) |
| 227 |
sizes = {os.path.basename(x): os.path.getsize(x) for x in files} |
| 228 |
self.record_sizes(sizes) |
| 229 |
tdir = mkdtemp() |
| 230 |
try: |
| 231 |
self.upload_to_staging(tdir, files) |
| 232 |
self.upload_to_calibre() |
| 233 |
if opts.replace: |
| 234 |
upload_signatures() |
| 235 |
check_call('ssh code /apps/update-calibre-version.py'.split()) |
| 236 |
# self.upload_to_sourceforge() |
| 237 |
# upload_to_fosshub() |
| 238 |
self.upload_to_github(opts.replace) |
| 239 |
finally: |
| 240 |
shutil.rmtree(tdir, ignore_errors=True) |
| 241 |
|
| 242 |
def record_sizes(self, sizes): |
| 243 |
print('\nRecording dist sizes') |
| 244 |
args = [f'{__version__}:{fname}:{size}' for fname, size in sizes.items()] |
| 245 |
check_call(['ssh', 'code', '/usr/local/bin/dist_sizes'] + args) |
| 246 |
|
| 247 |
def upload_to_staging(self, tdir, files): |
| 248 |
os.mkdir(tdir + '/dist') |
| 249 |
hosting = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'hosting.py') |
| 250 |
shutil.copyfile(hosting, os.path.join(tdir, 'hosting.py')) |
| 251 |
|
| 252 |
for f in files: |
| 253 |
for x in (tdir + '/dist',): |
| 254 |
dest = os.path.join(x, os.path.basename(f)) |
| 255 |
shutil.copy2(f, x) |
| 256 |
os.chmod(dest, stat.S_IREAD | stat.S_IWRITE | stat.S_IRGRP | stat.S_IROTH) |
| 257 |
|
| 258 |
with open(os.path.join(tdir, 'fmap'), 'wb') as fo: |
| 259 |
for f, desc in files.items(): |
| 260 |
fo.write((f'{f}: {desc}\n').encode()) |
| 261 |
|
| 262 |
while True: |
| 263 |
try: |
| 264 |
send_data(tdir) |
| 265 |
except Exception: |
| 266 |
print('\nUpload to staging failed, retrying in a minute') |
| 267 |
time.sleep(60) |
| 268 |
else: |
| 269 |
break |
| 270 |
|
| 271 |
while True: |
| 272 |
try: |
| 273 |
send_to_backup(tdir) |
| 274 |
except Exception: |
| 275 |
print('\nUpload to backup failed, retrying in a minute') |
| 276 |
time.sleep(60) |
| 277 |
else: |
| 278 |
break |
| 279 |
|
| 280 |
def upload_to_github(self, replace): |
| 281 |
data = get_github_data() |
| 282 |
args, safe = gh_cmdline(__version__, data) |
| 283 |
if replace: |
| 284 |
args = ['--replace'] + args |
| 285 |
safe = ['--replace'] + safe |
| 286 |
run_remote_upload(args, safe) |
| 287 |
|
| 288 |
def upload_to_sourceforge(self): |
| 289 |
sdata = get_sourceforge_data() |
| 290 |
args = sf_cmdline(__version__, sdata) |
| 291 |
run_remote_upload(args) |
| 292 |
|
| 293 |
def upload_to_calibre(self): |
| 294 |
run_remote_upload(calibre_cmdline(__version__)) |
| 295 |
|
| 296 |
|
| 297 |
# }}} |
| 298 |
|
| 299 |
|
| 300 |
class UploadUserManual(Command): # {{{ |
| 301 |
description = 'Build and upload the User Manual' |
| 302 |
sub_commands = ['manual'] |
| 303 |
|
| 304 |
def build_plugin_example(self, path): |
| 305 |
from calibre import CurrentDir |
| 306 |
|
| 307 |
with NamedTemporaryFile(suffix='.zip') as f: |
| 308 |
os.fchmod(f.fileno(), stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH | stat.S_IWRITE) |
| 309 |
with CurrentDir(path): |
| 310 |
with ZipFile(f, 'w') as zf: |
| 311 |
for x in os.listdir('.'): |
| 312 |
if x.endswith('.swp'): |
| 313 |
continue |
| 314 |
zf.write(x) |
| 315 |
if os.path.isdir(x): |
| 316 |
for y in os.listdir(x): |
| 317 |
zf.write(os.path.join(x, y)) |
| 318 |
bname = self.b(path) + '_plugin.zip' |
| 319 |
dest = f'{DOWNLOADS}/{bname}' |
| 320 |
subprocess.check_call(['scp', f.name, 'main:' + dest]) |
| 321 |
|
| 322 |
def run(self, opts): |
| 323 |
path = self.j(self.SRC, '..', 'manual', 'plugin_examples') |
| 324 |
for x in glob.glob(self.j(path, '*')): |
| 325 |
self.build_plugin_example(x) |
| 326 |
|
| 327 |
srcdir = self.j(manual_build_dir(), 'en', 'html') + '/' |
| 328 |
check_call(' '.join(['rsync', '-zz', '-rl', '--info=progress2', srcdir, 'main:/srv/manual/']), shell=True) |
| 329 |
check_call('ssh main chown -R http:http /srv/manual'.split()) |
| 330 |
|
| 331 |
|
| 332 |
# }}} |
| 333 |
|
| 334 |
|
| 335 |
class UploadDemo(Command): # {{{ |
| 336 |
description = 'Rebuild and upload various demos' |
| 337 |
|
| 338 |
def run(self, opts): |
| 339 |
check_call( |
| 340 |
f'''ebook-convert {self.j(self.SRC, HTML2LRF)}/demo.html /tmp/html2lrf.lrf ''' |
| 341 |
'''--title='Demonstration of html2lrf' --authors='Kovid Goyal' ''' |
| 342 |
'''--header ''' |
| 343 |
'''--serif-family "/usr/share/fonts/corefonts, Times New Roman" ''' |
| 344 |
'''--mono-family "/usr/share/fonts/corefonts, Andale Mono" ''', |
| 345 |
shell=True, |
| 346 |
) |
| 347 |
|
| 348 |
lrf = self.j(self.SRC, 'calibre', 'ebooks', 'lrf', 'html', 'demo') |
| 349 |
check_call(f'cd {lrf} && zip -j /tmp/html-demo.zip * /tmp/html2lrf.lrf', shell=True) |
| 350 |
|
| 351 |
check_call(f'scp /tmp/html-demo.zip main:{DOWNLOADS}/', shell=True) |
| 352 |
|
| 353 |
|
| 354 |
# }}} |
| 355 |
|
| 356 |
|
| 357 |
class UploadToServer(Command): # {{{ |
| 358 |
description = 'Upload miscellaneous data to calibre server' |
| 359 |
|
| 360 |
def run(self, opts): |
| 361 |
check_call('scp translations/website/locales.zip main:/srv/main/'.split()) |
| 362 |
check_call('scp translations/changelog/locales.zip main:/srv/main/changelog-locales.zip'.split()) |
| 363 |
check_call('ssh main /apps/static/generate.py'.split()) |
| 364 |
src_file = glob.glob('dist/calibre-*.tar.xz')[0] |
| 365 |
upload_signatures() |
| 366 |
check_call(['git', 'push']) |
| 367 |
check_call([os.environ['PENV'] + '/gpg-as-kovid', '--armor', '--yes', '--detach-sign', src_file]) |
| 368 |
check_call(['scp', src_file + '.asc', 'code:/srv/code/signatures/']) |
| 369 |
check_call('ssh code /usr/local/bin/update-calibre-code.py'.split()) |
| 370 |
check_call(('ssh code /apps/update-calibre-version.py ' + __version__).split()) |
| 371 |
check_call( |
| 372 |
(f'ssh main /usr/local/bin/update-calibre-version.py {__version__} && /usr/local/bin/update-calibre-code.py && /apps/static/generate.py').split() |
| 373 |
) |
| 374 |
|
| 375 |
|
| 376 |
# }}} |