Debian Codegraph @

setup/upload.py

raw
1 #!/usr/bin/env python
2
3
4 __license__ = 'GPL v3'
5 __copyright__ = '2009, Kovid Goyal <[email protected]>'
6 __docformat__ = 'restructuredtext en'
7
8 import glob
9 import hashlib
10 import json
11 import os
12 import shutil
13 import stat
14 import subprocess
15 import sys
16 import time
17 from subprocess import check_call
18 from tempfile import NamedTemporaryFile, mkdtemp
19 from urllib.request import Request, urlopen
20 from zipfile import ZipFile
21
22 if __name__ == '__main__':
23 d = os.path.dirname
24 sys.path.insert(0, d(d(os.path.abspath(__file__))))
25
26 from setup import Command, __appname__, __version__, installer_names, manual_build_dir
27
28 DOWNLOADS = '/srv/main/downloads'
29 HTML2LRF = 'calibre/ebooks/lrf/html/demo'
30 TXT2LRF = 'src/calibre/ebooks/lrf/txt/demo'
31 STAGING_HOST = 'download.calibre-ebook.com'
32 BACKUP_HOST = 'code.calibre-ebook.com'
33 STAGING_USER = BACKUP_USER = 'root'
34 STAGING_DIR = '/root/staging'
35 BACKUP_DIR = '/binaries'
36
37
38 def installer_description(fname):
39 if fname.endswith('.tar.xz'):
40 return 'Source code'
41 if fname.endswith('.txz'):
42 return ('ARM' if 'arm64' in fname else 'AMD') + ' 64-bit Linux binary'
43 if fname.endswith('.msi'):
44 return 'Windows installer'
45 if fname.endswith('.dmg'):
46 return 'macOS dmg'
47 if fname.endswith('.exe'):
48 return 'Calibre Portable'
49 return 'Unknown file'
50
51
52 def upload_signatures():
53 tdir = mkdtemp()
54 scp = ['scp']
55 try:
56 for installer in installer_names():
57 if not os.path.exists(installer):
58 continue
59 sig = os.path.join(tdir, os.path.basename(installer + '.sig'))
60 scp.append(sig)
61 check_call([os.environ['PENV'] + '/gpg-as-kovid', '--output', sig, '--detach-sig', installer])
62 with open(installer, 'rb') as f:
63 raw = f.read()
64 fingerprint = hashlib.sha512(raw).hexdigest()
65 sha512 = os.path.join(tdir, os.path.basename(installer + '.sha512'))
66 with open(sha512, 'w') as f:
67 f.write(fingerprint)
68 scp.append(sha512)
69 for srv in 'code main'.split():
70 check_call(scp + [f'{srv}:/srv/{srv}/signatures/'])
71 check_call(['ssh', srv, 'chown', '-R', 'http:http', f'/srv/{srv}/signatures'])
72 finally:
73 shutil.rmtree(tdir)
74
75
76 class ReUpload(Command): # {{{
77 description = 'Re-upload any installers present in dist/'
78
79 sub_commands = ['upload_installers']
80
81 def pre_sub_commands(self, opts):
82 opts.replace = True
83 exists = {x for x in installer_names() if os.path.exists(x)}
84 if not exists:
85 print('There appear to be no installers!')
86 raise SystemExit(1)
87
88 def run(self, opts):
89 for x in installer_names():
90 if os.path.exists(x):
91 os.remove(x)
92
93
94 # }}}
95
96
97 # Data {{{
98 def get_github_data():
99 with open(os.environ['PENV'] + '/github-token', 'rb') as f:
100 un, pw = f.read().decode('utf-8').strip().split(':')
101 return {'username': un, 'password': pw}
102
103
104 def get_sourceforge_data():
105 return {'username': 'kovidgoyal', 'project': 'calibre'}
106
107
108 def get_fosshub_data():
109 with open(os.environ['PENV'] + '/fosshub', 'rb') as f:
110 return f.read().decode('utf-8').strip()
111
112
113 def send_data(loc):
114 subprocess.check_call([
115 'rsync',
116 '--inplace',
117 '--delete',
118 '-r',
119 '-zz',
120 '-h',
121 '--info=progress2',
122 '-e',
123 'ssh -x',
124 loc + '/',
125 f'{STAGING_USER}@{STAGING_HOST}:{STAGING_DIR}',
126 ])
127
128
129 def send_to_backup(loc):
130 host = f'{BACKUP_USER}@{BACKUP_HOST}'
131 dest = f'{BACKUP_DIR}/{__version__}'
132 subprocess.check_call(['ssh', '-x', host, 'mkdir', '-p', dest])
133 subprocess.check_call(['rsync', '--inplace', '--delete', '-r', '-zz', '-h', '--info=progress2', '-e', 'ssh -x', loc + '/', f'{host}:{dest}/'])
134
135
136 def gh_cmdline(ver, data):
137 safe = [
138 __appname__,
139 ver,
140 'fmap',
141 'github',
142 __appname__,
143 data['username'],
144 ]
145 return safe + [data['password']], safe + ['PASSWORD_REDACTED']
146
147
148 def sf_cmdline(ver, sdata):
149 return [__appname__, ver, 'fmap', 'sourceforge', sdata['project'], sdata['username']]
150
151
152 def calibre_cmdline(ver):
153 return [__appname__, ver, 'fmap', 'calibre']
154
155
156 def run_remote_upload(args, safe=None):
157 print('Running remotely:', ' '.join(safe or args))
158 subprocess.check_call(['ssh', '-x', f'{STAGING_USER}@{STAGING_HOST}', 'cd', STAGING_DIR, '&&', 'python', 'hosting.py'] + args)
159
160
161 # }}}
162
163
164 def upload_to_fosshub():
165 # https://devzone.fosshub.com/dashboard/restApi
166 # fosshub has no API to do partial uploads, so we always upload all files.
167 api_key = get_fosshub_data()
168
169 def request(path, data=None):
170 r = Request(
171 'https://api.fosshub.com/rest/' + path.lstrip('/'), headers={'Content-Type': 'application/json', 'X-auth-key': api_key, 'User-Agent': 'calibre'}
172 )
173 res = urlopen(r, data=data)
174 ans = json.loads(res.read())
175 if ans.get('error'):
176 raise SystemExit(ans['error'])
177 if res.getcode() != 200:
178 raise SystemExit(f'Request to {path} failed with response code: {res.getcode()}')
179 # from pprint import pprint
180 # pprint(ans)
181 return ans['status'] if 'status' in ans else ans['data']
182
183 print('Sending upload request to fosshub...')
184 project_id = None
185
186 for project in request('projects'):
187 if project['name'].lower() == 'calibre':
188 project_id = project['id']
189 break
190 else:
191 raise SystemExit('No calibre project found')
192
193 files = set(installer_names())
194 entries = []
195 for fname in files:
196 desc = installer_description(fname)
197 url = f'https://download.calibre-ebook.com/{__version__}/{os.path.basename(fname)}'
198 entries.append({
199 'fileUrl': url,
200 'type': desc,
201 'version': __version__,
202 })
203 jq = {
204 'version': __version__,
205 'files': entries,
206 'publish': True,
207 'isOldRelease': False,
208 }
209 data = json.dumps(jq)
210 # print(data)
211 data = data.encode('utf-8')
212 if not request(f'projects/{project_id}/releases/', data=data):
213 raise SystemExit('Failed to queue publish job with fosshub')
214
215
216 class UploadInstallers(Command): # {{{
217 def add_options(self, parser):
218 parser.add_option('--replace', default=False, action='store_true', help='Replace existing installers')
219
220 def run(self, opts):
221 # return upload_to_fosshub()
222 all_possible = set(installer_names())
223 available = set(glob.glob('dist/*'))
224 files = {x: installer_description(x) for x in all_possible.intersection(available)}
225 for x in files:
226 os.chmod(x, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH)
227 sizes = {os.path.basename(x): os.path.getsize(x) for x in files}
228 self.record_sizes(sizes)
229 tdir = mkdtemp()
230 try:
231 self.upload_to_staging(tdir, files)
232 self.upload_to_calibre()
233 if opts.replace:
234 upload_signatures()
235 check_call('ssh code /apps/update-calibre-version.py'.split())
236 # self.upload_to_sourceforge()
237 # upload_to_fosshub()
238 self.upload_to_github(opts.replace)
239 finally:
240 shutil.rmtree(tdir, ignore_errors=True)
241
242 def record_sizes(self, sizes):
243 print('\nRecording dist sizes')
244 args = [f'{__version__}:{fname}:{size}' for fname, size in sizes.items()]
245 check_call(['ssh', 'code', '/usr/local/bin/dist_sizes'] + args)
246
247 def upload_to_staging(self, tdir, files):
248 os.mkdir(tdir + '/dist')
249 hosting = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'hosting.py')
250 shutil.copyfile(hosting, os.path.join(tdir, 'hosting.py'))
251
252 for f in files:
253 for x in (tdir + '/dist',):
254 dest = os.path.join(x, os.path.basename(f))
255 shutil.copy2(f, x)
256 os.chmod(dest, stat.S_IREAD | stat.S_IWRITE | stat.S_IRGRP | stat.S_IROTH)
257
258 with open(os.path.join(tdir, 'fmap'), 'wb') as fo:
259 for f, desc in files.items():
260 fo.write((f'{f}: {desc}\n').encode())
261
262 while True:
263 try:
264 send_data(tdir)
265 except Exception:
266 print('\nUpload to staging failed, retrying in a minute')
267 time.sleep(60)
268 else:
269 break
270
271 while True:
272 try:
273 send_to_backup(tdir)
274 except Exception:
275 print('\nUpload to backup failed, retrying in a minute')
276 time.sleep(60)
277 else:
278 break
279
280 def upload_to_github(self, replace):
281 data = get_github_data()
282 args, safe = gh_cmdline(__version__, data)
283 if replace:
284 args = ['--replace'] + args
285 safe = ['--replace'] + safe
286 run_remote_upload(args, safe)
287
288 def upload_to_sourceforge(self):
289 sdata = get_sourceforge_data()
290 args = sf_cmdline(__version__, sdata)
291 run_remote_upload(args)
292
293 def upload_to_calibre(self):
294 run_remote_upload(calibre_cmdline(__version__))
295
296
297 # }}}
298
299
300 class UploadUserManual(Command): # {{{
301 description = 'Build and upload the User Manual'
302 sub_commands = ['manual']
303
304 def build_plugin_example(self, path):
305 from calibre import CurrentDir
306
307 with NamedTemporaryFile(suffix='.zip') as f:
308 os.fchmod(f.fileno(), stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH | stat.S_IWRITE)
309 with CurrentDir(path):
310 with ZipFile(f, 'w') as zf:
311 for x in os.listdir('.'):
312 if x.endswith('.swp'):
313 continue
314 zf.write(x)
315 if os.path.isdir(x):
316 for y in os.listdir(x):
317 zf.write(os.path.join(x, y))
318 bname = self.b(path) + '_plugin.zip'
319 dest = f'{DOWNLOADS}/{bname}'
320 subprocess.check_call(['scp', f.name, 'main:' + dest])
321
322 def run(self, opts):
323 path = self.j(self.SRC, '..', 'manual', 'plugin_examples')
324 for x in glob.glob(self.j(path, '*')):
325 self.build_plugin_example(x)
326
327 srcdir = self.j(manual_build_dir(), 'en', 'html') + '/'
328 check_call(' '.join(['rsync', '-zz', '-rl', '--info=progress2', srcdir, 'main:/srv/manual/']), shell=True)
329 check_call('ssh main chown -R http:http /srv/manual'.split())
330
331
332 # }}}
333
334
335 class UploadDemo(Command): # {{{
336 description = 'Rebuild and upload various demos'
337
338 def run(self, opts):
339 check_call(
340 f'''ebook-convert {self.j(self.SRC, HTML2LRF)}/demo.html /tmp/html2lrf.lrf '''
341 '''--title='Demonstration of html2lrf' --authors='Kovid Goyal' '''
342 '''--header '''
343 '''--serif-family "/usr/share/fonts/corefonts, Times New Roman" '''
344 '''--mono-family "/usr/share/fonts/corefonts, Andale Mono" ''',
345 shell=True,
346 )
347
348 lrf = self.j(self.SRC, 'calibre', 'ebooks', 'lrf', 'html', 'demo')
349 check_call(f'cd {lrf} && zip -j /tmp/html-demo.zip * /tmp/html2lrf.lrf', shell=True)
350
351 check_call(f'scp /tmp/html-demo.zip main:{DOWNLOADS}/', shell=True)
352
353
354 # }}}
355
356
357 class UploadToServer(Command): # {{{
358 description = 'Upload miscellaneous data to calibre server'
359
360 def run(self, opts):
361 check_call('scp translations/website/locales.zip main:/srv/main/'.split())
362 check_call('scp translations/changelog/locales.zip main:/srv/main/changelog-locales.zip'.split())
363 check_call('ssh main /apps/static/generate.py'.split())
364 src_file = glob.glob('dist/calibre-*.tar.xz')[0]
365 upload_signatures()
366 check_call(['git', 'push'])
367 check_call([os.environ['PENV'] + '/gpg-as-kovid', '--armor', '--yes', '--detach-sign', src_file])
368 check_call(['scp', src_file + '.asc', 'code:/srv/code/signatures/'])
369 check_call('ssh code /usr/local/bin/update-calibre-code.py'.split())
370 check_call(('ssh code /apps/update-calibre-version.py ' + __version__).split())
371 check_call(
372 (f'ssh main /usr/local/bin/update-calibre-version.py {__version__} && /usr/local/bin/update-calibre-code.py && /apps/static/generate.py').split()
373 )
374
375
376 # }}}