Debian Codegraph @

setup/hosting.py

raw
1 #!/usr/bin/env python
2
3 __license__ = 'GPL v3'
4 __copyright__ = '2011, Kovid Goyal <[email protected]>'
5 __docformat__ = 'restructuredtext en'
6
7 import json
8 import mimetypes
9 import os
10 import shutil
11 import sys
12 import time
13 from argparse import ArgumentParser, FileType
14 from collections import OrderedDict
15 from pprint import pprint
16 from subprocess import check_call
17
18
19 class ReadFileWithProgressReporting: # {{{
20 def __init__(self, path, mode='rb'):
21 self.fobj = open(path, mode)
22 self.fobj.seek(0, os.SEEK_END)
23 self._total = self.fobj.tell()
24 self.fobj.seek(0)
25 self.start_time = time.time()
26
27 def tell(self, *a):
28 return self.fobj.tell(*a)
29
30 def __enter__(self):
31 return self
32
33 def __exit__(self, *a):
34 self.fobj.close()
35 del self.fobj
36
37 def __len__(self):
38 return self._total
39
40 def read(self, size):
41 data = self.fobj.read(size)
42 if data:
43 self.report_progress(len(data))
44 return data
45
46 def report_progress(self, size):
47 sys.stdout.write('\x1b[s')
48 sys.stdout.write('\x1b[K')
49 frac = float(self.tell()) / self._total
50 mb_pos = self.tell() / float(1024**2)
51 mb_tot = self._total / float(1024**2)
52 kb_pos = self.tell() / 1024.0
53 kb_rate = kb_pos / (time.time() - self.start_time)
54 bit_rate = kb_rate * 1024
55 eta = int((self._total - self.tell()) / bit_rate) + 1
56 eta_m, eta_s = eta / 60, eta % 60
57 sys.stdout.write(f' {frac * 100:.1f}% {mb_pos:.1f}/{mb_tot:.1f}MB {kb_rate:.1f} KB/sec {eta_m} minutes, {eta_s} seconds left')
58 sys.stdout.write('\x1b[u')
59 if self.tell() >= self._total:
60 sys.stdout.write('\n')
61 t = int(time.time() - self.start_time) + 1
62 print(f'Upload took {t / 60} minutes and {t % 60} seconds at {kb_rate:.1f} KB/sec')
63 sys.stdout.flush()
64
65
66 # }}}
67
68
69 class Base: # {{{
70 def __init__(self):
71 self.d = os.path.dirname
72 self.j = os.path.join
73 self.a = os.path.abspath
74 self.b = os.path.basename
75 self.s = os.path.splitext
76 self.e = os.path.exists
77
78 def info(self, *args, **kwargs):
79 print(*args, **kwargs)
80 sys.stdout.flush()
81
82 def warn(self, *args, **kwargs):
83 print('\n' + '_' * 20, 'WARNING', '_' * 20)
84 print(*args, **kwargs)
85 print('_' * 50)
86 sys.stdout.flush()
87
88
89 # }}}
90
91
92 class SourceForge(Base): # {{{
93 # Note that you should manually ssh once to username,[email protected]
94 # on the staging server so that the host key is setup
95
96 def __init__(self, files, project, version, username, replace=False):
97 self.username, self.project, self.version = username, project, version
98 self.base = '/home/frs/project/c/ca/' + project
99 self.rdir = self.base + '/' + version
100 self.files = files
101
102 def __call__(self):
103 for x in self.files:
104 start = time.time()
105 self.info('Uploading', x)
106 for i in range(5):
107 try:
108 check_call([
109 'rsync',
110 '-h',
111 '-zz',
112 '--progress',
113 '-e',
114 'ssh -x',
115 x,
116 '{},{}@frs.sourceforge.net:{}'.format(self.username, self.project, self.rdir + '/'),
117 ])
118 except KeyboardInterrupt:
119 raise SystemExit(1)
120 except Exception:
121 print('\nUpload failed, trying again in 30 seconds')
122 time.sleep(30)
123 else:
124 break
125 print('Uploaded in', int(time.time() - start), 'seconds\n\n')
126
127
128 # }}}
129
130
131 class GitHub(Base): # {{{
132 API = 'https://api.github.com/'
133
134 def __init__(self, files, reponame, version, username, password, replace=False):
135 self.files, self.reponame, self.version, self.username, self.password, self.replace = (files, reponame, version, username, password, replace)
136 self.current_tag_name = 'v' + self.version
137 import requests
138
139 self.requests = s = requests.Session()
140 s.auth = (self.username, self.password)
141 s.headers.update({'Accept': 'application/vnd.github.v3+json'})
142
143 def __call__(self):
144 releases = self.releases()
145 self.clean_older_releases(releases)
146 release = self.create_release(releases)
147 upload_url = release['upload_url'].partition('{')[0]
148 existing_assets = self.existing_assets(release['id'])
149 for path, desc in self.files.items():
150 self.info('')
151 url = self.API + f'repos/{self.username}/{self.reponame}/releases/assets/{{}}'
152 fname = os.path.basename(path)
153 if fname in existing_assets:
154 self.info(f'Deleting {fname} from GitHub with id: {existing_assets[fname]}')
155 r = self.requests.delete(url.format(existing_assets[fname]))
156 if r.status_code != 204:
157 self.fail(r, f'Failed to delete {fname} from GitHub')
158 r = self.do_upload(upload_url, path, desc, fname)
159 if r.status_code != 201:
160 self.fail(r, f'Failed to upload file: {fname}')
161 try:
162 r = self.requests.patch(url.format(r.json()['id']), data=json.dumps({'name': fname, 'label': desc}))
163 except Exception:
164 time.sleep(15)
165 r = self.requests.patch(url.format(r.json()['id']), data=json.dumps({'name': fname, 'label': desc}))
166 if r.status_code != 200:
167 self.fail(r, f'Failed to set label for {fname}')
168
169 def clean_older_releases(self, releases):
170 for release in releases:
171 if release.get('assets', None) and release['tag_name'] != self.current_tag_name:
172 self.info('\nDeleting old released installers from: {}'.format(release['tag_name']))
173 for asset in release['assets']:
174 r = self.requests.delete(self.API + 'repos/{}/{}/releases/assets/{}'.format(self.username, self.reponame, asset['id']))
175 if r.status_code != 204:
176 self.fail(r, 'Failed to delete obsolete asset: {} for release: {}'.format(asset['name'], release['tag_name']))
177
178 def do_upload(self, url, path, desc, fname):
179 mime_type = mimetypes.guess_type(fname)[0] or 'application/octet-stream'
180 self.info(f'Uploading to GitHub: {fname} ({mime_type})')
181 with ReadFileWithProgressReporting(path) as f:
182 return self.requests.post(url, headers={'Content-Type': mime_type, 'Content-Length': str(f._total)}, params={'name': fname}, data=f)
183
184 def fail(self, r, msg):
185 print(msg, f' Status Code: {r.status_code}', file=sys.stderr)
186 print('JSON from response:', file=sys.stderr)
187 pprint(dict(r.json()), stream=sys.stderr)
188 raise SystemExit(1)
189
190 def already_exists(self, r):
191 error_code = r.json().get('errors', [{}])[0].get('code', None)
192 return error_code == 'already_exists'
193
194 def existing_assets(self, release_id):
195 url = self.API + f'repos/{self.username}/{self.reponame}/releases/{release_id}/assets'
196 r = self.requests.get(url)
197 if r.status_code != 200:
198 self.fail('Failed to get assets for release')
199 return {asset['name']: asset['id'] for asset in r.json()}
200
201 def releases(self):
202 url = self.API + f'repos/{self.username}/{self.reponame}/releases'
203 r = self.requests.get(url)
204 if r.status_code != 200:
205 self.fail(r, 'Failed to list releases')
206 return r.json()
207
208 def create_release(self, releases):
209 "Create a release on GitHub or if it already exists, return the existing release"
210 for release in releases:
211 # Check for existing release
212 if release['tag_name'] == self.current_tag_name:
213 return release
214 url = self.API + f'repos/{self.username}/{self.reponame}/releases'
215 r = self.requests.post(
216 url,
217 data=json.dumps({
218 'tag_name': self.current_tag_name,
219 'target_commitish': 'master',
220 'name': f'version {self.version}',
221 'body': f'Release version {self.version}',
222 'draft': False,
223 'prerelease': False,
224 }),
225 )
226 if r.status_code != 201:
227 self.fail(r, f'Failed to create release for version: {self.version}')
228 return r.json()
229
230
231 # }}}
232
233
234 def generate_index(): # {{{
235 os.chdir('/srv/download')
236 releases = set()
237 for x in os.listdir('.'):
238 if os.path.isdir(x) and '.' in x:
239 releases.add(tuple(int(y) for y in x.split('.')))
240 rmap = OrderedDict()
241 for rnum in sorted(releases, reverse=True):
242 series = rnum[:2] if rnum[0] == 0 else rnum[:1]
243 if series not in rmap:
244 rmap[series] = []
245 rmap[series].append(rnum)
246
247 template = '''<!DOCTYPE html>\n<html lang="en"> <head> <meta charset="utf-8"> <title>{title}</title><link rel="icon" type="image/png" href="//calibre-ebook.com/favicon.png" /> <style type="text/css"> {style} </style> </head> <body> <h1>{title}</h1> <p>{msg}</p> {body} </body> </html> ''' # noqa: E501
248 style = '''
249 body { font-family: sans-serif; background-color: #eee; }
250 a { text-decoration: none; }
251 a:visited { color: blue }
252 a:hover { color: red }
253 ul { list-style-type: none }
254 li { padding-bottom: 1ex }
255 dd li { text-indent: 0; margin: 0 }
256 dd ul { padding: 0; margin: 0 }
257 dt { font-weight: bold }
258 dd { margin-bottom: 2ex }
259 '''
260 body = []
261 for series in rmap:
262 body.append(
263 '<li><a href="{0}.html" title="Releases in the {0}.x series">{0}.x</a>\xa0\xa0\xa0<span style="font-size:smaller">[{1} releases]</span></li>'.format( # noqa: E501
264 '.'.join(map(str, series)), len(rmap[series])
265 )
266 )
267 body = '<ul>{}</ul>'.format(' '.join(body))
268 index = template.format(title='Previous calibre releases', style=style, msg='Choose a series of calibre releases', body=body)
269 with open('index.html', 'wb') as f:
270 f.write(index.encode('utf-8'))
271
272 for series, releases in rmap.items():
273 sname = '.'.join(map(str, series))
274 body = ['<li><a href="{0}/" title="Release {0}">{0}</a></li>'.format('.'.join(map(str, r))) for r in releases]
275 body = '<ul class="release-list">{}</ul>'.format(' '.join(body))
276 index = template.format(title=f'Previous calibre releases ({sname}.x)', style=style, msg='Choose a calibre release', body=body)
277 with open(f'{sname}.html', 'wb') as f:
278 f.write(index.encode('utf-8'))
279
280 for r in releases:
281 rname = '.'.join(map(str, r))
282 os.chdir(rname)
283 try:
284 body = []
285 files = os.listdir('.')
286 windows = [x for x in files if x.endswith('.msi')]
287 if windows:
288
289 def wdesc(x):
290 return 'Windows ' + ('64-bit' if '-64bit-' in x else '32-bit') + ' Installer'
291
292 windows = ['<li><a href="{0}" title="{1}">{1}</a></li>'.format(x, wdesc(x)) for x in windows]
293 body.append('<dt>Windows</dt><dd><ul>{}</ul></dd>'.format(' '.join(windows)))
294 portable = [x for x in files if '-portable-' in x]
295 if portable:
296 body.append('<dt>Calibre Portable</dt><dd><a href="{0}" title="{1}">{1}</a></dd>'.format(portable[0], 'Calibre Portable Installer'))
297 osx = [x for x in files if x.endswith('.dmg')]
298 if osx:
299 body.append('<dt>Apple Mac</dt><dd><a href="{0}" title="{1}">{1}</a></dd>'.format(osx[0], 'OS X Disk Image (.dmg)'))
300 linux = [x for x in files if x.endswith(('.txz', 'tar.bz2'))]
301 if linux:
302
303 def ldesc(x):
304 if 'i686' in x:
305 return 'Linux Intel 32-bit binary'
306 if 'arm64' in x:
307 return 'Linux ARM 64-bit binary'
308 return 'Linux Intel 64-bit binary'
309
310 linux = ['<li><a href="{0}" title="{1}">{1}</a></li>'.format(x, ldesc(x)) for x in linux]
311 body.append('<dt>Linux</dt><dd><ul>{}</ul></dd>'.format(' '.join(linux)))
312 source = [x for x in files if x.endswith(('.xz', '.gz'))]
313 if source:
314 body.append('<dt>Source Code</dt><dd><a href="{0}" title="{1}">{1}</a></dd>'.format(source[0], 'Source code (all platforms)'))
315
316 body = '<dl>{}</dl>'.format(''.join(body))
317 index = template.format(title=f'calibre release ({rname})', style=style, msg='', body=body)
318 with open('index.html', 'wb') as f:
319 f.write(index.encode('utf-8'))
320 finally:
321 os.chdir('..')
322
323
324 # }}}
325
326
327 SERVER_BASE = '/srv/download/'
328
329
330 def upload_to_servers(files, version): # {{{
331 base = SERVER_BASE
332 dest = os.path.join(base, version)
333 if not os.path.exists(dest):
334 os.mkdir(dest)
335 for src in files:
336 shutil.copyfile(src, os.path.join(dest, os.path.basename(src)))
337 cwd = os.getcwd()
338 try:
339 generate_index()
340 finally:
341 os.chdir(cwd)
342
343 # for server, rdir in {'files':'/srv/download/'}.items():
344 # print('Uploading to server:', server)
345 # server = f'{server}.calibre-ebook.com'
346 # # Copy the generated index files
347 # print('Copying generated index')
348 # check_call(['rsync', '-hza', '-e', 'ssh -x', '--include', '*.html',
349 # '--filter', '-! */', base, f'root@{server}:{rdir}'])
350 # # Copy the release files
351 # rdir = f'{rdir}{version}/'
352 # for x in files:
353 # start = time.time()
354 # print('Uploading', x)
355 # for i in range(5):
356 # try:
357 # check_call(['rsync', '-h', '-z', '--progress', '-e', 'ssh -x', x,
358 # f'root@{server}:{rdir}'])
359 # except KeyboardInterrupt:
360 # raise SystemExit(1)
361 # except Exception:
362 # print('\nUpload failed, trying again in 30 seconds')
363 # time.sleep(30)
364 # else:
365 # break
366 # print('Uploaded in', int(time.time() - start), 'seconds\n\n')
367
368
369 # }}}
370
371
372 # CLI {{{
373 def cli_parser():
374 epilog = 'Copyright Kovid Goyal 2012'
375
376 p = ArgumentParser(description='Upload project files to a hosting service automatically', epilog=epilog)
377 a = p.add_argument
378 a('appname', help='The name of the application, all files to upload should begin with this name')
379 a('version', help='The version of the application, all files to upload should contain this version')
380 a(
381 'file_map',
382 type=FileType('r'),
383 help='A file containing a mapping of files to be uploaded to '
384 'descriptions of the files. The descriptions will be visible '
385 'to users trying to get the file from the hosting service. '
386 'The format of the file is filename: description, with one per '
387 'line. filename can be a path to the file relative to the current '
388 'directory.',
389 )
390 a('--replace', action='store_true', default=False, help='If specified, existing files are replaced, otherwise they are skipped.')
391
392 subparsers = p.add_subparsers(help='Where to upload to', dest='service', title='Service', description='Hosting service to upload to')
393 sf = subparsers.add_parser('sourceforge', help='Upload to sourceforge', epilog=epilog)
394 gh = subparsers.add_parser('github', help='Upload to GitHub', epilog=epilog)
395 subparsers.add_parser('calibre', help='Upload to calibre file servers')
396
397 a = sf.add_argument
398 a('project', help='The name of the project on sourceforge we are uploading to')
399 a('username', help='Sourceforge username')
400
401 a = gh.add_argument
402 a('project', help='The name of the repository on GitHub we are uploading to')
403 a('username', help='Username to log into your GitHub account')
404 a('password', help='Password to log into your GitHub account')
405
406 return p
407
408
409 def main(args=None):
410 cli = cli_parser()
411 args = cli.parse_args(args)
412 files = {}
413 with args.file_map as f:
414 for line in f:
415 fname, _, desc = line.partition(':')
416 fname, desc = fname.strip(), desc.strip()
417 if fname and desc:
418 files[fname] = desc
419
420 ofiles = OrderedDict()
421 for x in sorted(files, key=lambda x: os.stat(x).st_size, reverse=True):
422 ofiles[x] = files[x]
423
424 if args.service == 'sourceforge':
425 sf = SourceForge(ofiles, args.project, args.version, args.username, replace=args.replace)
426 sf()
427 elif args.service == 'github':
428 gh = GitHub(ofiles, args.project, args.version, args.username, args.password, replace=args.replace)
429 gh()
430 elif args.service == 'calibre':
431 upload_to_servers(ofiles, args.version)
432
433
434 if __name__ == '__main__':
435 main()
436 # }}}