Debian Codegraph @

setup/publish.py

raw
1 #!/usr/bin/env python
2
3
4 __license__ = 'GPL v3'
5 __copyright__ = '2009, Kovid Goyal <[email protected]>'
6 __docformat__ = 'restructuredtext en'
7
8 import filecmp
9 import json
10 import os
11 import shutil
12 import subprocess
13 import sys
14 import time
15
16 from setup import Command, __version__, installer_names, manual_build_dir, require_clean_git, require_git_master
17 from setup.parallel_build import create_job, parallel_build
18
19
20 class Stage1(Command):
21 description = 'Stage 1 of the publish process'
22
23 sub_commands = [
24 'cacerts',
25 'resources',
26 'iso639',
27 'iso3166',
28 'gui',
29 'recent_uas',
30 ]
31
32
33 class Stage2(Command):
34 description = 'Stage 2 of the publish process, builds the binaries'
35
36 def run(self, opts):
37 base = os.path.join(self.d(self.SRC))
38 for x in ('dist', 'build'):
39 x = os.path.join(base, x)
40 if os.path.exists(x):
41 shutil.rmtree(x)
42 os.mkdir(x)
43
44 self.info('Starting builds for all platforms, this will take a while...')
45
46 session = ['layout vertical']
47 platforms = 'linux64', 'linuxarm64', 'osx', 'win'
48 for x in platforms:
49 cmd = (
50 f'''{sys.executable} -c "import subprocess; subprocess.Popen(['{sys.executable}', './setup.py', '{x}']).wait() != 0 and'''
51 f''' input('Build of {x} failed, press Enter to exit');"'''
52 )
53 session.append('title ' + x)
54 session.append('launch ' + cmd)
55
56 p = subprocess.Popen(
57 ['kitty', '-o', 'enabled_layouts=vertical,stack', '-o', 'scrollback_lines=20000', '-o', 'close_on_child_death=y', '--session=-'],
58 stdin=subprocess.PIPE,
59 )
60
61 p.communicate('\n'.join(session).encode('utf-8'))
62 p.wait()
63
64 for installer in installer_names(include_source=False):
65 installer = self.j(self.d(self.SRC), installer)
66 if not os.path.exists(installer) or os.path.getsize(installer) < 10000:
67 raise SystemExit(f'The installer {os.path.basename(installer)} does not exist')
68 os.chmod(installer, 0o644)
69
70
71 class Stage3(Command):
72 description = 'Stage 3 of the publish process'
73 sub_commands = ['upload_user_manual', 'upload_demo', 'sdist', 'tag_release']
74
75
76 class Stage4(Command):
77 description = 'Stage 4 of the publish process'
78 sub_commands = ['upload_installers']
79
80
81 class Stage5(Command):
82 description = 'Stage 5 of the publish process'
83 sub_commands = ['upload_to_server']
84
85 def run(self, opts):
86 subprocess.check_call('rm -rf build/* dist/*', shell=True)
87
88
89 def require_hsm_for_signing():
90 cp = subprocess.run(['pkcs11-tool', '-L'], stdout=subprocess.DEVNULL)
91 if cp.returncode != 0:
92 raise SystemExit('Attach the HSM for authenticode signing first')
93
94
95 class Publish(Command):
96 description = 'Publish a new calibre release'
97 sub_commands = [
98 'stage1',
99 'stage2',
100 'stage3',
101 'stage4',
102 'stage5',
103 ]
104
105 def pre_sub_commands(self, opts):
106 require_git_master()
107 require_clean_git()
108 require_hsm_for_signing()
109 version = tuple(map(int, __version__.split('.'))) # noqa: RUF048
110 if version[2] > 99:
111 raise SystemExit(f'The version number {__version__} indicates a preview release, did you mean to run ./setup.py publish_preview?')
112 if 'PUBLISH_BUILD_DONE' not in os.environ:
113 subprocess.check_call([sys.executable, 'setup.py', 'check'])
114 subprocess.check_call([sys.executable, 'setup.py', 'build'])
115 if 'SKIP_CALIBRE_TESTS' not in os.environ:
116 subprocess.check_call([sys.executable, 'setup.py', 'test'])
117 subprocess.check_call([sys.executable, 'setup.py', 'pot'])
118 subprocess.check_call([sys.executable, 'setup.py', 'translations'])
119 os.environ['PUBLISH_BUILD_DONE'] = '1'
120 os.execl(os.path.abspath('setup.py'), './setup.py', 'publish')
121
122
123 class PublishBetas(Command):
124 sub_commands = ['stage1', 'stage2', 'sdist']
125
126 def pre_sub_commands(self, opts):
127 require_clean_git()
128 require_hsm_for_signing()
129 # require_git_master()
130
131 def run(self, opts):
132 dist = self.a(self.j(self.d(self.SRC), 'dist'))
133 subprocess.check_call((f'rsync --partial -rh --info=progress2 --delete-after {dist}/ download.calibre-ebook.com:/srv/download/betas/').split())
134
135
136 class PublishPreview(Command):
137 sub_commands = ['stage1', 'stage2', 'sdist']
138
139 def pre_sub_commands(self, opts):
140 version = tuple(map(int, __version__.split('.'))) # noqa: RUF048
141 if version[2] < 100:
142 raise SystemExit('Must set calibre version to have patch level greater than 100')
143 require_clean_git()
144 require_git_master()
145 require_hsm_for_signing()
146
147 def run(self, opts):
148 dist = self.a(self.j(self.d(self.SRC), 'dist'))
149 with open(os.path.join(dist, 'README.txt'), 'w') as f:
150 print(
151 '''\
152 These are preview releases of changes to calibre since the last normal release.
153 Preview releases are typically released every Friday, they serve as a way
154 for users to test upcoming features/fixes in the next calibre release.
155 ''',
156 file=f,
157 )
158 subprocess.check_call((f'rsync -rh --info=progress2 --delete-after --delete {dist}/ download.calibre-ebook.com:/srv/download/preview/').split())
159
160
161 class Manual(Command):
162 description = '''Build the User Manual '''
163
164 def add_options(self, parser):
165 parser.add_option(
166 '-l',
167 '--language',
168 action='append',
169 default=[],
170 help=('Build translated versions for only the specified languages (can be specified multiple times)'),
171 )
172 parser.add_option('--serve', action='store_true', default=False, help='Run a webserver on the built manual files')
173
174 def run(self, opts):
175 tdir = manual_build_dir()
176 if os.path.exists(tdir):
177 shutil.rmtree(tdir)
178 os.mkdir(tdir)
179 st = time.time()
180 base = self.j(self.d(self.SRC), 'manual')
181 for d in ('generated',):
182 d = self.j(base, d)
183 if os.path.exists(d):
184 shutil.rmtree(d)
185 os.makedirs(d)
186 jobs = []
187 languages = opts.language or list(json.load(open(self.j(base, 'locale', 'completed.json'), 'rb')))
188 languages = set(languages) - {'en'}
189 languages.discard('ta') # Tamil translations break Sphinx
190 languages = ['en'] + list(languages)
191 os.environ['ALL_USER_MANUAL_LANGUAGES'] = ' '.join(languages)
192 for language in languages:
193 jobs.append(
194 create_job(
195 [sys.executable, self.j(self.d(self.SRC), 'manual', 'build.py'), language, self.j(tdir, language)],
196 f'\n\n**************** Building translations for: {language}',
197 )
198 )
199 self.info(f'Building manual for {len(jobs)} languages')
200 subprocess.check_call(jobs[0].cmd)
201 if not parallel_build(jobs[1:], self.info):
202 raise SystemExit(1)
203 cwd = os.getcwd()
204 with open('resources/localization/website-languages.txt') as wl:
205 languages = frozenset(filter(None, (x.strip() for x in wl.read().split())))
206 try:
207 os.chdir(self.j(tdir, 'en', 'html'))
208 for x in os.listdir(tdir):
209 if x != 'en':
210 shutil.copytree(self.j(tdir, x, 'html'), x)
211 self.replace_with_symlinks(x)
212 else:
213 os.symlink('.', 'en')
214 for x in languages:
215 if x and not os.path.exists(x):
216 os.symlink('.', x)
217 self.info(f'Built manual for {len(jobs)} languages in {int((time.time() - st) / 60.0)} minutes')
218 finally:
219 os.chdir(cwd)
220
221 if opts.serve:
222 self.serve_manual(self.j(tdir, 'en', 'html'))
223
224 def serve_manual(self, root):
225 os.chdir(root)
226 from http.server import HTTPServer, SimpleHTTPRequestHandler
227
228 HandlerClass = SimpleHTTPRequestHandler
229 ServerClass = HTTPServer
230 Protocol = 'HTTP/1.0'
231 server_address = ('127.0.0.1', 8000)
232
233 HandlerClass.protocol_version = Protocol
234 httpd = ServerClass(server_address, HandlerClass)
235
236 print('Serving User Manual on localhost:8000')
237 from calibre.gui2 import open_url
238
239 open_url('http://localhost:8000')
240 httpd.serve_forever()
241
242 def replace_with_symlinks(self, lang_dir):
243 "Replace all identical files with symlinks to save disk space/upload bandwidth"
244 from calibre import walk
245
246 base = self.a(lang_dir)
247 for f in walk(base):
248 r = os.path.relpath(f, base)
249 orig = self.j(self.d(base), r)
250 try:
251 sz = os.stat(orig).st_size
252 except OSError:
253 continue
254 if sz == os.stat(f).st_size and filecmp._do_cmp(f, orig):
255 os.remove(f)
256 os.symlink(os.path.relpath(orig, self.d(f)), f)
257
258 def clean(self):
259 path = os.path.join(self.SRC, 'calibre', 'manual', '.build')
260 if os.path.exists(path):
261 shutil.rmtree(path)
262
263
264 class ManPages(Command):
265 description = '''Build the man pages '''
266
267 def add_options(self, parser):
268 parser.add_option('--man-dir', help='Where to generate the man pages')
269 parser.add_option('--compress-man-pages', default=False, action='store_true', help='Compress the generated man pages')
270
271 def run(self, opts):
272 self.build_man_pages(opts.man_dir or 'man-pages', opts.compress_man_pages)
273
274 def build_man_pages(self, dest, compress=False):
275 from calibre.utils.localization import available_translations
276
277 dest = os.path.abspath(dest)
278 if os.path.exists(dest):
279 shutil.rmtree(dest)
280 os.makedirs(dest)
281 base = self.j(self.d(self.SRC), 'manual')
282 languages = set(available_translations())
283 languages.discard('ta') # Tamil translatins are completely borked break sphinx
284 languages.discard('id') # Indonesian man page fails to build
285 languages = ['en'] + list(languages - {'en', 'en_GB'})
286 os.environ['ALL_USER_MANUAL_LANGUAGES'] = ' '.join(languages)
287 try:
288 os.makedirs(dest)
289 except OSError:
290 pass
291 jobs = []
292 for l in languages:
293 jobs.append(create_job([sys.executable, self.j(base, 'build.py'), '--man-pages', l, dest], f'\n\n**************** Building translations for: {l}'))
294 self.info(f'\tCreating man pages in {dest} for {len(jobs)} languages...')
295 subprocess.check_call(jobs[0].cmd)
296 if not parallel_build(jobs[1:], self.info, verbose=False):
297 raise SystemExit(1)
298 cwd = os.getcwd()
299 os.chdir(dest)
300 try:
301 for x in tuple(os.listdir('.')):
302 if x in languages:
303 if x == 'en':
304 os.rename(x, 'man1')
305 else:
306 os.mkdir(self.j(x, 'man1'))
307 for y in os.listdir(x):
308 if y != 'man1':
309 os.rename(self.j(x, y), self.j(x, 'man1', y))
310 else:
311 shutil.rmtree(x) if os.path.isdir(x) else os.remove(x)
312 if compress:
313 jobs = []
314 for dirpath, dirnames, filenames in os.walk('.'):
315 for f in filenames:
316 if f.endswith('.1'):
317 jobs.append(create_job(['gzip', '--best', self.j(dirpath, f)], ''))
318 if not parallel_build(jobs, self.info, verbose=False):
319 raise SystemExit(1)
320 finally:
321 os.chdir(cwd)
322
323
324 class TagRelease(Command):
325 description = 'Tag a new release in git'
326
327 def run(self, opts):
328 self.info('Tagging release')
329 subprocess.check_call(f'git tag -s v{__version__} -m "version-{__version__}"'.split())
330 subprocess.check_call(f'git push origin v{__version__}'.split())