| 1 |
#!/usr/bin/env python |
| 2 |
|
| 3 |
|
| 4 |
__license__ = 'GPL v3' |
| 5 |
__copyright__ = '2009, Kovid Goyal <[email protected]>' |
| 6 |
__docformat__ = 'restructuredtext en' |
| 7 |
|
| 8 |
import filecmp |
| 9 |
import json |
| 10 |
import os |
| 11 |
import shutil |
| 12 |
import subprocess |
| 13 |
import sys |
| 14 |
import time |
| 15 |
|
| 16 |
from setup import Command, __version__, installer_names, manual_build_dir, require_clean_git, require_git_master |
| 17 |
from setup.parallel_build import create_job, parallel_build |
| 18 |
|
| 19 |
|
| 20 |
class Stage1(Command): |
| 21 |
description = 'Stage 1 of the publish process' |
| 22 |
|
| 23 |
sub_commands = [ |
| 24 |
'cacerts', |
| 25 |
'resources', |
| 26 |
'iso639', |
| 27 |
'iso3166', |
| 28 |
'gui', |
| 29 |
'recent_uas', |
| 30 |
] |
| 31 |
|
| 32 |
|
| 33 |
class Stage2(Command): |
| 34 |
description = 'Stage 2 of the publish process, builds the binaries' |
| 35 |
|
| 36 |
def run(self, opts): |
| 37 |
base = os.path.join(self.d(self.SRC)) |
| 38 |
for x in ('dist', 'build'): |
| 39 |
x = os.path.join(base, x) |
| 40 |
if os.path.exists(x): |
| 41 |
shutil.rmtree(x) |
| 42 |
os.mkdir(x) |
| 43 |
|
| 44 |
self.info('Starting builds for all platforms, this will take a while...') |
| 45 |
|
| 46 |
session = ['layout vertical'] |
| 47 |
platforms = 'linux64', 'linuxarm64', 'osx', 'win' |
| 48 |
for x in platforms: |
| 49 |
cmd = ( |
| 50 |
f'''{sys.executable} -c "import subprocess; subprocess.Popen(['{sys.executable}', './setup.py', '{x}']).wait() != 0 and''' |
| 51 |
f''' input('Build of {x} failed, press Enter to exit');"''' |
| 52 |
) |
| 53 |
session.append('title ' + x) |
| 54 |
session.append('launch ' + cmd) |
| 55 |
|
| 56 |
p = subprocess.Popen( |
| 57 |
['kitty', '-o', 'enabled_layouts=vertical,stack', '-o', 'scrollback_lines=20000', '-o', 'close_on_child_death=y', '--session=-'], |
| 58 |
stdin=subprocess.PIPE, |
| 59 |
) |
| 60 |
|
| 61 |
p.communicate('\n'.join(session).encode('utf-8')) |
| 62 |
p.wait() |
| 63 |
|
| 64 |
for installer in installer_names(include_source=False): |
| 65 |
installer = self.j(self.d(self.SRC), installer) |
| 66 |
if not os.path.exists(installer) or os.path.getsize(installer) < 10000: |
| 67 |
raise SystemExit(f'The installer {os.path.basename(installer)} does not exist') |
| 68 |
os.chmod(installer, 0o644) |
| 69 |
|
| 70 |
|
| 71 |
class Stage3(Command): |
| 72 |
description = 'Stage 3 of the publish process' |
| 73 |
sub_commands = ['upload_user_manual', 'upload_demo', 'sdist', 'tag_release'] |
| 74 |
|
| 75 |
|
| 76 |
class Stage4(Command): |
| 77 |
description = 'Stage 4 of the publish process' |
| 78 |
sub_commands = ['upload_installers'] |
| 79 |
|
| 80 |
|
| 81 |
class Stage5(Command): |
| 82 |
description = 'Stage 5 of the publish process' |
| 83 |
sub_commands = ['upload_to_server'] |
| 84 |
|
| 85 |
def run(self, opts): |
| 86 |
subprocess.check_call('rm -rf build/* dist/*', shell=True) |
| 87 |
|
| 88 |
|
| 89 |
def require_hsm_for_signing(): |
| 90 |
cp = subprocess.run(['pkcs11-tool', '-L'], stdout=subprocess.DEVNULL) |
| 91 |
if cp.returncode != 0: |
| 92 |
raise SystemExit('Attach the HSM for authenticode signing first') |
| 93 |
|
| 94 |
|
| 95 |
class Publish(Command): |
| 96 |
description = 'Publish a new calibre release' |
| 97 |
sub_commands = [ |
| 98 |
'stage1', |
| 99 |
'stage2', |
| 100 |
'stage3', |
| 101 |
'stage4', |
| 102 |
'stage5', |
| 103 |
] |
| 104 |
|
| 105 |
def pre_sub_commands(self, opts): |
| 106 |
require_git_master() |
| 107 |
require_clean_git() |
| 108 |
require_hsm_for_signing() |
| 109 |
version = tuple(map(int, __version__.split('.'))) # noqa: RUF048 |
| 110 |
if version[2] > 99: |
| 111 |
raise SystemExit(f'The version number {__version__} indicates a preview release, did you mean to run ./setup.py publish_preview?') |
| 112 |
if 'PUBLISH_BUILD_DONE' not in os.environ: |
| 113 |
subprocess.check_call([sys.executable, 'setup.py', 'check']) |
| 114 |
subprocess.check_call([sys.executable, 'setup.py', 'build']) |
| 115 |
if 'SKIP_CALIBRE_TESTS' not in os.environ: |
| 116 |
subprocess.check_call([sys.executable, 'setup.py', 'test']) |
| 117 |
subprocess.check_call([sys.executable, 'setup.py', 'pot']) |
| 118 |
subprocess.check_call([sys.executable, 'setup.py', 'translations']) |
| 119 |
os.environ['PUBLISH_BUILD_DONE'] = '1' |
| 120 |
os.execl(os.path.abspath('setup.py'), './setup.py', 'publish') |
| 121 |
|
| 122 |
|
| 123 |
class PublishBetas(Command): |
| 124 |
sub_commands = ['stage1', 'stage2', 'sdist'] |
| 125 |
|
| 126 |
def pre_sub_commands(self, opts): |
| 127 |
require_clean_git() |
| 128 |
require_hsm_for_signing() |
| 129 |
# require_git_master() |
| 130 |
|
| 131 |
def run(self, opts): |
| 132 |
dist = self.a(self.j(self.d(self.SRC), 'dist')) |
| 133 |
subprocess.check_call((f'rsync --partial -rh --info=progress2 --delete-after {dist}/ download.calibre-ebook.com:/srv/download/betas/').split()) |
| 134 |
|
| 135 |
|
| 136 |
class PublishPreview(Command): |
| 137 |
sub_commands = ['stage1', 'stage2', 'sdist'] |
| 138 |
|
| 139 |
def pre_sub_commands(self, opts): |
| 140 |
version = tuple(map(int, __version__.split('.'))) # noqa: RUF048 |
| 141 |
if version[2] < 100: |
| 142 |
raise SystemExit('Must set calibre version to have patch level greater than 100') |
| 143 |
require_clean_git() |
| 144 |
require_git_master() |
| 145 |
require_hsm_for_signing() |
| 146 |
|
| 147 |
def run(self, opts): |
| 148 |
dist = self.a(self.j(self.d(self.SRC), 'dist')) |
| 149 |
with open(os.path.join(dist, 'README.txt'), 'w') as f: |
| 150 |
print( |
| 151 |
'''\ |
| 152 |
These are preview releases of changes to calibre since the last normal release. |
| 153 |
Preview releases are typically released every Friday, they serve as a way |
| 154 |
for users to test upcoming features/fixes in the next calibre release. |
| 155 |
''', |
| 156 |
file=f, |
| 157 |
) |
| 158 |
subprocess.check_call((f'rsync -rh --info=progress2 --delete-after --delete {dist}/ download.calibre-ebook.com:/srv/download/preview/').split()) |
| 159 |
|
| 160 |
|
| 161 |
class Manual(Command): |
| 162 |
description = '''Build the User Manual ''' |
| 163 |
|
| 164 |
def add_options(self, parser): |
| 165 |
parser.add_option( |
| 166 |
'-l', |
| 167 |
'--language', |
| 168 |
action='append', |
| 169 |
default=[], |
| 170 |
help=('Build translated versions for only the specified languages (can be specified multiple times)'), |
| 171 |
) |
| 172 |
parser.add_option('--serve', action='store_true', default=False, help='Run a webserver on the built manual files') |
| 173 |
|
| 174 |
def run(self, opts): |
| 175 |
tdir = manual_build_dir() |
| 176 |
if os.path.exists(tdir): |
| 177 |
shutil.rmtree(tdir) |
| 178 |
os.mkdir(tdir) |
| 179 |
st = time.time() |
| 180 |
base = self.j(self.d(self.SRC), 'manual') |
| 181 |
for d in ('generated',): |
| 182 |
d = self.j(base, d) |
| 183 |
if os.path.exists(d): |
| 184 |
shutil.rmtree(d) |
| 185 |
os.makedirs(d) |
| 186 |
jobs = [] |
| 187 |
languages = opts.language or list(json.load(open(self.j(base, 'locale', 'completed.json'), 'rb'))) |
| 188 |
languages = set(languages) - {'en'} |
| 189 |
languages.discard('ta') # Tamil translations break Sphinx |
| 190 |
languages = ['en'] + list(languages) |
| 191 |
os.environ['ALL_USER_MANUAL_LANGUAGES'] = ' '.join(languages) |
| 192 |
for language in languages: |
| 193 |
jobs.append( |
| 194 |
create_job( |
| 195 |
[sys.executable, self.j(self.d(self.SRC), 'manual', 'build.py'), language, self.j(tdir, language)], |
| 196 |
f'\n\n**************** Building translations for: {language}', |
| 197 |
) |
| 198 |
) |
| 199 |
self.info(f'Building manual for {len(jobs)} languages') |
| 200 |
subprocess.check_call(jobs[0].cmd) |
| 201 |
if not parallel_build(jobs[1:], self.info): |
| 202 |
raise SystemExit(1) |
| 203 |
cwd = os.getcwd() |
| 204 |
with open('resources/localization/website-languages.txt') as wl: |
| 205 |
languages = frozenset(filter(None, (x.strip() for x in wl.read().split()))) |
| 206 |
try: |
| 207 |
os.chdir(self.j(tdir, 'en', 'html')) |
| 208 |
for x in os.listdir(tdir): |
| 209 |
if x != 'en': |
| 210 |
shutil.copytree(self.j(tdir, x, 'html'), x) |
| 211 |
self.replace_with_symlinks(x) |
| 212 |
else: |
| 213 |
os.symlink('.', 'en') |
| 214 |
for x in languages: |
| 215 |
if x and not os.path.exists(x): |
| 216 |
os.symlink('.', x) |
| 217 |
self.info(f'Built manual for {len(jobs)} languages in {int((time.time() - st) / 60.0)} minutes') |
| 218 |
finally: |
| 219 |
os.chdir(cwd) |
| 220 |
|
| 221 |
if opts.serve: |
| 222 |
self.serve_manual(self.j(tdir, 'en', 'html')) |
| 223 |
|
| 224 |
def serve_manual(self, root): |
| 225 |
os.chdir(root) |
| 226 |
from http.server import HTTPServer, SimpleHTTPRequestHandler |
| 227 |
|
| 228 |
HandlerClass = SimpleHTTPRequestHandler |
| 229 |
ServerClass = HTTPServer |
| 230 |
Protocol = 'HTTP/1.0' |
| 231 |
server_address = ('127.0.0.1', 8000) |
| 232 |
|
| 233 |
HandlerClass.protocol_version = Protocol |
| 234 |
httpd = ServerClass(server_address, HandlerClass) |
| 235 |
|
| 236 |
print('Serving User Manual on localhost:8000') |
| 237 |
from calibre.gui2 import open_url |
| 238 |
|
| 239 |
open_url('http://localhost:8000') |
| 240 |
httpd.serve_forever() |
| 241 |
|
| 242 |
def replace_with_symlinks(self, lang_dir): |
| 243 |
"Replace all identical files with symlinks to save disk space/upload bandwidth" |
| 244 |
from calibre import walk |
| 245 |
|
| 246 |
base = self.a(lang_dir) |
| 247 |
for f in walk(base): |
| 248 |
r = os.path.relpath(f, base) |
| 249 |
orig = self.j(self.d(base), r) |
| 250 |
try: |
| 251 |
sz = os.stat(orig).st_size |
| 252 |
except OSError: |
| 253 |
continue |
| 254 |
if sz == os.stat(f).st_size and filecmp._do_cmp(f, orig): |
| 255 |
os.remove(f) |
| 256 |
os.symlink(os.path.relpath(orig, self.d(f)), f) |
| 257 |
|
| 258 |
def clean(self): |
| 259 |
path = os.path.join(self.SRC, 'calibre', 'manual', '.build') |
| 260 |
if os.path.exists(path): |
| 261 |
shutil.rmtree(path) |
| 262 |
|
| 263 |
|
| 264 |
class ManPages(Command): |
| 265 |
description = '''Build the man pages ''' |
| 266 |
|
| 267 |
def add_options(self, parser): |
| 268 |
parser.add_option('--man-dir', help='Where to generate the man pages') |
| 269 |
parser.add_option('--compress-man-pages', default=False, action='store_true', help='Compress the generated man pages') |
| 270 |
|
| 271 |
def run(self, opts): |
| 272 |
self.build_man_pages(opts.man_dir or 'man-pages', opts.compress_man_pages) |
| 273 |
|
| 274 |
def build_man_pages(self, dest, compress=False): |
| 275 |
from calibre.utils.localization import available_translations |
| 276 |
|
| 277 |
dest = os.path.abspath(dest) |
| 278 |
if os.path.exists(dest): |
| 279 |
shutil.rmtree(dest) |
| 280 |
os.makedirs(dest) |
| 281 |
base = self.j(self.d(self.SRC), 'manual') |
| 282 |
languages = set(available_translations()) |
| 283 |
languages.discard('ta') # Tamil translatins are completely borked break sphinx |
| 284 |
languages.discard('id') # Indonesian man page fails to build |
| 285 |
languages = ['en'] + list(languages - {'en', 'en_GB'}) |
| 286 |
os.environ['ALL_USER_MANUAL_LANGUAGES'] = ' '.join(languages) |
| 287 |
try: |
| 288 |
os.makedirs(dest) |
| 289 |
except OSError: |
| 290 |
pass |
| 291 |
jobs = [] |
| 292 |
for l in languages: |
| 293 |
jobs.append(create_job([sys.executable, self.j(base, 'build.py'), '--man-pages', l, dest], f'\n\n**************** Building translations for: {l}')) |
| 294 |
self.info(f'\tCreating man pages in {dest} for {len(jobs)} languages...') |
| 295 |
subprocess.check_call(jobs[0].cmd) |
| 296 |
if not parallel_build(jobs[1:], self.info, verbose=False): |
| 297 |
raise SystemExit(1) |
| 298 |
cwd = os.getcwd() |
| 299 |
os.chdir(dest) |
| 300 |
try: |
| 301 |
for x in tuple(os.listdir('.')): |
| 302 |
if x in languages: |
| 303 |
if x == 'en': |
| 304 |
os.rename(x, 'man1') |
| 305 |
else: |
| 306 |
os.mkdir(self.j(x, 'man1')) |
| 307 |
for y in os.listdir(x): |
| 308 |
if y != 'man1': |
| 309 |
os.rename(self.j(x, y), self.j(x, 'man1', y)) |
| 310 |
else: |
| 311 |
shutil.rmtree(x) if os.path.isdir(x) else os.remove(x) |
| 312 |
if compress: |
| 313 |
jobs = [] |
| 314 |
for dirpath, dirnames, filenames in os.walk('.'): |
| 315 |
for f in filenames: |
| 316 |
if f.endswith('.1'): |
| 317 |
jobs.append(create_job(['gzip', '--best', self.j(dirpath, f)], '')) |
| 318 |
if not parallel_build(jobs, self.info, verbose=False): |
| 319 |
raise SystemExit(1) |
| 320 |
finally: |
| 321 |
os.chdir(cwd) |
| 322 |
|
| 323 |
|
| 324 |
class TagRelease(Command): |
| 325 |
description = 'Tag a new release in git' |
| 326 |
|
| 327 |
def run(self, opts): |
| 328 |
self.info('Tagging release') |
| 329 |
subprocess.check_call(f'git tag -s v{__version__} -m "version-{__version__}"'.split()) |
| 330 |
subprocess.check_call(f'git push origin v{__version__}'.split()) |