| 1 |
#!/usr/bin/env python |
| 2 |
|
| 3 |
|
| 4 |
__license__ = 'GPL v3' |
| 5 |
__copyright__ = '2009, Kovid Goyal <[email protected]>' |
| 6 |
__docformat__ = 'restructuredtext en' |
| 7 |
|
| 8 |
import errno |
| 9 |
import hashlib |
| 10 |
import os |
| 11 |
import re |
| 12 |
import shutil |
| 13 |
import subprocess |
| 14 |
import sys |
| 15 |
import tempfile |
| 16 |
import time |
| 17 |
from contextlib import contextmanager, suppress |
| 18 |
from functools import lru_cache |
| 19 |
|
| 20 |
iswindows = re.search(r'win(32|64)', sys.platform) |
| 21 |
ismacos = 'darwin' in sys.platform |
| 22 |
isfreebsd = 'freebsd' in sys.platform |
| 23 |
isnetbsd = 'netbsd' in sys.platform |
| 24 |
isdragonflybsd = 'dragonfly' in sys.platform |
| 25 |
isbsd = isnetbsd or isfreebsd or isdragonflybsd |
| 26 |
ishaiku = 'haiku' in sys.platform |
| 27 |
islinux = not ismacos and not iswindows and not isbsd and not ishaiku |
| 28 |
is_ci = os.environ.get('CI', '').lower() == 'true' |
| 29 |
sys.setup_dir = os.path.dirname(os.path.abspath(__file__)) |
| 30 |
SRC = os.path.abspath(os.path.join(os.path.dirname(sys.setup_dir), 'src')) |
| 31 |
sys.path.insert(0, SRC) |
| 32 |
sys.resources_location = os.path.join(os.path.dirname(SRC), 'resources') |
| 33 |
sys.extensions_location = os.path.abspath(os.environ.get('CALIBRE_SETUP_EXTENSIONS_PATH', os.path.join(SRC, 'calibre', 'plugins'))) |
| 34 |
sys.running_from_setup = True |
| 35 |
|
| 36 |
__version__ = __appname__ = modules = functions = basenames = scripts = None |
| 37 |
|
| 38 |
_cache_dir_built = False |
| 39 |
|
| 40 |
|
| 41 |
def newer(targets, sources): |
| 42 |
if hasattr(targets, 'rjust'): |
| 43 |
targets = [targets] |
| 44 |
if hasattr(sources, 'rjust'): |
| 45 |
sources = [sources] |
| 46 |
for f in targets: |
| 47 |
if not os.path.exists(f): |
| 48 |
return True |
| 49 |
ttimes = (os.stat(x).st_mtime for x in targets) |
| 50 |
oldest_target = min(ttimes) |
| 51 |
try: |
| 52 |
stimes = (os.stat(x).st_mtime for x in sources) |
| 53 |
newest_source = max(stimes) |
| 54 |
except FileNotFoundError: |
| 55 |
newest_source = oldest_target + 1 |
| 56 |
return newest_source > oldest_target |
| 57 |
|
| 58 |
|
| 59 |
def dump_json(obj, path, indent=4): |
| 60 |
import json |
| 61 |
|
| 62 |
with open(path, 'wb') as f: |
| 63 |
data = json.dumps(obj, indent=indent) |
| 64 |
if not isinstance(data, bytes): |
| 65 |
data = data.encode('utf-8') |
| 66 |
f.write(data) |
| 67 |
|
| 68 |
|
| 69 |
@lru_cache |
| 70 |
def curl_supports_etags(): |
| 71 |
return '--etag-compare' in subprocess.check_output(['curl', '--help', 'all']).decode('utf-8') |
| 72 |
|
| 73 |
|
| 74 |
def _download_securely(url, ignore_cache): |
| 75 |
# We use curl here as on some OSes (OS X) when bootstrapping calibre, |
| 76 |
# python will be unable to validate certificates until after cacerts is |
| 77 |
# installed |
| 78 |
if is_ci and iswindows: |
| 79 |
# curl is failing for wikipedia urls on CI (used for browser_data) |
| 80 |
from urllib.request import urlopen |
| 81 |
|
| 82 |
with urlopen(url) as f: |
| 83 |
return f.read() |
| 84 |
if not curl_supports_etags(): |
| 85 |
return subprocess.check_output(['curl', '-fsSL', url]) |
| 86 |
url_hash = hashlib.sha1(url.encode('utf-8')).hexdigest() |
| 87 |
cache_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), '.cache', 'download', url_hash) |
| 88 |
os.makedirs(cache_dir, exist_ok=True) |
| 89 |
staging = os.path.join(cache_dir, 'data.bin.staging') |
| 90 |
etag = os.path.join(cache_dir, 'etag.txt') |
| 91 |
if ignore_cache: |
| 92 |
with suppress(FileNotFoundError): |
| 93 |
os.remove(etag) |
| 94 |
try: |
| 95 |
subprocess.check_call(['curl', '-fsSL', '--etag-compare', etag, '--etag-save', etag, '-o', staging, url]) |
| 96 |
except Exception: |
| 97 |
if os.path.exists(staging): |
| 98 |
os.remove(staging) |
| 99 |
if os.path.exists(etag): |
| 100 |
os.remove(etag) |
| 101 |
raise |
| 102 |
final = os.path.join(cache_dir, 'data.bin') |
| 103 |
if os.path.exists(staging): |
| 104 |
os.replace(staging, final) |
| 105 |
with open(final, 'rb') as f: |
| 106 |
return f.read() |
| 107 |
|
| 108 |
|
| 109 |
def download_securely(url, retry_count: int = 5 if is_ci else 3, sleep_time: float = 1, ignore_cache: bool = False) -> bytes: |
| 110 |
for i in range(retry_count): |
| 111 |
try: |
| 112 |
return _download_securely(url, ignore_cache) |
| 113 |
except Exception as err: |
| 114 |
if i >= retry_count - 1: |
| 115 |
raise |
| 116 |
print(f'Download of {url} failed with error {err}, retrying...', file=sys.stderr) |
| 117 |
time.sleep(sleep_time) |
| 118 |
|
| 119 |
|
| 120 |
def build_cache_dir(): |
| 121 |
global _cache_dir_built |
| 122 |
ans = os.path.join(os.path.dirname(SRC), '.build-cache') |
| 123 |
if not _cache_dir_built: |
| 124 |
_cache_dir_built = True |
| 125 |
try: |
| 126 |
os.mkdir(ans) |
| 127 |
except OSError as err: |
| 128 |
if err.errno != errno.EEXIST: |
| 129 |
raise |
| 130 |
return ans |
| 131 |
|
| 132 |
|
| 133 |
def require_git_master(branch='master'): |
| 134 |
if subprocess.check_output(['git', 'symbolic-ref', '--short', 'HEAD']).decode('utf-8').strip() != branch: |
| 135 |
raise SystemExit(f'You must be in the {branch} git branch') |
| 136 |
|
| 137 |
|
| 138 |
def require_clean_git(): |
| 139 |
c = subprocess.check_call |
| 140 |
p = subprocess.Popen |
| 141 |
c('git rev-parse --verify HEAD'.split(), stdout=subprocess.DEVNULL) |
| 142 |
c('git update-index -q --ignore-submodules --refresh'.split()) |
| 143 |
if p('git diff-files --quiet --ignore-submodules'.split()).wait() != 0: |
| 144 |
raise SystemExit('You have unstaged changes in your working tree') |
| 145 |
if p('git diff-index --cached --quiet --ignore-submodules HEAD --'.split()).wait() != 0: |
| 146 |
raise SystemExit('Your git index contains uncommitted changes') |
| 147 |
|
| 148 |
|
| 149 |
def initialize_constants(): |
| 150 |
global __version__, __appname__, modules, functions, basenames, scripts |
| 151 |
|
| 152 |
with open(os.path.join(SRC, 'calibre/constants.py'), 'rb') as f: |
| 153 |
src = f.read().decode('utf-8') |
| 154 |
nv = re.search(r'numeric_version\s+=\s+\((\d+), (\d+), (\d+)\)', src) |
| 155 |
__version__ = '.'.join((nv.group(1), nv.group(2), nv.group(3))) |
| 156 |
__appname__ = re.search(r'__appname__\s+=\s+(u{0,1})[\'"]([^\'"]+)[\'"]', src).group(2) |
| 157 |
with open(os.path.join(SRC, 'calibre/linux.py'), 'rb') as sf: |
| 158 |
epsrc = re.compile(r'entry_points = (\{.*?\})', re.DOTALL).search(sf.read().decode('utf-8')).group(1) |
| 159 |
entry_points = eval(epsrc, {'__appname__': __appname__}) |
| 160 |
|
| 161 |
def e2b(ep): |
| 162 |
return re.search(r'\s*(.*?)\s*=', ep).group(1).strip() |
| 163 |
|
| 164 |
def e2s(ep, base='src'): |
| 165 |
return (base + os.path.sep + re.search(r'.*=\s*(.*?):', ep).group(1).replace('.', '/') + '.py').strip() |
| 166 |
|
| 167 |
def e2m(ep): |
| 168 |
return re.search(r'.*=\s*(.*?)\s*:', ep).group(1).strip() |
| 169 |
|
| 170 |
def e2f(ep): |
| 171 |
return ep[ep.rindex(':') + 1 :].strip() |
| 172 |
|
| 173 |
basenames, functions, modules, scripts = {}, {}, {}, {} |
| 174 |
for x in ('console', 'gui'): |
| 175 |
y = x + '_scripts' |
| 176 |
basenames[x] = list(map(e2b, entry_points[y])) |
| 177 |
functions[x] = list(map(e2f, entry_points[y])) |
| 178 |
modules[x] = list(map(e2m, entry_points[y])) |
| 179 |
scripts[x] = list(map(e2s, entry_points[y])) |
| 180 |
|
| 181 |
|
| 182 |
initialize_constants() |
| 183 |
preferred_encoding = 'utf-8' |
| 184 |
prints = print |
| 185 |
warnings = [] |
| 186 |
|
| 187 |
|
| 188 |
def get_warnings(): |
| 189 |
return list(warnings) |
| 190 |
|
| 191 |
|
| 192 |
def edit_file(path): |
| 193 |
return subprocess.Popen([os.environ.get('EDITOR', 'vim'), '-S', os.path.join(SRC, '../session.vim'), '-f', path]).wait() == 0 |
| 194 |
|
| 195 |
|
| 196 |
class Command: |
| 197 |
SRC = SRC |
| 198 |
PROJECT_ROOT = os.path.dirname(SRC) |
| 199 |
RESOURCES = os.path.join(PROJECT_ROOT, 'resources') |
| 200 |
|
| 201 |
description = '' |
| 202 |
usage_help = '' |
| 203 |
drop_privileges_for_subcommands = False |
| 204 |
require_venv = False |
| 205 |
|
| 206 |
sub_commands = [] |
| 207 |
|
| 208 |
def __init__(self): |
| 209 |
self.d = os.path.dirname |
| 210 |
self.j = os.path.join |
| 211 |
self.a = os.path.abspath |
| 212 |
self.b = os.path.basename |
| 213 |
self.s = os.path.splitext |
| 214 |
self.e = os.path.exists |
| 215 |
self.orig_euid = os.geteuid() if hasattr(os, 'geteuid') else None |
| 216 |
self.real_uid = os.environ.get('SUDO_UID', None) |
| 217 |
self.real_gid = os.environ.get('SUDO_GID', None) |
| 218 |
self.real_user = os.environ.get('SUDO_USER', None) |
| 219 |
|
| 220 |
def pre_sub_commands(self, opts): |
| 221 |
pass |
| 222 |
|
| 223 |
def running(self, cmd): |
| 224 |
from setup.commands import command_names |
| 225 |
|
| 226 |
if is_ci: |
| 227 |
self.info('::group::' + command_names[cmd]) |
| 228 |
self.info('\n*') |
| 229 |
self.info('* Running', command_names[cmd]) |
| 230 |
self.info('*\n') |
| 231 |
|
| 232 |
def run_cmd(self, cmd, opts): |
| 233 |
from setup.commands import command_names |
| 234 |
|
| 235 |
cmd.pre_sub_commands(opts) |
| 236 |
if self.drop_privileges_for_subcommands and self.orig_euid is not None and os.getuid() == 0 and self.real_uid is not None: |
| 237 |
if self.real_user is not None: |
| 238 |
self.info('Dropping privileges to those of', self.real_user + ':', self.real_uid) |
| 239 |
|
| 240 |
pid = os.fork() |
| 241 |
if pid == 0: |
| 242 |
if self.real_gid is not None: |
| 243 |
os.setgid(int(self.real_gid)) |
| 244 |
if self.real_uid is not None: |
| 245 |
os.setuid(int(self.real_uid)) |
| 246 |
for scmd in cmd.sub_commands: |
| 247 |
self.run_cmd(scmd, opts) |
| 248 |
raise SystemExit(0) |
| 249 |
else: |
| 250 |
rpid, st = os.waitpid(pid, 0) |
| 251 |
if code := os.waitstatus_to_exitcode(st) != 0: |
| 252 |
sys.exit(code) |
| 253 |
else: |
| 254 |
for scmd in cmd.sub_commands: |
| 255 |
self.run_cmd(scmd, opts) |
| 256 |
|
| 257 |
st = time.time() |
| 258 |
self.running(cmd) |
| 259 |
if cmd.require_venv: |
| 260 |
self.ensure_venv() |
| 261 |
cmd.run(opts) |
| 262 |
self.info(f'* {command_names[cmd]} took {time.time() - st:.1f} seconds') |
| 263 |
if is_ci: |
| 264 |
self.info('::endgroup::') |
| 265 |
|
| 266 |
def run_all(self, opts): |
| 267 |
self.run_cmd(self, opts) |
| 268 |
|
| 269 |
def add_command_options(self, command, parser): |
| 270 |
from setup import commands |
| 271 |
|
| 272 |
command.sub_commands = [getattr(commands, cmd) for cmd in command.sub_commands] |
| 273 |
for cmd in command.sub_commands: |
| 274 |
self.add_command_options(cmd, parser) |
| 275 |
|
| 276 |
command.add_options(parser) |
| 277 |
|
| 278 |
def add_all_options(self, parser): |
| 279 |
self.add_command_options(self, parser) |
| 280 |
|
| 281 |
def run(self, opts): |
| 282 |
pass |
| 283 |
|
| 284 |
def add_options(self, parser): |
| 285 |
pass |
| 286 |
|
| 287 |
def clean(self): |
| 288 |
pass |
| 289 |
|
| 290 |
@lru_cache(maxsize=2) |
| 291 |
def ensure_venv(self) -> None: |
| 292 |
venv_dir = self.j(self.PROJECT_ROOT, '.venv') |
| 293 |
if not self.e(venv_dir): |
| 294 |
subprocess.check_call(['uv', 'venv'], cwd=self.PROJECT_ROOT) |
| 295 |
deps = subprocess.check_output(['uv', 'pip', 'compile', 'pyproject.toml', '--group', 'dev'], cwd=self.PROJECT_ROOT) |
| 296 |
subprocess.run(['uv', 'pip', 'sync', '-'], check=True, input=deps, cwd=self.PROJECT_ROOT) |
| 297 |
|
| 298 |
@classmethod |
| 299 |
def newer(cls, targets, sources): |
| 300 |
""" |
| 301 |
Return True if sources is newer that targets or if targets |
| 302 |
does not exist. |
| 303 |
""" |
| 304 |
return newer(targets, sources) |
| 305 |
|
| 306 |
def info(self, *args, **kwargs): |
| 307 |
prints(*args, **kwargs) |
| 308 |
sys.stdout.flush() |
| 309 |
|
| 310 |
def warn(self, *args, **kwargs): |
| 311 |
print('\n' + '_' * 20, 'WARNING', '_' * 20) |
| 312 |
prints(*args, **kwargs) |
| 313 |
print('_' * 50) |
| 314 |
warnings.append((args, kwargs)) |
| 315 |
sys.stdout.flush() |
| 316 |
|
| 317 |
@contextmanager |
| 318 |
def temp_dir(self, **kw): |
| 319 |
ans = tempfile.mkdtemp(**kw) |
| 320 |
try: |
| 321 |
yield ans |
| 322 |
finally: |
| 323 |
shutil.rmtree(ans) |
| 324 |
|
| 325 |
|
| 326 |
def installer_names(include_source=True): |
| 327 |
base = f'dist/{__appname__}' |
| 328 |
yield f'{base}-64bit-{__version__}.msi' |
| 329 |
yield f'{base}-{__version__}.dmg' |
| 330 |
yield f'{base}-portable-installer-{__version__}.exe' |
| 331 |
for arch in ('x86_64', 'arm64'): |
| 332 |
yield f'{base}-{__version__}-{arch}.txz' |
| 333 |
if include_source: |
| 334 |
yield f'{base}-{__version__}.tar.xz' |
| 335 |
|
| 336 |
|
| 337 |
@lru_cache |
| 338 |
def manual_build_dir(): |
| 339 |
# cant use tempfile.gettempdir() as calibre.startup overrides it |
| 340 |
return os.path.join('/tmp', 'user-manual-build') |