Debian Codegraph @

setup/__init__.py

raw
1 #!/usr/bin/env python
2
3
4 __license__ = 'GPL v3'
5 __copyright__ = '2009, Kovid Goyal <[email protected]>'
6 __docformat__ = 'restructuredtext en'
7
8 import errno
9 import hashlib
10 import os
11 import re
12 import shutil
13 import subprocess
14 import sys
15 import tempfile
16 import time
17 from contextlib import contextmanager, suppress
18 from functools import lru_cache
19
20 iswindows = re.search(r'win(32|64)', sys.platform)
21 ismacos = 'darwin' in sys.platform
22 isfreebsd = 'freebsd' in sys.platform
23 isnetbsd = 'netbsd' in sys.platform
24 isdragonflybsd = 'dragonfly' in sys.platform
25 isbsd = isnetbsd or isfreebsd or isdragonflybsd
26 ishaiku = 'haiku' in sys.platform
27 islinux = not ismacos and not iswindows and not isbsd and not ishaiku
28 is_ci = os.environ.get('CI', '').lower() == 'true'
29 sys.setup_dir = os.path.dirname(os.path.abspath(__file__))
30 SRC = os.path.abspath(os.path.join(os.path.dirname(sys.setup_dir), 'src'))
31 sys.path.insert(0, SRC)
32 sys.resources_location = os.path.join(os.path.dirname(SRC), 'resources')
33 sys.extensions_location = os.path.abspath(os.environ.get('CALIBRE_SETUP_EXTENSIONS_PATH', os.path.join(SRC, 'calibre', 'plugins')))
34 sys.running_from_setup = True
35
36 __version__ = __appname__ = modules = functions = basenames = scripts = None
37
38 _cache_dir_built = False
39
40
41 def newer(targets, sources):
42 if hasattr(targets, 'rjust'):
43 targets = [targets]
44 if hasattr(sources, 'rjust'):
45 sources = [sources]
46 for f in targets:
47 if not os.path.exists(f):
48 return True
49 ttimes = (os.stat(x).st_mtime for x in targets)
50 oldest_target = min(ttimes)
51 try:
52 stimes = (os.stat(x).st_mtime for x in sources)
53 newest_source = max(stimes)
54 except FileNotFoundError:
55 newest_source = oldest_target + 1
56 return newest_source > oldest_target
57
58
59 def dump_json(obj, path, indent=4):
60 import json
61
62 with open(path, 'wb') as f:
63 data = json.dumps(obj, indent=indent)
64 if not isinstance(data, bytes):
65 data = data.encode('utf-8')
66 f.write(data)
67
68
69 @lru_cache
70 def curl_supports_etags():
71 return '--etag-compare' in subprocess.check_output(['curl', '--help', 'all']).decode('utf-8')
72
73
74 def _download_securely(url, ignore_cache):
75 # We use curl here as on some OSes (OS X) when bootstrapping calibre,
76 # python will be unable to validate certificates until after cacerts is
77 # installed
78 if is_ci and iswindows:
79 # curl is failing for wikipedia urls on CI (used for browser_data)
80 from urllib.request import urlopen
81
82 with urlopen(url) as f:
83 return f.read()
84 if not curl_supports_etags():
85 return subprocess.check_output(['curl', '-fsSL', url])
86 url_hash = hashlib.sha1(url.encode('utf-8')).hexdigest()
87 cache_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), '.cache', 'download', url_hash)
88 os.makedirs(cache_dir, exist_ok=True)
89 staging = os.path.join(cache_dir, 'data.bin.staging')
90 etag = os.path.join(cache_dir, 'etag.txt')
91 if ignore_cache:
92 with suppress(FileNotFoundError):
93 os.remove(etag)
94 try:
95 subprocess.check_call(['curl', '-fsSL', '--etag-compare', etag, '--etag-save', etag, '-o', staging, url])
96 except Exception:
97 if os.path.exists(staging):
98 os.remove(staging)
99 if os.path.exists(etag):
100 os.remove(etag)
101 raise
102 final = os.path.join(cache_dir, 'data.bin')
103 if os.path.exists(staging):
104 os.replace(staging, final)
105 with open(final, 'rb') as f:
106 return f.read()
107
108
109 def download_securely(url, retry_count: int = 5 if is_ci else 3, sleep_time: float = 1, ignore_cache: bool = False) -> bytes:
110 for i in range(retry_count):
111 try:
112 return _download_securely(url, ignore_cache)
113 except Exception as err:
114 if i >= retry_count - 1:
115 raise
116 print(f'Download of {url} failed with error {err}, retrying...', file=sys.stderr)
117 time.sleep(sleep_time)
118
119
120 def build_cache_dir():
121 global _cache_dir_built
122 ans = os.path.join(os.path.dirname(SRC), '.build-cache')
123 if not _cache_dir_built:
124 _cache_dir_built = True
125 try:
126 os.mkdir(ans)
127 except OSError as err:
128 if err.errno != errno.EEXIST:
129 raise
130 return ans
131
132
133 def require_git_master(branch='master'):
134 if subprocess.check_output(['git', 'symbolic-ref', '--short', 'HEAD']).decode('utf-8').strip() != branch:
135 raise SystemExit(f'You must be in the {branch} git branch')
136
137
138 def require_clean_git():
139 c = subprocess.check_call
140 p = subprocess.Popen
141 c('git rev-parse --verify HEAD'.split(), stdout=subprocess.DEVNULL)
142 c('git update-index -q --ignore-submodules --refresh'.split())
143 if p('git diff-files --quiet --ignore-submodules'.split()).wait() != 0:
144 raise SystemExit('You have unstaged changes in your working tree')
145 if p('git diff-index --cached --quiet --ignore-submodules HEAD --'.split()).wait() != 0:
146 raise SystemExit('Your git index contains uncommitted changes')
147
148
149 def initialize_constants():
150 global __version__, __appname__, modules, functions, basenames, scripts
151
152 with open(os.path.join(SRC, 'calibre/constants.py'), 'rb') as f:
153 src = f.read().decode('utf-8')
154 nv = re.search(r'numeric_version\s+=\s+\((\d+), (\d+), (\d+)\)', src)
155 __version__ = '.'.join((nv.group(1), nv.group(2), nv.group(3)))
156 __appname__ = re.search(r'__appname__\s+=\s+(u{0,1})[\'"]([^\'"]+)[\'"]', src).group(2)
157 with open(os.path.join(SRC, 'calibre/linux.py'), 'rb') as sf:
158 epsrc = re.compile(r'entry_points = (\{.*?\})', re.DOTALL).search(sf.read().decode('utf-8')).group(1)
159 entry_points = eval(epsrc, {'__appname__': __appname__})
160
161 def e2b(ep):
162 return re.search(r'\s*(.*?)\s*=', ep).group(1).strip()
163
164 def e2s(ep, base='src'):
165 return (base + os.path.sep + re.search(r'.*=\s*(.*?):', ep).group(1).replace('.', '/') + '.py').strip()
166
167 def e2m(ep):
168 return re.search(r'.*=\s*(.*?)\s*:', ep).group(1).strip()
169
170 def e2f(ep):
171 return ep[ep.rindex(':') + 1 :].strip()
172
173 basenames, functions, modules, scripts = {}, {}, {}, {}
174 for x in ('console', 'gui'):
175 y = x + '_scripts'
176 basenames[x] = list(map(e2b, entry_points[y]))
177 functions[x] = list(map(e2f, entry_points[y]))
178 modules[x] = list(map(e2m, entry_points[y]))
179 scripts[x] = list(map(e2s, entry_points[y]))
180
181
182 initialize_constants()
183 preferred_encoding = 'utf-8'
184 prints = print
185 warnings = []
186
187
188 def get_warnings():
189 return list(warnings)
190
191
192 def edit_file(path):
193 return subprocess.Popen([os.environ.get('EDITOR', 'vim'), '-S', os.path.join(SRC, '../session.vim'), '-f', path]).wait() == 0
194
195
196 class Command:
197 SRC = SRC
198 PROJECT_ROOT = os.path.dirname(SRC)
199 RESOURCES = os.path.join(PROJECT_ROOT, 'resources')
200
201 description = ''
202 usage_help = ''
203 drop_privileges_for_subcommands = False
204 require_venv = False
205
206 sub_commands = []
207
208 def __init__(self):
209 self.d = os.path.dirname
210 self.j = os.path.join
211 self.a = os.path.abspath
212 self.b = os.path.basename
213 self.s = os.path.splitext
214 self.e = os.path.exists
215 self.orig_euid = os.geteuid() if hasattr(os, 'geteuid') else None
216 self.real_uid = os.environ.get('SUDO_UID', None)
217 self.real_gid = os.environ.get('SUDO_GID', None)
218 self.real_user = os.environ.get('SUDO_USER', None)
219
220 def pre_sub_commands(self, opts):
221 pass
222
223 def running(self, cmd):
224 from setup.commands import command_names
225
226 if is_ci:
227 self.info('::group::' + command_names[cmd])
228 self.info('\n*')
229 self.info('* Running', command_names[cmd])
230 self.info('*\n')
231
232 def run_cmd(self, cmd, opts):
233 from setup.commands import command_names
234
235 cmd.pre_sub_commands(opts)
236 if self.drop_privileges_for_subcommands and self.orig_euid is not None and os.getuid() == 0 and self.real_uid is not None:
237 if self.real_user is not None:
238 self.info('Dropping privileges to those of', self.real_user + ':', self.real_uid)
239
240 pid = os.fork()
241 if pid == 0:
242 if self.real_gid is not None:
243 os.setgid(int(self.real_gid))
244 if self.real_uid is not None:
245 os.setuid(int(self.real_uid))
246 for scmd in cmd.sub_commands:
247 self.run_cmd(scmd, opts)
248 raise SystemExit(0)
249 else:
250 rpid, st = os.waitpid(pid, 0)
251 if code := os.waitstatus_to_exitcode(st) != 0:
252 sys.exit(code)
253 else:
254 for scmd in cmd.sub_commands:
255 self.run_cmd(scmd, opts)
256
257 st = time.time()
258 self.running(cmd)
259 if cmd.require_venv:
260 self.ensure_venv()
261 cmd.run(opts)
262 self.info(f'* {command_names[cmd]} took {time.time() - st:.1f} seconds')
263 if is_ci:
264 self.info('::endgroup::')
265
266 def run_all(self, opts):
267 self.run_cmd(self, opts)
268
269 def add_command_options(self, command, parser):
270 from setup import commands
271
272 command.sub_commands = [getattr(commands, cmd) for cmd in command.sub_commands]
273 for cmd in command.sub_commands:
274 self.add_command_options(cmd, parser)
275
276 command.add_options(parser)
277
278 def add_all_options(self, parser):
279 self.add_command_options(self, parser)
280
281 def run(self, opts):
282 pass
283
284 def add_options(self, parser):
285 pass
286
287 def clean(self):
288 pass
289
290 @lru_cache(maxsize=2)
291 def ensure_venv(self) -> None:
292 venv_dir = self.j(self.PROJECT_ROOT, '.venv')
293 if not self.e(venv_dir):
294 subprocess.check_call(['uv', 'venv'], cwd=self.PROJECT_ROOT)
295 deps = subprocess.check_output(['uv', 'pip', 'compile', 'pyproject.toml', '--group', 'dev'], cwd=self.PROJECT_ROOT)
296 subprocess.run(['uv', 'pip', 'sync', '-'], check=True, input=deps, cwd=self.PROJECT_ROOT)
297
298 @classmethod
299 def newer(cls, targets, sources):
300 """
301 Return True if sources is newer that targets or if targets
302 does not exist.
303 """
304 return newer(targets, sources)
305
306 def info(self, *args, **kwargs):
307 prints(*args, **kwargs)
308 sys.stdout.flush()
309
310 def warn(self, *args, **kwargs):
311 print('\n' + '_' * 20, 'WARNING', '_' * 20)
312 prints(*args, **kwargs)
313 print('_' * 50)
314 warnings.append((args, kwargs))
315 sys.stdout.flush()
316
317 @contextmanager
318 def temp_dir(self, **kw):
319 ans = tempfile.mkdtemp(**kw)
320 try:
321 yield ans
322 finally:
323 shutil.rmtree(ans)
324
325
326 def installer_names(include_source=True):
327 base = f'dist/{__appname__}'
328 yield f'{base}-64bit-{__version__}.msi'
329 yield f'{base}-{__version__}.dmg'
330 yield f'{base}-portable-installer-{__version__}.exe'
331 for arch in ('x86_64', 'arm64'):
332 yield f'{base}-{__version__}-{arch}.txz'
333 if include_source:
334 yield f'{base}-{__version__}.tar.xz'
335
336
337 @lru_cache
338 def manual_build_dir():
339 # cant use tempfile.gettempdir() as calibre.startup overrides it
340 return os.path.join('/tmp', 'user-manual-build')