| 1 |
#!/usr/bin/env python |
| 2 |
|
| 3 |
__license__ = 'GPL v3' |
| 4 |
__copyright__ = '2009, Kovid Goyal <[email protected]>' |
| 5 |
__docformat__ = 'restructuredtext en' |
| 6 |
|
| 7 |
import errno |
| 8 |
import glob |
| 9 |
import json |
| 10 |
import os |
| 11 |
import shlex |
| 12 |
import shutil |
| 13 |
import subprocess |
| 14 |
import sys |
| 15 |
import sysconfig |
| 16 |
import textwrap |
| 17 |
from functools import partial |
| 18 |
from typing import NamedTuple |
| 19 |
|
| 20 |
from setup import SRC, Command, isbsd, isfreebsd, ishaiku, islinux, ismacos, iswindows |
| 21 |
|
| 22 |
isunix = islinux or ismacos or isbsd or ishaiku |
| 23 |
|
| 24 |
py_lib = os.path.join(sys.prefix, 'libs', 'python{}{}.lib'.format(*sys.version_info[:2])) |
| 25 |
|
| 26 |
|
| 27 |
class CompileCommand(NamedTuple): |
| 28 |
cmd: list[str] |
| 29 |
src: str |
| 30 |
dest: str |
| 31 |
|
| 32 |
|
| 33 |
class LinkCommand(NamedTuple): |
| 34 |
cmd: list[str] |
| 35 |
objects: list[str] |
| 36 |
dest: str |
| 37 |
|
| 38 |
|
| 39 |
def walk(path='.'): |
| 40 |
for dirpath, dirnames, filenames in os.walk(path): |
| 41 |
for f in filenames: |
| 42 |
yield os.path.join(dirpath, f) |
| 43 |
|
| 44 |
|
| 45 |
def init_symbol_name(name): |
| 46 |
prefix = 'PyInit_' |
| 47 |
return prefix + name |
| 48 |
|
| 49 |
|
| 50 |
def absolutize(paths): |
| 51 |
return list({x if os.path.isabs(x) else os.path.join(SRC, x.replace('/', os.sep)) for x in paths}) |
| 52 |
|
| 53 |
|
| 54 |
class Extension: |
| 55 |
def __init__(self, name, sources, **kwargs): |
| 56 |
self.data = d = {} |
| 57 |
self.name = d['name'] = name |
| 58 |
self.sources = d['sources'] = absolutize(sources) |
| 59 |
self.needs_cxx = d['needs_cxx'] = bool([1 for x in self.sources if os.path.splitext(x)[1] in ('.cpp', '.c++', '.cxx')]) |
| 60 |
self.needs_py2 = d['needs_py2'] = kwargs.get('needs_py2', False) |
| 61 |
self.headers = d['headers'] = absolutize(kwargs.get('headers', [])) |
| 62 |
self.sip_files = d['sip_files'] = absolutize(kwargs.get('sip_files', [])) |
| 63 |
self.needs_exceptions = d['needs_exceptions'] = kwargs.get('needs_exceptions', False) |
| 64 |
self.qt_modules = d['qt_modules'] = kwargs.get('qt_modules', ['widgets']) |
| 65 |
self.inc_dirs = d['inc_dirs'] = absolutize(kwargs.get('inc_dirs', [])) |
| 66 |
self.lib_dirs = d['lib_dirs'] = absolutize(kwargs.get('lib_dirs', [])) |
| 67 |
self.extra_objs = d['extra_objs'] = absolutize(kwargs.get('extra_objs', [])) |
| 68 |
self.error = d['error'] = kwargs.get('error', None) |
| 69 |
self.libraries = d['libraries'] = kwargs.get('libraries', []) |
| 70 |
self.cflags = d['cflags'] = kwargs.get('cflags', []) |
| 71 |
self.uses_icu = 'icuuc' in self.libraries |
| 72 |
self.ldflags = d['ldflags'] = kwargs.get('ldflags', []) |
| 73 |
self.optional = d['options'] = kwargs.get('optional', False) |
| 74 |
self.needs_cxx_std = kwargs.get('needs_c++') |
| 75 |
self.needs_c_std = kwargs.get('needs_c') |
| 76 |
self.only_build_for = kwargs.get('only', '') |
| 77 |
|
| 78 |
|
| 79 |
def lazy_load(name): |
| 80 |
name = name.removeprefix('!') |
| 81 |
from setup import build_environment |
| 82 |
|
| 83 |
try: |
| 84 |
return getattr(build_environment, name) |
| 85 |
except AttributeError: |
| 86 |
raise ImportError(f'The setup.build_environment module has no symbol named: {name}') |
| 87 |
|
| 88 |
|
| 89 |
def expand_file_list(items, is_paths=True, cross_compile_for='native'): |
| 90 |
if not items: |
| 91 |
return [] |
| 92 |
ans = [] |
| 93 |
for item in items: |
| 94 |
if item.startswith('!'): |
| 95 |
if cross_compile_for == 'native' or not item.endswith('_dirs'): |
| 96 |
item = lazy_load(item) |
| 97 |
if hasattr(item, 'rjust'): |
| 98 |
item = [item] |
| 99 |
items = expand_file_list(item, is_paths=is_paths, cross_compile_for=cross_compile_for) |
| 100 |
else: |
| 101 |
pkg, category = item[1:].split('_')[:2] |
| 102 |
if category == 'inc': |
| 103 |
category = 'include' |
| 104 |
items = [f'bypy/b/windows/64/{pkg}/{category}'] |
| 105 |
items = expand_file_list(item, is_paths=is_paths, cross_compile_for=cross_compile_for) |
| 106 |
ans.extend(items) |
| 107 |
elif '*' in item: |
| 108 |
ans.extend(expand_file_list(sorted(glob.glob(os.path.join(SRC, item))), is_paths=is_paths, cross_compile_for=cross_compile_for)) |
| 109 |
else: |
| 110 |
item = [item] |
| 111 |
if is_paths: |
| 112 |
item = absolutize(item) |
| 113 |
ans.extend(item) |
| 114 |
return ans |
| 115 |
|
| 116 |
|
| 117 |
def is_ext_allowed(cross_compile_for: str, ext: Extension) -> bool: |
| 118 |
only = ext.only_build_for |
| 119 |
if only: |
| 120 |
if islinux and only == cross_compile_for: |
| 121 |
return True |
| 122 |
only = set(only.split()) |
| 123 |
q = set(filter(lambda x: globals()['is' + x], ['bsd', 'freebsd', 'haiku', 'linux', 'macos', 'windows'])) |
| 124 |
return len(q.intersection(only)) > 0 |
| 125 |
return True |
| 126 |
|
| 127 |
|
| 128 |
def parse_extension(ext, compiling_for='native'): |
| 129 |
ext = ext.copy() |
| 130 |
only = ext.pop('only', None) |
| 131 |
kw = {} |
| 132 |
name = ext.pop('name') |
| 133 |
get_key = 'linux_' |
| 134 |
if iswindows: |
| 135 |
get_key = 'windows_' |
| 136 |
elif ismacos: |
| 137 |
get_key = 'macos_' |
| 138 |
elif isbsd: |
| 139 |
get_key = 'bsd_' |
| 140 |
elif isfreebsd: |
| 141 |
get_key = 'freebsd_' |
| 142 |
elif ishaiku: |
| 143 |
get_key = 'haiku_' |
| 144 |
if compiling_for == 'windows': |
| 145 |
get_key = 'windows_' |
| 146 |
|
| 147 |
def get(k, default=''): |
| 148 |
ans = ext.pop(k, default) |
| 149 |
ans = ext.pop(get_key + k, ans) |
| 150 |
return ans |
| 151 |
|
| 152 |
for k in 'libraries qt_private ldflags cflags error'.split(): |
| 153 |
kw[k] = expand_file_list(get(k).split(), is_paths=False) |
| 154 |
defines = get('defines') |
| 155 |
if defines: |
| 156 |
if 'cflags' not in kw: |
| 157 |
kw['cflags'] = [] |
| 158 |
cflags = kw['cflags'] |
| 159 |
prefix = '/D' if get_key == 'windows_' else '-D' |
| 160 |
cflags.extend(prefix + x for x in defines.split()) |
| 161 |
for k in 'inc_dirs lib_dirs sources headers sip_files'.split(): |
| 162 |
v = get(k) |
| 163 |
if v: |
| 164 |
kw[k] = expand_file_list(v.split()) |
| 165 |
kw.update(ext) |
| 166 |
kw['only'] = only |
| 167 |
return Extension(name, **kw) |
| 168 |
|
| 169 |
|
| 170 |
def read_extensions(): |
| 171 |
if hasattr(read_extensions, 'extensions'): |
| 172 |
return read_extensions.extensions |
| 173 |
with open(os.path.dirname(os.path.abspath(__file__)) + '/extensions.json', 'rb') as f: |
| 174 |
ans = read_extensions.extensions = json.load(f) |
| 175 |
return ans |
| 176 |
|
| 177 |
|
| 178 |
def get_python_include_paths(): |
| 179 |
ans = [] |
| 180 |
for name in sysconfig.get_path_names(): |
| 181 |
if 'include' in name: |
| 182 |
ans.append(name) |
| 183 |
|
| 184 |
def gp(x): |
| 185 |
return sysconfig.get_path(x) |
| 186 |
|
| 187 |
return sorted(frozenset(filter(None, map(gp, sorted(ans))))) |
| 188 |
|
| 189 |
|
| 190 |
is_macos_universal_build = ismacos and 'universal2' in sysconfig.get_platform() |
| 191 |
|
| 192 |
|
| 193 |
def basic_windows_flags(debug=False): |
| 194 |
cflags = '/c /nologo /W3 /EHsc /O2 /utf-8'.split() |
| 195 |
cflags.append('/Zi' if debug else '/DNDEBUG') |
| 196 |
suffix = 'd' if debug else '' |
| 197 |
cflags.append('/MD' + suffix) |
| 198 |
ldflags = f'/DLL /nologo /INCREMENTAL:NO /NODEFAULTLIB:libcmt{suffix}.lib'.split() |
| 199 |
if debug: |
| 200 |
ldflags.append('/DEBUG') |
| 201 |
# cflags = '/c /nologo /Ox /MD /W3 /EHsc /Zi'.split() |
| 202 |
# ldflags = '/DLL /nologo /INCREMENTAL:NO /DEBUG'.split() |
| 203 |
cflags.append('/GS-') |
| 204 |
return cflags, ldflags |
| 205 |
|
| 206 |
|
| 207 |
class Environment(NamedTuple): |
| 208 |
cc: str |
| 209 |
cxx: str |
| 210 |
linker: str |
| 211 |
base_cflags: list[str] |
| 212 |
base_cxxflags: list[str] |
| 213 |
base_ldflags: list[str] |
| 214 |
cflags: list[str] |
| 215 |
ldflags: list[str] |
| 216 |
make: str |
| 217 |
internal_inc_prefix: str |
| 218 |
external_inc_prefix: str |
| 219 |
libdir_prefix: str |
| 220 |
lib_prefix: str |
| 221 |
lib_suffix: str |
| 222 |
obj_suffix: str |
| 223 |
cc_input_c_flag: str |
| 224 |
cc_input_cpp_flag: str |
| 225 |
cc_output_flag: str |
| 226 |
platform_name: str |
| 227 |
dest_ext: str |
| 228 |
std_prefix: str |
| 229 |
|
| 230 |
def inc_dirs_to_cflags(self, dirs) -> list[str]: |
| 231 |
return [self.external_inc_prefix + x for x in dirs] |
| 232 |
|
| 233 |
def lib_dirs_to_ldflags(self, dirs) -> list[str]: |
| 234 |
return [self.libdir_prefix + x for x in dirs if x] |
| 235 |
|
| 236 |
def libraries_to_ldflags(self, libs): |
| 237 |
def map_name(x): |
| 238 |
if '/' in x: |
| 239 |
return x |
| 240 |
return self.lib_prefix + x + self.lib_suffix |
| 241 |
|
| 242 |
return list(map(map_name, libs)) |
| 243 |
|
| 244 |
|
| 245 |
def init_env(debug=False, sanitize=False, compiling_for='native'): |
| 246 |
from setup.build_environment import NMAKE, win_cc, win_inc, win_ld, win_lib |
| 247 |
|
| 248 |
linker = None |
| 249 |
internal_inc_prefix = external_inc_prefix = '-I' |
| 250 |
libdir_prefix = '-L' |
| 251 |
lib_prefix = '-l' |
| 252 |
lib_suffix = '' |
| 253 |
std_prefix = '-std=' |
| 254 |
obj_suffix = '.o' |
| 255 |
cc_input_c_flag = cc_input_cpp_flag = '-c' |
| 256 |
cc_output_flag = '-o' |
| 257 |
platform_name = 'linux' |
| 258 |
dest_ext = '.so' |
| 259 |
if isunix: |
| 260 |
cc = os.environ.get('CC', 'gcc') |
| 261 |
cxx = os.environ.get('CXX', 'g++') |
| 262 |
debug = '-ggdb' if debug else '' |
| 263 |
cflags = os.environ.get('OVERRIDE_CFLAGS', f'-Wall -DNDEBUG {debug} -fno-strict-aliasing -pipe -O3') |
| 264 |
cflags = shlex.split(cflags) + ['-fPIC'] |
| 265 |
ldflags = os.environ.get('OVERRIDE_LDFLAGS', '-Wall') |
| 266 |
ldflags = shlex.split(ldflags) |
| 267 |
base_cflags = shlex.split(os.environ.get('CFLAGS', '')) |
| 268 |
base_cxxflags = shlex.split(os.environ.get('CXXFLAGS', '')) |
| 269 |
base_ldflags = shlex.split(os.environ.get('LDFLAGS', '')) |
| •270 |
|
| 271 |
base_cflags += shlex.split(os.environ.get('CPPFLAGS', '')) |
| 272 |
base_cxxflags += shlex.split(os.environ.get('CPPFLAGS', '')) |
| 273 |
|
| 274 |
cflags += base_cflags |
| 275 |
ldflags += base_ldflags |
| 276 |
cflags += ['-fvisibility=hidden'] |
| 277 |
if sanitize: |
| 278 |
cflags.append('-fsanitize=address') |
| 279 |
|
| 280 |
if islinux: |
| 281 |
cflags.append('-pthread') |
| 282 |
if sys.stdout.isatty(): |
| 283 |
base_cflags.append('-fdiagnostics-color=always') |
| 284 |
cflags.append('-fdiagnostics-color=always') |
| 285 |
ldflags.append('-shared') |
| 286 |
|
| 287 |
if isbsd: |
| 288 |
cflags.append('-pthread') |
| 289 |
ldflags.append('-shared') |
| 290 |
|
| 291 |
if ishaiku: |
| 292 |
cflags.append('-lpthread') |
| 293 |
ldflags.append('-shared') |
| 294 |
|
| 295 |
if islinux or isbsd or ishaiku: |
| 296 |
cflags.extend('-I' + x for x in get_python_include_paths()) |
| 297 |
ldlib = sysconfig.get_config_var('LIBDIR') |
| 298 |
if ldlib: |
| 299 |
ldflags += ['-L' + ldlib] |
| 300 |
ldlib = sysconfig.get_config_var('VERSION') |
| 301 |
if ldlib: |
| 302 |
ldflags += ['-lpython' + ldlib + sys.abiflags] |
| 303 |
ldflags += (sysconfig.get_config_var('LINKFORSHARED') or '').split() |
| 304 |
|
| 305 |
if ismacos: |
| 306 |
platform_name = 'macos' |
| 307 |
if is_macos_universal_build: |
| 308 |
cflags.extend(['-arch', 'x86_64', '-arch', 'arm64']) |
| 309 |
ldflags.extend(['-arch', 'x86_64', '-arch', 'arm64']) |
| 310 |
cflags.append('-D_OSX') |
| 311 |
ldflags.extend('-bundle -undefined dynamic_lookup'.split()) |
| 312 |
cflags.extend(['-fno-common', '-dynamic']) |
| 313 |
cflags.extend('-I' + x for x in get_python_include_paths()) |
| 314 |
# Qt headers use __yield() without including <arm_acle.h>, suppress the |
| 315 |
# resulting error from newer clang versions |
| 316 |
base_cxxflags.append('-Wno-implicit-function-declaration') |
| 317 |
|
| 318 |
if iswindows or compiling_for == 'windows': |
| 319 |
platform_name = 'windows' |
| 320 |
std_prefix = '/std:' |
| 321 |
cc = cxx = win_cc |
| 322 |
linker = win_ld |
| 323 |
cflags, ldflags = basic_windows_flags(debug) |
| 324 |
base_cflags, base_cxxflags, base_ldflags = [], [], [] |
| 325 |
if compiling_for == 'windows': |
| 326 |
cc = cxx = 'clang-cl' |
| 327 |
linker = 'lld-link' |
| 328 |
splat = '.build-cache/xwin/root' |
| 329 |
cflags.append('-fcolor-diagnostics') |
| 330 |
cflags.append('-fansi-escape-codes') |
| 331 |
for I in 'sdk/include/um sdk/include/cppwinrt sdk/include/shared sdk/include/ucrt crt/include sdk/include/winrt'.split(): |
| 332 |
cflags.append('/external:I') |
| 333 |
cflags.append(f'{splat}/{I}') |
| 334 |
for L in 'sdk/lib/um crt/lib sdk/lib/ucrt'.split(): |
| 335 |
ldflags.append(f'/libpath:{splat}/{L}') |
| 336 |
for L in os.environ.get('CROSS_LIBDIRS', '').split(os.pathsep): |
| 337 |
ldflags.append(f'/libpath:{L}') |
| 338 |
else: |
| 339 |
for p in win_inc: |
| 340 |
cflags.append('-I' + p) |
| 341 |
for p in win_lib: |
| 342 |
if p: |
| 343 |
ldflags.append('/LIBPATH:' + p) |
| 344 |
internal_inc_prefix = external_inc_prefix = '/I' |
| 345 |
libdir_prefix = '/libpath:' |
| 346 |
lib_prefix = '' |
| 347 |
lib_suffix = '.lib' |
| 348 |
cc_input_c_flag = '/Tc' |
| 349 |
cc_input_cpp_flag = '/Tp' |
| 350 |
cc_output_flag = '/Fo' |
| 351 |
obj_suffix = '.obj' |
| 352 |
dest_ext = '.pyd' |
| 353 |
if compiling_for == 'windows': |
| 354 |
external_inc_prefix = '/external:I' |
| 355 |
dest_ext = '.cross-windows-x64' + dest_ext |
| 356 |
obj_suffix = '.cross-windows-x64' + obj_suffix |
| 357 |
cflags.append('/external:I') |
| 358 |
cflags.append('bypy/b/windows/64/pkg/python/private/python/include') |
| 359 |
ldflags.append('/libpath:' + 'bypy/b/windows/64/pkg/python/private/python/libs') |
| 360 |
else: |
| 361 |
cflags.extend('-I' + x for x in get_python_include_paths()) |
| 362 |
ldflags.append('/LIBPATH:' + os.path.join(sysconfig.get_config_var('prefix'), 'libs')) |
| 363 |
return Environment( |
| 364 |
platform_name=platform_name, |
| 365 |
dest_ext=dest_ext, |
| 366 |
std_prefix=std_prefix, |
| 367 |
base_cflags=base_cflags, |
| 368 |
base_cxxflags=base_cxxflags, |
| 369 |
base_ldflags=base_ldflags, |
| 370 |
cc=cc, |
| 371 |
cxx=cxx, |
| 372 |
cflags=cflags, |
| 373 |
ldflags=ldflags, |
| 374 |
linker=linker, |
| 375 |
make=NMAKE if iswindows else 'make', |
| 376 |
lib_prefix=lib_prefix, |
| 377 |
obj_suffix=obj_suffix, |
| 378 |
cc_input_c_flag=cc_input_c_flag, |
| 379 |
cc_input_cpp_flag=cc_input_cpp_flag, |
| 380 |
cc_output_flag=cc_output_flag, |
| 381 |
internal_inc_prefix=internal_inc_prefix, |
| 382 |
external_inc_prefix=external_inc_prefix, |
| 383 |
libdir_prefix=libdir_prefix, |
| 384 |
lib_suffix=lib_suffix, |
| 385 |
) |
| 386 |
|
| 387 |
|
| 388 |
class Build(Command): |
| 389 |
short_description = 'Build calibre C/C++ extension modules' |
| 390 |
DEFAULT_OUTPUTDIR = os.path.abspath(os.path.join(SRC, 'calibre', 'plugins')) |
| 391 |
DEFAULT_BUILDDIR = os.path.abspath(os.path.join(os.path.dirname(SRC), 'build')) |
| 392 |
|
| 393 |
description = textwrap.dedent('''\ |
| 394 |
calibre depends on several python extensions written in C/C++. |
| 395 |
This command will compile them. You can influence the compile |
| 396 |
process by several environment variables, listed below: |
| 397 |
|
| 398 |
CC - C Compiler defaults to gcc |
| 399 |
CXX - C++ Compiler, defaults to g++ |
| 400 |
CFLAGS - Extra compiler flags |
| 401 |
LDFLAGS - Extra linker flags |
| 402 |
|
| 403 |
POPPLER_INC_DIR - poppler header files |
| 404 |
POPPLER_LIB_DIR - poppler-qt4 library |
| 405 |
|
| 406 |
PODOFO_INC_DIR - podofo header files |
| 407 |
PODOFO_LIB_DIR - podofo library files |
| 408 |
|
| 409 |
QMAKE - Path to qmake |
| 410 |
''') |
| 411 |
|
| 412 |
def add_options(self, parser): |
| 413 |
choices = [e['name'] for e in read_extensions()] + ['all', 'headless'] |
| 414 |
parser.add_option( |
| 415 |
'-1', '--only', choices=choices, default='all', help=('Build only the named extension. Available: ' + ', '.join(choices) + '. Default:%default') |
| 416 |
) |
| 417 |
parser.add_option('--no-compile', default=False, action='store_true', help='Skip compiling all C/C++ extensions.') |
| 418 |
parser.add_option('--build-dir', default=None, help='Path to directory in which to place object files during the build process, defaults to "build"') |
| 419 |
parser.add_option('--output-dir', default=None, help='Path to directory in which to place the built extensions. Defaults to src/calibre/plugins') |
| 420 |
parser.add_option('--debug', default=False, action='store_true', help='Build in debug mode') |
| 421 |
parser.add_option( |
| 422 |
'--sanitize', default=False, action='store_true', help='Build with sanitization support. Run with LD_PRELOAD=$(gcc -print-file-name=libasan.so)' |
| 423 |
) |
| 424 |
parser.add_option( |
| 425 |
'--cross-compile-extensions', |
| 426 |
choices='windows disabled'.split(), |
| 427 |
default='disabled', |
| 428 |
help=( |
| 429 |
'Cross compile extensions for other platforms. Useful for development.' |
| 430 |
' Currently supports of windows extensions on Linux. Remember to run ./setup.py xwin first to install the Windows SDK locally. ' |
| 431 |
), |
| 432 |
) |
| 433 |
|
| 434 |
def dump_db(self, name, db): |
| 435 |
os.makedirs('build', exist_ok=True) |
| 436 |
existing = [] |
| 437 |
try: |
| 438 |
with open(f'build/{name}_commands.json', 'rb') as f: |
| 439 |
existing = json.load(f) |
| 440 |
except FileNotFoundError: |
| 441 |
pass |
| 442 |
combined = {x['output']: x for x in existing} |
| 443 |
for x in db: |
| 444 |
combined[x['output']] = x |
| 445 |
try: |
| 446 |
with open(f'build/{name}_commands.json', 'w') as f: |
| 447 |
json.dump(tuple(combined.values()), f, indent=2) |
| 448 |
except OSError as err: |
| 449 |
if err.errno != errno.EROFS: |
| 450 |
raise |
| 451 |
|
| 452 |
def run(self, opts): |
| 453 |
from setup.parallel_build import create_job, parallel_build |
| 454 |
|
| 455 |
if opts.no_compile: |
| 456 |
self.info('--no-compile specified, skipping compilation') |
| 457 |
return |
| 458 |
self.compiling_for = 'native' |
| 459 |
if islinux and opts.cross_compile_extensions == 'windows': |
| 460 |
self.compiling_for = 'windows' |
| 461 |
if not os.path.exists('.build-cache/xwin/root'): |
| 462 |
subprocess.check_call([sys.executable, 'setup.py', 'xwin']) |
| 463 |
self.env = init_env(debug=opts.debug) |
| 464 |
self.windows_cross_env = init_env(debug=opts.debug, compiling_for='windows') |
| 465 |
all_extensions = tuple(map(partial(parse_extension, compiling_for=self.compiling_for), read_extensions())) |
| 466 |
self.build_dir = os.path.abspath(opts.build_dir or self.DEFAULT_BUILDDIR) |
| 467 |
self.output_dir = os.path.abspath(opts.output_dir or self.DEFAULT_OUTPUTDIR) |
| 468 |
self.obj_dir = os.path.join(self.build_dir, 'objects') |
| 469 |
for x in (self.output_dir, self.obj_dir): |
| 470 |
os.makedirs(x, exist_ok=True) |
| 471 |
pyqt_extensions, extensions = [], [] |
| 472 |
for ext in all_extensions: |
| 473 |
if opts.only not in {'all', ext.name}: |
| 474 |
continue |
| 475 |
if not is_ext_allowed(self.compiling_for, ext): |
| 476 |
continue |
| 477 |
if ext.error: |
| 478 |
if ext.optional: |
| 479 |
self.warn(ext.error) |
| 480 |
continue |
| 481 |
else: |
| 482 |
raise Exception(ext.error) |
| 483 |
(pyqt_extensions if ext.sip_files else extensions).append(ext) |
| 484 |
|
| 485 |
jobs = [] |
| 486 |
objects_map = {} |
| 487 |
self.info(f'Building {len(extensions) + len(pyqt_extensions)} extensions') |
| 488 |
ccdb = [] |
| 489 |
for ext in all_extensions: |
| 490 |
if ext in pyqt_extensions: |
| 491 |
continue |
| 492 |
cmds, objects = self.get_compile_commands(ext, ccdb) |
| 493 |
objects_map[id(ext)] = objects |
| 494 |
if ext in extensions: |
| 495 |
for cmd in cmds: |
| 496 |
jobs.append(create_job(cmd.cmd)) |
| 497 |
self.dump_db('compile', ccdb) |
| 498 |
if jobs: |
| 499 |
self.info(f'Compiling {len(jobs)} files...') |
| 500 |
if not parallel_build(jobs, self.info): |
| 501 |
raise SystemExit(1) |
| 502 |
jobs, link_commands, lddb = [], [], [] |
| 503 |
for ext in all_extensions: |
| 504 |
if ext in pyqt_extensions: |
| 505 |
continue |
| 506 |
objects = objects_map[id(ext)] |
| 507 |
cmd = self.get_link_command(ext, objects, lddb) |
| 508 |
if ext in extensions and cmd is not None: |
| 509 |
link_commands.append(cmd) |
| 510 |
jobs.append(create_job(cmd.cmd)) |
| 511 |
self.dump_db('link', lddb) |
| 512 |
if jobs: |
| 513 |
self.info(f'Linking {len(jobs)} files...') |
| 514 |
if not parallel_build(jobs, self.info): |
| 515 |
raise SystemExit(1) |
| 516 |
for cmd in link_commands: |
| 517 |
self.post_link_cleanup(cmd) |
| 518 |
|
| 519 |
jobs = [] |
| 520 |
sbf_map = {} |
| 521 |
for ext in pyqt_extensions: |
| 522 |
cmd, sbf, cwd = self.get_sip_commands(ext) |
| 523 |
sbf_map[id(ext)] = sbf |
| 524 |
if cmd is not None: |
| 525 |
jobs.append(create_job(cmd, cwd=cwd)) |
| 526 |
if jobs: |
| 527 |
self.info(f'SIPing {len(jobs)} files...') |
| 528 |
if not parallel_build(jobs, self.info): |
| 529 |
raise SystemExit(1) |
| 530 |
for ext in pyqt_extensions: |
| 531 |
sbf = sbf_map[id(ext)] |
| 532 |
if not os.path.exists(sbf): |
| 533 |
self.build_pyqt_extension(ext, sbf) |
| 534 |
|
| 535 |
if opts.only in {'all', 'headless'}: |
| 536 |
self.build_headless() |
| 537 |
|
| 538 |
def dest(self, ext, env): |
| 539 |
return os.path.join(self.output_dir, getattr(ext, 'name', ext)) + env.dest_ext |
| 540 |
|
| 541 |
def env_for_compilation_db(self, ext): |
| 542 |
if is_ext_allowed('native', ext): |
| 543 |
return self.env |
| 544 |
if ext.only_build_for == 'windows': |
| 545 |
return self.windows_cross_env |
| 546 |
|
| 547 |
def get_compile_commands(self, ext, db): |
| 548 |
obj_dir = self.j(self.obj_dir, ext.name) |
| 549 |
|
| 550 |
def get(src: str, env: Environment, for_tooling: bool = False) -> CompileCommand: |
| 551 |
compiler = env.cxx if ext.needs_cxx else env.cc |
| 552 |
obj = self.j(obj_dir, os.path.splitext(self.b(src))[0] + env.obj_suffix) |
| 553 |
inf = env.cc_input_cpp_flag if src.endswith(('.cpp', '.cxx')) else env.cc_input_c_flag |
| 554 |
sinc = [inf, src] |
| 555 |
if env.cc_output_flag.startswith('/'): |
| 556 |
if for_tooling: # clangd gets confused by cl.exe style source and output flags |
| 557 |
oinc = ['-o', obj] |
| 558 |
else: |
| 559 |
oinc = [env.cc_output_flag + obj] |
| 560 |
sinc = [inf + src] |
| 561 |
else: |
| 562 |
oinc = [env.cc_output_flag, obj] |
| 563 |
einc = env.inc_dirs_to_cflags(ext.inc_dirs) |
| 564 |
if env.cc_output_flag.startswith('/'): |
| 565 |
cflags = ['/DCALIBRE_MODINIT_FUNC=PyMODINIT_FUNC'] |
| 566 |
else: |
| 567 |
return_type = 'PyObject*' |
| 568 |
extern_decl = 'extern "C"' if ext.needs_cxx else '' |
| 569 |
cflags = [f'-DCALIBRE_MODINIT_FUNC={extern_decl} __attribute__ ((visibility ("default"))) {return_type}'] |
| 570 |
if ext.needs_cxx and ext.needs_cxx_std: |
| 571 |
if env.cc_output_flag.startswith('/') and ext.needs_cxx == '11': |
| 572 |
ext.needs_cxx = '14' |
| 573 |
cflags.append(env.std_prefix + 'c++' + ext.needs_cxx_std) |
| 574 |
|
| 575 |
if ext.needs_c_std and not env.std_prefix.startswith('/'): |
| 576 |
cflags.append(env.std_prefix + 'c' + ext.needs_c_std) |
| 577 |
|
| 578 |
cmd = [compiler] + env.cflags + cflags + ext.cflags + einc + sinc + oinc |
| 579 |
return CompileCommand(cmd, src, obj) |
| 580 |
|
| 581 |
objects = [] |
| 582 |
ans = [] |
| 583 |
os.makedirs(obj_dir, exist_ok=True) |
| 584 |
|
| 585 |
for src in ext.sources: |
| 586 |
cc = get(src, self.windows_cross_env if self.compiling_for == 'windows' else self.env) |
| 587 |
objects.append(cc.dest) |
| 588 |
if self.newer(cc.dest, [src] + ext.headers): |
| 589 |
ans.append(cc) |
| 590 |
env = self.env_for_compilation_db(ext) |
| 591 |
if env is not None: |
| 592 |
cc = get(src, env, for_tooling=True) |
| 593 |
db.append({ |
| 594 |
'arguments': cc.cmd, |
| 595 |
'directory': os.getcwd(), |
| 596 |
'file': os.path.relpath(src, os.getcwd()), |
| 597 |
'output': os.path.relpath(cc.dest, os.getcwd()), |
| 598 |
}) |
| 599 |
return ans, objects |
| 600 |
|
| 601 |
def get_link_command(self, ext, objects, lddb): |
| 602 |
|
| 603 |
def get(env: Environment) -> LinkCommand: |
| 604 |
dest = self.dest(ext, env) |
| 605 |
compiler = env.cxx if ext.needs_cxx else env.cc |
| 606 |
linker = env.linker or compiler |
| 607 |
cmd = [linker] |
| 608 |
elib = env.lib_dirs_to_ldflags(ext.lib_dirs) |
| 609 |
xlib = env.libraries_to_ldflags(ext.libraries) |
| 610 |
all_objects = sorted(objects + ext.extra_objs) |
| 611 |
if iswindows or env is self.windows_cross_env: |
| 612 |
pre_ld_flags = [] |
| 613 |
if ext.uses_icu: |
| 614 |
# windows has its own ICU libs that don't work |
| 615 |
pre_ld_flags = elib |
| 616 |
cmd += pre_ld_flags + env.ldflags + ext.ldflags + elib + xlib + ['/EXPORT:' + init_symbol_name(ext.name)] + all_objects + ['/OUT:' + dest] |
| 617 |
else: |
| 618 |
cmd += all_objects + ['-o', dest] + env.ldflags + ext.ldflags + elib + xlib |
| 619 |
return LinkCommand(cmd, objects, dest) |
| 620 |
|
| 621 |
env = self.env_for_compilation_db(ext) |
| 622 |
if env is not None: |
| 623 |
ld = get(env) |
| 624 |
lddb.append({'arguments': ld.cmd, 'directory': os.getcwd(), 'output': os.path.relpath(ld.dest, os.getcwd())}) |
| 625 |
|
| 626 |
env = self.windows_cross_env if self.compiling_for == 'windows' else self.env |
| 627 |
lc = get(env) |
| 628 |
if self.newer(lc.dest, objects + ext.extra_objs): |
| 629 |
return lc |
| 630 |
|
| 631 |
def post_link_cleanup(self, link_command): |
| 632 |
if iswindows: |
| 633 |
dest = link_command.dest |
| 634 |
for x in ('.exp', '.lib'): |
| 635 |
x = os.path.splitext(dest)[0] + x |
| 636 |
if os.path.exists(x): |
| 637 |
os.remove(x) |
| 638 |
|
| 639 |
def check_call(self, *args, **kwargs): |
| 640 |
"""print cmdline if an error occurred |
| 641 |
|
| 642 |
If something is missing (cmake e.g.) you get a non-informative error |
| 643 |
self.check_call(qmc + [ext.name+'.pro']) |
| 644 |
so you would have to look at the source to see the actual command. |
| 645 |
""" |
| 646 |
try: |
| 647 |
subprocess.check_call(*args, **kwargs) |
| 648 |
except Exception: |
| 649 |
cmdline = ' '.join([f'"{arg}"' if ' ' in arg else arg for arg in args[0]]) |
| 650 |
print(f'Error while executing: {cmdline}\n') |
| 651 |
raise |
| 652 |
|
| 653 |
def build_headless(self): |
| 654 |
from setup.parallel_build import cpu_count |
| 655 |
|
| 656 |
if iswindows or ishaiku: |
| 657 |
return # Don't have headless operation on these platforms |
| 658 |
from setup.build_environment import CMAKE, sw |
| 659 |
|
| 660 |
self.info('\n####### Building headless QPA plugin', '#' * 7) |
| 661 |
a = absolutize |
| 662 |
headers = a([ |
| 663 |
'calibre/headless/headless_backingstore.h', |
| 664 |
'calibre/headless/headless_integration.h', |
| 665 |
]) |
| 666 |
sources = a([ |
| 667 |
'calibre/headless/main.cpp', |
| 668 |
'calibre/headless/headless_backingstore.cpp', |
| 669 |
'calibre/headless/headless_integration.cpp', |
| 670 |
]) |
| 671 |
others = a(['calibre/headless/headless.json']) |
| 672 |
target = self.dest('headless', self.env) |
| 673 |
if not ismacos: |
| 674 |
target = target.replace('headless', 'libheadless') |
| 675 |
if not self.newer(target, headers + sources + others): |
| 676 |
return |
| 677 |
|
| 678 |
bdir = self.j(self.build_dir, 'headless') |
| 679 |
if os.path.exists(bdir): |
| 680 |
shutil.rmtree(bdir) |
| 681 |
sdir = os.path.join(bdir, 'src') |
| 682 |
shutil.copytree(os.path.dirname(sources[0]), sdir) |
| 683 |
with open(os.path.join(sdir, 'CMakeLists.txt'), 'r+') as f: |
| 684 |
raw = f.read() |
| 685 |
qt = lazy_load('qt') |
| 686 |
if qt['version'] >= (6, 10): |
| 687 |
fp = 'find_package(Qt6 REQUIRED COMPONENTS Gui GuiPrivate Core CorePrivate)' |
| 688 |
ll = 'target_link_libraries(headless PRIVATE Qt6::Gui Qt6::GuiPrivate Qt6::Core Qt6::CorePrivate)' |
| 689 |
else: |
| 690 |
fp = 'find_package(Qt6Gui REQUIRED)' |
| 691 |
ll = 'target_link_libraries(headless PRIVATE Qt::Gui Qt::GuiPrivate Qt::Core Qt::CorePrivate)' |
| 692 |
raw = raw.replace('__FIND_GUI__', fp) |
| 693 |
raw = raw.replace('__LINK_TARGETS__', ll) |
| 694 |
f.seek(0), f.truncate() |
| 695 |
f.write(raw) |
| 696 |
bdir = os.path.join(bdir, 'build') |
| 697 |
cmd = [CMAKE] |
| 698 |
if is_macos_universal_build: |
| 699 |
cmd += ['-DCMAKE_OSX_ARCHITECTURES=x86_64;arm64'] |
| 700 |
if sw and os.path.exists(os.path.join(sw, 'qt')): |
| 701 |
cmd += ['-DCMAKE_SYSTEM_PREFIX_PATH=' + os.path.join(sw, 'qt').replace(os.sep, '/')] |
| 702 |
os.makedirs(bdir) |
| 703 |
cwd = os.getcwd() |
| 704 |
os.chdir(bdir) |
| 705 |
try: |
| 706 |
self.check_call(cmd + ['-S', sdir]) |
| 707 |
self.check_call([self.env.make] + [f'-j{cpu_count or 1}']) |
| 708 |
finally: |
| 709 |
os.chdir(cwd) |
| 710 |
os.rename(self.j(bdir, 'libheadless.so'), target) |
| 711 |
|
| 712 |
def create_sip_build_skeleton(self, src_dir, ext): |
| 713 |
from setup.build_environment import pyqt_sip_abi_version |
| 714 |
|
| 715 |
abi_version = '' |
| 716 |
if pyqt_sip_abi_version(): |
| 717 |
abi_version = f'abi-version = "{pyqt_sip_abi_version()}"' |
| 718 |
sipf = ext.sip_files[0] |
| 719 |
needs_exceptions = 'true' if ext.needs_exceptions else 'false' |
| 720 |
with open(os.path.join(src_dir, 'pyproject.toml'), 'w') as f: |
| 721 |
f.write(f''' |
| 722 |
[build-system] |
| 723 |
requires = ["sip >=5.3", "PyQt-builder >=1"] |
| 724 |
build-backend = "sipbuild.api" |
| 725 |
|
| 726 |
[tool.sip] |
| 727 |
project-factory = "pyqtbuild:PyQtProject" |
| 728 |
|
| 729 |
[tool.sip.project] |
| 730 |
sip-files-dir = "." |
| 731 |
{abi_version} |
| •732 |
sip-include-dirs = ["/usr/lib/python3/dist-packages/PyQt6/bindings"] |
| 733 |
|
| 734 |
[project] |
| 735 |
name = "{ext.name}" |
| 736 |
|
| 737 |
[tool.sip.builder] |
| 738 |
qmake-settings = [ |
| 739 |
"""QMAKE_CC = {self.env.cc}""", |
| 740 |
"""QMAKE_CXX = {self.env.cxx}""", |
| 741 |
"""QMAKE_LINK = {self.env.linker or self.env.cxx}""", |
| 742 |
"""QMAKE_CFLAGS += {shlex.join(self.env.base_cflags)}""", |
| 743 |
"""QMAKE_CXXFLAGS += {shlex.join(self.env.base_cxxflags)}""", |
| 744 |
"""QMAKE_LFLAGS += {shlex.join(self.env.base_ldflags)}""", |
| 745 |
] |
| 746 |
|
| 747 |
[tool.sip.bindings.{ext.name}] |
| 748 |
headers = {sorted(ext.headers)} |
| 749 |
sources = {sorted(ext.sources)} |
| 750 |
exceptions = {needs_exceptions} |
| 751 |
include-dirs = {ext.inc_dirs} |
| 752 |
qmake-QT = {ext.qt_modules} |
| 753 |
sip-file = {os.path.basename(sipf)!r} |
| 754 |
''') |
| 755 |
shutil.copy2(sipf, src_dir) |
| 756 |
|
| 757 |
def get_sip_commands(self, ext): |
| 758 |
from setup.build_environment import QMAKE |
| 759 |
|
| 760 |
pyqt_dir = self.j(self.build_dir, 'pyqt') |
| 761 |
src_dir = self.j(pyqt_dir, ext.name) |
| 762 |
# TODO: Handle building extensions with multiple SIP files. |
| 763 |
sipf = ext.sip_files[0] |
| 764 |
sbf = self.j(src_dir, self.b(sipf) + '.sbf') |
| 765 |
cmd = None |
| 766 |
cwd = None |
| 767 |
if self.newer(sbf, [sipf] + ext.headers + ext.sources): |
| 768 |
shutil.rmtree(src_dir, ignore_errors=True) |
| 769 |
os.makedirs(src_dir) |
| 770 |
self.create_sip_build_skeleton(src_dir, ext) |
| 771 |
cwd = src_dir |
| 772 |
cmd = [sys.executable, '-m', 'sipbuild.tools.build', '--verbose', '--no-make', '--qmake', QMAKE] |
| 773 |
return cmd, sbf, cwd |
| 774 |
|
| 775 |
def build_pyqt_extension(self, ext, sbf): |
| 776 |
self.info(f'\n####### Building {ext.name} extension', '#' * 7) |
| 777 |
src_dir = os.path.dirname(sbf) |
| 778 |
cwd = os.getcwd() |
| 779 |
try: |
| 780 |
os.chdir(os.path.join(src_dir, 'build')) |
| •781 |
for q in walk('.'): |
| 782 |
if os.path.basename(q) in ('Makefile',): |
| 783 |
with open(q, 'r+') as f: |
| 784 |
raw = f.read() |
| 785 |
raw = raw.replace('CFLAGS =', 'CFLAGS = ' + os.environ.get('CFLAGS', '') + ' ' + os.environ.get('CPPFLAGS', '')) |
| 786 |
raw = raw.replace('CXXFLAGS =', 'CXXFLAGS = ' + os.environ.get('CXXFLAGS', '') + ' ' + os.environ.get('CPPFLAGS', '')) |
| 787 |
raw = raw.replace('LFLAGS =', 'LFLAGS = ' + os.environ.get('LDFLAGS', '')) |
| 788 |
f.seek(0), f.truncate() |
| 789 |
f.write(raw) |
| 790 |
env = os.environ.copy() |
| 791 |
if is_macos_universal_build: |
| 792 |
env['ARCHS'] = 'x86_64 arm64' |
| 793 |
self.check_call([self.env.make] + ([] if iswindows else [f'-j{os.cpu_count() or 1}']), env=env) |
| 794 |
e = 'pyd' if iswindows else 'so' |
| 795 |
m = glob.glob(f'{ext.name}/{ext.name}.*{e}') |
| 796 |
if not m: |
| 797 |
raise SystemExit(f'No built PyQt extension file in {os.path.join(os.getcwd(), ext.name)}') |
| 798 |
if len(m) != 1: |
| 799 |
raise SystemExit(f'Found extra PyQt extension files: {m}') |
| 800 |
shutil.copy2(m[0], self.dest(ext, self.env)) |
| 801 |
with open(sbf, 'w') as f: |
| 802 |
f.write('done') |
| 803 |
finally: |
| 804 |
os.chdir(cwd) |
| 805 |
|
| 806 |
def clean(self): |
| 807 |
self.output_dir = self.DEFAULT_OUTPUTDIR |
| 808 |
extensions = map(parse_extension, read_extensions()) |
| 809 |
env = init_env() |
| 810 |
for ext in extensions: |
| 811 |
dest = self.dest(ext, env) |
| 812 |
b, d = os.path.basename(dest), os.path.dirname(dest) |
| 813 |
b = b.split('.')[0] + '.*' |
| 814 |
for x in glob.glob(os.path.join(d, b)): |
| 815 |
if os.path.exists(x): |
| 816 |
os.remove(x) |
| 817 |
build_dir = self.DEFAULT_BUILDDIR |
| 818 |
if os.path.exists(build_dir): |
| 819 |
shutil.rmtree(build_dir) |