Debian Codegraph @

setup/wincross.py

raw
1 #!/usr/bin/env python
2 # License: GPLv3 Copyright: 2023, Kovid Goyal <kovid at kovidgoyal.net>
3
4 # See https://github.com/mstorsjo/msvc-wine/blob/master/vsdownload.py and
5 # https://github.com/Jake-Shadle/xwin/blob/main/src/lib.rs for the basic logic
6 # used to download and install the needed VisualStudio packages
7
8 import argparse
9 import concurrent.futures
10 import glob
11 import hashlib
12 import json
13 import os
14 import re
15 import shutil
16 import subprocess
17 import sys
18 from collections import defaultdict
19 from dataclasses import dataclass
20 from functools import partial
21 from pprint import pprint
22 from tempfile import TemporaryDirectory
23 from urllib.parse import unquote
24 from urllib.request import urlopen
25 from zipfile import ZipFile
26
27
28 @dataclass
29 class File:
30 filename: str
31 url: str
32 size: int
33 sha256: str
34
35 def __init__(self, pf, filename=''):
36 self.filename = filename or pf['fileName']
37 self.url = pf['url']
38 self.size = pf['size']
39 self.sha256 = pf['sha256'].lower()
40
41
42 def package_sort_key(p):
43 chip = 0 if p.get('chip', '').lower() == 'x64' else 1
44 language = 0 if p.get('language', '').lower().startswith('en-') else 1
45 return chip, language
46
47
48 def llvm_arch_to_ms_arch(arch):
49 return {'x86_64': 'x64', 'aarch64': 'arm64', 'x64': 'x64', 'arm64': 'arm64'}[arch]
50
51
52 class Packages:
53 def __init__(self, manifest_raw, crt_variant, arch):
54 arch = llvm_arch_to_ms_arch(arch)
55 self.manifest = json.loads(manifest_raw)
56 self.packages = defaultdict(list)
57 self.cabinet_entries = {}
58 for p in self.manifest['packages']:
59 pid = p['id'].lower()
60 self.packages[pid].append(p)
61 for v in self.packages.values():
62 v.sort(key=package_sort_key)
63
64 build_tools = self.packages['Microsoft.VisualStudio.Product.BuildTools'.lower()][0]
65 pat = re.compile(r'Microsoft\.VisualStudio\.Component\.VC\.(.+)\.x86\.x64')
66 latest = (0, 0, 0, 0)
67 self.crt_version = ''
68 for dep in build_tools['dependencies']:
69 m = pat.match(dep)
70 if m is not None:
71 parts = m.group(1).split('.')
72 if len(parts) > 1:
73 q = tuple(map(int, parts))
74 if q > latest:
75 self.crt_version = m.group(1)
76 latest = q
77 if not self.crt_version:
78 raise KeyError('Failed to find CRT version from build tools deps')
79 self.files_to_download = []
80
81 def add_package(key):
82 p = self.packages.get(key.lower())
83 if not p:
84 raise KeyError(f'No package named {key} found')
85 for pf in p[0]['payloads']:
86 self.files_to_download.append(File(pf))
87
88 # CRT headers
89 add_package(f'Microsoft.VC.{self.crt_version}.CRT.Headers.base')
90 # CRT libs
91 prefix = f'Microsoft.VC.{self.crt_version}.CRT.{arch}.'.lower()
92 variants = {}
93 for pid in self.packages:
94 if pid.startswith(prefix):
95 parts = pid[len(prefix) :].split('.')
96 if parts[-1] == 'base':
97 variant = parts[0]
98 if variant not in variants or 'spectre' in parts:
99 # spectre variant contains both spectre and regular libs
100 variants[variant] = pid
101 add_package(variants[crt_variant])
102 # ATL headers
103 add_package(f'Microsoft.VC.{self.crt_version}.ATL.Headers.base')
104 # ATL libs
105 add_package(f'Microsoft.VC.{self.crt_version}.ATL.{arch}.Spectre.base')
106 add_package(f'Microsoft.VC.{self.crt_version}.ATL.{arch}.base')
107 # WinSDK
108 pat = re.compile(r'Win(\d+)SDK_(\d.+)', re.IGNORECASE)
109 latest_sdk = (0, 0, 0)
110 self.sdk_version = ''
111 sdk_pid = ''
112 for pid in self.packages:
113 m = pat.match(pid)
114 if m is not None:
115 ver = tuple(map(int, m.group(2).split('.')))
116 if ver > latest_sdk:
117 self.sdk_version = m.group(2)
118 latest_sdk = ver
119 sdk_pid = pid
120 if not self.sdk_version:
121 raise KeyError('Failed to find SDK package')
122 # headers are in x86 package and arch specific package
123 for pf in self.packages[sdk_pid][0]['payloads']:
124 fname = pf['fileName'].split('\\')[-1]
125 if fname.startswith('Windows SDK Desktop Headers '):
126 q = fname[len('Windows SDK Desktop Headers ') :]
127 if q.lower() == 'x86-x86_en-us.msi':
128 self.files_to_download.append(File(pf, filename=f'{sdk_pid}_headers.msi'))
129 elif q.lower() == f'{arch}-x86_en-us.msi':
130 self.files_to_download.append(File(pf, filename=f'{sdk_pid}_{arch}_headers.msi'))
131 elif fname == 'Windows SDK for Windows Store Apps Headers-x86_en-us.msi':
132 self.files_to_download.append(File(pf, filename=f'{sdk_pid}_store_headers.msi'))
133 elif fname.startswith('Windows SDK Desktop Libs '):
134 q = fname[len('Windows SDK Desktop Libs ') :]
135 if q == f'{arch}-x86_en-us.msi':
136 self.files_to_download.append(File(pf, filename=f'{sdk_pid}_libs_x64.msi'))
137 elif fname == 'Windows SDK for Windows Store Apps Libs-x86_en-us.msi':
138 self.files_to_download.append(File(pf, filename=f'{sdk_pid}_store_libs.msi'))
139 elif (fl := fname.lower()).endswith('.cab'):
140 self.cabinet_entries[fl] = File(pf, filename=fl)
141 # UCRT
142 for pf in self.packages['Microsoft.Windows.UniversalCRT.HeadersLibsSources.Msi'.lower()][0]['payloads']:
143 fname = pf['fileName'].split('\\')[-1]
144 if fname == 'Universal CRT Headers Libraries and Sources-x86_en-us.msi':
145 self.files_to_download.append(File(pf))
146 self.files_to_download[-1].filename = 'ucrt.msi'
147 elif (fl := fname.lower()).endswith('.cab'):
148 self.cabinet_entries[fl] = File(pf, filename=fl)
149
150
151 def download_item(dest_dir: str, file: File):
152 dest = os.path.join(dest_dir, file.filename)
153 m = hashlib.sha256()
154 with urlopen(file.url) as src, open(dest, 'wb') as d:
155 with memoryview(bytearray(shutil.COPY_BUFSIZE)) as buf:
156 while True:
157 n = src.readinto(buf)
158 if not n:
159 break
160 elif n < shutil.COPY_BUFSIZE:
161 with buf[:n] as smv:
162 d.write(smv)
163 m.update(smv)
164 else:
165 d.write(buf)
166 m.update(buf)
167 if m.hexdigest() != file.sha256:
168 raise SystemExit(f'The hash for {file.filename} does not match. {m.hexdigest()} != {file.sha256}')
169
170
171 def cabinets_in_msi(path):
172 raw = subprocess.check_output(['msiinfo', 'export', path, 'Media']).decode('utf-8')
173 return re.findall(r'\S+\.cab', raw)
174
175
176 def download(dest_dir, manifest_version=17, manifest_type='release', manifest_path='', crt_variant='desktop', arch='x86_64'):
177 if manifest_path:
178 manifest = open(manifest_path, 'rb').read()
179 else:
180 url = f'https://aka.ms/vs/{manifest_version}/{manifest_type}/channel'
181 print('Downloading top-level manifest from', url)
182 tm = json.loads(urlopen(url).read())
183 print('Got toplevel manifest for', (tm['info']['productDisplayVersion']))
184 for item in tm['channelItems']:
185 if item.get('type') == 'Manifest':
186 url = item['payloads'][0]['url']
187 print('Downloading actual manifest...')
188 manifest = urlopen(url).read()
189
190 pkgs = Packages(manifest, crt_variant, arch)
191 os.makedirs(dest_dir, exist_ok=True)
192 total = sum(x.size for x in pkgs.files_to_download)
193 print('Downloading', int(total / (1024 * 1024)), 'MB in', len(pkgs.files_to_download), 'files...')
194 with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor:
195 for _ in executor.map(partial(download_item, dest_dir), pkgs.files_to_download):
196 pass
197 cabs = []
198 for x in os.listdir(dest_dir):
199 if x.lower().endswith('.msi'):
200 for cab in cabinets_in_msi(os.path.join(dest_dir, x)):
201 cabs.append(pkgs.cabinet_entries[cab])
202 total = sum(x.size for x in cabs)
203 print('Downloading', int(total / (1024 * 1024)), 'MB in', len(cabs), 'files...')
204 with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor:
205 for _ in executor.map(partial(download_item, dest_dir), cabs):
206 pass
207
208
209 def merge_trees(src, dest):
210 if not os.path.isdir(src):
211 return
212 if not os.path.isdir(dest):
213 shutil.move(src, dest)
214 return
215 destnames = {n.lower(): n for n in os.listdir(dest)}
216 for d in os.scandir(src):
217 n = d.name
218 srcname = os.path.join(src, n)
219 destname = os.path.join(dest, n)
220 if d.is_dir():
221 if os.path.isdir(destname):
222 merge_trees(srcname, destname)
223 elif n.lower() in destnames:
224 merge_trees(srcname, os.path.join(dest, destnames[n.lower()]))
225 else:
226 shutil.move(srcname, destname)
227 else:
228 shutil.move(srcname, destname)
229
230
231 def extract_msi(path, dest_dir):
232 print('Extracting', os.path.basename(path), '...')
233 with open(os.path.join(dest_dir, os.path.basename(path) + '.listing'), 'w') as log:
234 subprocess.check_call(['msiextract', '-C', dest_dir, path], stdout=log)
235
236
237 def extract_zipfile(zf, dest_dir):
238 tmp = os.path.join(dest_dir, 'extract')
239 os.mkdir(tmp)
240 for f in zf.infolist():
241 name = unquote(f.filename)
242 dest = os.path.join(dest_dir, name)
243 extracted = zf.extract(f, tmp)
244 os.makedirs(os.path.dirname(dest), exist_ok=True)
245 shutil.move(extracted, dest)
246 shutil.rmtree(tmp)
247
248
249 def extract_vsix(path, dest_dir):
250 print('Extracting', os.path.basename(path), '...')
251 with TemporaryDirectory(dir=dest_dir) as tdir, ZipFile(path, 'r') as zf:
252 extract_zipfile(zf, tdir)
253 contents = os.path.join(tdir, 'Contents')
254 merge_trees(contents, dest_dir)
255 names = zf.namelist()
256 with open(os.path.join(dest_dir, os.path.basename(path) + '.listing'), 'w') as ls:
257 ls.write('\n'.join(names))
258
259
260 def move_unpacked_trees(src_dir, dest_dir):
261 # CRT
262 crt_src = os.path.dirname(glob.glob(os.path.join(src_dir, 'VC/Tools/MSVC/*/include'))[0])
263 crt_dest = os.path.join(dest_dir, 'crt')
264 os.makedirs(crt_dest)
265 merge_trees(os.path.join(crt_src, 'include'), os.path.join(crt_dest, 'include'))
266 merge_trees(os.path.join(crt_src, 'lib'), os.path.join(crt_dest, 'lib'))
267 merge_trees(os.path.join(crt_src, 'atlmfc', 'include'), os.path.join(crt_dest, 'include'))
268 merge_trees(os.path.join(crt_src, 'atlmfc', 'lib'), os.path.join(crt_dest, 'lib'))
269
270 # SDK
271 sdk_ver = glob.glob(os.path.join(src_dir, 'win11sdk_*_headers.msi.listing'))[0]
272 sdk_ver = sdk_ver.split('_')[1]
273 for x in glob.glob(os.path.join(src_dir, 'Program Files/Windows Kits/*/Include/*')):
274 if os.path.basename(x).startswith(sdk_ver):
275 sdk_ver = os.path.basename(x)
276 break
277 else:
278 raise SystemExit(f'Failed to find sdk_ver: {sdk_ver}')
279 sdk_include_src = glob.glob(os.path.join(src_dir, f'Program Files/Windows Kits/*/Include/{sdk_ver}'))[0]
280 sdk_dest = os.path.join(dest_dir, 'sdk')
281 os.makedirs(sdk_dest)
282 merge_trees(sdk_include_src, os.path.join(sdk_dest, 'include'))
283 sdk_lib_src = glob.glob(os.path.join(src_dir, f'Program Files/Windows Kits/*/Lib/{sdk_ver}'))[0]
284 merge_trees(sdk_lib_src, os.path.join(sdk_dest, 'lib'))
285
286 # UCRT
287 if os.path.exists(os.path.join(sdk_include_src, 'ucrt')):
288 return
289 ucrt_include_src = glob.glob(os.path.join(src_dir, 'Program Files/Windows Kits/*/Include/*/ucrt'))[0]
290 merge_trees(ucrt_include_src, os.path.join(sdk_dest, 'include', 'ucrt'))
291 ucrt_lib_src = glob.glob(os.path.join(src_dir, 'Program Files/Windows Kits/*/Lib/*/ucrt'))[0]
292 merge_trees(ucrt_lib_src, os.path.join(sdk_dest, 'lib', 'ucrt'))
293
294
295 def unpack(src_dir, dest_dir):
296 if os.path.exists(dest_dir):
297 shutil.rmtree(dest_dir)
298 extract_dir = os.path.join(dest_dir, 'extract')
299 os.makedirs(extract_dir)
300 for x in os.listdir(src_dir):
301 path = os.path.join(src_dir, x)
302 ext = os.path.splitext(x)[1].lower()
303 if ext == '.msi':
304 extract_msi(path, extract_dir)
305 elif ext == '.vsix':
306 extract_vsix(path, extract_dir)
307 elif ext == '.cab':
308 continue
309 else:
310 raise SystemExit(f'Unknown downloaded file type: {x}')
311 move_unpacked_trees(extract_dir, dest_dir)
312 shutil.rmtree(extract_dir)
313
314
315 def symlink_transformed(path, transform=str.lower):
316 base, name = os.path.split(path)
317 lname = transform(name)
318 if lname != name:
319 npath = os.path.join(base, lname)
320 if not os.path.lexists(npath):
321 os.symlink(name, npath)
322
323
324 def clone_tree(src_dir, dest_dir):
325 os.makedirs(dest_dir)
326 for dirpath, dirnames, filenames in os.walk(src_dir):
327 for d in dirnames:
328 path = os.path.join(dirpath, d)
329 rpath = os.path.relpath(path, src_dir)
330 dpath = os.path.join(dest_dir, rpath)
331 os.makedirs(dpath)
332 symlink_transformed(dpath)
333 for f in filenames:
334 if f.lower().endswith('.pdb'):
335 continue
336 path = os.path.join(dirpath, f)
337 rpath = os.path.relpath(path, src_dir)
338 dpath = os.path.join(dest_dir, rpath)
339 os.link(path, dpath)
340 symlink_transformed(dpath)
341
342
343 def files_in(path):
344 for dirpath, _, filenames in os.walk(path):
345 for f in filenames:
346 yield os.path.relpath(os.path.join(dirpath, f), path)
347
348
349 def create_include_symlinks(path, include_root, include_files):
350 "Create symlinks for include entries in header files whose case does not match"
351 with open(path, 'rb') as f:
352 src = f.read()
353 for m in re.finditer(rb'^#include\s+([<"])(.+?)[>"]', src, flags=re.M):
354 spec = m.group(2).decode().replace('\\', '/')
355 lspec = spec.lower()
356 if spec == lspec:
357 continue
358 is_local = m.group(1).decode() == '"'
359 found = ''
360 lmatches = []
361 for ir, specs in include_files.items():
362 if spec in specs:
363 found = ir
364 break
365 if lspec in specs:
366 lmatches.append(ir)
367
368 if found and (not is_local or found == include_root):
369 continue
370 if lmatches:
371 if is_local and include_root in lmatches:
372 fr = include_root
373 else:
374 fr = lmatches[0]
375 symlink_transformed(os.path.join(fr, lspec), lambda n: os.path.basename(spec))
376
377
378 def setup(splat_dir, root_dir, arch):
379 print('Creating symlinks...')
380 msarch = llvm_arch_to_ms_arch(arch)
381 if os.path.exists(root_dir):
382 shutil.rmtree(root_dir)
383 os.makedirs(root_dir)
384 # CRT
385 clone_tree(os.path.join(splat_dir, 'crt', 'include'), os.path.join(root_dir, 'crt', 'include'))
386 clone_tree(os.path.join(splat_dir, 'crt', 'lib', 'spectre', msarch), os.path.join(root_dir, 'crt', 'lib'))
387 # SDK
388 clone_tree(os.path.join(splat_dir, 'sdk', 'include'), os.path.join(root_dir, 'sdk', 'include'))
389 for x in glob.glob(os.path.join(splat_dir, 'sdk', 'lib', '*', msarch)):
390 clone_tree(x, os.path.join(root_dir, 'sdk', 'lib', os.path.basename(os.path.dirname(x))))
391 include_roots = [x for x in glob.glob(os.path.join(root_dir, 'sdk', 'include', '*')) if os.path.isdir(x)]
392 include_roots.append(os.path.join(root_dir, 'crt', 'include'))
393 include_files = {x: set(files_in(x)) for x in include_roots}
394 for ir, files in include_files.items():
395 files_to_check = []
396 for relpath in files:
397 path = os.path.join(ir, relpath)
398 if not os.path.islink(path):
399 files_to_check.append(path)
400 for path in files_to_check:
401 create_include_symlinks(path, ir, include_files)
402
403
404 def main(args=sys.argv[1:]):
405 stages = ('download', 'unpack', 'setup')
406 p = argparse.ArgumentParser(description='Setup the headers and libraries for cross-compilation of windows binaries')
407 p.add_argument('stages', metavar='STAGES', nargs='*', help=(f'The stages to run by default all stages are run. Stages are: {" ".join(stages)}'))
408 p.add_argument('--manifest-version', default=17, type=int, help='The manifest version to use to find the packages to install')
409 p.add_argument('--manifest-path', default='', help='Path to a local manifest file to use. Causes --manifest-version to be ignored.')
410 p.add_argument('--crt-variant', default='desktop', choices=('desktop', 'store', 'onecore'), help='The type of CRT to download')
411 p.add_argument('--arch', default='x86_64', choices=('x86_64', 'aarch64'), help='The architecture to install')
412 p.add_argument('--dest', default='.', help='The directory to install into')
413 args = p.parse_args(args)
414 if args.dest == '.':
415 args.dest = os.getcwd()
416 stages = args.stages or stages
417 dl_dir = os.path.join(args.dest, 'dl')
418 splat_dir = os.path.join(args.dest, 'splat')
419 root_dir = os.path.join(args.dest, 'root')
420 for stage in stages:
421 if stage == 'download':
422 download(dl_dir, manifest_version=args.manifest_version, manifest_path=args.manifest_path, crt_variant=args.crt_variant, arch=args.arch)
423 elif stage == 'unpack':
424 unpack(dl_dir, splat_dir)
425 elif stage == 'setup':
426 setup(splat_dir, root_dir, args.arch)
427 else:
428 raise SystemExit(f'Unknown stage: {stage}')
429
430
431 if __name__ == '__main__':
432 pprint
433 main()