Debian Codegraph @

project/bundles/homebrew/package_lib.py

raw
1 #!/usr/bin/env python3
2 from __future__ import print_function
3 import sys, os, traceback, shutil, subprocess, re, pickle, glob
4 from pathlib import Path
5 from ast import literal_eval
6
7 """Copy dylibs into the build folder.
8
9 Copyright (c)2024 Michael Miller michael underscore miller at msn dot com
10
11 Based heavily on copy_dylibs.py by Andy Duplain trojanfoe at gmail dot com
12
13 """
14
15 # Any dependencies outside of these directories need copying and fixing
16 good_dirs = ["/System/", "/usr/lib/"]
17
18 # preferred directories to find libraries, in order
19 preferred_dir_prefix = ["lib/", "share/", "opt/", "bin/", "Cellar/"]
20 preferred_dirs = []
21
22 # Turn on for more output
23 debug = True
24
25 homebrew_prefix = None
26
27 custom_lib_dir = None
28
29 lib_dir = "@executable_path/../"
30
31 frameworks_dir = None
32
33 bundle_dir = None
34
35 # Module and library install_name changes:
36 # {
37 # dylib_path : [ old_name, new_name ], ..., [old_name, new_name]
38 # ...
39 # }
40 install_names = set()
41 install_names_cache = set()
42
43 # List of dylibs copied
44 processed_dylibs = set()
45 found_dylibs = dict()
46 copied_dylibs = set()
47
48 sign_binaries = False
49 copy_binaries = False
50 follow_symlinks = False
51 preserve_rpath = False
52 force = False
53 force_depth = 1
54 current_depth = 0
55
56 def echo(message):
57 print(message)
58 sys.stdout.flush()
59
60
61 def echon(message):
62 sys.stdout.write(message)
63 sys.stdout.flush()
64
65 def normalize_path(path):
66 new_path = os.path.abspath(path)
67 if not new_path.endswith("/"):
68 new_path = new_path + "/"
69 return new_path
70
71 def is_file_good(file):
72 for dir in good_dirs:
73 if file.startswith(dir):
74 return True
75 return False
76
77 def add_rpath(dylib):
78 global bundle_dir
79 global install_names
80 global install_names_cache
81 global sign_binaries
82
83 if not sign_binaries or (install_names_cache and dylib in install_names_cache):
84 return
85
86 install_names.add(dylib)
87
88
89
90 # change id based on the new rpath of the binary
91 rpath = dylib[len(bundle_dir):]
92 echo("Changing ID :@rpath/" + rpath)
93 cmdline = ["install_name_tool", "-id", "@rpath/" + rpath, dylib]
94 if debug:
95 echo("Running: " + " ".join(cmdline))
96 exitcode = subprocess.call(cmdline)
97
98 # depth = rpath.count("/")
99 # rpath = "@loader_path"
100 # for i in range(depth):
101 # rpath = rpath + "/.."
102
103 echo("Adding @rpath: @executable_path/../")
104 cmdline = ["install_name_tool", "-add_rpath", "@executable_path/../", dylib]
105 if debug:
106 echo("Running: " + " ".join(cmdline))
107 exitcode = subprocess.call(cmdline)
108
109 # path to ./lib relative to the executable
110 echo("Adding @rpath: @executable_path/../lib")
111 cmdline = ["install_name_tool", "-add_rpath", "@executable_path/../lib", dylib]
112 if debug:
113 echo("Running: " + " ".join(cmdline))
114 exitcode = subprocess.call(cmdline)
115
116 # # path to ./lib relative to the executable
117 # dylib_dir = dylib[len(bundle_dir):]
118 # if not dylib_dir.startswith("/"):
119 # dylib_dir = "/" + dylib_dir
120
121 # rpath = "@loader_path/"
122
123 # depth = dylib_dir.count("/")
124 # for i in range(dylib_dir.count("/")-1):
125 # echo("Adding @rpath: " + rpath)
126 # cmdline = ["install_name_tool", "-add_rpath", rpath, dylib]
127 # if debug:
128 # echo("Running: " + " ".join(cmdline))
129 # exitcode = subprocess.call(cmdline)
130 # cmdline = ["install_name_tool", "-add_rpath", rpath + "lib", dylib]
131 # if debug:
132 # echo("Running: " + " ".join(cmdline))
133 # exitcode = subprocess.call(cmdline)
134 # cmdline = ["install_name_tool", "-add_rpath", rpath + "opt", dylib]
135 # if debug:
136 # echo("Running: " + " ".join(cmdline))
137 # exitcode = subprocess.call(cmdline)
138 # rpath = rpath + "../"
139
140 def expand_symlink_path(path):
141 # subdirs = path.split("/")
142 # for subdir in subdirs:
143 # if
144 return
145
146 def expand_symlinks(dylib):
147 return
148
149
150 def change_install_names(old_name, new_name, dylib):
151 global install_names
152 global sign_binaries
153 global preserve_rpath
154 # check if anything needs to change
155 if str(old_name).lower().startswith("@loader_path") or not sign_binaries or (install_names_cache and dylib in install_names_cache):
156 return
157
158 if str(old_name).startswith("@"):
159 if preserve_rpath:
160 return
161 # check the old rpath matches the new path or it doesn't have any path
162 if str(old_name).startswith("@rpath"):
163 prefix = "@rpath/"
164 else:
165 if str(old_name).startswith("@@executable_path"):
166 prefix = "@executable_path/"
167
168 file = old_name[len(prefix):]
169 rel_path = new_name[len(bundle_dir):]
170
171 # nothing to do, return
172 if file == rel_path:
173 return
174
175 # dylib_dir = dylib[len(bundle_dir):]
176 # if not dylib_dir.startswith("/"):
177 # dylib_dir = "/" + dylib_dir
178
179 new_lib_name = new_name[len(bundle_dir):]
180 if not new_lib_name.startswith("/"):
181 new_lib_name = "/" + new_lib_name
182
183 # depth = dylib_dir.count("/")
184 # for i in range(dylib_dir.count("/")-1):
185 # new_lib_name = "/.." + new_lib_name
186 new_lib_name = "@rpath" + new_lib_name
187
188 install_names.add(os.path.realpath(dylib))
189
190 # new_lib_name = simplify_path(new_lib_name, dylib)
191
192 cmdline = ["install_name_tool", "-change", old_name, new_lib_name, dylib]
193 if debug:
194 echo("Running: " + " ".join(cmdline))
195 exitcode = subprocess.call(cmdline)
196 if exitcode != 0:
197 raise RuntimeError(
198 "Failed to change '{0}' to '{1}' in '{2}".format(
199 old_name, new_lib_name, dylib
200 )
201 )
202
203 def find_file(file, parent):
204 global found_dylibs
205 global bundle_dir
206 global follow_symlinks
207 global preferred_dirs
208
209 dylib = found_dylibs.get(file)
210 if dylib:
211 return dylib
212
213 # # return the same file if it exists, but avoid things in the Cellar if we can
214 # if file.find("Cellar") == -1 and os.path.exists(file) and (copy_binaries or file.startswith(bundle_dir)):
215 # # check if file lives in a valid lib path
216 # for path in preferred_dirs:
217 # if file.startswith(path):
218 # found_dylibs[file] = file
219 # return file
220
221 # if file.startswith("@loader_path"):
222 # file = file.replace("@loader_path", os.path.dirname(parent))
223 # if os.path.exists(file):
224 # found_dylibs[os.path.basename(file)] = file
225 # return file
226
227 echo("Finding: " + os.path.basename(file))
228
229 # Shortcut for bundle path to avoid duplicates
230 found_files = glob.glob(os.path.join(bundle_dir + "/**/", os.path.basename(file)), recursive=True)
231 for found_file in found_files:
232 dylib_path = os.path.dirname(found_file)
233 if not "dSYM" in dylib_path and not "include" in dylib_path and not "bindings" in dylib_path and not "Headers" in dylib_path and os.path.isfile(found_file):
234 # if not "dSYM" in dylib_path and not "include" in dylib_path and not "bindings" in dylib_path and os.path.isfile(found_file):
235 found_dylibs[file] = found_file
236 return found_file
237
238 if os.path.exists(file):
239 found_dylibs[file] = file
240 return file
241
242 new_file = os.path.basename(file)
243
244 parent_path = os.path.dirname(parent)
245 if os.path.exists(os.path.join(parent_path, new_file)) and os.path.isfile(os.path.join(parent_path, new_file)):
246 echo("Found in parent: " + os.path.basename(file))
247 found_dylibs[file] = os.path.join(parent_path, new_file)
248 return os.path.join(parent_path, new_file)
249
250 tmp_preferred_dirs = []
251 if copy_binaries:
252 tmp_preferred_dirs = preferred_dirs
253 else:
254 tmp_preferred_dirs.append(bundle_dir)
255
256 for pref_dir in tmp_preferred_dirs:
257 search_path = pref_dir + "/**/" + new_file
258 echo("Searching: " + search_path)
259 found_files = glob.glob(search_path, recursive=True)
260 for found_file in found_files:
261 dylib_path = os.path.dirname(found_file)
262 if not "dSYM" in dylib_path and not "include" in dylib_path and not "bindings" in dylib_path and not "Headers" in dylib_path and os.path.isfile(found_file):
263 found_dylibs[file] = found_file
264 return found_file
265
266 raise RuntimeError("Unable to find file: '{0}'\nDo you need --copy=true?".format(file))
267
268 def copy_dylib(src):
269 global copied_dylibs
270 global copy_binaries
271
272 #(dylib_path, dylib_filename) = os.path.split(src)
273
274 dest = None
275
276 src = src.replace("//", "/")
277
278 # shortcut for Hoembrew libs
279 if homebrew_prefix and src.startswith(homebrew_prefix):
280 dest = src.replace(homebrew_prefix, bundle_dir)
281 else:
282 # figure out where we found the file
283 if src.startswith(bundle_dir):
284 dest = src
285 else:
286 for path in preferred_dirs:
287 if src.startswith(path):
288 dest = src.replace(path, bundle_dir)
289 break
290
291 # if dest.find("Cellar") != -1:
292 # dest = dest.replace("Cellar", "share")
293 echo("Source file: " + src)
294 #if dest:
295 dest = os.path.abspath(dest)
296
297 if not dest in copied_dylibs and not os.path.exists(dest) and copy_binaries:
298 if debug:
299 echo("Copying {0} into bundle".format(src))
300 dest_dir = os.path.dirname(dest)
301 if not os.path.exists(dest_dir):
302 os.makedirs(dest_dir)
303 attrs = os.stat(src)
304 shutil.copyfile(src, dest)
305 os.chmod(dest, attrs.st_mode)
306
307
308 if dest in copied_dylibs:
309 ret = True
310 else:
311 ret = False
312 copied_dylibs.add(dest)
313
314 return dest, ret
315
316 def copy_dependencies(file, dep_file):
317 global processed_dylibs
318 global force
319 global force_depth
320 global current_depth
321
322 if not file in processed_dylibs and current_depth < force_depth:
323 if (force):
324 current_depth += 1
325 echo("Current depth: " + str(current_depth))
326 # process the file once if it's a symlinked file
327 if file.startswith("@"):
328 processed_dylibs.add(file)
329 else:
330 processed_dylibs.add(os.path.realpath(file))
331
332 dep_lines = []
333
334 file_filename = os.path.basename(file)
335 echo("Examining {0}".format(file_filename))
336 #found = 0
337 pipe = subprocess.Popen(["otool", "-L", file], stdout=subprocess.PIPE)
338 while True:
339 line = pipe.stdout.readline().decode("utf-8")
340 if line != "":
341 dep_lines.append(line)
342 else:
343 break
344
345 for line in dep_lines:
346 m = re.match(r"\s*(\S+)\s*\(compatibility version .+\)$", line)
347 if m:
348 dep = m.group(1)
349 # process only good file, and avoid reprocessing current file
350 if not is_file_good(dep) and os.path.split(file)[1] != os.path.split(dep)[1]:
351 #(dep_path, dep_filename) = os.path.split(dep)
352 # echo(dep)
353 found_dep = find_file(dep, dep_file)
354 if found_dep :
355 (bundled_dep, processed) = copy_dylib(found_dep)
356 if bundled_dep:
357 change_install_names(dep, bundled_dep, file)
358 if not processed:
359 # use the dep moved to the bundle. Don't update the original
360 copy_dependencies(bundled_dep, found_dep)
361
362
363
364 def codesign():
365 global sign_binaries
366 if not sign_binaries:
367 return
368
369 for dylib in install_names:
370 if not install_names_cache or not dylib in install_names_cache:
371 add_rpath(dylib)
372
373 echo("Codesigning {0}".format(os.path.basename(dylib)))
374 cmdline = [
375 "/usr/bin/codesign",
376 "--force",
377 "-s",
378 "-",
379 dylib,
380 ]
381 if debug:
382 echo("Running: " + " ".join(cmdline))
383 exitcode = subprocess.call(cmdline)
384 if exitcode != 0:
385 raise RuntimeError("Failed to codesign '{0}'".format(dylib))
386
387 def main(args):
388 global frameworks_dir
389 global bundle_dir
390 global homebrew_prefix
391 global preferred_dirs
392 global install_names
393 global install_names_cache
394 global processed_dylibs
395 global sign_binaries
396 global copy_binaries
397 global found_dylibs
398 global follow_symlinks
399 global preserve_rpath
400 global force
401 global force_depth
402
403 processed_dylib_cache_mode = "none"
404 found_dylib_cache_mode = "none"
405 signed_dylib_cache_mode = "none"
406
407
408 # process command line parameters
409 for arg in args:
410 param = str(arg)
411 if param.find("=") == -1:
412 continue
413 (arg_name, arg_value) = param.split("=")
414 arg_value = arg_value.replace("\"", "")
415 match arg_name.lower():
416 case "--file":
417 root_full_path = os.path.abspath(arg_value)
418 case "--bundle-root":
419 bundle_dir = normalize_path(arg_value)
420 preferred_dirs.append(bundle_dir)
421 case "--homebrew":
422 homebrew_prefix = normalize_path(arg_value)
423 for pref_dir in preferred_dir_prefix:
424 preferred_dirs.append(normalize_path(os.path.join(homebrew_prefix, pref_dir)))
425 case "--other-libs":
426 preferred_dirs.append(normalize_path(arg_value))
427 case "--processed-cache":
428 processed_dylib_cache_mode = arg_value
429 case "--found-cache":
430 found_dylib_cache_mode = arg_value
431 case "--signed-cache":
432 signed_dylib_cache_mode = arg_value
433 case "--update-binary":
434 sign_binaries = (arg_value.lower() == "true" or arg_value == "1")
435 case "--copy":
436 copy_binaries = (arg_value.lower() == "true" or arg_value == "1")
437 case "--preserve_rpath":
438 preserve_rpath = (arg_value.lower() == "true" or arg_value == "1")
439 case "--force":
440 force = (arg_value.lower() == "true" or arg_value == "1")
441 case _:
442 echo("Invalid parameter: " + arg_name)
443 quit()
444
445 # check we have the minimum params
446 if not root_full_path or not bundle_dir:
447 # we need a file to process and the bundle path at a minimum
448 echo( "Please supply --file and --bundle-path")
449
450 # get the path to the script to store the cache files
451 script_path = os.path.dirname(os.path.realpath(__file__))
452
453 # load the caches
454 processed_dylibs_file = script_path + "/processed_dylibs_cache.pkl"
455 if (processed_dylib_cache_mode == "update" or processed_dylib_cache_mode == "use") and os.path.exists(processed_dylibs_file):
456 with open(processed_dylibs_file, 'rb') as f:
457 processed_dylibs = pickle.load(f)
458
459 found_dylibs_file = script_path + "/found_dylibs_cache.pkl"
460 if (found_dylib_cache_mode == "update" or found_dylib_cache_mode == "use") and os.path.exists(found_dylibs_file):
461 with open(found_dylibs_file, 'rb') as f:
462 found_dylibs = pickle.load(f)
463
464 signed_dylibs_file = script_path + "/signed_dylibs_cache.pkl"
465 if (signed_dylib_cache_mode == "update" or signed_dylib_cache_mode == "use") and os.path.exists(signed_dylibs_file):
466 with open(signed_dylibs_file, 'rb') as f:
467 install_names_cache = pickle.load(f)
468
469
470 # force remove the file from the caches to reprocess it but preserving the rest of the cache
471 if (force):
472 for file in glob.glob(root_full_path):
473 try:
474 processed_dylibs.remove(file)
475 except:
476 print(file + " not in processed cache")
477 try:
478 found_dylibs.remove(file)
479 except:
480 print(file + " not in found cache")
481 try:
482 install_names_cache.remove(file)
483 except:
484 print(file + " not in signed cache")
485
486 # main loop to process files
487 for file in glob.glob(root_full_path):
488 print(file)
489 copy_dependencies(file, file)
490
491 # codesign the binaries
492 codesign()
493
494 # persist working data for the caches
495 if processed_dylib_cache_mode == "new" or processed_dylib_cache_mode == "update":
496 with open(processed_dylibs_file, 'wb') as f:
497 pickle.dump(processed_dylibs, f)
498
499 if found_dylib_cache_mode == "new" or found_dylib_cache_mode == "update":
500 with open(found_dylibs_file, 'wb') as f:
501 pickle.dump(found_dylibs, f)
502
503 if signed_dylib_cache_mode == "new" or signed_dylib_cache_mode == "update":
504 install_names_cache = install_names_cache.union(install_names)
505 with open(signed_dylibs_file, 'wb') as f:
506 pickle.dump(install_names_cache, f)
507
508 if __name__ == "__main__":
509 exitcode = 99
510 try:
511 exitcode = main(sys.argv)
512 except Exception as e:
513 echo(traceback.format_exc())
514 sys.exit(exitcode)