Debian Codegraph @

setup/parallel_build.py

raw
1 #!/usr/bin/env python
2
3
4 __license__ = 'GPL v3'
5 __copyright__ = '2014, Kovid Goyal <kovid at kovidgoyal.net>'
6
7 import itertools
8 import json
9 import os
10 import shlex
11 import subprocess
12 import sys
13 from collections import namedtuple
14 from contextlib import closing
15 from functools import partial
16 from multiprocessing.pool import ThreadPool as Pool
17 from threading import Thread
18
19 from polyglot.builtins import as_bytes
20
21 Job = namedtuple('Job', 'cmd human_text cwd')
22
23 cpu_count = min(16, max(1, os.cpu_count()))
24
25
26 def run_worker(job, decorate=True):
27 cmd, human_text = job.cmd, job.human_text
28 human_text = human_text or shlex.join(cmd)
29 cwd = job.cwd
30 if cmd[0].lower().endswith('cl.exe'):
31 cwd = os.environ.get('COMPILER_CWD')
32 try:
33 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=cwd)
34 except Exception as err:
35 return False, human_text, str(err)
36 stdout, stderr = p.communicate()
37 if stdout:
38 stdout = stdout.decode('utf-8')
39 if stderr:
40 stderr = stderr.decode('utf-8')
41 if decorate:
42 stdout = human_text + '\n' + (stdout or '')
43 ok = p.returncode == 0
44 return ok, stdout, (stderr or '')
45
46
47 def create_job(cmd, human_text=None, cwd=None):
48 return Job(cmd, human_text, cwd)
49
50
51 def parallel_build(jobs, log, verbose=True):
52 p = Pool(cpu_count)
53 with closing(p):
54 for ok, stdout, stderr in p.imap(run_worker, jobs):
55 if verbose or not ok:
56 log(stdout)
57 if stderr:
58 log(stderr)
59 if not ok:
60 return False
61 return True
62
63
64 def parallel_build_silent(jobs):
65 p = Pool(cpu_count)
66 results = []
67 failed = False
68 with closing(p):
69 for (ok, stdout, stderr), job in zip(p.imap(partial(run_worker, decorate=False), jobs), jobs):
70 results.append((ok, job.cmd, job.human_text, stdout, stderr))
71 if not ok:
72 failed = True
73 return failed, results
74
75
76 def parallel_check_output(jobs, log):
77 p = Pool(cpu_count)
78 with closing(p):
79 for ok, stdout, stderr in p.imap(partial(run_worker, decorate=False), ((j, '') for j in jobs)):
80 if not ok:
81 log(stdout)
82 if stderr:
83 log(stderr)
84 raise SystemExit(1)
85 yield stdout
86
87
88 def get_tasks(it):
89 it = tuple(it)
90 size, extra = divmod(len(it), cpu_count)
91 if extra:
92 size += 1
93 it = iter(it)
94 while 1:
95 x = tuple(itertools.islice(it, size))
96 if not x:
97 return
98 yield x
99
100
101 def batched_parallel_jobs(cmd, jobs, cwd=None):
102 workers = []
103
104 def get_output(p):
105 p.output = p.communicate(as_bytes(json.dumps(p.jobs_batch)))
106
107 for batch in get_tasks(jobs):
108 p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd)
109 p.jobs_batch = batch
110 p.output_thread = t = Thread(target=get_output, args=(p,))
111 t.daemon = True
112 t.start()
113 workers.append(p)
114
115 failed = False
116 ans = []
117 for p in workers:
118 p.output_thread.join()
119 if p.wait() != 0:
120 sys.stderr.buffer.write(p.output[1])
121 sys.stderr.buffer.flush()
122 failed = True
123 else:
124 ans.extend(json.loads(p.output[0]))
125 if failed:
126 raise SystemExit('Worker process failed')
127 return ans
128
129
130 def threaded_func_jobs(func, jobs_args):
131
132 def f(args):
133 return func(*args)
134
135 p = Pool(cpu_count)
136 with closing(p):
137 return p.map(f, jobs_args)