| 1 |
#!/usr/bin/env python |
| 2 |
|
| 3 |
|
| 4 |
__license__ = 'GPL v3' |
| 5 |
__copyright__ = '2014, Kovid Goyal <kovid at kovidgoyal.net>' |
| 6 |
|
| 7 |
import itertools |
| 8 |
import json |
| 9 |
import os |
| 10 |
import shlex |
| 11 |
import subprocess |
| 12 |
import sys |
| 13 |
from collections import namedtuple |
| 14 |
from contextlib import closing |
| 15 |
from functools import partial |
| 16 |
from multiprocessing.pool import ThreadPool as Pool |
| 17 |
from threading import Thread |
| 18 |
|
| 19 |
from polyglot.builtins import as_bytes |
| 20 |
|
| 21 |
Job = namedtuple('Job', 'cmd human_text cwd') |
| 22 |
|
| 23 |
cpu_count = min(16, max(1, os.cpu_count())) |
| 24 |
|
| 25 |
|
| 26 |
def run_worker(job, decorate=True): |
| 27 |
cmd, human_text = job.cmd, job.human_text |
| 28 |
human_text = human_text or shlex.join(cmd) |
| 29 |
cwd = job.cwd |
| 30 |
if cmd[0].lower().endswith('cl.exe'): |
| 31 |
cwd = os.environ.get('COMPILER_CWD') |
| 32 |
try: |
| 33 |
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=cwd) |
| 34 |
except Exception as err: |
| 35 |
return False, human_text, str(err) |
| 36 |
stdout, stderr = p.communicate() |
| 37 |
if stdout: |
| 38 |
stdout = stdout.decode('utf-8') |
| 39 |
if stderr: |
| 40 |
stderr = stderr.decode('utf-8') |
| 41 |
if decorate: |
| 42 |
stdout = human_text + '\n' + (stdout or '') |
| 43 |
ok = p.returncode == 0 |
| 44 |
return ok, stdout, (stderr or '') |
| 45 |
|
| 46 |
|
| 47 |
def create_job(cmd, human_text=None, cwd=None): |
| 48 |
return Job(cmd, human_text, cwd) |
| 49 |
|
| 50 |
|
| 51 |
def parallel_build(jobs, log, verbose=True): |
| 52 |
p = Pool(cpu_count) |
| 53 |
with closing(p): |
| 54 |
for ok, stdout, stderr in p.imap(run_worker, jobs): |
| 55 |
if verbose or not ok: |
| 56 |
log(stdout) |
| 57 |
if stderr: |
| 58 |
log(stderr) |
| 59 |
if not ok: |
| 60 |
return False |
| 61 |
return True |
| 62 |
|
| 63 |
|
| 64 |
def parallel_build_silent(jobs): |
| 65 |
p = Pool(cpu_count) |
| 66 |
results = [] |
| 67 |
failed = False |
| 68 |
with closing(p): |
| 69 |
for (ok, stdout, stderr), job in zip(p.imap(partial(run_worker, decorate=False), jobs), jobs): |
| 70 |
results.append((ok, job.cmd, job.human_text, stdout, stderr)) |
| 71 |
if not ok: |
| 72 |
failed = True |
| 73 |
return failed, results |
| 74 |
|
| 75 |
|
| 76 |
def parallel_check_output(jobs, log): |
| 77 |
p = Pool(cpu_count) |
| 78 |
with closing(p): |
| 79 |
for ok, stdout, stderr in p.imap(partial(run_worker, decorate=False), ((j, '') for j in jobs)): |
| 80 |
if not ok: |
| 81 |
log(stdout) |
| 82 |
if stderr: |
| 83 |
log(stderr) |
| 84 |
raise SystemExit(1) |
| 85 |
yield stdout |
| 86 |
|
| 87 |
|
| 88 |
def get_tasks(it): |
| 89 |
it = tuple(it) |
| 90 |
size, extra = divmod(len(it), cpu_count) |
| 91 |
if extra: |
| 92 |
size += 1 |
| 93 |
it = iter(it) |
| 94 |
while 1: |
| 95 |
x = tuple(itertools.islice(it, size)) |
| 96 |
if not x: |
| 97 |
return |
| 98 |
yield x |
| 99 |
|
| 100 |
|
| 101 |
def batched_parallel_jobs(cmd, jobs, cwd=None): |
| 102 |
workers = [] |
| 103 |
|
| 104 |
def get_output(p): |
| 105 |
p.output = p.communicate(as_bytes(json.dumps(p.jobs_batch))) |
| 106 |
|
| 107 |
for batch in get_tasks(jobs): |
| 108 |
p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd) |
| 109 |
p.jobs_batch = batch |
| 110 |
p.output_thread = t = Thread(target=get_output, args=(p,)) |
| 111 |
t.daemon = True |
| 112 |
t.start() |
| 113 |
workers.append(p) |
| 114 |
|
| 115 |
failed = False |
| 116 |
ans = [] |
| 117 |
for p in workers: |
| 118 |
p.output_thread.join() |
| 119 |
if p.wait() != 0: |
| 120 |
sys.stderr.buffer.write(p.output[1]) |
| 121 |
sys.stderr.buffer.flush() |
| 122 |
failed = True |
| 123 |
else: |
| 124 |
ans.extend(json.loads(p.output[0])) |
| 125 |
if failed: |
| 126 |
raise SystemExit('Worker process failed') |
| 127 |
return ans |
| 128 |
|
| 129 |
|
| 130 |
def threaded_func_jobs(func, jobs_args): |
| 131 |
|
| 132 |
def f(args): |
| 133 |
return func(*args) |
| 134 |
|
| 135 |
p = Pool(cpu_count) |
| 136 |
with closing(p): |
| 137 |
return p.map(f, jobs_args) |