Debian Codegraph @

waflib/extras/batched_cc.py

raw
1 #!/usr/bin/env python
2 # encoding: utf-8
3 # Thomas Nagy, 2006-2015 (ita)
4
5 """
6 Instead of compiling object files one by one, c/c++ compilers are often able to compile at once:
7 cc -c ../file1.c ../file2.c ../file3.c
8
9 Files are output on the directory where the compiler is called, and dependencies are more difficult
10 to track (do not run the command on all source files if only one file changes)
11 As such, we do as if the files were compiled one by one, but no command is actually run:
12 replace each cc/cpp Task by a TaskSlave. A new task called TaskMaster collects the
13 signatures from each slave and finds out the command-line to run.
14
15 Just import this module to start using it:
16 def build(bld):
17 bld.load('batched_cc')
18
19 Note that this is provided as an example, unity builds are recommended
20 for best performance results (fewer tasks and fewer jobs to execute).
21 See waflib/extras/unity.py.
22 """
23
24 from waflib import Task, Utils
25 from waflib.TaskGen import extension, feature, after_method
26 from waflib.Tools import c, cxx
27
28 MAX_BATCH = 50
29
30 c_str = '${CC} ${ARCH_ST:ARCH} ${CFLAGS} ${FRAMEWORKPATH_ST:FRAMEWORKPATH} ${tsk.batch_incpaths()} ${DEFINES_ST:DEFINES} -c ${SRCLST} ${CXX_TGT_F_BATCHED} ${CPPFLAGS}'
31 c_fun, _ = Task.compile_fun_noshell(c_str)
32
33 cxx_str = '${CXX} ${ARCH_ST:ARCH} ${CXXFLAGS} ${FRAMEWORKPATH_ST:FRAMEWORKPATH} ${tsk.batch_incpaths()} ${DEFINES_ST:DEFINES} -c ${SRCLST} ${CXX_TGT_F_BATCHED} ${CPPFLAGS}'
34 cxx_fun, _ = Task.compile_fun_noshell(cxx_str)
35
36 count = 70000
37 class batch(Task.Task):
38 color = 'PINK'
39
40 after = ['c', 'cxx']
41 before = ['cprogram', 'cshlib', 'cstlib', 'cxxprogram', 'cxxshlib', 'cxxstlib']
42
43 def uid(self):
44 return Utils.h_list([Task.Task.uid(self), self.generator.idx, self.generator.path.abspath(), self.generator.target])
45
46 def __str__(self):
47 return 'Batch compilation for %d slaves' % len(self.slaves)
48
49 def __init__(self, *k, **kw):
50 Task.Task.__init__(self, *k, **kw)
51 self.slaves = []
52 self.inputs = []
53 self.hasrun = 0
54
55 global count
56 count += 1
57 self.idx = count
58
59 def add_slave(self, slave):
60 self.slaves.append(slave)
61 self.set_run_after(slave)
62
63 def runnable_status(self):
64 for t in self.run_after:
65 if not t.hasrun:
66 return Task.ASK_LATER
67
68 for t in self.slaves:
69 #if t.executed:
70 if t.hasrun != Task.SKIPPED:
71 return Task.RUN_ME
72
73 return Task.SKIP_ME
74
75 def get_cwd(self):
76 return self.slaves[0].outputs[0].parent
77
78 def batch_incpaths(self):
79 st = self.env.CPPPATH_ST
80 return [st % node.abspath() for node in self.generator.includes_nodes]
81
82 def run(self):
83 self.outputs = []
84
85 srclst = []
86 slaves = []
87 for t in self.slaves:
88 if t.hasrun != Task.SKIPPED:
89 slaves.append(t)
90 srclst.append(t.inputs[0].abspath())
91
92 self.env.SRCLST = srclst
93
94 if self.slaves[0].__class__.__name__ == 'c':
95 ret = c_fun(self)
96 else:
97 ret = cxx_fun(self)
98
99 if ret:
100 return ret
101
102 for t in slaves:
103 t.old_post_run()
104
105 def hook(cls_type):
106 def n_hook(self, node):
107
108 ext = '.obj' if self.env.CC_NAME == 'msvc' else '.o'
109 name = node.name
110 k = name.rfind('.')
111 if k >= 0:
112 basename = name[:k] + ext
113 else:
114 basename = name + ext
115
116 outdir = node.parent.get_bld().make_node('%d' % self.idx)
117 outdir.mkdir()
118 out = outdir.find_or_declare(basename)
119
120 task = self.create_task(cls_type, node, out)
121
122 try:
123 self.compiled_tasks.append(task)
124 except AttributeError:
125 self.compiled_tasks = [task]
126
127 if not getattr(self, 'masters', None):
128 self.masters = {}
129 self.allmasters = []
130
131 def fix_path(tsk):
132 if self.env.CC_NAME == 'msvc':
133 tsk.env.append_unique('CXX_TGT_F_BATCHED', '/Fo%s\\' % outdir.abspath())
134
135 if not node.parent in self.masters:
136 m = self.masters[node.parent] = self.master = self.create_task('batch')
137 fix_path(m)
138 self.allmasters.append(m)
139 else:
140 m = self.masters[node.parent]
141 if len(m.slaves) > MAX_BATCH:
142 m = self.masters[node.parent] = self.master = self.create_task('batch')
143 fix_path(m)
144 self.allmasters.append(m)
145 m.add_slave(task)
146 return task
147 return n_hook
148
149 extension('.c')(hook('c'))
150 extension('.cpp','.cc','.cxx','.C','.c++')(hook('cxx'))
151
152 @feature('cprogram', 'cshlib', 'cstaticlib', 'cxxprogram', 'cxxshlib', 'cxxstlib')
153 @after_method('apply_link')
154 def link_after_masters(self):
155 if getattr(self, 'allmasters', None):
156 for m in self.allmasters:
157 self.link_task.set_run_after(m)
158
159 # Modify the c and cxx task classes - in theory it would be best to
160 # create subclasses and to re-map the c/c++ extensions
161 for x in ('c', 'cxx'):
162 t = Task.classes[x]
163 def run(self):
164 pass
165
166 def post_run(self):
167 pass
168
169 setattr(t, 'oldrun', getattr(t, 'run', None))
170 setattr(t, 'run', run)
171 setattr(t, 'old_post_run', t.post_run)
172 setattr(t, 'post_run', post_run)
173