Debian Codegraph @

Tools/ftscalingbench/ftscalingbench.py

raw
1 # This script runs a set of small benchmarks to help identify scaling
2 # bottlenecks in the free-threaded interpreter. The benchmarks consist
3 # of patterns that ought to scale well, but haven't in the past. This is
4 # typically due to reference count contention or lock contention.
5 #
6 # This is not intended to be a general multithreading benchmark suite, nor
7 # are the benchmarks intended to be representative of real-world workloads.
8 #
9 # On Linux, to avoid confounding hardware effects, the script attempts to:
10 # * Use a single CPU socket (to avoid NUMA effects)
11 # * Use distinct physical cores (to avoid hyperthreading/SMT effects)
12 # * Use "performance" cores (Intel, ARM) on CPUs that have performance and
13 # efficiency cores
14 #
15 # It also helps to disable dynamic frequency scaling (i.e., "Turbo Boost")
16 #
17 # Intel:
18 # > echo "1" | sudo tee /sys/devices/system/cpu/intel_pstate/no_turbo
19 #
20 # AMD:
21 # > echo "0" | sudo tee /sys/devices/system/cpu/cpufreq/boost
22 #
23
24 import math
25 import os
26 import queue
27 import sys
28 import threading
29 import time
30 from dataclasses import dataclass
31
32 # The iterations in individual benchmarks are scaled by this factor.
33 WORK_SCALE = 100
34
35 ALL_BENCHMARKS = {}
36
37 threads = []
38 in_queues = []
39 out_queues = []
40
41
42 def register_benchmark(func):
43 ALL_BENCHMARKS[func.__name__] = func
44 return func
45
46 @register_benchmark
47 def object_cfunction():
48 accu = 0
49 tab = [1] * 100
50 for i in range(1000 * WORK_SCALE):
51 tab.pop(0)
52 tab.append(i)
53 accu += tab[50]
54 return accu
55
56 @register_benchmark
57 def cmodule_function():
58 N = 1000 * WORK_SCALE
59 for i in range(N):
60 math.cos(i / N)
61
62 @register_benchmark
63 def object_lookup_special():
64 # round() uses `_PyObject_LookupSpecial()` internally.
65 N = 1000 * WORK_SCALE
66 for i in range(N):
67 round(i / N)
68
69 class MyContextManager:
70 def __enter__(self):
71 pass
72 def __exit__(self, exc_type, exc_value, traceback):
73 pass
74
75 @register_benchmark
76 def context_manager():
77 N = 1000 * WORK_SCALE
78 for i in range(N):
79 with MyContextManager():
80 pass
81
82 @register_benchmark
83 def mult_constant():
84 x = 1.0
85 for i in range(3000 * WORK_SCALE):
86 x *= 1.01
87
88 def simple_gen():
89 for i in range(10):
90 yield i
91
92 @register_benchmark
93 def generator():
94 accu = 0
95 for i in range(100 * WORK_SCALE):
96 for v in simple_gen():
97 accu += v
98 return accu
99
100 class Counter:
101 def __init__(self):
102 self.i = 0
103
104 def next_number(self):
105 self.i += 1
106 return self.i
107
108 @register_benchmark
109 def pymethod():
110 c = Counter()
111 for i in range(1000 * WORK_SCALE):
112 c.next_number()
113 return c.i
114
115 def next_number(i):
116 return i + 1
117
118 @register_benchmark
119 def pyfunction():
120 accu = 0
121 for i in range(1000 * WORK_SCALE):
122 accu = next_number(i)
123 return accu
124
125 def double(x):
126 return x + x
127
128 module = sys.modules[__name__]
129
130 @register_benchmark
131 def module_function():
132 total = 0
133 for i in range(1000 * WORK_SCALE):
134 total += module.double(i)
135 return total
136
137 class MyObject:
138 pass
139
140 @register_benchmark
141 def load_string_const():
142 accu = 0
143 for i in range(1000 * WORK_SCALE):
144 if i == 'a string':
145 accu += 7
146 else:
147 accu += 1
148 return accu
149
150 @register_benchmark
151 def load_tuple_const():
152 accu = 0
153 for i in range(1000 * WORK_SCALE):
154 if i == (1, 2):
155 accu += 7
156 else:
157 accu += 1
158 return accu
159
160 @register_benchmark
161 def create_pyobject():
162 for i in range(1000 * WORK_SCALE):
163 o = MyObject()
164
165 @register_benchmark
166 def create_closure():
167 for i in range(1000 * WORK_SCALE):
168 def foo(x):
169 return x
170 foo(i)
171
172 @register_benchmark
173 def create_dict():
174 for i in range(1000 * WORK_SCALE):
175 d = {
176 "key": "value",
177 }
178
179 thread_local = threading.local()
180
181 @register_benchmark
182 def thread_local_read():
183 tmp = thread_local
184 tmp.x = 10
185 for i in range(500 * WORK_SCALE):
186 _ = tmp.x
187 _ = tmp.x
188 _ = tmp.x
189 _ = tmp.x
190 _ = tmp.x
191
192
193 @dataclass
194 class MyDataClass:
195 x: int
196 y: int
197 z: int
198
199 @register_benchmark
200 def instantiate_dataclass():
201 for _ in range(1000 * WORK_SCALE):
202 obj = MyDataClass(x=1, y=2, z=3)
203
204 @register_benchmark
205 def super_call():
206 # TODO: super() on the same class from multiple threads still doesn't
207 # scale well, so use a class per-thread here for now.
208 class Base:
209 def method(self):
210 return 1
211
212 class Derived(Base):
213 def method(self):
214 return super().method()
215
216 obj = Derived()
217 for _ in range(1000 * WORK_SCALE):
218 obj.method()
219
220
221 class MyClassMethod:
222 @classmethod
223 def my_classmethod(cls):
224 return cls
225
226 @staticmethod
227 def my_staticmethod():
228 pass
229
230 @register_benchmark
231 def classmethod_call():
232 obj = MyClassMethod()
233 for _ in range(1000 * WORK_SCALE):
234 obj.my_classmethod()
235
236 @register_benchmark
237 def staticmethod_call():
238 obj = MyClassMethod()
239 for _ in range(1000 * WORK_SCALE):
240 obj.my_staticmethod()
241
242 @register_benchmark
243 def setattr_non_interned():
244 prefix = "prefix"
245 obj = MyObject()
246 for _ in range(1000 * WORK_SCALE):
247 setattr(obj, f"{prefix}_a", None)
248 setattr(obj, f"{prefix}_b", None)
249 setattr(obj, f"{prefix}_c", None)
250
251
252 def bench_one_thread(func):
253 t0 = time.perf_counter_ns()
254 func()
255 t1 = time.perf_counter_ns()
256 return t1 - t0
257
258
259 def bench_parallel(func):
260 t0 = time.perf_counter_ns()
261 for inq in in_queues:
262 inq.put(func)
263 for outq in out_queues:
264 outq.get()
265 t1 = time.perf_counter_ns()
266 return t1 - t0
267
268
269 def benchmark(func):
270 delta_one_thread = bench_one_thread(func)
271 delta_many_threads = bench_parallel(func)
272
273 speedup = delta_one_thread * len(threads) / delta_many_threads
274 if speedup >= 1:
275 factor = speedup
276 direction = "faster"
277 else:
278 factor = 1 / speedup
279 direction = "slower"
280
281 use_color = hasattr(sys.stdout, 'isatty') and sys.stdout.isatty()
282 color = reset_color = ""
283 if use_color:
284 if speedup <= 1.1:
285 color = "\x1b[31m" # red
286 elif speedup < len(threads)/2:
287 color = "\x1b[33m" # yellow
288 reset_color = "\x1b[0m"
289
290 print(f"{color}{func.__name__:<25} {round(factor, 1):>4}x {direction}{reset_color}")
291
292 def determine_num_threads_and_affinity():
293 if sys.platform != "linux":
294 return [None] * os.cpu_count()
295
296 # Try to use `lscpu -p` on Linux
297 import subprocess
298 try:
299 output = subprocess.check_output(["lscpu", "-p=cpu,node,core,MAXMHZ"],
300 text=True, env={"LC_NUMERIC": "C"})
301 except (FileNotFoundError, subprocess.CalledProcessError):
302 return [None] * os.cpu_count()
303
304 table = []
305 for line in output.splitlines():
306 if line.startswith("#"):
307 continue
308 cpu, node, core, maxhz = line.split(",")
309 if maxhz == "":
310 maxhz = "0"
311 table.append((int(cpu), int(node), int(core), float(maxhz)))
312
313 cpus = []
314 cores = set()
315 max_mhz_all = max(row[3] for row in table)
316 for cpu, node, core, maxmhz in table:
317 # Choose only CPUs on the same node, unique cores, and try to avoid
318 # "efficiency" cores.
319 if node == 0 and core not in cores and maxmhz == max_mhz_all:
320 cpus.append(cpu)
321 cores.add(core)
322 return cpus
323
324
325 def thread_run(cpu, in_queue, out_queue):
326 if cpu is not None and hasattr(os, "sched_setaffinity"):
327 # Set the affinity for the current thread
328 os.sched_setaffinity(0, (cpu,))
329
330 while True:
331 func = in_queue.get()
332 if func is None:
333 break
334 func()
335 out_queue.put(None)
336
337
338 def initialize_threads(opts):
339 if opts.threads == -1:
340 cpus = determine_num_threads_and_affinity()
341 else:
342 cpus = [None] * opts.threads # don't set affinity
343
344 print(f"Running benchmarks with {len(cpus)} threads")
345 for cpu in cpus:
346 inq = queue.Queue()
347 outq = queue.Queue()
348 in_queues.append(inq)
349 out_queues.append(outq)
350 t = threading.Thread(target=thread_run, args=(cpu, inq, outq), daemon=True)
351 threads.append(t)
352 t.start()
353
354
355 def main(opts):
356 global WORK_SCALE
357 if not hasattr(sys, "_is_gil_enabled") or sys._is_gil_enabled():
358 sys.stderr.write("expected to be run with the GIL disabled\n")
359
360 benchmark_names = opts.benchmarks
361 if benchmark_names:
362 for name in benchmark_names:
363 if name not in ALL_BENCHMARKS:
364 sys.stderr.write(f"Unknown benchmark: {name}\n")
365 sys.exit(1)
366 else:
367 benchmark_names = ALL_BENCHMARKS.keys()
368
369 WORK_SCALE = opts.scale
370
371 if not opts.baseline_only:
372 initialize_threads(opts)
373
374 do_bench = not opts.baseline_only and not opts.parallel_only
375 for name in benchmark_names:
376 func = ALL_BENCHMARKS[name]
377 if do_bench:
378 benchmark(func)
379 continue
380
381 if opts.parallel_only:
382 delta_ns = bench_parallel(func)
383 else:
384 delta_ns = bench_one_thread(func)
385
386 time_ms = delta_ns / 1_000_000
387 print(f"{func.__name__:<18} {time_ms:.1f} ms")
388
389
390 if __name__ == "__main__":
391 import argparse
392
393 parser = argparse.ArgumentParser()
394 parser.add_argument("-t", "--threads", type=int, default=-1,
395 help="number of threads to use")
396 parser.add_argument("--scale", type=int, default=100,
397 help="work scale factor for the benchmark (default=100)")
398 parser.add_argument("--baseline-only", default=False, action="store_true",
399 help="only run the baseline benchmarks (single thread)")
400 parser.add_argument("--parallel-only", default=False, action="store_true",
401 help="only run the parallel benchmark (many threads)")
402 parser.add_argument("benchmarks", nargs="*",
403 help="benchmarks to run")
404 options = parser.parse_args()
405 main(options)