Debian Codegraph @

testsuite/systemtap.examples/process/mutex-contention.stp

raw
1 #! /usr/bin/env stap
2
3 // NOTE: a popup mutex is one that is seen in contention but the init
4 // of it was not detected not sure why there are popups - static mutex
5 // init?
6
7 // stack trace where mutex was initialized, if known
8 global mutex_init_stack
9 // stack trace where mutex was first referenced, if init stack not known
10 global mutex_uninit
11 // stats about the contention on this mutex
12 global mutex_contention
13 // usymdata for this mutex
14 global mutex_names
15 // stack trace of last contention for this mutex
16 global mutex_last_cont_stack
17
18 // NOTE: the way 389 works is that many common mutexes are initialized
19 // in the same place. For example - each connection object has its
20 // own mutex - there may be thousands of these. They are all
21 // initialized in the same place - same with entry objects, where
22 // there may be millions. So, if we want to get aggregate contention
23 // statistics for _all_ connection objects, not just each individual
24 // one, we have to keep track of each unique call stack. This is what
25 // mutex_cont_stack is used for.
26
27 // stats about unique stacks
28 global mutex_cont_stack
29 // report mutexes that had more than this many contentions
30 global cont_count_threshold = 1
31 // usec - report mutexes that had a max time more than this
32 global cont_max_threshold = 100000
33 global verbose = 0
34 global keep_stats = 1
35
36 /* ... and a variant of the futexes.stp script: */
37 global FUTEX_WAIT = 0 /*, FUTEX_WAKE = 1 */
38 global FUTEX_PRIVATE_FLAG = 128 /* linux 2.6.22+ */
39 global FUTEX_CLOCK_REALTIME = 256 /* linux 2.6.29+ */
40
41 function process_mutex_init(mutex, ppfunc)
42 {
43 if (verbose && (mutex in mutex_init_stack)) {
44 printf("error: %s: mutex %p is already initialized at\n%s\n",
45 ppfunc, mutex, mutex_init_stack[mutex])
46 }
47 if (keep_stats) {
48 mutex_init_stack[mutex] = sprint_ubacktrace()
49 mutex_names[mutex] = usymdata (mutex)
50 }
51 else {
52 printf("init %p at\n%s\n", mutex, sprint_ubacktrace())
53 }
54 if (verbose) {
55 printf("%s: mutex %p %s\n", ppfunc, mutex, mutex_names[mutex])
56 }
57 }
58
59 function show_contention(mutex, stack, type)
60 {
61 count = @count(mutex_contention[mutex])
62 max = @max(mutex_contention[mutex])
63 if ((count > cont_count_threshold) || (max > cont_max_threshold)) {
64 printf("=======================================\nmutex %p (%s) contended %d times, %d avg usec, %d max usec, %d total usec, %s at\n%s\n",
65 mutex, mutex_names[mutex], count, @avg(mutex_contention[mutex]),
66 max, @sum(mutex_contention[mutex]), type, stack)
67 if (mutex in mutex_last_cont_stack) {
68 printf("\nmutex was last contended at\n%s\n",
69 mutex_last_cont_stack[mutex])
70 }
71 }
72 }
73
74 // we use @defined($muex) and @defined($rwlock) here because
75 // on 64 bit systems, the wildcard search takes us through
76 // both 64 bit and 32 bit libc (which doesn't have debuginfo),
77 // this means the probe point resolved from the 32 bit library
78 // has no context info
79 probe process("/lib*/libc.so*").function("pthread_mutex_init")
80 {
81 if(@defined($mutex))
82 process_mutex_init($mutex, ppfunc())
83 }
84 probe process("/lib*/libpthread.so*").function("__pthread_mutex_init")
85 {
86 if(@defined($mutex))
87 process_mutex_init($mutex, ppfunc())
88 }
89 probe process("/lib*/libpthread.so*").function("__pthread_rwlock_init")
90 {
91 if(@defined($rwlock))
92 process_mutex_init($rwlock, ppfunc())
93 }
94
95 global entry_times%, uaddrs%
96
97 probe syscall.futex {
98 if (pid() != target()) next // skip irrelevant processes
99 if ((op & ~(FUTEX_PRIVATE_FLAG|FUTEX_CLOCK_REALTIME)) != FUTEX_WAIT) next
100 entry_times[tid()] = gettimeofday_us()
101 uaddrs[tid()] = futex_uaddr
102 }
103
104 probe syscall.futex.return
105 {
106 if (!(tid() in entry_times)) next
107 entry = entry_times[tid()]
108 delete entry_times[tid()]
109
110 uaddr = uaddrs[tid()]
111 delete uaddrs[tid()]
112
113 elapsed = gettimeofday_us() - entry
114 if (keep_stats) {
115 mutex_contention[uaddr] <<< elapsed
116 stack = sprint_ubacktrace()
117 mutex_last_cont_stack[uaddr] = stack
118 mutex_cont_stack[stack] <<< elapsed
119 if (uaddr in mutex_init_stack) {
120 if (verbose) {
121 printf("contention time %d on mutex %p initialized at\n%s\n",
122 elapsed, uaddr, mutex_init_stack[uaddr])
123 }
124 }
125 else if (uaddr in mutex_uninit) {
126 if (verbose) {
127 printf("contention time %d on popup mutex %p at\n%s\n",
128 elapsed, uaddr, stack)
129 }
130 }
131 else {
132 if (verbose) {
133 printf("contention time %d on popup mutex %p at\n%s\n",
134 elapsed, uaddr, stack)
135 }
136 mutex_uninit[uaddr] = stack
137 mutex_names[uaddr] = usymdata (uaddr)
138 }
139 }
140 else {
141 printf("contention %p elapsed %d at\n%s\n", uaddr, elapsed,
142 sprint_ubacktrace())
143 }
144 }
145
146 probe end
147 {
148 if (!keep_stats) {
149 printf("======== END\n")
150 next
151 }
152 printf("<<<<<<<< aggregate stats\n")
153 foreach (stack in mutex_cont_stack-) {
154 count = @count(mutex_cont_stack[stack])
155 max = @max(mutex_cont_stack[stack])
156 if ((count > cont_count_threshold) || (max > cont_max_threshold)) {
157 printf("=======================================\nstack contended %d times, %d avg usec, %d max usec, %d total usec, at\n%s\n",
158 @count(mutex_cont_stack[stack]), @avg(mutex_cont_stack[stack]),
159 @max(mutex_cont_stack[stack]), @sum(mutex_cont_stack[stack]),
160 stack)
161 }
162 }
163 printf(">>>>>>>> aggregate stats\n")
164 foreach (mutex in mutex_contention-) {
165 if (mutex in mutex_init_stack) {
166 stack = mutex_init_stack[mutex]
167 type = "init"
168 }
169 else if (mutex in mutex_uninit) {
170 stack = mutex_uninit[mutex]
171 type = "popup"
172 }
173 show_contention(mutex, stack, type)
174 }
175 }