| 1 |
#! /usr/bin/env stap |
| 2 |
|
| 3 |
// NOTE: a popup mutex is one that is seen in contention but the init |
| 4 |
// of it was not detected not sure why there are popups - static mutex |
| 5 |
// init? |
| 6 |
|
| 7 |
// stack trace where mutex was initialized, if known |
| 8 |
global mutex_init_stack |
| 9 |
// stack trace where mutex was first referenced, if init stack not known |
| 10 |
global mutex_uninit |
| 11 |
// stats about the contention on this mutex |
| 12 |
global mutex_contention |
| 13 |
// usymdata for this mutex |
| 14 |
global mutex_names |
| 15 |
// stack trace of last contention for this mutex |
| 16 |
global mutex_last_cont_stack |
| 17 |
|
| 18 |
// NOTE: the way 389 works is that many common mutexes are initialized |
| 19 |
// in the same place. For example - each connection object has its |
| 20 |
// own mutex - there may be thousands of these. They are all |
| 21 |
// initialized in the same place - same with entry objects, where |
| 22 |
// there may be millions. So, if we want to get aggregate contention |
| 23 |
// statistics for _all_ connection objects, not just each individual |
| 24 |
// one, we have to keep track of each unique call stack. This is what |
| 25 |
// mutex_cont_stack is used for. |
| 26 |
|
| 27 |
// stats about unique stacks |
| 28 |
global mutex_cont_stack |
| 29 |
// report mutexes that had more than this many contentions |
| 30 |
global cont_count_threshold = 1 |
| 31 |
// usec - report mutexes that had a max time more than this |
| 32 |
global cont_max_threshold = 100000 |
| 33 |
global verbose = 0 |
| 34 |
global keep_stats = 1 |
| 35 |
|
| 36 |
/* ... and a variant of the futexes.stp script: */ |
| 37 |
global FUTEX_WAIT = 0 /*, FUTEX_WAKE = 1 */ |
| 38 |
global FUTEX_PRIVATE_FLAG = 128 /* linux 2.6.22+ */ |
| 39 |
global FUTEX_CLOCK_REALTIME = 256 /* linux 2.6.29+ */ |
| 40 |
|
| 41 |
function process_mutex_init(mutex, ppfunc) |
| 42 |
{ |
| 43 |
if (verbose && (mutex in mutex_init_stack)) { |
| 44 |
printf("error: %s: mutex %p is already initialized at\n%s\n", |
| 45 |
ppfunc, mutex, mutex_init_stack[mutex]) |
| 46 |
} |
| 47 |
if (keep_stats) { |
| 48 |
mutex_init_stack[mutex] = sprint_ubacktrace() |
| 49 |
mutex_names[mutex] = usymdata (mutex) |
| 50 |
} |
| 51 |
else { |
| 52 |
printf("init %p at\n%s\n", mutex, sprint_ubacktrace()) |
| 53 |
} |
| 54 |
if (verbose) { |
| 55 |
printf("%s: mutex %p %s\n", ppfunc, mutex, mutex_names[mutex]) |
| 56 |
} |
| 57 |
} |
| 58 |
|
| 59 |
function show_contention(mutex, stack, type) |
| 60 |
{ |
| 61 |
count = @count(mutex_contention[mutex]) |
| 62 |
max = @max(mutex_contention[mutex]) |
| 63 |
if ((count > cont_count_threshold) || (max > cont_max_threshold)) { |
| 64 |
printf("=======================================\nmutex %p (%s) contended %d times, %d avg usec, %d max usec, %d total usec, %s at\n%s\n", |
| 65 |
mutex, mutex_names[mutex], count, @avg(mutex_contention[mutex]), |
| 66 |
max, @sum(mutex_contention[mutex]), type, stack) |
| 67 |
if (mutex in mutex_last_cont_stack) { |
| 68 |
printf("\nmutex was last contended at\n%s\n", |
| 69 |
mutex_last_cont_stack[mutex]) |
| 70 |
} |
| 71 |
} |
| 72 |
} |
| 73 |
|
| 74 |
// we use @defined($muex) and @defined($rwlock) here because |
| 75 |
// on 64 bit systems, the wildcard search takes us through |
| 76 |
// both 64 bit and 32 bit libc (which doesn't have debuginfo), |
| 77 |
// this means the probe point resolved from the 32 bit library |
| 78 |
// has no context info |
| 79 |
probe process("/lib*/libc.so*").function("pthread_mutex_init") |
| 80 |
{ |
| 81 |
if(@defined($mutex)) |
| 82 |
process_mutex_init($mutex, ppfunc()) |
| 83 |
} |
| 84 |
probe process("/lib*/libpthread.so*").function("__pthread_mutex_init") |
| 85 |
{ |
| 86 |
if(@defined($mutex)) |
| 87 |
process_mutex_init($mutex, ppfunc()) |
| 88 |
} |
| 89 |
probe process("/lib*/libpthread.so*").function("__pthread_rwlock_init") |
| 90 |
{ |
| 91 |
if(@defined($rwlock)) |
| 92 |
process_mutex_init($rwlock, ppfunc()) |
| 93 |
} |
| 94 |
|
| 95 |
global entry_times%, uaddrs% |
| 96 |
|
| 97 |
probe syscall.futex { |
| 98 |
if (pid() != target()) next // skip irrelevant processes |
| 99 |
if ((op & ~(FUTEX_PRIVATE_FLAG|FUTEX_CLOCK_REALTIME)) != FUTEX_WAIT) next |
| 100 |
entry_times[tid()] = gettimeofday_us() |
| 101 |
uaddrs[tid()] = futex_uaddr |
| 102 |
} |
| 103 |
|
| 104 |
probe syscall.futex.return |
| 105 |
{ |
| 106 |
if (!(tid() in entry_times)) next |
| 107 |
entry = entry_times[tid()] |
| 108 |
delete entry_times[tid()] |
| 109 |
|
| 110 |
uaddr = uaddrs[tid()] |
| 111 |
delete uaddrs[tid()] |
| 112 |
|
| 113 |
elapsed = gettimeofday_us() - entry |
| 114 |
if (keep_stats) { |
| 115 |
mutex_contention[uaddr] <<< elapsed |
| 116 |
stack = sprint_ubacktrace() |
| 117 |
mutex_last_cont_stack[uaddr] = stack |
| 118 |
mutex_cont_stack[stack] <<< elapsed |
| 119 |
if (uaddr in mutex_init_stack) { |
| 120 |
if (verbose) { |
| 121 |
printf("contention time %d on mutex %p initialized at\n%s\n", |
| 122 |
elapsed, uaddr, mutex_init_stack[uaddr]) |
| 123 |
} |
| 124 |
} |
| 125 |
else if (uaddr in mutex_uninit) { |
| 126 |
if (verbose) { |
| 127 |
printf("contention time %d on popup mutex %p at\n%s\n", |
| 128 |
elapsed, uaddr, stack) |
| 129 |
} |
| 130 |
} |
| 131 |
else { |
| 132 |
if (verbose) { |
| 133 |
printf("contention time %d on popup mutex %p at\n%s\n", |
| 134 |
elapsed, uaddr, stack) |
| 135 |
} |
| 136 |
mutex_uninit[uaddr] = stack |
| 137 |
mutex_names[uaddr] = usymdata (uaddr) |
| 138 |
} |
| 139 |
} |
| 140 |
else { |
| 141 |
printf("contention %p elapsed %d at\n%s\n", uaddr, elapsed, |
| 142 |
sprint_ubacktrace()) |
| 143 |
} |
| 144 |
} |
| 145 |
|
| 146 |
probe end |
| 147 |
{ |
| 148 |
if (!keep_stats) { |
| 149 |
printf("======== END\n") |
| 150 |
next |
| 151 |
} |
| 152 |
printf("<<<<<<<< aggregate stats\n") |
| 153 |
foreach (stack in mutex_cont_stack-) { |
| 154 |
count = @count(mutex_cont_stack[stack]) |
| 155 |
max = @max(mutex_cont_stack[stack]) |
| 156 |
if ((count > cont_count_threshold) || (max > cont_max_threshold)) { |
| 157 |
printf("=======================================\nstack contended %d times, %d avg usec, %d max usec, %d total usec, at\n%s\n", |
| 158 |
@count(mutex_cont_stack[stack]), @avg(mutex_cont_stack[stack]), |
| 159 |
@max(mutex_cont_stack[stack]), @sum(mutex_cont_stack[stack]), |
| 160 |
stack) |
| 161 |
} |
| 162 |
} |
| 163 |
printf(">>>>>>>> aggregate stats\n") |
| 164 |
foreach (mutex in mutex_contention-) { |
| 165 |
if (mutex in mutex_init_stack) { |
| 166 |
stack = mutex_init_stack[mutex] |
| 167 |
type = "init" |
| 168 |
} |
| 169 |
else if (mutex in mutex_uninit) { |
| 170 |
stack = mutex_uninit[mutex] |
| 171 |
type = "popup" |
| 172 |
} |
| 173 |
show_contention(mutex, stack, type) |
| 174 |
} |
| 175 |
} |