| 1 |
#! /usr/bin/env stap |
| 2 |
# |
| 3 |
# Copyright (C) 2010 Red Hat, Inc. |
| 4 |
# Written by William Cohen <[email protected]> |
| 5 |
# |
| 6 |
# This script records the time that a process has spent asleep and attempt |
| 7 |
# to provide a reason for that that sleep. The script needs to be run with the |
| 8 |
# '--all-modules' option to map the backtrace addresses to function names |
| 9 |
# and the function names back to reasons. |
| 10 |
# |
| 11 |
# stap --all-modules latencytap.stp |
| 12 |
|
| 13 |
global priority, reason; |
| 14 |
global traced_pid, sleep_time, dequeue; |
| 15 |
global this_sleep; |
| 16 |
global debug = 0; |
| 17 |
global reason_len; |
| 18 |
|
| 19 |
function log_event:long (p:long) |
| 20 |
{ |
| 21 |
return (!traced_pid || traced_pid == task_pid(p)) |
| 22 |
} |
| 23 |
|
| 24 |
#func names from hex addresses |
| 25 |
function func_backtrace:string (ips:string) |
| 26 |
{ |
| 27 |
ret = ""; |
| 28 |
for (ip=tokenize(ips, " "); ip!=""; ip=tokenize("", " ")) |
| 29 |
ret = ret . symname(strtol(ip,16)) . " "; |
| 30 |
return ret; |
| 31 |
} |
| 32 |
|
| 33 |
# pick out a reason based on the backtrace function names |
| 34 |
function translate:string (bt:string) |
| 35 |
{ |
| 36 |
ip=tokenize(bt, " "); |
| 37 |
func = symname(strtol(ip,16)) |
| 38 |
pri = 1; rea=""; |
| 39 |
while (ip!="") { |
| 40 |
p = priority[func] |
| 41 |
if (p >= pri) { |
| 42 |
pri = priority[func]; |
| 43 |
rea = reason[func]; |
| 44 |
} |
| 45 |
ip=tokenize("", " ") |
| 46 |
func = symname(strtol(ip,16)) |
| 47 |
} |
| 48 |
return rea; |
| 49 |
} |
| 50 |
|
| 51 |
probe kernel.trace("deactivate_task") !, |
| 52 |
kernel.function("deactivate_task") { |
| 53 |
s = task_state($p) |
| 54 |
# check to see if task is in appropriate state: |
| 55 |
# TASK_INTERRUPTIBLE 1 |
| 56 |
# TASK_UNINTERRUPTIBLE 2 |
| 57 |
if (log_event($p) && (s & 3)) { |
| 58 |
dequeue[$p] = gettimeofday_us(); |
| 59 |
} |
| 60 |
} |
| 61 |
|
| 62 |
probe kernel.trace("activate_task") !, |
| 63 |
kernel.function("activate_task") { |
| 64 |
if (!log_event($p)) next |
| 65 |
|
| 66 |
a = gettimeofday_us() |
| 67 |
d = dequeue[$p] |
| 68 |
delete dequeue[$p] |
| 69 |
if (d) { |
| 70 |
sleep = a - d |
| 71 |
if (sleep > 0) { |
| 72 |
this_sleep[$p] = sleep |
| 73 |
# sleep_time[task_backtrace($p)] <<< sleep |
| 74 |
} |
| 75 |
} |
| 76 |
} |
| 77 |
|
| 78 |
#FIXME: It would be nicer to get backtrace information in activate_task probe. |
| 79 |
# This would eliminate the overhead of probing every context switch |
| 80 |
# and this_sleep associate array. However, this needs a properly working |
| 81 |
# task_backtrace() to eliminate this probe. |
| 82 |
probe scheduler.cpu_on { |
| 83 |
p = task_current() |
| 84 |
t = this_sleep[p] |
| 85 |
if (t){ |
| 86 |
delete this_sleep[p] |
| 87 |
sleep_time[backtrace()] <<< t |
| 88 |
} |
| 89 |
} |
| 90 |
|
| 91 |
global sort_latencies; |
| 92 |
|
| 93 |
function global_report_latencies() { |
| 94 |
total = 0 |
| 95 |
|
| 96 |
foreach ([backtrace] in sleep_time) { |
| 97 |
val = @sum(sleep_time[backtrace]); |
| 98 |
sort_latencies[backtrace] = val; |
| 99 |
total += val; |
| 100 |
} |
| 101 |
|
| 102 |
ansi_clear_screen(); |
| 103 |
printf("%-*s %12s %12s %12s %7s%%\n", |
| 104 |
@max(reason_len)+1, "Reason", "Count", "Average(us)", "Maximum(us)", "Percent"); |
| 105 |
foreach ([backtrace] in sort_latencies- limit 20) { |
| 106 |
#print reason count average max percentage |
| 107 |
cause = translate (backtrace); |
| 108 |
# if can't find reason print out a backtrace |
| 109 |
if (cause == "" && debug) |
| 110 |
printf("%s\n", func_backtrace(backtrace)); |
| 111 |
printf("%-*s ", @max(reason_len)+1, cause); |
| 112 |
p = (@sum(sleep_time[backtrace])*100)/total; |
| 113 |
printf("%12d %12d %12d %7d%%\n", |
| 114 |
@count(sleep_time[backtrace]), |
| 115 |
@avg(sleep_time[backtrace]), |
| 116 |
@max(sleep_time[backtrace]), |
| 117 |
p); |
| 118 |
} |
| 119 |
delete sort_latencies; |
| 120 |
} |
| 121 |
|
| 122 |
probe timer.s(30) { |
| 123 |
global_report_latencies() |
| 124 |
delete sleep_time |
| 125 |
} |
| 126 |
|
| 127 |
probe begin { |
| 128 |
traced_pid = target() |
| 129 |
} |
| 130 |
|
| 131 |
# Set up the translations and priorities for each function. |
| 132 |
# The vast majority of translations were mechanically generated from the |
| 133 |
# latencytop (http://www.latencytop.org/) translation file: |
| 134 |
# /usr/share/latencytop/latencytop.trans . |
| 135 |
|
| 136 |
function explain(symbol,pri,blah) { |
| 137 |
priority[symbol] = pri |
| 138 |
reason[symbol] = blah |
| 139 |
reason_len <<< strlen(blah) |
| 140 |
} |
| 141 |
|
| 142 |
probe begin { |
| 143 |
explain("common_nsleep",3,"Nanosleep delay"); |
| 144 |
explain("SyS_epoll_wait",3,"Waiting for event (epoll)"); |
| 145 |
explain("SyS_flock",3,"flock() on a file"); |
| 146 |
explain("SyS_nanosleep",5,"Application requested delay"); |
| 147 |
explain("SyS_pause",5,"Application requested delay"); |
| 148 |
explain("__bread",2,"Synchronous buffer read"); |
| 149 |
explain("__ext3_get_inode_loc",3,"Reading EXT3 inode"); |
| 150 |
explain("__log_wait_for_space",5,"Waiting for EXT3 journal space"); |
| 151 |
explain("__mark_inode_dirty",1,"Marking inode dirty"); |
| 152 |
explain("__pollwait",2,"Waiting for event (poll)"); |
| 153 |
explain("__sys_accept4",2,"Waiting for a connection on a socket (accept/accept4)"); |
| 154 |
explain("__stop_machine_run",3,"Freezing the kernel (for module load)"); |
| 155 |
explain("__wait_on_buffer",2,"Waiting for buffer IO to complete"); |
| 156 |
explain("acpi_ec_transaction",3,"ACPI hardware access"); |
| 157 |
explain("acquire_console_sem",3,"Waiting for console access"); |
| 158 |
explain("alloc_page_vma",3,"Allocating a VMA"); |
| 159 |
explain("ata_exec_internal_sg",3,"Executing internal ATA command"); |
| 160 |
explain("ata_scsi_ioctl",3,"ATA/SCSI disk ioctl"); |
| 161 |
explain("bdi_forker_task",2,"bdi-* kernel thread"); |
| 162 |
explain("bdi_sync_supers",1,"Sync superblock"); |
| 163 |
explain("bdi_writeback_task",2,"Flush data to backing device"); |
| 164 |
explain("bdi_writeback_thread",2,"Flush data to backing dev"); |
| 165 |
explain("blk_execute_rq",3,"Submitting block IO"); |
| 166 |
explain("blkdev_direct_IO",3,"Direct block device IO"); |
| 167 |
explain("blkdev_ioctl",2,"block device IOCTL"); |
| 168 |
explain("blocking_notifier_call_chain",1,"Blocking notifier"); |
| 169 |
explain("btrfs_sync_file",2,"btrfs sync file"); |
| 170 |
explain("cdrom_open",3,"opening cdrom device"); |
| 171 |
explain("cleaner_kthread",3,"btrfs cleaner_kthread()"); |
| 172 |
explain("core_sys_select",2,"Waiting for event (select)"); |
| 173 |
explain("dm_wait_for_completion",2,"Waiting for device mapper ops"); |
| 174 |
explain("do_epoll_wait",3,"Waiting for event (epoll)"); |
| 175 |
explain("do_fork",3,"Fork() system call"); |
| 176 |
explain("do_fsync",5,"fsync() on a file"); |
| 177 |
explain("do_futex",3,"Userspace lock contention"); |
| 178 |
explain("do_generic_mapping_read",2,"Reading file data"); |
| 179 |
explain("do_page_fault",5,"Page fault"); |
| 180 |
explain("do_rmdir",3,"Removing directory"); |
| 181 |
explain("do_sync_write",2,"synchronous write"); |
| 182 |
explain("do_sys_open",2,"Opening file"); |
| 183 |
explain("do_sys_poll",2,"Waiting for event (poll)"); |
| 184 |
explain("do_truncate",3,"Truncating file"); |
| 185 |
explain("do_unlinkat",2,"Unlinking file"); |
| 186 |
explain("do_writepages",2,"Writing back dirty pages"); |
| 187 |
explain("evdev_read",5,"Reading keyboard/mouse input"); |
| 188 |
explain("ext3_bread",3,"Synchronous EXT3 read"); |
| 189 |
explain("ext3_create",3,"EXT3 Creating a file"); |
| 190 |
explain("ext3_delete_inode",3,"EXT3 deleting inode"); |
| 191 |
explain("ext3_find_entry",3,"EXT3: Looking for file"); |
| 192 |
explain("ext3_free_branches",3,"Unlinking file on EXT3"); |
| 193 |
explain("ext3_get_branch",3,"Reading EXT3 indirect blocks"); |
| 194 |
explain("ext3_mark_inode_dirty ",3,"EXT3 marking inode dirty"); |
| 195 |
explain("ext3_mkdir",3,"EXT3: Creating a directory"); |
| 196 |
explain("ext3_orphan_add ",3,"EXT3 adding orphan"); |
| 197 |
explain("ext3_read_inode",3,"Reading EXT3 inode"); |
| 198 |
explain("ext3_readdir",3,"Reading EXT3 directory"); |
| 199 |
explain("ext3_unlink ",3,"EXT3 unlinking file"); |
| 200 |
explain("filemap_fault",5,"Page fault"); |
| 201 |
explain("filp_close",3,"Closing a file"); |
| 202 |
explain("generic_delete_inode",3,"Deleting an inode"); |
| 203 |
explain("generic_make_request",3,"Creating block layer request"); |
| 204 |
explain("get_request_wait",3,"Creating block layer request"); |
| 205 |
explain("handle_mm_fault",5,"Page fault"); |
| 206 |
explain("htree_dirblock_to_tree",3,"Reading EXT3 directory htree"); |
| 207 |
explain("i915_gem_throttle_ioctl",3,"Throttling GPU while waiting for commands"); |
| 208 |
explain("i915_irq_wait",3,"Waiting for GPU interrupt"); |
| 209 |
explain("i915_wait_request",2,"Waiting for GPU command to complete"); |
| 210 |
explain("inet_sendmsg",3,"Sending TCP/IP data"); |
| 211 |
explain("inotify_inode_queue_event",3,"Inotify event"); |
| 212 |
explain("input_close_device",3,"Closing input device"); |
| 213 |
explain("journal_commit_transaction",3,"EXT3: committing transaction"); |
| 214 |
explain("journal_get_undo_access",3,"Waiting for EXT3 journal undo operation"); |
| 215 |
explain("journal_get_write_access",3,"EXT3: Waiting for journal access"); |
| 216 |
explain("kauditd_thread",2,"kernel audit daemon"); |
| 217 |
explain("kcompactd",2,"kcompactd() kernel thread"); |
| 218 |
explain("khugepaged",2,"khugepaged() kernel thread"); |
| 219 |
explain("kjournald",2,"kjournald() kernel thread"); |
| 220 |
explain("kjournald2",2,"jdb2() kernel thread"); |
| 221 |
explain("kmsg_read",3,"Reading from dmesg"); |
| 222 |
explain("ksoftirqd",2,"Waking ksoftirqd"); |
| 223 |
explain("kswapd",2,"kswapd() kernel thread"); |
| 224 |
explain("kthreadd",2,"kthreadd kernel thread"); |
| 225 |
explain("kvm_vcpu_ioctl",3,"KVM ioctl"); |
| 226 |
explain("link_path_walk",3,"Following symlink"); |
| 227 |
explain("lock_kernel",3,"Big Kernel Lock contention"); |
| 228 |
explain("lock_super",1,"Superblock lock contention"); |
| 229 |
explain("log_do_checkpoint",3,"EXT3 journal checkpoint"); |
| 230 |
explain("lookup_create",3,"Creating file"); |
| 231 |
explain("md_thread",2,"Raid resync kernel thread"); |
| 232 |
explain("migration_thread",2,"migration() kernel thread"); |
| 233 |
explain("mousedev_close_device",3,"Closing mouse device"); |
| 234 |
explain("mousedev_open",3,"Opening mouse device"); |
| 235 |
explain("mousedev_release",3,"Closing mouse device"); |
| 236 |
explain("page_cache_sync_readahead",3,"Pagecache sync readahead"); |
| 237 |
explain("path_walk",3,"Walking directory tree"); |
| 238 |
explain("pdflush",2,"pdflush() kernel thread"); |
| 239 |
explain("pipe_read",3,"Reading from a pipe"); |
| 240 |
explain("pipe_wait",3,"Waiting for pipe data"); |
| 241 |
explain("pipe_write",3,"Writing to a pipe"); |
| 242 |
explain("proc_delete_inode",3,"Removing /proc file"); |
| 243 |
explain("proc_lookup",3,"Looking up /proc file"); |
| 244 |
explain("proc_reg_read",2,"Reading from /proc file"); |
| 245 |
explain("proc_root_readdir",2,"Reading /proc directory"); |
| 246 |
explain("psi_poll_worker",3,"Pressure stall information (psimon) kthread"); |
| 247 |
explain("rcu_gp_kthread",2,"RCU grace period kernel thread"); |
| 248 |
explain("read_block_bitmap",3,"Reading EXT3 block bitmaps"); |
| 249 |
explain("request_module",2,"Loading a kernel module"); |
| 250 |
explain("run_ksoftirqd",2,"Waking ksoftirqd"); |
| 251 |
explain("scsi_cmd_ioctl",3,"SCSI ioctl command"); |
| 252 |
explain("scsi_error_handler",2,"SCSI error handler"); |
| 253 |
explain("scsi_execute_req",3,"Executing raw SCSI command"); |
| 254 |
explain("sd_ioctl",3,"SCSI disk ioctl"); |
| 255 |
explain("shrink_icache_memory",3,"reducing inode cache memory footprint"); |
| 256 |
explain("sigsuspend",3,"waiting for a signal"); |
| 257 |
explain("smpboot_thread_fn",2,"percpu hotplug kernel thread"); |
| 258 |
explain("sock_sendmsg",2,"Sending data over socket"); |
| 259 |
explain("sr_cd_check",3,"Checking CDROM media present"); |
| 260 |
explain("sr_do_ioctl",3,"SCSI cdrom ioctl"); |
| 261 |
explain("sr_media_change",3,"Checking for media change"); |
| 262 |
explain("sync_buffer",3,"Writing buffer to disk (synchronous)"); |
| 263 |
explain("sync_filesystems",5,"Syncing filesystem"); |
| 264 |
explain("sync_inode",3,"(f)syncing an inode to disk"); |
| 265 |
explain("sync_page",3,"Writing a page to disk"); |
| 266 |
explain("sync_sb_inodes",3,"Syncing inodes"); |
| 267 |
explain("synchronize_rcu",3,"Waiting for RCU"); |
| 268 |
explain("sys_epoll_wait",3,"Waiting for event (epoll)"); |
| 269 |
explain("sys_execve",3,"Executing a program"); |
| 270 |
explain("sys_fcntl",2,"FCNTL system call"); |
| 271 |
explain("sys_flock",3,"flock() on a file"); |
| 272 |
explain("sys_futex",3,"Userspace lock contention"); |
| 273 |
explain("sys_getdents",3,"Reading directory content"); |
| 274 |
explain("sys_mkdirat",3,"Creating directory"); |
| 275 |
explain("sys_mmap",3,"mmaping memory"); |
| 276 |
explain("sys_mprotect",3,"mprotect() system call"); |
| 277 |
explain("sys_munmap",3,"unmapping memory"); |
| 278 |
explain("sys_nanosleep",5,"Application requested delay"); |
| 279 |
explain("sys_pause",5,"Application requested delay"); |
| 280 |
explain("sys_wait4",3,"Waiting for a process to die"); |
| 281 |
explain("tcp_connect",3,"TCP/IP connect"); |
| 282 |
explain("tcp_recvmsg",3,"Receiving TCP/IP data"); |
| 283 |
explain("transaction_kthread",3,"btrfs transaction_kthread()"); |
| 284 |
explain("tty_ioctl",2,"TTY IOCTL"); |
| 285 |
explain("tty_poll",3,"Waiting for TTY data"); |
| 286 |
explain("tty_read",3,"Waiting for TTY input"); |
| 287 |
explain("tty_wait_until_sent",3,"Waiting for TTY to finish sending"); |
| 288 |
explain("tty_write",3,"Writing data to TTY"); |
| 289 |
explain("udp_recvmsg",3,"Receiving UDP/IP data"); |
| 290 |
explain("unix_stream_recvmsg",3,"Waiting for data on unix socket"); |
| 291 |
explain("update_atime",3,"Updating inode atime"); |
| 292 |
explain("usb_autoresume_device",2,"Waking up USB device"); |
| 293 |
explain("usb_port_resume",2,"Waking up USB device"); |
| 294 |
explain("usb_stor_control_thread",2,"usb storage control kernel thread"); |
| 295 |
explain("usbdev_open",3,"opening USB device"); |
| 296 |
explain("vfs_create",1,"Creating a file"); |
| 297 |
explain("vfs_read",1,"Reading from file"); |
| 298 |
explain("vfs_lock_file",1,"Locking a file"); |
| 299 |
explain("vfs_readdir",1,"Reading directory content"); |
| 300 |
explain("vfs_rmdir",3,"Removing directory"); |
| 301 |
explain("vfs_stat_fd",3,"stat() operation"); |
| 302 |
explain("vfs_unlink",1,"Unlinking file"); |
| 303 |
explain("vfs_write",1,"Writing to file"); |
| 304 |
explain("vt_waitactive",3,"vt_waitactive IOCTL"); |
| 305 |
explain("worker_thread",2,"kernel workqueue thread"); |
| 306 |
explain("writeback_inodes",3,"Writing back inodes"); |
| 307 |
} |