| 1 |
#!/usr/bin/env python |
| 2 |
# |
| 3 |
# Copyright (C) 2016 The Android Open Source Project |
| 4 |
# |
| 5 |
# Licensed under the Apache License, Version 2.0 (the "License"); |
| 6 |
# you may not use this file except in compliance with the License. |
| 7 |
# You may obtain a copy of the License at |
| 8 |
# |
| 9 |
# http://www.apache.org/licenses/LICENSE-2.0 |
| 10 |
# |
| 11 |
# Unless required by applicable law or agreed to in writing, software |
| 12 |
# distributed under the License is distributed on an "AS IS" BASIS, |
| 13 |
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 |
# See the License for the specific language governing permissions and |
| 15 |
# limitations under the License. |
| 16 |
# |
| 17 |
|
| 18 |
"""app_profiler.py: Record cpu profiling data of an android app or native program. |
| 19 |
|
| 20 |
It downloads simpleperf on device, uses it to collect profiling data on the selected app, |
| 21 |
and pulls profiling data and related binaries on host. |
| 22 |
""" |
| 23 |
|
| 24 |
from __future__ import print_function |
| 25 |
import argparse |
| 26 |
import os |
| 27 |
import os.path |
| 28 |
import subprocess |
| 29 |
import sys |
| 30 |
import time |
| 31 |
|
| 32 |
from utils import AdbHelper, bytes_to_str, extant_dir, get_script_dir, get_target_binary_path |
| 33 |
from utils import log_debug, log_info, log_exit, ReadElf, remove, str_to_bytes |
| 34 |
|
| 35 |
NATIVE_LIBS_DIR_ON_DEVICE = '/data/local/tmp/native_libs/' |
| 36 |
|
| 37 |
class HostElfEntry(object): |
| 38 |
"""Represent a native lib on host in NativeLibDownloader.""" |
| 39 |
def __init__(self, path, name, score): |
| 40 |
self.path = path |
| 41 |
self.name = name |
| 42 |
self.score = score |
| 43 |
|
| 44 |
def __repr__(self): |
| 45 |
return self.__str__() |
| 46 |
|
| 47 |
def __str__(self): |
| 48 |
return '[path: %s, name %s, score %s]' % (self.path, self.name, self.score) |
| 49 |
|
| 50 |
|
| 51 |
class NativeLibDownloader(object): |
| 52 |
"""Download native libs on device. |
| 53 |
|
| 54 |
1. Collect info of all native libs in the native_lib_dir on host. |
| 55 |
2. Check the available native libs in /data/local/tmp/native_libs on device. |
| 56 |
3. Sync native libs on device. |
| 57 |
""" |
| 58 |
def __init__(self, ndk_path, device_arch, adb): |
| 59 |
self.adb = adb |
| 60 |
self.readelf = ReadElf(ndk_path) |
| 61 |
self.device_arch = device_arch |
| 62 |
self.need_archs = self._get_need_archs() |
| 63 |
self.host_build_id_map = {} # Map from build_id to HostElfEntry. |
| 64 |
self.device_build_id_map = {} # Map from build_id to relative_path on device. |
| 65 |
self.name_count_map = {} # Used to give a unique name for each library. |
| 66 |
self.dir_on_device = NATIVE_LIBS_DIR_ON_DEVICE |
| 67 |
self.build_id_list_file = 'build_id_list' |
| 68 |
|
| 69 |
def _get_need_archs(self): |
| 70 |
"""Return the archs of binaries needed on device.""" |
| 71 |
if self.device_arch == 'arm64': |
| 72 |
return ['arm', 'arm64'] |
| 73 |
if self.device_arch == 'arm': |
| 74 |
return ['arm'] |
| 75 |
if self.device_arch == 'x86_64': |
| 76 |
return ['x86', 'x86_64'] |
| 77 |
if self.device_arch == 'x86': |
| 78 |
return ['x86'] |
| 79 |
return [] |
| 80 |
|
| 81 |
def collect_native_libs_on_host(self, native_lib_dir): |
| 82 |
self.host_build_id_map.clear() |
| 83 |
for root, _, files in os.walk(native_lib_dir): |
| 84 |
for name in files: |
| 85 |
if not name.endswith('.so'): |
| 86 |
continue |
| 87 |
self.add_native_lib_on_host(os.path.join(root, name), name) |
| 88 |
|
| 89 |
def add_native_lib_on_host(self, path, name): |
| 90 |
build_id = self.readelf.get_build_id(path) |
| 91 |
if not build_id: |
| 92 |
return |
| 93 |
arch = self.readelf.get_arch(path) |
| 94 |
if arch not in self.need_archs: |
| 95 |
return |
| 96 |
sections = self.readelf.get_sections(path) |
| 97 |
score = 0 |
| 98 |
if '.debug_info' in sections: |
| 99 |
score = 3 |
| 100 |
elif '.gnu_debugdata' in sections: |
| 101 |
score = 2 |
| 102 |
elif '.symtab' in sections: |
| 103 |
score = 1 |
| 104 |
entry = self.host_build_id_map.get(build_id) |
| 105 |
if entry: |
| 106 |
if entry.score < score: |
| 107 |
entry.path = path |
| 108 |
entry.score = score |
| 109 |
else: |
| 110 |
repeat_count = self.name_count_map.get(name, 0) |
| 111 |
self.name_count_map[name] = repeat_count + 1 |
| 112 |
unique_name = name if repeat_count == 0 else name + '_' + str(repeat_count) |
| 113 |
self.host_build_id_map[build_id] = HostElfEntry(path, unique_name, score) |
| 114 |
|
| 115 |
def collect_native_libs_on_device(self): |
| 116 |
self.device_build_id_map.clear() |
| 117 |
self.adb.check_run(['shell', 'mkdir', '-p', self.dir_on_device]) |
| 118 |
if os.path.exists(self.build_id_list_file): |
| 119 |
os.remove(self.build_id_list_file) |
| 120 |
self.adb.run(['pull', self.dir_on_device + self.build_id_list_file]) |
| 121 |
if os.path.exists(self.build_id_list_file): |
| 122 |
with open(self.build_id_list_file, 'rb') as fh: |
| 123 |
for line in fh.readlines(): |
| 124 |
line = bytes_to_str(line).strip() |
| 125 |
items = line.split('=') |
| 126 |
if len(items) == 2: |
| 127 |
self.device_build_id_map[items[0]] = items[1] |
| 128 |
remove(self.build_id_list_file) |
| 129 |
|
| 130 |
def sync_natives_libs_on_device(self): |
| 131 |
# Push missing native libs on device. |
| 132 |
for build_id in self.host_build_id_map: |
| 133 |
if build_id not in self.device_build_id_map: |
| 134 |
entry = self.host_build_id_map[build_id] |
| 135 |
self.adb.check_run(['push', entry.path, self.dir_on_device + entry.name]) |
| 136 |
# Remove native libs not exist on host. |
| 137 |
for build_id in self.device_build_id_map: |
| 138 |
if build_id not in self.host_build_id_map: |
| 139 |
name = self.device_build_id_map[build_id] |
| 140 |
self.adb.run(['shell', 'rm', self.dir_on_device + name]) |
| 141 |
# Push new build_id_list on device. |
| 142 |
with open(self.build_id_list_file, 'wb') as fh: |
| 143 |
for build_id in self.host_build_id_map: |
| 144 |
s = str_to_bytes('%s=%s\n' % (build_id, self.host_build_id_map[build_id].name)) |
| 145 |
fh.write(s) |
| 146 |
self.adb.check_run(['push', self.build_id_list_file, |
| 147 |
self.dir_on_device + self.build_id_list_file]) |
| 148 |
os.remove(self.build_id_list_file) |
| 149 |
|
| 150 |
|
| 151 |
class ProfilerBase(object): |
| 152 |
"""Base class of all Profilers.""" |
| 153 |
def __init__(self, args): |
| 154 |
self.args = args |
| 155 |
self.adb = AdbHelper(enable_switch_to_root=not args.disable_adb_root) |
| 156 |
self.is_root_device = self.adb.switch_to_root() |
| 157 |
self.android_version = self.adb.get_android_version() |
| 158 |
if self.android_version < 7: |
| 159 |
log_exit("""app_profiler.py isn't supported on Android < N, please switch to use |
| 160 |
simpleperf binary directly.""") |
| 161 |
self.device_arch = self.adb.get_device_arch() |
| 162 |
self.record_subproc = None |
| 163 |
|
| 164 |
def profile(self): |
| 165 |
log_info('prepare profiling') |
| 166 |
self.prepare() |
| 167 |
log_info('start profiling') |
| 168 |
self.start() |
| 169 |
self.wait_profiling() |
| 170 |
log_info('collect profiling data') |
| 171 |
self.collect_profiling_data() |
| 172 |
log_info('profiling is finished.') |
| 173 |
|
| 174 |
def prepare(self): |
| 175 |
"""Prepare recording. """ |
| 176 |
self.download_simpleperf() |
| 177 |
if self.args.native_lib_dir: |
| 178 |
self.download_libs() |
| 179 |
|
| 180 |
def download_simpleperf(self): |
| 181 |
simpleperf_binary = get_target_binary_path(self.device_arch, 'simpleperf') |
| 182 |
self.adb.check_run(['push', simpleperf_binary, '/data/local/tmp']) |
| 183 |
self.adb.check_run(['shell', 'chmod', 'a+x', '/data/local/tmp/simpleperf']) |
| 184 |
|
| 185 |
def download_libs(self): |
| 186 |
downloader = NativeLibDownloader(self.args.ndk_path, self.device_arch, self.adb) |
| 187 |
downloader.collect_native_libs_on_host(self.args.native_lib_dir) |
| 188 |
downloader.collect_native_libs_on_device() |
| 189 |
downloader.sync_natives_libs_on_device() |
| 190 |
|
| 191 |
def start(self): |
| 192 |
raise NotImplementedError |
| 193 |
|
| 194 |
def start_profiling(self, target_args): |
| 195 |
"""Start simpleperf reocrd process on device.""" |
| 196 |
args = ['/data/local/tmp/simpleperf', 'record', '-o', '/data/local/tmp/perf.data', |
| 197 |
self.args.record_options] |
| 198 |
if self.adb.run(['shell', 'ls', NATIVE_LIBS_DIR_ON_DEVICE]): |
| 199 |
args += ['--symfs', NATIVE_LIBS_DIR_ON_DEVICE] |
| 200 |
args += target_args |
| 201 |
adb_args = [self.adb.adb_path, 'shell'] + args |
| 202 |
log_debug('run adb cmd: %s' % adb_args) |
| 203 |
self.record_subproc = subprocess.Popen(adb_args) |
| 204 |
|
| 205 |
def wait_profiling(self): |
| 206 |
"""Wait until profiling finishes, or stop profiling when user presses Ctrl-C.""" |
| 207 |
returncode = None |
| 208 |
try: |
| 209 |
returncode = self.record_subproc.wait() |
| 210 |
except KeyboardInterrupt: |
| 211 |
self.stop_profiling() |
| 212 |
self.record_subproc = None |
| 213 |
# Don't check return value of record_subproc. Because record_subproc also |
| 214 |
# receives Ctrl-C, and always returns non-zero. |
| 215 |
returncode = 0 |
| 216 |
log_debug('profiling result [%s]' % (returncode == 0)) |
| 217 |
if returncode != 0: |
| 218 |
log_exit('Failed to record profiling data.') |
| 219 |
|
| 220 |
def stop_profiling(self): |
| 221 |
"""Stop profiling by sending SIGINT to simpleperf, and wait until it exits |
| 222 |
to make sure perf.data is completely generated.""" |
| 223 |
has_killed = False |
| 224 |
while True: |
| 225 |
(result, _) = self.adb.run_and_return_output(['shell', 'pidof', 'simpleperf']) |
| 226 |
if not result: |
| 227 |
break |
| 228 |
if not has_killed: |
| 229 |
has_killed = True |
| 230 |
self.adb.run_and_return_output(['shell', 'pkill', '-l', '2', 'simpleperf']) |
| 231 |
time.sleep(1) |
| 232 |
|
| 233 |
def collect_profiling_data(self): |
| 234 |
self.adb.check_run_and_return_output(['pull', '/data/local/tmp/perf.data', |
| 235 |
self.args.perf_data_path]) |
| 236 |
if not self.args.skip_collect_binaries: |
| 237 |
binary_cache_args = [sys.executable, |
| 238 |
os.path.join(get_script_dir(), 'binary_cache_builder.py')] |
| 239 |
binary_cache_args += ['-i', self.args.perf_data_path] |
| 240 |
if self.args.native_lib_dir: |
| 241 |
binary_cache_args += ['-lib', self.args.native_lib_dir] |
| 242 |
if self.args.disable_adb_root: |
| 243 |
binary_cache_args += ['--disable_adb_root'] |
| 244 |
if self.args.ndk_path: |
| 245 |
binary_cache_args += ['--ndk_path', self.args.ndk_path] |
| 246 |
subprocess.check_call(binary_cache_args) |
| 247 |
|
| 248 |
|
| 249 |
class AppProfiler(ProfilerBase): |
| 250 |
"""Profile an Android app.""" |
| 251 |
def prepare(self): |
| 252 |
super(AppProfiler, self).prepare() |
| 253 |
if self.args.compile_java_code: |
| 254 |
self.compile_java_code() |
| 255 |
|
| 256 |
def compile_java_code(self): |
| 257 |
self.kill_app_process() |
| 258 |
# Fully compile Java code on Android >= N. |
| 259 |
self.adb.set_property('debug.generate-debug-info', 'true') |
| 260 |
self.adb.check_run(['shell', 'cmd', 'package', 'compile', '-f', '-m', 'speed', |
| 261 |
self.args.app]) |
| 262 |
|
| 263 |
def kill_app_process(self): |
| 264 |
if self.find_app_process(): |
| 265 |
self.adb.check_run(['shell', 'am', 'force-stop', self.args.app]) |
| 266 |
count = 0 |
| 267 |
while True: |
| 268 |
time.sleep(1) |
| 269 |
pid = self.find_app_process() |
| 270 |
if not pid: |
| 271 |
break |
| 272 |
# When testing on Android N, `am force-stop` sometimes can't kill |
| 273 |
# com.example.simpleperf.simpleperfexampleofkotlin. So use kill when this happens. |
| 274 |
count += 1 |
| 275 |
if count >= 3: |
| 276 |
self.run_in_app_dir(['kill', '-9', str(pid)]) |
| 277 |
|
| 278 |
def find_app_process(self): |
| 279 |
result, output = self.adb.run_and_return_output(['shell', 'pidof', self.args.app]) |
| 280 |
return int(output) if result else None |
| 281 |
|
| 282 |
def run_in_app_dir(self, args): |
| 283 |
if self.is_root_device: |
| 284 |
adb_args = ['shell', 'cd /data/data/' + self.args.app + ' && ' + (' '.join(args))] |
| 285 |
else: |
| 286 |
adb_args = ['shell', 'run-as', self.args.app] + args |
| 287 |
return self.adb.run_and_return_output(adb_args, log_output=False) |
| 288 |
|
| 289 |
def start(self): |
| 290 |
if self.args.activity or self.args.test: |
| 291 |
self.kill_app_process() |
| 292 |
self.start_profiling(['--app', self.args.app]) |
| 293 |
if self.args.activity: |
| 294 |
self.start_activity() |
| 295 |
elif self.args.test: |
| 296 |
self.start_test() |
| 297 |
# else: no need to start an activity or test. |
| 298 |
|
| 299 |
def start_activity(self): |
| 300 |
activity = self.args.app + '/' + self.args.activity |
| 301 |
result = self.adb.run(['shell', 'am', 'start', '-n', activity]) |
| 302 |
if not result: |
| 303 |
self.record_subproc.terminate() |
| 304 |
log_exit("Can't start activity %s" % activity) |
| 305 |
|
| 306 |
def start_test(self): |
| 307 |
runner = self.args.app + '/androidx.test.runner.AndroidJUnitRunner' |
| 308 |
result = self.adb.run(['shell', 'am', 'instrument', '-e', 'class', |
| 309 |
self.args.test, runner]) |
| 310 |
if not result: |
| 311 |
self.record_subproc.terminate() |
| 312 |
log_exit("Can't start instrumentation test %s" % self.args.test) |
| 313 |
|
| 314 |
|
| 315 |
class NativeProgramProfiler(ProfilerBase): |
| 316 |
"""Profile a native program.""" |
| 317 |
def start(self): |
| 318 |
pid = int(self.adb.check_run_and_return_output(['shell', 'pidof', |
| 319 |
self.args.native_program])) |
| 320 |
self.start_profiling(['-p', str(pid)]) |
| 321 |
|
| 322 |
|
| 323 |
class NativeCommandProfiler(ProfilerBase): |
| 324 |
"""Profile running a native command.""" |
| 325 |
def start(self): |
| 326 |
self.start_profiling([self.args.cmd]) |
| 327 |
|
| 328 |
|
| 329 |
class NativeProcessProfiler(ProfilerBase): |
| 330 |
"""Profile processes given their pids.""" |
| 331 |
def start(self): |
| 332 |
self.start_profiling(['-p', ','.join(self.args.pid)]) |
| 333 |
|
| 334 |
|
| 335 |
class NativeThreadProfiler(ProfilerBase): |
| 336 |
"""Profile threads given their tids.""" |
| 337 |
def start(self): |
| 338 |
self.start_profiling(['-t', ','.join(self.args.tid)]) |
| 339 |
|
| 340 |
|
| 341 |
class SystemWideProfiler(ProfilerBase): |
| 342 |
"""Profile system wide.""" |
| 343 |
def start(self): |
| 344 |
self.start_profiling(['-a']) |
| 345 |
|
| 346 |
|
| 347 |
def main(): |
| 348 |
parser = argparse.ArgumentParser(description=__doc__, |
| 349 |
formatter_class=argparse.RawDescriptionHelpFormatter) |
| 350 |
|
| 351 |
target_group = parser.add_argument_group(title='Select profiling target' |
| 352 |
).add_mutually_exclusive_group(required=True) |
| 353 |
target_group.add_argument('-p', '--app', help="""Profile an Android app, given the package name. |
| 354 |
Like `-p com.example.android.myapp`.""") |
| 355 |
|
| 356 |
target_group.add_argument('-np', '--native_program', help="""Profile a native program running on |
| 357 |
the Android device. Like `-np surfaceflinger`.""") |
| 358 |
|
| 359 |
target_group.add_argument('-cmd', help="""Profile running a command on the Android device. |
| 360 |
Like `-cmd "pm -l"`.""") |
| 361 |
|
| 362 |
target_group.add_argument('--pid', nargs='+', help="""Profile native processes running on device |
| 363 |
given their process ids.""") |
| 364 |
|
| 365 |
target_group.add_argument('--tid', nargs='+', help="""Profile native threads running on device |
| 366 |
given their thread ids.""") |
| 367 |
|
| 368 |
target_group.add_argument('--system_wide', action='store_true', help="""Profile system wide.""") |
| 369 |
|
| 370 |
app_target_group = parser.add_argument_group(title='Extra options for profiling an app') |
| 371 |
app_target_group.add_argument('--compile_java_code', action='store_true', help="""Used with -p. |
| 372 |
On Android N and Android O, we need to compile Java code into |
| 373 |
native instructions to profile Java code. Android O also needs |
| 374 |
wrap.sh in the apk to use the native instructions.""") |
| 375 |
|
| 376 |
app_start_group = app_target_group.add_mutually_exclusive_group() |
| 377 |
app_start_group.add_argument('-a', '--activity', help="""Used with -p. Profile the launch time |
| 378 |
of an activity in an Android app. The app will be started or |
| 379 |
restarted to run the activity. Like `-a .MainActivity`.""") |
| 380 |
|
| 381 |
app_start_group.add_argument('-t', '--test', help="""Used with -p. Profile the launch time of an |
| 382 |
instrumentation test in an Android app. The app will be started or |
| 383 |
restarted to run the instrumentation test. Like |
| 384 |
`-t test_class_name`.""") |
| 385 |
|
| 386 |
record_group = parser.add_argument_group('Select recording options') |
| 387 |
record_group.add_argument('-r', '--record_options', |
| 388 |
default='-e task-clock:u -f 1000 -g --duration 10', help="""Set |
| 389 |
recording options for `simpleperf record` command. Use |
| 390 |
`run_simpleperf_on_device.py record -h` to see all accepted options. |
| 391 |
Default is "-e task-clock:u -f 1000 -g --duration 10".""") |
| 392 |
|
| 393 |
record_group.add_argument('-lib', '--native_lib_dir', type=extant_dir, |
| 394 |
help="""When profiling an Android app containing native libraries, |
| 395 |
the native libraries are usually stripped and lake of symbols |
| 396 |
and debug information to provide good profiling result. By |
| 397 |
using -lib, you tell app_profiler.py the path storing |
| 398 |
unstripped native libraries, and app_profiler.py will search |
| 399 |
all shared libraries with suffix .so in the directory. Then |
| 400 |
the native libraries will be downloaded on device and |
| 401 |
collected in build_cache.""") |
| 402 |
|
| 403 |
record_group.add_argument('-o', '--perf_data_path', default='perf.data', |
| 404 |
help='The path to store profiling data. Default is perf.data.') |
| 405 |
|
| 406 |
record_group.add_argument('-nb', '--skip_collect_binaries', action='store_true', |
| 407 |
help="""By default we collect binaries used in profiling data from |
| 408 |
device to binary_cache directory. It can be used to annotate |
| 409 |
source code and disassembly. This option skips it.""") |
| 410 |
|
| 411 |
other_group = parser.add_argument_group('Other options') |
| 412 |
other_group.add_argument('--ndk_path', type=extant_dir, |
| 413 |
help="""Set the path of a ndk release. app_profiler.py needs some |
| 414 |
tools in ndk, like readelf.""") |
| 415 |
|
| 416 |
other_group.add_argument('--disable_adb_root', action='store_true', |
| 417 |
help="""Force adb to run in non root mode. By default, app_profiler.py |
| 418 |
will try to switch to root mode to be able to profile released |
| 419 |
Android apps.""") |
| 420 |
|
| 421 |
def check_args(args): |
| 422 |
if (not args.app) and (args.compile_java_code or args.activity or args.test): |
| 423 |
log_exit('--compile_java_code, -a, -t can only be used when profiling an Android app.') |
| 424 |
|
| 425 |
args = parser.parse_args() |
| 426 |
check_args(args) |
| 427 |
if args.app: |
| 428 |
profiler = AppProfiler(args) |
| 429 |
elif args.native_program: |
| 430 |
profiler = NativeProgramProfiler(args) |
| 431 |
elif args.cmd: |
| 432 |
profiler = NativeCommandProfiler(args) |
| 433 |
elif args.pid: |
| 434 |
profiler = NativeProcessProfiler(args) |
| 435 |
elif args.tid: |
| 436 |
profiler = NativeThreadProfiler(args) |
| 437 |
elif args.system_wide: |
| 438 |
profiler = SystemWideProfiler(args) |
| 439 |
profiler.profile() |
| 440 |
|
| 441 |
if __name__ == '__main__': |
| 442 |
main() |