Debian Codegraph @

perfprofd/scripts/perf_proto_json2sqlite.py

raw
1 #!/usr/bin/python
2 #
3 # Copyright (C) 2018 The Android Open Source Project
4 #
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
8 #
9 # http://www.apache.org/licenses/LICENSE-2.0
10 #
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
16
17 import argparse
18 import itertools
19 import json
20 import sqlite3
21
22
23 class SqliteWriter(object):
24 def __init__(self):
25 self.sample_count = 0
26 self.dso_map = {}
27 self.pid_map = {}
28 self.tid_map = {}
29 self.symbol_map = {}
30
31 def open(self, out):
32 self._conn = sqlite3.connect(out)
33 self._c = self._conn.cursor()
34 # Ensure tables exist
35 # The sample replicates pid and tid.
36 try:
37 self._c.execute('''CREATE TABLE pids (id integer PRIMARY KEY AUTOINCREMENT,
38 name text)''')
39 self._c.execute('''CREATE TABLE tids (id integer PRIMARY KEY AUTOINCREMENT,
40 name text)''')
41 self._c.execute('''CREATE TABLE syms (id integer PRIMARY KEY AUTOINCREMENT,
42 name text)''')
43 self._c.execute('''CREATE TABLE dsos (id integer PRIMARY KEY AUTOINCREMENT,
44 name text)''')
45 self._c.execute('''CREATE TABLE samples (id integer PRIMARY KEY AUTOINCREMENT,
46 pid_id int not null,
47 tid_id int not null)
48 ''')
49 self._c.execute('''CREATE TABLE stacks (sample_id int not null,
50 depth int not null,
51 dso_id int not null,
52 sym_id int not null,
53 offset int not null,
54 primary key (sample_id, depth))
55 ''')
56 except sqlite3.OperationalError:
57 pass # ignore
58
59 def close(self):
60 self._conn.commit()
61 self._conn.close()
62
63 def insert_into_tmp_or_get(self, name, table_dict, table_dict_tmp):
64 if name in table_dict:
65 return table_dict[name]
66 if name in table_dict_tmp:
67 return table_dict_tmp[name]
68 index = len(table_dict) + len(table_dict_tmp)
69 table_dict_tmp[name] = index
70 return index
71
72 def prepare(self):
73 self.dso_tmp_map = {}
74 self.pid_tmp_map = {}
75 self.tid_tmp_map = {}
76 self.symbol_tmp_map = {}
77 self.samples_tmp_list = []
78 self.stacks_tmp_list = []
79
80 def write_sqlite_index_table(self, table_dict, table_name):
81 for key, value in table_dict.iteritems():
82 self._c.execute("insert into {tn} values (?,?)".format(tn=table_name), (value, key))
83
84 def flush(self):
85 self.write_sqlite_index_table(self.pid_tmp_map, 'pids')
86 self.write_sqlite_index_table(self.tid_tmp_map, 'tids')
87 self.write_sqlite_index_table(self.dso_tmp_map, 'dsos')
88 self.write_sqlite_index_table(self.symbol_tmp_map, 'syms')
89
90 for sample in self.samples_tmp_list:
91 self._c.execute("insert into samples values (?,?,?)", sample)
92 for stack in self.stacks_tmp_list:
93 self._c.execute("insert into stacks values (?,?,?,?,?)", stack)
94
95 self.pid_map.update(self.pid_tmp_map)
96 self.tid_map.update(self.tid_tmp_map)
97 self.dso_map.update(self.dso_tmp_map)
98 self.symbol_map.update(self.symbol_tmp_map)
99
100 self.dso_tmp_map = {}
101 self.pid_tmp_map = {}
102 self.tid_tmp_map = {}
103 self.symbol_tmp_map = {}
104 self.samples_tmp_list = []
105 self.stacks_tmp_list = []
106
107 def add_sample(self, sample, tid_name_map):
108 sample_id = self.sample_count
109 self.sample_count = self.sample_count + 1
110
111 def get_name(pid, name_map):
112 if pid in name_map:
113 return name_map[pid]
114 pid_str = str(pid)
115 if pid_str in name_map:
116 return name_map[pid_str]
117 if pid == 0:
118 return "[kernel]"
119 return "[unknown]"
120
121 pid_name = get_name(sample[0], tid_name_map)
122 pid_id = self.insert_into_tmp_or_get(pid_name, self.pid_map, self.pid_tmp_map)
123 tid_name = get_name(sample[1], tid_name_map)
124 tid_id = self.insert_into_tmp_or_get(tid_name, self.tid_map, self.tid_tmp_map)
125
126 self.samples_tmp_list.append((sample_id, pid_id, tid_id))
127
128 stack_depth = 0
129 for entry in sample[2]:
130 sym_id = self.insert_into_tmp_or_get(entry[0], self.symbol_map, self.symbol_tmp_map)
131 dso = entry[2]
132 if dso is None:
133 dso = "None"
134 dso_id = self.insert_into_tmp_or_get(dso, self.dso_map, self.dso_tmp_map)
135
136 self.stacks_tmp_list.append((sample_id, stack_depth, dso_id, sym_id, entry[1]))
137
138 stack_depth = stack_depth + 1
139
140
141 if __name__ == "__main__":
142 parser = argparse.ArgumentParser(description='''Process a set of perfprofd JSON files produced
143 by perf_proto_stack.py into SQLite database''')
144
145 parser.add_argument('file', help='JSON files to parse and combine', metavar='file', nargs='+')
146
147 parser.add_argument('--sqlite-out', help='SQLite database output', type=str,
148 default='sqlite.db')
149
150 args = parser.parse_args()
151 if args is not None:
152 sql_out = SqliteWriter()
153 sql_out.open(args.sqlite_out)
154 sql_out.prepare()
155
156 for f in args.file:
157 print 'Processing %s' % (f)
158 fp = open(f, 'r')
159 data = json.load(fp)
160 fp.close()
161
162 for sample in data['samples']:
163 sql_out.add_sample(sample, data['names'])
164
165 sql_out.flush()
166
167 sql_out.close()