Debian Codegraph @

script/attr_count_read

raw
1 #!/usr/bin/env python3
2 #
3 # Copyright (C) Catalyst IT Ltd. 2019
4 #
5 # This program is free software; you can redistribute it and/or modify
6 # it under the terms of the GNU General Public License as published by
7 # the Free Software Foundation; either version 3 of the License, or
8 # (at your option) any later version.
9 #
10 # This program is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU General Public License for more details.
14 #
15 # You should have received a copy of the GNU General Public License
16 # along with this program. If not, see <http://www.gnu.org/licenses/>.
17 #
18
19 import sys
20 import argparse
21 import struct
22 import os
23 from collections import OrderedDict, Counter
24 from pprint import pprint
25
26 sys.path.insert(0, "bin/python")
27 import tdb
28
29
30 def unpack_uint(filename, casefold=True):
31 db = tdb.Tdb(filename)
32 d = {}
33 for k in db:
34 v = struct.unpack("I", db[k])[0]
35 k2 = k.decode('utf-8')
36 if casefold:
37 k2 = k2.lower()
38 if k2 in d: # because casefold
39 d[k2] += v
40 else:
41 d[k2] = v
42 return d
43
44
45 def unpack_ssize_t_pair(filename, casefold):
46 db = tdb.Tdb(filename)
47 pairs = []
48 for k in db:
49 key = struct.unpack("nn", k)
50 v = struct.unpack("I", db[k])[0]
51 pairs.append((v, key))
52
53 pairs.sort(reverse=True)
54 #print(pairs)
55 return [(k, v) for (v, k) in pairs]
56
57
58 DATABASES = [
59 ('requested', "debug/attr_counts_requested.tdb", unpack_uint,
60 "The attribute was specifically requested."),
61 ('duplicates', "debug/attr_counts_duplicates.tdb", unpack_uint,
62 "Requested more than once in the same request."),
63 ('empty request', "debug/attr_counts_empty_req.tdb", unpack_uint,
64 "No attributes were requested, but these were returned"),
65 ('null request', "debug/attr_counts_null_req.tdb", unpack_uint,
66 "The attribute list was NULL and these were returned."),
67 ('found', "debug/attr_counts_found.tdb", unpack_uint,
68 "The attribute was specifically requested and it was found."),
69 ('not found', "debug/attr_counts_not_found.tdb", unpack_uint,
70 "The attribute was specifically requested but was not found."),
71 ('unwanted', "debug/attr_counts_unwanted.tdb", unpack_uint,
72 "The attribute was not requested and it was found."),
73 ('star match', "debug/attr_counts_star_match.tdb", unpack_uint,
74 'The attribute was not specifically requested but "*" was.'),
75 ('req vs found', "debug/attr_counts_req_vs_found.tdb", unpack_ssize_t_pair,
76 "How many attributes were requested versus how many were returned."),
77 ]
78
79
80 def plot_pair_data(name, data, doc, lim=90):
81 # Note we keep the matplotlib import internal to this function for
82 # two reasons:
83 # 1. Some people won't have matplotlib, but might want to run the
84 # script.
85 # 2. The import takes hundreds of milliseconds, which is a
86 # nuisance if you don't want graphs.
87 #
88 # This plot could be improved!
89 import matplotlib.pylab as plt
90 fig, ax = plt.subplots()
91 if lim:
92 data2 = []
93 for p, c in data:
94 if p[0] > lim or p[1] > lim:
95 print("not plotting %s: %s" % (p, c))
96 continue
97 data2.append((p, c))
98 skipped = len(data) - len(data2)
99 if skipped:
100 name += " (excluding %d out of range values)" % skipped
101 data = data2
102 xy, counts = zip(*data)
103 x, y = zip(*xy)
104 bins_x = max(x) + 4
105 bins_y = max(y)
106 ax.set_title(name)
107 ax.scatter(x, y, c=counts)
108 plt.show()
109
110
111 def print_pair_data(name, data, doc):
112 print(name)
113 print(doc)
114 t = "%14s | %14s | %14s"
115 print(t % ("requested", "returned", "count"))
116 print(t % (('-' * 14,) * 3))
117
118 for xy, count in data:
119 x, y = xy
120 if x == -2:
121 x = 'NULL'
122 elif x == -4:
123 x = '*'
124 print(t % (x, y, count))
125
126
127 def print_counts(count_data):
128 all_attrs = Counter()
129 for c in count_data:
130 all_attrs.update(c[1])
131
132 print("found %d attrs" % len(all_attrs))
133 longest = max(len(x) for x in all_attrs)
134
135 #pprint(all_attrs)
136 rows = OrderedDict()
137 for a, _ in all_attrs.most_common():
138 rows[a] = [a]
139
140 for col_name, counts, doc in count_data:
141 for attr, row in rows.items():
142 d = counts.get(attr, '')
143 row.append(d)
144
145 print("%15s: %s" % (col_name, doc))
146 print()
147
148 t = "%{}s".format(longest)
149 for c in count_data:
150 t += " | %{}s".format(max(len(c[0]), 7))
151
152 h = t % (("attribute",) + tuple(c[0] for c in count_data))
153 print(h)
154 print("-" * len(h))
155
156 for attr, row in rows.items():
157 print(t % tuple(row))
158
159
160 def main():
161 parser = argparse.ArgumentParser()
162 parser.add_argument('LDB_PRIVATE_DIR',
163 help="read attr counts in this directory")
164 parser.add_argument('--plot', action="store_true",
165 help='attempt to draw graphs')
166 parser.add_argument('--no-casefold', action="store_false",
167 default=True, dest="casefold",
168 help='See all the encountered case variants')
169 args = parser.parse_args()
170
171 if not os.path.isdir(args.LDB_PRIVATE_DIR):
172 parser.print_usage()
173 sys.exit(1)
174
175 count_data = []
176 pair_data = []
177 for k, filename, unpacker, doc in DATABASES:
178 filename = os.path.join(args.LDB_PRIVATE_DIR, filename)
179 try:
180 d = unpacker(filename, casefold=args.casefold)
181 except (RuntimeError, IOError) as e:
182 print("could not parse %s: %s" % (filename, e))
183 continue
184 if unpacker is unpack_ssize_t_pair:
185 pair_data.append((k, d, doc))
186 else:
187 count_data.append((k, d, doc))
188
189 for k, v, doc in pair_data:
190 if args.plot:
191 plot_pair_data(k, v, doc)
192 print_pair_data(k, v, doc)
193
194 print()
195 print_counts(count_data)
196
197 main()