| 1 |
#!/usr/bin/env python3 |
| 2 |
# |
| 3 |
# Copyright (C) Catalyst IT Ltd. 2019 |
| 4 |
# |
| 5 |
# This program is free software; you can redistribute it and/or modify |
| 6 |
# it under the terms of the GNU General Public License as published by |
| 7 |
# the Free Software Foundation; either version 3 of the License, or |
| 8 |
# (at your option) any later version. |
| 9 |
# |
| 10 |
# This program is distributed in the hope that it will be useful, |
| 11 |
# but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 12 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 13 |
# GNU General Public License for more details. |
| 14 |
# |
| 15 |
# You should have received a copy of the GNU General Public License |
| 16 |
# along with this program. If not, see <http://www.gnu.org/licenses/>. |
| 17 |
# |
| 18 |
|
| 19 |
import sys |
| 20 |
import argparse |
| 21 |
import struct |
| 22 |
import os |
| 23 |
from collections import OrderedDict, Counter |
| 24 |
from pprint import pprint |
| 25 |
|
| 26 |
sys.path.insert(0, "bin/python") |
| 27 |
import tdb |
| 28 |
|
| 29 |
|
| 30 |
def unpack_uint(filename, casefold=True): |
| 31 |
db = tdb.Tdb(filename) |
| 32 |
d = {} |
| 33 |
for k in db: |
| 34 |
v = struct.unpack("I", db[k])[0] |
| 35 |
k2 = k.decode('utf-8') |
| 36 |
if casefold: |
| 37 |
k2 = k2.lower() |
| 38 |
if k2 in d: # because casefold |
| 39 |
d[k2] += v |
| 40 |
else: |
| 41 |
d[k2] = v |
| 42 |
return d |
| 43 |
|
| 44 |
|
| 45 |
def unpack_ssize_t_pair(filename, casefold): |
| 46 |
db = tdb.Tdb(filename) |
| 47 |
pairs = [] |
| 48 |
for k in db: |
| 49 |
key = struct.unpack("nn", k) |
| 50 |
v = struct.unpack("I", db[k])[0] |
| 51 |
pairs.append((v, key)) |
| 52 |
|
| 53 |
pairs.sort(reverse=True) |
| 54 |
#print(pairs) |
| 55 |
return [(k, v) for (v, k) in pairs] |
| 56 |
|
| 57 |
|
| 58 |
DATABASES = [ |
| 59 |
('requested', "debug/attr_counts_requested.tdb", unpack_uint, |
| 60 |
"The attribute was specifically requested."), |
| 61 |
('duplicates', "debug/attr_counts_duplicates.tdb", unpack_uint, |
| 62 |
"Requested more than once in the same request."), |
| 63 |
('empty request', "debug/attr_counts_empty_req.tdb", unpack_uint, |
| 64 |
"No attributes were requested, but these were returned"), |
| 65 |
('null request', "debug/attr_counts_null_req.tdb", unpack_uint, |
| 66 |
"The attribute list was NULL and these were returned."), |
| 67 |
('found', "debug/attr_counts_found.tdb", unpack_uint, |
| 68 |
"The attribute was specifically requested and it was found."), |
| 69 |
('not found', "debug/attr_counts_not_found.tdb", unpack_uint, |
| 70 |
"The attribute was specifically requested but was not found."), |
| 71 |
('unwanted', "debug/attr_counts_unwanted.tdb", unpack_uint, |
| 72 |
"The attribute was not requested and it was found."), |
| 73 |
('star match', "debug/attr_counts_star_match.tdb", unpack_uint, |
| 74 |
'The attribute was not specifically requested but "*" was.'), |
| 75 |
('req vs found', "debug/attr_counts_req_vs_found.tdb", unpack_ssize_t_pair, |
| 76 |
"How many attributes were requested versus how many were returned."), |
| 77 |
] |
| 78 |
|
| 79 |
|
| 80 |
def plot_pair_data(name, data, doc, lim=90): |
| 81 |
# Note we keep the matplotlib import internal to this function for |
| 82 |
# two reasons: |
| 83 |
# 1. Some people won't have matplotlib, but might want to run the |
| 84 |
# script. |
| 85 |
# 2. The import takes hundreds of milliseconds, which is a |
| 86 |
# nuisance if you don't want graphs. |
| 87 |
# |
| 88 |
# This plot could be improved! |
| 89 |
import matplotlib.pylab as plt |
| 90 |
fig, ax = plt.subplots() |
| 91 |
if lim: |
| 92 |
data2 = [] |
| 93 |
for p, c in data: |
| 94 |
if p[0] > lim or p[1] > lim: |
| 95 |
print("not plotting %s: %s" % (p, c)) |
| 96 |
continue |
| 97 |
data2.append((p, c)) |
| 98 |
skipped = len(data) - len(data2) |
| 99 |
if skipped: |
| 100 |
name += " (excluding %d out of range values)" % skipped |
| 101 |
data = data2 |
| 102 |
xy, counts = zip(*data) |
| 103 |
x, y = zip(*xy) |
| 104 |
bins_x = max(x) + 4 |
| 105 |
bins_y = max(y) |
| 106 |
ax.set_title(name) |
| 107 |
ax.scatter(x, y, c=counts) |
| 108 |
plt.show() |
| 109 |
|
| 110 |
|
| 111 |
def print_pair_data(name, data, doc): |
| 112 |
print(name) |
| 113 |
print(doc) |
| 114 |
t = "%14s | %14s | %14s" |
| 115 |
print(t % ("requested", "returned", "count")) |
| 116 |
print(t % (('-' * 14,) * 3)) |
| 117 |
|
| 118 |
for xy, count in data: |
| 119 |
x, y = xy |
| 120 |
if x == -2: |
| 121 |
x = 'NULL' |
| 122 |
elif x == -4: |
| 123 |
x = '*' |
| 124 |
print(t % (x, y, count)) |
| 125 |
|
| 126 |
|
| 127 |
def print_counts(count_data): |
| 128 |
all_attrs = Counter() |
| 129 |
for c in count_data: |
| 130 |
all_attrs.update(c[1]) |
| 131 |
|
| 132 |
print("found %d attrs" % len(all_attrs)) |
| 133 |
longest = max(len(x) for x in all_attrs) |
| 134 |
|
| 135 |
#pprint(all_attrs) |
| 136 |
rows = OrderedDict() |
| 137 |
for a, _ in all_attrs.most_common(): |
| 138 |
rows[a] = [a] |
| 139 |
|
| 140 |
for col_name, counts, doc in count_data: |
| 141 |
for attr, row in rows.items(): |
| 142 |
d = counts.get(attr, '') |
| 143 |
row.append(d) |
| 144 |
|
| 145 |
print("%15s: %s" % (col_name, doc)) |
| 146 |
print() |
| 147 |
|
| 148 |
t = "%{}s".format(longest) |
| 149 |
for c in count_data: |
| 150 |
t += " | %{}s".format(max(len(c[0]), 7)) |
| 151 |
|
| 152 |
h = t % (("attribute",) + tuple(c[0] for c in count_data)) |
| 153 |
print(h) |
| 154 |
print("-" * len(h)) |
| 155 |
|
| 156 |
for attr, row in rows.items(): |
| 157 |
print(t % tuple(row)) |
| 158 |
|
| 159 |
|
| 160 |
def main(): |
| 161 |
parser = argparse.ArgumentParser() |
| 162 |
parser.add_argument('LDB_PRIVATE_DIR', |
| 163 |
help="read attr counts in this directory") |
| 164 |
parser.add_argument('--plot', action="store_true", |
| 165 |
help='attempt to draw graphs') |
| 166 |
parser.add_argument('--no-casefold', action="store_false", |
| 167 |
default=True, dest="casefold", |
| 168 |
help='See all the encountered case variants') |
| 169 |
args = parser.parse_args() |
| 170 |
|
| 171 |
if not os.path.isdir(args.LDB_PRIVATE_DIR): |
| 172 |
parser.print_usage() |
| 173 |
sys.exit(1) |
| 174 |
|
| 175 |
count_data = [] |
| 176 |
pair_data = [] |
| 177 |
for k, filename, unpacker, doc in DATABASES: |
| 178 |
filename = os.path.join(args.LDB_PRIVATE_DIR, filename) |
| 179 |
try: |
| 180 |
d = unpacker(filename, casefold=args.casefold) |
| 181 |
except (RuntimeError, IOError) as e: |
| 182 |
print("could not parse %s: %s" % (filename, e)) |
| 183 |
continue |
| 184 |
if unpacker is unpack_ssize_t_pair: |
| 185 |
pair_data.append((k, d, doc)) |
| 186 |
else: |
| 187 |
count_data.append((k, d, doc)) |
| 188 |
|
| 189 |
for k, v, doc in pair_data: |
| 190 |
if args.plot: |
| 191 |
plot_pair_data(k, v, doc) |
| 192 |
print_pair_data(k, v, doc) |
| 193 |
|
| 194 |
print() |
| 195 |
print_counts(count_data) |
| 196 |
|
| 197 |
main() |