| 1 |
local rspamd_util = require "rspamd_util" |
| 2 |
local lua_util = require "lua_util" |
| 3 |
local opts = {} |
| 4 |
|
| 5 |
local argparse = require "argparse" |
| 6 |
local parser = argparse() |
| 7 |
:name "rspamadm control fuzzystat" |
| 8 |
:description "Shows help for the specified configuration options" |
| 9 |
:help_description_margin(32) |
| 10 |
parser:flag "--no-ips" |
| 11 |
:description "No IPs stats" |
| 12 |
parser:flag "--no-keys" |
| 13 |
:description "No keys stats" |
| 14 |
parser:flag "--short" |
| 15 |
:description "Short output mode" |
| 16 |
parser:flag "-n --number" |
| 17 |
:description "Disable numbers humanization" |
| 18 |
parser:option "--sort" |
| 19 |
:description "Sort order" |
| 20 |
:convert { |
| 21 |
checked = "checked", |
| 22 |
matched = "matched", |
| 23 |
errors = "errors", |
| 24 |
name = "name" |
| 25 |
} |
| 26 |
|
| 27 |
local function add_data(target, src) |
| 28 |
for k, v in pairs(src) do |
| 29 |
if type(v) == 'number' then |
| 30 |
if target[k] then |
| 31 |
target[k] = target[k] + v |
| 32 |
else |
| 33 |
target[k] = v |
| 34 |
end |
| 35 |
elseif k == 'ips' then |
| 36 |
if not target['ips'] then |
| 37 |
target['ips'] = {} |
| 38 |
end |
| 39 |
-- Iterate over IPs |
| 40 |
for ip, st in pairs(v) do |
| 41 |
if not target['ips'][ip] then |
| 42 |
target['ips'][ip] = {} |
| 43 |
end |
| 44 |
add_data(target['ips'][ip], st) |
| 45 |
end |
| 46 |
elseif k == 'flags' then |
| 47 |
if not target['flags'] then |
| 48 |
target['flags'] = {} |
| 49 |
end |
| 50 |
-- Iterate over Flags |
| 51 |
for flag, st in pairs(v) do |
| 52 |
if not target['flags'][flag] then |
| 53 |
target['flags'][flag] = {} |
| 54 |
end |
| 55 |
add_data(target['flags'][flag], st) |
| 56 |
end |
| 57 |
elseif k == 'keypair' then |
| 58 |
if type(v.extensions) == 'table' then |
| 59 |
if type(v.extensions.name) == 'string' then |
| 60 |
target.name = v.extensions.name |
| 61 |
end |
| 62 |
if type(v.extensions.email) == 'string' then |
| 63 |
target.email = v.extensions.email |
| 64 |
end |
| 65 |
if type(v.extensions.ratelimit) == 'table' then |
| 66 |
if not target.ratelimit then |
| 67 |
target.ratelimit = { |
| 68 |
cur = { |
| 69 |
last = 0, |
| 70 |
count = 0 |
| 71 |
}, |
| 72 |
} |
| 73 |
end |
| 74 |
-- Passed as {burst = x, rate = y} |
| 75 |
target.ratelimit.limit = v.extensions.ratelimit |
| 76 |
end |
| 77 |
end |
| 78 |
elseif k == 'ratelimit' then |
| 79 |
if not target.ratelimit then |
| 80 |
target.ratelimit = { |
| 81 |
cur = { |
| 82 |
last = 0, |
| 83 |
count = 0 |
| 84 |
}, |
| 85 |
} |
| 86 |
end |
| 87 |
-- Ratelimit is passed as {cur = count, last = time} |
| 88 |
target.ratelimit.cur.count = v.cur + target.ratelimit.cur.count |
| 89 |
target.ratelimit.cur.last = math.max(v.last, target.ratelimit.cur.last) |
| 90 |
end |
| 91 |
end |
| 92 |
end |
| 93 |
|
| 94 |
local function print_num(num) |
| 95 |
if num then |
| 96 |
if opts['n'] or opts['number'] then |
| 97 |
return tostring(num) |
| 98 |
else |
| 99 |
return rspamd_util.humanize_number(num) |
| 100 |
end |
| 101 |
else |
| 102 |
return 'na' |
| 103 |
end |
| 104 |
end |
| 105 |
|
| 106 |
local function print_stat(st, tabs) |
| 107 |
if st['checked'] then |
| 108 |
if st.checked_per_hour then |
| 109 |
print(string.format('%sChecked: %s (%s per hour in average)', tabs, |
| 110 |
print_num(st['checked']), print_num(st['checked_per_hour']))) |
| 111 |
else |
| 112 |
print(string.format('%sChecked: %s', tabs, print_num(st['checked']))) |
| 113 |
end |
| 114 |
end |
| 115 |
if st['matched'] then |
| 116 |
if st.checked and st.checked > 0 and st.checked <= st.matched then |
| 117 |
local percentage = st.matched / st.checked * 100.0 |
| 118 |
if st.matched_per_hour then |
| 119 |
print(string.format('%sMatched: %s - %s percent (%s per hour in average)', tabs, |
| 120 |
print_num(st['matched']), percentage, print_num(st['matched_per_hour']))) |
| 121 |
else |
| 122 |
print(string.format('%sMatched: %s - %s percent', tabs, print_num(st['matched']), percentage)) |
| 123 |
end |
| 124 |
else |
| 125 |
if st.matched_per_hour then |
| 126 |
print(string.format('%sMatched: %s (%s per hour in average)', tabs, |
| 127 |
print_num(st['matched']), print_num(st['matched_per_hour']))) |
| 128 |
else |
| 129 |
print(string.format('%sMatched: %s', tabs, print_num(st['matched']))) |
| 130 |
end |
| 131 |
end |
| 132 |
end |
| 133 |
if st['errors'] then |
| 134 |
print(string.format('%sErrors: %s', tabs, print_num(st['errors']))) |
| 135 |
end |
| 136 |
if st['added'] then |
| 137 |
print(string.format('%sAdded: %s', tabs, print_num(st['added']))) |
| 138 |
end |
| 139 |
if st['deleted'] then |
| 140 |
print(string.format('%sDeleted: %s', tabs, print_num(st['deleted']))) |
| 141 |
end |
| 142 |
end |
| 143 |
|
| 144 |
-- Sort by checked |
| 145 |
local function sort_hash_table(tbl, sort_opts, key_key) |
| 146 |
local res = {} |
| 147 |
for k, v in pairs(tbl) do |
| 148 |
table.insert(res, { [key_key] = k, data = v }) |
| 149 |
end |
| 150 |
|
| 151 |
local function sort_order(elt) |
| 152 |
local key = 'checked' |
| 153 |
local sort_res = 0 |
| 154 |
|
| 155 |
if sort_opts['sort'] then |
| 156 |
if sort_opts['sort'] == 'matched' then |
| 157 |
key = 'matched' |
| 158 |
elseif sort_opts['sort'] == 'errors' then |
| 159 |
key = 'errors' |
| 160 |
elseif sort_opts['sort'] == 'name' then |
| 161 |
return elt[key_key] |
| 162 |
end |
| 163 |
end |
| 164 |
|
| 165 |
if elt.data[key] then |
| 166 |
sort_res = elt.data[key] |
| 167 |
end |
| 168 |
|
| 169 |
return sort_res |
| 170 |
end |
| 171 |
|
| 172 |
table.sort(res, function(a, b) |
| 173 |
return sort_order(a) > sort_order(b) |
| 174 |
end) |
| 175 |
|
| 176 |
return res |
| 177 |
end |
| 178 |
|
| 179 |
local function add_result(dst, src, k) |
| 180 |
if type(src) == 'table' then |
| 181 |
if type(dst) == 'number' then |
| 182 |
-- Convert dst to table |
| 183 |
dst = { dst } |
| 184 |
elseif type(dst) == 'nil' then |
| 185 |
dst = {} |
| 186 |
end |
| 187 |
|
| 188 |
for i, v in ipairs(src) do |
| 189 |
if dst[i] and k ~= 'fuzzy_stored' then |
| 190 |
dst[i] = dst[i] + v |
| 191 |
else |
| 192 |
dst[i] = v |
| 193 |
end |
| 194 |
end |
| 195 |
else |
| 196 |
if type(dst) == 'table' then |
| 197 |
if k ~= 'fuzzy_stored' then |
| 198 |
dst[1] = dst[1] + src |
| 199 |
else |
| 200 |
dst[1] = src |
| 201 |
end |
| 202 |
else |
| 203 |
if dst and k ~= 'fuzzy_stored' then |
| 204 |
dst = dst + src |
| 205 |
else |
| 206 |
dst = src |
| 207 |
end |
| 208 |
end |
| 209 |
end |
| 210 |
|
| 211 |
return dst |
| 212 |
end |
| 213 |
|
| 214 |
local function print_result(r) |
| 215 |
local function num_to_epoch(num) |
| 216 |
if num == 1 then |
| 217 |
return 'v0.6' |
| 218 |
elseif num == 2 then |
| 219 |
return 'v0.8' |
| 220 |
elseif num == 3 then |
| 221 |
return 'v0.9' |
| 222 |
elseif num == 4 then |
| 223 |
return 'v1.0+' |
| 224 |
elseif num == 5 then |
| 225 |
return 'v1.7+' |
| 226 |
end |
| 227 |
return '???' |
| 228 |
end |
| 229 |
if type(r) == 'table' then |
| 230 |
local res = {} |
| 231 |
for i, num in ipairs(r) do |
| 232 |
res[i] = string.format('(%s: %s)', num_to_epoch(i), print_num(num)) |
| 233 |
end |
| 234 |
|
| 235 |
return table.concat(res, ', ') |
| 236 |
end |
| 237 |
|
| 238 |
return print_num(r) |
| 239 |
end |
| 240 |
|
| 241 |
return function(args, res) |
| 242 |
local res_ips = {} |
| 243 |
local res_databases = {} |
| 244 |
local wrk = res['workers'] |
| 245 |
opts = parser:parse(args) |
| 246 |
|
| 247 |
if wrk then |
| 248 |
for _, pr in pairs(wrk) do |
| 249 |
-- processes cycle |
| 250 |
if pr['data'] then |
| 251 |
local id = pr['id'] |
| 252 |
|
| 253 |
if id then |
| 254 |
local res_db = res_databases[id] |
| 255 |
if not res_db then |
| 256 |
res_db = { |
| 257 |
keys = {} |
| 258 |
} |
| 259 |
res_databases[id] = res_db |
| 260 |
end |
| 261 |
|
| 262 |
-- General stats |
| 263 |
for k, v in pairs(pr['data']) do |
| 264 |
if k ~= 'keys' and k ~= 'errors_ips' then |
| 265 |
res_db[k] = add_result(res_db[k], v, k) |
| 266 |
elseif k == 'errors_ips' then |
| 267 |
-- Errors ips |
| 268 |
if not res_db['errors_ips'] then |
| 269 |
res_db['errors_ips'] = {} |
| 270 |
end |
| 271 |
for ip, nerrors in pairs(v) do |
| 272 |
if not res_db['errors_ips'][ip] then |
| 273 |
res_db['errors_ips'][ip] = nerrors |
| 274 |
else |
| 275 |
res_db['errors_ips'][ip] = nerrors + res_db['errors_ips'][ip] |
| 276 |
end |
| 277 |
end |
| 278 |
end |
| 279 |
end |
| 280 |
|
| 281 |
if pr['data']['keys'] then |
| 282 |
local res_keys = res_db['keys'] |
| 283 |
if not res_keys then |
| 284 |
res_keys = {} |
| 285 |
res_db['keys'] = res_keys |
| 286 |
end |
| 287 |
-- Go through keys in input |
| 288 |
for k, elts in pairs(pr['data']['keys']) do |
| 289 |
-- keys cycle |
| 290 |
if not res_keys[k] then |
| 291 |
res_keys[k] = {} |
| 292 |
end |
| 293 |
|
| 294 |
add_data(res_keys[k], elts) |
| 295 |
|
| 296 |
if elts['ips'] then |
| 297 |
for ip, v in pairs(elts['ips']) do |
| 298 |
if not res_ips[ip] then |
| 299 |
res_ips[ip] = {} |
| 300 |
end |
| 301 |
add_data(res_ips[ip], v) |
| 302 |
end |
| 303 |
end |
| 304 |
end |
| 305 |
end |
| 306 |
end |
| 307 |
end |
| 308 |
end |
| 309 |
end |
| 310 |
|
| 311 |
-- General stats |
| 312 |
for db, st in pairs(res_databases) do |
| 313 |
print(string.format('Statistics for storage %s', db)) |
| 314 |
|
| 315 |
for k, v in pairs(st) do |
| 316 |
if k ~= 'keys' and k ~= 'errors_ips' then |
| 317 |
print(string.format('%s: %s', k, print_result(v))) |
| 318 |
end |
| 319 |
end |
| 320 |
print('') |
| 321 |
|
| 322 |
local res_keys = st['keys'] |
| 323 |
if res_keys and not opts['no_keys'] and not opts['short'] then |
| 324 |
print('Keys statistics:') |
| 325 |
-- Convert into an array to allow sorting |
| 326 |
local sorted_keys = sort_hash_table(res_keys, opts, 'key') |
| 327 |
|
| 328 |
for _, key in ipairs(sorted_keys) do |
| 329 |
local key_stat = key.data |
| 330 |
if key_stat.name then |
| 331 |
print(string.format('Key id: %s, name: %s (email: %s)', key.key, key_stat.name, |
| 332 |
key_stat.email or 'unknown')) |
| 333 |
else |
| 334 |
print(string.format('Key id: %s', key.key)) |
| 335 |
end |
| 336 |
|
| 337 |
print_stat(key_stat, '\t') |
| 338 |
|
| 339 |
if key_stat['ips'] and not opts['no_ips'] then |
| 340 |
print('') |
| 341 |
print('\tIPs stat:') |
| 342 |
local sorted_ips = sort_hash_table(key_stat['ips'], opts, 'ip') |
| 343 |
|
| 344 |
for _, v in ipairs(sorted_ips) do |
| 345 |
print(string.format('\t%s', v['ip'])) |
| 346 |
print_stat(v['data'], '\t\t') |
| 347 |
print('') |
| 348 |
end |
| 349 |
end |
| 350 |
|
| 351 |
if key_stat.flags then |
| 352 |
print('') |
| 353 |
print('\tFlags stat:') |
| 354 |
for flag, v in pairs(key_stat.flags) do |
| 355 |
print(string.format('\t[%s]:', flag)) |
| 356 |
-- Remove irrelevant fields |
| 357 |
v.checked = nil |
| 358 |
print_stat(v, '\t\t') |
| 359 |
print('') |
| 360 |
end |
| 361 |
end |
| 362 |
|
| 363 |
if key_stat.ratelimit then |
| 364 |
print('') |
| 365 |
print('\tRatelimit stat:') |
| 366 |
print(string.format('\tLimit: %s (%.2f per hour leak rate)', |
| 367 |
print_num(key_stat.ratelimit.limit.burst), (key_stat.ratelimit.limit.rate or 0.0) * 3600)) |
| 368 |
print(string.format('\tCurrent: %s (%s last)', |
| 369 |
print_num(key_stat.ratelimit.cur.count), os.date('%c', key_stat.ratelimit.cur.last))) |
| 370 |
print('') |
| 371 |
end |
| 372 |
|
| 373 |
print('') |
| 374 |
end |
| 375 |
end |
| 376 |
if st['errors_ips'] and not opts['no_ips'] and not opts['short'] then |
| 377 |
print('') |
| 378 |
print('Errors IPs statistics:') |
| 379 |
local ip_stat = st['errors_ips'] |
| 380 |
local ips = lua_util.keys(ip_stat) |
| 381 |
-- Reverse sort by number of errors |
| 382 |
table.sort(ips, function(a, b) |
| 383 |
return ip_stat[a] > ip_stat[b] |
| 384 |
end) |
| 385 |
for _, ip in ipairs(ips) do |
| 386 |
print(string.format('%s: %s', ip, print_result(ip_stat[ip]))) |
| 387 |
end |
| 388 |
print('') |
| 389 |
end |
| 390 |
end |
| 391 |
|
| 392 |
if not opts['no_ips'] and not opts['short'] then |
| 393 |
print('') |
| 394 |
print('IPs statistics:') |
| 395 |
|
| 396 |
local sorted_ips = sort_hash_table(res_ips, opts, 'ip') |
| 397 |
for _, v in ipairs(sorted_ips) do |
| 398 |
print(string.format('%s', v['ip'])) |
| 399 |
print_stat(v['data'], '\t') |
| 400 |
print('') |
| 401 |
end |
| 402 |
end |
| 403 |
end |
| 404 |
|