Debian Codegraph @

lualib/rspamadm/fuzzy_stat.lua

raw
1 local rspamd_util = require "rspamd_util"
2 local lua_util = require "lua_util"
3 local opts = {}
4
5 local argparse = require "argparse"
6 local parser = argparse()
7 :name "rspamadm control fuzzystat"
8 :description "Shows help for the specified configuration options"
9 :help_description_margin(32)
10 parser:flag "--no-ips"
11 :description "No IPs stats"
12 parser:flag "--no-keys"
13 :description "No keys stats"
14 parser:flag "--short"
15 :description "Short output mode"
16 parser:flag "-n --number"
17 :description "Disable numbers humanization"
18 parser:option "--sort"
19 :description "Sort order"
20 :convert {
21 checked = "checked",
22 matched = "matched",
23 errors = "errors",
24 name = "name"
25 }
26
27 local function add_data(target, src)
28 for k, v in pairs(src) do
29 if type(v) == 'number' then
30 if target[k] then
31 target[k] = target[k] + v
32 else
33 target[k] = v
34 end
35 elseif k == 'ips' then
36 if not target['ips'] then
37 target['ips'] = {}
38 end
39 -- Iterate over IPs
40 for ip, st in pairs(v) do
41 if not target['ips'][ip] then
42 target['ips'][ip] = {}
43 end
44 add_data(target['ips'][ip], st)
45 end
46 elseif k == 'flags' then
47 if not target['flags'] then
48 target['flags'] = {}
49 end
50 -- Iterate over Flags
51 for flag, st in pairs(v) do
52 if not target['flags'][flag] then
53 target['flags'][flag] = {}
54 end
55 add_data(target['flags'][flag], st)
56 end
57 elseif k == 'keypair' then
58 if type(v.extensions) == 'table' then
59 if type(v.extensions.name) == 'string' then
60 target.name = v.extensions.name
61 end
62 if type(v.extensions.email) == 'string' then
63 target.email = v.extensions.email
64 end
65 if type(v.extensions.ratelimit) == 'table' then
66 if not target.ratelimit then
67 target.ratelimit = {
68 cur = {
69 last = 0,
70 count = 0
71 },
72 }
73 end
74 -- Passed as {burst = x, rate = y}
75 target.ratelimit.limit = v.extensions.ratelimit
76 end
77 end
78 elseif k == 'ratelimit' then
79 if not target.ratelimit then
80 target.ratelimit = {
81 cur = {
82 last = 0,
83 count = 0
84 },
85 }
86 end
87 -- Ratelimit is passed as {cur = count, last = time}
88 target.ratelimit.cur.count = v.cur + target.ratelimit.cur.count
89 target.ratelimit.cur.last = math.max(v.last, target.ratelimit.cur.last)
90 end
91 end
92 end
93
94 local function print_num(num)
95 if num then
96 if opts['n'] or opts['number'] then
97 return tostring(num)
98 else
99 return rspamd_util.humanize_number(num)
100 end
101 else
102 return 'na'
103 end
104 end
105
106 local function print_stat(st, tabs)
107 if st['checked'] then
108 if st.checked_per_hour then
109 print(string.format('%sChecked: %s (%s per hour in average)', tabs,
110 print_num(st['checked']), print_num(st['checked_per_hour'])))
111 else
112 print(string.format('%sChecked: %s', tabs, print_num(st['checked'])))
113 end
114 end
115 if st['matched'] then
116 if st.checked and st.checked > 0 and st.checked <= st.matched then
117 local percentage = st.matched / st.checked * 100.0
118 if st.matched_per_hour then
119 print(string.format('%sMatched: %s - %s percent (%s per hour in average)', tabs,
120 print_num(st['matched']), percentage, print_num(st['matched_per_hour'])))
121 else
122 print(string.format('%sMatched: %s - %s percent', tabs, print_num(st['matched']), percentage))
123 end
124 else
125 if st.matched_per_hour then
126 print(string.format('%sMatched: %s (%s per hour in average)', tabs,
127 print_num(st['matched']), print_num(st['matched_per_hour'])))
128 else
129 print(string.format('%sMatched: %s', tabs, print_num(st['matched'])))
130 end
131 end
132 end
133 if st['errors'] then
134 print(string.format('%sErrors: %s', tabs, print_num(st['errors'])))
135 end
136 if st['added'] then
137 print(string.format('%sAdded: %s', tabs, print_num(st['added'])))
138 end
139 if st['deleted'] then
140 print(string.format('%sDeleted: %s', tabs, print_num(st['deleted'])))
141 end
142 end
143
144 -- Sort by checked
145 local function sort_hash_table(tbl, sort_opts, key_key)
146 local res = {}
147 for k, v in pairs(tbl) do
148 table.insert(res, { [key_key] = k, data = v })
149 end
150
151 local function sort_order(elt)
152 local key = 'checked'
153 local sort_res = 0
154
155 if sort_opts['sort'] then
156 if sort_opts['sort'] == 'matched' then
157 key = 'matched'
158 elseif sort_opts['sort'] == 'errors' then
159 key = 'errors'
160 elseif sort_opts['sort'] == 'name' then
161 return elt[key_key]
162 end
163 end
164
165 if elt.data[key] then
166 sort_res = elt.data[key]
167 end
168
169 return sort_res
170 end
171
172 table.sort(res, function(a, b)
173 return sort_order(a) > sort_order(b)
174 end)
175
176 return res
177 end
178
179 local function add_result(dst, src, k)
180 if type(src) == 'table' then
181 if type(dst) == 'number' then
182 -- Convert dst to table
183 dst = { dst }
184 elseif type(dst) == 'nil' then
185 dst = {}
186 end
187
188 for i, v in ipairs(src) do
189 if dst[i] and k ~= 'fuzzy_stored' then
190 dst[i] = dst[i] + v
191 else
192 dst[i] = v
193 end
194 end
195 else
196 if type(dst) == 'table' then
197 if k ~= 'fuzzy_stored' then
198 dst[1] = dst[1] + src
199 else
200 dst[1] = src
201 end
202 else
203 if dst and k ~= 'fuzzy_stored' then
204 dst = dst + src
205 else
206 dst = src
207 end
208 end
209 end
210
211 return dst
212 end
213
214 local function print_result(r)
215 local function num_to_epoch(num)
216 if num == 1 then
217 return 'v0.6'
218 elseif num == 2 then
219 return 'v0.8'
220 elseif num == 3 then
221 return 'v0.9'
222 elseif num == 4 then
223 return 'v1.0+'
224 elseif num == 5 then
225 return 'v1.7+'
226 end
227 return '???'
228 end
229 if type(r) == 'table' then
230 local res = {}
231 for i, num in ipairs(r) do
232 res[i] = string.format('(%s: %s)', num_to_epoch(i), print_num(num))
233 end
234
235 return table.concat(res, ', ')
236 end
237
238 return print_num(r)
239 end
240
241 return function(args, res)
242 local res_ips = {}
243 local res_databases = {}
244 local wrk = res['workers']
245 opts = parser:parse(args)
246
247 if wrk then
248 for _, pr in pairs(wrk) do
249 -- processes cycle
250 if pr['data'] then
251 local id = pr['id']
252
253 if id then
254 local res_db = res_databases[id]
255 if not res_db then
256 res_db = {
257 keys = {}
258 }
259 res_databases[id] = res_db
260 end
261
262 -- General stats
263 for k, v in pairs(pr['data']) do
264 if k ~= 'keys' and k ~= 'errors_ips' then
265 res_db[k] = add_result(res_db[k], v, k)
266 elseif k == 'errors_ips' then
267 -- Errors ips
268 if not res_db['errors_ips'] then
269 res_db['errors_ips'] = {}
270 end
271 for ip, nerrors in pairs(v) do
272 if not res_db['errors_ips'][ip] then
273 res_db['errors_ips'][ip] = nerrors
274 else
275 res_db['errors_ips'][ip] = nerrors + res_db['errors_ips'][ip]
276 end
277 end
278 end
279 end
280
281 if pr['data']['keys'] then
282 local res_keys = res_db['keys']
283 if not res_keys then
284 res_keys = {}
285 res_db['keys'] = res_keys
286 end
287 -- Go through keys in input
288 for k, elts in pairs(pr['data']['keys']) do
289 -- keys cycle
290 if not res_keys[k] then
291 res_keys[k] = {}
292 end
293
294 add_data(res_keys[k], elts)
295
296 if elts['ips'] then
297 for ip, v in pairs(elts['ips']) do
298 if not res_ips[ip] then
299 res_ips[ip] = {}
300 end
301 add_data(res_ips[ip], v)
302 end
303 end
304 end
305 end
306 end
307 end
308 end
309 end
310
311 -- General stats
312 for db, st in pairs(res_databases) do
313 print(string.format('Statistics for storage %s', db))
314
315 for k, v in pairs(st) do
316 if k ~= 'keys' and k ~= 'errors_ips' then
317 print(string.format('%s: %s', k, print_result(v)))
318 end
319 end
320 print('')
321
322 local res_keys = st['keys']
323 if res_keys and not opts['no_keys'] and not opts['short'] then
324 print('Keys statistics:')
325 -- Convert into an array to allow sorting
326 local sorted_keys = sort_hash_table(res_keys, opts, 'key')
327
328 for _, key in ipairs(sorted_keys) do
329 local key_stat = key.data
330 if key_stat.name then
331 print(string.format('Key id: %s, name: %s (email: %s)', key.key, key_stat.name,
332 key_stat.email or 'unknown'))
333 else
334 print(string.format('Key id: %s', key.key))
335 end
336
337 print_stat(key_stat, '\t')
338
339 if key_stat['ips'] and not opts['no_ips'] then
340 print('')
341 print('\tIPs stat:')
342 local sorted_ips = sort_hash_table(key_stat['ips'], opts, 'ip')
343
344 for _, v in ipairs(sorted_ips) do
345 print(string.format('\t%s', v['ip']))
346 print_stat(v['data'], '\t\t')
347 print('')
348 end
349 end
350
351 if key_stat.flags then
352 print('')
353 print('\tFlags stat:')
354 for flag, v in pairs(key_stat.flags) do
355 print(string.format('\t[%s]:', flag))
356 -- Remove irrelevant fields
357 v.checked = nil
358 print_stat(v, '\t\t')
359 print('')
360 end
361 end
362
363 if key_stat.ratelimit then
364 print('')
365 print('\tRatelimit stat:')
366 print(string.format('\tLimit: %s (%.2f per hour leak rate)',
367 print_num(key_stat.ratelimit.limit.burst), (key_stat.ratelimit.limit.rate or 0.0) * 3600))
368 print(string.format('\tCurrent: %s (%s last)',
369 print_num(key_stat.ratelimit.cur.count), os.date('%c', key_stat.ratelimit.cur.last)))
370 print('')
371 end
372
373 print('')
374 end
375 end
376 if st['errors_ips'] and not opts['no_ips'] and not opts['short'] then
377 print('')
378 print('Errors IPs statistics:')
379 local ip_stat = st['errors_ips']
380 local ips = lua_util.keys(ip_stat)
381 -- Reverse sort by number of errors
382 table.sort(ips, function(a, b)
383 return ip_stat[a] > ip_stat[b]
384 end)
385 for _, ip in ipairs(ips) do
386 print(string.format('%s: %s', ip, print_result(ip_stat[ip])))
387 end
388 print('')
389 end
390 end
391
392 if not opts['no_ips'] and not opts['short'] then
393 print('')
394 print('IPs statistics:')
395
396 local sorted_ips = sort_hash_table(res_ips, opts, 'ip')
397 for _, v in ipairs(sorted_ips) do
398 print(string.format('%s', v['ip']))
399 print_stat(v['data'], '\t')
400 print('')
401 end
402 end
403 end
404