Debian Codegraph @

lualib/rspamadm/logstats.lua

raw
1 --[[
2 Copyright (c) 2026, Vsevolod Stakhov <[email protected]>
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 ]]--
16
17 local argparse = require "argparse"
18 local rspamd_regexp = require "rspamd_regexp"
19 local ucl = require "ucl"
20 local log_utils = require "lua_log_utils"
21 local ansicolors = require "ansicolors"
22
23 local action_colors = {
24 reject = ansicolors.red,
25 ['add header'] = ansicolors.yellow,
26 ['rewrite subject'] = ansicolors.yellow,
27 ['soft reject'] = ansicolors.magenta,
28 greylist = ansicolors.cyan,
29 ['no action'] = ansicolors.green,
30 }
31
32 local parser = argparse()
33 :name "rspamadm logstats"
34 :description "Analyze Rspamd rules by parsing log files"
35 :help_description_margin(32)
36
37 parser:argument "log"
38 :description "Log file or directory to read (stdin if omitted)"
39 :args "?"
40 :default ""
41 parser:option "-r --reject-score"
42 :description "Reject threshold"
43 :argname "<score>"
44 :default "15.0"
45 :convert(tonumber)
46 parser:option "-j --junk-score"
47 :description "Junk score threshold"
48 :argname "<score>"
49 :default "6.0"
50 :convert(tonumber)
51 parser:option "-s --symbol"
52 :description "Check specified symbol (regexp, '.*' by default)"
53 :argname "<sym>"
54 :count "*"
55 parser:option "-S --symbol-bidir"
56 :description "Bidirectional symbol (splits into SYM_SPAM/SYM_HAM)"
57 :argname "<sym>"
58 :count "*"
59 parser:option "-X --exclude"
60 :description "Exclude log lines if symbol fires"
61 :argname "<sym>"
62 :count "*"
63 parser:option "--ignore"
64 :description "Ignore symbol in correlations"
65 :argname "<sym>"
66 :count "*"
67 parser:option "-g --group"
68 :description "Group symbols (comma-separated)"
69 :argname "<syms>"
70 :count "*"
71 parser:option "--mult"
72 :description "Multiply symbol score (sym=number)"
73 :argname "<sym=num>"
74 :count "*"
75 parser:option "-a --alpha-score"
76 :description "Ignore score threshold"
77 :argname "<score>"
78 :default "0.1"
79 :convert(tonumber)
80 parser:flag "-c --correlations"
81 :description "Enable correlations report"
82 parser:option "--nrelated"
83 :description "Number of related symbols to show"
84 :argname "<n>"
85 :default "10"
86 :convert(tonumber)
87 parser:option "--search-pattern"
88 :description "Do not process input until pattern is found"
89 :argname "<pattern>"
90 :default ""
91 parser:option "--start"
92 :description "Starting time for log parsing"
93 :argname "<time>"
94 :default ""
95 parser:option "--end"
96 :description "Ending time for log parsing"
97 :argname "<time>"
98 parser:option "-n --num-logs"
99 :description "Number of recent logfiles to analyze"
100 :argname "<n>"
101 :convert(tonumber)
102 parser:option "-x --exclude-logs"
103 :description "Number of latest logs to exclude"
104 :argname "<n>"
105 :default "0"
106 :convert(tonumber)
107 parser:flag "--json"
108 :description "Print JSON output"
109
110 local function compile_ignored(ignored_list)
111 local compiled = {}
112 for _, ex in ipairs(ignored_list) do
113 local re = rspamd_regexp.create('^' .. ex .. '$')
114 if re then
115 table.insert(compiled, re)
116 end
117 end
118 return compiled
119 end
120
121 local function is_ignored(sym, ignored_compiled)
122 for _, re in ipairs(ignored_compiled) do
123 if re:match(sym) then
124 return true
125 end
126 end
127 return false
128 end
129
130 local function gen_related(htb, target_sym, nrelated)
131 local sorted = {}
132 for sym, count in pairs(htb) do
133 if sym ~= target_sym then
134 table.insert(sorted, { sym, count })
135 end
136 end
137 table.sort(sorted, function(a, b) return a[2] > b[2] end)
138
139 local result = {}
140 for i = 1, math.min(#sorted, nrelated) do
141 result[i] = sorted[i]
142 end
143 return result
144 end
145
146 local function stringify_related(ar, total)
147 local parts = {}
148 for _, elt in ipairs(ar) do
149 table.insert(parts, string.format("\t%s(%d: %.1f%%)",
150 elt[1], elt[2], elt[2] / (total * 1.0) * 100.0))
151 end
152 return table.concat(parts, "\n")
153 end
154
155 local function parse_mult_options(mult_list)
156 local result = {}
157 for _, m in ipairs(mult_list) do
158 local sym, num = m:match('^([^=]+)=(.+)$')
159 if sym and num then
160 result[sym] = tonumber(num) or 1.0
161 end
162 end
163 return result
164 end
165
166 local re_sym_parse = rspamd_regexp.create('/^([^(]+)(\\(([^)]+)\\))?/')
167
168 local function process_related(symbols, target, source, groups, symbols_ignored,
169 symbols_mult, diff_alpha, bidir_match)
170 for _, s in ipairs(symbols) do
171 local results = re_sym_parse:search(s, false, true)
172 if not results or #results == 0 then
173 goto continue
174 end
175 local caps = results[1]
176 if not caps or #caps < 2 then
177 goto continue
178 end
179
180 local sym_name = tostring(caps[2])
181 local sym_score = 0
182
183 if groups[sym_name] then
184 sym_name = groups[sym_name]
185 end
186
187 if source == sym_name then
188 goto continue
189 end
190
191 if is_ignored(sym_name, symbols_ignored) then
192 goto continue
193 end
194
195 if caps[4] then
196 sym_score = (tonumber(tostring(caps[4])) or 0) * (symbols_mult[sym_name] or 1.0)
197 if math.abs(sym_score) < diff_alpha then
198 goto continue
199 end
200 local bm = bidir_match[sym_name]
201 if bm then
202 if sym_score >= 0 then
203 sym_name = bm.spam
204 else
205 sym_name = bm.ham
206 end
207 end
208 end
209
210 target[sym_name] = (target[sym_name] or 0) + 1
211
212 ::continue::
213 end
214 end
215
216 local function handler(args)
217 local res = parser:parse(args)
218
219 local reject_score = res['reject_score']
220 local junk_score = res['junk_score']
221 local symbols_search = res['symbol'] or {}
222 local symbols_bidir = res['symbol_bidir'] or {}
223 local symbols_exclude = res['exclude'] or {}
224 local symbols_ignored = compile_ignored(res['ignore'] or {})
225 local symbols_groups = res['group'] or {}
226 local symbols_mult = parse_mult_options(res['mult'] or {})
227 local diff_alpha = res['alpha_score']
228 local correlations = res['correlations']
229 local nrelated = res['nrelated']
230 local json_output = res['json']
231
232 local bidir_match = {}
233 for _, s in ipairs(symbols_bidir) do
234 bidir_match[s] = {
235 spam = s .. '_SPAM',
236 ham = s .. '_HAM',
237 }
238 local found = false
239 for _, existing in ipairs(symbols_search) do
240 if existing == s then found = true; break end
241 end
242 if not found then
243 table.insert(symbols_search, s)
244 end
245 end
246
247 local groups = {}
248 local group_id = 0
249 for _, g in ipairs(symbols_groups) do
250 local syms = {}
251 for sym in g:gmatch('[^,]+') do
252 table.insert(syms, sym)
253 end
254 local group_name = 'group' .. group_id
255 group_id = group_id + 1
256 for _, s in ipairs(syms) do
257 groups[s] = group_name
258 local found = false
259 for _, existing in ipairs(symbols_search) do
260 if existing == s then found = true; break end
261 end
262 if not found then
263 table.insert(symbols_search, s)
264 end
265 end
266 end
267
268 if #symbols_search == 0 then
269 symbols_search = { '.*' }
270 end
271
272 -- Compile search patterns
273 local search_res = {}
274 for _, s in ipairs(symbols_search) do
275 local re = rspamd_regexp.create(s)
276 if re then
277 table.insert(search_res, { pattern = s, re = re })
278 end
279 end
280
281 -- Compile exclude patterns
282 local exclude_res = {}
283 for _, ex in ipairs(symbols_exclude) do
284 local re = rspamd_regexp.create('^' .. ex)
285 if re then
286 table.insert(exclude_res, re)
287 end
288 end
289
290 local total = 0
291 local total_spam = 0
292 local total_junk = 0
293 local sym_res = {}
294 local alpha_filtered = {}
295 local actions = {}
296 local timeStamp = {}
297 local scanTime = { max = 0, total = 0 }
298
299 local function process_callback(ts, act, score, symbols_str, scan_time)
300 -- Split symbols: split on , but accounting for {options} blocks
301 local symbols_raw = {}
302 local tmp = symbols_str
303 -- Split handling {}-enclosed options
304 while tmp and #tmp > 0 do
305 local sym_part, rest = tmp:match('^([^,{]+%b{})(.*)')
306 if not sym_part then
307 sym_part, rest = tmp:match('^([^,]+)(.*)')
308 end
309 if sym_part then
310 table.insert(symbols_raw, sym_part)
311 if rest and rest:sub(1, 1) == ',' then
312 rest = rest:sub(2)
313 end
314 tmp = rest
315 else
316 break
317 end
318 end
319
320 -- Check excludes
321 for _, sym in ipairs(symbols_raw) do
322 for _, ex_re in ipairs(exclude_res) do
323 if ex_re:match(sym) then
324 return
325 end
326 end
327 end
328
329 -- Update timestamps
330 if not timeStamp['end'] or ts > timeStamp['end'] then
331 timeStamp['end'] = ts
332 end
333 if not timeStamp['start'] or ts < timeStamp['start'] then
334 timeStamp['start'] = ts
335 end
336
337 -- Update scan times
338 if not scanTime['min'] or scan_time < scanTime['min'] then
339 scanTime['min'] = scan_time
340 end
341 if scan_time > scanTime['max'] then
342 scanTime['max'] = scan_time
343 end
344 scanTime['total'] = scanTime['total'] + scan_time
345
346 actions[act] = (actions[act] or 0) + 1
347 total = total + 1
348
349 local is_spam = false
350 local is_junk = false
351 if score >= reject_score then
352 total_spam = total_spam + 1
353 is_spam = true
354 elseif score >= junk_score then
355 total_junk = total_junk + 1
356 is_junk = true
357 end
358
359 local sym_names = {}
360
361 for _, sr in ipairs(search_res) do
362 for _, sym in ipairs(symbols_raw) do
363 if sr.re:match(sym) then
364 local results = re_sym_parse:search(sym, false, true)
365 if not results or #results == 0 then
366 goto continue_sym
367 end
368 local caps = results[1]
369 if not caps or #caps < 2 then
370 goto continue_sym
371 end
372
373 local sym_name = tostring(caps[2])
374 local sym_score = 0
375 local orig_name = sym_name
376
377 if caps[4] then
378 sym_score = (tonumber(tostring(caps[4])) or 0) * (symbols_mult[sym_name] or 1.0)
379 if math.abs(sym_score) < diff_alpha then
380 alpha_filtered[sym_name] = (alpha_filtered[sym_name] or 0) + 1
381 goto continue_sym
382 end
383 local bm = bidir_match[sym_name]
384 if bm then
385 if sym_score >= 0 then
386 sym_name = bm.spam
387 else
388 sym_name = bm.ham
389 end
390 end
391 end
392
393 -- Check that original name matches the search pattern
394 local match_re = rspamd_regexp.create('^' .. sr.pattern)
395 if match_re and not match_re:match(orig_name) then
396 goto continue_sym
397 end
398
399 if groups[sr.pattern] then
400 sym_name = groups[sr.pattern]
401 end
402
403 table.insert(sym_names, sym_name)
404
405 if not sym_res[sym_name] then
406 sym_res[sym_name] = {
407 hits = 0,
408 spam_hits = 0,
409 junk_hits = 0,
410 spam_change = 0,
411 junk_change = 0,
412 weight = 0,
413 corr = {},
414 symbols_met_spam = {},
415 symbols_met_ham = {},
416 symbols_met_junk = {},
417 }
418 end
419
420 local r = sym_res[sym_name]
421 r.hits = r.hits + 1
422 r.weight = r.weight + sym_score
423
424 if is_spam then
425 r.spam_hits = r.spam_hits + 1
426 if correlations then
427 process_related(symbols_raw, r.symbols_met_spam, sym_name,
428 groups, symbols_ignored, symbols_mult, diff_alpha, bidir_match)
429 end
430 elseif is_junk then
431 r.junk_hits = r.junk_hits + 1
432 if correlations then
433 process_related(symbols_raw, r.symbols_met_junk, sym_name,
434 groups, symbols_ignored, symbols_mult, diff_alpha, bidir_match)
435 end
436 else
437 if correlations then
438 process_related(symbols_raw, r.symbols_met_ham, sym_name,
439 groups, symbols_ignored, symbols_mult, diff_alpha, bidir_match)
440 end
441 end
442
443 if sym_score ~= 0 then
444 local score_without = score - sym_score
445 if sym_score > 0 then
446 if is_spam and score_without < reject_score then
447 r.spam_change = r.spam_change + 1
448 end
449 if is_junk and score_without < junk_score then
450 r.junk_change = r.junk_change + 1
451 end
452 else
453 if not is_spam and score_without >= reject_score then
454 r.spam_change = r.spam_change + 1
455 end
456 if not is_junk and score_without >= junk_score then
457 r.junk_change = r.junk_change + 1
458 end
459 end
460 end
461 ::continue_sym::
462 end
463 end
464 end
465
466 if correlations then
467 for _, sym in ipairs(sym_names) do
468 if not is_ignored(sym, symbols_ignored) then
469 local r = sym_res[sym]
470 for _, corr_sym in ipairs(sym_names) do
471 if corr_sym ~= sym then
472 r.corr[corr_sym] = (r.corr[corr_sym] or 0) + 1
473 end
474 end
475 end
476 end
477 end
478 end
479
480 log_utils.process_logs(res['log'], res['start'] or '', res['end'], process_callback, {
481 search_pattern = res['search_pattern'],
482 num_logs = res['num_logs'],
483 exclude_logs = res['exclude_logs'],
484 })
485
486 local total_ham = total - (total_spam + total_junk)
487
488 if json_output then
489 local result = {}
490 result.total = total
491 if timeStamp['start'] then
492 result.start = timeStamp['start']
493 end
494 if timeStamp['end'] then
495 result['end'] = timeStamp['end']
496 end
497 result.actions = actions
498 result.symbols = {}
499
500 if total > 0 then
501 for s, r in pairs(sym_res) do
502 if r.hits > 0 then
503 local th = r.hits
504 local sh = r.spam_hits
505 local jh = r.junk_hits
506 local hh = th - sh - jh
507 local htp = (total_ham ~= 0) and (hh * 100.0 / total_ham) or 0
508 local stp = (total_spam ~= 0) and (sh * 100.0 / total_spam) or 0
509 local jtp = (total_junk ~= 0) and (jh * 100.0 / total_junk) or 0
510
511 local sym_data = {
512 avg_weight = r.weight / th,
513 hits = th,
514 hits_percentage = th / total,
515 spam_hits = sh,
516 spam_to_total = sh / th,
517 spam_percentage = stp / 100.0,
518 ham_hits = hh,
519 ham_to_total = hh / th,
520 ham_percentage = htp / 100.0,
521 junk_hits = jh,
522 junk_to_total = jh / th,
523 junk_percentage = jtp / 100.0,
524 }
525
526 if r.weight ~= 0 then
527 sym_data.spam_change = r.spam_change
528 sym_data.junk_change = r.junk_change
529 end
530
531 if correlations then
532 local corr_data = {}
533 for cs, hits in pairs(r.corr) do
534 local corr_prob = hits / total
535 local sym_prob = r.hits / total
536 corr_data[cs] = corr_prob / sym_prob
537 end
538 sym_data.correlations = corr_data
539 end
540
541 result.symbols[s] = sym_data
542 end
543 end
544 end
545
546 io.write(ucl.to_format(result, 'json'))
547 io.write('\n')
548 else
549 -- Human-readable output
550 if total > 0 then
551 for s, r in pairs(sym_res) do
552 if r.hits > 0 then
553 local th = r.hits
554 local sh = r.spam_hits
555 local jh = r.junk_hits
556 local hh = th - sh - jh
557 local htp = (total_ham ~= 0) and (hh * 100.0 / total_ham) or 0
558 local stp = (total_spam ~= 0) and (sh * 100.0 / total_spam) or 0
559 local jtp = (total_junk ~= 0) and (jh * 100.0 / total_junk) or 0
560
561 io.write(string.format(
562 "%s avg. weight %.3f, hits %d (%.3f%%):\n" ..
563 " %s %7.3f%%, %6d/%-6d (%7.3f%%)\n" ..
564 " %s %7.3f%%, %6d/%-6d (%7.3f%%)\n" ..
565 " %s %7.3f%%, %6d/%-6d (%7.3f%%)\n",
566 ansicolors.bright .. s .. ansicolors.reset,
567 r.weight / th, th, (th / total * 100),
568 ansicolors.green .. "Ham " .. ansicolors.reset,
569 (hh / th * 100), hh, total_ham, htp,
570 ansicolors.red .. "Spam" .. ansicolors.reset,
571 (sh / th * 100), sh, total_spam, stp,
572 ansicolors.yellow .. "Junk" .. ansicolors.reset,
573 (jh / th * 100), jh, total_junk, jtp))
574
575 local schp = (total_spam > 0) and (r.spam_change / total_spam * 100.0) or 0
576 local jchp = (total_junk > 0) and (r.junk_change / total_junk * 100.0) or 0
577
578 if r.weight ~= 0 then
579 local spam_desc, junk_desc
580 if r.weight > 0 then
581 spam_desc = "Spam changes (ham/junk -> spam)"
582 junk_desc = "Junk changes (ham -> junk)"
583 else
584 spam_desc = "Spam changes (spam -> junk/ham)"
585 junk_desc = "Junk changes (junk -> ham) "
586 end
587 io.write(string.format(
588 "\n%s: %6d/%-6d (%7.3f%%)\n" ..
589 "Spam changes / total spam hits: %6d/%-6d (%7.3f%%)\n" ..
590 "%s: %6d/%-6d (%7.3f%%)\n" ..
591 "Junk changes / total junk hits: %6d/%-6d (%7.3f%%)\n",
592 spam_desc, r.spam_change, th, (r.spam_change / th * 100),
593 r.spam_change, total_spam, schp,
594 junk_desc, r.junk_change, th, (r.junk_change / th * 100),
595 r.junk_change, total_junk, jchp))
596 end
597
598 if correlations then
599 io.write("Correlations report:\n")
600 for cs, _ in pairs(r.corr) do
601 local corr_prob = r.hits / total
602 local merged_hits = 0
603 if r.symbols_met_spam[cs] then
604 merged_hits = merged_hits + r.symbols_met_spam[cs]
605 end
606 if r.symbols_met_junk[cs] then
607 merged_hits = merged_hits + r.symbols_met_junk[cs]
608 end
609 if r.symbols_met_ham[cs] then
610 merged_hits = merged_hits + r.symbols_met_ham[cs]
611 end
612 if merged_hits > 0 then
613 io.write(string.format("Probability of %s when %s fires: %.3f\n",
614 cs, s, ((merged_hits / total) / corr_prob)))
615 end
616 end
617
618 local spam_related = gen_related(r.symbols_met_spam, s, nrelated)
619 local junk_related = gen_related(r.symbols_met_junk, s, nrelated)
620 local ham_related = gen_related(r.symbols_met_ham, s, nrelated)
621
622 io.write("Related symbols report:\n")
623 io.write(string.format("Top related in spam:\n %s\n",
624 stringify_related(spam_related, r.spam_hits)))
625 io.write(string.format("Top related in junk:\n %s\n",
626 stringify_related(junk_related, r.junk_hits)))
627 io.write(string.format("Top related in ham:\n %s\n",
628 stringify_related(ham_related, r.hits - r.spam_hits - r.junk_hits)))
629 end
630 else
631 io.write(string.format("Symbol %s has not been met\n", s))
632 end
633
634 io.write(string.rep('-', 80) .. '\n')
635 end
636 end
637
638 if next(alpha_filtered) then
639 io.write(string.format(
640 "\n%s the following symbols were found but ignored" ..
641 " due to score < alpha_score (%.2f):\n",
642 ansicolors.yellow .. "WARNING:" .. ansicolors.reset, diff_alpha))
643 for sym, count in pairs(alpha_filtered) do
644 io.write(string.format(" %s: %d hit(s)\n",
645 ansicolors.bright .. sym .. ansicolors.reset, count))
646 end
647 io.write("Use --alpha-score 0 to include them.\n")
648 end
649
650 io.write(string.format("\n%s\nMessages scanned: %d",
651 ansicolors.bright .. "=== Summary " .. string.rep('=', 68) .. ansicolors.reset,
652 total))
653 if timeStamp['start'] then
654 io.write(string.format(" [ %s / %s ]\n", timeStamp['start'], timeStamp['end']))
655 else
656 io.write('\n')
657 end
658 io.write('\n')
659 local sorted_actions = {}
660 for a, _ in pairs(actions) do
661 table.insert(sorted_actions, a)
662 end
663 table.sort(sorted_actions)
664 for _, a in ipairs(sorted_actions) do
665 local color = action_colors[a] or ansicolors.white
666 io.write(string.format("%s: %6.2f%%, %d\n",
667 color .. string.format("%11s", a) .. ansicolors.reset,
668 100 * actions[a] / total, actions[a]))
669 end
670 io.write('\n')
671 if scanTime['min'] then
672 io.write(string.format("scan time min/avg/max = %.2f/%.2f/%.2f s\n",
673 scanTime['min'] / 1000,
674 (total > 0) and (scanTime['total'] / total / 1000) or 0,
675 scanTime['max'] / 1000))
676 end
677 io.write(string.rep('=', 80) .. '\n')
678 end
679 end
680
681 return {
682 handler = handler,
683 description = parser._description,
684 name = 'logstats'
685 }