Debian Codegraph @

lualib/rspamadm/autolearnstats.lua

raw
1 --[[
2 Copyright (c) 2026, Alexander Moisseev <[email protected]>
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 ]]--
16
17 local argparse = require "argparse"
18 local rspamd_regexp = require "rspamd_regexp"
19 local rspamd_util = require "rspamd_util"
20 local log_utils = require "lua_log_utils"
21 local ansicolors = require "ansicolors"
22
23 local parser = argparse()
24 :name "rspamadm autolearnstats"
25 :description "Report Bayes autolearn events from rspamd log"
26 :help_description_margin(32)
27
28 parser:argument "log"
29 :description "Log file or directory to read (stdin if omitted)"
30 :args "?"
31 :default ""
32 parser:option "--start"
33 :description "Starting time for log parsing"
34 :argname "<time>"
35 :default ""
36 parser:option "--end"
37 :description "Ending time for log parsing"
38 :argname "<time>"
39 parser:option "-n --num-logs"
40 :description "Number of recent logfiles to analyze"
41 :argname "<n>"
42 :convert(tonumber)
43 parser:option "-x --exclude-logs"
44 :description "Number of latest logs to exclude"
45 :argname "<n>"
46 :default "0"
47 :convert(tonumber)
48
49 -- Lua-side "can autolearn" log line (lua_bayes_learn.lua)
50 -- Module is always "lua" regardless of worker type.
51 -- Captures: req_id, from, verdict, score, op, threshold, mime_rcpts
52 local re_lua = rspamd_regexp.create(
53 '/\\([^)]+\\) (<[0-9a-fA-F]+>); lua; lua_bayes_learn\\.lua:\\d+: ' ..
54 'id: <[^>]*>, from: <([^>]*)>: can autolearn (\\w+): score (-?[\\d.]+) ' ..
55 '([^\\s,]+) (-?[\\d.]+), mime_rcpts: <([^>]*)>/')
56
57 -- C-side "autolearn confirmed" line (rspamd_stat_check_autolearn).
58 -- Module is "proxy" for rspamd_proxy worker, "task" for normal worker.
59 -- Matched only against the success messages, which start with "<MSG-ID>: autolearn",
60 -- to avoid false positives from error messages emitted by the same function.
61 -- Captures: req_id
62 local re_confirmed = rspamd_regexp.create(
63 '/\\([^)]+\\) (<[0-9a-fA-F]+>); \\w+; rspamd_stat_check_autolearn: <[^>]*>: autolearn /')
64
65 -- Task result line for sender IP extraction (rspamd_task_write_log)
66 -- Module is "proxy" for rspamd_proxy worker, "task" for normal worker.
67 -- Captures: req_id, ip
68 local re_ip = rspamd_regexp.create(
69 '/\\([^)]+\\) (<[0-9a-fA-F]+>); \\w+; rspamd_task_write_log: ' ..
70 'id: [^,]+,(?:\\s*qid: [^,]+,)?\\s*ip: ([^,\\s]+),/')
71
72 -- Color support: only when stdout is a TTY
73 local isatty = rspamd_util.isatty()
74
75 -- lua_bayes_learn.lua only logs "can autolearn" for spam, junk, and ham.
76 local verdict_colors = {
77 spam = ansicolors.red,
78 junk = ansicolors.yellow,
79 ham = ansicolors.green,
80 }
81
82 local function get_verdict_color(verdict)
83 return verdict_colors[verdict] or ansicolors.white
84 end
85
86 local function colored(s, color)
87 if not isatty or not color or color == '' then return s end
88 return color .. s .. ansicolors.reset
89 end
90
91 local function pad(s, n)
92 local len = #s
93 if len >= n then return s end
94 return s .. string.rep(' ', n - len)
95 end
96
97 local function iterate_bayes_log(handle, start_time, end_time, candidates, learned, ips)
98 local ts_format = nil
99
100 for line in handle:lines() do
101 log_utils.spinner()
102
103 if not ts_format then
104 local fmt = log_utils.detect_log_format(line)
105 if fmt == false then
106 io.stderr:write("Unknown log format\n")
107 return
108 elseif fmt == nil then
109 goto continue
110 else
111 ts_format = fmt
112 end
113 end
114
115 -- "can autolearn" line: collect candidate with full metadata
116 local r = re_lua:search(line, false, true)
117 if r and r[1] then
118 local caps = r[1]
119 local ts = log_utils.extract_timestamp(line, ts_format)
120 if ts then
121 if start_time ~= '' and ts < start_time then goto continue end
122 if end_time and ts > end_time then goto continue end
123 local req_id = tostring(caps[2])
124 if candidates[req_id] then
125 io.stderr:write(string.format(
126 "Warning: duplicate req_id %s, overwriting earlier entry\n", req_id))
127 end
128 candidates[req_id] = {
129 ts = ts,
130 from = tostring(caps[3]),
131 verdict = tostring(caps[4]),
132 score = tostring(caps[5]),
133 op = tostring(caps[6]),
134 thr = tostring(caps[7]),
135 rcpts = tostring(caps[8]),
136 }
137 end
138 goto continue
139 end
140
141 -- "autolearn confirmed" line: mark request as actually learned
142 r = re_confirmed:search(line, false, true)
143 if r and r[1] then
144 learned[tostring(r[1][2])] = true
145 goto continue
146 end
147
148 -- Task result line: extract sender IP.
149 -- No time-window filter here: only IPs for req_ids present in candidates
150 -- (which are already time-filtered) will be used in the output.
151 r = re_ip:search(line, false, true)
152 if r and r[1] then
153 ips[tostring(r[1][2])] = tostring(r[1][3])
154 end
155
156 ::continue::
157 end
158 end
159
160 local function process_logs(log_file, start_time, end_time, candidates, learned, ips, opts)
161 opts = opts or {}
162 local num_logs = opts.num_logs
163 local exclude_logs = opts.exclude_logs or 0
164
165 start_time = log_utils.normalized_time(start_time or '')
166 end_time = log_utils.normalized_time(end_time or '')
167 if end_time == '' then end_time = nil end
168
169 if not log_file or log_file == '-' or log_file == '' then
170 iterate_bayes_log(io.stdin, start_time, end_time, candidates, learned, ips)
171 return
172 end
173
174 local err, st = rspamd_util.stat(log_file)
175 if err then
176 io.stderr:write(string.format("Cannot stat %s: %s\n", log_file, err))
177 os.exit(1)
178 end
179
180 if st.type == 'directory' then
181 local logs = log_utils.get_logfiles_list(log_file, num_logs, exclude_logs)
182 for idx, fname in ipairs(logs) do
183 local path = log_file .. '/' .. fname
184 local h, open_err = log_utils.open_log_file(path)
185 if not h then
186 io.stderr:write(string.format("Cannot open %s: %s\n", path, open_err or 'unknown'))
187 else
188 if isatty then
189 io.stderr:write(string.format("\027[J Parsing log files: [%d/%d] %s\027[G",
190 idx, #logs, fname))
191 else
192 io.stderr:write(string.format(" Parsing log files: [%d/%d] %s\n",
193 idx, #logs, fname))
194 end
195 log_utils.reset_spinner()
196 iterate_bayes_log(h, start_time, end_time, candidates, learned, ips)
197 h:close()
198 end
199 end
200 if isatty then io.stderr:write("\027[J\027[G") end
201 else
202 local h, open_err = log_utils.open_log_file(log_file)
203 if not h then
204 io.stderr:write(string.format("Cannot open %s: %s\n", log_file, open_err or 'unknown'))
205 os.exit(1)
206 end
207 log_utils.reset_spinner()
208 iterate_bayes_log(h, start_time, end_time, candidates, learned, ips)
209 h:close()
210 end
211 end
212
213 local function handler(args)
214 local res = parser:parse(args)
215
216 local candidates = {}
217 local learned = {}
218 local ips = {}
219
220 -- res['end'] uses bracket syntax because 'end' is a Lua reserved keyword.
221 process_logs(
222 res['log'],
223 res['start'] or '',
224 res['end'] or '',
225 candidates, learned, ips,
226 {
227 num_logs = res['num_logs'],
228 exclude_logs = res['exclude_logs'] or 0,
229 }
230 )
231
232 local sorted = {}
233 for req_id, c in pairs(candidates) do
234 table.insert(sorted, { req_id = req_id, c = c })
235 end
236 table.sort(sorted, function(a, b) return a.c.ts < b.c.ts end)
237
238 -- Compute column widths from actual data (plain values, no ANSI codes)
239 local col = {
240 verdict = #'Verd',
241 score = #'Score',
242 ts = 19, -- timestamp format is always 19 chars
243 tid = #'Task',
244 ip = #'IP',
245 from = #'From',
246 rcpts = #'Recipients',
247 }
248 for _, entry in ipairs(sorted) do
249 local c = entry.c
250 col.verdict = math.max(col.verdict, #c.verdict)
251 col.score = math.max(col.score, #(c.score .. c.op .. c.thr))
252 col.tid = math.max(col.tid, #entry.req_id - 2) -- strip < >
253 col.ip = math.max(col.ip, #(ips[entry.req_id] or '-'))
254 col.from = math.max(col.from, #c.from)
255 col.rcpts = math.max(col.rcpts, #c.rcpts)
256 end
257
258 local sep = ' '
259
260 if #sorted == 0 then
261 io.write('No autolearn candidates found.\n')
262 return
263 end
264
265 local sep_width = 3 + #sep + col.verdict + #sep + col.score + #sep +
266 col.ts + #sep + col.tid + #sep + col.ip + #sep + col.from + #sep + col.rcpts
267
268 -- Header: [L] Verd Score Timestamp Task IP From Recipients
269 io.write(string.format("%-3s" .. sep .. "%-" .. col.verdict .. "s" .. sep ..
270 "%-" .. col.score .. "s" .. sep .. "%-" .. col.ts .. "s" .. sep ..
271 "%-" .. col.tid .. "s" .. sep .. "%-" .. col.ip .. "s" .. sep ..
272 "%-" .. col.from .. "s" .. sep .. "%-" .. col.rcpts .. "s\n",
273 '', 'Verd', 'Score', 'Timestamp', 'Task', 'IP', 'From', 'Recipients'))
274 io.write(string.rep('-', sep_width) .. '\n')
275
276 local n_learned = 0
277 local class_stats = {}
278
279 for _, entry in ipairs(sorted) do
280 local req_id = entry.req_id
281 local c = entry.c
282 local tid = req_id:gsub('[<>]', '')
283 local score_str = c.score .. c.op .. c.thr
284 local from_ip = ips[req_id] or '-'
285 local is_learned = learned[req_id]
286 local vcolor = get_verdict_color(c.verdict)
287
288 if not class_stats[c.verdict] then
289 class_stats[c.verdict] = { candidates = 0, learned = 0 }
290 end
291 class_stats[c.verdict].candidates = class_stats[c.verdict].candidates + 1
292 if is_learned then
293 n_learned = n_learned + 1
294 class_stats[c.verdict].learned = class_stats[c.verdict].learned + 1
295 end
296
297 local marker = is_learned and '[L]' or ' '
298 io.write(
299 colored(pad(marker, 3), is_learned and ansicolors.green or '') .. sep ..
300 colored(pad(c.verdict, col.verdict), vcolor) .. sep ..
301 pad(score_str, col.score) .. sep ..
302 pad(c.ts, col.ts) .. sep ..
303 pad(tid, col.tid) .. sep ..
304 pad(from_ip, col.ip) .. sep ..
305 pad(c.from, col.from) .. sep ..
306 c.rcpts .. '\n'
307 )
308 end
309
310 if #sorted > 0 then
311 io.write(string.format("\nTotal autolearn candidates: %d Learned: %d\n",
312 #sorted, n_learned))
313
314 local sorted_classes = {}
315 for cls in pairs(class_stats) do
316 table.insert(sorted_classes, cls)
317 end
318 table.sort(sorted_classes)
319
320 local cls_label_w = 0
321 for _, cls in ipairs(sorted_classes) do
322 cls_label_w = math.max(cls_label_w, #cls)
323 end
324 for _, cls in ipairs(sorted_classes) do
325 local s = class_stats[cls]
326 io.write(colored(
327 string.format(" %-" .. cls_label_w .. "s %d candidates / %d learned\n",
328 cls, s.candidates, s.learned),
329 get_verdict_color(cls)))
330 end
331 end
332 end
333
334 return {
335 handler = handler,
336 description = parser._description,
337 name = 'autolearnstats'
338 }