| 1 |
--[[ |
| 2 |
Copyright (c) 2026, Alexander Moisseev <[email protected]> |
| 3 |
|
| 4 |
Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 |
you may not use this file except in compliance with the License. |
| 6 |
You may obtain a copy of the License at |
| 7 |
|
| 8 |
http://www.apache.org/licenses/LICENSE-2.0 |
| 9 |
|
| 10 |
Unless required by applicable law or agreed to in writing, software |
| 11 |
distributed under the License is distributed on an "AS IS" BASIS, |
| 12 |
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 |
See the License for the specific language governing permissions and |
| 14 |
limitations under the License. |
| 15 |
]]-- |
| 16 |
|
| 17 |
local argparse = require "argparse" |
| 18 |
local rspamd_regexp = require "rspamd_regexp" |
| 19 |
local rspamd_util = require "rspamd_util" |
| 20 |
local log_utils = require "lua_log_utils" |
| 21 |
local ansicolors = require "ansicolors" |
| 22 |
|
| 23 |
local parser = argparse() |
| 24 |
:name "rspamadm autolearnstats" |
| 25 |
:description "Report Bayes autolearn events from rspamd log" |
| 26 |
:help_description_margin(32) |
| 27 |
|
| 28 |
parser:argument "log" |
| 29 |
:description "Log file or directory to read (stdin if omitted)" |
| 30 |
:args "?" |
| 31 |
:default "" |
| 32 |
parser:option "--start" |
| 33 |
:description "Starting time for log parsing" |
| 34 |
:argname "<time>" |
| 35 |
:default "" |
| 36 |
parser:option "--end" |
| 37 |
:description "Ending time for log parsing" |
| 38 |
:argname "<time>" |
| 39 |
parser:option "-n --num-logs" |
| 40 |
:description "Number of recent logfiles to analyze" |
| 41 |
:argname "<n>" |
| 42 |
:convert(tonumber) |
| 43 |
parser:option "-x --exclude-logs" |
| 44 |
:description "Number of latest logs to exclude" |
| 45 |
:argname "<n>" |
| 46 |
:default "0" |
| 47 |
:convert(tonumber) |
| 48 |
|
| 49 |
-- Lua-side "can autolearn" log line (lua_bayes_learn.lua) |
| 50 |
-- Module is always "lua" regardless of worker type. |
| 51 |
-- Captures: req_id, from, verdict, score, op, threshold, mime_rcpts |
| 52 |
local re_lua = rspamd_regexp.create( |
| 53 |
'/\\([^)]+\\) (<[0-9a-fA-F]+>); lua; lua_bayes_learn\\.lua:\\d+: ' .. |
| 54 |
'id: <[^>]*>, from: <([^>]*)>: can autolearn (\\w+): score (-?[\\d.]+) ' .. |
| 55 |
'([^\\s,]+) (-?[\\d.]+), mime_rcpts: <([^>]*)>/') |
| 56 |
|
| 57 |
-- C-side "autolearn confirmed" line (rspamd_stat_check_autolearn). |
| 58 |
-- Module is "proxy" for rspamd_proxy worker, "task" for normal worker. |
| 59 |
-- Matched only against the success messages, which start with "<MSG-ID>: autolearn", |
| 60 |
-- to avoid false positives from error messages emitted by the same function. |
| 61 |
-- Captures: req_id |
| 62 |
local re_confirmed = rspamd_regexp.create( |
| 63 |
'/\\([^)]+\\) (<[0-9a-fA-F]+>); \\w+; rspamd_stat_check_autolearn: <[^>]*>: autolearn /') |
| 64 |
|
| 65 |
-- Task result line for sender IP extraction (rspamd_task_write_log) |
| 66 |
-- Module is "proxy" for rspamd_proxy worker, "task" for normal worker. |
| 67 |
-- Captures: req_id, ip |
| 68 |
local re_ip = rspamd_regexp.create( |
| 69 |
'/\\([^)]+\\) (<[0-9a-fA-F]+>); \\w+; rspamd_task_write_log: ' .. |
| 70 |
'id: [^,]+,(?:\\s*qid: [^,]+,)?\\s*ip: ([^,\\s]+),/') |
| 71 |
|
| 72 |
-- Color support: only when stdout is a TTY |
| 73 |
local isatty = rspamd_util.isatty() |
| 74 |
|
| 75 |
-- lua_bayes_learn.lua only logs "can autolearn" for spam, junk, and ham. |
| 76 |
local verdict_colors = { |
| 77 |
spam = ansicolors.red, |
| 78 |
junk = ansicolors.yellow, |
| 79 |
ham = ansicolors.green, |
| 80 |
} |
| 81 |
|
| 82 |
local function get_verdict_color(verdict) |
| 83 |
return verdict_colors[verdict] or ansicolors.white |
| 84 |
end |
| 85 |
|
| 86 |
local function colored(s, color) |
| 87 |
if not isatty or not color or color == '' then return s end |
| 88 |
return color .. s .. ansicolors.reset |
| 89 |
end |
| 90 |
|
| 91 |
local function pad(s, n) |
| 92 |
local len = #s |
| 93 |
if len >= n then return s end |
| 94 |
return s .. string.rep(' ', n - len) |
| 95 |
end |
| 96 |
|
| 97 |
local function iterate_bayes_log(handle, start_time, end_time, candidates, learned, ips) |
| 98 |
local ts_format = nil |
| 99 |
|
| 100 |
for line in handle:lines() do |
| 101 |
log_utils.spinner() |
| 102 |
|
| 103 |
if not ts_format then |
| 104 |
local fmt = log_utils.detect_log_format(line) |
| 105 |
if fmt == false then |
| 106 |
io.stderr:write("Unknown log format\n") |
| 107 |
return |
| 108 |
elseif fmt == nil then |
| 109 |
goto continue |
| 110 |
else |
| 111 |
ts_format = fmt |
| 112 |
end |
| 113 |
end |
| 114 |
|
| 115 |
-- "can autolearn" line: collect candidate with full metadata |
| 116 |
local r = re_lua:search(line, false, true) |
| 117 |
if r and r[1] then |
| 118 |
local caps = r[1] |
| 119 |
local ts = log_utils.extract_timestamp(line, ts_format) |
| 120 |
if ts then |
| 121 |
if start_time ~= '' and ts < start_time then goto continue end |
| 122 |
if end_time and ts > end_time then goto continue end |
| 123 |
local req_id = tostring(caps[2]) |
| 124 |
if candidates[req_id] then |
| 125 |
io.stderr:write(string.format( |
| 126 |
"Warning: duplicate req_id %s, overwriting earlier entry\n", req_id)) |
| 127 |
end |
| 128 |
candidates[req_id] = { |
| 129 |
ts = ts, |
| 130 |
from = tostring(caps[3]), |
| 131 |
verdict = tostring(caps[4]), |
| 132 |
score = tostring(caps[5]), |
| 133 |
op = tostring(caps[6]), |
| 134 |
thr = tostring(caps[7]), |
| 135 |
rcpts = tostring(caps[8]), |
| 136 |
} |
| 137 |
end |
| 138 |
goto continue |
| 139 |
end |
| 140 |
|
| 141 |
-- "autolearn confirmed" line: mark request as actually learned |
| 142 |
r = re_confirmed:search(line, false, true) |
| 143 |
if r and r[1] then |
| 144 |
learned[tostring(r[1][2])] = true |
| 145 |
goto continue |
| 146 |
end |
| 147 |
|
| 148 |
-- Task result line: extract sender IP. |
| 149 |
-- No time-window filter here: only IPs for req_ids present in candidates |
| 150 |
-- (which are already time-filtered) will be used in the output. |
| 151 |
r = re_ip:search(line, false, true) |
| 152 |
if r and r[1] then |
| 153 |
ips[tostring(r[1][2])] = tostring(r[1][3]) |
| 154 |
end |
| 155 |
|
| 156 |
::continue:: |
| 157 |
end |
| 158 |
end |
| 159 |
|
| 160 |
local function process_logs(log_file, start_time, end_time, candidates, learned, ips, opts) |
| 161 |
opts = opts or {} |
| 162 |
local num_logs = opts.num_logs |
| 163 |
local exclude_logs = opts.exclude_logs or 0 |
| 164 |
|
| 165 |
start_time = log_utils.normalized_time(start_time or '') |
| 166 |
end_time = log_utils.normalized_time(end_time or '') |
| 167 |
if end_time == '' then end_time = nil end |
| 168 |
|
| 169 |
if not log_file or log_file == '-' or log_file == '' then |
| 170 |
iterate_bayes_log(io.stdin, start_time, end_time, candidates, learned, ips) |
| 171 |
return |
| 172 |
end |
| 173 |
|
| 174 |
local err, st = rspamd_util.stat(log_file) |
| 175 |
if err then |
| 176 |
io.stderr:write(string.format("Cannot stat %s: %s\n", log_file, err)) |
| 177 |
os.exit(1) |
| 178 |
end |
| 179 |
|
| 180 |
if st.type == 'directory' then |
| 181 |
local logs = log_utils.get_logfiles_list(log_file, num_logs, exclude_logs) |
| 182 |
for idx, fname in ipairs(logs) do |
| 183 |
local path = log_file .. '/' .. fname |
| 184 |
local h, open_err = log_utils.open_log_file(path) |
| 185 |
if not h then |
| 186 |
io.stderr:write(string.format("Cannot open %s: %s\n", path, open_err or 'unknown')) |
| 187 |
else |
| 188 |
if isatty then |
| 189 |
io.stderr:write(string.format("\027[J Parsing log files: [%d/%d] %s\027[G", |
| 190 |
idx, #logs, fname)) |
| 191 |
else |
| 192 |
io.stderr:write(string.format(" Parsing log files: [%d/%d] %s\n", |
| 193 |
idx, #logs, fname)) |
| 194 |
end |
| 195 |
log_utils.reset_spinner() |
| 196 |
iterate_bayes_log(h, start_time, end_time, candidates, learned, ips) |
| 197 |
h:close() |
| 198 |
end |
| 199 |
end |
| 200 |
if isatty then io.stderr:write("\027[J\027[G") end |
| 201 |
else |
| 202 |
local h, open_err = log_utils.open_log_file(log_file) |
| 203 |
if not h then |
| 204 |
io.stderr:write(string.format("Cannot open %s: %s\n", log_file, open_err or 'unknown')) |
| 205 |
os.exit(1) |
| 206 |
end |
| 207 |
log_utils.reset_spinner() |
| 208 |
iterate_bayes_log(h, start_time, end_time, candidates, learned, ips) |
| 209 |
h:close() |
| 210 |
end |
| 211 |
end |
| 212 |
|
| 213 |
local function handler(args) |
| 214 |
local res = parser:parse(args) |
| 215 |
|
| 216 |
local candidates = {} |
| 217 |
local learned = {} |
| 218 |
local ips = {} |
| 219 |
|
| 220 |
-- res['end'] uses bracket syntax because 'end' is a Lua reserved keyword. |
| 221 |
process_logs( |
| 222 |
res['log'], |
| 223 |
res['start'] or '', |
| 224 |
res['end'] or '', |
| 225 |
candidates, learned, ips, |
| 226 |
{ |
| 227 |
num_logs = res['num_logs'], |
| 228 |
exclude_logs = res['exclude_logs'] or 0, |
| 229 |
} |
| 230 |
) |
| 231 |
|
| 232 |
local sorted = {} |
| 233 |
for req_id, c in pairs(candidates) do |
| 234 |
table.insert(sorted, { req_id = req_id, c = c }) |
| 235 |
end |
| 236 |
table.sort(sorted, function(a, b) return a.c.ts < b.c.ts end) |
| 237 |
|
| 238 |
-- Compute column widths from actual data (plain values, no ANSI codes) |
| 239 |
local col = { |
| 240 |
verdict = #'Verd', |
| 241 |
score = #'Score', |
| 242 |
ts = 19, -- timestamp format is always 19 chars |
| 243 |
tid = #'Task', |
| 244 |
ip = #'IP', |
| 245 |
from = #'From', |
| 246 |
rcpts = #'Recipients', |
| 247 |
} |
| 248 |
for _, entry in ipairs(sorted) do |
| 249 |
local c = entry.c |
| 250 |
col.verdict = math.max(col.verdict, #c.verdict) |
| 251 |
col.score = math.max(col.score, #(c.score .. c.op .. c.thr)) |
| 252 |
col.tid = math.max(col.tid, #entry.req_id - 2) -- strip < > |
| 253 |
col.ip = math.max(col.ip, #(ips[entry.req_id] or '-')) |
| 254 |
col.from = math.max(col.from, #c.from) |
| 255 |
col.rcpts = math.max(col.rcpts, #c.rcpts) |
| 256 |
end |
| 257 |
|
| 258 |
local sep = ' ' |
| 259 |
|
| 260 |
if #sorted == 0 then |
| 261 |
io.write('No autolearn candidates found.\n') |
| 262 |
return |
| 263 |
end |
| 264 |
|
| 265 |
local sep_width = 3 + #sep + col.verdict + #sep + col.score + #sep + |
| 266 |
col.ts + #sep + col.tid + #sep + col.ip + #sep + col.from + #sep + col.rcpts |
| 267 |
|
| 268 |
-- Header: [L] Verd Score Timestamp Task IP From Recipients |
| 269 |
io.write(string.format("%-3s" .. sep .. "%-" .. col.verdict .. "s" .. sep .. |
| 270 |
"%-" .. col.score .. "s" .. sep .. "%-" .. col.ts .. "s" .. sep .. |
| 271 |
"%-" .. col.tid .. "s" .. sep .. "%-" .. col.ip .. "s" .. sep .. |
| 272 |
"%-" .. col.from .. "s" .. sep .. "%-" .. col.rcpts .. "s\n", |
| 273 |
'', 'Verd', 'Score', 'Timestamp', 'Task', 'IP', 'From', 'Recipients')) |
| 274 |
io.write(string.rep('-', sep_width) .. '\n') |
| 275 |
|
| 276 |
local n_learned = 0 |
| 277 |
local class_stats = {} |
| 278 |
|
| 279 |
for _, entry in ipairs(sorted) do |
| 280 |
local req_id = entry.req_id |
| 281 |
local c = entry.c |
| 282 |
local tid = req_id:gsub('[<>]', '') |
| 283 |
local score_str = c.score .. c.op .. c.thr |
| 284 |
local from_ip = ips[req_id] or '-' |
| 285 |
local is_learned = learned[req_id] |
| 286 |
local vcolor = get_verdict_color(c.verdict) |
| 287 |
|
| 288 |
if not class_stats[c.verdict] then |
| 289 |
class_stats[c.verdict] = { candidates = 0, learned = 0 } |
| 290 |
end |
| 291 |
class_stats[c.verdict].candidates = class_stats[c.verdict].candidates + 1 |
| 292 |
if is_learned then |
| 293 |
n_learned = n_learned + 1 |
| 294 |
class_stats[c.verdict].learned = class_stats[c.verdict].learned + 1 |
| 295 |
end |
| 296 |
|
| 297 |
local marker = is_learned and '[L]' or ' ' |
| 298 |
io.write( |
| 299 |
colored(pad(marker, 3), is_learned and ansicolors.green or '') .. sep .. |
| 300 |
colored(pad(c.verdict, col.verdict), vcolor) .. sep .. |
| 301 |
pad(score_str, col.score) .. sep .. |
| 302 |
pad(c.ts, col.ts) .. sep .. |
| 303 |
pad(tid, col.tid) .. sep .. |
| 304 |
pad(from_ip, col.ip) .. sep .. |
| 305 |
pad(c.from, col.from) .. sep .. |
| 306 |
c.rcpts .. '\n' |
| 307 |
) |
| 308 |
end |
| 309 |
|
| 310 |
if #sorted > 0 then |
| 311 |
io.write(string.format("\nTotal autolearn candidates: %d Learned: %d\n", |
| 312 |
#sorted, n_learned)) |
| 313 |
|
| 314 |
local sorted_classes = {} |
| 315 |
for cls in pairs(class_stats) do |
| 316 |
table.insert(sorted_classes, cls) |
| 317 |
end |
| 318 |
table.sort(sorted_classes) |
| 319 |
|
| 320 |
local cls_label_w = 0 |
| 321 |
for _, cls in ipairs(sorted_classes) do |
| 322 |
cls_label_w = math.max(cls_label_w, #cls) |
| 323 |
end |
| 324 |
for _, cls in ipairs(sorted_classes) do |
| 325 |
local s = class_stats[cls] |
| 326 |
io.write(colored( |
| 327 |
string.format(" %-" .. cls_label_w .. "s %d candidates / %d learned\n", |
| 328 |
cls, s.candidates, s.learned), |
| 329 |
get_verdict_color(cls))) |
| 330 |
end |
| 331 |
end |
| 332 |
end |
| 333 |
|
| 334 |
return { |
| 335 |
handler = handler, |
| 336 |
description = parser._description, |
| 337 |
name = 'autolearnstats' |
| 338 |
} |