Debian Codegraph @

lualib/rspamadm/mapstats.lua

raw
1 --[[
2 Copyright (c) 2026, Vsevolod Stakhov <[email protected]>
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 ]]--
16
17 local argparse = require "argparse"
18 local rspamd_regexp = require "rspamd_regexp"
19 local rspamd_ip = require "rspamd_ip"
20 local log_utils = require "lua_log_utils"
21 local ansicolors = require "ansicolors"
22
23 local parser = argparse()
24 :name "rspamadm mapstats"
25 :description "Count Rspamd multimap matches by parsing log files"
26 :help_description_margin(32)
27
28 parser:argument "log"
29 :description "Log file or directory to read (stdin if omitted)"
30 :args "?"
31 :default ""
32 parser:option "-c --config"
33 :description "Path to config file"
34 :argname "<file>"
35 :default(rspamd_paths and rspamd_paths["CONFDIR"] and
36 (rspamd_paths["CONFDIR"] .. "/" .. "rspamd.conf") or
37 "/etc/rspamd/rspamd.conf")
38 parser:option "--start"
39 :description "Starting time for log parsing"
40 :argname "<time>"
41 :default ""
42 parser:option "--end"
43 :description "Ending time for log parsing"
44 :argname "<time>"
45 parser:option "-n --num-logs"
46 :description "Number of recent logfiles to analyze"
47 :argname "<n>"
48 :convert(tonumber)
49 parser:option "-x --exclude-logs"
50 :description "Number of latest logs to exclude"
51 :argname "<n>"
52 :default "0"
53 :convert(tonumber)
54
55 local re_non_file_url = rspamd_regexp.create('/^.*(?<!file):\\/\\//')
56 local re_regexp_line = rspamd_regexp.create('/^\\/(.+)\\/(\\S?)(?:\\s+(\\d+\\.?\\d*))?$/')
57 local re_plain_line = rspamd_regexp.create('/^(\\S+)(?:\\s+(\\d+\\.?\\d*))?$/')
58 local re_sym_with_opts = rspamd_regexp.create('/([^(]+)\\([.0-9]+\\)\\{([^;]+);\\}/')
59
60 local function get_multimap_config(config_path)
61 local _r, err = rspamd_config:load_ucl(config_path)
62 if not _r then
63 io.stderr:write(string.format("Cannot load config %s: %s\n", config_path, err))
64 os.exit(1)
65 end
66 _r, err = rspamd_config:parse_rcl({ 'logging', 'worker' })
67 if not _r then
68 io.stderr:write(string.format("Cannot parse config %s: %s\n", config_path, err))
69 os.exit(1)
70 end
71
72 local multimap_opts = rspamd_config:get_all_opt('multimap')
73 if not multimap_opts then
74 io.stderr:write("No multimap configuration found.\n")
75 os.exit(1)
76 end
77
78 return multimap_opts
79 end
80
81 local function validate_regex_flags(flags, map_file, line_num)
82 if flags and #flags > 0 then
83 local bad = flags:match('[^imsxurOL]')
84 if bad then
85 io.stderr:write(string.format(
86 "Invalid regex flag in %s at line %d: '%s' (supported: imsxurOL)\n",
87 map_file, line_num, flags))
88 return false
89 end
90 end
91 return true
92 end
93
94 local function get_map(symbol_cfg, map_file)
95 local fh, err = io.open(map_file, 'r')
96 if not fh then
97 io.stderr:write(string.format("Cannot open map file %s: %s\n", map_file, err or 'unknown'))
98 return {}
99 end
100
101 local entries = {}
102 local line_num = 0
103 local is_regexp = symbol_cfg.regexp and true or false
104
105 for line in fh:lines() do
106 line_num = line_num + 1
107
108 local trimmed = line:match('^%s*(.-)%s*$')
109 if trimmed == '' or trimmed:match('^#') then
110 table.insert(entries, {
111 line_num = line_num,
112 is_comment = true,
113 content = line,
114 })
115 else
116 -- Extract inline comment before regex parsing to avoid
117 -- rspamd_regexp capture truncation on unmatched optional groups
118 local comment = nil
119 local body = trimmed
120 if is_regexp then
121 -- For regexes, comment must come after the closing / and optional flags/score
122 -- Find the last # that's preceded by whitespace (outside the regex pattern)
123 local close_slash = trimmed:find('/', 2)
124 if close_slash then
125 local after = trimmed:sub(close_slash + 1)
126 local cmt_pos = after:find('%s+#%s*')
127 if cmt_pos then
128 -- Find actual position of # in after
129 local hash_pos = after:find('#', cmt_pos)
130 if hash_pos then
131 comment = after:sub(hash_pos + 1):match('^%s*(.*)')
132 body = trimmed:sub(1, close_slash) .. after:sub(1, cmt_pos - 1)
133 end
134 end
135 end
136 else
137 local pre, cmt = trimmed:match('^(.-)%s+#%s*(.*)$')
138 if pre and cmt then
139 comment = cmt
140 body = pre
141 end
142 end
143
144 if is_regexp then
145 local results = re_regexp_line:search(body, false, true)
146 if not results or #results == 0 then
147 io.stderr:write(string.format("Syntax error in %s at line %d\n", map_file, line_num))
148 fh:close()
149 return {}
150 end
151 local caps = results[1]
152 if not caps or #caps < 2 then
153 io.stderr:write(string.format("Syntax error in %s at line %d\n", map_file, line_num))
154 fh:close()
155 return {}
156 end
157
158 local pattern = tostring(caps[2])
159 local flags = caps[3] and tostring(caps[3]) or ''
160 local result = caps[4] and tostring(caps[4]) or nil
161
162 if not validate_regex_flags(flags, map_file, line_num) then
163 fh:close()
164 return {}
165 end
166
167 -- Compile with rspamd_regexp (handles all rspamd flags natively)
168 local re_pattern = '/' .. pattern .. '/' .. flags
169 local compiled = rspamd_regexp.create(re_pattern)
170 if not compiled then
171 io.stderr:write(string.format("Invalid regex in %s at line %d\n", map_file, line_num))
172 fh:close()
173 return {}
174 end
175
176 table.insert(entries, {
177 line_num = line_num,
178 pattern = pattern,
179 flag = flags,
180 compiled = compiled,
181 result = result,
182 comment = comment,
183 count = 0,
184 })
185 else
186 local results = re_plain_line:search(body, false, true)
187 if not results or #results == 0 then
188 io.stderr:write(string.format("Syntax error in %s at line %d\n", map_file, line_num))
189 fh:close()
190 return {}
191 end
192 local caps = results[1]
193 if not caps or #caps < 2 then
194 io.stderr:write(string.format("Syntax error in %s at line %d\n", map_file, line_num))
195 fh:close()
196 return {}
197 end
198
199 local pattern = tostring(caps[2])
200 local result = caps[3] and tostring(caps[3]) or nil
201
202 table.insert(entries, {
203 line_num = line_num,
204 pattern = pattern,
205 result = result,
206 comment = comment,
207 count = 0,
208 })
209 end
210 end
211 end
212
213 fh:close()
214 return entries
215 end
216
217 local function ip_within(ip_obj, cidr_str)
218 -- Extract mask from CIDR notation before parsing
219 local ip_part, mask_str = cidr_str:match('^(.+)/(%d+)$')
220 if ip_part then
221 local cidr_ip = rspamd_ip.from_string(ip_part)
222 if not cidr_ip or not cidr_ip:is_valid() then
223 return false
224 end
225 local mask = tonumber(mask_str)
226 -- apply_mask returns a new IP object with the mask applied
227 local ip_masked = ip_obj:apply_mask(mask)
228 local cidr_masked = cidr_ip:apply_mask(mask)
229 if not ip_masked or not cidr_masked then
230 return false
231 end
232 return ip_masked == cidr_masked
233 else
234 local cidr_ip = rspamd_ip.from_string(cidr_str)
235 if not cidr_ip or not cidr_ip:is_valid() then
236 return false
237 end
238 return ip_obj == cidr_ip
239 end
240 end
241
242 local function handler(args)
243 local res = parser:parse(args)
244
245 local multimap = get_multimap_config(res['config'])
246
247 local map = {}
248 local symbols_search = {}
249 local unmatched = {}
250
251 for symbol, cfg in pairs(multimap) do
252 if type(cfg) ~= 'table' then
253 goto continue_sym
254 end
255
256 local maps_list = cfg['map']
257 if not maps_list then
258 goto continue_sym
259 end
260
261 if type(maps_list) ~= 'table' then
262 maps_list = { maps_list }
263 elseif maps_list[1] == nil then
264 -- It's a single map object, not an array
265 maps_list = { maps_list }
266 end
267
268 map[symbol] = {
269 type = cfg['type'] or 'string',
270 is_regexp = cfg['regexp'] and true or false,
271 maps = {},
272 }
273
274 local has_valid_maps = false
275 for _, map_source in ipairs(maps_list) do
276 if type(map_source) == 'table' then
277 map_source = map_source['url'] or map_source['name'] or ''
278 end
279 if type(map_source) ~= 'string' then
280 goto continue_map
281 end
282
283 -- Skip non-file maps
284 if re_non_file_url:match(map_source) then
285 io.write(string.format("%s: %s %s\n",
286 ansicolors.bright .. symbol .. ansicolors.reset, map_source,
287 ansicolors.yellow .. "[SKIPPED]" .. ansicolors.reset))
288 goto continue_map
289 end
290
291 -- Strip file:// prefix
292 local file_path = map_source:gsub('^fallback%+', ''):gsub('^file://', '')
293
294 local entries = get_map(cfg, file_path)
295 if #entries == 0 then
296 io.write(string.format("%s: %s %s\n",
297 ansicolors.bright .. symbol .. ansicolors.reset, map_source,
298 ansicolors.red .. "[FAILED]" .. ansicolors.reset))
299 goto continue_map
300 end
301
302 local entry_count = 0
303 for _, e in ipairs(entries) do
304 if not e.is_comment then
305 entry_count = entry_count + 1
306 end
307 end
308 io.write(string.format("%s: %s %s - %d entries\n",
309 ansicolors.bright .. symbol .. ansicolors.reset, map_source,
310 ansicolors.green .. "[OK]" .. ansicolors.reset, entry_count))
311
312 table.insert(map[symbol].maps, {
313 source = map_source,
314 entries = entries,
315 })
316 has_valid_maps = true
317
318 ::continue_map::
319 end
320
321 if has_valid_maps then
322 table.insert(symbols_search, symbol)
323 end
324
325 ::continue_sym::
326 end
327
328 if #symbols_search == 0 then
329 io.stderr:write("No file-based multimap symbols found. Nothing to analyze.\n")
330 os.exit(1)
331 end
332
333 io.write("====== maps added =====\n")
334
335 -- Process logs
336 local function process_callback(ts, act, score, symbols_str, scan_time)
337 if symbols_str == '' then
338 return
339 end
340
341 local symbols_raw = {}
342 for sym in symbols_str:gmatch('[^,]+') do
343 table.insert(symbols_raw, sym)
344 end
345
346 for _, s in ipairs(symbols_search) do
347 for _, sym in ipairs(symbols_raw) do
348 if not sym:find(s, 1, true) then
349 goto continue_inner
350 end
351
352 local results = re_sym_with_opts:search(sym, false, true)
353 if not results or #results == 0 then
354 unmatched[sym] = (unmatched[sym] or 0) + 1
355 goto continue_inner
356 end
357 local caps = results[1]
358 if not caps or #caps < 3 then
359 unmatched[sym] = (unmatched[sym] or 0) + 1
360 goto continue_inner
361 end
362
363 local sym_name = tostring(caps[2])
364 local sym_opt = tostring(caps[3])
365
366 if sym_name ~= s then
367 goto continue_inner
368 end
369
370 local ip_obj
371 if map[sym_name].type == 'ip' then
372 ip_obj = rspamd_ip.from_string(sym_opt)
373 if not ip_obj or not ip_obj:is_valid() then
374 io.stderr:write(string.format("Invalid IP address in symbol %s: %s\n", sym_name, sym_opt))
375 goto continue_inner
376 end
377 end
378
379 local matched = false
380 for _, map_entry in ipairs(map[sym_name].maps) do
381 for _, entry in ipairs(map_entry.entries) do
382 if entry.is_comment then
383 goto continue_entry
384 end
385
386 if map[sym_name].type == 'ip' then
387 if ip_obj and ip_within(ip_obj, entry.pattern) then
388 entry.count = entry.count + 1
389 matched = true
390 break
391 end
392 elseif map[sym_name].is_regexp then
393 if entry.compiled:match(sym_opt) then
394 entry.count = entry.count + 1
395 matched = true
396 break
397 end
398 else
399 if sym_opt == entry.pattern then
400 entry.count = entry.count + 1
401 matched = true
402 break
403 end
404 end
405
406 ::continue_entry::
407 end
408 if matched then break end
409 end
410
411 if not matched then
412 unmatched[sym] = (unmatched[sym] or 0) + 1
413 end
414
415 ::continue_inner::
416 end
417 end
418 end
419
420 log_utils.process_logs(res['log'], res['start'] or '', res['end'], process_callback, {
421 num_logs = res['num_logs'],
422 exclude_logs = res['exclude_logs'],
423 })
424
425 -- Output results
426 for _, symbol in ipairs(symbols_search) do
427 io.write(string.format("%s:\n", ansicolors.bright .. symbol .. ansicolors.reset))
428 io.write(string.format(" type=%s\n", map[symbol].type))
429
430 for _, map_entry in ipairs(map[symbol].maps) do
431 io.write(string.format("\nMap: %s\n", map_entry.source))
432 io.write("Pattern\t\t\tMatches\t\tComment\n")
433 io.write(string.rep('-', 80) .. '\n')
434
435 for _, entry in ipairs(map_entry.entries) do
436 if entry.is_comment then
437 io.write(entry.content .. '\n')
438 else
439 if map[symbol].is_regexp then
440 io.write(string.format("%-23s", '/' .. entry.pattern .. '/' .. entry.flag))
441 else
442 io.write(string.format("%-23s", entry.pattern))
443 end
444
445 if entry.count and entry.count > 0 then
446 io.write(string.format("\t%s",
447 ansicolors.green .. tostring(entry.count) .. ansicolors.reset))
448 else
449 io.write("\t-")
450 end
451
452 if entry.comment then
453 io.write(string.format("\t\t# %s", entry.comment))
454 end
455
456 io.write('\n')
457 end
458 end
459 end
460
461 io.write(string.rep('=', 80) .. '\n')
462 end
463
464 -- Unmatched report
465 if next(unmatched) then
466 io.write(string.format("\n%s\n",
467 ansicolors.yellow .. "Symbols with unmatched values:" .. ansicolors.reset))
468 io.write(string.rep('-', 80) .. '\n')
469
470 local grouped = {}
471 for key, count in pairs(unmatched) do
472 local sym_name = key:match('^(%w+)%(')
473 if sym_name then
474 if not grouped[sym_name] then
475 grouped[sym_name] = {}
476 end
477 table.insert(grouped[sym_name], { full = key, count = count })
478 end
479 end
480
481 local sorted_groups = {}
482 for sym_name in pairs(grouped) do
483 table.insert(sorted_groups, sym_name)
484 end
485 table.sort(sorted_groups)
486
487 for _, symbol in ipairs(sorted_groups) do
488 local entries = grouped[symbol]
489 table.sort(entries, function(a, b) return a.count > b.count end)
490
491 io.write(string.format("\n%s: %s\n",
492 ansicolors.bright .. symbol .. ansicolors.reset,
493 ansicolors.yellow .. string.format("%d unmatched value(s)", #entries) .. ansicolors.reset))
494 local limit = math.min(#entries, 5)
495 for i = 1, limit do
496 io.write(string.format(" %dx: %s\n", entries[i].count, entries[i].full))
497 end
498 if #entries > 5 then
499 io.write(" ...\n")
500 end
501 end
502 end
503 end
504
505 return {
506 handler = handler,
507 description = parser._description,
508 name = 'mapstats'
509 }