Debian Codegraph @

lualib/lua_log_utils.lua

raw
1 --[[
2 Copyright (c) 2026, Vsevolod Stakhov <[email protected]>
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 ]]--
16
17 local rspamd_regexp = require "rspamd_regexp"
18 local rspamd_util = require "rspamd_util"
19
20 local exports = {}
21
22 local isatty = rspamd_util.isatty()
23
24 local decompressor = {
25 bz2 = 'bzip2 -cd',
26 gz = 'gzip -cd',
27 xz = 'xz -cd',
28 zst = 'zstd -cd',
29 }
30
31 local month_map = {
32 Jan = 0, Feb = 1, Mar = 2, Apr = 3, May = 4, Jun = 5,
33 Jul = 6, Aug = 7, Sep = 8, Oct = 9, Nov = 10, Dec = 11,
34 }
35
36 local spinner_chars = { '/', '-', '\\', '|' }
37 local spinner_update_time = 0
38
39 function exports.spinner()
40 if not isatty then
41 return
42 end
43 local now = os.time()
44 if (now - spinner_update_time) < 1 then
45 return
46 end
47 spinner_update_time = now
48 io.stderr:write(string.format("%s\r", spinner_chars[(now % #spinner_chars) + 1]))
49 io.stderr:flush()
50 end
51
52 function exports.reset_spinner()
53 spinner_update_time = 0
54 end
55
56 local re_rspamd_fmt = rspamd_regexp.create(
57 '^\\d{4}-\\d\\d-\\d\\d \\d\\d:\\d\\d:\\d\\d(?:\\.\\d{3,5})? #\\d+\\(')
58 local re_syslog_fmt1 = rspamd_regexp.create(
59 '^\\w{3} \\s?\\d\\d? \\d\\d:\\d\\d:\\d\\d #\\d+\\(')
60 local re_syslog_fmt2 = rspamd_regexp.create(
61 '^\\w{3} \\s?\\d\\d? \\d\\d:\\d\\d:\\d\\d \\S+ rspamd\\[\\d+\\]')
62 local re_syslog5424_fmt = rspamd_regexp.create(
63 '\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{1,6})?(?:Z|[-+]\\d{2}:\\d{2}) \\S+ rspamd\\[\\d+\\]')
64 local re_newsyslog = rspamd_regexp.create(
65 '^\\w{3} \\s?\\d\\d? \\d\\d:\\d\\d:\\d\\d \\S+ newsyslog\\[\\d+\\]: logfile turned over$')
66 local re_journalctl = rspamd_regexp.create(
67 '^-- Logs begin at \\w{3} \\d{4}-\\d\\d-\\d\\d \\d\\d:\\d\\d:\\d\\d [A-Z]{3},' ..
68 ' end at \\w{3} \\d{4}-\\d\\d-\\d\\d \\d\\d:\\d\\d:\\d\\d [A-Z]{3}\\. --$')
69
70 function exports.detect_log_format(line)
71 if re_rspamd_fmt:match(line) then
72 return 'rspamd'
73 elseif re_syslog_fmt1:match(line) or re_syslog_fmt2:match(line) then
74 return 'syslog'
75 elseif re_syslog5424_fmt:match(line) then
76 return 'syslog5424'
77 elseif re_newsyslog:match(line) or re_journalctl:match(line) then
78 return nil -- skip line
79 else
80 return false -- unknown
81 end
82 end
83
84 function exports.syslog2iso(ts_str)
85 local month_s, day, hh, mm, ss = ts_str:match('^(%a+)%s+(%d+)%s+(%d+):(%d+):(%d+)')
86 if not month_s then
87 return nil
88 end
89 local mon = month_map[month_s]
90 if not mon then
91 return nil
92 end
93 local now = os.time()
94 local t = os.date('*t', now)
95 local year = t.year
96 local epoch = os.time({
97 year = year, month = mon + 1, day = tonumber(day),
98 hour = tonumber(hh), min = tonumber(mm), sec = tonumber(ss)
99 })
100 if epoch > now then
101 year = year - 1
102 end
103 return string.format('%04d-%02d-%02d %02d:%02d:%02d',
104 year, mon + 1, tonumber(day), tonumber(hh), tonumber(mm), tonumber(ss))
105 end
106
107 function exports.extract_timestamp(line, ts_format)
108 if ts_format == 'syslog' then
109 local ts_str = line:match('^(%a+%s+%d+%s+%d+:%d+:%d+)')
110 if ts_str then
111 return exports.syslog2iso(ts_str)
112 end
113 elseif ts_format == 'syslog5424' then
114 local date, time = line:match('^(%d%d%d%d%-%d%d%-%d%d)T(%d%d:%d%d:%d%d)')
115 if date and time then
116 return date .. ' ' .. time
117 end
118 else
119 local d, t = line:match('^(%d%d%d%d%-%d%d%-%d%d)%s+(%d%d:%d%d:%d%d)')
120 if d and t then
121 return d .. ' ' .. t
122 end
123 end
124 return nil
125 end
126
127 function exports.normalized_time(s)
128 if not s or s == '' then
129 return ''
130 end
131 if s:match('^%d%d[:%d]*$') then
132 local t = os.date('*t')
133 return string.format('%04d-%02d-%02d %s', t.year, t.month, t.day, s)
134 end
135 return s
136 end
137
138 local function shell_quote(s)
139 return "'" .. s:gsub("'", "'\\''") .. "'"
140 end
141
142 function exports.open_log_file(path)
143 local ext = path:match('%.([^%.]+)$')
144 local dc = decompressor[ext]
145 if dc then
146 return io.popen(dc .. ' ' .. shell_quote(path), 'r')
147 else
148 return io.open(path, 'r')
149 end
150 end
151
152 local re_numbered_log = rspamd_regexp.create([[\.\d+(?:\.(?:bz2|gz|xz|zst))?$]])
153
154 local function numeric_index(fname)
155 local idx = fname:match('%.(%d+)%.')
156 if not idx then
157 idx = fname:match('%.(%d+)$')
158 end
159 return tonumber(idx) or 0
160 end
161
162 function exports.get_logfiles_list(dir, num_logs, exclude_logs)
163 exclude_logs = exclude_logs or 0
164
165 local all_files = rspamd_util.glob(dir .. '/*')
166 if not all_files or #all_files == 0 then
167 io.stderr:write(string.format("No files found in directory: %s\n", dir))
168 return {}
169 end
170
171 local unnumbered = {}
172 local numbered = {}
173
174 for _, full_path in ipairs(all_files) do
175 local err, st = rspamd_util.stat(full_path)
176 if not err and st and st.type == 'regular' then
177 local fname = full_path:match('[^/]+$')
178 if re_numbered_log:match(fname) then
179 table.insert(numbered, fname)
180 else
181 table.insert(unnumbered, fname)
182 end
183 end
184 end
185
186 table.sort(numbered, function(a, b)
187 return numeric_index(a) < numeric_index(b)
188 end)
189
190 local logs = {}
191 for _, f in ipairs(unnumbered) do
192 table.insert(logs, f)
193 end
194 for _, f in ipairs(numbered) do
195 table.insert(logs, f)
196 end
197
198 -- Apply exclude_logs and num_logs (splice from exclude_logs+1, take num_logs)
199 local start_idx = exclude_logs + 1
200 local end_idx = num_logs and (start_idx + num_logs - 1) or #logs
201 if end_idx > #logs then
202 end_idx = #logs
203 end
204
205 local selected = {}
206 for i = start_idx, end_idx do
207 table.insert(selected, logs[i])
208 end
209
210 -- Reverse order (newest last -> oldest first for processing)
211 local reversed = {}
212 for i = #selected, 1, -1 do
213 table.insert(reversed, selected[i])
214 end
215
216 io.stderr:write("\nLog files to process:\n")
217 for _, f in ipairs(reversed) do
218 io.stderr:write(string.format(" %s\n", f))
219 end
220 io.stderr:write("\n")
221
222 return reversed
223 end
224
225 local re_task_log = rspamd_regexp.create([[rspamd_task_write_log]])
226 local re_log_line = rspamd_regexp.create(
227 '/\\(([^()]+)\\): \\[(NaN|-?\\d+(?:\\.\\d+)?)\\/(-?\\d+(?:\\.\\d+)?)\\]\\s+\\[([^]]+)\\].*time: (\\d+\\.\\d+)ms/')
228
229 function exports.iterate_log(handle, start_time, end_time, callback, opts)
230 opts = opts or {}
231 local search_pattern = opts.search_pattern
232 local search_re
233 if search_pattern and search_pattern ~= '' then
234 search_re = rspamd_regexp.create(search_pattern)
235 end
236
237 local ts_format = nil
238 local enabled = (not search_re)
239
240 for line in handle:lines() do
241 if not ts_format then
242 local fmt = exports.detect_log_format(line)
243 if fmt == false then
244 io.stderr:write("Unknown log format\n")
245 return
246 elseif fmt == nil then
247 goto continue
248 else
249 ts_format = fmt
250 end
251 end
252
253 if not enabled then
254 if search_re and search_re:match(line) then
255 enabled = true
256 else
257 goto continue
258 end
259 end
260
261 if re_task_log:match(line) then
262 exports.spinner()
263
264 local ts = exports.extract_timestamp(line, ts_format)
265 if not ts then
266 goto continue
267 end
268
269 if start_time ~= '' and ts < start_time then
270 goto continue
271 end
272 if end_time and end_time ~= '' and ts > end_time then
273 goto continue
274 end
275
276 local results = re_log_line:search(line, false, true)
277 if not results or #results == 0 then
278 goto continue
279 end
280
281 local captures = results[1]
282 if not captures or #captures < 6 then
283 goto continue
284 end
285
286 local act = tostring(captures[2])
287 local score_str = tostring(captures[3])
288 local symbols_str = tostring(captures[5])
289 local scan_time_str = tostring(captures[6])
290
291 local score = tonumber(score_str) or 0
292 local scan_time = tonumber(scan_time_str) or 0
293
294 callback(ts, act, score, symbols_str, scan_time, line)
295 end
296
297 ::continue::
298 end
299 end
300
301 function exports.process_logs(log_file, start_time, end_time, callback, opts)
302 opts = opts or {}
303 local num_logs = opts.num_logs
304 local exclude_logs = opts.exclude_logs or 0
305
306 start_time = exports.normalized_time(start_time or '')
307 end_time = exports.normalized_time(end_time or '')
308 if end_time == '' then end_time = nil end
309
310 if not log_file or log_file == '-' or log_file == '' then
311 exports.iterate_log(io.stdin, start_time, end_time, callback, opts)
312 else
313 local err, st = rspamd_util.stat(log_file)
314 if err then
315 io.stderr:write(string.format("Cannot stat %s: %s\n", log_file, err))
316 os.exit(1)
317 end
318
319 if st.type == 'directory' then
320 local logs = exports.get_logfiles_list(log_file, num_logs, exclude_logs)
321 for idx, fname in ipairs(logs) do
322 local path = log_file .. '/' .. fname
323 local h, open_err = exports.open_log_file(path)
324 if not h then
325 io.stderr:write(string.format("Cannot open %s: %s\n", path, open_err or 'unknown error'))
326 else
327 if isatty then
328 io.stderr:write(string.format("\027[J Parsing log files: [%d/%d] %s\027[G",
329 idx, #logs, fname))
330 else
331 io.stderr:write(string.format(" Parsing log files: [%d/%d] %s\n",
332 idx, #logs, fname))
333 end
334 exports.reset_spinner()
335 exports.spinner()
336 exports.iterate_log(h, start_time, end_time, callback, opts)
337 h:close()
338 end
339 end
340 if isatty then
341 io.stderr:write("\027[J\027[G")
342 end
343 else
344 local h, open_err = exports.open_log_file(log_file)
345 if not h then
346 io.stderr:write(string.format("Cannot open %s: %s\n", log_file, open_err or 'unknown error'))
347 os.exit(1)
348 end
349 exports.reset_spinner()
350 exports.spinner()
351 exports.iterate_log(h, start_time, end_time, callback, opts)
352 h:close()
353 end
354 end
355 end
356
357 return exports