Debian Codegraph @

lualib/lua_scanners/common.lua

raw
1 --[[
2 Copyright (c) 2022, Vsevolod Stakhov <[email protected]>
3 Copyright (c) 2019, Carsten Rosenberg <[email protected]>
4
5 Licensed under the Apache License, Version 2.0 (the "License");
6 you may not use this file except in compliance with the License.
7 You may obtain a copy of the License at
8
9 http://www.apache.org/licenses/LICENSE-2.0
10
11 Unless required by applicable law or agreed to in writing, software
12 distributed under the License is distributed on an "AS IS" BASIS,
13 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 See the License for the specific language governing permissions and
15 limitations under the License.
16 ]] --
17
18 --[[[
19 -- @module lua_scanners_common
20 -- This module contains common external scanners functions
21 --]]
22
23 local rspamd_logger = require "rspamd_logger"
24 local rspamd_regexp = require "rspamd_regexp"
25 local lua_util = require "lua_util"
26 local lua_redis = require "lua_redis"
27 local lua_magic_types = require "lua_magic/types"
28 local fun = require "fun"
29
30 local exports = {}
31
32 local function log_clean(task, rule, msg)
33 msg = msg or 'message or mime_part is clean'
34
35 if rule.log_clean then
36 rspamd_logger.infox(task, '%s: %s', rule.log_prefix, msg)
37 else
38 lua_util.debugm(rule.name, task, '%s: %s', rule.log_prefix, msg)
39 end
40 end
41
42 local function match_patterns(default_sym, found, patterns, dyn_weight)
43 if type(patterns) ~= 'table' then
44 return default_sym, dyn_weight
45 end
46 if not patterns[1] then
47 for sym, pat in pairs(patterns) do
48 if pat:match(found) then
49 return sym, '1'
50 end
51 end
52 return default_sym, dyn_weight
53 else
54 for _, p in ipairs(patterns) do
55 for sym, pat in pairs(p) do
56 if pat:match(found) then
57 return sym, '1'
58 end
59 end
60 end
61 return default_sym, dyn_weight
62 end
63 end
64
65 local function yield_result(task, rule, vname, dyn_weight, is_fail, maybe_part)
66 local all_whitelisted = true
67 local patterns
68 local symbol
69 local threat_table
70 local threat_info
71 local flags
72
73 if type(vname) == 'string' then
74 threat_table = { vname }
75 elseif type(vname) == 'table' then
76 threat_table = vname
77 end
78
79
80 -- This should be more generic
81 if not is_fail then
82 patterns = rule.patterns
83 symbol = rule.symbol
84 threat_info = rule.detection_category .. 'found'
85 if not dyn_weight then
86 dyn_weight = 1.0
87 end
88 elseif is_fail == 'fail' then
89 patterns = rule.patterns_fail
90 symbol = rule.symbol_fail
91 threat_info = "FAILED with error"
92 dyn_weight = 0.0
93 elseif is_fail == 'encrypted' then
94 patterns = rule.patterns
95 symbol = rule.symbol_encrypted
96 threat_info = "Scan has returned that input was encrypted"
97 dyn_weight = 1.0
98 elseif is_fail == 'macro' then
99 patterns = rule.patterns
100 symbol = rule.symbol_macro
101 threat_info = "Scan has returned that input contains macros"
102 dyn_weight = 1.0
103 end
104
105 for _, tm in ipairs(threat_table) do
106 local symname, symscore = match_patterns(symbol, tm, patterns, dyn_weight)
107 if rule.whitelist and rule.whitelist:get_key(tm) then
108 rspamd_logger.infox(task, '%s: "%s" is in whitelist', rule.log_prefix, tm)
109 else
110 all_whitelisted = false
111 rspamd_logger.infox(task, '%s: result - %s: "%s - score: %s"',
112 rule.log_prefix, threat_info, tm, symscore)
113
114 if maybe_part and rule.show_attachments and maybe_part:get_filename() then
115 local fname = maybe_part:get_filename()
116 task:insert_result(symname, symscore, string.format("%s|%s",
117 tm, fname))
118 else
119 task:insert_result(symname, symscore, tm)
120 end
121 end
122 end
123
124 if rule.action and is_fail ~= 'fail' and not all_whitelisted then
125 threat_table = table.concat(threat_table, '; ')
126 if rule.action ~= 'reject' then
127 flags = 'least'
128 end
129 task:set_pre_result(rule.action,
130 lua_util.template(rule.message or 'Rejected', {
131 SCANNER = rule.name,
132 VIRUS = threat_table,
133 }), rule.name, nil, nil, flags)
134 end
135 end
136
137 local function message_not_too_large(task, content, rule)
138 local max_size = tonumber(rule.max_size)
139 if not max_size then
140 return true
141 end
142 if #content > max_size then
143 rspamd_logger.infox(task, "skip %s check as it is too large: %s (%s is allowed)",
144 rule.log_prefix, #content, max_size)
145 return false
146 end
147 return true
148 end
149
150 local function message_not_too_small(task, content, rule)
151 local min_size = tonumber(rule.min_size)
152 if not min_size then
153 return true
154 end
155 if #content < min_size then
156 rspamd_logger.infox(task, "skip %s check as it is too small: %s (%s is allowed)",
157 rule.log_prefix, #content, min_size)
158 return false
159 end
160 return true
161 end
162
163 local function message_min_words(task, rule)
164 if rule.text_part_min_words and tonumber(rule.text_part_min_words) > 0 then
165 local text_part_above_limit = false
166 local text_parts = task:get_text_parts()
167
168 local filter_func = function(p)
169 return p:get_words_count() >= tonumber(rule.text_part_min_words)
170 end
171
172 fun.each(function(p)
173 text_part_above_limit = true
174 end, fun.filter(filter_func, text_parts))
175
176 if not text_part_above_limit then
177 rspamd_logger.infox(task, '%s: #words in all text parts is below text_part_min_words limit: %s',
178 rule.log_prefix, rule.text_part_min_words)
179 end
180
181 return text_part_above_limit
182 else
183 return true
184 end
185 end
186
187 local function dynamic_scan(task, rule)
188 if rule.dynamic_scan then
189 if rule.action ~= 'reject' then
190 local metric_result = task:get_metric_score()
191 local metric_action = task:get_metric_action()
192 local has_pre_result = task:has_pre_result()
193 -- ToDo: needed?
194 -- Sometimes leads to FPs
195 --if rule.symbol_type == 'postfilter' and metric_action == 'reject' then
196 -- rspamd_logger.infox(task, '%s: aborting: %s', rule.log_prefix, "result is already reject")
197 -- return false
198 --elseif metric_result[1] > metric_result[2]*2 then
199 if metric_result[1] > metric_result[2] * 2 then
200 rspamd_logger.infox(task, '%s: aborting: %s', rule.log_prefix, 'score > 2 * reject_level: ' .. metric_result[1])
201 return false
202 elseif has_pre_result and metric_action == 'reject' then
203 rspamd_logger.infox(task, '%s: aborting: %s', rule.log_prefix, 'pre_result reject is set')
204 return false
205 else
206 return true, 'undecided'
207 end
208 else
209 return true, 'dynamic_scan is not possible with config `action=reject;`'
210 end
211 else
212 return true
213 end
214 end
215
216 local function need_check(task, content, rule, digest, fn, maybe_part)
217 local uncached = true
218 local key = digest
219
220 local function redis_av_cb(err, data)
221 if data and type(data) == 'string' then
222 -- Cached
223 data = lua_util.str_split(data, '\t')
224 local threat_string = lua_util.str_split(data[1], '\v')
225 local score = data[2] or rule.default_score
226
227 if threat_string[1] ~= 'OK' then
228 if threat_string[1] == 'MACRO' then
229 yield_result(task, rule, 'File contains macros',
230 0.0, 'macro', maybe_part)
231 elseif threat_string[1] == 'ENCRYPTED' then
232 yield_result(task, rule, 'File is encrypted',
233 0.0, 'encrypted', maybe_part)
234 else
235 -- Check if cached data contains symbol name (for category-based scanners)
236 -- Format: "SYMBOL_NAME\vdetails" or just "details"
237 if #threat_string >= 2 and rule.symbols then
238 -- New format with symbol name
239 local symbol_name = threat_string[1]
240 local details = threat_string[2]
241 lua_util.debugm(rule.name, task, '%s: got cached threat result for %s: %s - %s',
242 rule.log_prefix, key, symbol_name, details)
243 task:insert_result(symbol_name, 1.0, details)
244 else
245 -- Old format without symbol name
246 lua_util.debugm(rule.name, task, '%s: got cached threat result for %s: %s - score: %s',
247 rule.log_prefix, key, threat_string[1], score)
248 yield_result(task, rule, threat_string, score, false, maybe_part)
249 end
250 end
251 else
252 lua_util.debugm(rule.name, task, '%s: got cached negative result for %s: %s',
253 rule.log_prefix, key, threat_string[1])
254 end
255 uncached = false
256 else
257 if err then
258 rspamd_logger.errx(task, 'got error checking cache: %s', err)
259 end
260 end
261
262 local f_message_not_too_large = message_not_too_large(task, content, rule)
263 local f_message_not_too_small = message_not_too_small(task, content, rule)
264 local f_message_min_words = message_min_words(task, rule)
265 local f_dynamic_scan = dynamic_scan(task, rule)
266
267 if uncached and
268 f_message_not_too_large and
269 f_message_not_too_small and
270 f_message_min_words and
271 f_dynamic_scan then
272 fn()
273 end
274 end
275
276 if rule.redis_params and not rule.no_cache then
277 key = rule.prefix .. key
278
279 if lua_redis.redis_make_request(task,
280 rule.redis_params, -- connect params
281 key, -- hash key
282 false, -- is write
283 redis_av_cb, --callback
284 'GET', -- command
285 { key } -- arguments)
286 ) then
287 return true
288 end
289 end
290
291 return false
292 end
293
294 local function save_cache(task, digest, rule, to_save, dyn_weight, maybe_part)
295 local key = digest
296 if not dyn_weight then
297 dyn_weight = 1.0
298 end
299
300 local function redis_set_cb(err)
301 -- Do nothing
302 if err then
303 rspamd_logger.errx(task, 'failed to save %s cache for %s -> "%s": %s',
304 rule.detection_category, to_save, key, err)
305 else
306 lua_util.debugm(rule.name, task, '%s: saved cached result for %s: %s - score %s - ttl %s',
307 rule.log_prefix, key, to_save, dyn_weight, rule.cache_expire)
308 end
309 end
310
311 if type(to_save) == 'table' then
312 to_save = table.concat(to_save, '\v')
313 end
314
315 local value_tbl = { to_save, dyn_weight }
316 if maybe_part and rule.show_attachments and maybe_part:get_filename() then
317 local fname = maybe_part:get_filename()
318 table.insert(value_tbl, fname)
319 end
320 local value = table.concat(value_tbl, '\t')
321
322 if rule.redis_params and rule.prefix then
323 key = rule.prefix .. key
324
325 lua_redis.redis_make_request(task,
326 rule.redis_params, -- connect params
327 key, -- hash key
328 true, -- is write
329 redis_set_cb, --callback
330 'SETEX', -- command
331 { key, rule.cache_expire or 0, value }
332 )
333 end
334
335 return false
336 end
337
338 local function create_regex_table(patterns)
339 local regex_table = {}
340 if patterns[1] then
341 for i, p in ipairs(patterns) do
342 if type(p) == 'table' then
343 local new_set = {}
344 for k, v in pairs(p) do
345 new_set[k] = rspamd_regexp.create_cached(v)
346 end
347 regex_table[i] = new_set
348 else
349 regex_table[i] = {}
350 end
351 end
352 else
353 for k, v in pairs(patterns) do
354 regex_table[k] = rspamd_regexp.create_cached(v)
355 end
356 end
357 return regex_table
358 end
359
360 local function match_filter(task, rule, found, patterns, pat_type)
361 if type(patterns) ~= 'table' or not found then
362 return false
363 end
364 if not patterns[1] then
365 for _, pat in pairs(patterns) do
366 if pat_type == 'ext' and tostring(pat) == tostring(found) then
367 return true
368 elseif pat_type == 'regex' and pat:match(found) then
369 return true
370 end
371 end
372 return false
373 else
374 for _, p in ipairs(patterns) do
375 for _, pat in ipairs(p) do
376 if pat_type == 'ext' and tostring(pat) == tostring(found) then
377 return true
378 elseif pat_type == 'regex' and pat:match(found) then
379 return true
380 end
381 end
382 end
383 return false
384 end
385 end
386
387 -- borrowed from mime_types.lua
388 -- ext is the last extension, LOWERCASED
389 -- ext2 is the one before last extension LOWERCASED
390 local function gen_extension(fname)
391 local filename_parts = lua_util.str_split(fname, '.')
392
393 local ext = {}
394 for n = 1, 2 do
395 ext[n] = #filename_parts > n and string.lower(filename_parts[#filename_parts + 1 - n]) or nil
396 end
397 return ext[1], ext[2], filename_parts
398 end
399
400 local function check_parts_match(task, rule)
401 local filter_func = function(p)
402 local mtype, msubtype = p:get_type()
403 local detected_ext = p:get_detected_ext()
404 local fname = p:get_filename()
405 local ext, ext2
406
407 if rule.scan_all_mime_parts == false then
408 -- check file extension and filename regex matching
409 --lua_util.debugm(rule.name, task, '%s: filename: |%s|%s|', rule.log_prefix, fname)
410 if fname ~= nil then
411 ext, ext2 = gen_extension(fname)
412 --lua_util.debugm(rule.name, task, '%s: extension, fname: |%s|%s|%s|', rule.log_prefix, ext, ext2, fname)
413 if match_filter(task, rule, ext, rule.mime_parts_filter_ext, 'ext')
414 or match_filter(task, rule, ext2, rule.mime_parts_filter_ext, 'ext') then
415 lua_util.debugm(rule.name, task, '%s: extension matched: |%s|%s|', rule.log_prefix, ext, ext2)
416 return true
417 elseif match_filter(task, rule, fname, rule.mime_parts_filter_regex, 'regex') then
418 lua_util.debugm(rule.name, task, '%s: filename regex matched', rule.log_prefix)
419 return true
420 end
421 end
422 -- check content type string regex matching
423 if mtype ~= nil and msubtype ~= nil then
424 local ct = string.format('%s/%s', mtype, msubtype):lower()
425 if match_filter(task, rule, ct, rule.mime_parts_filter_regex, 'regex') then
426 lua_util.debugm(rule.name, task, '%s: regex content-type: %s', rule.log_prefix, ct)
427 return true
428 end
429 end
430 -- check detected content type (libmagic) regex matching
431 if detected_ext then
432 local magic = lua_magic_types[detected_ext] or {}
433 if match_filter(task, rule, detected_ext, rule.mime_parts_filter_ext, 'ext') then
434 lua_util.debugm(rule.name, task, '%s: detected extension matched: |%s|', rule.log_prefix, detected_ext)
435 return true
436 elseif magic.ct and match_filter(task, rule, magic.ct, rule.mime_parts_filter_regex, 'regex') then
437 lua_util.debugm(rule.name, task, '%s: regex detected libmagic content-type: %s',
438 rule.log_prefix, magic.ct)
439 return true
440 end
441 end
442 -- check filenames in archives
443 if p:is_archive() then
444 local arch = p:get_archive()
445 local filelist = arch:get_files_full(1000)
446 for _, f in ipairs(filelist) do
447 ext, ext2 = gen_extension(f.name)
448 if match_filter(task, rule, ext, rule.mime_parts_filter_ext, 'ext')
449 or match_filter(task, rule, ext2, rule.mime_parts_filter_ext, 'ext') then
450 lua_util.debugm(rule.name, task, '%s: extension matched in archive: |%s|%s|', rule.log_prefix, ext, ext2)
451 --lua_util.debugm(rule.name, task, '%s: extension matched in archive: %s', rule.log_prefix, ext)
452 return true
453 elseif match_filter(task, rule, f.name, rule.mime_parts_filter_regex, 'regex') then
454 lua_util.debugm(rule.name, task, '%s: filename regex matched in archive', rule.log_prefix)
455 return true
456 end
457 end
458 end
459 end
460
461 -- check text_part has more words than text_part_min_words_check
462 if rule.scan_text_mime and rule.text_part_min_words and p:is_text() and
463 p:get_text():get_words_count() >= tonumber(rule.text_part_min_words) then
464 return true
465 end
466
467 if rule.scan_image_mime and p:is_image() then
468 return true
469 end
470
471 if rule.scan_all_mime_parts ~= false then
472 local is_part_checkable = (p:is_attachment() and (not p:is_image() or rule.scan_image_mime))
473 if detected_ext then
474 -- We know what to scan!
475 local magic = lua_magic_types[detected_ext] or {}
476
477 if magic.av_check ~= false or is_part_checkable then
478 return true
479 end
480 elseif is_part_checkable then
481 -- Just rely on attachment property
482 return true
483 end
484 end
485
486 return false
487 end
488
489 return fun.filter(filter_func, task:get_parts())
490 end
491
492 local function check_metric_results(task, rule)
493 if rule.action ~= 'reject' then
494 local metric_result = task:get_metric_score()
495 local metric_action = task:get_metric_action()
496 local has_pre_result = task:has_pre_result()
497
498 if rule.symbol_type == 'postfilter' and metric_action == 'reject' then
499 return true, 'result is already reject'
500 elseif metric_result[1] > metric_result[2] * 2 then
501 return true, 'score > 2 * reject_level: ' .. metric_result[1]
502 elseif has_pre_result and metric_action == 'reject' then
503 return true, 'pre_result reject is set'
504 else
505 return false, 'undecided'
506 end
507 else
508 return false, 'dynamic_scan is not possible with config `action=reject;`'
509 end
510 end
511
512 exports.log_clean = log_clean
513 exports.yield_result = yield_result
514 exports.match_patterns = match_patterns
515 exports.condition_check_and_continue = need_check
516 exports.save_cache = save_cache
517 exports.create_regex_table = create_regex_table
518 exports.check_parts_match = check_parts_match
519 exports.check_metric_results = check_metric_results
520
521 setmetatable(exports, {
522 __call = function(t, override)
523 for k, v in pairs(t) do
524 if _G[k] ~= nil then
525 local msg = 'function ' .. k .. ' already exists in global scope.'
526 if override then
527 _G[k] = v
528 print('WARNING: ' .. msg .. ' Overwritten.')
529 else
530 print('NOTICE: ' .. msg .. ' Skipped.')
531 end
532 else
533 _G[k] = v
534 end
535 end
536 end,
537 })
538
539 return exports