Debian Codegraph @

lualib/lua_fuzzy.lua

raw
1 --[[
2 Copyright (c) 2022, Vsevolod Stakhov <[email protected]>
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 ]]--
16
17 --[[[
18 -- @module lua_fuzzy
19 -- This module contains helper functions for supporting fuzzy check module
20 --]]
21
22
23 local N = "lua_fuzzy"
24 local lua_util = require "lua_util"
25 local rspamd_regexp = require "rspamd_regexp"
26 local fun = require "fun"
27 local rspamd_logger = require "rspamd_logger"
28 local T = require "lua_shape.core"
29
30 -- Filled by C code, indexed by number in this table
31 local rules = {}
32
33 -- Pre-defined rules options
34 local policies = {
35 recommended = {
36 min_bytes = 1024,
37 min_height = 500,
38 min_width = 500,
39 min_length = 64,
40 text_multiplier = 4.0, -- divide min_bytes by 4 for texts
41 mime_types = { "application/*" },
42 scan_archives = true,
43 short_text_direct_hash = true,
44 text_shingles = true,
45 text_hashes = true,
46 skip_images = false,
47 }
48 }
49
50 local default_policy = policies.recommended
51
52 local schema_fields = {
53 min_bytes = T.transform(
54 T.one_of({T.number(), T.string()}),
55 tonumber
56 ),
57 min_height = T.transform(
58 T.one_of({T.number(), T.string()}),
59 tonumber
60 ),
61 min_width = T.transform(
62 T.one_of({T.number(), T.string()}),
63 tonumber
64 ),
65 min_length = T.transform(
66 T.one_of({T.number(), T.string()}),
67 tonumber
68 ),
69 text_multiplier = T.number(),
70 mime_types = T.array(T.string()),
71 scan_archives = T.boolean(),
72 short_text_direct_hash = T.boolean(),
73 text_shingles = T.boolean(),
74 text_hashes = T.boolean(),
75 skip_images = T.boolean(),
76 }
77 local policy_schema = T.table(schema_fields)
78
79 local policy_schema_open = T.table(schema_fields, {
80 open = true,
81 })
82
83 local exports = {}
84
85 local function apply_checks_overrides(rule)
86 local checks = rule.checks
87
88 if type(checks) ~= 'table' then
89 return
90 end
91
92 local function find_section(name)
93 local lname = name:lower()
94
95 for k, v in pairs(checks) do
96 if type(k) == 'string' and k:lower() == lname then
97 return v
98 end
99 end
100
101 return nil
102 end
103
104 local function bool_opt(section, key)
105 if type(section) ~= 'table' then
106 return nil
107 end
108
109 if section[key] == nil then
110 return nil
111 end
112
113 return lua_util.toboolean(section[key])
114 end
115
116 local function number_opt(section, key)
117 if type(section) ~= 'table' then
118 return nil
119 end
120
121 if section[key] == nil then
122 return nil
123 end
124
125 return tonumber(section[key])
126 end
127
128 local text_section = find_section('text')
129
130 if text_section then
131 local enabled = bool_opt(text_section, 'enabled')
132
133 if enabled == nil then
134 enabled = true
135 end
136
137 rule.text_hashes = enabled
138
139 local opt = bool_opt(text_section, 'no_subject')
140
141 if opt ~= nil then
142 rule.no_subject = opt
143 end
144
145 opt = bool_opt(text_section, 'short_text_direct_hash')
146
147 if opt ~= nil then
148 rule.short_text_direct_hash = opt
149 end
150
151 local num = number_opt(text_section, 'min_length')
152
153 if num ~= nil then
154 rule.min_length = num
155 end
156
157 num = number_opt(text_section, 'text_multiplier')
158
159 if num ~= nil then
160 rule.text_multiplier = num
161 end
162 end
163
164 local html_section = find_section('html')
165
166 if html_section then
167 local enabled = bool_opt(html_section, 'enabled')
168
169 if enabled == nil then
170 enabled = true
171 end
172
173 rule.html_shingles = enabled
174
175 local num = number_opt(html_section, 'min_html_tags')
176
177 if num == nil then
178 num = number_opt(html_section, 'min_tags')
179 end
180
181 if num ~= nil then
182 rule.min_html_tags = num
183 end
184
185 num = number_opt(html_section, 'html_weight')
186
187 if num == nil then
188 num = number_opt(html_section, 'weight')
189 end
190
191 if num ~= nil then
192 rule.html_weight = num
193 end
194 end
195
196 local image_section = find_section('image') or find_section('images')
197
198 if image_section then
199 local enabled = bool_opt(image_section, 'enabled')
200
201 if enabled == nil then
202 enabled = true
203 end
204
205 rule.skip_images = not enabled
206
207 local num = number_opt(image_section, 'min_height')
208
209 if num ~= nil then
210 rule.min_height = num
211 end
212
213 num = number_opt(image_section, 'min_width')
214
215 if num ~= nil then
216 rule.min_width = num
217 end
218 end
219
220 local archive_section = find_section('archive') or find_section('archives')
221
222 if archive_section then
223 local enabled = bool_opt(archive_section, 'enabled')
224
225 if enabled == nil then
226 enabled = true
227 end
228
229 rule.scan_archives = enabled
230 end
231
232 rule.checks = nil
233 end
234
235 --[[[
236 -- @function lua_fuzzy.register_policy(name, policy)
237 -- Adds a new policy with name `name`. Must be valid, checked using policy_schema
238 --]]
239 exports.register_policy = function(name, policy)
240 if policies[name] then
241 rspamd_logger.warnx(rspamd_config, "overriding policy %s", name)
242 end
243
244 local parsed_policy, err = policy_schema:transform(policy)
245
246 if not parsed_policy then
247 rspamd_logger.errx(rspamd_config, 'invalid fuzzy rule policy %s: %s',
248 name, err)
249
250 return
251 else
252 policies.name = parsed_policy
253 end
254 end
255
256 --[[[
257 -- @function lua_fuzzy.process_rule(rule)
258 -- Processes fuzzy rule (applying policies or defaults if needed). Returns policy id
259 --]]
260 exports.process_rule = function(rule)
261 local processed_rule = lua_util.shallowcopy(rule)
262 local policy = default_policy
263
264 if processed_rule.policy then
265 policy = policies[processed_rule.policy]
266 end
267
268 if policy then
269 processed_rule = lua_util.override_defaults(policy, processed_rule)
270
271 apply_checks_overrides(processed_rule)
272
273 local parsed_policy, err = policy_schema_open:transform(processed_rule)
274
275 if not parsed_policy then
276 rspamd_logger.errx(rspamd_config, 'invalid fuzzy rule default fields: %s', err)
277 else
278 processed_rule = parsed_policy
279 end
280 else
281 rspamd_logger.warnx(rspamd_config, "unknown policy %s", processed_rule.policy)
282 apply_checks_overrides(processed_rule)
283 end
284
285 if processed_rule.mime_types then
286 processed_rule.mime_types = fun.totable(fun.map(function(gl)
287 return rspamd_regexp.import_glob(gl, 'i')
288 end, processed_rule.mime_types))
289 end
290
291 table.insert(rules, processed_rule)
292 return #rules
293 end
294
295 -- CJK languages use multi-byte characters (3 bytes per char in UTF-8) and
296 -- carry more semantic content per token than Latin languages.
297 local function is_cjk_language(lang)
298 return lang and (lang == 'ja' or lang == 'zh' or lang == 'ko')
299 end
300
301 local function check_length(task, part, rule)
302 local bytes = part:get_length()
303 local length_ok = bytes > 0
304
305 local id = part:get_id()
306 lua_util.debugm(N, task, 'check size of part %s', id)
307
308 if length_ok and rule.min_bytes > 0 then
309
310 local adjusted_bytes = bytes
311
312 if part:is_text() then
313 -- Fuzzy plugin uses stripped utf content to get an exact hash, that
314 -- corresponds to `get_content_oneline()`
315 -- However, in the case of empty parts this method returns `nil`, so extra
316 -- sanity check is required.
317 bytes = #(part:get_text():get_content_oneline() or '')
318
319 -- Short hashing algorithm also use subject unless explicitly denied
320 if not rule.no_subject then
321 local subject = task:get_subject() or ''
322 bytes = bytes + #subject
323 end
324
325 if rule.text_multiplier then
326 local multiplier = rule.text_multiplier
327
328 -- CJK characters are 3 bytes in UTF-8, so the same semantic content
329 -- takes ~3x more bytes than Latin text; boost the multiplier to compensate
330 local lang = part:get_text():get_language()
331 if is_cjk_language(lang) then
332 multiplier = multiplier * 3.0
333 lua_util.debugm(N, task, 'CJK language %s: boosted text_multiplier to %s',
334 lang, multiplier)
335 end
336
337 adjusted_bytes = bytes * multiplier
338 end
339 end
340
341 if rule.min_bytes > adjusted_bytes then
342 lua_util.debugm(N, task, 'skip part of length %s (%s adjusted) ' ..
343 'as it has less than %s bytes',
344 bytes, adjusted_bytes, rule.min_bytes)
345 length_ok = false
346 else
347 lua_util.debugm(N, task, 'allow part of length %s (%s adjusted)',
348 bytes, adjusted_bytes, rule.min_bytes)
349 end
350 else
351 lua_util.debugm(N, task, 'allow part %s, no length limits', id)
352 end
353
354 return length_ok
355 end
356
357 local function check_text_part(task, part, rule, text)
358 local allow_direct, allow_shingles = false, false
359
360 local id = part:get_id()
361 lua_util.debugm(N, task, 'check text part %s', id)
362
363 if rule.text_hashes == false then
364 lua_util.debugm(N, task, 'text hashes disabled, relying on HTML for part %s', id)
365 return rule.html_shingles == true, false
366 end
367
368 local wcnt = text:get_words_count()
369
370 if rule.text_shingles then
371 -- Check number of words
372 local min_words = rule.min_length or 0
373 local min_floor = 32
374
375 -- CJK morphemes carry higher semantic density per token, so fewer words
376 -- are needed for meaningful shingle generation (3-word window still works
377 -- well with as few as 12 tokens producing 10 windows)
378 local lang = text:get_language()
379 if is_cjk_language(lang) then
380 min_words = math.floor(min_words / 3)
381 min_floor = 12
382 lua_util.debugm(N, task, 'CJK language %s: adjusted min_words to %s (floor %s)',
383 lang, min_words, min_floor)
384 end
385
386 if min_words < min_floor then
387 min_words = min_floor
388 end
389 if wcnt < min_words then
390 lua_util.debugm(N, task, 'text has less than %s words: %s; disable shingles',
391 min_words, wcnt)
392 allow_shingles = false
393 else
394 lua_util.debugm(N, task, 'allow shingles in text %s, %s words',
395 id, wcnt)
396 allow_shingles = true
397 end
398
399 if not rule.short_text_direct_hash and not allow_shingles then
400 allow_direct = false
401 else
402 if not allow_shingles then
403 lua_util.debugm(N, task,
404 'allow direct hash for short text %s, %s words',
405 id, wcnt)
406 allow_direct = check_length(task, part, rule)
407 else
408 allow_direct = wcnt > 0
409 end
410 end
411 else
412 lua_util.debugm(N, task,
413 'disable shingles in text %s', id)
414 allow_direct = check_length(task, part, rule)
415 end
416
417 return allow_direct, allow_shingles
418 end
419
420 --local function has_sane_text_parts(task)
421 -- local text_parts = task:get_text_parts() or {}
422 -- return fun.any(function(tp) return tp:get_words_count() > 32 end, text_parts)
423 --end
424
425 local function check_image_part(task, part, rule, image)
426 if rule.skip_images then
427 lua_util.debugm(N, task, 'skip image part as images are disabled')
428 return false, false
429 end
430
431 local id = part:get_id()
432 lua_util.debugm(N, task, 'check image part %s', id)
433
434 if rule.min_width > 0 or rule.min_height > 0 then
435 -- Check dimensions
436 local min_width = rule.min_width or rule.min_height
437 local min_height = rule.min_height or rule.min_width
438 local height = image:get_height()
439 local width = image:get_width()
440
441 if height and width then
442 if height < min_height or width < min_width then
443 lua_util.debugm(N, task, 'skip image part %s as it does not meet minimum sizes: %sx%s < %sx%s',
444 id, width, height, min_width, min_height)
445 return false, false
446 else
447 lua_util.debugm(N, task, 'allow image part %s: %sx%s',
448 id, width, height)
449 end
450 end
451 end
452
453 return check_length(task, part, rule), false
454 end
455
456 local function mime_types_check(task, part, rule)
457 local t, st = part:get_type()
458
459 if not t then
460 return false, false
461 end
462
463 local ct = string.format('%s/%s', t, st)
464
465 local detected_ct
466 t, st = part:get_detected_type()
467 if t then
468 detected_ct = string.format('%s/%s', t, st)
469 else
470 detected_ct = ct
471 end
472
473 local id = part:get_id()
474 lua_util.debugm(N, task, 'check binary part %s: %s', id, ct)
475
476 -- For bad mime parts we implicitly enable fuzzy check
477 local mime_trace = (task:get_symbol('MIME_TRACE') or {})[1]
478 local opts = {}
479
480 if mime_trace then
481 opts = mime_trace.options or opts
482 end
483 opts = fun.tomap(fun.map(function(opt)
484 local elts = lua_util.str_split(opt, ':')
485 return elts[1], elts[2]
486 end, opts))
487
488 if opts[id] and opts[id] == '-' then
489 lua_util.debugm(N, task, 'explicitly check binary part %s: bad mime type %s', id, ct)
490 return check_length(task, part, rule), false
491 end
492
493 if rule.mime_types then
494
495 if fun.any(function(gl_re)
496 if gl_re:match(ct) or (detected_ct and gl_re:match(detected_ct)) then
497 return true
498 else
499 return false
500 end
501 end, rule.mime_types) then
502 lua_util.debugm(N, task, 'found mime type match for part %s: %s (%s detected)',
503 id, ct, detected_ct)
504 return check_length(task, part, rule), false
505 end
506
507 return false, false
508 end
509
510 return false, false
511 end
512
513 exports.check_mime_part = function(task, part, rule_id)
514 local rule = rules[rule_id]
515
516 if not rule then
517 rspamd_logger.errx(task, 'cannot find rule with id %s', rule_id)
518
519 return false, false
520 end
521
522 if part:is_text() then
523 return check_text_part(task, part, rule, part:get_text())
524 end
525
526 if part:is_image() then
527 return check_image_part(task, part, rule, part:get_image())
528 end
529
530 if part:is_archive() and rule.scan_archives then
531 -- Always send archives
532 lua_util.debugm(N, task, 'check archive part %s', part:get_id())
533
534 return true, false
535 end
536
537 if part:is_specific() then
538 local sp = part:get_specific()
539
540 if type(sp) == 'table' and sp.fuzzy_hashes then
541 lua_util.debugm(N, task, 'check specific part %s', part:get_id())
542 return true, false
543 end
544 end
545
546 if part:is_attachment() then
547 return mime_types_check(task, part, rule)
548 end
549
550 return false, false
551 end
552
553 exports.cleanup_rules = function()
554 rules = {}
555 end
556
557 return exports