Debian Codegraph @

lualib/lua_bayes_learn.lua

raw
1 --[[
2 Copyright (c) 2022, Vsevolod Stakhov <[email protected]>
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 ]]--
16
17 -- This file contains functions to simplify bayes classifier auto-learning
18
19 local lua_util = require "lua_util"
20 local lua_verdict = require "lua_verdict"
21 local logger = require "rspamd_logger"
22 local N = "lua_bayes"
23
24 local exports = {}
25
26 local function as_set(tbl, case_sensitive)
27 if not tbl then
28 return nil
29 end
30
31 local res = {}
32
33 for k, v in pairs(tbl) do
34 if type(k) == 'number' then
35 if type(v) == 'string' then
36 res[case_sensitive and v or v:lower()] = true
37 else
38 res[v] = true
39 end
40 else
41 if type(v) == 'boolean' then
42 res[case_sensitive and k or k:lower()] = v
43 else
44 res[case_sensitive and k or k:lower()] = true
45 end
46 end
47 end
48
49 return res
50 end
51
52 local function merge_options(defaults, module_defaults, overrides)
53 local merged = lua_util.override_defaults(defaults, module_defaults or {})
54
55 if overrides then
56 merged = lua_util.override_defaults(merged, overrides)
57 end
58
59 return merged
60 end
61
62 local function interpret_guard_result(res, reason, extra)
63 local ok = res
64 local msg = reason
65 local meta = extra
66
67 if type(res) == 'table' then
68 ok = res.ok
69 msg = res.reason or res.message or reason
70 if res.meta then
71 meta = res.meta
72 else
73 meta = res
74 end
75
76 if ok == nil then
77 if res.allow ~= nil then
78 ok = res.allow
79 elseif res.block ~= nil then
80 ok = not res.block
81 end
82 end
83
84 if ok == nil then
85 ok = true
86 end
87
88 if res.stop ~= nil then
89 meta = meta or {}
90 meta.stop = res.stop and true or nil
91 end
92 else
93 if ok == nil then
94 ok = true
95 end
96 end
97
98 return ok, msg, meta
99 end
100
101 local function execute_guards(guards, ctx, default_reason)
102 if not guards then
103 return true
104 end
105
106 for _, guard in ipairs(guards) do
107 local ok, msg, meta = interpret_guard_result(guard.cb(ctx), default_reason, nil)
108
109 if not ok then
110 if ctx.result then
111 ctx.result.guard = guard.name
112 ctx.result.reason = msg or default_reason
113 if meta then
114 ctx.result.meta = meta
115 end
116 end
117
118 return false, msg, meta
119 end
120
121 if meta and meta.stop then
122 return true, msg, meta
123 end
124 end
125
126 return true
127 end
128
129 local function register_guard(registry, name, cb, opts)
130 if type(name) == 'function' then
131 cb = name
132 name = nil
133 end
134
135 if type(cb) ~= 'function' then
136 return nil, 'guard callback must be a function'
137 end
138
139 local guard = {
140 name = name or string.format('guard_%d', (#registry) + 1),
141 cb = cb,
142 priority = opts and opts.priority or 0,
143 }
144
145 registry[#registry + 1] = guard
146 table.sort(registry, function(a, b)
147 return (a.priority or 0) > (b.priority or 0)
148 end)
149
150 return guard.name
151 end
152
153 local function unregister_guard(registry, name)
154 for i = #registry, 1, -1 do
155 if registry[i].name == name then
156 table.remove(registry, i)
157 end
158 end
159 end
160
161 local can_learn_defaults = {}
162 local autolearn_defaults = {}
163
164 local can_learn_guards = {}
165 local autolearn_guards = {}
166
167 local default_can_learn_settings = {
168 bypass_header = {
169 enabled = true,
170 header = 'Learn-Type',
171 values = {'bulk'},
172 behaviour = 'skip_probability',
173 case_sensitive = false,
174 },
175 probability_check = {
176 enabled = true,
177 -- 'can_learn_prob' is written by the C layer (rspamd_stat_classifier_is_skipped)
178 -- before invoking this condition. It holds the per-classifier, per-class
179 -- probability so each classifier's can_learn decision is independent.
180 variable = 'can_learn_prob',
181 ctype = 'double',
182 -- Unified threshold: >= min_prob means "already in this class, skip learning".
183 -- Replaces the old asymmetric spam_min/ham_max pair.
184 min_prob = 0.95,
185 skip_for_unlearn = false,
186 require_value = false,
187 },
188 }
189
190 local default_autolearn_settings = {
191 require_queue_id = true,
192 check_local = true,
193 check_authed = true,
194 verdict_source = {
195 name = 'bayes',
196 },
197 logging = {
198 enabled = true,
199 },
200 thresholds = {
201 spam = nil,
202 ham = nil,
203 junk = nil,
204 },
205 learn_verdict = false,
206 balance = {
207 enabled = false,
208 min_balance = 0.9,
209 spam_key = 'spam_learns',
210 ham_key = 'ham_learns',
211 value_type = 'int64',
212 },
213 }
214
215 --- Allows global overrides for can_learn defaults (e.g. from config)
216 -- @param opts table of default overrides
217 exports.configure_can_learn = function(opts)
218 if opts then
219 can_learn_defaults = lua_util.override_defaults(can_learn_defaults, opts)
220 end
221 end
222
223 --- Allows global overrides for autolearn defaults (e.g. from config)
224 -- @param opts table of default overrides
225 exports.configure_autolearn = function(opts)
226 if opts then
227 autolearn_defaults = lua_util.override_defaults(autolearn_defaults, opts)
228 end
229 end
230
231 --- Register an extra guard for can_learn checks
232 -- @param name string or callback (name optional)
233 -- @param cb guard callback
234 -- @param opts optional table with priority
235 exports.register_can_learn_guard = function(name, cb, opts)
236 return register_guard(can_learn_guards, name, cb, opts)
237 end
238
239 --- Remove a previously registered can_learn guard by name
240 exports.unregister_can_learn_guard = function(name)
241 unregister_guard(can_learn_guards, name)
242 end
243
244 --- Register an extra guard for autolearn decisions
245 -- @param name string or callback (name optional)
246 -- @param cb guard callback
247 -- @param opts optional table with priority
248 exports.register_autolearn_guard = function(name, cb, opts)
249 return register_guard(autolearn_guards, name, cb, opts)
250 end
251
252 --- Remove a previously registered autolearn guard by name
253 exports.unregister_autolearn_guard = function(name)
254 unregister_guard(autolearn_guards, name)
255 end
256
257
258 --- Determines if a message can be learned by Bayes
259 -- @param task rspamd_task
260 -- @param is_spam boolean indicates target class
261 -- @param is_unlearn boolean indicates unlearn operation
262 -- @param overrides optional per-call overrides
263 exports.can_learn = function(task, is_spam, is_unlearn, overrides)
264 local opts = merge_options(default_can_learn_settings, can_learn_defaults, overrides)
265
266 if opts.bypass_header and opts.bypass_header.values then
267 opts.bypass_header._set = as_set(opts.bypass_header.values, opts.bypass_header.case_sensitive)
268 end
269
270 local ctx = {
271 task = task,
272 is_spam = is_spam,
273 is_unlearn = is_unlearn,
274 options = opts,
275 state = {},
276 result = {},
277 }
278
279 if overrides and overrides.guards then
280 local ok, msg, meta = execute_guards(overrides.guards, ctx, 'blocked by can_learn guard')
281
282 if not ok then
283 return false, msg, ctx.result
284 end
285
286 if meta and meta.stop then
287 return true, nil, ctx.result
288 end
289 end
290
291 local ok, msg = execute_guards(can_learn_guards, ctx, 'blocked by can_learn guard')
292
293 if not ok then
294 return false, msg, ctx.result
295 end
296
297 local probability_opts = opts.probability_check
298 local skip_probability = false
299
300 if opts.bypass_header and opts.bypass_header.enabled ~= false then
301 local header_name = opts.bypass_header.header or 'Learn-Type'
302 local header_value = task:get_request_header(header_name)
303
304 if header_value then
305 header_value = tostring(header_value)
306 if opts.bypass_header.case_sensitive ~= true then
307 header_value = header_value:lower()
308 end
309
310 local matched
311 if opts.bypass_header._set then
312 matched = opts.bypass_header._set[header_value]
313 else
314 matched = false
315 end
316
317 if matched then
318 if opts.bypass_header.behaviour == 'allow' then
319 ctx.result.reason = 'bypass header matched'
320 return true, nil, ctx.result
321 elseif opts.bypass_header.behaviour == 'deny' then
322 ctx.result.reason = opts.bypass_header.reason or 'bypass header denies learning'
323 ctx.result.guard = 'bypass_header'
324 return false, ctx.result.reason, ctx.result
325 else
326 skip_probability = true
327 end
328 end
329 end
330 end
331
332 if ctx.state.skip_probability ~= nil then
333 skip_probability = ctx.state.skip_probability
334 end
335
336 if probability_opts and probability_opts.enabled ~= false and not skip_probability then
337 if is_unlearn and probability_opts.skip_for_unlearn then
338 ctx.result.reason = 'probability check skipped for unlearn'
339 else
340 local prob
341
342 if probability_opts.resolver and type(probability_opts.resolver) == 'function' then
343 prob = probability_opts.resolver(ctx)
344 else
345 prob = task:get_mempool():get_variable(probability_opts.variable or 'bayes_prob',
346 probability_opts.ctype or 'double')
347 end
348
349 ctx.result.probability = prob
350
351 if prob == nil then
352 if probability_opts.require_value then
353 local reason = probability_opts.missing_reason or 'probability value is missing'
354 ctx.result.guard = 'probability_check'
355 ctx.result.reason = reason
356 return false, reason, ctx.result
357 end
358 else
359 local in_class
360 local guard_msg
361
362 if probability_opts.check and type(probability_opts.check) == 'function' then
363 in_class, guard_msg = probability_opts.check(ctx, prob)
364 else
365 -- Unified check: high probability means the message is already confidently
366 -- in the target class (works for both binary spam/ham and multiclass).
367 -- can_learn_prob is set per-classifier by C so there is no cross-contamination.
368 in_class = prob >= (probability_opts.min_prob or
369 probability_opts.spam_min or 0.95)
370 end
371
372 if in_class then
373 -- class name is written by C before invoking this condition
374 local cl = task:get_mempool():get_variable('can_learn_class') or
375 (is_spam and 'spam' or 'ham')
376 local reason
377
378 if probability_opts.message_formatter and type(probability_opts.message_formatter) == 'function' then
379 reason = probability_opts.message_formatter(ctx, prob, cl) or guard_msg
380 end
381
382 reason = reason or guard_msg or
383 string.format('already in class %s; probability %.2f%%', cl, prob * 100.0)
384
385 ctx.result.guard = 'probability_check'
386 ctx.result.reason = reason
387
388 return false, reason, ctx.result
389 end
390 end
391 end
392 end
393
394 ctx.result.guard = ctx.result.guard or 'can_learn'
395
396 return true, nil, ctx.result
397 end
398
399 --- Decide if a message should be auto-learned and return class
400 -- @param task rspamd_task
401 -- @param conf classifier autolearn configuration
402 -- @param overrides optional per-call overrides
403 exports.autolearn = function(task, conf, overrides)
404 local opts = merge_options(default_autolearn_settings, autolearn_defaults, overrides)
405 opts = merge_options(opts, {}, conf)
406
407 if opts.check_balance ~= nil then
408 opts.balance = opts.balance or {}
409 if opts.balance.enabled == nil then
410 opts.balance.enabled = opts.check_balance and true or false
411 end
412 end
413
414 if opts.min_balance ~= nil then
415 opts.balance = opts.balance or {}
416 if opts.balance.min_balance == nil then
417 opts.balance.min_balance = opts.min_balance
418 end
419 end
420
421 local external_options = opts.options
422 if external_options ~= nil then
423 opts.options = nil
424
425 if type(external_options) == 'function' then
426 local ok, res = pcall(external_options, task, opts)
427
428 if ok and type(res) == 'table' then
429 opts = merge_options(opts, {}, res)
430 else
431 lua_util.debugm(N, task, 'autolearn options callback failed: %s', res)
432 end
433 elseif type(external_options) == 'table' then
434 opts = merge_options(opts, {}, external_options)
435 else
436 lua_util.debugm(N, task, 'autolearn options must be a table or function, got %s',
437 type(external_options))
438 end
439 end
440
441 local ctx = {
442 task = task,
443 conf = opts,
444 state = {},
445 result = {},
446 }
447
448 if overrides and overrides.guards then
449 local ok, msg, meta = execute_guards(overrides.guards, ctx, 'blocked by autolearn guard')
450
451 if not ok then
452 return nil, msg, ctx.result
453 end
454
455 if meta and meta.stop then
456 return ctx.result.decision, msg, ctx.result
457 end
458 end
459
460 local ok, msg = execute_guards(autolearn_guards, ctx, 'blocked by autolearn guard')
461
462 if not ok then
463 return nil, msg, ctx.result
464 end
465
466 if opts.require_queue_id and not task:get_queue_id() then
467 lua_util.debugm(N, task, 'no need to autolearn - queue id is missing')
468 ctx.result.reason = 'queue id is missing'
469 return nil, ctx.result.reason, ctx.result
470 end
471
472 local skip_conf = {opts.check_local, opts.check_authed}
473 if lua_util.is_skip_local_or_authed(task, skip_conf) then
474 lua_util.debugm(N, task, 'skip autolearn for local or authed network')
475 ctx.result.reason = 'local or authed network'
476 return nil, ctx.result.reason, ctx.result
477 end
478
479 local verdict_source = opts.verdict_source or {}
480 local verdict, score
481
482 if verdict_source.extractor and type(verdict_source.extractor) == 'function' then
483 verdict, score = verdict_source.extractor(ctx)
484 else
485 verdict, score = lua_verdict.get_specific_verdict(verdict_source.name or 'bayes', task)
486 end
487
488 ctx.result.verdict = verdict
489 ctx.result.score = score
490
491 if verdict == 'passthrough' then
492 lua_util.debugm(N, task, 'no need to autolearn - verdict: %s', verdict)
493 ctx.result.reason = 'verdict passthrough'
494 return nil, ctx.result.reason, ctx.result
495 end
496
497 local learn_spam, learn_ham = false, false
498
499 local thresholds = opts.thresholds or {}
500 thresholds.spam = thresholds.spam or opts.spam_threshold
501 thresholds.ham = thresholds.ham or opts.ham_threshold
502 thresholds.junk = thresholds.junk or opts.junk_threshold
503
504 local log_opts = opts.logging or {}
505 local function log_can_autolearn(verdict_name, score_value, threshold)
506 if log_opts.enabled == false then
507 return
508 end
509
510 local from = task:get_from('smtp')
511 local mime_rcpts = 'undef'
512 local mr = task:get_recipients('mime')
513 if mr then
514 local r_addrs = {}
515 for _, r in ipairs(mr) do
516 r_addrs[#r_addrs + 1] = r.addr
517 end
518 if #r_addrs > 0 then
519 mime_rcpts = table.concat(r_addrs, ',')
520 end
521 end
522
523 logger.info(task, 'id: %s, from: <%s>: can autolearn %s: score %s %s %s, mime_rcpts: <%s>',
524 task:get_header('Message-Id') or '<undef>',
525 from and from[1].addr or 'undef',
526 verdict_name,
527 string.format('%.2f', score_value or 0),
528 verdict_name == 'ham' and '<=' or verdict_name == 'spam' and '>=' or '/',
529 threshold,
530 mime_rcpts)
531 end
532
533 if thresholds.spam and thresholds.ham then
534 if verdict == 'spam' then
535 if score and score >= thresholds.spam then
536 log_can_autolearn(verdict, score, thresholds.spam)
537 learn_spam = true
538 end
539 elseif verdict == 'junk' then
540 if thresholds.junk and score and score >= thresholds.junk then
541 log_can_autolearn(verdict, score, thresholds.junk)
542 learn_spam = true
543 end
544 elseif verdict == 'ham' then
545 if score and score <= thresholds.ham then
546 log_can_autolearn(verdict, score, thresholds.ham)
547 learn_ham = true
548 end
549 end
550 elseif opts.learn_verdict then
551 if verdict == 'spam' or verdict == 'junk' then
552 learn_spam = true
553 elseif verdict == 'ham' then
554 learn_ham = true
555 end
556 elseif opts.evaluate and type(opts.evaluate) == 'function' then
557 local decision = opts.evaluate(ctx)
558
559 if decision == 'spam' then
560 learn_spam = true
561 elseif decision == 'ham' then
562 learn_ham = true
563 end
564 end
565
566 if opts.balance and opts.balance.enabled then
567 local balance_opts = opts.balance
568 local spam_learns = task:get_mempool():get_variable(balance_opts.spam_key or 'spam_learns', balance_opts.value_type or 'int64') or 0
569 local ham_learns = task:get_mempool():get_variable(balance_opts.ham_key or 'ham_learns', balance_opts.value_type or 'int64') or 0
570
571 local min_balance = balance_opts.min_balance or 0.9
572
573 if spam_learns > 0 or ham_learns > 0 then
574 local max_ratio = 1.0 / min_balance
575 local spam_learns_ratio = spam_learns / (ham_learns + 1)
576 if spam_learns_ratio > max_ratio and learn_spam then
577 lua_util.debugm(N, task,
578 'skip learning spam, balance is not satisfied: %s < %s; %s spam learns; %s ham learns',
579 spam_learns_ratio, min_balance, spam_learns, ham_learns)
580 learn_spam = false
581 ctx.result.reason = 'spam balance check failed'
582 end
583
584 local ham_learns_ratio = ham_learns / (spam_learns + 1)
585 if ham_learns_ratio > max_ratio and learn_ham then
586 lua_util.debugm(N, task,
587 'skip learning ham, balance is not satisfied: %s < %s; %s spam learns; %s ham learns',
588 ham_learns_ratio, min_balance, spam_learns, ham_learns)
589 learn_ham = false
590 ctx.result.reason = 'ham balance check failed'
591 end
592 end
593 end
594
595 if learn_spam then
596 ctx.result.decision = 'spam'
597 return 'spam', nil, ctx.result
598 elseif learn_ham then
599 ctx.result.decision = 'ham'
600 return 'ham', nil, ctx.result
601 end
602
603 return nil, ctx.result.reason, ctx.result
604 end
605
606 return exports