Debian Codegraph @

lualib/lua_neural_learn.lua

raw
1 --[[
2 Copyright (c) 2024, Vsevolod Stakhov <[email protected]>
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 ]]--
16
17 --[[
18 Neural network autolearn helpers.
19
20 This module provides configurable autolearn conditions for neural networks,
21 particularly useful for LLM-based providers where automatic learning needs
22 careful control.
23
24 Similar to lua_bayes_learn.lua, this provides:
25 - Guards system for pluggable checks
26 - Expression-based conditions (rspamd_expression)
27 - Score/action/symbol-based thresholds
28 - Hooks for custom logic via rspamd_plugins
29
30 Usage in neural.lua:
31 local neural_learn = require "lua_neural_learn"
32 local can_learn, reason = neural_learn.can_autolearn(task, rule, 'spam')
33 ]]--
34
35 local lua_util = require "lua_util"
36 local rspamd_expression = require "rspamd_expression"
37 local rspamd_logger = require "rspamd_logger"
38
39 local N = "lua_neural_learn"
40
41 local exports = {}
42
43 -- Global defaults that can be overridden via configure()
44 local global_defaults = {}
45
46 -- Registered guards (callbacks that can block learning)
47 local autolearn_guards = {}
48
49 -- Cached compiled expressions per rule
50 local expression_cache = {}
51
52 -- Default autolearn settings
53 local default_autolearn_settings = {
54 -- Master enable/disable
55 enabled = false,
56
57 -- Require minimum score magnitude for learning
58 spam_score = nil, -- e.g., 6.0 - learn spam if score >= 6.0
59 ham_score = nil, -- e.g., -2.0 - learn ham if score <= -2.0
60
61 -- Require specific actions
62 spam_action = nil, -- e.g., 'reject' - only learn spam on reject
63 ham_action = nil, -- e.g., 'no action' - only learn ham on no action
64
65 -- Expression-based conditions (rspamd_expression syntax)
66 -- Examples:
67 -- "BAYES_SPAM & !WHITELIST_SENDER"
68 -- "DMARC_POLICY_REJECT | (RBL_SPAMHAUS_SBL & SURBL_MULTI)"
69 spam_condition = nil,
70 ham_condition = nil,
71
72 -- Required symbols (all must be present)
73 spam_symbols = nil, -- e.g., {'BAYES_SPAM', 'DKIM_VALID'}
74 ham_symbols = nil,
75
76 -- Forbidden symbols (any blocks learning)
77 skip_symbols = nil, -- e.g., {'WHITELIST_SENDER', 'GREYLIST'}
78
79 -- Minimum symbol weight sum
80 spam_symbol_weight = nil, -- e.g., 5.0 - sum of spam_symbols scores >= 5.0
81 ham_symbol_weight = nil, -- e.g., -3.0 - sum of ham_symbols scores <= -3.0
82
83 -- Probability-based check (skip if already confident)
84 probability_check = {
85 enabled = false,
86 variable = 'neural_prob', -- mempool variable name
87 spam_min = 0.95, -- skip if already 95% spam
88 ham_max = 0.05, -- skip if already 95% ham
89 },
90
91 -- Rate limiting
92 rate_limit = {
93 enabled = false,
94 max_daily = 1000, -- per class per day
95 redis_prefix = 'neural_autolearn',
96 },
97
98 -- Sampling (probabilistic training reduction)
99 sampling = {
100 spam_prob = 1.0, -- 1.0 = always, 0.5 = 50% chance
101 ham_prob = 1.0,
102 },
103
104 -- Exclusion conditions (matching RBL module naming)
105 exclude_local = false, -- exclude local network messages from autolearn
106 exclude_users = true, -- exclude authenticated users from autolearn
107 }
108
109 -- Helper: convert array to set
110 local function as_set(tbl)
111 if not tbl then
112 return nil
113 end
114 local res = {}
115 for _, v in ipairs(tbl) do
116 res[v] = true
117 end
118 return res
119 end
120
121 -- Helper: merge options with defaults
122 local function merge_options(defaults, overrides)
123 local merged = lua_util.override_defaults(defaults, global_defaults)
124 if overrides then
125 merged = lua_util.override_defaults(merged, overrides)
126 end
127 return merged
128 end
129
130 -- Guard execution
131 local function execute_guards(task, learn_type, ctx)
132 for _, guard in ipairs(autolearn_guards) do
133 local ok, reason = guard.cb(task, learn_type, ctx)
134 if not ok then
135 return false, reason or guard.name
136 end
137 end
138 return true
139 end
140
141 --- Register a guard callback for autolearn decisions
142 -- @param name string guard name
143 -- @param cb function(task, learn_type, ctx) -> bool, reason
144 -- @param opts table optional {priority = number}
145 function exports.register_guard(name, cb, opts)
146 if type(name) == 'function' then
147 cb = name
148 name = string.format('guard_%d', #autolearn_guards + 1)
149 end
150
151 if type(cb) ~= 'function' then
152 rspamd_logger.errx(rspamd_config, '%s: guard callback must be a function', N)
153 return nil
154 end
155
156 local guard = {
157 name = name,
158 cb = cb,
159 priority = opts and opts.priority or 0,
160 }
161
162 autolearn_guards[#autolearn_guards + 1] = guard
163 table.sort(autolearn_guards, function(a, b)
164 return (a.priority or 0) > (b.priority or 0)
165 end)
166
167 lua_util.debugm(N, rspamd_config, 'registered autolearn guard: %s', name)
168 return name
169 end
170
171 --- Unregister a guard by name
172 function exports.unregister_guard(name)
173 for i = #autolearn_guards, 1, -1 do
174 if autolearn_guards[i].name == name then
175 table.remove(autolearn_guards, i)
176 return true
177 end
178 end
179 return false
180 end
181
182 --- Configure global defaults
183 -- @param opts table of default overrides
184 function exports.configure(opts)
185 if opts then
186 global_defaults = lua_util.override_defaults(global_defaults, opts)
187 lua_util.debugm(N, rspamd_config, 'configured neural autolearn defaults')
188 end
189 end
190
191 -- Compile and cache expression
192 local function get_expression(rule_name, expr_str, pool)
193 local cache_key = rule_name .. ':' .. expr_str
194 if expression_cache[cache_key] then
195 return expression_cache[cache_key]
196 end
197
198 local function parse_atom(str)
199 local atom = ''
200 for c in str:gmatch('.') do
201 if c:match('[%w_]') then
202 atom = atom .. c
203 else
204 break
205 end
206 end
207 return atom
208 end
209
210 local function process_atom(atom, task)
211 if task:has_symbol(atom) then
212 local sym = task:get_symbol(atom)
213 if sym and sym[1] then
214 local score = math.abs(sym[1].score or 0)
215 return score > 0.001 and score or 0.001
216 end
217 return 0.001
218 end
219 return 0
220 end
221
222 local expr, err = rspamd_expression.create(expr_str, { parse_atom, process_atom }, pool)
223 if err then
224 rspamd_logger.errx(rspamd_config, '%s: cannot create expression [%s]: %s', N, expr_str, err)
225 return nil
226 end
227
228 expression_cache[cache_key] = expr
229 return expr
230 end
231
232 -- Check if all required symbols are present
233 local function check_required_symbols(task, symbols)
234 if not symbols or #symbols == 0 then
235 return true
236 end
237 for _, sym in ipairs(symbols) do
238 if not task:has_symbol(sym) then
239 return false, string.format('missing required symbol: %s', sym)
240 end
241 end
242 return true
243 end
244
245 -- Check if any forbidden symbols are present
246 local function check_forbidden_symbols(task, symbols)
247 if not symbols then
248 return true
249 end
250 local skip_set = as_set(symbols)
251 if not skip_set then
252 return true
253 end
254 for sym, _ in pairs(skip_set) do
255 if task:has_symbol(sym) then
256 return false, string.format('has forbidden symbol: %s', sym)
257 end
258 end
259 return true
260 end
261
262 -- Calculate sum of symbol scores
263 local function get_symbols_weight(task, symbols)
264 if not symbols or #symbols == 0 then
265 return 0
266 end
267 local total = 0
268 for _, sym in ipairs(symbols) do
269 local s = task:get_symbol(sym)
270 if s and s[1] then
271 total = total + (s[1].score or 0)
272 end
273 end
274 return total
275 end
276
277 --- Main function: determine if a message should be autolearned
278 -- @param task rspamd_task
279 -- @param rule neural rule configuration
280 -- @param learn_type 'spam' or 'ham'
281 -- @param overrides optional per-call config overrides
282 -- @return bool can_learn, string reason
283 function exports.can_autolearn(task, rule, learn_type, overrides)
284 local autolearn_opts = rule.autolearn or {}
285 local opts = merge_options(default_autolearn_settings, autolearn_opts)
286
287 if overrides then
288 opts = merge_options(opts, overrides)
289 end
290
291 -- Master enable check
292 if not opts.enabled then
293 return false, 'autolearn disabled'
294 end
295
296 local score = task:get_metric_score()[1]
297 local action = task:get_metric_action()
298
299 local ctx = {
300 task = task,
301 rule = rule,
302 learn_type = learn_type,
303 score = score,
304 action = action,
305 options = opts,
306 }
307
308 -- Execute registered guards first
309 local guard_ok, guard_reason = execute_guards(task, learn_type, ctx)
310 if not guard_ok then
311 return false, string.format('blocked by guard: %s', guard_reason)
312 end
313
314 -- Exclusion checks (matching RBL module naming)
315 if opts.exclude_local and task:get_from_ip() and task:get_from_ip():is_local() then
316 return false, 'local network message'
317 end
318
319 if opts.exclude_users and task:get_user() then
320 return false, 'authenticated user'
321 end
322
323 -- Forbidden symbols check
324 local skip_ok, skip_reason = check_forbidden_symbols(task, opts.skip_symbols)
325 if not skip_ok then
326 return false, skip_reason
327 end
328
329 -- Learn type specific checks
330 if learn_type == 'spam' then
331 -- Score threshold
332 if opts.spam_score and score < opts.spam_score then
333 return false, string.format('score %.2f < spam_score %.2f', score, opts.spam_score)
334 end
335
336 -- Action requirement
337 if opts.spam_action and action ~= opts.spam_action then
338 return false, string.format('action %s != required %s', action, opts.spam_action)
339 end
340
341 -- Required symbols
342 local sym_ok, sym_reason = check_required_symbols(task, opts.spam_symbols)
343 if not sym_ok then
344 return false, sym_reason
345 end
346
347 -- Symbol weight threshold
348 if opts.spam_symbol_weight then
349 local weight = get_symbols_weight(task, opts.spam_symbols)
350 if weight < opts.spam_symbol_weight then
351 return false, string.format('spam symbol weight %.2f < %.2f', weight, opts.spam_symbol_weight)
352 end
353 end
354
355 -- Expression condition
356 if opts.spam_condition then
357 local expr = get_expression(rule.prefix or 'default', opts.spam_condition, rspamd_config:get_mempool())
358 if expr then
359 local result = expr:process(task)
360 if result <= 0 then
361 return false, string.format('spam_condition not satisfied: %s', opts.spam_condition)
362 end
363 end
364 end
365
366 elseif learn_type == 'ham' then
367 -- Score threshold
368 if opts.ham_score and score > opts.ham_score then
369 return false, string.format('score %.2f > ham_score %.2f', score, opts.ham_score)
370 end
371
372 -- Action requirement
373 if opts.ham_action and action ~= opts.ham_action then
374 return false, string.format('action %s != required %s', action, opts.ham_action)
375 end
376
377 -- Required symbols
378 local sym_ok, sym_reason = check_required_symbols(task, opts.ham_symbols)
379 if not sym_ok then
380 return false, sym_reason
381 end
382
383 -- Symbol weight threshold
384 if opts.ham_symbol_weight then
385 local weight = get_symbols_weight(task, opts.ham_symbols)
386 if weight > opts.ham_symbol_weight then
387 return false, string.format('ham symbol weight %.2f > %.2f', weight, opts.ham_symbol_weight)
388 end
389 end
390
391 -- Expression condition
392 if opts.ham_condition then
393 local expr = get_expression(rule.prefix or 'default', opts.ham_condition, rspamd_config:get_mempool())
394 if expr then
395 local result = expr:process(task)
396 if result <= 0 then
397 return false, string.format('ham_condition not satisfied: %s', opts.ham_condition)
398 end
399 end
400 end
401 end
402
403 -- Probability check (skip if already confident)
404 if opts.probability_check and opts.probability_check.enabled then
405 local prob_var = opts.probability_check.variable or 'neural_prob'
406 local prob = task:get_mempool():get_variable(prob_var, 'double')
407 if prob then
408 if learn_type == 'spam' and prob >= opts.probability_check.spam_min then
409 return false, string.format('already confident spam: %.2f >= %.2f', prob, opts.probability_check.spam_min)
410 elseif learn_type == 'ham' and prob <= opts.probability_check.ham_max then
411 return false, string.format('already confident ham: %.2f <= %.2f', prob, opts.probability_check.ham_max)
412 end
413 end
414 end
415
416 -- Probabilistic sampling
417 if opts.sampling then
418 local sample_prob = learn_type == 'spam' and opts.sampling.spam_prob or opts.sampling.ham_prob
419 if sample_prob and sample_prob < 1.0 then
420 local coin = math.random()
421 if coin > sample_prob then
422 return false, string.format('sampled out: %.2f > %.2f', coin, sample_prob)
423 end
424 end
425 end
426
427 return true, nil
428 end
429
430 --- Determine learn type based on score/action/symbols
431 -- @param task rspamd_task
432 -- @param rule neural rule configuration
433 -- @return string learn_type ('spam', 'ham', or nil), string reason
434 function exports.get_learn_type(task, rule)
435 local autolearn_opts = rule.autolearn or {}
436 local opts = merge_options(default_autolearn_settings, autolearn_opts)
437
438 if not opts.enabled then
439 return nil, 'autolearn disabled'
440 end
441
442 -- Try spam first
443 local spam_ok, spam_reason = exports.can_autolearn(task, rule, 'spam')
444 if spam_ok then
445 return 'spam', 'autolearn spam'
446 end
447
448 -- Try ham
449 local ham_ok, ham_reason = exports.can_autolearn(task, rule, 'ham')
450 if ham_ok then
451 return 'ham', 'autolearn ham'
452 end
453
454 -- Neither qualifies
455 return nil, spam_reason or ham_reason or 'no autolearn condition matched'
456 end
457
458 --- Set autolearn class in mempool (for integration with neural.lua)
459 -- @param task rspamd_task
460 -- @param learn_type 'spam' or 'ham'
461 function exports.set_autolearn_class(task, learn_type)
462 task:get_mempool():set_variable('neural_autolearn_class', learn_type)
463 lua_util.debugm(N, task, 'set neural autolearn class: %s', learn_type)
464 end
465
466 --- Get autolearn class from mempool
467 -- @param task rspamd_task
468 -- @return string learn_type or nil
469 function exports.get_autolearn_class(task)
470 return task:get_mempool():get_variable('neural_autolearn_class')
471 end
472
473 --- Clear expression cache (useful for config reload)
474 function exports.clear_cache()
475 expression_cache = {}
476 end
477
478 -- Register module in rspamd_plugins for user hooks
479 if rspamd_plugins then
480 rspamd_plugins['neural_learn'] = {
481 register_guard = exports.register_guard,
482 unregister_guard = exports.unregister_guard,
483 configure = exports.configure,
484 can_autolearn = exports.can_autolearn,
485 get_learn_type = exports.get_learn_type,
486 set_autolearn_class = exports.set_autolearn_class,
487 get_autolearn_class = exports.get_autolearn_class,
488 }
489 end
490
491 return exports