Debian Codegraph @

lualib/plugins/neural.lua

raw
1 --[[
2 Copyright (c) 2022, Vsevolod Stakhov <[email protected]>
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 ]] --
16
17 local fun = require "fun"
18 local lua_redis = require "lua_redis"
19 local lua_settings = require "lua_settings"
20 local lua_util = require "lua_util"
21 local meta_functions = require "lua_meta"
22 local rspamd_kann = require "rspamd_kann"
23 local rspamd_logger = require "rspamd_logger"
24 local rspamd_tensor = require "rspamd_tensor"
25 local rspamd_util = require "rspamd_util"
26 local ucl = require "ucl"
27 local neural_external = require "lua_neural_external"
28
29 local N = 'neural'
30
31 -- Used in prefix to avoid wrong ANN to be loaded
32 local plugin_ver = '3'
33
34 -- Module vars
35 local default_options = {
36 train = {
37 max_trains = 1000,
38 max_epoch = 1000,
39 max_usages = 10,
40 max_iterations = 25, -- Torch style
41 mse = 0.001,
42 autotrain = true,
43 train_prob = 1.0,
44 learn_threads = 1,
45 learn_mode = 'balanced', -- Possible values: balanced, proportional
46 learning_rate = 0.01,
47 classes_bias = 0.0, -- balanced mode: what difference is allowed between classes (1:1 proportion means 0 bias)
48 spam_skip_prob = 0.0, -- proportional mode: spam skip probability (0-1)
49 ham_skip_prob = 0.0, -- proportional mode: ham skip probability
50 store_pool_only = false, -- store tokens in cache only (disables autotrain);
51 store_set_only = false, -- store ham and spam sets in Redis, but do not train ANN (autotrain must be enabled);
52 -- neural_vec_mpack stores vector of training data in messagepack neural_profile_digest stores profile digest
53 },
54 watch_interval = 60.0,
55 lock_expire = 600,
56 learning_spawned = false,
57 ann_expire = 60 * 60 * 24 * 2, -- 2 days
58 hidden_layer_mult = 1.5, -- number of neurons in the hidden layer (symbol-based mode)
59 -- Multi-layer architecture settings (for LLM embeddings mode)
60 layers = nil, -- layer size multipliers (auto-computed based on input dim if nil)
61 dropout = nil, -- dropout rate (0.2 default for embeddings, nil=disabled for symbols)
62 use_layernorm = nil, -- enable layer normalization (true default for embeddings)
63 activation = nil, -- activation function: 'relu' or 'gelu' (default: gelu for embeddings, relu for symbols)
64 roc_enabled = false, -- Use ROC to find the best possible thresholds for ham and spam. If spam_score_threshold or ham_score_threshold is defined, it takes precedence over ROC thresholds.
65 roc_misclassification_cost = 0.5, -- Cost of misclassifying a spam message (must be 0..1).
66 spam_score_threshold = nil, -- neural score threshold for spam (must be 0..1 or nil to disable)
67 ham_score_threshold = nil, -- neural score threshold for ham (must be 0..1 or nil to disable)
68 flat_threshold_curve = false, -- use binary classification 0/1 when threshold is reached
69 symbol_spam = 'NEURAL_SPAM',
70 symbol_ham = 'NEURAL_HAM',
71 max_inputs = nil, -- when PCA is used
72 blacklisted_symbols = {}, -- list of symbols skipped in neural processing
73 -- Phase 0 additions (scaffolding for feature providers)
74 providers = nil, -- list of provider configs; if nil, fallback to symbols-only provider
75 fusion = {
76 normalization = 'none', -- none|unit|zscore (zscore requires stats)
77 per_provider_pca = false, -- if true, apply PCA per provider before fusion (not active yet)
78 },
79 disable_symbols_input = false, -- when true, do not use symbols provider unless explicitly listed
80 -- External pretrained model support
81 external_model = nil, -- external model configuration (see lua_neural_external)
82 }
83
84 -- Rule structure:
85 -- * static config fields (see `default_options`)
86 -- * prefix - name or defined prefix
87 -- * settings - table of settings indexed by settings id, -1 is used when no settings defined
88
89 -- Rule settings element defines elements for specific settings id:
90 -- * symbols - static symbols profile (defined by config or extracted from symcache)
91 -- * name - name of settings id
92 -- * digest - digest of all symbols
93 -- * ann - dynamic ANN configuration loaded from Redis
94 -- * train - train data for ANN (e.g. the currently trained ANN)
95
96 -- Settings ANN table is loaded from Redis and represents dynamic profile for ANN
97 -- Some elements are directly stored in Redis, ANN is, in turn loaded dynamically
98 -- * version - version of ANN loaded from redis
99 -- * redis_key - name of ANN key in Redis
100 -- * symbols - symbols in THIS PARTICULAR ANN (might be different from set.symbols)
101 -- * distance - distance between set.symbols and set.ann.symbols
102 -- * ann - kann object
103
104 local settings = {
105 rules = {},
106 prefix = 'rn', -- Neural network default prefix
107 max_profiles = 3, -- Maximum number of NN profiles stored
108 }
109
110 -- Get module & Redis configuration
111 local module_config = rspamd_config:get_all_opt(N)
112 settings = lua_util.override_defaults(settings, module_config)
113 local redis_params = lua_redis.parse_redis_server('neural')
114
115 local redis_lua_script_vectors_len = "neural_train_size.lua"
116 local redis_lua_script_maybe_invalidate = "neural_maybe_invalidate.lua"
117 local redis_lua_script_maybe_lock = "neural_maybe_lock.lua"
118 local redis_lua_script_save_unlock = "neural_save_unlock.lua"
119
120 local redis_script_id = {}
121
122 -- Provider registry (Phase 0 scaffolding)
123 local registered_providers = {}
124
125 --- Registers a feature provider implementation
126 -- @param name string
127 -- @param provider table with function collect(task, ctx) -> vector(table of numbers), meta(table)
128 local function register_provider(name, provider)
129 registered_providers[name] = provider
130 end
131
132 local function get_provider(name)
133 return registered_providers[name]
134 end
135
136 -- Forward declaration
137 local result_to_vector
138
139 -- Built-in symbols provider (compatibility path)
140 register_provider('symbols', {
141 collect = function(task, ctx)
142 local vec = result_to_vector(task, ctx.profile)
143 return vec, { name = 'symbols', type = 'symbols', dim = #vec, weight = ctx.weight or 1.0 }
144 end,
145 collect_async = function(task, ctx, cont)
146 local vec = result_to_vector(task, ctx.profile)
147 cont(vec, { name = 'symbols', type = 'symbols', dim = #vec, weight = ctx.weight or 1.0 })
148 end,
149 })
150
151 -- Metatokens-only provider for contexts where symbols are not available
152 register_provider('metatokens', {
153 collect = function(task, ctx)
154 local mt = meta_functions.rspamd_gen_metatokens(task)
155 -- Convert to table of numbers
156 local vec = {}
157 for i = 1, #mt do
158 vec[i] = tonumber(mt[i]) or 0.0
159 end
160 return vec, { name = 'metatokens', type = 'metatokens', dim = #vec, weight = ctx.weight or 1.0 }
161 end,
162 collect_async = function(task, ctx, cont)
163 local mt = meta_functions.rspamd_gen_metatokens(task)
164 -- Convert to table of numbers
165 local vec = {}
166 for i = 1, #mt do
167 vec[i] = tonumber(mt[i]) or 0.0
168 end
169 cont(vec, { name = 'metatokens', type = 'metatokens', dim = #vec, weight = ctx.weight or 1.0 })
170 end,
171 })
172
173 local function load_scripts()
174 local err
175 redis_script_id.vectors_len, err = lua_redis.load_redis_script_from_file(redis_lua_script_vectors_len,
176 redis_params)
177 if err then
178 rspamd_logger.errx(rspamd_config, err)
179 end
180 redis_script_id.maybe_invalidate, err = lua_redis.load_redis_script_from_file(redis_lua_script_maybe_invalidate,
181 redis_params)
182 if err then
183 rspamd_logger.errx(rspamd_config, err)
184 end
185 redis_script_id.maybe_lock, err = lua_redis.load_redis_script_from_file(redis_lua_script_maybe_lock,
186 redis_params)
187 if err then
188 rspamd_logger.errx(rspamd_config, err)
189 end
190 redis_script_id.save_unlock, err = lua_redis.load_redis_script_from_file(redis_lua_script_save_unlock,
191 redis_params)
192 if err then
193 rspamd_logger.errx(rspamd_config, err)
194 end
195 end
196
197 -- Creates a simple single-layer ANN for symbol-based inputs (backward compatible)
198 local function create_symbol_ann(n, rule)
199 local nhidden = math.floor(n * (rule.hidden_layer_mult or 1.0) + 1.0)
200 local t = rspamd_kann.layer.input(n)
201 t = rspamd_kann.transform.relu(t)
202 t = rspamd_kann.layer.dense(t, nhidden)
203 t = rspamd_kann.layer.cost(t, 1, rspamd_kann.cost.ceb_neg)
204 return rspamd_kann.new.kann(t)
205 end
206
207 -- Creates a multi-layer funnel ANN optimized for high-dimensional embeddings
208 -- Architecture: Input → [Dense → LayerNorm → Activation → Dropout]* → Cost
209 local function create_embedding_ann(n, rule)
210 local t = rspamd_kann.layer.input(n)
211
212 -- Get architecture settings with smart defaults based on input dimension
213 local layers = rule.layers
214 if not layers then
215 -- Auto-compute layer sizes based on input dimension
216 if n > 512 then
217 layers = { 0.5, 0.25, 0.125 } -- 3 layers for large embeddings (e.g., 1024-dim)
218 elseif n > 256 then
219 layers = { 0.5, 0.25 } -- 2 layers for medium embeddings
220 else
221 layers = { 0.5 } -- 1 layer for small embeddings
222 end
223 end
224
225 local dropout_rate = rule.dropout
226 if dropout_rate == nil then
227 dropout_rate = 0.2 -- Default dropout for regularization
228 end
229
230 local use_layernorm = rule.use_layernorm
231 if use_layernorm == nil then
232 use_layernorm = true -- Default: enable layer normalization
233 end
234
235 -- Select activation function: GELU for embeddings (better for high-dim), ReLU as fallback
236 local activation = rule.activation
237 if not activation then
238 -- Default to GELU for embeddings if available
239 activation = rspamd_kann.transform.gelu and 'gelu' or 'relu'
240 end
241 local activate_fn = (activation == 'gelu' and rspamd_kann.transform.gelu) or rspamd_kann.transform.relu
242
243 lua_util.debugm(N, rspamd_config, 'embedding ANN: %s layers, dropout=%s, layernorm=%s, activation=%s',
244 #layers, dropout_rate, use_layernorm, activation)
245
246 -- Build funnel architecture with graduated dimension reduction
247 for i, layer_mult in ipairs(layers) do
248 local layer_size = math.max(math.floor(n * layer_mult), 32)
249
250 -- Dense layer
251 t = rspamd_kann.layer.dense(t, layer_size)
252
253 -- Layer normalization for training stability
254 if use_layernorm then
255 t = rspamd_kann.layer.layernorm(t)
256 end
257
258 -- Activation function (GELU or ReLU)
259 t = activate_fn(t)
260
261 -- Dropout for regularization (less on final hidden layer)
262 if dropout_rate > 0 then
263 local rate = (i == #layers) and (dropout_rate * 0.5) or dropout_rate
264 t = rspamd_kann.layer.dropout(t, rate)
265 end
266 end
267
268 t = rspamd_kann.layer.cost(t, 1, rspamd_kann.cost.ceb_neg)
269 return rspamd_kann.new.kann(t)
270 end
271
272 -- Conv1d ANN: uses the enhanced embedding architecture.
273 -- The actual convolution (multi-scale max-over-time pooling) is done in the
274 -- fasttext_embed provider, which produces compact feature vectors (n_scales * channels).
275 -- The ANN itself is a simple dense network on these pre-convolved features.
276 local function create_conv1d_ann(n, rule)
277 lua_util.debugm(N, rspamd_config,
278 'creating conv1d ANN: %s pre-convolved inputs', n)
279 return create_embedding_ann(n, rule)
280 end
281
282 -- Detects if rule uses LLM embeddings provider
283 local function uses_llm_embeddings(rule)
284 if not rule.providers then
285 return false
286 end
287 for _, p in ipairs(rule.providers) do
288 if p.type == 'llm' then
289 return true
290 end
291 end
292 return false
293 end
294
295 -- Main ANN factory function - auto-selects architecture based on rule configuration
296 local function create_ann(n, nlayers, rule)
297 -- Check for conv1d architecture first
298 if rule.conv1d then
299 lua_util.debugm(N, rspamd_config, 'creating conv1d ANN with %s inputs', n)
300 return create_conv1d_ann(n, rule)
301 end
302
303 -- Check if we should use the enhanced embedding architecture
304 -- Conditions: has LLM provider, or explicit multi-layer config, or large input dimension
305 local use_embedding_arch = uses_llm_embeddings(rule)
306 or rule.layers ~= nil
307 or rule.use_layernorm ~= nil
308 or rule.dropout ~= nil
309
310 if use_embedding_arch then
311 lua_util.debugm(N, rspamd_config, 'creating multi-layer embedding ANN with %s inputs', n)
312 return create_embedding_ann(n, rule)
313 else
314 lua_util.debugm(N, rspamd_config, 'creating simple symbol ANN with %s inputs', n)
315 return create_symbol_ann(n, rule)
316 end
317 end
318
319 -- Fills ANN data for a specific settings element
320 local function fill_set_ann(set, ann_key)
321 if not set.ann then
322 set.ann = {
323 symbols = set.symbols,
324 distance = 0,
325 digest = set.digest,
326 redis_key = ann_key,
327 version = 0,
328 }
329 end
330 end
331
332 -- This function takes all inputs, applies PCA transformation and returns the final
333 -- PCA matrix as rspamd_tensor
334 local function learn_pca(inputs, max_inputs)
335 local scatter_matrix = rspamd_tensor.scatter_matrix(rspamd_tensor.fromtable(inputs))
336 local eigenvals = scatter_matrix:eigen()
337 -- scatter matrix is not filled with eigenvectors
338 lua_util.debugm(N, 'eigenvalues: %s', eigenvals)
339 local w = rspamd_tensor.new(2, max_inputs, #scatter_matrix[1])
340 for i = 1, max_inputs do
341 w[i] = scatter_matrix[#scatter_matrix - i + 1]
342 end
343
344 lua_util.debugm(N, 'pca matrix: %s', w)
345
346 return w
347 end
348
349 -- Build providers metadata for storage alongside ANN
350 local function build_providers_meta(metas)
351 if not metas or #metas == 0 then return nil end
352 local out = {}
353 for i, m in ipairs(metas) do
354 out[i] = {
355 name = m.name,
356 type = m.type,
357 dim = m.dim,
358 weight = m.weight,
359 model = m.model,
360 provider = m.provider,
361 }
362 end
363 return out
364 end
365
366 -- Normalization helpers
367 local function l2_normalize_vector(vec)
368 local sumsq = 0.0
369 for i = 1, #vec do
370 local v = vec[i]
371 sumsq = sumsq + v * v
372 end
373 if sumsq > 0 then
374 local inv = 1.0 / math.sqrt(sumsq)
375 for i = 1, #vec do
376 vec[i] = vec[i] * inv
377 end
378 end
379 return vec
380 end
381
382 local function compute_zscore_stats(inputs)
383 local n = #inputs
384 if n == 0 then return nil end
385 local d = #inputs[1]
386 local mean = {}
387 local m2 = {}
388 for j = 1, d do
389 mean[j] = 0.0
390 m2[j] = 0.0
391 end
392 for i = 1, n do
393 local x = inputs[i]
394 for j = 1, d do
395 local delta = x[j] - mean[j]
396 mean[j] = mean[j] + delta / i
397 m2[j] = m2[j] + delta * (x[j] - mean[j])
398 end
399 end
400 local std = {}
401 for j = 1, d do
402 std[j] = math.sqrt((n > 1 and (m2[j] / (n - 1))) or 0.0)
403 if std[j] == 0 or std[j] ~= std[j] then
404 std[j] = 1.0 -- avoid division by zero and NaN
405 end
406 end
407 return { mode = 'zscore', mean = mean, std = std }
408 end
409
410 local function apply_normalization(vec, norm_stats_or_mode)
411 if not norm_stats_or_mode then return vec end
412 if type(norm_stats_or_mode) == 'string' then
413 if norm_stats_or_mode == 'unit' then
414 return l2_normalize_vector(vec)
415 else
416 return vec
417 end
418 else
419 if norm_stats_or_mode.mode == 'unit' then
420 return l2_normalize_vector(vec)
421 elseif norm_stats_or_mode.mode == 'zscore' and norm_stats_or_mode.mean and norm_stats_or_mode.std then
422 local mean = norm_stats_or_mode.mean
423 local std = norm_stats_or_mode.std
424 for i = 1, math.min(#vec, #mean) do
425 vec[i] = (vec[i] - (mean[i] or 0.0)) / (std[i] or 1.0)
426 end
427 return vec
428 else
429 return vec
430 end
431 end
432 end
433
434 -- This function computes optimal threshold using ROC for the given set of inputs.
435 -- Returns a threshold that minimizes:
436 -- alpha * (false_positive_rate) + beta * (false_negative_rate)
437 -- Where alpha is cost of false positive result
438 -- beta is cost of false negative result
439 local function get_roc_thresholds(ann, inputs, outputs, alpha, beta)
440 -- Sorts list x and list y based on the values in list x.
441 local sort_relative = function(x, y)
442 local r = {}
443
444 assert(#x == #y)
445 local n = #x
446
447 local a = {}
448 local b = {}
449 for i = 1, n do
450 r[i] = i
451 end
452
453 local cmp = function(p, q)
454 return p < q
455 end
456
457 table.sort(r, function(p, q)
458 return cmp(x[p], x[q])
459 end)
460
461 for i = 1, n do
462 a[i] = x[r[i]]
463 b[i] = y[r[i]]
464 end
465
466 return a, b
467 end
468
469 local function get_scores(nn, input_vectors)
470 local scores = {}
471 for i = 1, #inputs do
472 local score = nn:apply1(input_vectors[i], nn.pca)[1]
473 scores[#scores + 1] = score
474 end
475
476 return scores
477 end
478
479 local fpr = {}
480 local fnr = {}
481 local scores = get_scores(ann, inputs)
482
483 scores, outputs = sort_relative(scores, outputs)
484
485 local n_samples = #outputs
486 local n_spam = 0
487 local n_ham = 0
488 local ham_count_ahead = {}
489 local spam_count_ahead = {}
490 local ham_count_behind = {}
491 local spam_count_behind = {}
492
493 ham_count_ahead[n_samples + 1] = 0
494 spam_count_ahead[n_samples + 1] = 0
495
496 for i = n_samples, 1, -1 do
497 -- Labels are -1.0 for ham and 1.0 for spam (ceb_neg cost function)
498 if outputs[i][1] < 0 then
499 n_ham = n_ham + 1
500 ham_count_ahead[i] = 1
501 spam_count_ahead[i] = 0
502 else
503 n_spam = n_spam + 1
504 ham_count_ahead[i] = 0
505 spam_count_ahead[i] = 1
506 end
507
508 ham_count_ahead[i] = ham_count_ahead[i] + ham_count_ahead[i + 1]
509 spam_count_ahead[i] = spam_count_ahead[i] + spam_count_ahead[i + 1]
510 end
511
512 for i = 1, n_samples do
513 -- Labels are -1.0 for ham and 1.0 for spam (ceb_neg cost function)
514 if outputs[i][1] < 0 then
515 ham_count_behind[i] = 1
516 spam_count_behind[i] = 0
517 else
518 ham_count_behind[i] = 0
519 spam_count_behind[i] = 1
520 end
521
522 if i ~= 1 then
523 ham_count_behind[i] = ham_count_behind[i] + ham_count_behind[i - 1]
524 spam_count_behind[i] = spam_count_behind[i] + spam_count_behind[i - 1]
525 end
526 end
527
528 for i = 1, n_samples do
529 fpr[i] = 0
530 fnr[i] = 0
531
532 if (ham_count_ahead[i + 1] + ham_count_behind[i]) ~= 0 then
533 fpr[i] = ham_count_ahead[i + 1] / (ham_count_ahead[i + 1] + ham_count_behind[i])
534 end
535
536 if (spam_count_behind[i] + spam_count_ahead[i + 1]) ~= 0 then
537 fnr[i] = spam_count_behind[i] / (spam_count_behind[i] + spam_count_ahead[i + 1])
538 end
539 end
540
541 local p = n_spam / (n_spam + n_ham)
542
543 local cost = {}
544 local min_cost_idx = 0
545 local min_cost = math.huge
546 for i = 1, n_samples do
547 cost[i] = ((1 - p) * alpha * fpr[i]) + (p * beta * fnr[i])
548 if min_cost >= cost[i] then
549 min_cost = cost[i]
550 min_cost_idx = i
551 end
552 end
553
554 return scores[min_cost_idx]
555 end
556
557 -- This function is intended to extend lock for ANN during training
558 -- It registers periodic that increases locked key each 30 seconds unless
559 -- `set.learning_spawned` is set to `true`
560 local function register_lock_extender(rule, set, ev_base, ann_key)
561 rspamd_config:add_periodic(ev_base, 30.0,
562 function()
563 local function redis_lock_extend_cb(err, _)
564 if err then
565 rspamd_logger.errx(rspamd_config, 'cannot lock ANN %s from redis: %s',
566 ann_key, err)
567 else
568 rspamd_logger.infox(rspamd_config, 'extend lock for ANN %s for 30 seconds',
569 ann_key)
570 end
571 end
572
573 if set.learning_spawned then
574 lua_redis.redis_make_request_taskless(ev_base,
575 rspamd_config,
576 rule.redis,
577 nil,
578 true, -- is write
579 redis_lock_extend_cb, --callback
580 'HINCRBY', -- command
581 { ann_key, 'lock', '30' }
582 )
583 else
584 lua_util.debugm(N, rspamd_config, "stop lock extension as learning_spawned is false")
585 return false -- do not plan any more updates
586 end
587
588 return true
589 end
590 )
591 end
592
593 local function can_push_train_vector(rule, task, learn_type, nspam, nham)
594 local train_opts = rule.train
595 local coin = math.random()
596
597 if train_opts.train_prob and coin < 1.0 - train_opts.train_prob then
598 rspamd_logger.infox(task, 'probabilistically skip sample: %s', coin)
599 return false
600 end
601
602 if train_opts.learn_mode == 'balanced' then
603 -- Keep balanced training set based on number of spam and ham samples
604 if learn_type == 'spam' then
605 if nspam <= train_opts.max_trains then
606 if nspam > nham then
607 -- Apply sampling
608 local skip_rate = 1.0 - nham / (nspam + 1)
609 if coin < skip_rate - train_opts.classes_bias then
610 rspamd_logger.infox(task,
611 'skip %s sample to keep spam/ham balance; probability %s; %s spam and %s ham vectors stored',
612 learn_type,
613 skip_rate - train_opts.classes_bias,
614 nspam, nham)
615 return false
616 end
617 end
618 return true
619 else
620 -- Enough learns
621 rspamd_logger.infox(task, 'skip %s sample to keep spam/ham balance; too many spam samples: %s',
622 learn_type,
623 nspam)
624 end
625 else
626 if nham <= train_opts.max_trains then
627 if nham > nspam then
628 -- Apply sampling
629 local skip_rate = 1.0 - nspam / (nham + 1)
630 if coin < skip_rate - train_opts.classes_bias then
631 rspamd_logger.infox(task,
632 'skip %s sample to keep spam/ham balance; probability %s; %s spam and %s ham vectors stored',
633 learn_type,
634 skip_rate - train_opts.classes_bias,
635 nspam, nham)
636 return false
637 end
638 end
639 return true
640 else
641 rspamd_logger.infox(task, 'skip %s sample to keep spam/ham balance; too many ham samples: %s', learn_type,
642 nham)
643 end
644 end
645 else
646 -- Probabilistic learn mode, we just skip learn if we already have enough samples or
647 -- if our coin drop is less than desired probability
648 if learn_type == 'spam' then
649 if nspam <= train_opts.max_trains then
650 if train_opts.spam_skip_prob then
651 if coin <= train_opts.spam_skip_prob then
652 rspamd_logger.infox(task, 'skip %s sample probabilistically; probability %s (%s skip chance)', learn_type,
653 coin, train_opts.spam_skip_prob)
654 return false
655 end
656
657 return true
658 end
659 else
660 rspamd_logger.infox(task, 'skip %s sample; too many spam samples: %s (%s limit)', learn_type,
661 nspam, train_opts.max_trains)
662 end
663 else
664 if nham <= train_opts.max_trains then
665 if train_opts.ham_skip_prob then
666 if coin <= train_opts.ham_skip_prob then
667 rspamd_logger.infox(task, 'skip %s sample probabilistically; probability %s (%s skip chance)', learn_type,
668 coin, train_opts.ham_skip_prob)
669 return false
670 end
671
672 return true
673 end
674 else
675 rspamd_logger.infox(task, 'skip %s sample; too many ham samples: %s (%s limit)', learn_type,
676 nham, train_opts.max_trains)
677 end
678 end
679 end
680
681 return false
682 end
683
684 -- Closure generator for unlock function
685 local function gen_unlock_cb(rule, set, ann_key)
686 return function(err)
687 if err then
688 rspamd_logger.errx(rspamd_config, 'cannot unlock ANN %s:%s at %s from redis: %s',
689 rule.prefix, set.name, ann_key, err)
690 else
691 lua_util.debugm(N, rspamd_config, 'unlocked ANN %s:%s at %s',
692 rule.prefix, set.name, ann_key)
693 end
694 end
695 end
696
697 -- Used to generate new ANN key for specific profile
698 local function new_ann_key(rule, set, version)
699 local ann_key = string.format('%s_%s_%s_%s_%d', settings.prefix,
700 rule.prefix, set.name, set.digest:sub(1, 8), version)
701
702 return ann_key
703 end
704
705 local function redis_ann_prefix(rule, settings_name)
706 -- We also need to count metatokens:
707 -- Note: meta_functions.version represents the metatoken format version
708 local n = meta_functions.version
709 return string.format('%s%d_%s_%d_%s',
710 settings.prefix, plugin_ver, rule.prefix, n, settings_name)
711 end
712
713 -- Returns a stable key for pending training vectors (version-independent)
714 -- Used for batch/manual training to avoid version mismatch issues
715 local function pending_train_key(rule, set)
716 return string.format('%s_%s_%s_pending',
717 settings.prefix, rule.prefix, set.name)
718 end
719
720 -- Compute a stable digest for providers configuration
721 local function providers_config_digest(providers_cfg, rule)
722 if not providers_cfg then return nil end
723 -- Normalize minimal subset of fields to keep digest stable across equivalent configs
724 local norm = { providers = {} }
725
726 local fusion = rule and rule.fusion or nil
727 if rule then
728 local effective_fusion = {
729 normalization = (fusion and fusion.normalization) or 'none',
730 include_meta = fusion and fusion.include_meta,
731 meta_weight = fusion and fusion.meta_weight,
732 per_provider_pca = fusion and fusion.per_provider_pca,
733 }
734 if effective_fusion.include_meta == nil then
735 effective_fusion.include_meta = true
736 end
737 if effective_fusion.meta_weight == nil then
738 effective_fusion.meta_weight = 1.0
739 end
740 if effective_fusion.per_provider_pca == nil then
741 effective_fusion.per_provider_pca = false
742 end
743 norm.fusion = effective_fusion
744 end
745
746 if rule and rule.max_inputs then
747 norm.max_inputs = rule.max_inputs
748 end
749
750 local gpt_settings = rspamd_config:get_all_opt('gpt') or {}
751
752 for i, p in ipairs(providers_cfg) do
753 local ptype = p.type or p.name or 'unknown'
754 local entry = {
755 type = ptype,
756 weight = p.weight or 1.0,
757 dim = p.dim,
758 }
759
760 if ptype == 'llm' then
761 local llm_type = p.llm_type or p.api or p.backend or gpt_settings.type
762 local model = p.model or gpt_settings.model
763 local max_tokens = p.max_tokens
764 if not max_tokens and gpt_settings.model_parameters and model then
765 local model_cfg = gpt_settings.model_parameters[model] or {}
766 max_tokens = model_cfg.max_completion_tokens or model_cfg.max_tokens
767 end
768 if not max_tokens then
769 max_tokens = gpt_settings.max_tokens
770 end
771
772 entry.llm_type = llm_type
773 entry.model = model
774 entry.max_tokens = max_tokens
775 end
776
777 -- Conv1d feature extraction settings affect output dimensions
778 if p.output_mode == 'conv1d' then
779 entry.output_mode = 'conv1d'
780 entry.max_words = p.max_words or 32
781 entry.kernel_sizes = p.kernel_sizes or { 1, 3, 5 }
782 entry.conv_pooling = p.conv_pooling or 'max'
783 end
784
785 norm.providers[i] = entry
786 end
787 return lua_util.unordered_table_digest(norm)
788 end
789
790 -- If no providers configured, fallback to symbols provider unless disabled
791 -- phase: 'infer' | 'train'
792 -- Removed synchronous collect_features; use collect_features_async instead
793
794 -- Async version: runs providers in parallel and calls cb(fused, meta) when done
795 local function collect_features_async(task, rule, profile_or_set, phase, cb)
796 local providers_cfg = rule.providers
797 if not providers_cfg or #providers_cfg == 0 then
798 if rule.disable_symbols_input then
799 cb(nil, { providers = {}, total_dim = 0, digest = providers_config_digest(providers_cfg, rule) })
800 return
801 end
802 local prov = get_provider('symbols')
803 if prov and prov.collect_async then
804 prov.collect_async(task, { profile = profile_or_set, weight = 1.0, phase = phase }, function(vec, meta)
805 local metas = {}
806 if vec then
807 metas[1] = meta
808 end
809 local fused = {}
810 if vec then
811 local w = (meta and meta.weight) or 1.0
812 local norm_mode = (rule.fusion and rule.fusion.normalization) or 'none'
813 if norm_mode ~= 'none' then
814 vec = apply_normalization(vec, norm_mode)
815 end
816 for _, x in ipairs(vec) do
817 fused[#fused + 1] = x * w
818 end
819 end
820 cb(#fused > 0 and fused or nil, {
821 providers = build_providers_meta(metas) or metas,
822 total_dim = #fused,
823 digest = providers_config_digest(providers_cfg, rule),
824 })
825 end)
826 return
827 end
828 -- Fallback: direct symbols compute
829 local vec = result_to_vector(task, profile_or_set)
830 local meta = { name = 'symbols', type = 'symbols', dim = #vec, weight = 1.0 }
831 local fused = {}
832 local w = 1.0
833 local norm_mode = (rule.fusion and rule.fusion.normalization) or 'none'
834 if norm_mode ~= 'none' then
835 vec = apply_normalization(vec, norm_mode)
836 end
837 for _, x in ipairs(vec) do
838 fused[#fused + 1] = x * w
839 end
840 cb(fused,
841 {
842 providers = build_providers_meta({ meta }) or { meta },
843 total_dim = #fused,
844 digest = providers_config_digest(
845 providers_cfg, rule)
846 })
847 return
848 end
849
850 local vectors = {}
851 local metas = {}
852 local remaining = 0
853
854 local function maybe_finish()
855 remaining = remaining - 1
856 if remaining == 0 then
857 -- Fuse
858 local fused = {}
859 for i, v in ipairs(vectors) do
860 if v then
861 local w = (metas[i] and metas[i].weight) or 1.0
862 local norm_mode = (rule.fusion and rule.fusion.normalization) or 'none'
863 if norm_mode ~= 'none' then
864 v = apply_normalization(v, norm_mode)
865 end
866 for _, x in ipairs(v) do
867 fused[#fused + 1] = x * w
868 end
869 end
870 end
871 local meta = {
872 providers = build_providers_meta(metas) or metas,
873 total_dim = #fused,
874 digest = providers_config_digest(providers_cfg, rule),
875 }
876 if #fused == 0 then
877 cb(nil, meta)
878 else
879 cb(fused, meta)
880 end
881 end
882 end
883
884 local function start_provider(i, pcfg)
885 local prov = get_provider(pcfg.type or pcfg.name)
886 if not prov or not prov.collect_async then
887 maybe_finish()
888 return
889 end
890 prov.collect_async(task, {
891 profile = profile_or_set,
892 set = profile_or_set,
893 rule = rule,
894 config = pcfg,
895 weight = pcfg.weight or 1.0,
896 phase = phase,
897 }, function(vec, meta)
898 if vec then
899 metas[i] = meta or { name = pcfg.name or pcfg.type, type = pcfg.type, dim = #vec, weight = pcfg.weight or 1.0 }
900 vectors[i] = vec
901 end
902 maybe_finish()
903 end)
904 end
905
906 -- Include symbols provider (which includes both symbols AND metatokens) as an extra provider
907 -- The name 'include_meta' is historical but it actually includes the full symbols provider
908 -- For backward compatibility, include symbols by default unless explicitly disabled
909 local include_meta = false
910 if not providers_cfg or #providers_cfg == 0 then
911 -- No providers, always use symbols (which includes metatokens)
912 include_meta = true
913 elseif rule.fusion then
914 -- Explicit fusion config takes precedence
915 include_meta = rule.fusion.include_meta
916 if include_meta == nil then
917 -- Default to true for backward compatibility when fusion is configured but include_meta not specified
918 include_meta = true
919 end
920 else
921 -- Providers configured but no fusion settings - default to including symbols+metatokens
922 include_meta = true
923 end
924
925 local meta_weight = (rule.fusion and rule.fusion.meta_weight) or 1.0
926
927 remaining = #providers_cfg + (include_meta and 1 or 0)
928
929 -- Start all configured providers
930 for i, pcfg in ipairs(providers_cfg) do
931 start_provider(i, pcfg)
932 end
933
934 if include_meta then
935 -- Always use metatokens provider for consistency
936 -- This ensures same dimensions whether called from controller or full scan
937 local prov = get_provider('metatokens')
938
939 if prov and prov.collect_async then
940 local meta_index = #providers_cfg + 1 -- Metatokens always come after providers
941 prov.collect_async(task, { profile = profile_or_set, set = profile_or_set, weight = meta_weight, phase = phase },
942 function(vec, meta)
943 if vec then
944 metas[meta_index] = meta
945 vectors[meta_index] = vec
946 end
947 maybe_finish()
948 end)
949 else
950 maybe_finish()
951 end
952 end
953 end
954
955 -- This function receives training vectors, checks them, spawn learning and saves ANN in Redis
956 local function spawn_train(params)
957 -- Prevent concurrent training (flag may be set by do_train_ann or needs to be set here for direct calls)
958 if params.set.learning_spawned then
959 lua_util.debugm(N, rspamd_config, 'spawn_train: training already in progress for %s:%s, skipping',
960 params.rule.prefix, params.set.name)
961 return
962 end
963 params.set.learning_spawned = true
964
965 -- Check training data sanity
966 -- Now we need to join inputs and create the appropriate test vectors
967 local n
968
969 -- When using providers, derive dimension from actual vectors
970 if params.rule.providers and #params.rule.providers > 0 and
971 (#params.spam_vec > 0 or #params.ham_vec > 0) then
972 -- Use dimension from stored vectors
973 if #params.spam_vec > 0 then
974 n = #params.spam_vec[1]
975 else
976 n = #params.ham_vec[1]
977 end
978 lua_util.debugm(N, rspamd_config, 'spawn_train: using vector dimension %s from stored vectors', n)
979 else
980 -- Traditional symbol-based dimension
981 n = #params.set.symbols + meta_functions.rspamd_count_metatokens()
982 lua_util.debugm(N, rspamd_config, 'spawn_train: using symbol dimension %s symbols + %s metatokens = %s',
983 #params.set.symbols, meta_functions.rspamd_count_metatokens(), n)
984 end
985
986 -- Now we can train ann - wrap in pcall to catch KANN errors
987 local create_ok, train_ann = pcall(create_ann, params.rule.max_inputs or n, 3, params.rule)
988 if not create_ok then
989 rspamd_logger.errx(rspamd_config, 'failed to create ANN for %s:%s: %s',
990 params.rule.prefix, params.set.name, train_ann)
991 params.set.learning_spawned = false
992 return
993 end
994
995 if #params.ham_vec + #params.spam_vec < params.rule.train.max_trains / 2 then
996 -- Insufficient training data, reset flag and return
997 rspamd_logger.errx(rspamd_config, 'insufficient training data for ANN %s:%s: spam=%s ham=%s (need at least %s total)',
998 params.rule.prefix, params.set.name,
999 #params.spam_vec, #params.ham_vec, params.rule.train.max_trains / 2)
1000 params.set.learning_spawned = false
1001 return
1002 else
1003 local inputs, outputs = {}, {}
1004
1005 -- Used to show parsed vectors in a convenient format (for debugging only)
1006 local function debug_vec(t)
1007 local ret = {}
1008 for i, v in ipairs(t) do
1009 if v ~= 0 then
1010 ret[#ret + 1] = string.format('%d=%.2f', i, v)
1011 end
1012 end
1013
1014 return ret
1015 end
1016
1017 -- Make training set by joining vectors
1018 -- KANN automatically shuffles those samples
1019 -- 1.0 is used for spam and -1.0 is used for ham
1020 -- It implies that output layer can express that (e.g. tanh output)
1021 for _, e in ipairs(params.spam_vec) do
1022 inputs[#inputs + 1] = e
1023 outputs[#outputs + 1] = { 1.0 }
1024 --rspamd_logger.debugm(N, rspamd_config, 'spam vector: %s', debug_vec(e))
1025 end
1026 for _, e in ipairs(params.ham_vec) do
1027 inputs[#inputs + 1] = e
1028 outputs[#outputs + 1] = { -1.0 }
1029 --rspamd_logger.debugm(N, rspamd_config, 'ham vector: %s', debug_vec(e))
1030 end
1031
1032 -- Called in child process
1033 local function train()
1034 local log_thresh = params.rule.train.max_iterations / 10
1035 local seen_nan = false
1036
1037 local function train_cb(iter, train_cost, value_cost)
1038 if (iter * (params.rule.train.max_iterations / log_thresh)) % (params.rule.train.max_iterations) == 0 then
1039 if train_cost ~= train_cost and not seen_nan then
1040 -- We have nan :( try to log lot's of stuff to dig into a problem
1041 seen_nan = true
1042 rspamd_logger.errx(rspamd_config, 'ANN %s:%s: train error: observed nan in error cost!; value cost = %s',
1043 params.rule.prefix, params.set.name,
1044 value_cost)
1045 for i, e in ipairs(inputs) do
1046 lua_util.debugm(N, rspamd_config, 'train vector %s -> %s',
1047 debug_vec(e), outputs[i][1])
1048 end
1049 end
1050
1051 rspamd_logger.infox(rspamd_config,
1052 "ANN %s:%s: learned from %s redis key in %s iterations, error: %s, value cost: %s",
1053 params.rule.prefix, params.set.name,
1054 params.ann_key,
1055 iter,
1056 train_cost,
1057 value_cost)
1058 end
1059 end
1060
1061 lua_util.debugm(N, rspamd_config, "subprocess to learn ANN %s:%s has been started",
1062 params.rule.prefix, params.set.name)
1063
1064 local pca
1065 if params.rule.max_inputs then
1066 -- Train PCA in the main process, presumably it is not that long
1067 lua_util.debugm(N, rspamd_config, "start PCA train for ANN %s:%s",
1068 params.rule.prefix, params.set.name)
1069 pca = learn_pca(inputs, params.rule.max_inputs)
1070 end
1071
1072 -- Compute normalization stats if requested
1073 local norm_stats
1074 if params.rule.fusion and params.rule.fusion.normalization == 'zscore' then
1075 norm_stats = compute_zscore_stats(inputs)
1076 elseif params.rule.fusion and params.rule.fusion.normalization == 'unit' then
1077 norm_stats = { mode = 'unit' }
1078 end
1079
1080 if norm_stats then
1081 for i = 1, #inputs do
1082 inputs[i] = apply_normalization(inputs[i], norm_stats)
1083 end
1084 end
1085
1086 lua_util.debugm(N, rspamd_config, "start neural train for ANN %s:%s",
1087 params.rule.prefix, params.set.name)
1088 local ret, err = pcall(train_ann.train1, train_ann,
1089 inputs, outputs, {
1090 lr = params.rule.train.learning_rate,
1091 max_epoch = params.rule.train.max_iterations,
1092 cb = train_cb,
1093 pca = pca
1094 })
1095
1096 if not ret then
1097 rspamd_logger.errx(rspamd_config, "cannot train ann %s:%s: %s",
1098 params.rule.prefix, params.set.name, err)
1099
1100 return nil
1101 else
1102 lua_util.debugm(N, rspamd_config, "finished neural train for ANN %s:%s",
1103 params.rule.prefix, params.set.name)
1104 end
1105
1106 local roc_thresholds = {}
1107 if params.rule.roc_enabled then
1108 local spam_threshold = get_roc_thresholds(train_ann,
1109 inputs,
1110 outputs,
1111 1 - params.rule.roc_misclassification_cost,
1112 params.rule.roc_misclassification_cost)
1113 local ham_threshold = get_roc_thresholds(train_ann,
1114 inputs,
1115 outputs,
1116 params.rule.roc_misclassification_cost,
1117 1 - params.rule.roc_misclassification_cost)
1118 roc_thresholds = { spam_threshold, ham_threshold }
1119
1120 rspamd_logger.messagex(rspamd_config, "ROC thresholds: (spam_threshold: %s, ham_threshold: %s)",
1121 roc_thresholds[1], roc_thresholds[2])
1122 end
1123
1124 if not seen_nan then
1125 -- Convert to strings as ucl cannot rspamd_text properly
1126 local pca_data
1127 if pca then
1128 pca_data = tostring(pca:save())
1129 end
1130 local out = {
1131 ann_data = tostring(train_ann:save()),
1132 pca_data = pca_data,
1133 roc_thresholds = roc_thresholds,
1134 norm_stats = norm_stats,
1135 }
1136
1137 local final_data = ucl.to_format(out, 'msgpack')
1138 lua_util.debugm(N, rspamd_config, "subprocess for ANN %s:%s returned %s bytes",
1139 params.rule.prefix, params.set.name, #final_data)
1140 return final_data
1141 else
1142 return nil
1143 end
1144 end
1145
1146 local function redis_save_cb(err)
1147 if err then
1148 rspamd_logger.errx(rspamd_config, 'cannot save ANN %s:%s to redis key %s: %s',
1149 params.rule.prefix, params.set.name, params.ann_key, err)
1150 lua_redis.redis_make_request_taskless(params.ev_base,
1151 rspamd_config,
1152 params.rule.redis,
1153 nil,
1154 false, -- is write
1155 gen_unlock_cb(params.rule, params.set, params.ann_key), --callback
1156 'HDEL', -- command
1157 { params.ann_key, 'lock' }
1158 )
1159 else
1160 rspamd_logger.infox(rspamd_config, 'saved ANN %s:%s to redis: %s',
1161 params.rule.prefix, params.set.name, params.set.ann.redis_key)
1162
1163 -- Clean up pending training keys if they were used
1164 if params.pending_key then
1165 local function cleanup_cb(cleanup_err)
1166 if cleanup_err then
1167 lua_util.debugm(N, rspamd_config, 'failed to cleanup pending keys: %s', cleanup_err)
1168 else
1169 lua_util.debugm(N, rspamd_config, 'cleaned up pending training keys for %s',
1170 params.pending_key)
1171 end
1172 end
1173 -- Delete both spam and ham pending sets
1174 lua_redis.redis_make_request_taskless(params.ev_base,
1175 rspamd_config,
1176 params.rule.redis,
1177 nil,
1178 true, -- is write
1179 cleanup_cb,
1180 'DEL',
1181 { params.pending_key .. '_spam_set', params.pending_key .. '_ham_set' }
1182 )
1183 end
1184 end
1185 end
1186
1187 local function ann_trained(err, data)
1188 params.set.learning_spawned = false
1189 if err then
1190 rspamd_logger.errx(rspamd_config, 'cannot train ANN %s:%s : %s',
1191 params.rule.prefix, params.set.name, err)
1192 lua_redis.redis_make_request_taskless(params.ev_base,
1193 rspamd_config,
1194 params.rule.redis,
1195 nil,
1196 true, -- is write
1197 gen_unlock_cb(params.rule, params.set, params.ann_key), --callback
1198 'HDEL', -- command
1199 { params.ann_key, 'lock' }
1200 )
1201 else
1202 local parser = ucl.parser()
1203 local ok, parse_err = parser:parse_text(data, 'msgpack')
1204 if not ok then
1205 rspamd_logger.errx(rspamd_config, 'cannot parse training result for ANN %s:%s: %s (data size: %s)',
1206 params.rule.prefix, params.set.name, parse_err, #data)
1207 lua_redis.redis_make_request_taskless(params.ev_base,
1208 rspamd_config,
1209 params.rule.redis,
1210 nil,
1211 true,
1212 gen_unlock_cb(params.rule, params.set, params.ann_key),
1213 'HDEL',
1214 { params.ann_key, 'lock' }
1215 )
1216 return
1217 end
1218 local parsed = parser:get_object()
1219 local ann_data = rspamd_util.zstd_compress(parsed.ann_data)
1220 local pca_data = parsed.pca_data
1221 local roc_thresholds = parsed.roc_thresholds
1222 local norm_stats = parsed.norm_stats
1223
1224 fill_set_ann(params.set, params.ann_key)
1225 if pca_data then
1226 params.set.ann.pca = rspamd_tensor.load(pca_data)
1227 pca_data = rspamd_util.zstd_compress(pca_data)
1228 end
1229
1230 if roc_thresholds then
1231 params.set.ann.roc_thresholds = roc_thresholds
1232 end
1233
1234
1235 -- Deserialise ANN from the child process
1236 local loaded_ann = rspamd_kann.load(parsed.ann_data)
1237 local version = (params.set.ann.version or 0) + 1
1238 params.set.ann.version = version
1239 params.set.ann.ann = loaded_ann
1240 params.set.ann.symbols = params.set.symbols
1241 params.set.ann.redis_key = new_ann_key(params.rule, params.set, version)
1242
1243 local profile = {
1244 symbols = params.set.symbols,
1245 digest = params.set.digest,
1246 redis_key = params.set.ann.redis_key,
1247 version = version,
1248 providers_digest = providers_config_digest(params.rule.providers, params.rule),
1249 }
1250
1251 local profile_serialized = ucl.to_format(profile, 'json-compact', true)
1252 local roc_thresholds_serialized = ucl.to_format(roc_thresholds, 'json-compact', true)
1253 local providers_meta_serialized
1254 if params.rule.providers then
1255 providers_meta_serialized = ucl.to_format(
1256 build_providers_meta(params.set.ann.providers or params.rule.providers), 'json-compact', true)
1257 end
1258
1259 rspamd_logger.infox(rspamd_config,
1260 'trained ANN %s:%s, %s bytes (%s compressed); %s rows in pca (%sb compressed); redis key: %s (old key %s)',
1261 params.rule.prefix, params.set.name,
1262 #data, #ann_data,
1263 #(params.set.ann.pca or {}), #(pca_data or {}),
1264 params.set.ann.redis_key, params.ann_key)
1265
1266 -- Ensure all arguments are non-nil for Lua 5.4 compatibility
1267 -- (nil values in tables cause length/iteration issues)
1268 lua_redis.exec_redis_script(redis_script_id.save_unlock,
1269 { ev_base = params.ev_base, is_write = true },
1270 redis_save_cb,
1271 { profile.redis_key,
1272 redis_ann_prefix(params.rule, params.set.name),
1273 params.ann_key, -- old key to unlock...
1274 },
1275 { ann_data,
1276 profile_serialized,
1277 tostring(params.rule.ann_expire),
1278 tostring(os.time()),
1279 roc_thresholds_serialized or '',
1280 pca_data or '',
1281 providers_meta_serialized or '',
1282 ucl.to_format(norm_stats, 'json-compact', true) or '',
1283 })
1284 end
1285 -- Force GC to clean up training temporaries (parsed data, compressed buffers, etc.)
1286 -- to prevent LuaJIT GC atomic phase stalls on a bloated heap
1287 collectgarbage('collect')
1288 end
1289
1290 if params.rule.max_inputs then
1291 fill_set_ann(params.set, params.ann_key)
1292 end
1293
1294 params.worker:spawn_process {
1295 func = train,
1296 on_complete = ann_trained,
1297 proctitle = string.format("ANN train for %s/%s", params.rule.prefix, params.set.name),
1298 }
1299 -- Register lock extension (learning_spawned already set at start of spawn_train)
1300 register_lock_extender(params.rule, params.set, params.ev_base, params.ann_key)
1301 return
1302 end
1303 end
1304
1305 -- This function is used to adjust profiles and allowed setting ids for each rule
1306 -- It must be called when all settings are already registered (e.g. at post-init for config)
1307 local function process_rules_settings()
1308 local function process_settings_elt(rule, selt)
1309 local profile = rule.profile[selt.name]
1310 if profile then
1311 -- Use static user defined profile
1312 -- Ensure that we have an array...
1313 lua_util.debugm(N, rspamd_config, "use static profile for %s (%s): %s",
1314 rule.prefix, selt.name, profile)
1315 if not profile[1] then
1316 profile = lua_util.keys(profile)
1317 end
1318 selt.symbols = profile
1319 else
1320 lua_util.debugm(N, rspamd_config, "use dynamic cfg based profile for %s (%s)",
1321 rule.prefix, selt.name)
1322 end
1323
1324 local function filter_symbols_predicate(sname)
1325 if settings.blacklisted_symbols and settings.blacklisted_symbols[sname] then
1326 return false
1327 end
1328 local fl = rspamd_config:get_symbol_flags(sname)
1329 if fl then
1330 fl = lua_util.list_to_hash(fl)
1331
1332 return not (fl.nostat or fl.idempotent or fl.skip or fl.composite)
1333 end
1334
1335 return true
1336 end
1337
1338 -- Generic stuff
1339 if not profile then
1340 -- Do filtering merely if we are using a dynamic profile
1341 selt.symbols = fun.totable(fun.filter(filter_symbols_predicate, selt.symbols))
1342 end
1343
1344 table.sort(selt.symbols)
1345
1346 selt.digest = lua_util.table_digest(selt.symbols)
1347 selt.prefix = redis_ann_prefix(rule, selt.name)
1348
1349 rspamd_logger.messagex(rspamd_config,
1350 'use NN prefix for rule %s; settings id "%s"; symbols digest: "%s"',
1351 selt.prefix, selt.name, selt.digest)
1352
1353 lua_redis.register_prefix(selt.prefix, N,
1354 string.format('NN prefix for rule "%s"; settings id "%s"',
1355 selt.prefix, selt.name), {
1356 persistent = true,
1357 type = 'zlist',
1358 })
1359 -- Versions
1360 lua_redis.register_prefix(selt.prefix .. '_\\d+', N,
1361 string.format('NN storage for rule "%s"; settings id "%s"',
1362 selt.prefix, selt.name), {
1363 persistent = true,
1364 type = 'hash',
1365 })
1366 lua_redis.register_prefix(selt.prefix .. '_\\d+_spam_set', N,
1367 string.format('NN learning set (spam) for rule "%s"; settings id "%s"',
1368 selt.prefix, selt.name), {
1369 persistent = true,
1370 type = 'set',
1371 })
1372 lua_redis.register_prefix(selt.prefix .. '_\\d+_ham_set', N,
1373 string.format('NN learning set (ham) for rule "%s"; settings id "%s"',
1374 rule.prefix, selt.name), {
1375 persistent = true,
1376 type = 'set',
1377 })
1378 end
1379
1380 for k, rule in pairs(settings.rules) do
1381 if not rule.allowed_settings then
1382 rule.allowed_settings = {}
1383 elseif rule.allowed_settings == 'all' then
1384 -- Extract all settings ids
1385 rule.allowed_settings = lua_util.keys(lua_settings.all_settings())
1386 end
1387
1388 -- Convert to a map <setting_id> -> true
1389 rule.allowed_settings = lua_util.list_to_hash(rule.allowed_settings)
1390
1391 -- Check if we can work without settings
1392 if k == 'default' or type(rule.default) ~= 'boolean' then
1393 rule.default = true
1394 end
1395
1396 rule.settings = {}
1397
1398 if rule.default then
1399 local default_settings = {
1400 symbols = lua_settings.default_symbols(),
1401 name = 'default'
1402 }
1403
1404 process_settings_elt(rule, default_settings)
1405 rule.settings[-1] = default_settings -- Magic constant, but OK as settings are positive int32
1406 end
1407
1408 -- Now, for each allowed settings, we store sorted symbols + digest
1409 -- We set table rule.settings[id] -> { name = name, symbols = symbols, digest = digest }
1410 for s, _ in pairs(rule.allowed_settings) do
1411 -- Here, we have a name, set of symbols and
1412 local settings_id = s
1413 if type(settings_id) ~= 'number' then
1414 settings_id = lua_settings.numeric_settings_id(s)
1415 end
1416 local selt = lua_settings.settings_by_id(settings_id)
1417
1418 local nelt = {
1419 symbols = selt.symbols, -- Already sorted
1420 name = selt.name
1421 }
1422
1423 process_settings_elt(rule, nelt)
1424 for id, ex in pairs(rule.settings) do
1425 if type(ex) == 'table' then
1426 if nelt and lua_util.distance_sorted(ex.symbols, nelt.symbols) == 0 then
1427 -- Equal symbols, add reference
1428 lua_util.debugm(N, rspamd_config,
1429 'added reference from settings id %s to %s; same symbols',
1430 nelt.name, ex.name)
1431 rule.settings[settings_id] = id
1432 nelt = nil
1433 end
1434 end
1435 end
1436
1437 if nelt then
1438 rule.settings[settings_id] = nelt
1439 lua_util.debugm(N, rspamd_config, 'added new settings id %s(%s) to %s',
1440 nelt.name, settings_id, rule.prefix)
1441 end
1442 end
1443 end
1444 end
1445
1446 -- Extract settings element for a specific settings id
1447 local function get_rule_settings(task, rule)
1448 local sid = task:get_settings_id() or -1
1449 local set = rule.settings[sid]
1450
1451 if not set then
1452 return nil
1453 end
1454
1455 while type(set) == 'number' do
1456 -- Reference to another settings!
1457 set = rule.settings[set]
1458 end
1459
1460 return set
1461 end
1462
1463 result_to_vector = function(task, profile)
1464 if not profile.zeros then
1465 -- Fill zeros vector
1466 local zeros = {}
1467 for i = 1, meta_functions.rspamd_count_metatokens() do
1468 zeros[i] = 0.0
1469 end
1470 for _, _ in ipairs(profile.symbols) do
1471 zeros[#zeros + 1] = 0.0
1472 end
1473 profile.zeros = zeros
1474 end
1475
1476 local vec = lua_util.shallowcopy(profile.zeros)
1477 local mt = meta_functions.rspamd_gen_metatokens(task)
1478
1479 for i, v in ipairs(mt) do
1480 vec[i] = v
1481 end
1482
1483 task:process_ann_tokens(profile.symbols, vec, #mt, 0.1)
1484
1485 return vec
1486 end
1487
1488 return {
1489 can_push_train_vector = can_push_train_vector,
1490 collect_features_async = collect_features_async,
1491 create_ann = create_ann,
1492 default_options = default_options,
1493 build_providers_meta = build_providers_meta,
1494 apply_normalization = apply_normalization,
1495 gen_unlock_cb = gen_unlock_cb,
1496 get_provider = get_provider,
1497 get_rule_settings = get_rule_settings,
1498 load_scripts = load_scripts,
1499 module_config = module_config,
1500 new_ann_key = new_ann_key,
1501 pending_train_key = pending_train_key,
1502 providers_config_digest = providers_config_digest,
1503 register_provider = register_provider,
1504 plugin_ver = plugin_ver,
1505 process_rules_settings = process_rules_settings,
1506 redis_ann_prefix = redis_ann_prefix,
1507 redis_params = redis_params,
1508 redis_script_id = redis_script_id,
1509 result_to_vector = result_to_vector,
1510 settings = settings,
1511 spawn_train = spawn_train,
1512 -- External model support
1513 neural_external = neural_external,
1514 }