| 1 |
--[[ |
| 2 |
Copyright (c) 2022, Vsevolod Stakhov <[email protected]> |
| 3 |
|
| 4 |
Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 |
you may not use this file except in compliance with the License. |
| 6 |
You may obtain a copy of the License at |
| 7 |
|
| 8 |
http://www.apache.org/licenses/LICENSE-2.0 |
| 9 |
|
| 10 |
Unless required by applicable law or agreed to in writing, software |
| 11 |
distributed under the License is distributed on an "AS IS" BASIS, |
| 12 |
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 |
See the License for the specific language governing permissions and |
| 14 |
limitations under the License. |
| 15 |
]] -- |
| 16 |
|
| 17 |
local fun = require "fun" |
| 18 |
local lua_redis = require "lua_redis" |
| 19 |
local lua_settings = require "lua_settings" |
| 20 |
local lua_util = require "lua_util" |
| 21 |
local meta_functions = require "lua_meta" |
| 22 |
local rspamd_kann = require "rspamd_kann" |
| 23 |
local rspamd_logger = require "rspamd_logger" |
| 24 |
local rspamd_tensor = require "rspamd_tensor" |
| 25 |
local rspamd_util = require "rspamd_util" |
| 26 |
local ucl = require "ucl" |
| 27 |
local neural_external = require "lua_neural_external" |
| 28 |
|
| 29 |
local N = 'neural' |
| 30 |
|
| 31 |
-- Used in prefix to avoid wrong ANN to be loaded |
| 32 |
local plugin_ver = '3' |
| 33 |
|
| 34 |
-- Module vars |
| 35 |
local default_options = { |
| 36 |
train = { |
| 37 |
max_trains = 1000, |
| 38 |
max_epoch = 1000, |
| 39 |
max_usages = 10, |
| 40 |
max_iterations = 25, -- Torch style |
| 41 |
mse = 0.001, |
| 42 |
autotrain = true, |
| 43 |
train_prob = 1.0, |
| 44 |
learn_threads = 1, |
| 45 |
learn_mode = 'balanced', -- Possible values: balanced, proportional |
| 46 |
learning_rate = 0.01, |
| 47 |
classes_bias = 0.0, -- balanced mode: what difference is allowed between classes (1:1 proportion means 0 bias) |
| 48 |
spam_skip_prob = 0.0, -- proportional mode: spam skip probability (0-1) |
| 49 |
ham_skip_prob = 0.0, -- proportional mode: ham skip probability |
| 50 |
store_pool_only = false, -- store tokens in cache only (disables autotrain); |
| 51 |
store_set_only = false, -- store ham and spam sets in Redis, but do not train ANN (autotrain must be enabled); |
| 52 |
-- neural_vec_mpack stores vector of training data in messagepack neural_profile_digest stores profile digest |
| 53 |
}, |
| 54 |
watch_interval = 60.0, |
| 55 |
lock_expire = 600, |
| 56 |
learning_spawned = false, |
| 57 |
ann_expire = 60 * 60 * 24 * 2, -- 2 days |
| 58 |
hidden_layer_mult = 1.5, -- number of neurons in the hidden layer (symbol-based mode) |
| 59 |
-- Multi-layer architecture settings (for LLM embeddings mode) |
| 60 |
layers = nil, -- layer size multipliers (auto-computed based on input dim if nil) |
| 61 |
dropout = nil, -- dropout rate (0.2 default for embeddings, nil=disabled for symbols) |
| 62 |
use_layernorm = nil, -- enable layer normalization (true default for embeddings) |
| 63 |
activation = nil, -- activation function: 'relu' or 'gelu' (default: gelu for embeddings, relu for symbols) |
| 64 |
roc_enabled = false, -- Use ROC to find the best possible thresholds for ham and spam. If spam_score_threshold or ham_score_threshold is defined, it takes precedence over ROC thresholds. |
| 65 |
roc_misclassification_cost = 0.5, -- Cost of misclassifying a spam message (must be 0..1). |
| 66 |
spam_score_threshold = nil, -- neural score threshold for spam (must be 0..1 or nil to disable) |
| 67 |
ham_score_threshold = nil, -- neural score threshold for ham (must be 0..1 or nil to disable) |
| 68 |
flat_threshold_curve = false, -- use binary classification 0/1 when threshold is reached |
| 69 |
symbol_spam = 'NEURAL_SPAM', |
| 70 |
symbol_ham = 'NEURAL_HAM', |
| 71 |
max_inputs = nil, -- when PCA is used |
| 72 |
blacklisted_symbols = {}, -- list of symbols skipped in neural processing |
| 73 |
-- Phase 0 additions (scaffolding for feature providers) |
| 74 |
providers = nil, -- list of provider configs; if nil, fallback to symbols-only provider |
| 75 |
fusion = { |
| 76 |
normalization = 'none', -- none|unit|zscore (zscore requires stats) |
| 77 |
per_provider_pca = false, -- if true, apply PCA per provider before fusion (not active yet) |
| 78 |
}, |
| 79 |
disable_symbols_input = false, -- when true, do not use symbols provider unless explicitly listed |
| 80 |
-- External pretrained model support |
| 81 |
external_model = nil, -- external model configuration (see lua_neural_external) |
| 82 |
} |
| 83 |
|
| 84 |
-- Rule structure: |
| 85 |
-- * static config fields (see `default_options`) |
| 86 |
-- * prefix - name or defined prefix |
| 87 |
-- * settings - table of settings indexed by settings id, -1 is used when no settings defined |
| 88 |
|
| 89 |
-- Rule settings element defines elements for specific settings id: |
| 90 |
-- * symbols - static symbols profile (defined by config or extracted from symcache) |
| 91 |
-- * name - name of settings id |
| 92 |
-- * digest - digest of all symbols |
| 93 |
-- * ann - dynamic ANN configuration loaded from Redis |
| 94 |
-- * train - train data for ANN (e.g. the currently trained ANN) |
| 95 |
|
| 96 |
-- Settings ANN table is loaded from Redis and represents dynamic profile for ANN |
| 97 |
-- Some elements are directly stored in Redis, ANN is, in turn loaded dynamically |
| 98 |
-- * version - version of ANN loaded from redis |
| 99 |
-- * redis_key - name of ANN key in Redis |
| 100 |
-- * symbols - symbols in THIS PARTICULAR ANN (might be different from set.symbols) |
| 101 |
-- * distance - distance between set.symbols and set.ann.symbols |
| 102 |
-- * ann - kann object |
| 103 |
|
| 104 |
local settings = { |
| 105 |
rules = {}, |
| 106 |
prefix = 'rn', -- Neural network default prefix |
| 107 |
max_profiles = 3, -- Maximum number of NN profiles stored |
| 108 |
} |
| 109 |
|
| 110 |
-- Get module & Redis configuration |
| 111 |
local module_config = rspamd_config:get_all_opt(N) |
| 112 |
settings = lua_util.override_defaults(settings, module_config) |
| 113 |
local redis_params = lua_redis.parse_redis_server('neural') |
| 114 |
|
| 115 |
local redis_lua_script_vectors_len = "neural_train_size.lua" |
| 116 |
local redis_lua_script_maybe_invalidate = "neural_maybe_invalidate.lua" |
| 117 |
local redis_lua_script_maybe_lock = "neural_maybe_lock.lua" |
| 118 |
local redis_lua_script_save_unlock = "neural_save_unlock.lua" |
| 119 |
|
| 120 |
local redis_script_id = {} |
| 121 |
|
| 122 |
-- Provider registry (Phase 0 scaffolding) |
| 123 |
local registered_providers = {} |
| 124 |
|
| 125 |
--- Registers a feature provider implementation |
| 126 |
-- @param name string |
| 127 |
-- @param provider table with function collect(task, ctx) -> vector(table of numbers), meta(table) |
| 128 |
local function register_provider(name, provider) |
| 129 |
registered_providers[name] = provider |
| 130 |
end |
| 131 |
|
| 132 |
local function get_provider(name) |
| 133 |
return registered_providers[name] |
| 134 |
end |
| 135 |
|
| 136 |
-- Forward declaration |
| 137 |
local result_to_vector |
| 138 |
|
| 139 |
-- Built-in symbols provider (compatibility path) |
| 140 |
register_provider('symbols', { |
| 141 |
collect = function(task, ctx) |
| 142 |
local vec = result_to_vector(task, ctx.profile) |
| 143 |
return vec, { name = 'symbols', type = 'symbols', dim = #vec, weight = ctx.weight or 1.0 } |
| 144 |
end, |
| 145 |
collect_async = function(task, ctx, cont) |
| 146 |
local vec = result_to_vector(task, ctx.profile) |
| 147 |
cont(vec, { name = 'symbols', type = 'symbols', dim = #vec, weight = ctx.weight or 1.0 }) |
| 148 |
end, |
| 149 |
}) |
| 150 |
|
| 151 |
-- Metatokens-only provider for contexts where symbols are not available |
| 152 |
register_provider('metatokens', { |
| 153 |
collect = function(task, ctx) |
| 154 |
local mt = meta_functions.rspamd_gen_metatokens(task) |
| 155 |
-- Convert to table of numbers |
| 156 |
local vec = {} |
| 157 |
for i = 1, #mt do |
| 158 |
vec[i] = tonumber(mt[i]) or 0.0 |
| 159 |
end |
| 160 |
return vec, { name = 'metatokens', type = 'metatokens', dim = #vec, weight = ctx.weight or 1.0 } |
| 161 |
end, |
| 162 |
collect_async = function(task, ctx, cont) |
| 163 |
local mt = meta_functions.rspamd_gen_metatokens(task) |
| 164 |
-- Convert to table of numbers |
| 165 |
local vec = {} |
| 166 |
for i = 1, #mt do |
| 167 |
vec[i] = tonumber(mt[i]) or 0.0 |
| 168 |
end |
| 169 |
cont(vec, { name = 'metatokens', type = 'metatokens', dim = #vec, weight = ctx.weight or 1.0 }) |
| 170 |
end, |
| 171 |
}) |
| 172 |
|
| 173 |
local function load_scripts() |
| 174 |
local err |
| 175 |
redis_script_id.vectors_len, err = lua_redis.load_redis_script_from_file(redis_lua_script_vectors_len, |
| 176 |
redis_params) |
| 177 |
if err then |
| 178 |
rspamd_logger.errx(rspamd_config, err) |
| 179 |
end |
| 180 |
redis_script_id.maybe_invalidate, err = lua_redis.load_redis_script_from_file(redis_lua_script_maybe_invalidate, |
| 181 |
redis_params) |
| 182 |
if err then |
| 183 |
rspamd_logger.errx(rspamd_config, err) |
| 184 |
end |
| 185 |
redis_script_id.maybe_lock, err = lua_redis.load_redis_script_from_file(redis_lua_script_maybe_lock, |
| 186 |
redis_params) |
| 187 |
if err then |
| 188 |
rspamd_logger.errx(rspamd_config, err) |
| 189 |
end |
| 190 |
redis_script_id.save_unlock, err = lua_redis.load_redis_script_from_file(redis_lua_script_save_unlock, |
| 191 |
redis_params) |
| 192 |
if err then |
| 193 |
rspamd_logger.errx(rspamd_config, err) |
| 194 |
end |
| 195 |
end |
| 196 |
|
| 197 |
-- Creates a simple single-layer ANN for symbol-based inputs (backward compatible) |
| 198 |
local function create_symbol_ann(n, rule) |
| 199 |
local nhidden = math.floor(n * (rule.hidden_layer_mult or 1.0) + 1.0) |
| 200 |
local t = rspamd_kann.layer.input(n) |
| 201 |
t = rspamd_kann.transform.relu(t) |
| 202 |
t = rspamd_kann.layer.dense(t, nhidden) |
| 203 |
t = rspamd_kann.layer.cost(t, 1, rspamd_kann.cost.ceb_neg) |
| 204 |
return rspamd_kann.new.kann(t) |
| 205 |
end |
| 206 |
|
| 207 |
-- Creates a multi-layer funnel ANN optimized for high-dimensional embeddings |
| 208 |
-- Architecture: Input → [Dense → LayerNorm → Activation → Dropout]* → Cost |
| 209 |
local function create_embedding_ann(n, rule) |
| 210 |
local t = rspamd_kann.layer.input(n) |
| 211 |
|
| 212 |
-- Get architecture settings with smart defaults based on input dimension |
| 213 |
local layers = rule.layers |
| 214 |
if not layers then |
| 215 |
-- Auto-compute layer sizes based on input dimension |
| 216 |
if n > 512 then |
| 217 |
layers = { 0.5, 0.25, 0.125 } -- 3 layers for large embeddings (e.g., 1024-dim) |
| 218 |
elseif n > 256 then |
| 219 |
layers = { 0.5, 0.25 } -- 2 layers for medium embeddings |
| 220 |
else |
| 221 |
layers = { 0.5 } -- 1 layer for small embeddings |
| 222 |
end |
| 223 |
end |
| 224 |
|
| 225 |
local dropout_rate = rule.dropout |
| 226 |
if dropout_rate == nil then |
| 227 |
dropout_rate = 0.2 -- Default dropout for regularization |
| 228 |
end |
| 229 |
|
| 230 |
local use_layernorm = rule.use_layernorm |
| 231 |
if use_layernorm == nil then |
| 232 |
use_layernorm = true -- Default: enable layer normalization |
| 233 |
end |
| 234 |
|
| 235 |
-- Select activation function: GELU for embeddings (better for high-dim), ReLU as fallback |
| 236 |
local activation = rule.activation |
| 237 |
if not activation then |
| 238 |
-- Default to GELU for embeddings if available |
| 239 |
activation = rspamd_kann.transform.gelu and 'gelu' or 'relu' |
| 240 |
end |
| 241 |
local activate_fn = (activation == 'gelu' and rspamd_kann.transform.gelu) or rspamd_kann.transform.relu |
| 242 |
|
| 243 |
lua_util.debugm(N, rspamd_config, 'embedding ANN: %s layers, dropout=%s, layernorm=%s, activation=%s', |
| 244 |
#layers, dropout_rate, use_layernorm, activation) |
| 245 |
|
| 246 |
-- Build funnel architecture with graduated dimension reduction |
| 247 |
for i, layer_mult in ipairs(layers) do |
| 248 |
local layer_size = math.max(math.floor(n * layer_mult), 32) |
| 249 |
|
| 250 |
-- Dense layer |
| 251 |
t = rspamd_kann.layer.dense(t, layer_size) |
| 252 |
|
| 253 |
-- Layer normalization for training stability |
| 254 |
if use_layernorm then |
| 255 |
t = rspamd_kann.layer.layernorm(t) |
| 256 |
end |
| 257 |
|
| 258 |
-- Activation function (GELU or ReLU) |
| 259 |
t = activate_fn(t) |
| 260 |
|
| 261 |
-- Dropout for regularization (less on final hidden layer) |
| 262 |
if dropout_rate > 0 then |
| 263 |
local rate = (i == #layers) and (dropout_rate * 0.5) or dropout_rate |
| 264 |
t = rspamd_kann.layer.dropout(t, rate) |
| 265 |
end |
| 266 |
end |
| 267 |
|
| 268 |
t = rspamd_kann.layer.cost(t, 1, rspamd_kann.cost.ceb_neg) |
| 269 |
return rspamd_kann.new.kann(t) |
| 270 |
end |
| 271 |
|
| 272 |
-- Conv1d ANN: uses the enhanced embedding architecture. |
| 273 |
-- The actual convolution (multi-scale max-over-time pooling) is done in the |
| 274 |
-- fasttext_embed provider, which produces compact feature vectors (n_scales * channels). |
| 275 |
-- The ANN itself is a simple dense network on these pre-convolved features. |
| 276 |
local function create_conv1d_ann(n, rule) |
| 277 |
lua_util.debugm(N, rspamd_config, |
| 278 |
'creating conv1d ANN: %s pre-convolved inputs', n) |
| 279 |
return create_embedding_ann(n, rule) |
| 280 |
end |
| 281 |
|
| 282 |
-- Detects if rule uses LLM embeddings provider |
| 283 |
local function uses_llm_embeddings(rule) |
| 284 |
if not rule.providers then |
| 285 |
return false |
| 286 |
end |
| 287 |
for _, p in ipairs(rule.providers) do |
| 288 |
if p.type == 'llm' then |
| 289 |
return true |
| 290 |
end |
| 291 |
end |
| 292 |
return false |
| 293 |
end |
| 294 |
|
| 295 |
-- Main ANN factory function - auto-selects architecture based on rule configuration |
| 296 |
local function create_ann(n, nlayers, rule) |
| 297 |
-- Check for conv1d architecture first |
| 298 |
if rule.conv1d then |
| 299 |
lua_util.debugm(N, rspamd_config, 'creating conv1d ANN with %s inputs', n) |
| 300 |
return create_conv1d_ann(n, rule) |
| 301 |
end |
| 302 |
|
| 303 |
-- Check if we should use the enhanced embedding architecture |
| 304 |
-- Conditions: has LLM provider, or explicit multi-layer config, or large input dimension |
| 305 |
local use_embedding_arch = uses_llm_embeddings(rule) |
| 306 |
or rule.layers ~= nil |
| 307 |
or rule.use_layernorm ~= nil |
| 308 |
or rule.dropout ~= nil |
| 309 |
|
| 310 |
if use_embedding_arch then |
| 311 |
lua_util.debugm(N, rspamd_config, 'creating multi-layer embedding ANN with %s inputs', n) |
| 312 |
return create_embedding_ann(n, rule) |
| 313 |
else |
| 314 |
lua_util.debugm(N, rspamd_config, 'creating simple symbol ANN with %s inputs', n) |
| 315 |
return create_symbol_ann(n, rule) |
| 316 |
end |
| 317 |
end |
| 318 |
|
| 319 |
-- Fills ANN data for a specific settings element |
| 320 |
local function fill_set_ann(set, ann_key) |
| 321 |
if not set.ann then |
| 322 |
set.ann = { |
| 323 |
symbols = set.symbols, |
| 324 |
distance = 0, |
| 325 |
digest = set.digest, |
| 326 |
redis_key = ann_key, |
| 327 |
version = 0, |
| 328 |
} |
| 329 |
end |
| 330 |
end |
| 331 |
|
| 332 |
-- This function takes all inputs, applies PCA transformation and returns the final |
| 333 |
-- PCA matrix as rspamd_tensor |
| 334 |
local function learn_pca(inputs, max_inputs) |
| 335 |
local scatter_matrix = rspamd_tensor.scatter_matrix(rspamd_tensor.fromtable(inputs)) |
| 336 |
local eigenvals = scatter_matrix:eigen() |
| 337 |
-- scatter matrix is not filled with eigenvectors |
| 338 |
lua_util.debugm(N, 'eigenvalues: %s', eigenvals) |
| 339 |
local w = rspamd_tensor.new(2, max_inputs, #scatter_matrix[1]) |
| 340 |
for i = 1, max_inputs do |
| 341 |
w[i] = scatter_matrix[#scatter_matrix - i + 1] |
| 342 |
end |
| 343 |
|
| 344 |
lua_util.debugm(N, 'pca matrix: %s', w) |
| 345 |
|
| 346 |
return w |
| 347 |
end |
| 348 |
|
| 349 |
-- Build providers metadata for storage alongside ANN |
| 350 |
local function build_providers_meta(metas) |
| 351 |
if not metas or #metas == 0 then return nil end |
| 352 |
local out = {} |
| 353 |
for i, m in ipairs(metas) do |
| 354 |
out[i] = { |
| 355 |
name = m.name, |
| 356 |
type = m.type, |
| 357 |
dim = m.dim, |
| 358 |
weight = m.weight, |
| 359 |
model = m.model, |
| 360 |
provider = m.provider, |
| 361 |
} |
| 362 |
end |
| 363 |
return out |
| 364 |
end |
| 365 |
|
| 366 |
-- Normalization helpers |
| 367 |
local function l2_normalize_vector(vec) |
| 368 |
local sumsq = 0.0 |
| 369 |
for i = 1, #vec do |
| 370 |
local v = vec[i] |
| 371 |
sumsq = sumsq + v * v |
| 372 |
end |
| 373 |
if sumsq > 0 then |
| 374 |
local inv = 1.0 / math.sqrt(sumsq) |
| 375 |
for i = 1, #vec do |
| 376 |
vec[i] = vec[i] * inv |
| 377 |
end |
| 378 |
end |
| 379 |
return vec |
| 380 |
end |
| 381 |
|
| 382 |
local function compute_zscore_stats(inputs) |
| 383 |
local n = #inputs |
| 384 |
if n == 0 then return nil end |
| 385 |
local d = #inputs[1] |
| 386 |
local mean = {} |
| 387 |
local m2 = {} |
| 388 |
for j = 1, d do |
| 389 |
mean[j] = 0.0 |
| 390 |
m2[j] = 0.0 |
| 391 |
end |
| 392 |
for i = 1, n do |
| 393 |
local x = inputs[i] |
| 394 |
for j = 1, d do |
| 395 |
local delta = x[j] - mean[j] |
| 396 |
mean[j] = mean[j] + delta / i |
| 397 |
m2[j] = m2[j] + delta * (x[j] - mean[j]) |
| 398 |
end |
| 399 |
end |
| 400 |
local std = {} |
| 401 |
for j = 1, d do |
| 402 |
std[j] = math.sqrt((n > 1 and (m2[j] / (n - 1))) or 0.0) |
| 403 |
if std[j] == 0 or std[j] ~= std[j] then |
| 404 |
std[j] = 1.0 -- avoid division by zero and NaN |
| 405 |
end |
| 406 |
end |
| 407 |
return { mode = 'zscore', mean = mean, std = std } |
| 408 |
end |
| 409 |
|
| 410 |
local function apply_normalization(vec, norm_stats_or_mode) |
| 411 |
if not norm_stats_or_mode then return vec end |
| 412 |
if type(norm_stats_or_mode) == 'string' then |
| 413 |
if norm_stats_or_mode == 'unit' then |
| 414 |
return l2_normalize_vector(vec) |
| 415 |
else |
| 416 |
return vec |
| 417 |
end |
| 418 |
else |
| 419 |
if norm_stats_or_mode.mode == 'unit' then |
| 420 |
return l2_normalize_vector(vec) |
| 421 |
elseif norm_stats_or_mode.mode == 'zscore' and norm_stats_or_mode.mean and norm_stats_or_mode.std then |
| 422 |
local mean = norm_stats_or_mode.mean |
| 423 |
local std = norm_stats_or_mode.std |
| 424 |
for i = 1, math.min(#vec, #mean) do |
| 425 |
vec[i] = (vec[i] - (mean[i] or 0.0)) / (std[i] or 1.0) |
| 426 |
end |
| 427 |
return vec |
| 428 |
else |
| 429 |
return vec |
| 430 |
end |
| 431 |
end |
| 432 |
end |
| 433 |
|
| 434 |
-- This function computes optimal threshold using ROC for the given set of inputs. |
| 435 |
-- Returns a threshold that minimizes: |
| 436 |
-- alpha * (false_positive_rate) + beta * (false_negative_rate) |
| 437 |
-- Where alpha is cost of false positive result |
| 438 |
-- beta is cost of false negative result |
| 439 |
local function get_roc_thresholds(ann, inputs, outputs, alpha, beta) |
| 440 |
-- Sorts list x and list y based on the values in list x. |
| 441 |
local sort_relative = function(x, y) |
| 442 |
local r = {} |
| 443 |
|
| 444 |
assert(#x == #y) |
| 445 |
local n = #x |
| 446 |
|
| 447 |
local a = {} |
| 448 |
local b = {} |
| 449 |
for i = 1, n do |
| 450 |
r[i] = i |
| 451 |
end |
| 452 |
|
| 453 |
local cmp = function(p, q) |
| 454 |
return p < q |
| 455 |
end |
| 456 |
|
| 457 |
table.sort(r, function(p, q) |
| 458 |
return cmp(x[p], x[q]) |
| 459 |
end) |
| 460 |
|
| 461 |
for i = 1, n do |
| 462 |
a[i] = x[r[i]] |
| 463 |
b[i] = y[r[i]] |
| 464 |
end |
| 465 |
|
| 466 |
return a, b |
| 467 |
end |
| 468 |
|
| 469 |
local function get_scores(nn, input_vectors) |
| 470 |
local scores = {} |
| 471 |
for i = 1, #inputs do |
| 472 |
local score = nn:apply1(input_vectors[i], nn.pca)[1] |
| 473 |
scores[#scores + 1] = score |
| 474 |
end |
| 475 |
|
| 476 |
return scores |
| 477 |
end |
| 478 |
|
| 479 |
local fpr = {} |
| 480 |
local fnr = {} |
| 481 |
local scores = get_scores(ann, inputs) |
| 482 |
|
| 483 |
scores, outputs = sort_relative(scores, outputs) |
| 484 |
|
| 485 |
local n_samples = #outputs |
| 486 |
local n_spam = 0 |
| 487 |
local n_ham = 0 |
| 488 |
local ham_count_ahead = {} |
| 489 |
local spam_count_ahead = {} |
| 490 |
local ham_count_behind = {} |
| 491 |
local spam_count_behind = {} |
| 492 |
|
| 493 |
ham_count_ahead[n_samples + 1] = 0 |
| 494 |
spam_count_ahead[n_samples + 1] = 0 |
| 495 |
|
| 496 |
for i = n_samples, 1, -1 do |
| 497 |
-- Labels are -1.0 for ham and 1.0 for spam (ceb_neg cost function) |
| 498 |
if outputs[i][1] < 0 then |
| 499 |
n_ham = n_ham + 1 |
| 500 |
ham_count_ahead[i] = 1 |
| 501 |
spam_count_ahead[i] = 0 |
| 502 |
else |
| 503 |
n_spam = n_spam + 1 |
| 504 |
ham_count_ahead[i] = 0 |
| 505 |
spam_count_ahead[i] = 1 |
| 506 |
end |
| 507 |
|
| 508 |
ham_count_ahead[i] = ham_count_ahead[i] + ham_count_ahead[i + 1] |
| 509 |
spam_count_ahead[i] = spam_count_ahead[i] + spam_count_ahead[i + 1] |
| 510 |
end |
| 511 |
|
| 512 |
for i = 1, n_samples do |
| 513 |
-- Labels are -1.0 for ham and 1.0 for spam (ceb_neg cost function) |
| 514 |
if outputs[i][1] < 0 then |
| 515 |
ham_count_behind[i] = 1 |
| 516 |
spam_count_behind[i] = 0 |
| 517 |
else |
| 518 |
ham_count_behind[i] = 0 |
| 519 |
spam_count_behind[i] = 1 |
| 520 |
end |
| 521 |
|
| 522 |
if i ~= 1 then |
| 523 |
ham_count_behind[i] = ham_count_behind[i] + ham_count_behind[i - 1] |
| 524 |
spam_count_behind[i] = spam_count_behind[i] + spam_count_behind[i - 1] |
| 525 |
end |
| 526 |
end |
| 527 |
|
| 528 |
for i = 1, n_samples do |
| 529 |
fpr[i] = 0 |
| 530 |
fnr[i] = 0 |
| 531 |
|
| 532 |
if (ham_count_ahead[i + 1] + ham_count_behind[i]) ~= 0 then |
| 533 |
fpr[i] = ham_count_ahead[i + 1] / (ham_count_ahead[i + 1] + ham_count_behind[i]) |
| 534 |
end |
| 535 |
|
| 536 |
if (spam_count_behind[i] + spam_count_ahead[i + 1]) ~= 0 then |
| 537 |
fnr[i] = spam_count_behind[i] / (spam_count_behind[i] + spam_count_ahead[i + 1]) |
| 538 |
end |
| 539 |
end |
| 540 |
|
| 541 |
local p = n_spam / (n_spam + n_ham) |
| 542 |
|
| 543 |
local cost = {} |
| 544 |
local min_cost_idx = 0 |
| 545 |
local min_cost = math.huge |
| 546 |
for i = 1, n_samples do |
| 547 |
cost[i] = ((1 - p) * alpha * fpr[i]) + (p * beta * fnr[i]) |
| 548 |
if min_cost >= cost[i] then |
| 549 |
min_cost = cost[i] |
| 550 |
min_cost_idx = i |
| 551 |
end |
| 552 |
end |
| 553 |
|
| 554 |
return scores[min_cost_idx] |
| 555 |
end |
| 556 |
|
| 557 |
-- This function is intended to extend lock for ANN during training |
| 558 |
-- It registers periodic that increases locked key each 30 seconds unless |
| 559 |
-- `set.learning_spawned` is set to `true` |
| 560 |
local function register_lock_extender(rule, set, ev_base, ann_key) |
| 561 |
rspamd_config:add_periodic(ev_base, 30.0, |
| 562 |
function() |
| 563 |
local function redis_lock_extend_cb(err, _) |
| 564 |
if err then |
| 565 |
rspamd_logger.errx(rspamd_config, 'cannot lock ANN %s from redis: %s', |
| 566 |
ann_key, err) |
| 567 |
else |
| 568 |
rspamd_logger.infox(rspamd_config, 'extend lock for ANN %s for 30 seconds', |
| 569 |
ann_key) |
| 570 |
end |
| 571 |
end |
| 572 |
|
| 573 |
if set.learning_spawned then |
| 574 |
lua_redis.redis_make_request_taskless(ev_base, |
| 575 |
rspamd_config, |
| 576 |
rule.redis, |
| 577 |
nil, |
| 578 |
true, -- is write |
| 579 |
redis_lock_extend_cb, --callback |
| 580 |
'HINCRBY', -- command |
| 581 |
{ ann_key, 'lock', '30' } |
| 582 |
) |
| 583 |
else |
| 584 |
lua_util.debugm(N, rspamd_config, "stop lock extension as learning_spawned is false") |
| 585 |
return false -- do not plan any more updates |
| 586 |
end |
| 587 |
|
| 588 |
return true |
| 589 |
end |
| 590 |
) |
| 591 |
end |
| 592 |
|
| 593 |
local function can_push_train_vector(rule, task, learn_type, nspam, nham) |
| 594 |
local train_opts = rule.train |
| 595 |
local coin = math.random() |
| 596 |
|
| 597 |
if train_opts.train_prob and coin < 1.0 - train_opts.train_prob then |
| 598 |
rspamd_logger.infox(task, 'probabilistically skip sample: %s', coin) |
| 599 |
return false |
| 600 |
end |
| 601 |
|
| 602 |
if train_opts.learn_mode == 'balanced' then |
| 603 |
-- Keep balanced training set based on number of spam and ham samples |
| 604 |
if learn_type == 'spam' then |
| 605 |
if nspam <= train_opts.max_trains then |
| 606 |
if nspam > nham then |
| 607 |
-- Apply sampling |
| 608 |
local skip_rate = 1.0 - nham / (nspam + 1) |
| 609 |
if coin < skip_rate - train_opts.classes_bias then |
| 610 |
rspamd_logger.infox(task, |
| 611 |
'skip %s sample to keep spam/ham balance; probability %s; %s spam and %s ham vectors stored', |
| 612 |
learn_type, |
| 613 |
skip_rate - train_opts.classes_bias, |
| 614 |
nspam, nham) |
| 615 |
return false |
| 616 |
end |
| 617 |
end |
| 618 |
return true |
| 619 |
else |
| 620 |
-- Enough learns |
| 621 |
rspamd_logger.infox(task, 'skip %s sample to keep spam/ham balance; too many spam samples: %s', |
| 622 |
learn_type, |
| 623 |
nspam) |
| 624 |
end |
| 625 |
else |
| 626 |
if nham <= train_opts.max_trains then |
| 627 |
if nham > nspam then |
| 628 |
-- Apply sampling |
| 629 |
local skip_rate = 1.0 - nspam / (nham + 1) |
| 630 |
if coin < skip_rate - train_opts.classes_bias then |
| 631 |
rspamd_logger.infox(task, |
| 632 |
'skip %s sample to keep spam/ham balance; probability %s; %s spam and %s ham vectors stored', |
| 633 |
learn_type, |
| 634 |
skip_rate - train_opts.classes_bias, |
| 635 |
nspam, nham) |
| 636 |
return false |
| 637 |
end |
| 638 |
end |
| 639 |
return true |
| 640 |
else |
| 641 |
rspamd_logger.infox(task, 'skip %s sample to keep spam/ham balance; too many ham samples: %s', learn_type, |
| 642 |
nham) |
| 643 |
end |
| 644 |
end |
| 645 |
else |
| 646 |
-- Probabilistic learn mode, we just skip learn if we already have enough samples or |
| 647 |
-- if our coin drop is less than desired probability |
| 648 |
if learn_type == 'spam' then |
| 649 |
if nspam <= train_opts.max_trains then |
| 650 |
if train_opts.spam_skip_prob then |
| 651 |
if coin <= train_opts.spam_skip_prob then |
| 652 |
rspamd_logger.infox(task, 'skip %s sample probabilistically; probability %s (%s skip chance)', learn_type, |
| 653 |
coin, train_opts.spam_skip_prob) |
| 654 |
return false |
| 655 |
end |
| 656 |
|
| 657 |
return true |
| 658 |
end |
| 659 |
else |
| 660 |
rspamd_logger.infox(task, 'skip %s sample; too many spam samples: %s (%s limit)', learn_type, |
| 661 |
nspam, train_opts.max_trains) |
| 662 |
end |
| 663 |
else |
| 664 |
if nham <= train_opts.max_trains then |
| 665 |
if train_opts.ham_skip_prob then |
| 666 |
if coin <= train_opts.ham_skip_prob then |
| 667 |
rspamd_logger.infox(task, 'skip %s sample probabilistically; probability %s (%s skip chance)', learn_type, |
| 668 |
coin, train_opts.ham_skip_prob) |
| 669 |
return false |
| 670 |
end |
| 671 |
|
| 672 |
return true |
| 673 |
end |
| 674 |
else |
| 675 |
rspamd_logger.infox(task, 'skip %s sample; too many ham samples: %s (%s limit)', learn_type, |
| 676 |
nham, train_opts.max_trains) |
| 677 |
end |
| 678 |
end |
| 679 |
end |
| 680 |
|
| 681 |
return false |
| 682 |
end |
| 683 |
|
| 684 |
-- Closure generator for unlock function |
| 685 |
local function gen_unlock_cb(rule, set, ann_key) |
| 686 |
return function(err) |
| 687 |
if err then |
| 688 |
rspamd_logger.errx(rspamd_config, 'cannot unlock ANN %s:%s at %s from redis: %s', |
| 689 |
rule.prefix, set.name, ann_key, err) |
| 690 |
else |
| 691 |
lua_util.debugm(N, rspamd_config, 'unlocked ANN %s:%s at %s', |
| 692 |
rule.prefix, set.name, ann_key) |
| 693 |
end |
| 694 |
end |
| 695 |
end |
| 696 |
|
| 697 |
-- Used to generate new ANN key for specific profile |
| 698 |
local function new_ann_key(rule, set, version) |
| 699 |
local ann_key = string.format('%s_%s_%s_%s_%d', settings.prefix, |
| 700 |
rule.prefix, set.name, set.digest:sub(1, 8), version) |
| 701 |
|
| 702 |
return ann_key |
| 703 |
end |
| 704 |
|
| 705 |
local function redis_ann_prefix(rule, settings_name) |
| 706 |
-- We also need to count metatokens: |
| 707 |
-- Note: meta_functions.version represents the metatoken format version |
| 708 |
local n = meta_functions.version |
| 709 |
return string.format('%s%d_%s_%d_%s', |
| 710 |
settings.prefix, plugin_ver, rule.prefix, n, settings_name) |
| 711 |
end |
| 712 |
|
| 713 |
-- Returns a stable key for pending training vectors (version-independent) |
| 714 |
-- Used for batch/manual training to avoid version mismatch issues |
| 715 |
local function pending_train_key(rule, set) |
| 716 |
return string.format('%s_%s_%s_pending', |
| 717 |
settings.prefix, rule.prefix, set.name) |
| 718 |
end |
| 719 |
|
| 720 |
-- Compute a stable digest for providers configuration |
| 721 |
local function providers_config_digest(providers_cfg, rule) |
| 722 |
if not providers_cfg then return nil end |
| 723 |
-- Normalize minimal subset of fields to keep digest stable across equivalent configs |
| 724 |
local norm = { providers = {} } |
| 725 |
|
| 726 |
local fusion = rule and rule.fusion or nil |
| 727 |
if rule then |
| 728 |
local effective_fusion = { |
| 729 |
normalization = (fusion and fusion.normalization) or 'none', |
| 730 |
include_meta = fusion and fusion.include_meta, |
| 731 |
meta_weight = fusion and fusion.meta_weight, |
| 732 |
per_provider_pca = fusion and fusion.per_provider_pca, |
| 733 |
} |
| 734 |
if effective_fusion.include_meta == nil then |
| 735 |
effective_fusion.include_meta = true |
| 736 |
end |
| 737 |
if effective_fusion.meta_weight == nil then |
| 738 |
effective_fusion.meta_weight = 1.0 |
| 739 |
end |
| 740 |
if effective_fusion.per_provider_pca == nil then |
| 741 |
effective_fusion.per_provider_pca = false |
| 742 |
end |
| 743 |
norm.fusion = effective_fusion |
| 744 |
end |
| 745 |
|
| 746 |
if rule and rule.max_inputs then |
| 747 |
norm.max_inputs = rule.max_inputs |
| 748 |
end |
| 749 |
|
| 750 |
local gpt_settings = rspamd_config:get_all_opt('gpt') or {} |
| 751 |
|
| 752 |
for i, p in ipairs(providers_cfg) do |
| 753 |
local ptype = p.type or p.name or 'unknown' |
| 754 |
local entry = { |
| 755 |
type = ptype, |
| 756 |
weight = p.weight or 1.0, |
| 757 |
dim = p.dim, |
| 758 |
} |
| 759 |
|
| 760 |
if ptype == 'llm' then |
| 761 |
local llm_type = p.llm_type or p.api or p.backend or gpt_settings.type |
| 762 |
local model = p.model or gpt_settings.model |
| 763 |
local max_tokens = p.max_tokens |
| 764 |
if not max_tokens and gpt_settings.model_parameters and model then |
| 765 |
local model_cfg = gpt_settings.model_parameters[model] or {} |
| 766 |
max_tokens = model_cfg.max_completion_tokens or model_cfg.max_tokens |
| 767 |
end |
| 768 |
if not max_tokens then |
| 769 |
max_tokens = gpt_settings.max_tokens |
| 770 |
end |
| 771 |
|
| 772 |
entry.llm_type = llm_type |
| 773 |
entry.model = model |
| 774 |
entry.max_tokens = max_tokens |
| 775 |
end |
| 776 |
|
| 777 |
-- Conv1d feature extraction settings affect output dimensions |
| 778 |
if p.output_mode == 'conv1d' then |
| 779 |
entry.output_mode = 'conv1d' |
| 780 |
entry.max_words = p.max_words or 32 |
| 781 |
entry.kernel_sizes = p.kernel_sizes or { 1, 3, 5 } |
| 782 |
entry.conv_pooling = p.conv_pooling or 'max' |
| 783 |
end |
| 784 |
|
| 785 |
norm.providers[i] = entry |
| 786 |
end |
| 787 |
return lua_util.unordered_table_digest(norm) |
| 788 |
end |
| 789 |
|
| 790 |
-- If no providers configured, fallback to symbols provider unless disabled |
| 791 |
-- phase: 'infer' | 'train' |
| 792 |
-- Removed synchronous collect_features; use collect_features_async instead |
| 793 |
|
| 794 |
-- Async version: runs providers in parallel and calls cb(fused, meta) when done |
| 795 |
local function collect_features_async(task, rule, profile_or_set, phase, cb) |
| 796 |
local providers_cfg = rule.providers |
| 797 |
if not providers_cfg or #providers_cfg == 0 then |
| 798 |
if rule.disable_symbols_input then |
| 799 |
cb(nil, { providers = {}, total_dim = 0, digest = providers_config_digest(providers_cfg, rule) }) |
| 800 |
return |
| 801 |
end |
| 802 |
local prov = get_provider('symbols') |
| 803 |
if prov and prov.collect_async then |
| 804 |
prov.collect_async(task, { profile = profile_or_set, weight = 1.0, phase = phase }, function(vec, meta) |
| 805 |
local metas = {} |
| 806 |
if vec then |
| 807 |
metas[1] = meta |
| 808 |
end |
| 809 |
local fused = {} |
| 810 |
if vec then |
| 811 |
local w = (meta and meta.weight) or 1.0 |
| 812 |
local norm_mode = (rule.fusion and rule.fusion.normalization) or 'none' |
| 813 |
if norm_mode ~= 'none' then |
| 814 |
vec = apply_normalization(vec, norm_mode) |
| 815 |
end |
| 816 |
for _, x in ipairs(vec) do |
| 817 |
fused[#fused + 1] = x * w |
| 818 |
end |
| 819 |
end |
| 820 |
cb(#fused > 0 and fused or nil, { |
| 821 |
providers = build_providers_meta(metas) or metas, |
| 822 |
total_dim = #fused, |
| 823 |
digest = providers_config_digest(providers_cfg, rule), |
| 824 |
}) |
| 825 |
end) |
| 826 |
return |
| 827 |
end |
| 828 |
-- Fallback: direct symbols compute |
| 829 |
local vec = result_to_vector(task, profile_or_set) |
| 830 |
local meta = { name = 'symbols', type = 'symbols', dim = #vec, weight = 1.0 } |
| 831 |
local fused = {} |
| 832 |
local w = 1.0 |
| 833 |
local norm_mode = (rule.fusion and rule.fusion.normalization) or 'none' |
| 834 |
if norm_mode ~= 'none' then |
| 835 |
vec = apply_normalization(vec, norm_mode) |
| 836 |
end |
| 837 |
for _, x in ipairs(vec) do |
| 838 |
fused[#fused + 1] = x * w |
| 839 |
end |
| 840 |
cb(fused, |
| 841 |
{ |
| 842 |
providers = build_providers_meta({ meta }) or { meta }, |
| 843 |
total_dim = #fused, |
| 844 |
digest = providers_config_digest( |
| 845 |
providers_cfg, rule) |
| 846 |
}) |
| 847 |
return |
| 848 |
end |
| 849 |
|
| 850 |
local vectors = {} |
| 851 |
local metas = {} |
| 852 |
local remaining = 0 |
| 853 |
|
| 854 |
local function maybe_finish() |
| 855 |
remaining = remaining - 1 |
| 856 |
if remaining == 0 then |
| 857 |
-- Fuse |
| 858 |
local fused = {} |
| 859 |
for i, v in ipairs(vectors) do |
| 860 |
if v then |
| 861 |
local w = (metas[i] and metas[i].weight) or 1.0 |
| 862 |
local norm_mode = (rule.fusion and rule.fusion.normalization) or 'none' |
| 863 |
if norm_mode ~= 'none' then |
| 864 |
v = apply_normalization(v, norm_mode) |
| 865 |
end |
| 866 |
for _, x in ipairs(v) do |
| 867 |
fused[#fused + 1] = x * w |
| 868 |
end |
| 869 |
end |
| 870 |
end |
| 871 |
local meta = { |
| 872 |
providers = build_providers_meta(metas) or metas, |
| 873 |
total_dim = #fused, |
| 874 |
digest = providers_config_digest(providers_cfg, rule), |
| 875 |
} |
| 876 |
if #fused == 0 then |
| 877 |
cb(nil, meta) |
| 878 |
else |
| 879 |
cb(fused, meta) |
| 880 |
end |
| 881 |
end |
| 882 |
end |
| 883 |
|
| 884 |
local function start_provider(i, pcfg) |
| 885 |
local prov = get_provider(pcfg.type or pcfg.name) |
| 886 |
if not prov or not prov.collect_async then |
| 887 |
maybe_finish() |
| 888 |
return |
| 889 |
end |
| 890 |
prov.collect_async(task, { |
| 891 |
profile = profile_or_set, |
| 892 |
set = profile_or_set, |
| 893 |
rule = rule, |
| 894 |
config = pcfg, |
| 895 |
weight = pcfg.weight or 1.0, |
| 896 |
phase = phase, |
| 897 |
}, function(vec, meta) |
| 898 |
if vec then |
| 899 |
metas[i] = meta or { name = pcfg.name or pcfg.type, type = pcfg.type, dim = #vec, weight = pcfg.weight or 1.0 } |
| 900 |
vectors[i] = vec |
| 901 |
end |
| 902 |
maybe_finish() |
| 903 |
end) |
| 904 |
end |
| 905 |
|
| 906 |
-- Include symbols provider (which includes both symbols AND metatokens) as an extra provider |
| 907 |
-- The name 'include_meta' is historical but it actually includes the full symbols provider |
| 908 |
-- For backward compatibility, include symbols by default unless explicitly disabled |
| 909 |
local include_meta = false |
| 910 |
if not providers_cfg or #providers_cfg == 0 then |
| 911 |
-- No providers, always use symbols (which includes metatokens) |
| 912 |
include_meta = true |
| 913 |
elseif rule.fusion then |
| 914 |
-- Explicit fusion config takes precedence |
| 915 |
include_meta = rule.fusion.include_meta |
| 916 |
if include_meta == nil then |
| 917 |
-- Default to true for backward compatibility when fusion is configured but include_meta not specified |
| 918 |
include_meta = true |
| 919 |
end |
| 920 |
else |
| 921 |
-- Providers configured but no fusion settings - default to including symbols+metatokens |
| 922 |
include_meta = true |
| 923 |
end |
| 924 |
|
| 925 |
local meta_weight = (rule.fusion and rule.fusion.meta_weight) or 1.0 |
| 926 |
|
| 927 |
remaining = #providers_cfg + (include_meta and 1 or 0) |
| 928 |
|
| 929 |
-- Start all configured providers |
| 930 |
for i, pcfg in ipairs(providers_cfg) do |
| 931 |
start_provider(i, pcfg) |
| 932 |
end |
| 933 |
|
| 934 |
if include_meta then |
| 935 |
-- Always use metatokens provider for consistency |
| 936 |
-- This ensures same dimensions whether called from controller or full scan |
| 937 |
local prov = get_provider('metatokens') |
| 938 |
|
| 939 |
if prov and prov.collect_async then |
| 940 |
local meta_index = #providers_cfg + 1 -- Metatokens always come after providers |
| 941 |
prov.collect_async(task, { profile = profile_or_set, set = profile_or_set, weight = meta_weight, phase = phase }, |
| 942 |
function(vec, meta) |
| 943 |
if vec then |
| 944 |
metas[meta_index] = meta |
| 945 |
vectors[meta_index] = vec |
| 946 |
end |
| 947 |
maybe_finish() |
| 948 |
end) |
| 949 |
else |
| 950 |
maybe_finish() |
| 951 |
end |
| 952 |
end |
| 953 |
end |
| 954 |
|
| 955 |
-- This function receives training vectors, checks them, spawn learning and saves ANN in Redis |
| 956 |
local function spawn_train(params) |
| 957 |
-- Prevent concurrent training (flag may be set by do_train_ann or needs to be set here for direct calls) |
| 958 |
if params.set.learning_spawned then |
| 959 |
lua_util.debugm(N, rspamd_config, 'spawn_train: training already in progress for %s:%s, skipping', |
| 960 |
params.rule.prefix, params.set.name) |
| 961 |
return |
| 962 |
end |
| 963 |
params.set.learning_spawned = true |
| 964 |
|
| 965 |
-- Check training data sanity |
| 966 |
-- Now we need to join inputs and create the appropriate test vectors |
| 967 |
local n |
| 968 |
|
| 969 |
-- When using providers, derive dimension from actual vectors |
| 970 |
if params.rule.providers and #params.rule.providers > 0 and |
| 971 |
(#params.spam_vec > 0 or #params.ham_vec > 0) then |
| 972 |
-- Use dimension from stored vectors |
| 973 |
if #params.spam_vec > 0 then |
| 974 |
n = #params.spam_vec[1] |
| 975 |
else |
| 976 |
n = #params.ham_vec[1] |
| 977 |
end |
| 978 |
lua_util.debugm(N, rspamd_config, 'spawn_train: using vector dimension %s from stored vectors', n) |
| 979 |
else |
| 980 |
-- Traditional symbol-based dimension |
| 981 |
n = #params.set.symbols + meta_functions.rspamd_count_metatokens() |
| 982 |
lua_util.debugm(N, rspamd_config, 'spawn_train: using symbol dimension %s symbols + %s metatokens = %s', |
| 983 |
#params.set.symbols, meta_functions.rspamd_count_metatokens(), n) |
| 984 |
end |
| 985 |
|
| 986 |
-- Now we can train ann - wrap in pcall to catch KANN errors |
| 987 |
local create_ok, train_ann = pcall(create_ann, params.rule.max_inputs or n, 3, params.rule) |
| 988 |
if not create_ok then |
| 989 |
rspamd_logger.errx(rspamd_config, 'failed to create ANN for %s:%s: %s', |
| 990 |
params.rule.prefix, params.set.name, train_ann) |
| 991 |
params.set.learning_spawned = false |
| 992 |
return |
| 993 |
end |
| 994 |
|
| 995 |
if #params.ham_vec + #params.spam_vec < params.rule.train.max_trains / 2 then |
| 996 |
-- Insufficient training data, reset flag and return |
| 997 |
rspamd_logger.errx(rspamd_config, 'insufficient training data for ANN %s:%s: spam=%s ham=%s (need at least %s total)', |
| 998 |
params.rule.prefix, params.set.name, |
| 999 |
#params.spam_vec, #params.ham_vec, params.rule.train.max_trains / 2) |
| 1000 |
params.set.learning_spawned = false |
| 1001 |
return |
| 1002 |
else |
| 1003 |
local inputs, outputs = {}, {} |
| 1004 |
|
| 1005 |
-- Used to show parsed vectors in a convenient format (for debugging only) |
| 1006 |
local function debug_vec(t) |
| 1007 |
local ret = {} |
| 1008 |
for i, v in ipairs(t) do |
| 1009 |
if v ~= 0 then |
| 1010 |
ret[#ret + 1] = string.format('%d=%.2f', i, v) |
| 1011 |
end |
| 1012 |
end |
| 1013 |
|
| 1014 |
return ret |
| 1015 |
end |
| 1016 |
|
| 1017 |
-- Make training set by joining vectors |
| 1018 |
-- KANN automatically shuffles those samples |
| 1019 |
-- 1.0 is used for spam and -1.0 is used for ham |
| 1020 |
-- It implies that output layer can express that (e.g. tanh output) |
| 1021 |
for _, e in ipairs(params.spam_vec) do |
| 1022 |
inputs[#inputs + 1] = e |
| 1023 |
outputs[#outputs + 1] = { 1.0 } |
| 1024 |
--rspamd_logger.debugm(N, rspamd_config, 'spam vector: %s', debug_vec(e)) |
| 1025 |
end |
| 1026 |
for _, e in ipairs(params.ham_vec) do |
| 1027 |
inputs[#inputs + 1] = e |
| 1028 |
outputs[#outputs + 1] = { -1.0 } |
| 1029 |
--rspamd_logger.debugm(N, rspamd_config, 'ham vector: %s', debug_vec(e)) |
| 1030 |
end |
| 1031 |
|
| 1032 |
-- Called in child process |
| 1033 |
local function train() |
| 1034 |
local log_thresh = params.rule.train.max_iterations / 10 |
| 1035 |
local seen_nan = false |
| 1036 |
|
| 1037 |
local function train_cb(iter, train_cost, value_cost) |
| 1038 |
if (iter * (params.rule.train.max_iterations / log_thresh)) % (params.rule.train.max_iterations) == 0 then |
| 1039 |
if train_cost ~= train_cost and not seen_nan then |
| 1040 |
-- We have nan :( try to log lot's of stuff to dig into a problem |
| 1041 |
seen_nan = true |
| 1042 |
rspamd_logger.errx(rspamd_config, 'ANN %s:%s: train error: observed nan in error cost!; value cost = %s', |
| 1043 |
params.rule.prefix, params.set.name, |
| 1044 |
value_cost) |
| 1045 |
for i, e in ipairs(inputs) do |
| 1046 |
lua_util.debugm(N, rspamd_config, 'train vector %s -> %s', |
| 1047 |
debug_vec(e), outputs[i][1]) |
| 1048 |
end |
| 1049 |
end |
| 1050 |
|
| 1051 |
rspamd_logger.infox(rspamd_config, |
| 1052 |
"ANN %s:%s: learned from %s redis key in %s iterations, error: %s, value cost: %s", |
| 1053 |
params.rule.prefix, params.set.name, |
| 1054 |
params.ann_key, |
| 1055 |
iter, |
| 1056 |
train_cost, |
| 1057 |
value_cost) |
| 1058 |
end |
| 1059 |
end |
| 1060 |
|
| 1061 |
lua_util.debugm(N, rspamd_config, "subprocess to learn ANN %s:%s has been started", |
| 1062 |
params.rule.prefix, params.set.name) |
| 1063 |
|
| 1064 |
local pca |
| 1065 |
if params.rule.max_inputs then |
| 1066 |
-- Train PCA in the main process, presumably it is not that long |
| 1067 |
lua_util.debugm(N, rspamd_config, "start PCA train for ANN %s:%s", |
| 1068 |
params.rule.prefix, params.set.name) |
| 1069 |
pca = learn_pca(inputs, params.rule.max_inputs) |
| 1070 |
end |
| 1071 |
|
| 1072 |
-- Compute normalization stats if requested |
| 1073 |
local norm_stats |
| 1074 |
if params.rule.fusion and params.rule.fusion.normalization == 'zscore' then |
| 1075 |
norm_stats = compute_zscore_stats(inputs) |
| 1076 |
elseif params.rule.fusion and params.rule.fusion.normalization == 'unit' then |
| 1077 |
norm_stats = { mode = 'unit' } |
| 1078 |
end |
| 1079 |
|
| 1080 |
if norm_stats then |
| 1081 |
for i = 1, #inputs do |
| 1082 |
inputs[i] = apply_normalization(inputs[i], norm_stats) |
| 1083 |
end |
| 1084 |
end |
| 1085 |
|
| 1086 |
lua_util.debugm(N, rspamd_config, "start neural train for ANN %s:%s", |
| 1087 |
params.rule.prefix, params.set.name) |
| 1088 |
local ret, err = pcall(train_ann.train1, train_ann, |
| 1089 |
inputs, outputs, { |
| 1090 |
lr = params.rule.train.learning_rate, |
| 1091 |
max_epoch = params.rule.train.max_iterations, |
| 1092 |
cb = train_cb, |
| 1093 |
pca = pca |
| 1094 |
}) |
| 1095 |
|
| 1096 |
if not ret then |
| 1097 |
rspamd_logger.errx(rspamd_config, "cannot train ann %s:%s: %s", |
| 1098 |
params.rule.prefix, params.set.name, err) |
| 1099 |
|
| 1100 |
return nil |
| 1101 |
else |
| 1102 |
lua_util.debugm(N, rspamd_config, "finished neural train for ANN %s:%s", |
| 1103 |
params.rule.prefix, params.set.name) |
| 1104 |
end |
| 1105 |
|
| 1106 |
local roc_thresholds = {} |
| 1107 |
if params.rule.roc_enabled then |
| 1108 |
local spam_threshold = get_roc_thresholds(train_ann, |
| 1109 |
inputs, |
| 1110 |
outputs, |
| 1111 |
1 - params.rule.roc_misclassification_cost, |
| 1112 |
params.rule.roc_misclassification_cost) |
| 1113 |
local ham_threshold = get_roc_thresholds(train_ann, |
| 1114 |
inputs, |
| 1115 |
outputs, |
| 1116 |
params.rule.roc_misclassification_cost, |
| 1117 |
1 - params.rule.roc_misclassification_cost) |
| 1118 |
roc_thresholds = { spam_threshold, ham_threshold } |
| 1119 |
|
| 1120 |
rspamd_logger.messagex(rspamd_config, "ROC thresholds: (spam_threshold: %s, ham_threshold: %s)", |
| 1121 |
roc_thresholds[1], roc_thresholds[2]) |
| 1122 |
end |
| 1123 |
|
| 1124 |
if not seen_nan then |
| 1125 |
-- Convert to strings as ucl cannot rspamd_text properly |
| 1126 |
local pca_data |
| 1127 |
if pca then |
| 1128 |
pca_data = tostring(pca:save()) |
| 1129 |
end |
| 1130 |
local out = { |
| 1131 |
ann_data = tostring(train_ann:save()), |
| 1132 |
pca_data = pca_data, |
| 1133 |
roc_thresholds = roc_thresholds, |
| 1134 |
norm_stats = norm_stats, |
| 1135 |
} |
| 1136 |
|
| 1137 |
local final_data = ucl.to_format(out, 'msgpack') |
| 1138 |
lua_util.debugm(N, rspamd_config, "subprocess for ANN %s:%s returned %s bytes", |
| 1139 |
params.rule.prefix, params.set.name, #final_data) |
| 1140 |
return final_data |
| 1141 |
else |
| 1142 |
return nil |
| 1143 |
end |
| 1144 |
end |
| 1145 |
|
| 1146 |
local function redis_save_cb(err) |
| 1147 |
if err then |
| 1148 |
rspamd_logger.errx(rspamd_config, 'cannot save ANN %s:%s to redis key %s: %s', |
| 1149 |
params.rule.prefix, params.set.name, params.ann_key, err) |
| 1150 |
lua_redis.redis_make_request_taskless(params.ev_base, |
| 1151 |
rspamd_config, |
| 1152 |
params.rule.redis, |
| 1153 |
nil, |
| 1154 |
false, -- is write |
| 1155 |
gen_unlock_cb(params.rule, params.set, params.ann_key), --callback |
| 1156 |
'HDEL', -- command |
| 1157 |
{ params.ann_key, 'lock' } |
| 1158 |
) |
| 1159 |
else |
| 1160 |
rspamd_logger.infox(rspamd_config, 'saved ANN %s:%s to redis: %s', |
| 1161 |
params.rule.prefix, params.set.name, params.set.ann.redis_key) |
| 1162 |
|
| 1163 |
-- Clean up pending training keys if they were used |
| 1164 |
if params.pending_key then |
| 1165 |
local function cleanup_cb(cleanup_err) |
| 1166 |
if cleanup_err then |
| 1167 |
lua_util.debugm(N, rspamd_config, 'failed to cleanup pending keys: %s', cleanup_err) |
| 1168 |
else |
| 1169 |
lua_util.debugm(N, rspamd_config, 'cleaned up pending training keys for %s', |
| 1170 |
params.pending_key) |
| 1171 |
end |
| 1172 |
end |
| 1173 |
-- Delete both spam and ham pending sets |
| 1174 |
lua_redis.redis_make_request_taskless(params.ev_base, |
| 1175 |
rspamd_config, |
| 1176 |
params.rule.redis, |
| 1177 |
nil, |
| 1178 |
true, -- is write |
| 1179 |
cleanup_cb, |
| 1180 |
'DEL', |
| 1181 |
{ params.pending_key .. '_spam_set', params.pending_key .. '_ham_set' } |
| 1182 |
) |
| 1183 |
end |
| 1184 |
end |
| 1185 |
end |
| 1186 |
|
| 1187 |
local function ann_trained(err, data) |
| 1188 |
params.set.learning_spawned = false |
| 1189 |
if err then |
| 1190 |
rspamd_logger.errx(rspamd_config, 'cannot train ANN %s:%s : %s', |
| 1191 |
params.rule.prefix, params.set.name, err) |
| 1192 |
lua_redis.redis_make_request_taskless(params.ev_base, |
| 1193 |
rspamd_config, |
| 1194 |
params.rule.redis, |
| 1195 |
nil, |
| 1196 |
true, -- is write |
| 1197 |
gen_unlock_cb(params.rule, params.set, params.ann_key), --callback |
| 1198 |
'HDEL', -- command |
| 1199 |
{ params.ann_key, 'lock' } |
| 1200 |
) |
| 1201 |
else |
| 1202 |
local parser = ucl.parser() |
| 1203 |
local ok, parse_err = parser:parse_text(data, 'msgpack') |
| 1204 |
if not ok then |
| 1205 |
rspamd_logger.errx(rspamd_config, 'cannot parse training result for ANN %s:%s: %s (data size: %s)', |
| 1206 |
params.rule.prefix, params.set.name, parse_err, #data) |
| 1207 |
lua_redis.redis_make_request_taskless(params.ev_base, |
| 1208 |
rspamd_config, |
| 1209 |
params.rule.redis, |
| 1210 |
nil, |
| 1211 |
true, |
| 1212 |
gen_unlock_cb(params.rule, params.set, params.ann_key), |
| 1213 |
'HDEL', |
| 1214 |
{ params.ann_key, 'lock' } |
| 1215 |
) |
| 1216 |
return |
| 1217 |
end |
| 1218 |
local parsed = parser:get_object() |
| 1219 |
local ann_data = rspamd_util.zstd_compress(parsed.ann_data) |
| 1220 |
local pca_data = parsed.pca_data |
| 1221 |
local roc_thresholds = parsed.roc_thresholds |
| 1222 |
local norm_stats = parsed.norm_stats |
| 1223 |
|
| 1224 |
fill_set_ann(params.set, params.ann_key) |
| 1225 |
if pca_data then |
| 1226 |
params.set.ann.pca = rspamd_tensor.load(pca_data) |
| 1227 |
pca_data = rspamd_util.zstd_compress(pca_data) |
| 1228 |
end |
| 1229 |
|
| 1230 |
if roc_thresholds then |
| 1231 |
params.set.ann.roc_thresholds = roc_thresholds |
| 1232 |
end |
| 1233 |
|
| 1234 |
|
| 1235 |
-- Deserialise ANN from the child process |
| 1236 |
local loaded_ann = rspamd_kann.load(parsed.ann_data) |
| 1237 |
local version = (params.set.ann.version or 0) + 1 |
| 1238 |
params.set.ann.version = version |
| 1239 |
params.set.ann.ann = loaded_ann |
| 1240 |
params.set.ann.symbols = params.set.symbols |
| 1241 |
params.set.ann.redis_key = new_ann_key(params.rule, params.set, version) |
| 1242 |
|
| 1243 |
local profile = { |
| 1244 |
symbols = params.set.symbols, |
| 1245 |
digest = params.set.digest, |
| 1246 |
redis_key = params.set.ann.redis_key, |
| 1247 |
version = version, |
| 1248 |
providers_digest = providers_config_digest(params.rule.providers, params.rule), |
| 1249 |
} |
| 1250 |
|
| 1251 |
local profile_serialized = ucl.to_format(profile, 'json-compact', true) |
| 1252 |
local roc_thresholds_serialized = ucl.to_format(roc_thresholds, 'json-compact', true) |
| 1253 |
local providers_meta_serialized |
| 1254 |
if params.rule.providers then |
| 1255 |
providers_meta_serialized = ucl.to_format( |
| 1256 |
build_providers_meta(params.set.ann.providers or params.rule.providers), 'json-compact', true) |
| 1257 |
end |
| 1258 |
|
| 1259 |
rspamd_logger.infox(rspamd_config, |
| 1260 |
'trained ANN %s:%s, %s bytes (%s compressed); %s rows in pca (%sb compressed); redis key: %s (old key %s)', |
| 1261 |
params.rule.prefix, params.set.name, |
| 1262 |
#data, #ann_data, |
| 1263 |
#(params.set.ann.pca or {}), #(pca_data or {}), |
| 1264 |
params.set.ann.redis_key, params.ann_key) |
| 1265 |
|
| 1266 |
-- Ensure all arguments are non-nil for Lua 5.4 compatibility |
| 1267 |
-- (nil values in tables cause length/iteration issues) |
| 1268 |
lua_redis.exec_redis_script(redis_script_id.save_unlock, |
| 1269 |
{ ev_base = params.ev_base, is_write = true }, |
| 1270 |
redis_save_cb, |
| 1271 |
{ profile.redis_key, |
| 1272 |
redis_ann_prefix(params.rule, params.set.name), |
| 1273 |
params.ann_key, -- old key to unlock... |
| 1274 |
}, |
| 1275 |
{ ann_data, |
| 1276 |
profile_serialized, |
| 1277 |
tostring(params.rule.ann_expire), |
| 1278 |
tostring(os.time()), |
| 1279 |
roc_thresholds_serialized or '', |
| 1280 |
pca_data or '', |
| 1281 |
providers_meta_serialized or '', |
| 1282 |
ucl.to_format(norm_stats, 'json-compact', true) or '', |
| 1283 |
}) |
| 1284 |
end |
| 1285 |
-- Force GC to clean up training temporaries (parsed data, compressed buffers, etc.) |
| 1286 |
-- to prevent LuaJIT GC atomic phase stalls on a bloated heap |
| 1287 |
collectgarbage('collect') |
| 1288 |
end |
| 1289 |
|
| 1290 |
if params.rule.max_inputs then |
| 1291 |
fill_set_ann(params.set, params.ann_key) |
| 1292 |
end |
| 1293 |
|
| 1294 |
params.worker:spawn_process { |
| 1295 |
func = train, |
| 1296 |
on_complete = ann_trained, |
| 1297 |
proctitle = string.format("ANN train for %s/%s", params.rule.prefix, params.set.name), |
| 1298 |
} |
| 1299 |
-- Register lock extension (learning_spawned already set at start of spawn_train) |
| 1300 |
register_lock_extender(params.rule, params.set, params.ev_base, params.ann_key) |
| 1301 |
return |
| 1302 |
end |
| 1303 |
end |
| 1304 |
|
| 1305 |
-- This function is used to adjust profiles and allowed setting ids for each rule |
| 1306 |
-- It must be called when all settings are already registered (e.g. at post-init for config) |
| 1307 |
local function process_rules_settings() |
| 1308 |
local function process_settings_elt(rule, selt) |
| 1309 |
local profile = rule.profile[selt.name] |
| 1310 |
if profile then |
| 1311 |
-- Use static user defined profile |
| 1312 |
-- Ensure that we have an array... |
| 1313 |
lua_util.debugm(N, rspamd_config, "use static profile for %s (%s): %s", |
| 1314 |
rule.prefix, selt.name, profile) |
| 1315 |
if not profile[1] then |
| 1316 |
profile = lua_util.keys(profile) |
| 1317 |
end |
| 1318 |
selt.symbols = profile |
| 1319 |
else |
| 1320 |
lua_util.debugm(N, rspamd_config, "use dynamic cfg based profile for %s (%s)", |
| 1321 |
rule.prefix, selt.name) |
| 1322 |
end |
| 1323 |
|
| 1324 |
local function filter_symbols_predicate(sname) |
| 1325 |
if settings.blacklisted_symbols and settings.blacklisted_symbols[sname] then |
| 1326 |
return false |
| 1327 |
end |
| 1328 |
local fl = rspamd_config:get_symbol_flags(sname) |
| 1329 |
if fl then |
| 1330 |
fl = lua_util.list_to_hash(fl) |
| 1331 |
|
| 1332 |
return not (fl.nostat or fl.idempotent or fl.skip or fl.composite) |
| 1333 |
end |
| 1334 |
|
| 1335 |
return true |
| 1336 |
end |
| 1337 |
|
| 1338 |
-- Generic stuff |
| 1339 |
if not profile then |
| 1340 |
-- Do filtering merely if we are using a dynamic profile |
| 1341 |
selt.symbols = fun.totable(fun.filter(filter_symbols_predicate, selt.symbols)) |
| 1342 |
end |
| 1343 |
|
| 1344 |
table.sort(selt.symbols) |
| 1345 |
|
| 1346 |
selt.digest = lua_util.table_digest(selt.symbols) |
| 1347 |
selt.prefix = redis_ann_prefix(rule, selt.name) |
| 1348 |
|
| 1349 |
rspamd_logger.messagex(rspamd_config, |
| 1350 |
'use NN prefix for rule %s; settings id "%s"; symbols digest: "%s"', |
| 1351 |
selt.prefix, selt.name, selt.digest) |
| 1352 |
|
| 1353 |
lua_redis.register_prefix(selt.prefix, N, |
| 1354 |
string.format('NN prefix for rule "%s"; settings id "%s"', |
| 1355 |
selt.prefix, selt.name), { |
| 1356 |
persistent = true, |
| 1357 |
type = 'zlist', |
| 1358 |
}) |
| 1359 |
-- Versions |
| 1360 |
lua_redis.register_prefix(selt.prefix .. '_\\d+', N, |
| 1361 |
string.format('NN storage for rule "%s"; settings id "%s"', |
| 1362 |
selt.prefix, selt.name), { |
| 1363 |
persistent = true, |
| 1364 |
type = 'hash', |
| 1365 |
}) |
| 1366 |
lua_redis.register_prefix(selt.prefix .. '_\\d+_spam_set', N, |
| 1367 |
string.format('NN learning set (spam) for rule "%s"; settings id "%s"', |
| 1368 |
selt.prefix, selt.name), { |
| 1369 |
persistent = true, |
| 1370 |
type = 'set', |
| 1371 |
}) |
| 1372 |
lua_redis.register_prefix(selt.prefix .. '_\\d+_ham_set', N, |
| 1373 |
string.format('NN learning set (ham) for rule "%s"; settings id "%s"', |
| 1374 |
rule.prefix, selt.name), { |
| 1375 |
persistent = true, |
| 1376 |
type = 'set', |
| 1377 |
}) |
| 1378 |
end |
| 1379 |
|
| 1380 |
for k, rule in pairs(settings.rules) do |
| 1381 |
if not rule.allowed_settings then |
| 1382 |
rule.allowed_settings = {} |
| 1383 |
elseif rule.allowed_settings == 'all' then |
| 1384 |
-- Extract all settings ids |
| 1385 |
rule.allowed_settings = lua_util.keys(lua_settings.all_settings()) |
| 1386 |
end |
| 1387 |
|
| 1388 |
-- Convert to a map <setting_id> -> true |
| 1389 |
rule.allowed_settings = lua_util.list_to_hash(rule.allowed_settings) |
| 1390 |
|
| 1391 |
-- Check if we can work without settings |
| 1392 |
if k == 'default' or type(rule.default) ~= 'boolean' then |
| 1393 |
rule.default = true |
| 1394 |
end |
| 1395 |
|
| 1396 |
rule.settings = {} |
| 1397 |
|
| 1398 |
if rule.default then |
| 1399 |
local default_settings = { |
| 1400 |
symbols = lua_settings.default_symbols(), |
| 1401 |
name = 'default' |
| 1402 |
} |
| 1403 |
|
| 1404 |
process_settings_elt(rule, default_settings) |
| 1405 |
rule.settings[-1] = default_settings -- Magic constant, but OK as settings are positive int32 |
| 1406 |
end |
| 1407 |
|
| 1408 |
-- Now, for each allowed settings, we store sorted symbols + digest |
| 1409 |
-- We set table rule.settings[id] -> { name = name, symbols = symbols, digest = digest } |
| 1410 |
for s, _ in pairs(rule.allowed_settings) do |
| 1411 |
-- Here, we have a name, set of symbols and |
| 1412 |
local settings_id = s |
| 1413 |
if type(settings_id) ~= 'number' then |
| 1414 |
settings_id = lua_settings.numeric_settings_id(s) |
| 1415 |
end |
| 1416 |
local selt = lua_settings.settings_by_id(settings_id) |
| 1417 |
|
| 1418 |
local nelt = { |
| 1419 |
symbols = selt.symbols, -- Already sorted |
| 1420 |
name = selt.name |
| 1421 |
} |
| 1422 |
|
| 1423 |
process_settings_elt(rule, nelt) |
| 1424 |
for id, ex in pairs(rule.settings) do |
| 1425 |
if type(ex) == 'table' then |
| 1426 |
if nelt and lua_util.distance_sorted(ex.symbols, nelt.symbols) == 0 then |
| 1427 |
-- Equal symbols, add reference |
| 1428 |
lua_util.debugm(N, rspamd_config, |
| 1429 |
'added reference from settings id %s to %s; same symbols', |
| 1430 |
nelt.name, ex.name) |
| 1431 |
rule.settings[settings_id] = id |
| 1432 |
nelt = nil |
| 1433 |
end |
| 1434 |
end |
| 1435 |
end |
| 1436 |
|
| 1437 |
if nelt then |
| 1438 |
rule.settings[settings_id] = nelt |
| 1439 |
lua_util.debugm(N, rspamd_config, 'added new settings id %s(%s) to %s', |
| 1440 |
nelt.name, settings_id, rule.prefix) |
| 1441 |
end |
| 1442 |
end |
| 1443 |
end |
| 1444 |
end |
| 1445 |
|
| 1446 |
-- Extract settings element for a specific settings id |
| 1447 |
local function get_rule_settings(task, rule) |
| 1448 |
local sid = task:get_settings_id() or -1 |
| 1449 |
local set = rule.settings[sid] |
| 1450 |
|
| 1451 |
if not set then |
| 1452 |
return nil |
| 1453 |
end |
| 1454 |
|
| 1455 |
while type(set) == 'number' do |
| 1456 |
-- Reference to another settings! |
| 1457 |
set = rule.settings[set] |
| 1458 |
end |
| 1459 |
|
| 1460 |
return set |
| 1461 |
end |
| 1462 |
|
| 1463 |
result_to_vector = function(task, profile) |
| 1464 |
if not profile.zeros then |
| 1465 |
-- Fill zeros vector |
| 1466 |
local zeros = {} |
| 1467 |
for i = 1, meta_functions.rspamd_count_metatokens() do |
| 1468 |
zeros[i] = 0.0 |
| 1469 |
end |
| 1470 |
for _, _ in ipairs(profile.symbols) do |
| 1471 |
zeros[#zeros + 1] = 0.0 |
| 1472 |
end |
| 1473 |
profile.zeros = zeros |
| 1474 |
end |
| 1475 |
|
| 1476 |
local vec = lua_util.shallowcopy(profile.zeros) |
| 1477 |
local mt = meta_functions.rspamd_gen_metatokens(task) |
| 1478 |
|
| 1479 |
for i, v in ipairs(mt) do |
| 1480 |
vec[i] = v |
| 1481 |
end |
| 1482 |
|
| 1483 |
task:process_ann_tokens(profile.symbols, vec, #mt, 0.1) |
| 1484 |
|
| 1485 |
return vec |
| 1486 |
end |
| 1487 |
|
| 1488 |
return { |
| 1489 |
can_push_train_vector = can_push_train_vector, |
| 1490 |
collect_features_async = collect_features_async, |
| 1491 |
create_ann = create_ann, |
| 1492 |
default_options = default_options, |
| 1493 |
build_providers_meta = build_providers_meta, |
| 1494 |
apply_normalization = apply_normalization, |
| 1495 |
gen_unlock_cb = gen_unlock_cb, |
| 1496 |
get_provider = get_provider, |
| 1497 |
get_rule_settings = get_rule_settings, |
| 1498 |
load_scripts = load_scripts, |
| 1499 |
module_config = module_config, |
| 1500 |
new_ann_key = new_ann_key, |
| 1501 |
pending_train_key = pending_train_key, |
| 1502 |
providers_config_digest = providers_config_digest, |
| 1503 |
register_provider = register_provider, |
| 1504 |
plugin_ver = plugin_ver, |
| 1505 |
process_rules_settings = process_rules_settings, |
| 1506 |
redis_ann_prefix = redis_ann_prefix, |
| 1507 |
redis_params = redis_params, |
| 1508 |
redis_script_id = redis_script_id, |
| 1509 |
result_to_vector = result_to_vector, |
| 1510 |
settings = settings, |
| 1511 |
spawn_train = spawn_train, |
| 1512 |
-- External model support |
| 1513 |
neural_external = neural_external, |
| 1514 |
} |