| 1 |
--[[ |
| 2 |
Copyright (c) 2026, Vsevolod Stakhov <[email protected]> |
| 3 |
|
| 4 |
Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 |
you may not use this file except in compliance with the License. |
| 6 |
You may obtain a copy of the License at |
| 7 |
|
| 8 |
http://www.apache.org/licenses/LICENSE-2.0 |
| 9 |
|
| 10 |
Unless required by applicable law or agreed to in writing, software |
| 11 |
distributed under the License is distributed on an "AS IS" BASIS, |
| 12 |
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 |
See the License for the specific language governing permissions and |
| 14 |
limitations under the License. |
| 15 |
]]-- |
| 16 |
|
| 17 |
--[[ |
| 18 |
External neural model loading and merging. |
| 19 |
|
| 20 |
This module provides functionality to load pretrained neural models |
| 21 |
from external sources (HTTP/HTTPS) via the Maps infrastructure |
| 22 |
and merge them with locally trained weights. |
| 23 |
|
| 24 |
Model format (msgpack): |
| 25 |
{ |
| 26 |
magic = "RNM1", -- Rspamd Neural Model v1 |
| 27 |
version = 1, -- format version |
| 28 |
model_version = 123, -- model training version (incremented on retrain) |
| 29 |
providers_digest = "...", -- digest of providers config (must match local) |
| 30 |
ann_data = "...", -- serialized KANN (zstd compressed) |
| 31 |
pca_data = "...", -- optional PCA (zstd compressed) |
| 32 |
norm_stats = {...}, -- normalization stats |
| 33 |
roc_thresholds = {...}, -- ROC thresholds |
| 34 |
created_at = timestamp, |
| 35 |
} |
| 36 |
|
| 37 |
Usage in neural config: |
| 38 |
external_model = { |
| 39 |
url = "https://your-provider.com/models/<digest>"; |
| 40 |
sign_key = "your_key"; -- optional signature verification |
| 41 |
merge_alpha = 0.6; -- 60% external, 40% local |
| 42 |
}; |
| 43 |
]]-- |
| 44 |
|
| 45 |
local lua_redis = require "lua_redis" |
| 46 |
local rspamd_kann = require "rspamd_kann" |
| 47 |
local rspamd_logger = require "rspamd_logger" |
| 48 |
local rspamd_util = require "rspamd_util" |
| 49 |
local rspamd_text = require "rspamd_text" |
| 50 |
local ucl = require "ucl" |
| 51 |
|
| 52 |
-- Model format constants |
| 53 |
local MODEL_MAGIC = "RNM1" |
| 54 |
local MODEL_FORMAT_VERSION = 1 |
| 55 |
|
| 56 |
local exports = {} |
| 57 |
|
| 58 |
-- Cache of loaded external models: url -> { model, map, callbacks } |
| 59 |
local external_model_cache = {} |
| 60 |
|
| 61 |
--- Parse external model from msgpack data |
| 62 |
-- @param data raw msgpack data (rspamd_text, possibly zstd compressed) |
| 63 |
-- @return table with model data or nil, error message |
| 64 |
function exports.parse_model(data) |
| 65 |
if not data then |
| 66 |
return nil, "no data" |
| 67 |
end |
| 68 |
|
| 69 |
-- Convert rspamd_text to string if needed |
| 70 |
local data_str |
| 71 |
if type(data) == 'userdata' or (type(data) == 'table' and data.cookie) then |
| 72 |
data_str = tostring(data) |
| 73 |
else |
| 74 |
data_str = data |
| 75 |
end |
| 76 |
|
| 77 |
-- Try zstd decompression first |
| 78 |
local decompressed |
| 79 |
local err, decompressed_data = rspamd_util.zstd_decompress(data_str) |
| 80 |
if not err and decompressed_data then |
| 81 |
decompressed = tostring(decompressed_data) |
| 82 |
else |
| 83 |
-- Assume uncompressed |
| 84 |
decompressed = data_str |
| 85 |
end |
| 86 |
|
| 87 |
-- Parse msgpack |
| 88 |
local parser = ucl.parser() |
| 89 |
local ok, parse_err = parser:parse_text(decompressed, 'msgpack') |
| 90 |
if not ok then |
| 91 |
return nil, "failed to parse msgpack: " .. (parse_err or "unknown error") |
| 92 |
end |
| 93 |
|
| 94 |
local model = parser:get_object() |
| 95 |
|
| 96 |
-- Validate model format |
| 97 |
if model.magic ~= MODEL_MAGIC then |
| 98 |
return nil, string.format("invalid magic: expected %s, got %s", |
| 99 |
MODEL_MAGIC, model.magic or "nil") |
| 100 |
end |
| 101 |
|
| 102 |
if model.version ~= MODEL_FORMAT_VERSION then |
| 103 |
return nil, string.format("unsupported model version: %s (expected %s)", |
| 104 |
model.version or "nil", MODEL_FORMAT_VERSION) |
| 105 |
end |
| 106 |
|
| 107 |
return model |
| 108 |
end |
| 109 |
|
| 110 |
--- Load KANN from model data |
| 111 |
-- @param model parsed model table |
| 112 |
-- @return kann_t object or nil, error |
| 113 |
function exports.load_ann(model) |
| 114 |
if not model.ann_data then |
| 115 |
return nil, "no ann_data in model" |
| 116 |
end |
| 117 |
|
| 118 |
-- Decompress ann_data |
| 119 |
local ann_data_str = model.ann_data |
| 120 |
if type(ann_data_str) == 'userdata' or (type(ann_data_str) == 'table' and ann_data_str.cookie) then |
| 121 |
ann_data_str = tostring(ann_data_str) |
| 122 |
end |
| 123 |
|
| 124 |
local err, ann_data = rspamd_util.zstd_decompress(ann_data_str) |
| 125 |
if err then |
| 126 |
return nil, "failed to decompress ann_data: " .. err |
| 127 |
end |
| 128 |
|
| 129 |
local ann = rspamd_kann.load(ann_data) |
| 130 |
if not ann then |
| 131 |
return nil, "failed to load KANN from model" |
| 132 |
end |
| 133 |
|
| 134 |
return ann |
| 135 |
end |
| 136 |
|
| 137 |
--- Load PCA from model data |
| 138 |
-- @param model parsed model table |
| 139 |
-- @return tensor or nil |
| 140 |
function exports.load_pca(model) |
| 141 |
if not model.pca_data then |
| 142 |
return nil |
| 143 |
end |
| 144 |
|
| 145 |
local pca_data_str = model.pca_data |
| 146 |
if type(pca_data_str) == 'userdata' or (type(pca_data_str) == 'table' and pca_data_str.cookie) then |
| 147 |
pca_data_str = tostring(pca_data_str) |
| 148 |
end |
| 149 |
|
| 150 |
local err, pca_data = rspamd_util.zstd_decompress(pca_data_str) |
| 151 |
if err then |
| 152 |
rspamd_logger.warnx(rspamd_config, "failed to decompress pca_data: %s", err) |
| 153 |
return nil |
| 154 |
end |
| 155 |
|
| 156 |
local rspamd_tensor = require "rspamd_tensor" |
| 157 |
return rspamd_tensor.load(pca_data) |
| 158 |
end |
| 159 |
|
| 160 |
--- Check if model is compatible with local config |
| 161 |
-- @param model parsed model table |
| 162 |
-- @param providers_digest local providers digest |
| 163 |
-- @return boolean, reason |
| 164 |
function exports.is_compatible(model, providers_digest) |
| 165 |
if not model.providers_digest then |
| 166 |
return false, "model has no providers_digest" |
| 167 |
end |
| 168 |
|
| 169 |
if model.providers_digest ~= providers_digest then |
| 170 |
return false, string.format("providers digest mismatch: model=%s, local=%s", |
| 171 |
model.providers_digest:sub(1, 8), providers_digest:sub(1, 8)) |
| 172 |
end |
| 173 |
|
| 174 |
return true, "compatible" |
| 175 |
end |
| 176 |
|
| 177 |
--- Merge weights from external ANN into local ANN using interpolation |
| 178 |
-- w_new = alpha * w_external + (1-alpha) * w_local |
| 179 |
-- @param external_ann kann_t from external model |
| 180 |
-- @param local_ann kann_t from local training |
| 181 |
-- @param alpha weight for external (0.0 - 1.0) |
| 182 |
-- @return merged kann_t or nil, error |
| 183 |
function exports.merge_weights(external_ann, local_ann, alpha) |
| 184 |
if not external_ann or not local_ann then |
| 185 |
return nil, "missing ann" |
| 186 |
end |
| 187 |
|
| 188 |
alpha = alpha or 0.5 |
| 189 |
|
| 190 |
-- Check compatibility first |
| 191 |
local ok = external_ann:is_compatible(local_ann) |
| 192 |
if not ok then |
| 193 |
return nil, "incompatible ANN architectures" |
| 194 |
end |
| 195 |
|
| 196 |
-- Use the external ANN as base and merge local weights into it |
| 197 |
local merged, err = external_ann:merge_weights(local_ann, 1.0 - alpha) |
| 198 |
|
| 199 |
if not merged then |
| 200 |
return nil, "merge failed: " .. (err or "unknown") |
| 201 |
end |
| 202 |
|
| 203 |
return external_ann |
| 204 |
end |
| 205 |
|
| 206 |
--- Build URL for external model based on providers digest |
| 207 |
-- @param base_url base URL |
| 208 |
-- @param providers_digest digest of providers config |
| 209 |
-- @return full URL |
| 210 |
function exports.build_model_url(base_url, providers_digest) |
| 211 |
-- Remove trailing slash from base_url |
| 212 |
base_url = base_url:gsub('/+$', '') |
| 213 |
return string.format("%s/%s", base_url, providers_digest) |
| 214 |
end |
| 215 |
|
| 216 |
--- Register external model as a map |
| 217 |
-- This uses the Maps infrastructure for HTTP loading with signature verification |
| 218 |
-- @param cfg rspamd_config |
| 219 |
-- @param rule neural rule configuration |
| 220 |
-- @param providers_digest digest of providers config |
| 221 |
-- @param on_load_callback function(model_data, err) called when model is loaded/reloaded |
| 222 |
-- @return boolean success |
| 223 |
function exports.register_model_map(cfg, rule, providers_digest, on_load_callback) |
| 224 |
local ext_cfg = rule.external_model |
| 225 |
if not ext_cfg or not ext_cfg.url then |
| 226 |
return false |
| 227 |
end |
| 228 |
|
| 229 |
local url = ext_cfg.url |
| 230 |
local cache_key = url |
| 231 |
|
| 232 |
-- Check if already registered |
| 233 |
if external_model_cache[cache_key] then |
| 234 |
return true |
| 235 |
end |
| 236 |
|
| 237 |
-- Map callback: called when map data is loaded |
| 238 |
local function map_callback(data, map) |
| 239 |
if not data then |
| 240 |
rspamd_logger.errx(cfg, 'external neural model map returned no data for %s', url) |
| 241 |
if on_load_callback then |
| 242 |
on_load_callback(nil, "no data from map") |
| 243 |
end |
| 244 |
return |
| 245 |
end |
| 246 |
|
| 247 |
-- Parse model |
| 248 |
local model, parse_err = exports.parse_model(data) |
| 249 |
if not model then |
| 250 |
rspamd_logger.errx(cfg, 'failed to parse external neural model from %s: %s', url, parse_err) |
| 251 |
if on_load_callback then |
| 252 |
on_load_callback(nil, parse_err) |
| 253 |
end |
| 254 |
return |
| 255 |
end |
| 256 |
|
| 257 |
-- Check compatibility |
| 258 |
local compatible, reason = exports.is_compatible(model, providers_digest) |
| 259 |
if not compatible then |
| 260 |
rspamd_logger.errx(cfg, 'external neural model incompatible: %s', reason) |
| 261 |
if on_load_callback then |
| 262 |
on_load_callback(nil, reason) |
| 263 |
end |
| 264 |
return |
| 265 |
end |
| 266 |
|
| 267 |
rspamd_logger.infox(cfg, 'loaded external neural model from %s (version=%s)', |
| 268 |
url, model.model_version or 0) |
| 269 |
|
| 270 |
-- Update cache |
| 271 |
external_model_cache[cache_key] = { |
| 272 |
model = model, |
| 273 |
last_version = model.model_version, |
| 274 |
last_load = os.time(), |
| 275 |
} |
| 276 |
|
| 277 |
-- Call user callback |
| 278 |
if on_load_callback then |
| 279 |
on_load_callback(model, nil) |
| 280 |
end |
| 281 |
end |
| 282 |
|
| 283 |
-- Create callback map |
| 284 |
local map = cfg:add_map({ |
| 285 |
url = url, |
| 286 |
type = 'callback', |
| 287 |
description = string.format('External neural model for rule %s', rule.prefix or 'default'), |
| 288 |
callback = map_callback, |
| 289 |
opaque_data = true, -- Get data as rspamd_text |
| 290 |
}) |
| 291 |
|
| 292 |
if not map then |
| 293 |
rspamd_logger.errx(cfg, 'failed to register external neural model map for %s', url) |
| 294 |
return false |
| 295 |
end |
| 296 |
|
| 297 |
-- Set sign key if configured |
| 298 |
if ext_cfg.sign_key then |
| 299 |
map:set_sign_key(ext_cfg.sign_key) |
| 300 |
end |
| 301 |
|
| 302 |
external_model_cache[cache_key] = { |
| 303 |
map = map, |
| 304 |
callbacks = { on_load_callback }, |
| 305 |
} |
| 306 |
|
| 307 |
return true |
| 308 |
end |
| 309 |
|
| 310 |
--- Get cached model data for URL |
| 311 |
-- @param url model URL |
| 312 |
-- @return cached model data or nil |
| 313 |
function exports.get_cached_model(url) |
| 314 |
local cached = external_model_cache[url] |
| 315 |
if cached and cached.model then |
| 316 |
return cached.model |
| 317 |
end |
| 318 |
return nil |
| 319 |
end |
| 320 |
|
| 321 |
--- Create external model configuration for a neural rule |
| 322 |
-- @param rule neural rule configuration |
| 323 |
-- @param providers_digest digest of providers config |
| 324 |
-- @return table with external model config or nil |
| 325 |
function exports.create_external_config(rule, providers_digest) |
| 326 |
local ext = rule.external_model |
| 327 |
if not ext then |
| 328 |
return nil |
| 329 |
end |
| 330 |
|
| 331 |
local url = ext.url |
| 332 |
if not url then |
| 333 |
-- Build URL from digest if base_url is provided |
| 334 |
if ext.base_url then |
| 335 |
url = exports.build_model_url(ext.base_url, providers_digest) |
| 336 |
else |
| 337 |
rspamd_logger.errx(rspamd_config, 'external_model requires url or base_url') |
| 338 |
return nil |
| 339 |
end |
| 340 |
end |
| 341 |
|
| 342 |
return { |
| 343 |
url = url, |
| 344 |
sign_key = ext.sign_key, |
| 345 |
merge_strategy = ext.merge_strategy or "interpolate", |
| 346 |
merge_alpha = ext.merge_alpha or 0.5, |
| 347 |
check_interval = ext.check_interval or 86400, -- 24h |
| 348 |
local_fine_tune = ext.local_fine_tune ~= false, |
| 349 |
min_local_samples = ext.min_local_samples or 50, |
| 350 |
providers_digest = providers_digest, |
| 351 |
loaded = false, |
| 352 |
last_version = nil, |
| 353 |
last_check = nil, |
| 354 |
} |
| 355 |
end |
| 356 |
|
| 357 |
--- Store external model metadata in Redis for later merge |
| 358 |
-- @param redis redis params |
| 359 |
-- @param ev_base event base |
| 360 |
-- @param ann_key Redis key for the ANN |
| 361 |
-- @param model_data parsed model data |
| 362 |
-- @param callback function(err) |
| 363 |
function exports.store_base_model(redis, ev_base, ann_key, model_data, callback) |
| 364 |
-- Store base model version and compressed ann_data for re-merge |
| 365 |
local base_key = ann_key .. "_base" |
| 366 |
|
| 367 |
local function store_cb(err) |
| 368 |
if err then |
| 369 |
rspamd_logger.errx(rspamd_config, "failed to store base model: %s", err) |
| 370 |
end |
| 371 |
if callback then |
| 372 |
callback(err) |
| 373 |
end |
| 374 |
end |
| 375 |
|
| 376 |
-- Ensure ann_data is rspamd_text for opaque storage |
| 377 |
local ann_data = model_data.ann_data |
| 378 |
if type(ann_data) == 'string' then |
| 379 |
-- Already compressed, convert to text |
| 380 |
ann_data = rspamd_text.fromstring(ann_data) |
| 381 |
end |
| 382 |
|
| 383 |
-- Store base version and ann_data |
| 384 |
lua_redis.redis_make_request_taskless(ev_base, |
| 385 |
rspamd_config, |
| 386 |
redis, |
| 387 |
nil, |
| 388 |
true, -- is write |
| 389 |
store_cb, |
| 390 |
'HMSET', |
| 391 |
{ |
| 392 |
base_key, |
| 393 |
'version', tostring(model_data.model_version or 0), |
| 394 |
'ann_data', ann_data, |
| 395 |
'providers_digest', model_data.providers_digest or '', |
| 396 |
'created_at', tostring(model_data.created_at or os.time()), |
| 397 |
}, |
| 398 |
{ opaque_data = true } |
| 399 |
) |
| 400 |
end |
| 401 |
|
| 402 |
--- Load base model from Redis for re-merge |
| 403 |
-- @param redis redis params |
| 404 |
-- @param ev_base event base |
| 405 |
-- @param ann_key Redis key for the ANN |
| 406 |
-- @param callback function(err, model_data) |
| 407 |
function exports.load_base_model(redis, ev_base, ann_key, callback) |
| 408 |
local base_key = ann_key .. "_base" |
| 409 |
|
| 410 |
local function load_cb(err, data) |
| 411 |
if err then |
| 412 |
callback(err, nil) |
| 413 |
return |
| 414 |
end |
| 415 |
|
| 416 |
if type(data) ~= 'table' then |
| 417 |
callback("no base model found", nil) |
| 418 |
return |
| 419 |
end |
| 420 |
|
| 421 |
local model_data = { |
| 422 |
model_version = tonumber(data[1]) or 0, |
| 423 |
ann_data = data[2], -- rspamd_text |
| 424 |
providers_digest = data[3], |
| 425 |
created_at = tonumber(data[4]) or 0, |
| 426 |
} |
| 427 |
|
| 428 |
callback(nil, model_data) |
| 429 |
end |
| 430 |
|
| 431 |
lua_redis.redis_make_request_taskless(ev_base, |
| 432 |
rspamd_config, |
| 433 |
redis, |
| 434 |
nil, |
| 435 |
false, -- is write |
| 436 |
load_cb, |
| 437 |
'HMGET', |
| 438 |
{ base_key, 'version', 'ann_data', 'providers_digest', 'created_at' }, |
| 439 |
{ opaque_data = true } |
| 440 |
) |
| 441 |
end |
| 442 |
|
| 443 |
--- Serialize model to msgpack format |
| 444 |
-- @param ann kann_t object |
| 445 |
-- @param pca optional PCA tensor |
| 446 |
-- @param providers_digest providers config digest |
| 447 |
-- @param opts optional { norm_stats, roc_thresholds } |
| 448 |
-- @return rspamd_text with compressed msgpack data |
| 449 |
function exports.serialize_model(ann, pca, providers_digest, opts) |
| 450 |
opts = opts or {} |
| 451 |
|
| 452 |
-- Save ANN to memory |
| 453 |
local ann_text = ann:save() |
| 454 |
local ann_compressed = rspamd_util.zstd_compress(ann_text) |
| 455 |
|
| 456 |
local model = { |
| 457 |
magic = MODEL_MAGIC, |
| 458 |
version = MODEL_FORMAT_VERSION, |
| 459 |
model_version = opts.model_version or 1, |
| 460 |
providers_digest = providers_digest, |
| 461 |
ann_data = ann_compressed, |
| 462 |
created_at = os.time(), |
| 463 |
} |
| 464 |
|
| 465 |
if pca then |
| 466 |
local pca_text = pca:save() |
| 467 |
model.pca_data = rspamd_util.zstd_compress(pca_text) |
| 468 |
end |
| 469 |
|
| 470 |
if opts.norm_stats then |
| 471 |
model.norm_stats = opts.norm_stats |
| 472 |
end |
| 473 |
|
| 474 |
if opts.roc_thresholds then |
| 475 |
model.roc_thresholds = opts.roc_thresholds |
| 476 |
end |
| 477 |
|
| 478 |
return ucl.to_format(model, 'msgpack') |
| 479 |
end |
| 480 |
|
| 481 |
exports.MODEL_MAGIC = MODEL_MAGIC |
| 482 |
exports.MODEL_FORMAT_VERSION = MODEL_FORMAT_VERSION |
| 483 |
|
| 484 |
return exports |