Debian Codegraph @

lualib/lua_neural_external.lua

raw
1 --[[
2 Copyright (c) 2026, Vsevolod Stakhov <[email protected]>
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 ]]--
16
17 --[[
18 External neural model loading and merging.
19
20 This module provides functionality to load pretrained neural models
21 from external sources (HTTP/HTTPS) via the Maps infrastructure
22 and merge them with locally trained weights.
23
24 Model format (msgpack):
25 {
26 magic = "RNM1", -- Rspamd Neural Model v1
27 version = 1, -- format version
28 model_version = 123, -- model training version (incremented on retrain)
29 providers_digest = "...", -- digest of providers config (must match local)
30 ann_data = "...", -- serialized KANN (zstd compressed)
31 pca_data = "...", -- optional PCA (zstd compressed)
32 norm_stats = {...}, -- normalization stats
33 roc_thresholds = {...}, -- ROC thresholds
34 created_at = timestamp,
35 }
36
37 Usage in neural config:
38 external_model = {
39 url = "https://your-provider.com/models/<digest>";
40 sign_key = "your_key"; -- optional signature verification
41 merge_alpha = 0.6; -- 60% external, 40% local
42 };
43 ]]--
44
45 local lua_redis = require "lua_redis"
46 local rspamd_kann = require "rspamd_kann"
47 local rspamd_logger = require "rspamd_logger"
48 local rspamd_util = require "rspamd_util"
49 local rspamd_text = require "rspamd_text"
50 local ucl = require "ucl"
51
52 -- Model format constants
53 local MODEL_MAGIC = "RNM1"
54 local MODEL_FORMAT_VERSION = 1
55
56 local exports = {}
57
58 -- Cache of loaded external models: url -> { model, map, callbacks }
59 local external_model_cache = {}
60
61 --- Parse external model from msgpack data
62 -- @param data raw msgpack data (rspamd_text, possibly zstd compressed)
63 -- @return table with model data or nil, error message
64 function exports.parse_model(data)
65 if not data then
66 return nil, "no data"
67 end
68
69 -- Convert rspamd_text to string if needed
70 local data_str
71 if type(data) == 'userdata' or (type(data) == 'table' and data.cookie) then
72 data_str = tostring(data)
73 else
74 data_str = data
75 end
76
77 -- Try zstd decompression first
78 local decompressed
79 local err, decompressed_data = rspamd_util.zstd_decompress(data_str)
80 if not err and decompressed_data then
81 decompressed = tostring(decompressed_data)
82 else
83 -- Assume uncompressed
84 decompressed = data_str
85 end
86
87 -- Parse msgpack
88 local parser = ucl.parser()
89 local ok, parse_err = parser:parse_text(decompressed, 'msgpack')
90 if not ok then
91 return nil, "failed to parse msgpack: " .. (parse_err or "unknown error")
92 end
93
94 local model = parser:get_object()
95
96 -- Validate model format
97 if model.magic ~= MODEL_MAGIC then
98 return nil, string.format("invalid magic: expected %s, got %s",
99 MODEL_MAGIC, model.magic or "nil")
100 end
101
102 if model.version ~= MODEL_FORMAT_VERSION then
103 return nil, string.format("unsupported model version: %s (expected %s)",
104 model.version or "nil", MODEL_FORMAT_VERSION)
105 end
106
107 return model
108 end
109
110 --- Load KANN from model data
111 -- @param model parsed model table
112 -- @return kann_t object or nil, error
113 function exports.load_ann(model)
114 if not model.ann_data then
115 return nil, "no ann_data in model"
116 end
117
118 -- Decompress ann_data
119 local ann_data_str = model.ann_data
120 if type(ann_data_str) == 'userdata' or (type(ann_data_str) == 'table' and ann_data_str.cookie) then
121 ann_data_str = tostring(ann_data_str)
122 end
123
124 local err, ann_data = rspamd_util.zstd_decompress(ann_data_str)
125 if err then
126 return nil, "failed to decompress ann_data: " .. err
127 end
128
129 local ann = rspamd_kann.load(ann_data)
130 if not ann then
131 return nil, "failed to load KANN from model"
132 end
133
134 return ann
135 end
136
137 --- Load PCA from model data
138 -- @param model parsed model table
139 -- @return tensor or nil
140 function exports.load_pca(model)
141 if not model.pca_data then
142 return nil
143 end
144
145 local pca_data_str = model.pca_data
146 if type(pca_data_str) == 'userdata' or (type(pca_data_str) == 'table' and pca_data_str.cookie) then
147 pca_data_str = tostring(pca_data_str)
148 end
149
150 local err, pca_data = rspamd_util.zstd_decompress(pca_data_str)
151 if err then
152 rspamd_logger.warnx(rspamd_config, "failed to decompress pca_data: %s", err)
153 return nil
154 end
155
156 local rspamd_tensor = require "rspamd_tensor"
157 return rspamd_tensor.load(pca_data)
158 end
159
160 --- Check if model is compatible with local config
161 -- @param model parsed model table
162 -- @param providers_digest local providers digest
163 -- @return boolean, reason
164 function exports.is_compatible(model, providers_digest)
165 if not model.providers_digest then
166 return false, "model has no providers_digest"
167 end
168
169 if model.providers_digest ~= providers_digest then
170 return false, string.format("providers digest mismatch: model=%s, local=%s",
171 model.providers_digest:sub(1, 8), providers_digest:sub(1, 8))
172 end
173
174 return true, "compatible"
175 end
176
177 --- Merge weights from external ANN into local ANN using interpolation
178 -- w_new = alpha * w_external + (1-alpha) * w_local
179 -- @param external_ann kann_t from external model
180 -- @param local_ann kann_t from local training
181 -- @param alpha weight for external (0.0 - 1.0)
182 -- @return merged kann_t or nil, error
183 function exports.merge_weights(external_ann, local_ann, alpha)
184 if not external_ann or not local_ann then
185 return nil, "missing ann"
186 end
187
188 alpha = alpha or 0.5
189
190 -- Check compatibility first
191 local ok = external_ann:is_compatible(local_ann)
192 if not ok then
193 return nil, "incompatible ANN architectures"
194 end
195
196 -- Use the external ANN as base and merge local weights into it
197 local merged, err = external_ann:merge_weights(local_ann, 1.0 - alpha)
198
199 if not merged then
200 return nil, "merge failed: " .. (err or "unknown")
201 end
202
203 return external_ann
204 end
205
206 --- Build URL for external model based on providers digest
207 -- @param base_url base URL
208 -- @param providers_digest digest of providers config
209 -- @return full URL
210 function exports.build_model_url(base_url, providers_digest)
211 -- Remove trailing slash from base_url
212 base_url = base_url:gsub('/+$', '')
213 return string.format("%s/%s", base_url, providers_digest)
214 end
215
216 --- Register external model as a map
217 -- This uses the Maps infrastructure for HTTP loading with signature verification
218 -- @param cfg rspamd_config
219 -- @param rule neural rule configuration
220 -- @param providers_digest digest of providers config
221 -- @param on_load_callback function(model_data, err) called when model is loaded/reloaded
222 -- @return boolean success
223 function exports.register_model_map(cfg, rule, providers_digest, on_load_callback)
224 local ext_cfg = rule.external_model
225 if not ext_cfg or not ext_cfg.url then
226 return false
227 end
228
229 local url = ext_cfg.url
230 local cache_key = url
231
232 -- Check if already registered
233 if external_model_cache[cache_key] then
234 return true
235 end
236
237 -- Map callback: called when map data is loaded
238 local function map_callback(data, map)
239 if not data then
240 rspamd_logger.errx(cfg, 'external neural model map returned no data for %s', url)
241 if on_load_callback then
242 on_load_callback(nil, "no data from map")
243 end
244 return
245 end
246
247 -- Parse model
248 local model, parse_err = exports.parse_model(data)
249 if not model then
250 rspamd_logger.errx(cfg, 'failed to parse external neural model from %s: %s', url, parse_err)
251 if on_load_callback then
252 on_load_callback(nil, parse_err)
253 end
254 return
255 end
256
257 -- Check compatibility
258 local compatible, reason = exports.is_compatible(model, providers_digest)
259 if not compatible then
260 rspamd_logger.errx(cfg, 'external neural model incompatible: %s', reason)
261 if on_load_callback then
262 on_load_callback(nil, reason)
263 end
264 return
265 end
266
267 rspamd_logger.infox(cfg, 'loaded external neural model from %s (version=%s)',
268 url, model.model_version or 0)
269
270 -- Update cache
271 external_model_cache[cache_key] = {
272 model = model,
273 last_version = model.model_version,
274 last_load = os.time(),
275 }
276
277 -- Call user callback
278 if on_load_callback then
279 on_load_callback(model, nil)
280 end
281 end
282
283 -- Create callback map
284 local map = cfg:add_map({
285 url = url,
286 type = 'callback',
287 description = string.format('External neural model for rule %s', rule.prefix or 'default'),
288 callback = map_callback,
289 opaque_data = true, -- Get data as rspamd_text
290 })
291
292 if not map then
293 rspamd_logger.errx(cfg, 'failed to register external neural model map for %s', url)
294 return false
295 end
296
297 -- Set sign key if configured
298 if ext_cfg.sign_key then
299 map:set_sign_key(ext_cfg.sign_key)
300 end
301
302 external_model_cache[cache_key] = {
303 map = map,
304 callbacks = { on_load_callback },
305 }
306
307 return true
308 end
309
310 --- Get cached model data for URL
311 -- @param url model URL
312 -- @return cached model data or nil
313 function exports.get_cached_model(url)
314 local cached = external_model_cache[url]
315 if cached and cached.model then
316 return cached.model
317 end
318 return nil
319 end
320
321 --- Create external model configuration for a neural rule
322 -- @param rule neural rule configuration
323 -- @param providers_digest digest of providers config
324 -- @return table with external model config or nil
325 function exports.create_external_config(rule, providers_digest)
326 local ext = rule.external_model
327 if not ext then
328 return nil
329 end
330
331 local url = ext.url
332 if not url then
333 -- Build URL from digest if base_url is provided
334 if ext.base_url then
335 url = exports.build_model_url(ext.base_url, providers_digest)
336 else
337 rspamd_logger.errx(rspamd_config, 'external_model requires url or base_url')
338 return nil
339 end
340 end
341
342 return {
343 url = url,
344 sign_key = ext.sign_key,
345 merge_strategy = ext.merge_strategy or "interpolate",
346 merge_alpha = ext.merge_alpha or 0.5,
347 check_interval = ext.check_interval or 86400, -- 24h
348 local_fine_tune = ext.local_fine_tune ~= false,
349 min_local_samples = ext.min_local_samples or 50,
350 providers_digest = providers_digest,
351 loaded = false,
352 last_version = nil,
353 last_check = nil,
354 }
355 end
356
357 --- Store external model metadata in Redis for later merge
358 -- @param redis redis params
359 -- @param ev_base event base
360 -- @param ann_key Redis key for the ANN
361 -- @param model_data parsed model data
362 -- @param callback function(err)
363 function exports.store_base_model(redis, ev_base, ann_key, model_data, callback)
364 -- Store base model version and compressed ann_data for re-merge
365 local base_key = ann_key .. "_base"
366
367 local function store_cb(err)
368 if err then
369 rspamd_logger.errx(rspamd_config, "failed to store base model: %s", err)
370 end
371 if callback then
372 callback(err)
373 end
374 end
375
376 -- Ensure ann_data is rspamd_text for opaque storage
377 local ann_data = model_data.ann_data
378 if type(ann_data) == 'string' then
379 -- Already compressed, convert to text
380 ann_data = rspamd_text.fromstring(ann_data)
381 end
382
383 -- Store base version and ann_data
384 lua_redis.redis_make_request_taskless(ev_base,
385 rspamd_config,
386 redis,
387 nil,
388 true, -- is write
389 store_cb,
390 'HMSET',
391 {
392 base_key,
393 'version', tostring(model_data.model_version or 0),
394 'ann_data', ann_data,
395 'providers_digest', model_data.providers_digest or '',
396 'created_at', tostring(model_data.created_at or os.time()),
397 },
398 { opaque_data = true }
399 )
400 end
401
402 --- Load base model from Redis for re-merge
403 -- @param redis redis params
404 -- @param ev_base event base
405 -- @param ann_key Redis key for the ANN
406 -- @param callback function(err, model_data)
407 function exports.load_base_model(redis, ev_base, ann_key, callback)
408 local base_key = ann_key .. "_base"
409
410 local function load_cb(err, data)
411 if err then
412 callback(err, nil)
413 return
414 end
415
416 if type(data) ~= 'table' then
417 callback("no base model found", nil)
418 return
419 end
420
421 local model_data = {
422 model_version = tonumber(data[1]) or 0,
423 ann_data = data[2], -- rspamd_text
424 providers_digest = data[3],
425 created_at = tonumber(data[4]) or 0,
426 }
427
428 callback(nil, model_data)
429 end
430
431 lua_redis.redis_make_request_taskless(ev_base,
432 rspamd_config,
433 redis,
434 nil,
435 false, -- is write
436 load_cb,
437 'HMGET',
438 { base_key, 'version', 'ann_data', 'providers_digest', 'created_at' },
439 { opaque_data = true }
440 )
441 end
442
443 --- Serialize model to msgpack format
444 -- @param ann kann_t object
445 -- @param pca optional PCA tensor
446 -- @param providers_digest providers config digest
447 -- @param opts optional { norm_stats, roc_thresholds }
448 -- @return rspamd_text with compressed msgpack data
449 function exports.serialize_model(ann, pca, providers_digest, opts)
450 opts = opts or {}
451
452 -- Save ANN to memory
453 local ann_text = ann:save()
454 local ann_compressed = rspamd_util.zstd_compress(ann_text)
455
456 local model = {
457 magic = MODEL_MAGIC,
458 version = MODEL_FORMAT_VERSION,
459 model_version = opts.model_version or 1,
460 providers_digest = providers_digest,
461 ann_data = ann_compressed,
462 created_at = os.time(),
463 }
464
465 if pca then
466 local pca_text = pca:save()
467 model.pca_data = rspamd_util.zstd_compress(pca_text)
468 end
469
470 if opts.norm_stats then
471 model.norm_stats = opts.norm_stats
472 end
473
474 if opts.roc_thresholds then
475 model.roc_thresholds = opts.roc_thresholds
476 end
477
478 return ucl.to_format(model, 'msgpack')
479 end
480
481 exports.MODEL_MAGIC = MODEL_MAGIC
482 exports.MODEL_FORMAT_VERSION = MODEL_FORMAT_VERSION
483
484 return exports