Debian Codegraph @

lualib/rspamadm/neural_export.lua

raw
1 --[[
2 Copyright (c) 2026, Vsevolod Stakhov <[email protected]>
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 ]]--
16
17 --[[
18 Export neural network model from Redis to RNM1 format for distribution.
19
20 This tool extracts a trained neural model from Redis and saves it as
21 a portable msgpack file that can be loaded by other rspamd instances.
22
23 Usage:
24 rspamadm neural_export -o model.rnm
25
26 Options:
27 -c, --config Path to rspamd config (to get Redis connection)
28 -r, --rule Rule name to export (default: 'default')
29 -s, --settings Settings ID to export (default: 'default')
30 -o, --output Output file path (required)
31 -v, --version Specific model version to export (default: latest)
32 --list List available models instead of exporting
33 --digest Also print providers digest for URL construction
34 ]]--
35
36 local rspamd_logger = require "rspamd_logger"
37 local lua_redis = require "lua_redis"
38 local argparse = require "argparse"
39 local ucl = require "ucl"
40
41 local parser = argparse()
42 :name "rspamadm neural_export"
43 :description "Export neural network model from Redis to RNM1 format"
44 :help_description_margin(32)
45
46 parser:option "-c --config"
47 :description "Path to config file"
48 :argname("<cfg>")
49 :default(rspamd_paths["CONFDIR"] .. "/" .. "rspamd.conf")
50 parser:option "-r --rule"
51 :description "Rule name to export"
52 :argname("<rule>")
53 :default("default")
54 parser:option "-s --settings"
55 :description "Settings ID to export"
56 :argname("<id>")
57 :default("default")
58 parser:option "-o --output"
59 :description "Output file path"
60 :argname("<file>")
61 parser:option "-v --version"
62 :description "Specific model version to export"
63 :argname("<version>")
64 :convert(tonumber)
65 parser:flag "--list"
66 :description "List available models instead of exporting"
67 parser:flag "--digest"
68 :description "Print providers digest for URL construction"
69
70 -- Model format constants (must match lua_neural_external)
71 local MODEL_MAGIC = "RNM1"
72 local MODEL_FORMAT_VERSION = 1
73
74 --- Load config and initialize Redis connection
75 local function init_redis(opts)
76 local _r, err = rspamd_config:load_ucl(opts['config'])
77
78 if not _r then
79 rspamd_logger.errx('cannot parse %s: %s', opts['config'], err)
80 return nil
81 end
82
83 _r, err = rspamd_config:parse_rcl({ 'logging', 'worker' })
84 if not _r then
85 rspamd_logger.errx('cannot process %s: %s', opts['config'], err)
86 return nil
87 end
88
89 local redis_params = lua_redis.parse_redis_server('neural')
90 if not redis_params then
91 rspamd_logger.errx('cannot get Redis configuration for neural module')
92 return nil
93 end
94
95 return redis_params
96 end
97
98 --- Get neural rule configuration
99 local function get_neural_rule(opts)
100 local neural_opts = rspamd_config:get_all_opt('neural')
101 if not neural_opts then
102 rspamd_logger.errx('no neural configuration found')
103 return nil
104 end
105
106 local rules = neural_opts['rules'] or {}
107 local rule_name = opts['rule']
108
109 -- Handle legacy config (neural_opts itself is the rule)
110 if not rules['default'] and neural_opts.train then
111 rules['default'] = neural_opts
112 end
113
114 local rule = rules[rule_name]
115 if not rule then
116 rspamd_logger.errx('rule "%s" not found in neural config', rule_name)
117 return nil
118 end
119
120 if not rule.prefix then
121 rule.prefix = rule_name
122 end
123
124 if not rule.name then
125 rule.name = rule_name
126 end
127
128 return rule
129 end
130
131 --- Build Redis prefix for a rule/settings combination
132 local function build_redis_prefix(rule, settings_name)
133 local neural_common = require "plugins/neural"
134 return neural_common.redis_ann_prefix({
135 prefix = rule.prefix or 'default'
136 }, settings_name)
137 end
138
139 local function redis_connect(redis_params)
140 local redis_opts = {}
141 for k, v in pairs(redis_params) do
142 redis_opts[k] = v
143 end
144 redis_opts.opaque_data = true
145
146 local ok, conn = lua_redis.redis_connect_sync(redis_opts, false)
147
148 if not ok then
149 return nil, conn
150 end
151
152 return conn
153 end
154
155 local function maybe_convert_text(data)
156 if type(data) == 'userdata' or (type(data) == 'table' and data.cookie) then
157 return tostring(data)
158 end
159
160 return data
161 end
162
163 --- List available models from Redis
164 local function list_models(conn, prefix)
165 local ok, err = conn:add_cmd('ZREVRANGE', { prefix, '0', '-1', 'WITHSCORES' })
166
167 if not ok then
168 return nil, err
169 end
170
171 local ret, data = conn:exec()
172
173 if not ret then
174 return nil, data
175 end
176
177 return data
178 end
179
180 local function profile_has_ann(conn, ann_key)
181 local ok, err = conn:add_cmd('HEXISTS', { ann_key, 'ann' })
182
183 if not ok then
184 return false, err
185 end
186
187 local ret, data = conn:exec()
188
189 if not ret then
190 return false, data
191 end
192
193 return tonumber(data) == 1, nil
194 end
195
196 --- Load model data from Redis
197 local function load_model_from_redis(conn, ann_key)
198 local ok, err = conn:add_cmd('HMGET', {
199 ann_key, 'ann', 'roc_thresholds', 'pca', 'providers_meta', 'norm_stats'
200 })
201
202 if not ok then
203 return nil, err
204 end
205
206 local ret, data = conn:exec()
207
208 if not ret then
209 return nil, data
210 end
211
212 if type(data) ~= 'table' then
213 return nil, 'no data found at key: ' .. ann_key
214 end
215
216 -- data[1] = ann, data[2] = roc_thresholds, data[3] = pca,
217 -- data[4] = providers_meta, data[5] = norm_stats
218 local result = {
219 ann_data = maybe_convert_text(data[1]),
220 roc_thresholds = maybe_convert_text(data[2]),
221 pca_data = maybe_convert_text(data[3]),
222 providers_meta = maybe_convert_text(data[4]),
223 norm_stats = maybe_convert_text(data[5]),
224 }
225
226 if result.roc_thresholds then
227 local roc_parser = ucl.parser()
228 local parse_ok = roc_parser:parse_text(result.roc_thresholds)
229 if parse_ok then
230 result.roc_thresholds = roc_parser:get_object()
231 end
232 end
233
234 if result.norm_stats then
235 local norm_parser = ucl.parser()
236 local parse_ok = norm_parser:parse_text(result.norm_stats)
237 if parse_ok then
238 result.norm_stats = norm_parser:get_object()
239 end
240 end
241
242 return result
243 end
244
245 --- Load profile from Redis sorted set
246 local function load_profile(profile_str)
247 local profile_parser = ucl.parser()
248 local ok = profile_parser:parse_string(profile_str)
249 if not ok then
250 return nil
251 end
252 return profile_parser:get_object()
253 end
254
255 --- Serialize model to RNM1 format
256 local function serialize_model(model_data, providers_digest, opts)
257 opts = opts or {}
258
259 local model = {
260 magic = MODEL_MAGIC,
261 version = MODEL_FORMAT_VERSION,
262 model_version = opts.model_version or 1,
263 providers_digest = providers_digest,
264 ann_data = model_data.ann_data,
265 created_at = os.time(),
266 }
267
268 if model_data.pca_data then
269 model.pca_data = model_data.pca_data
270 end
271
272 if opts.norm_stats then
273 model.norm_stats = opts.norm_stats
274 end
275
276 if opts.roc_thresholds then
277 model.roc_thresholds = opts.roc_thresholds
278 end
279
280 return ucl.to_format(model, 'msgpack')
281 end
282
283 local function handler(args)
284 local opts = parser:parse(args)
285
286 -- Initialize
287 local redis_params = init_redis(opts)
288 if not redis_params then
289 os.exit(1)
290 end
291
292 local rule = get_neural_rule(opts)
293 if not rule then
294 os.exit(1)
295 end
296
297 -- Get providers digest
298 local neural_common = require "plugins/neural"
299 local providers_digest = neural_common.providers_config_digest(rule.providers, rule)
300
301 -- Print digest if requested
302 if opts['digest'] then
303 print(string.format("Providers digest: %s", providers_digest))
304 if not opts['list'] and not opts['output'] then
305 return
306 end
307 end
308
309 local settings_name = opts['settings']
310 local prefix = build_redis_prefix(rule, settings_name)
311 local conn, conn_err = redis_connect(redis_params)
312
313 if not conn then
314 rspamd_logger.errx('cannot connect to redis server: %s', conn_err or 'unknown error')
315 os.exit(1)
316 end
317
318 -- List mode: show available models
319 if opts['list'] then
320 print(string.format("\nAvailable models for rule '%s', settings '%s':", opts['rule'], settings_name))
321 print(string.format("Redis prefix: %s", prefix))
322 print("")
323
324 local data, err = list_models(conn, prefix)
325 if err then
326 rspamd_logger.errx('failed to list models: %s', err)
327 os.exit(1)
328 end
329
330 if not data or #data == 0 then
331 print("No models found")
332 return
333 end
334
335 for i = 1, #data, 2 do
336 local profile_str = data[i]
337 local score = tonumber(data[i + 1]) or 0
338 local profile = load_profile(profile_str)
339
340 if profile then
341 local has_ann = false
342 if profile.redis_key then
343 has_ann = profile_has_ann(conn, profile.redis_key)
344 end
345 local ts = os.date("%Y-%m-%d %H:%M:%S", score)
346 print(string.format(" Version: %s, Key: %s, Updated: %s%s",
347 profile.version or 0,
348 profile.redis_key or "?",
349 ts,
350 has_ann and '' or ' [stale]'))
351 end
352 end
353
354 return
355 end
356
357 -- Export mode: need output file
358 if not opts['output'] then
359 rspamd_logger.errx('output file is required for export (use -o <file>)')
360 os.exit(1)
361 end
362
363 local output_file = opts['output']
364 local target_version = opts['version']
365 local data, err = list_models(conn, prefix)
366 if err then
367 rspamd_logger.errx('failed to list models: %s', err)
368 os.exit(1)
369 end
370
371 if not data or #data == 0 then
372 rspamd_logger.errx('no models found in Redis')
373 os.exit(1)
374 end
375
376 local selected_profile
377 local selected_score = 0
378
379 for i = 1, #data, 2 do
380 local profile_str = data[i]
381 local score = tonumber(data[i + 1]) or 0
382 local profile = load_profile(profile_str)
383
384 if profile then
385 local has_ann = profile.redis_key and profile_has_ann(conn, profile.redis_key)
386
387 if not has_ann then
388 rspamd_logger.warnx('skip stale model profile: version=%s key=%s',
389 profile.version or 0, profile.redis_key or '?')
390 elseif target_version then
391 if profile.version == target_version then
392 selected_profile = profile
393 selected_score = score
394 break
395 end
396 elseif not selected_profile or score > selected_score then
397 selected_profile = profile
398 selected_score = score
399 end
400 end
401 end
402
403 if not selected_profile then
404 if target_version then
405 rspamd_logger.errx('model version %s not found', target_version)
406 else
407 rspamd_logger.errx('no suitable model found')
408 end
409 os.exit(1)
410 end
411
412 local ann_key = selected_profile.redis_key
413 rspamd_logger.messagex('Loading model from key: %s (version %s)',
414 ann_key, selected_profile.version or 0)
415
416 local model_data, load_err = load_model_from_redis(conn, ann_key)
417 if load_err then
418 rspamd_logger.errx('failed to load model data: %s', load_err)
419 os.exit(1)
420 end
421
422 if not model_data.ann_data then
423 rspamd_logger.errx('failed to load ANN data from Redis')
424 os.exit(1)
425 end
426
427 local rnm_data = serialize_model(model_data, providers_digest, {
428 model_version = selected_profile.version or 1,
429 norm_stats = model_data.norm_stats,
430 roc_thresholds = model_data.roc_thresholds,
431 })
432
433 local out, open_err = io.open(output_file, 'wb')
434 if not out then
435 rspamd_logger.errx('cannot open output file %s: %s', output_file, open_err)
436 os.exit(1)
437 end
438
439 local write_ok, write_err = out:write(rnm_data)
440 out:close()
441
442 if not write_ok then
443 rspamd_logger.errx('cannot write output file %s: %s', output_file, write_err)
444 os.exit(1)
445 end
446
447 rspamd_logger.messagex('Exported model to: %s', output_file)
448 rspamd_logger.messagex(' Model version: %s', selected_profile.version or 0)
449 rspamd_logger.messagex(' Providers digest: %s', providers_digest:sub(1, 16) .. '...')
450 rspamd_logger.messagex(' File size: %s bytes', #rnm_data)
451 end
452
453 return {
454 name = "neural_export",
455 aliases = { "neural_export" },
456 handler = handler,
457 description = parser._description
458 }