Debian Codegraph @

lualib/rspamadm/statistics_dump.lua

raw
1 --[[
2 Copyright (c) 2022, Vsevolod Stakhov <[email protected]>
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 ]]--
16
17 local lua_redis = require "lua_redis"
18 local rspamd_logger = require "rspamd_logger"
19 local argparse = require "argparse"
20 local rspamd_zstd = require "rspamd_zstd"
21 local rspamd_text = require "rspamd_text"
22 local rspamd_util = require "rspamd_util"
23 local rspamd_cdb = require "rspamd_cdb"
24 local lua_util = require "lua_util"
25 local rspamd_i64 = require "rspamd_int64"
26 local ucl = require "ucl"
27
28 local N = "statistics_dump"
29 local E = {}
30 local classifiers = {}
31
32 -- Define command line options
33 local parser = argparse()
34 :name "rspamadm statistics_dump"
35 :description "Dump/restore Rspamd statistics"
36 :help_description_margin(30)
37 :command_target("command")
38 :require_command(false)
39
40 parser:option "-c --config"
41 :description "Path to config file"
42 :argname("<cfg>")
43 :default(rspamd_paths["CONFDIR"] .. "/" .. "rspamd.conf")
44
45 parser:option "-b --batch-size"
46 :description "Number of entries to process at once"
47 :argname("<elts>")
48 :convert(tonumber)
49 :default(1000)
50
51 parser:option "-S --classifier"
52 :description "Classifier name (required when multiple classifiers configured)"
53 :argname("<name>")
54
55 -- Extract subcommand
56 local dump = parser:command "dump d"
57 :description "Dump bayes statistics"
58 dump:mutex(
59 dump:flag "-j --json"
60 :description "Json output",
61 dump:flag "-C --cdb"
62 :description "CDB output"
63 )
64 dump:flag "-c --compress"
65 :description "Compress output"
66 dump:option "-b --batch-size"
67 :description "Number of entries to process at once"
68 :argname("<elts>")
69 :convert(tonumber)
70 :default(1000)
71
72
73 -- Restore
74 local restore = parser:command "restore r"
75 :description "Restore bayes statistics"
76 restore:argument "file"
77 :description "Input file to process"
78 :argname "<file>"
79 :args "*"
80 restore:option "-b --batch-size"
81 :description "Number of entries to process at once"
82 :argname("<elts>")
83 :convert(tonumber)
84 :default(1000)
85 restore:option "-m --mode"
86 :description "Number of entries to process at once"
87 :argname("<append|subtract|replace>")
88 :convert {
89 ['append'] = 'append',
90 ['subtract'] = 'subtract',
91 ['replace'] = 'replace',
92 }
93 :default 'append'
94 restore:flag "-n --no-operation"
95 :description "Only show redis commands to be issued"
96
97 -- Migrate
98 local migrate = parser:command "migrate m"
99 :description "Migrate bayes data between shards (after hash algorithm change)"
100 migrate:flag "-n --dry-run"
101 :description "Only show what would be migrated, without writing"
102 migrate:flag "--no-delete"
103 :description "Copy keys to target shard without deleting from source"
104 migrate:option "-b --batch-size"
105 :description "Number of entries to process per SCAN batch"
106 :argname("<elts>")
107 :convert(tonumber)
108 :default(1000)
109
110 local function load_config(opts)
111 local _r, err = rspamd_config:load_ucl(opts['config'])
112
113 if not _r then
114 rspamd_logger.errx('cannot parse %s: %s', opts['config'], err)
115 os.exit(1)
116 end
117
118 _r, err = rspamd_config:parse_rcl({ 'logging', 'worker' })
119 if not _r then
120 rspamd_logger.errx('cannot process %s: %s', opts['config'], err)
121 os.exit(1)
122 end
123 end
124
125 local function check_redis_classifier(cls, cfg)
126 -- Skip old classifiers
127 if cls.new_schema then
128 local symbol_spam, symbol_ham
129 local symbols = {}
130 -- Load symbols from statfiles
131
132 local function get_class_label(class_name)
133 -- Check class_labels mapping in classifier config
134 if cls.class_labels and class_name then
135 local label = cls.class_labels[class_name]
136 if label then
137 return label
138 end
139 end
140 -- Default mapping: spam→S, ham→H, custom→class_name
141 if class_name == 'spam' then
142 return 'S'
143 elseif class_name == 'ham' then
144 return 'H'
145 end
146 return class_name
147 end
148
149 local function check_statfile_table(tbl, def_sym)
150 local symbol = tbl.symbol or def_sym
151
152 -- Determine class_name by priority:
153 -- 1. Explicit tbl.class
154 -- 2. Legacy tbl.spam boolean
155 -- 3. Heuristic from symbol name
156 local class_name
157 if tbl.class then
158 class_name = tbl.class
159 elseif tbl.spam then
160 class_name = 'spam'
161 else
162 if string.match(symbol:upper(), 'SPAM') then
163 class_name = 'spam'
164 else
165 class_name = 'ham'
166 end
167 end
168
169 local label = get_class_label(class_name)
170
171 -- Backward compat for binary classifiers
172 if class_name == 'spam' then
173 symbol_spam = symbol
174 elseif class_name == 'ham' then
175 symbol_ham = symbol
176 end
177
178 table.insert(symbols, {
179 symbol = symbol,
180 class_name = class_name,
181 label = label,
182 })
183 end
184
185 local statfiles = cls.statfile
186 if statfiles[1] then
187 for _, stf in ipairs(statfiles) do
188 if not stf.symbol then
189 for k, v in pairs(stf) do
190 check_statfile_table(v, k)
191 end
192 else
193 check_statfile_table(stf, 'undefined')
194 end
195 end
196 else
197 for stn, stf in pairs(statfiles) do
198 check_statfile_table(stf, stn)
199 end
200 end
201
202 local redis_params
203 -- Try load from classifier config
204 redis_params = lua_redis.try_load_redis_servers(cls,
205 rspamd_config, false, 'bayes')
206 if not redis_params then
207 -- Try load from statistics_dump module config
208 redis_params = lua_redis.try_load_redis_servers(cfg[N] or E,
209 rspamd_config, false, 'bayes')
210 if not redis_params then
211 redis_params = lua_redis.try_load_redis_servers(cfg[N] or E,
212 rspamd_config, true)
213 if not redis_params then
214 -- Try load from global redis config
215 redis_params = lua_redis.try_load_redis_servers(rspamd_config:get_all_opt('redis'),
216 rspamd_config, true)
217 if not redis_params then
218 return false
219 end
220 end
221 end
222 end
223
224 -- Derive classifier name: explicit name > first symbol
225 local cls_name = cls.name
226 if not cls_name then
227 if symbols[1] then
228 cls_name = symbols[1].symbol
229 else
230 cls_name = 'unknown'
231 end
232 end
233
234 table.insert(classifiers, {
235 name = cls_name,
236 symbol_spam = symbol_spam,
237 symbol_ham = symbol_ham,
238 symbols = symbols,
239 redis_params = redis_params,
240 })
241 end
242 end
243
244 local function redis_map_zip(ar)
245 local data = {}
246 for j = 1, #ar, 2 do
247 data[ar[j]] = ar[j + 1]
248 end
249
250 return data
251 end
252
253 -- Used to clear tables
254 local clear_fcn = table.clear or function(tbl)
255 local keys = lua_util.keys(tbl)
256 for _, k in ipairs(keys) do
257 tbl[k] = nil
258 end
259 end
260
261 local function connect_to_upstream(up, redis_params)
262 local rspamd_redis = require "rspamd_redis"
263 local ret, conn = rspamd_redis.connect_sync({
264 host = up:get_addr(),
265 timeout = redis_params.timeout,
266 config = rspamd_config,
267 ev_base = rspamadm_ev_base,
268 session = rspamadm_session,
269 })
270
271 if not ret or not conn then
272 rspamd_logger.errx("cannot connect to redis %s: %s", up:get_name(), conn)
273 return false, nil
274 end
275
276 local need_exec = false
277 if redis_params.username then
278 if redis_params.password then
279 conn:add_cmd('AUTH', { redis_params.username, redis_params.password })
280 need_exec = true
281 else
282 rspamd_logger.errx("redis requires a password when username is supplied")
283 return false, nil
284 end
285 elseif redis_params.password then
286 conn:add_cmd('AUTH', { redis_params.password })
287 need_exec = true
288 end
289
290 if redis_params.db then
291 conn:add_cmd('SELECT', { tostring(redis_params.db) })
292 need_exec = true
293 elseif redis_params.dbname then
294 conn:add_cmd('SELECT', { tostring(redis_params.dbname) })
295 need_exec = true
296 end
297
298 if need_exec then
299 local exec_ret, res = conn:exec()
300 if not exec_ret then
301 rspamd_logger.errx("cannot authenticate/select db on %s: %s", up:get_name(), res)
302 return false, nil
303 end
304 end
305
306 return true, conn
307 end
308
309 local compress_ctx
310
311 local function dump_out(out, opts, last)
312 if opts.compress and not compress_ctx then
313 compress_ctx = rspamd_zstd.compress_ctx()
314 end
315
316 if compress_ctx then
317 if last then
318 compress_ctx:stream(rspamd_text.fromtable(out), 'end'):write()
319 else
320 compress_ctx:stream(rspamd_text.fromtable(out), 'flush'):write()
321 end
322 else
323 for _, o in ipairs(out) do
324 io.write(o)
325 end
326 end
327 end
328
329 -- Maximum commands per pipeline exec() to avoid Lua stack overflow
330 local pipeline_max = 1000
331
332 local append_redis_hash_hmset
333 local exec_redis_commands
334
335 local function dump_cdb(out, opts, last, pattern, class_labels)
336 local results = out[pattern]
337
338 if not out.cdb_builder then
339 -- First invocation
340 out.cdb_builder = rspamd_cdb.build(string.format('%s.cdb', pattern))
341 -- Write learned counts for all class labels
342 for _, lbl in ipairs(class_labels or { 'S', 'H' }) do
343 local learned_key
344 if lbl == 'S' then
345 learned_key = 'learns_spam'
346 elseif lbl == 'H' then
347 learned_key = 'learns_ham'
348 else
349 learned_key = 'learns_' .. lbl
350 end
351 -- Pad CDB key to 8 bytes for consistent lookup
352 local cdb_key = string.format('_lrn%-4s', lbl)
353 out.cdb_builder:add(cdb_key, rspamd_i64.fromstring(results[learned_key] or '0'))
354 end
355 end
356
357 for _, o in ipairs(results.elts) do
358 out.cdb_builder:add(o.key, o.value)
359 end
360
361 if last then
362 out.cdb_builder:finalize()
363 out.cdb_builder = nil
364 end
365 end
366
367 local function dump_pattern(conn, pattern, opts, out, key, class_labels)
368 local cursor = 0
369
370 -- Build CDB pack format string from class labels
371 local cdb_fmt
372 if opts.cdb then
373 cdb_fmt = string.rep('f', #class_labels)
374 end
375
376 repeat
377 conn:add_cmd('SCAN', { tostring(cursor),
378 'MATCH', pattern,
379 'COUNT', tostring(opts.batch_size) })
380 local ret, results = conn:exec()
381
382 if not ret then
383 rspamd_logger.errx("cannot connect execute scan command: %s", results)
384 os.exit(1)
385 end
386
387 cursor = tonumber(results[1])
388
389 local elts = results[2]
390 local tokens = {}
391
392 -- Pipeline HGETALL in chunks to avoid stack overflow
393 for chunk_start = 1, #elts, pipeline_max do
394 local chunk_end = math.min(chunk_start + pipeline_max - 1, #elts)
395 for ei = chunk_start, chunk_end do
396 conn:add_cmd('HGETALL', { elts[ei] })
397 end
398 local all_results = { conn:exec() }
399
400 for i = 1, #all_results, 2 do
401 local r, hash_content = all_results[i], all_results[i + 1]
402 if r then
403 local data = redis_map_zip(hash_content)
404 tokens[#tokens + 1] = {
405 key = elts[chunk_start + (i - 1) / 2],
406 data = data,
407 }
408 end
409 end
410 all_results = nil
411 end
412
413 -- Output keeping track of the commas
414 for i, d in ipairs(tokens) do
415 if cursor == 0 and i == #tokens or not opts.json then
416 if opts.cdb then
417 -- Pack all class label values dynamically
418 local values = {}
419 for _, lbl in ipairs(class_labels) do
420 values[#values + 1] = tonumber(d.data[lbl] or '0') or 0
421 end
422 table.insert(out[key].elts, {
423 key = rspamd_i64.fromstring(string.match(d.key, '%d+')),
424 value = rspamd_util.pack(cdb_fmt, lua_util.unpack(values))
425 })
426 else
427 out[#out + 1] = rspamd_logger.slog('"%s": %s\n', d.key,
428 ucl.to_format(d.data, "json-compact"))
429 end
430 else
431 out[#out + 1] = rspamd_logger.slog('"%s": %s,\n', d.key,
432 ucl.to_format(d.data, "json-compact"))
433 end
434
435 end
436
437 if opts.json and cursor == 0 then
438 out[#out + 1] = '}}\n'
439 end
440
441 -- Do not write the last chunk of out as it will be processed afterwards
442 if cursor ~= 0 then
443 if opts.cdb then
444 dump_cdb(out, opts, false, key, class_labels)
445 out[key].elts = {}
446 else
447 dump_out(out, opts, false)
448 clear_fcn(out)
449 end
450 elseif opts.cdb then
451 dump_cdb(out, opts, true, key, class_labels)
452 end
453
454 until cursor == 0
455 end
456
457 local function select_classifier(opts)
458 if #classifiers == 0 then
459 rspamd_logger.errx("no redis classifiers found in config")
460 os.exit(1)
461 end
462
463 if #classifiers == 1 then
464 return { classifiers[1] }
465 end
466
467 -- Multiple classifiers: require --classifier
468 if not opts.classifier then
469 local names = {}
470 for _, cls in ipairs(classifiers) do
471 local syms = {}
472 for _, s in ipairs(cls.symbols) do
473 syms[#syms + 1] = s.symbol
474 end
475 names[#names + 1] = string.format(" %s (symbols: %s)", cls.name, table.concat(syms, ', '))
476 end
477 rspamd_logger.errx("multiple classifiers found, use --classifier to select one:\n%s",
478 table.concat(names, '\n'))
479 os.exit(1)
480 end
481
482 for _, cls in ipairs(classifiers) do
483 if cls.name == opts.classifier then
484 return { cls }
485 end
486 end
487
488 rspamd_logger.errx("classifier '%s' not found", opts.classifier)
489 os.exit(1)
490 end
491
492 local function dump_handler(opts)
493 local selected = select_classifier(opts)
494 local patterns_seen = {}
495 for _, cls in ipairs(selected) do
496 -- Collect class labels for CDB packing
497 local class_labels = {}
498 for _, s in ipairs(cls.symbols) do
499 class_labels[#class_labels + 1] = s.label
500 end
501
502 -- Connect to all shards to ensure complete dump
503 local connections = {}
504 local read_servers = cls.redis_params.read_servers
505 if read_servers then
506 local all_ups = read_servers:all_upstreams()
507 if all_ups and #all_ups > 0 then
508 for _, up in ipairs(all_ups) do
509 local res, conn = connect_to_upstream(up, cls.redis_params)
510 if res then
511 connections[#connections + 1] = { up = up, conn = conn }
512 else
513 rspamd_logger.errx("cannot connect to redis shard %s", up:get_name())
514 end
515 end
516 end
517 end
518
519 -- Fallback: single connection via round-robin
520 if #connections == 0 then
521 local res, conn = lua_redis.redis_connect_sync(cls.redis_params, false)
522 if not res then
523 rspamd_logger.errx("cannot connect to redis server: %s", cls.redis_params)
524 os.exit(1)
525 end
526 connections[#connections + 1] = { conn = conn }
527 end
528
529 local out = {}
530 local function check_keys(conn, sym)
531 local sym_keys_pattern = string.format("%s_keys", sym)
532 conn:add_cmd('SMEMBERS', { sym_keys_pattern })
533 local ret, keys = conn:exec()
534
535 if not ret then
536 rspamd_logger.errx("cannot execute command to get keys: %s", keys)
537 return
538 end
539
540 if not keys or #keys == 0 then
541 return
542 end
543
544 if not opts.json then
545 out[#out + 1] = string.format('"%s": %s\n', sym_keys_pattern,
546 ucl.to_format(keys, 'json-compact'))
547 end
548 for _, k in ipairs(keys) do
549 local pat = string.format('%s_*', k)
550 if not patterns_seen[pat] then
551 conn:add_cmd('HGETALL', { k })
552 local _ret, additional_keys = conn:exec()
553
554 if _ret then
555 if opts.json then
556 out[#out + 1] = string.format('{"pattern": "%s", "meta": %s, "elts": {\n',
557 k, ucl.to_format(redis_map_zip(additional_keys), 'json-compact'))
558 elseif opts.cdb then
559 out[k] = redis_map_zip(additional_keys)
560 out[k].elts = {}
561 else
562 out[#out + 1] = string.format('"%s": %s\n', k,
563 ucl.to_format(redis_map_zip(additional_keys), 'json-compact'))
564 end
565 dump_pattern(conn, pat, opts, out, k, class_labels)
566 patterns_seen[pat] = true
567 end
568 end
569 end
570 end
571
572 for _, c in ipairs(connections) do
573 for _, s in ipairs(cls.symbols) do
574 check_keys(c.conn, s.symbol)
575 end
576 end
577
578 if #out > 0 then
579 dump_out(out, opts, true)
580 end
581 end
582 end
583
584 local function obj_to_redis_arguments(obj, opts, cmd_pipe)
585 local key, value = next(obj)
586
587 if type(key) == 'string' then
588 if type(value) == 'table' then
589 if not value[1] then
590 if opts.mode == 'replace' then
591 local cmd = 'HMSET'
592 local params = { key }
593 for k, v in pairs(value) do
594 table.insert(params, k)
595 table.insert(params, v)
596 end
597 table.insert(cmd_pipe, { cmd, params })
598 else
599 local cmd = 'HINCRBYFLOAT'
600 local mult = 1.0
601 if opts.mode == 'subtract' then
602 mult = (-mult)
603 end
604
605 for k, v in pairs(value) do
606 if tonumber(v) then
607 v = tonumber(v)
608 table.insert(cmd_pipe, { cmd, { key, k, tostring(v * mult) } })
609 else
610 table.insert(cmd_pipe, { 'HSET', { key, k, v } })
611 end
612 end
613 end
614 else
615 -- Numeric table of elements (e.g. _keys) - it is actually a set in Redis
616 for _, elt in ipairs(value) do
617 table.insert(cmd_pipe, { 'SADD', { key, elt } })
618 end
619 end
620 end
621 end
622
623 return cmd_pipe
624 end
625
626 local function estimate_redis_commands(obj, opts)
627 local key, value = next(obj)
628
629 if type(key) ~= 'string' or type(value) ~= 'table' then
630 return 0
631 end
632
633 if not value[1] then
634 local n = 0
635
636 if opts.mode == 'replace' then
637 return 1
638 end
639
640 for _ in pairs(value) do
641 n = n + 1
642 end
643
644 return n
645 end
646
647 return #value
648 end
649
650 -- Send cmd_pipe commands to a single connection starting from start_idx.
651 -- Returns true on success, or (false, err, resume_idx) on failure where
652 -- resume_idx is the chunk start index that should be retried.
653 local function send_cmd_pipe(cmd_pipe, conn, start_idx)
654 for i = start_idx, #cmd_pipe, pipeline_max do
655 local chunk_end = math.min(i + pipeline_max - 1, #cmd_pipe)
656 local added = 0
657
658 for j = i, chunk_end do
659 local is_ok, err = conn:add_cmd(cmd_pipe[j][1], cmd_pipe[j][2])
660
661 if not is_ok then
662 rspamd_logger.errx("cannot add command: %s with args: %s: %s",
663 cmd_pipe[j][1], cmd_pipe[j][2], err)
664 -- add_cmd failed: no commands from this chunk were sent to Redis,
665 -- safe to retry from this chunk index
666 return false, err, i
667 end
668
669 added = added + 1
670 end
671
672 if added > 0 then
673 local ret, err = conn:exec()
674
675 if not ret then
676 local chunk_size = chunk_end - i + 1
677 rspamd_logger.errx("cannot execute restore batch: %s; skipping %s commands in failed chunk to avoid double-counting",
678 err, chunk_size)
679 -- exec() failed: some commands in this chunk may have been applied,
680 -- advance past this chunk to avoid double-counting
681 return false, err, i + pipeline_max
682 end
683 end
684 end
685
686 return true
687 end
688
689 local function reconnect_all(selected)
690 local new_conns = {}
691
692 for _, cls in ipairs(selected) do
693 local res, conn = lua_redis.redis_connect_sync(cls.redis_params, true)
694
695 if not res then
696 rspamd_logger.errx("cannot reconnect to redis server: %s", cls.redis_params)
697 return nil
698 end
699
700 table.insert(new_conns, conn)
701 end
702
703 return new_conns
704 end
705
706 local max_retries = 3
707
708 local function flush_restore_batch(batch, conns, selected, opts)
709 if #batch == 0 then
710 return true, conns
711 end
712
713 local cmd_pipe = {}
714 for _, cmd in ipairs(batch) do
715 obj_to_redis_arguments(cmd, opts, cmd_pipe)
716 end
717
718 if opts.no_operation then
719 for _, cmd in ipairs(cmd_pipe) do
720 rspamd_logger.messagex('%s %s', cmd[1], table.concat(cmd[2], ' '))
721 end
722 clear_fcn(batch)
723 return true, conns
724 end
725
726 for conn_idx, conn in ipairs(conns) do
727 local resume_idx = 1
728
729 for attempt = 1, max_retries do
730 local ok, err, next_idx = send_cmd_pipe(cmd_pipe, conn, resume_idx)
731
732 if ok then
733 break
734 end
735
736 if attempt == max_retries then
737 rspamd_logger.errx("batch failed after %s attempts: %s", max_retries, err)
738 return false, conns
739 end
740
741 resume_idx = next_idx or resume_idx
742 rspamd_logger.messagex("batch failed at command %s/%s, reconnecting (attempt %s/%s)",
743 resume_idx, #cmd_pipe, attempt, max_retries)
744
745 -- Brief pause before reconnecting to handle transient Redis unavailability
746 os.execute("sleep 1")
747
748 local new_conns = reconnect_all(selected)
749
750 if not new_conns then
751 rspamd_logger.errx("reconnection failed on attempt %s", attempt)
752 return false, conns
753 end
754
755 conns = new_conns
756 conn = conns[conn_idx]
757 end
758 end
759
760 clear_fcn(batch)
761 return true, conns
762 end
763
764 local function restore_handler(opts)
765 local selected = select_classifier(opts)
766 local files = opts.file or { '-' }
767 local conns = {}
768 local restore_pipeline_limit = math.max(100, math.min(opts.batch_size, pipeline_max))
769
770 for _, cls in ipairs(selected) do
771 local res, conn = lua_redis.redis_connect_sync(cls.redis_params, true)
772
773 if not res then
774 rspamd_logger.errx("cannot connect to redis server: %s", cls.redis_params)
775 os.exit(1)
776 end
777
778 table.insert(conns, conn)
779 end
780
781 local batch = {}
782 local pending_cmds = 0
783 local total_lines = 0
784 local total_cmds = 0
785 local total_batches = 0
786 local start_time = os.time()
787 local last_report_time = start_time
788
789 rspamd_logger.messagex("starting restore (batch_size=%s, pipeline_max=%s, mode=%s)",
790 opts.batch_size, pipeline_max, opts.mode or 'add')
791
792 for _, f in ipairs(files) do
793 local fd
794 if f ~= '-' then
795 fd = io.open(f, 'r')
796 io.input(fd)
797 end
798
799 rspamd_logger.messagex("processing file: %s", f)
800 local cur_line = 1
801 for line in io.lines() do
802 local ucl_parser = ucl.parser()
803 local res, err
804 res, err = ucl_parser:parse_string(line)
805
806 if not res then
807 rspamd_logger.errx("%s: cannot read line %s: %s", f, cur_line, err)
808 os.exit(1)
809 end
810
811 table.insert(batch, ucl_parser:get_object())
812 pending_cmds = pending_cmds + estimate_redis_commands(batch[#batch], opts)
813 cur_line = cur_line + 1
814
815 if #batch >= opts.batch_size or pending_cmds >= restore_pipeline_limit then
816 local ok
817 ok, conns = flush_restore_batch(batch, conns, selected, opts)
818 if not ok then
819 rspamd_logger.errx("restore failed at line %s (total restored: %s lines, %s commands in %s batches)",
820 total_lines + cur_line, total_lines, total_cmds, total_batches)
821 os.exit(1)
822 end
823 total_cmds = total_cmds + pending_cmds
824 total_batches = total_batches + 1
825 pending_cmds = 0
826
827 -- Incremental GC after each batch to spread collection cost
828 collectgarbage('step', 100)
829
830 local now = os.time()
831 if now - last_report_time >= 10 then
832 local elapsed = now - start_time
833 local rate = total_lines > 0 and math.floor(total_lines / elapsed) or 0
834 rspamd_logger.messagex("restored %s lines, %s commands in %s batches (%s lines/sec, %s KB lua mem)",
835 total_lines + cur_line - 1, total_cmds, total_batches, rate,
836 math.floor(collectgarbage('count')))
837 last_report_time = now
838 end
839 end
840 end
841
842 total_lines = total_lines + cur_line - 1
843
844 if fd then
845 fd:close()
846 end
847
848 -- Full GC between files
849 collectgarbage('collect')
850 end
851
852 if #batch > 0 then
853 local ok
854 ok, conns = flush_restore_batch(batch, conns, selected, opts)
855 if not ok then
856 rspamd_logger.errx("restore failed on final batch (total restored: %s lines, %s commands)",
857 total_lines, total_cmds)
858 os.exit(1)
859 end
860 total_cmds = total_cmds + pending_cmds
861 total_batches = total_batches + 1
862 end
863
864 local elapsed = os.time() - start_time
865 if elapsed == 0 then
866 elapsed = 1
867 end
868 rspamd_logger.messagex("restore complete: %s lines, %s commands in %s batches (%s sec, %s lines/sec)",
869 total_lines, total_cmds, total_batches, elapsed, math.floor(total_lines / elapsed))
870 end
871
872 -- Migrate a single prefix's token keys from source to target using pipelined commands.
873 -- SCAN on source, pipeline HGETALL, pipeline HMSET to target, pipeline DEL on source.
874 -- Returns number of tokens migrated.
875 local function collect_prefix_token_keys(src_conn, prefixes, batch_size)
876 local keys = {}
877 local seen = {}
878
879 for _, prefix in ipairs(prefixes) do
880 local scan_pattern = string.format('%s_*', prefix)
881 local cursor = "0"
882
883 repeat
884 src_conn:add_cmd('SCAN', { cursor, 'MATCH', scan_pattern,
885 'COUNT', tostring(batch_size) })
886 local ret, results = src_conn:exec()
887
888 if not ret then
889 rspamd_logger.errx("SCAN failed for %s: %s", prefix, results)
890 return nil, true
891 end
892
893 cursor = results[1]
894 local scanned = results[2]
895
896 if scanned and #scanned > 0 then
897 for _, k in ipairs(scanned) do
898 if not seen[k] then
899 seen[k] = true
900 keys[#keys + 1] = k
901 end
902 end
903 end
904 until cursor == "0"
905 end
906
907 return keys, false
908 end
909
910 local function migrate_token_keys(src_conn, dst_conn, keys, no_delete)
911 local total_tokens = 0
912
913 for i = 1, #keys, pipeline_max do
914 local chunk_end = math.min(i + pipeline_max - 1, #keys)
915
916 for j = i, chunk_end do
917 src_conn:add_cmd('HGETALL', { keys[j] })
918 end
919
920 local all_results = { src_conn:exec() }
921 local dst_cmds = {}
922 local src_del_cmds = {}
923
924 for j = i, chunk_end do
925 local idx = (j - i) * 2 + 1
926 local hret, hdata = all_results[idx], all_results[idx + 1]
927
928 if hret and append_redis_hash_hmset(dst_cmds, keys[j], hdata) then
929 total_tokens = total_tokens + 1
930 if not no_delete then
931 src_del_cmds[#src_del_cmds + 1] = { 'DEL', { keys[j] } }
932 end
933 end
934 end
935
936 all_results = nil
937
938 if not exec_redis_commands(dst_conn, dst_cmds) then
939 return total_tokens, true
940 end
941
942 if not no_delete and not exec_redis_commands(src_conn, src_del_cmds) then
943 return total_tokens, true
944 end
945 end
946
947 return total_tokens, false
948 end
949
950 append_redis_hash_hmset = function(cmds, key, hash_data)
951 if hash_data and #hash_data > 0 then
952 local args = { key }
953 for _, v in ipairs(hash_data) do
954 args[#args + 1] = v
955 end
956 cmds[#cmds + 1] = { 'HMSET', args }
957 return true
958 end
959
960 return false
961 end
962
963 exec_redis_commands = function(conn, cmds)
964 if #cmds == 0 then
965 return true
966 end
967
968 for i = 1, #cmds, pipeline_max do
969 local chunk_end = math.min(i + pipeline_max - 1, #cmds)
970
971 for j = i, chunk_end do
972 local is_ok, err = conn:add_cmd(cmds[j][1], cmds[j][2])
973
974 if not is_ok then
975 rspamd_logger.errx("cannot add command: %s with args: %s: %s",
976 cmds[j][1], cmds[j][2], err)
977 return false
978 end
979 end
980
981 local ret, err = conn:exec()
982 if not ret then
983 rspamd_logger.errx("cannot execute redis pipeline: %s", err)
984 return false
985 end
986 end
987
988 return true
989 end
990
991 local function migrate_prefix_group(prefixes, src_conn, dst_conn, sym_keys, batch_size, no_delete)
992 local stats = {
993 migrated = 0,
994 tokens = 0,
995 errors = 0,
996 }
997
998 if #prefixes == 0 then
999 return stats
1000 end
1001
1002 for i = 1, #prefixes, pipeline_max do
1003 local chunk_end = math.min(i + pipeline_max - 1, #prefixes)
1004
1005 for j = i, chunk_end do
1006 src_conn:add_cmd('HGETALL', { prefixes[j] })
1007 end
1008
1009 local all_results = { src_conn:exec() }
1010 local dst_meta_cmds = {}
1011 local dst_keys_cmds = {}
1012 local src_keys_cmds = {}
1013 local src_meta_del_cmds = {}
1014
1015 for j = i, chunk_end do
1016 local idx = (j - i) * 2 + 1
1017 local prefix = prefixes[j]
1018 local hret, hdata = all_results[idx], all_results[idx + 1]
1019
1020 if hret then
1021 append_redis_hash_hmset(dst_meta_cmds, prefix, hdata)
1022 dst_keys_cmds[#dst_keys_cmds + 1] = { 'SADD', { sym_keys, prefix } }
1023 if not no_delete then
1024 src_keys_cmds[#src_keys_cmds + 1] = { 'SREM', { sym_keys, prefix } }
1025 src_meta_del_cmds[#src_meta_del_cmds + 1] = { 'DEL', { prefix } }
1026 end
1027 stats.migrated = stats.migrated + 1
1028 else
1029 rspamd_logger.errx("cannot get prefix metadata for %s", prefix)
1030 stats.errors = stats.errors + 1
1031 end
1032 end
1033
1034 all_results = nil
1035
1036 if not exec_redis_commands(dst_conn, dst_meta_cmds) then
1037 stats.errors = stats.errors + (chunk_end - i + 1)
1038 return stats
1039 end
1040
1041 local chunk_prefixes = {}
1042 for j = i, chunk_end do
1043 chunk_prefixes[#chunk_prefixes + 1] = prefixes[j]
1044 end
1045
1046 local token_keys, scan_error = collect_prefix_token_keys(src_conn, chunk_prefixes, batch_size)
1047 if scan_error then
1048 stats.errors = stats.errors + #chunk_prefixes
1049 return stats
1050 end
1051
1052 if token_keys and #token_keys > 0 then
1053 local tok_count, had_error = migrate_token_keys(src_conn, dst_conn, token_keys, no_delete)
1054 stats.tokens = stats.tokens + tok_count
1055
1056 if had_error then
1057 stats.errors = stats.errors + #chunk_prefixes
1058 return stats
1059 end
1060 end
1061
1062 if not exec_redis_commands(dst_conn, dst_keys_cmds) then
1063 stats.errors = stats.errors + (chunk_end - i + 1)
1064 return stats
1065 end
1066
1067 if not no_delete then
1068 if not exec_redis_commands(src_conn, src_keys_cmds) then
1069 stats.errors = stats.errors + (chunk_end - i + 1)
1070 return stats
1071 end
1072
1073 if not exec_redis_commands(src_conn, src_meta_del_cmds) then
1074 stats.errors = stats.errors + (chunk_end - i + 1)
1075 return stats
1076 end
1077 end
1078 end
1079
1080 return stats
1081 end
1082
1083 local function migrate_handler(opts)
1084 local selected = select_classifier(opts)
1085 local stats = {
1086 checked = 0,
1087 correct = 0,
1088 migrated = 0,
1089 tokens = 0,
1090 errors = 0,
1091 }
1092
1093 for _, cls in ipairs(selected) do
1094 local write_servers = cls.redis_params.write_servers
1095 if not write_servers then
1096 rspamd_logger.errx("no write servers configured, cannot migrate")
1097 os.exit(1)
1098 end
1099
1100 local all_ups = write_servers:all_upstreams()
1101 if not all_ups or #all_ups <= 1 then
1102 rspamd_logger.messagex("only %s shard(s) configured, nothing to migrate",
1103 all_ups and #all_ups or 0)
1104 return
1105 end
1106
1107 rspamd_logger.messagex("found %s shards to check for migration", #all_ups)
1108
1109 -- Connect to every shard
1110 local shard_map = {}
1111 for _, up in ipairs(all_ups) do
1112 local res, conn = connect_to_upstream(up, cls.redis_params)
1113 if not res then
1114 rspamd_logger.errx("cannot connect to shard %s, aborting", up:get_name())
1115 os.exit(1)
1116 end
1117 shard_map[#shard_map + 1] = {
1118 name = up:get_name(),
1119 up = up,
1120 conn = conn,
1121 }
1122 end
1123
1124 -- Build name→shard index for fast lookup
1125 local shard_by_name = {}
1126 for _, shard in ipairs(shard_map) do
1127 shard_by_name[shard.name] = shard
1128 end
1129
1130 -- Phase 1: Collect all prefixes from all shards, determine migration plan
1131 for _, s in ipairs(cls.symbols) do
1132 local sym = s.symbol
1133 rspamd_logger.messagex("processing symbol: %s", sym)
1134 local sym_keys = string.format("%s_keys", sym)
1135
1136 -- Collect prefixes per shard and classify
1137 local misplaced = {} -- { {prefix, src_shard, dst_shard}, ... }
1138
1139 for shard_idx, shard in ipairs(shard_map) do
1140 shard.conn:add_cmd('SMEMBERS', { sym_keys })
1141 local ret, prefixes = shard.conn:exec()
1142
1143 if not ret then
1144 rspamd_logger.errx("cannot get %s from shard %s: %s",
1145 sym_keys, shard.name, prefixes)
1146 stats.errors = stats.errors + 1
1147 elseif prefixes and #prefixes > 0 then
1148 rspamd_logger.messagex(" shard %s [%s/%s]: %s prefix(es)",
1149 shard.name, shard_idx, #shard_map, #prefixes)
1150
1151 for _, prefix in ipairs(prefixes) do
1152 stats.checked = stats.checked + 1
1153 local target_up = write_servers:get_upstream_by_hash(prefix)
1154 local target_name = target_up:get_name()
1155
1156 if target_name == shard.name then
1157 stats.correct = stats.correct + 1
1158 else
1159 misplaced[#misplaced + 1] = {
1160 prefix = prefix,
1161 src = shard,
1162 dst = shard_by_name[target_name],
1163 }
1164 end
1165 end
1166 end
1167 end
1168
1169 if #misplaced == 0 then
1170 rspamd_logger.messagex(" all prefixes on correct shards")
1171 else
1172 rspamd_logger.messagex(" %s prefix(es) need migration", #misplaced)
1173 end
1174
1175 -- Phase 2: Migrate misplaced prefixes grouped by shard pair to reduce round-trips
1176 local grouped = {}
1177
1178 for pi, m in ipairs(misplaced) do
1179 if not m.dst then
1180 rspamd_logger.errx(" cannot find target shard for prefix '%s'", m.prefix)
1181 stats.errors = stats.errors + 1
1182 else
1183 rspamd_logger.messagex(" [%s/%s] '%s': %s -> %s",
1184 pi, #misplaced, m.prefix, m.src.name, m.dst.name)
1185
1186 stats.migrated = stats.migrated + 1
1187
1188 if not opts.dry_run then
1189 local group_key = string.format('%s\0%s', m.src.name, m.dst.name)
1190 local group = grouped[group_key]
1191
1192 if not group then
1193 group = {
1194 src = m.src,
1195 dst = m.dst,
1196 prefixes = {},
1197 }
1198 grouped[group_key] = group
1199 end
1200
1201 group.prefixes[#group.prefixes + 1] = m.prefix
1202 end
1203 end
1204
1205 if pi % 100 == 0 then
1206 collectgarbage('collect')
1207 end
1208 end
1209
1210 if not opts.dry_run then
1211 for _, group in pairs(grouped) do
1212 rspamd_logger.messagex(" migrating %s prefix(es): %s -> %s",
1213 #group.prefixes, group.src.name, group.dst.name)
1214 local group_stats = migrate_prefix_group(group.prefixes,
1215 group.src.conn, group.dst.conn, sym_keys, opts.batch_size, opts.no_delete)
1216 stats.tokens = stats.tokens + group_stats.tokens
1217 stats.errors = stats.errors + group_stats.errors
1218 end
1219 end
1220
1221 misplaced = nil
1222 collectgarbage('collect')
1223 end
1224 end
1225
1226 rspamd_logger.messagex("migration %s: checked=%s correct=%s migrated=%s tokens=%s errors=%s",
1227 opts.dry_run and "dry-run complete" or "complete",
1228 stats.checked, stats.correct, stats.migrated, stats.tokens, stats.errors)
1229 end
1230
1231 local function handler(args)
1232 local opts = parser:parse(args)
1233
1234 local command = opts.command or 'dump'
1235
1236 load_config(opts)
1237 rspamd_config:init_subsystem('stat')
1238
1239 local obj = rspamd_config:get_ucl()
1240
1241 local classifier = obj.classifier
1242
1243 if classifier then
1244 if classifier[1] then
1245 for _, cls in ipairs(classifier) do
1246 if cls.bayes then
1247 cls = cls.bayes
1248 end
1249 if cls.backend and cls.backend == 'redis' then
1250 check_redis_classifier(cls, obj)
1251 end
1252 end
1253 else
1254 if classifier.bayes then
1255
1256 classifier = classifier.bayes
1257 if classifier[1] then
1258 for _, cls in ipairs(classifier) do
1259 if cls.backend and cls.backend == 'redis' then
1260 check_redis_classifier(cls, obj)
1261 end
1262 end
1263 else
1264 if classifier.backend and classifier.backend == 'redis' then
1265 check_redis_classifier(classifier, obj)
1266 end
1267 end
1268 end
1269 end
1270 end
1271
1272 if type(opts.file) == 'string' then
1273 opts.file = { opts.file }
1274 elseif type(opts.file) == 'none' then
1275 opts.file = {}
1276 end
1277
1278 if command == 'dump' then
1279 dump_handler(opts)
1280 elseif command == 'restore' then
1281 restore_handler(opts)
1282 elseif command == 'migrate' then
1283 migrate_handler(opts)
1284 else
1285 parser:error('command %s is not implemented', command)
1286 end
1287 end
1288
1289 return {
1290 name = 'statistics_dump',
1291 aliases = { 'stat_dump', 'bayes_dump' },
1292 handler = handler,
1293 description = parser._description
1294 }