Debian Codegraph @

lualib/lua_redis.lua

raw
1 --[[
2 Copyright (c) 2022, Vsevolod Stakhov <[email protected]>
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 ]]--
16
17 local logger = require "rspamd_logger"
18 local lutil = require "lua_util"
19 local rspamd_util = require "rspamd_util"
20 local T = require "lua_shape.core"
21 local PluginSchema = require "lua_shape.plugin_schema"
22
23 local exports = {}
24
25 local E = {}
26 local N = "lua_redis"
27
28 local db_schema = T.one_of({
29 T.transform(T.number(), tostring),
30 T.string()
31 }):optional():doc({ summary = "Database number" })
32
33 local common_schema = T.table({
34 timeout = T.one_of({
35 T.number({ min = 0 }),
36 T.transform(T.string(), lutil.parse_time_interval)
37 }):optional():doc({ summary = "Connection timeout (seconds)" }),
38 db = db_schema,
39 database = db_schema,
40 dbname = db_schema,
41 prefix = T.string():optional():doc({ summary = "Key prefix" }),
42 username = T.string():optional():doc({ summary = "Username" }),
43 password = T.string():optional():doc({ summary = "Password" }),
44 -- TLS options
45 ssl = T.boolean():optional():doc({ summary = "Enable TLS to Redis" }),
46 no_ssl_verify = T.boolean():optional():doc({ summary = "Disable TLS certificate verification" }),
47 ssl_ca = T.string():optional():doc({ summary = "CA certificate file" }),
48 ssl_ca_dir = T.string():optional():doc({ summary = "CA certificates directory" }),
49 ssl_cert = T.string():optional():doc({ summary = "Client certificate file (PEM)" }),
50 ssl_key = T.string():optional():doc({ summary = "Client private key file (PEM)" }),
51 sni = T.string():optional():doc({ summary = "SNI server name override" }),
52 expand_keys = T.boolean():optional():doc({ summary = "Expand keys" }),
53 sentinels = T.one_of({
54 T.string(),
55 T.array(T.string())
56 }):optional():doc({ summary = "Sentinel servers" }),
57 sentinel_watch_time = T.one_of({
58 T.number({ min = 0 }),
59 T.transform(T.string(), lutil.parse_time_interval)
60 }):optional():doc({ summary = "Sentinel watch time" }),
61 sentinel_masters_pattern = T.string():optional():doc({ summary = "Sentinel masters pattern" }),
62 sentinel_master_maxerrors = T.one_of({
63 T.number(),
64 T.transform(T.string(), tonumber)
65 }):optional():doc({ summary = "Sentinel master max errors" }),
66 sentinel_username = T.string():optional():doc({ summary = "Sentinel username" }),
67 sentinel_password = T.string():optional():doc({ summary = "Sentinel password" }),
68 redis_version = T.one_of({
69 T.number(),
70 T.transform(T.string(), tonumber)
71 }):optional():doc({ summary = "Redis server version (6 or 7)" }),
72 }, { open = true })
73
74 local function extend_fields(...)
75 local res = {}
76 for i = 1, select('#', ...) do
77 local tbl = select(i, ...)
78 if tbl then
79 for k, v in pairs(tbl) do
80 res[k] = v
81 end
82 end
83 end
84 return res
85 end
86
87 -- Register common schema as redis_common first
88 PluginSchema.register("mixins.redis_common", common_schema)
89
90 -- Create full redis mixin schema for documentation
91 local redis_mixin_schema = T.one_of({
92 {
93 name = "common_only",
94 schema = T.table({}, {
95 open = true,
96 mixins = { T.mixin(common_schema, { as = "redis_common" }) }
97 }):doc({ summary = "Redis with no explicit servers (uses localhost)" })
98 },
99 {
100 name = "read_servers",
101 schema = T.table({
102 read_servers = T.one_of({
103 T.string():doc({ summary = "Single Redis server" }),
104 T.array(T.string()):doc({ summary = "Multiple Redis servers" })
105 }):doc({ summary = "Read servers list" })
106 }, {
107 open = true,
108 mixins = { T.mixin(common_schema, { as = "redis_common" }) }
109 }):doc({ summary = "Redis with read-only servers" })
110 },
111 {
112 name = "write_servers",
113 schema = T.table({
114 write_servers = T.one_of({
115 T.string():doc({ summary = "Single Redis server" }),
116 T.array(T.string()):doc({ summary = "Multiple Redis servers" })
117 }):doc({ summary = "Write servers list" })
118 }, {
119 open = true,
120 mixins = { T.mixin(common_schema, { as = "redis_common" }) }
121 }):doc({ summary = "Redis with write-only servers" })
122 },
123 {
124 name = "rw_servers",
125 schema = T.table({
126 read_servers = T.one_of({
127 T.string():doc({ summary = "Single Redis server" }),
128 T.array(T.string()):doc({ summary = "Multiple Redis servers" })
129 }):doc({ summary = "Read servers list" }),
130 write_servers = T.one_of({
131 T.string():doc({ summary = "Single Redis server" }),
132 T.array(T.string()):doc({ summary = "Multiple Redis servers" })
133 }):doc({ summary = "Write servers list" })
134 }, {
135 open = true,
136 mixins = { T.mixin(common_schema, { as = "redis_common" }) }
137 }):doc({ summary = "Redis with separate read and write servers" })
138 },
139 {
140 name = "servers",
141 schema = T.table({
142 servers = T.one_of({
143 T.string():doc({ summary = "Single Redis server" }),
144 T.array(T.string()):doc({ summary = "Multiple Redis servers" })
145 }):doc({ summary = "Redis servers list" })
146 }, {
147 open = true,
148 mixins = { T.mixin(common_schema, { as = "redis_common" }) }
149 }):doc({ summary = "Redis with server list" })
150 },
151 {
152 name = "server_legacy",
153 schema = T.table({
154 server = T.one_of({
155 T.string():doc({ summary = "Single Redis server" }),
156 T.array(T.string()):doc({ summary = "Multiple Redis servers" })
157 }):doc({ summary = "Redis server (legacy)" })
158 }, {
159 open = true,
160 mixins = { T.mixin(common_schema, { as = "redis_common" }) }
161 }):doc({ summary = "Redis with legacy server parameter" })
162 }
163 }):doc({ summary = "Redis connection configuration" })
164
165 PluginSchema.register("mixins.redis", redis_mixin_schema)
166
167 local enrich_schema = function(external)
168 local external_fields = external or {}
169
170 return T.one_of({
171 {
172 name = "common_only",
173 schema = T.table(extend_fields(external_fields), {
174 open = true,
175 mixins = {
176 T.mixin(common_schema, { as = "redis" })
177 }
178 })
179 },
180 {
181 name = "read_servers",
182 schema = T.table(extend_fields(external_fields, {
183 read_servers = T.one_of({
184 T.string(),
185 T.array(T.string())
186 })
187 }), {
188 open = true,
189 mixins = {
190 T.mixin(common_schema, { as = "redis" })
191 }
192 })
193 },
194 {
195 name = "write_servers",
196 schema = T.table(extend_fields(external_fields, {
197 write_servers = T.one_of({
198 T.string(),
199 T.array(T.string())
200 })
201 }), {
202 open = true,
203 mixins = {
204 T.mixin(common_schema, { as = "redis" })
205 }
206 })
207 },
208 {
209 name = "rw_servers",
210 schema = T.table(extend_fields(external_fields, {
211 read_servers = T.one_of({
212 T.string(),
213 T.array(T.string())
214 }),
215 write_servers = T.one_of({
216 T.string(),
217 T.array(T.string())
218 })
219 }), {
220 open = true,
221 mixins = {
222 T.mixin(common_schema, { as = "redis" })
223 }
224 })
225 },
226 {
227 name = "servers",
228 schema = T.table(extend_fields(external_fields, {
229 servers = T.one_of({
230 T.string(),
231 T.array(T.string())
232 })
233 }), {
234 open = true,
235 mixins = {
236 T.mixin(common_schema, { as = "redis" })
237 }
238 })
239 },
240 {
241 name = "server_legacy",
242 schema = T.table(extend_fields(external_fields, {
243 server = T.one_of({
244 T.string(),
245 T.array(T.string())
246 })
247 }), {
248 open = true,
249 mixins = {
250 T.mixin(common_schema, { as = "redis" })
251 }
252 })
253 }
254 })
255 end
256
257 exports.enrich_schema = enrich_schema
258
259 local function redis_query_sentinel(ev_base, params, initialised)
260 local function flatten_redis_table(tbl)
261 local res = {}
262 for i = 1, #tbl, 2 do
263 res[tbl[i]] = tbl[i + 1]
264 end
265
266 return res
267 end
268 -- Coroutines syntax
269 local rspamd_redis = require "rspamd_redis"
270 local sentinels = params.sentinels
271 local addr = sentinels:get_upstream_round_robin()
272
273 local host = addr:get_addr()
274 local masters = {}
275 local process_masters -- Function that is called to process masters data
276
277 local function masters_cb(err, result)
278 if not err and result and type(result) == 'table' then
279
280 local pending_subrequests = 0
281
282 for _, m in ipairs(result) do
283 local master = flatten_redis_table(m)
284
285 -- Wrap IPv6-addresses in brackets
286 if (master.ip:match(":")) then
287 master.ip = "[" .. master.ip .. "]"
288 end
289
290 if params.sentinel_masters_pattern then
291 if master.name:match(params.sentinel_masters_pattern) then
292 lutil.debugm(N, 'found master %s with ip %s and port %s',
293 master.name, master.ip, master.port)
294 masters[master.name] = master
295 else
296 lutil.debugm(N, 'skip master %s with ip %s and port %s, pattern %s',
297 master.name, master.ip, master.port, params.sentinel_masters_pattern)
298 end
299 else
300 lutil.debugm(N, 'found master %s with ip %s and port %s',
301 master.name, master.ip, master.port)
302 masters[master.name] = master
303 end
304 end
305
306 -- For each master we need to get a list of slaves
307 for k, v in pairs(masters) do
308 v.slaves = {}
309 local function slaves_cb(slave_err, slave_result)
310 if not slave_err and type(slave_result) == 'table' then
311 for _, s in ipairs(slave_result) do
312 local slave = flatten_redis_table(s)
313 lutil.debugm(N, rspamd_config,
314 'found slave for master %s with ip %s and port %s',
315 v.name, slave.ip, slave.port)
316 -- Wrap IPv6-addresses in brackets
317 if (slave.ip:match(":")) then
318 slave.ip = "[" .. slave.ip .. "]"
319 end
320 v.slaves[#v.slaves + 1] = slave
321 end
322 else
323 logger.errx('cannot get slaves data from Redis Sentinel %s: %s',
324 host:to_string(true), slave_err)
325 addr:fail()
326 end
327
328 pending_subrequests = pending_subrequests - 1
329
330 if pending_subrequests == 0 then
331 -- Finalize masters and slaves
332 process_masters()
333 end
334 end
335
336 local ret = rspamd_redis.make_request {
337 host = addr:get_addr(),
338 timeout = params.timeout,
339 username = params.sentinel_username,
340 password = params.sentinel_password,
341 config = rspamd_config,
342 ev_base = ev_base,
343 cmd = 'SENTINEL',
344 args = { 'slaves', k },
345 no_pool = true,
346 callback = slaves_cb
347 }
348
349 if not ret then
350 logger.errx(rspamd_config, 'cannot connect sentinel when query slaves at address: %s',
351 host:to_string(true))
352 addr:fail()
353 else
354 pending_subrequests = pending_subrequests + 1
355 end
356 end
357
358 addr:ok()
359 else
360 logger.errx('cannot get masters data from Redis Sentinel %s: %s',
361 host:to_string(true), err)
362 addr:fail()
363 end
364 end
365
366 local ret = rspamd_redis.make_request {
367 host = addr:get_addr(),
368 timeout = params.timeout,
369 config = rspamd_config,
370 ev_base = ev_base,
371 username = params.sentinel_username,
372 password = params.sentinel_password,
373 cmd = 'SENTINEL',
374 args = { 'masters' },
375 no_pool = true,
376 callback = masters_cb,
377 }
378
379 if not ret then
380 logger.errx(rspamd_config, 'cannot connect sentinel at address: %s',
381 host:to_string(true))
382 addr:fail()
383 end
384
385 process_masters = function()
386 -- We now form new strings for masters and slaves
387 local read_servers_tbl, write_servers_tbl = {}, {}
388
389 for _, master in pairs(masters) do
390 write_servers_tbl[#write_servers_tbl + 1] = string.format(
391 '%s:%s', master.ip, master.port
392 )
393 read_servers_tbl[#read_servers_tbl + 1] = string.format(
394 '%s:%s', master.ip, master.port
395 )
396
397 for _, slave in ipairs(master.slaves) do
398 if slave['master-link-status'] == 'ok' then
399 read_servers_tbl[#read_servers_tbl + 1] = string.format(
400 '%s:%s', slave.ip, slave.port
401 )
402 end
403 end
404 end
405
406 table.sort(read_servers_tbl)
407 table.sort(write_servers_tbl)
408
409 local read_servers_str = table.concat(read_servers_tbl, ',')
410 local write_servers_str = table.concat(write_servers_tbl, ',')
411
412 lutil.debugm(N, rspamd_config,
413 'new servers list: %s read; %s write',
414 read_servers_str,
415 write_servers_str)
416
417 if read_servers_str ~= params.read_servers_str then
418 local upstream_list = require "rspamd_upstream_list"
419
420 local read_upstreams = upstream_list.create(rspamd_config,
421 read_servers_str, 6379)
422
423 if read_upstreams then
424 logger.infox(rspamd_config, 'sentinel %s: replace read servers with new list: %s',
425 host:to_string(true), read_servers_str)
426 params.read_servers = read_upstreams
427 params.read_servers_str = read_servers_str
428 end
429 end
430
431 if write_servers_str ~= params.write_servers_str then
432 local upstream_list = require "rspamd_upstream_list"
433
434 local write_upstreams = upstream_list.create(rspamd_config,
435 write_servers_str, 6379)
436
437 if write_upstreams then
438 logger.infox(rspamd_config, 'sentinel %s: replace write servers with new list: %s',
439 host:to_string(true), write_servers_str)
440 params.write_servers = write_upstreams
441 params.write_servers_str = write_servers_str
442
443 local queried = false
444
445 local function monitor_failures(up, _, count)
446 if count > params.sentinel_master_maxerrors and not queried then
447 logger.infox(rspamd_config, 'sentinel: master with address %s, caused %s failures, try to query sentinel',
448 host:to_string(true), count)
449 queried = true -- Avoid multiple checks caused by this monitor
450 redis_query_sentinel(ev_base, params, true)
451 end
452 end
453
454 write_upstreams:add_watcher('failure', monitor_failures)
455 end
456 end
457 end
458
459 end
460
461 local function add_redis_sentinels(params)
462 local upstream_list = require "rspamd_upstream_list"
463
464 local upstreams_sentinels = upstream_list.create(rspamd_config,
465 params.sentinels, 5000)
466
467 if not upstreams_sentinels then
468 logger.errx(rspamd_config, 'cannot load redis sentinels string: %s',
469 params.sentinels)
470
471 return
472 end
473
474 params.sentinels = upstreams_sentinels
475
476 if not params.sentinel_watch_time then
477 params.sentinel_watch_time = 60 -- Each minute
478 end
479
480 if not params.sentinel_master_maxerrors then
481 params.sentinel_master_maxerrors = 2 -- Maximum number of errors before rechecking
482 end
483
484 rspamd_config:add_on_load(function(_, ev_base, worker)
485 local initialised = false
486 if worker:is_scanner() or worker:get_type() == 'fuzzy' then
487 rspamd_config:add_periodic(ev_base, 0.0, function()
488 redis_query_sentinel(ev_base, params, initialised)
489 initialised = true
490
491 return params.sentinel_watch_time
492 end, false)
493 end
494 end)
495 end
496
497 local cached_results = {}
498
499 local function calculate_redis_hash(params)
500 local cr = require "rspamd_cryptobox_hash"
501
502 local h = cr.create()
503
504 local function rec_hash(k, v)
505 if type(v) == 'string' then
506 h:update(k)
507 h:update(v)
508 elseif type(v) == 'number' then
509 h:update(k)
510 h:update(tostring(v))
511 elseif type(v) == 'table' then
512 for kk, vv in pairs(v) do
513 rec_hash(kk, vv)
514 end
515 end
516 end
517
518 rec_hash('top', params)
519
520 return h:base32()
521 end
522
523 local function process_redis_opts(options, redis_params)
524 local default_timeout = 1.0
525 local default_expand_keys = false
526
527 if not redis_params['timeout'] or redis_params['timeout'] == default_timeout then
528 if options['timeout'] then
529 redis_params['timeout'] = tonumber(options['timeout'])
530 else
531 redis_params['timeout'] = default_timeout
532 end
533 end
534
535 if options['prefix'] and not redis_params['prefix'] then
536 redis_params['prefix'] = options['prefix']
537 end
538
539 if options['redis_version'] and not redis_params['redis_version'] then
540 redis_params['redis_version'] = tonumber(options['redis_version'])
541 end
542
543 if type(options['expand_keys']) == 'boolean' then
544 redis_params['expand_keys'] = options['expand_keys']
545 else
546 redis_params['expand_keys'] = default_expand_keys
547 end
548
549 if not redis_params['db'] then
550 if options['db'] then
551 redis_params['db'] = tostring(options['db'])
552 elseif options['dbname'] then
553 redis_params['db'] = tostring(options['dbname'])
554 elseif options['database'] then
555 redis_params['db'] = tostring(options['database'])
556 end
557 end
558 if options['username'] and not redis_params['username'] then
559 redis_params['username'] = options['username']
560 end
561 if options['password'] and not redis_params['password'] then
562 redis_params['password'] = options['password']
563 end
564
565 -- TLS passthrough
566 if options['ssl'] ~= nil and redis_params['ssl'] == nil then
567 redis_params['ssl'] = options['ssl'] and true or false
568 end
569 if options['no_ssl_verify'] ~= nil and redis_params['no_ssl_verify'] == nil then
570 redis_params['no_ssl_verify'] = options['no_ssl_verify'] and true or false
571 end
572 if options['ssl_ca'] and not redis_params['ssl_ca'] then
573 redis_params['ssl_ca'] = options['ssl_ca']
574 end
575 if options['ssl_ca_dir'] and not redis_params['ssl_ca_dir'] then
576 redis_params['ssl_ca_dir'] = options['ssl_ca_dir']
577 end
578 if options['ssl_cert'] and not redis_params['ssl_cert'] then
579 redis_params['ssl_cert'] = options['ssl_cert']
580 end
581 if options['ssl_key'] and not redis_params['ssl_key'] then
582 redis_params['ssl_key'] = options['ssl_key']
583 end
584 if options['sni'] and not redis_params['sni'] then
585 redis_params['sni'] = options['sni']
586 end
587
588 if not redis_params.sentinels and options.sentinels then
589 redis_params.sentinels = options.sentinels
590 end
591
592 if options['sentinel_masters_pattern'] and not redis_params['sentinel_masters_pattern'] then
593 redis_params['sentinel_masters_pattern'] = options['sentinel_masters_pattern']
594 end
595
596 if options['sentinel_watch_time'] and not redis_params['sentinel_watch_time'] then
597 redis_params['sentinel_watch_time'] = options['sentinel_watch_time']
598 end
599
600 if options['sentinel_username'] and not redis_params['sentinel_username'] then
601 redis_params['sentinel_username'] = options['sentinel_username']
602 end
603
604 if options['sentinel_password'] and not redis_params['sentinel_password'] then
605 redis_params['sentinel_password'] = options['sentinel_password']
606 end
607
608 if options['sentinel_master_maxerrors'] and not redis_params['sentinel_master_maxerrors'] then
609 redis_params['sentinel_master_maxerrors'] = options['sentinel_master_maxerrors']
610 end
611
612 end
613
614 local function enrich_defaults(rspamd_config, module, redis_params)
615 if rspamd_config then
616 local opts = rspamd_config:get_all_opt('redis')
617
618 if opts then
619 if module then
620 if opts[module] then
621 process_redis_opts(opts[module], redis_params)
622 end
623 end
624
625 process_redis_opts(opts, redis_params)
626 end
627 end
628 end
629
630 local function maybe_return_cached(redis_params)
631 local h = calculate_redis_hash(redis_params)
632
633 if cached_results[h] then
634 lutil.debugm(N, 'reused redis server: %s', redis_params)
635 return cached_results[h]
636 end
637
638 redis_params.hash = h
639 cached_results[h] = redis_params
640
641 if not redis_params.read_only and redis_params.sentinels then
642 add_redis_sentinels(redis_params)
643 end
644
645 lutil.debugm(N, 'loaded new redis server: %s', redis_params)
646 return redis_params
647 end
648
649 --[[[
650 -- @module lua_redis
651 -- This module contains helper functions for working with Redis
652 --]]
653 local function process_redis_options(options, rspamd_config, result)
654 local default_port = 6379
655 local upstream_list = require "rspamd_upstream_list"
656 local read_only = true
657
658 -- Try to get read servers:
659 local upstreams_read, upstreams_write
660
661 if options['read_servers'] then
662 if rspamd_config then
663 upstreams_read = upstream_list.create(rspamd_config,
664 options['read_servers'], default_port)
665 else
666 upstreams_read = upstream_list.create(options['read_servers'],
667 default_port)
668 end
669
670 result.read_servers_str = options['read_servers']
671 elseif options['servers'] then
672 if rspamd_config then
673 upstreams_read = upstream_list.create(rspamd_config,
674 options['servers'], default_port)
675 else
676 upstreams_read = upstream_list.create(options['servers'], default_port)
677 end
678
679 result.read_servers_str = options['servers']
680 read_only = false
681 elseif options['server'] then
682 if rspamd_config then
683 upstreams_read = upstream_list.create(rspamd_config,
684 options['server'], default_port)
685 else
686 upstreams_read = upstream_list.create(options['server'], default_port)
687 end
688
689 result.read_servers_str = options['server']
690 read_only = false
691 end
692
693 if upstreams_read then
694 if options['write_servers'] then
695 if rspamd_config then
696 upstreams_write = upstream_list.create(rspamd_config,
697 options['write_servers'], default_port)
698 else
699 upstreams_write = upstream_list.create(options['write_servers'],
700 default_port)
701 end
702 result.write_servers_str = options['write_servers']
703 read_only = false
704 elseif not read_only then
705 upstreams_write = upstreams_read
706 result.write_servers_str = result.read_servers_str
707 end
708 end
709
710 -- Store options
711 process_redis_opts(options, result)
712
713 if read_only and not upstreams_write then
714 result.read_only = true
715 elseif upstreams_write then
716 result.read_only = false
717 end
718
719 if upstreams_read then
720 result.read_servers = upstreams_read
721
722 if upstreams_write then
723 result.write_servers = upstreams_write
724 end
725
726 return true
727 end
728
729 lutil.debugm(N, rspamd_config,
730 'cannot load redis server from obj: %s, processed to %s',
731 options, result)
732
733 return false
734 end
735
736 --[[[
737 @function try_load_redis_servers(options, rspamd_config, no_fallback)
738 Tries to load redis servers from the specified `options` object.
739 Returns `redis_params` table or nil in case of failure
740
741 --]]
742 exports.try_load_redis_servers = function(options, rspamd_config, no_fallback, module_name)
743 local result = {}
744
745 if process_redis_options(options, rspamd_config, result) then
746 if not no_fallback then
747 enrich_defaults(rspamd_config, module_name, result)
748 end
749 return maybe_return_cached(result)
750 end
751 end
752
753 -- This function parses redis server definition using either
754 -- specific server string for this module or global
755 -- redis section
756 local function rspamd_parse_redis_server(module_name, module_opts, no_fallback)
757 local result = {}
758
759 -- Try local options
760 local opts
761 lutil.debugm(N, rspamd_config, 'try load redis config for: %s', module_name)
762 if not module_opts then
763 opts = rspamd_config:get_all_opt(module_name)
764 else
765 opts = module_opts
766 end
767
768 if opts then
769 local ret
770
771 if opts.redis then
772 ret = process_redis_options(opts.redis, rspamd_config, result)
773
774 if ret then
775 if not no_fallback then
776 enrich_defaults(rspamd_config, module_name, result)
777 end
778 return maybe_return_cached(result)
779 end
780 end
781
782 ret = process_redis_options(opts, rspamd_config, result)
783
784 if ret then
785 if not no_fallback then
786 enrich_defaults(rspamd_config, module_name, result)
787 end
788 return maybe_return_cached(result)
789 end
790 end
791
792 if no_fallback then
793 logger.infox(rspamd_config, "cannot find Redis definitions for %s and fallback is disabled",
794 module_name)
795
796 return nil
797 end
798
799 -- Try global options
800 opts = rspamd_config:get_all_opt('redis')
801
802 if opts then
803 local ret
804
805 if opts[module_name] then
806 ret = process_redis_options(opts[module_name], rspamd_config, result)
807
808 if ret then
809 return maybe_return_cached(result)
810 end
811 else
812 ret = process_redis_options(opts, rspamd_config, result)
813
814 -- Exclude disabled
815 if opts['disabled_modules'] then
816 for _, v in ipairs(opts['disabled_modules']) do
817 if v == module_name then
818 logger.infox(rspamd_config, "NOT using default redis server for module %s: it is disabled",
819 module_name)
820
821 return nil
822 end
823 end
824 end
825
826 if ret then
827 logger.infox(rspamd_config, "use default Redis settings for %s",
828 module_name)
829 return maybe_return_cached(result)
830 end
831 end
832 end
833
834 if result.read_servers then
835 return maybe_return_cached(result)
836 end
837
838 return nil
839 end
840
841 --[[[
842 -- @function lua_redis.parse_redis_server(module_name, module_opts, no_fallback)
843 -- Extracts Redis server settings from configuration
844 -- @param {string} module_name name of module to get settings for
845 -- @param {table} module_opts settings for module or `nil` to fetch them from configuration
846 -- @param {boolean} no_fallback should be `true` if global settings must not be used
847 -- @return {table} redis server settings
848 -- @example
849 -- local rconfig = lua_redis.parse_redis_server('my_module')
850 -- -- rconfig contains upstream_list objects in ['write_servers'] and ['read_servers']
851 -- -- ['timeout'] contains timeout in seconds
852 -- -- ['expand_keys'] if true tells that redis key expansion is enabled
853 --]]
854
855 exports.rspamd_parse_redis_server = rspamd_parse_redis_server
856 exports.parse_redis_server = rspamd_parse_redis_server
857
858 local process_cmd = {
859 bitop = function(args)
860 local idx_l = {}
861 for i = 2, #args do
862 table.insert(idx_l, i)
863 end
864 return idx_l
865 end,
866 blpop = function(args)
867 local idx_l = {}
868 for i = 1, #args - 1 do
869 table.insert(idx_l, i)
870 end
871 return idx_l
872 end,
873 eval = function(args)
874 local idx_l = {}
875 local numkeys = args[2]
876 if numkeys and tonumber(numkeys) >= 1 then
877 for i = 3, numkeys + 2 do
878 table.insert(idx_l, i)
879 end
880 end
881 return idx_l
882 end,
883 set = function(args)
884 return { 1 }
885 end,
886 mget = function(args)
887 local idx_l = {}
888 for i = 1, #args do
889 table.insert(idx_l, i)
890 end
891 return idx_l
892 end,
893 mset = function(args)
894 local idx_l = {}
895 for i = 1, #args, 2 do
896 table.insert(idx_l, i)
897 end
898 return idx_l
899 end,
900 sdiffstore = function(args)
901 local idx_l = {}
902 for i = 2, #args do
903 table.insert(idx_l, i)
904 end
905 return idx_l
906 end,
907 smove = function(args)
908 return { 1, 2 }
909 end,
910 script = function()
911 end
912 }
913 process_cmd.append = process_cmd.set
914 process_cmd.auth = process_cmd.script
915 process_cmd.bgrewriteaof = process_cmd.script
916 process_cmd.bgsave = process_cmd.script
917 process_cmd.bitcount = process_cmd.set
918 process_cmd.bitfield = process_cmd.set
919 process_cmd.bitpos = process_cmd.set
920 process_cmd.brpop = process_cmd.blpop
921 process_cmd.brpoplpush = process_cmd.blpop
922 process_cmd.client = process_cmd.script
923 process_cmd.cluster = process_cmd.script
924 process_cmd.command = process_cmd.script
925 process_cmd.config = process_cmd.script
926 process_cmd.dbsize = process_cmd.script
927 process_cmd.debug = process_cmd.script
928 process_cmd.decr = process_cmd.set
929 process_cmd.decrby = process_cmd.set
930 process_cmd.del = process_cmd.mget
931 process_cmd.discard = process_cmd.script
932 process_cmd.dump = process_cmd.set
933 process_cmd.echo = process_cmd.script
934 process_cmd.evalsha = process_cmd.eval
935 process_cmd.exec = process_cmd.script
936 process_cmd.exists = process_cmd.mget
937 process_cmd.expire = process_cmd.set
938 process_cmd.expireat = process_cmd.set
939 process_cmd.flushall = process_cmd.script
940 process_cmd.flushdb = process_cmd.script
941 process_cmd.geoadd = process_cmd.set
942 process_cmd.geohash = process_cmd.set
943 process_cmd.geopos = process_cmd.set
944 process_cmd.geodist = process_cmd.set
945 process_cmd.georadius = process_cmd.set
946 process_cmd.georadiusbymember = process_cmd.set
947 process_cmd.get = process_cmd.set
948 process_cmd.getbit = process_cmd.set
949 process_cmd.getrange = process_cmd.set
950 process_cmd.getset = process_cmd.set
951 process_cmd.hdel = process_cmd.set
952 process_cmd.hexists = process_cmd.set
953 process_cmd.hget = process_cmd.set
954 process_cmd.hgetall = process_cmd.set
955 process_cmd.hincrby = process_cmd.set
956 process_cmd.hincrbyfloat = process_cmd.set
957 process_cmd.hkeys = process_cmd.set
958 process_cmd.hlen = process_cmd.set
959 process_cmd.hmget = process_cmd.set
960 process_cmd.hmset = process_cmd.set
961 process_cmd.hscan = process_cmd.set
962 process_cmd.hset = process_cmd.set
963 process_cmd.hsetnx = process_cmd.set
964 process_cmd.hstrlen = process_cmd.set
965 process_cmd.hvals = process_cmd.set
966 process_cmd.incr = process_cmd.set
967 process_cmd.incrby = process_cmd.set
968 process_cmd.incrbyfloat = process_cmd.set
969 process_cmd.info = process_cmd.script
970 process_cmd.keys = process_cmd.script
971 process_cmd.lastsave = process_cmd.script
972 process_cmd.lindex = process_cmd.set
973 process_cmd.linsert = process_cmd.set
974 process_cmd.llen = process_cmd.set
975 process_cmd.lpop = process_cmd.set
976 process_cmd.lpush = process_cmd.set
977 process_cmd.lpushx = process_cmd.set
978 process_cmd.lrange = process_cmd.set
979 process_cmd.lrem = process_cmd.set
980 process_cmd.lset = process_cmd.set
981 process_cmd.ltrim = process_cmd.set
982 process_cmd.migrate = process_cmd.script
983 process_cmd.monitor = process_cmd.script
984 process_cmd.move = process_cmd.set
985 process_cmd.msetnx = process_cmd.mset
986 process_cmd.multi = process_cmd.script
987 process_cmd.object = process_cmd.script
988 process_cmd.persist = process_cmd.set
989 process_cmd.pexpire = process_cmd.set
990 process_cmd.pexpireat = process_cmd.set
991 process_cmd.pfadd = process_cmd.set
992 process_cmd.pfcount = process_cmd.set
993 process_cmd.pfmerge = process_cmd.mget
994 process_cmd.ping = process_cmd.script
995 process_cmd.psetex = process_cmd.set
996 process_cmd.psubscribe = process_cmd.script
997 process_cmd.pubsub = process_cmd.script
998 process_cmd.pttl = process_cmd.set
999 process_cmd.publish = process_cmd.script
1000 process_cmd.punsubscribe = process_cmd.script
1001 process_cmd.quit = process_cmd.script
1002 process_cmd.randomkey = process_cmd.script
1003 process_cmd.readonly = process_cmd.script
1004 process_cmd.readwrite = process_cmd.script
1005 process_cmd.rename = process_cmd.mget
1006 process_cmd.renamenx = process_cmd.mget
1007 process_cmd.restore = process_cmd.set
1008 process_cmd.role = process_cmd.script
1009 process_cmd.rpop = process_cmd.set
1010 process_cmd.rpoplpush = process_cmd.mget
1011 process_cmd.rpush = process_cmd.set
1012 process_cmd.rpushx = process_cmd.set
1013 process_cmd.sadd = process_cmd.set
1014 process_cmd.save = process_cmd.script
1015 process_cmd.scard = process_cmd.set
1016 process_cmd.sdiff = process_cmd.mget
1017 process_cmd.select = process_cmd.script
1018 process_cmd.setbit = process_cmd.set
1019 process_cmd.setex = process_cmd.set
1020 process_cmd.setnx = process_cmd.set
1021 process_cmd.sinterstore = process_cmd.sdiff
1022 process_cmd.sismember = process_cmd.set
1023 process_cmd.slaveof = process_cmd.script
1024 process_cmd.slowlog = process_cmd.script
1025 process_cmd.smembers = process_cmd.script
1026 process_cmd.sort = process_cmd.set
1027 process_cmd.spop = process_cmd.set
1028 process_cmd.srandmember = process_cmd.set
1029 process_cmd.srem = process_cmd.set
1030 process_cmd.strlen = process_cmd.set
1031 process_cmd.subscribe = process_cmd.script
1032 process_cmd.sunion = process_cmd.mget
1033 process_cmd.sunionstore = process_cmd.mget
1034 process_cmd.swapdb = process_cmd.script
1035 process_cmd.sync = process_cmd.script
1036 process_cmd.time = process_cmd.script
1037 process_cmd.touch = process_cmd.mget
1038 process_cmd.ttl = process_cmd.set
1039 process_cmd.type = process_cmd.set
1040 process_cmd.unsubscribe = process_cmd.script
1041 process_cmd.unlink = process_cmd.mget
1042 process_cmd.unwatch = process_cmd.script
1043 process_cmd.wait = process_cmd.script
1044 process_cmd.watch = process_cmd.mget
1045 process_cmd.zadd = process_cmd.set
1046 process_cmd.zcard = process_cmd.set
1047 process_cmd.zcount = process_cmd.set
1048 process_cmd.zincrby = process_cmd.set
1049 process_cmd.zinterstore = process_cmd.eval
1050 process_cmd.zlexcount = process_cmd.set
1051 process_cmd.zrange = process_cmd.set
1052 process_cmd.zrangebylex = process_cmd.set
1053 process_cmd.zrank = process_cmd.set
1054 process_cmd.zrem = process_cmd.set
1055 process_cmd.zrembylex = process_cmd.set
1056 process_cmd.zrembyrank = process_cmd.set
1057 process_cmd.zrembyscore = process_cmd.set
1058 process_cmd.zrevrange = process_cmd.set
1059 process_cmd.zrevrangebyscore = process_cmd.set
1060 process_cmd.zrevrank = process_cmd.set
1061 process_cmd.zscore = process_cmd.set
1062 process_cmd.zunionstore = process_cmd.eval
1063 process_cmd.scan = process_cmd.script
1064 process_cmd.sscan = process_cmd.set
1065 process_cmd.hscan = process_cmd.set
1066 process_cmd.zscan = process_cmd.set
1067
1068 local function get_key_indexes(cmd, args)
1069 local idx_l = {}
1070 cmd = string.lower(cmd)
1071 if process_cmd[cmd] then
1072 idx_l = process_cmd[cmd](args)
1073 else
1074 logger.warnx(rspamd_config, "Don't know how to extract keys for %s Redis command", cmd)
1075 end
1076 return idx_l
1077 end
1078
1079 local gen_meta = {
1080 principal_recipient = function(task)
1081 return task:get_principal_recipient()
1082 end,
1083 principal_recipient_domain = function(task)
1084 local p = task:get_principal_recipient()
1085 if not p then
1086 return
1087 end
1088 return string.match(p, '.*@(.*)')
1089 end,
1090 ip = function(task)
1091 local i = task:get_ip()
1092 if i and i:is_valid() then
1093 return i:to_string()
1094 end
1095 end,
1096 from = function(task)
1097 return ((task:get_from('smtp') or E)[1] or E)['addr']
1098 end,
1099 from_domain = function(task)
1100 return ((task:get_from('smtp') or E)[1] or E)['domain']
1101 end,
1102 from_domain_or_helo_domain = function(task)
1103 local d = ((task:get_from('smtp') or E)[1] or E)['domain']
1104 if d and #d > 0 then
1105 return d
1106 end
1107 return task:get_helo()
1108 end,
1109 mime_from = function(task)
1110 return ((task:get_from('mime') or E)[1] or E)['addr']
1111 end,
1112 mime_from_domain = function(task)
1113 return ((task:get_from('mime') or E)[1] or E)['domain']
1114 end,
1115 }
1116
1117 local function gen_get_esld(f)
1118 return function(task)
1119 local d = f(task)
1120 if not d then
1121 return
1122 end
1123 return rspamd_util.get_tld(d)
1124 end
1125 end
1126
1127 gen_meta.smtp_from = gen_meta.from
1128 gen_meta.smtp_from_domain = gen_meta.from_domain
1129 gen_meta.smtp_from_domain_or_helo_domain = gen_meta.from_domain_or_helo_domain
1130 gen_meta.esld_principal_recipient_domain = gen_get_esld(gen_meta.principal_recipient_domain)
1131 gen_meta.esld_from_domain = gen_get_esld(gen_meta.from_domain)
1132 gen_meta.esld_smtp_from_domain = gen_meta.esld_from_domain
1133 gen_meta.esld_mime_from_domain = gen_get_esld(gen_meta.mime_from_domain)
1134 gen_meta.esld_from_domain_or_helo_domain = gen_get_esld(gen_meta.from_domain_or_helo_domain)
1135 gen_meta.esld_smtp_from_domain_or_helo_domain = gen_meta.esld_from_domain_or_helo_domain
1136
1137 local function get_key_expansion_metadata(task)
1138
1139 local md_mt = {
1140 __index = function(self, k)
1141 k = string.lower(k)
1142 local v = rawget(self, k)
1143 if v then
1144 return v
1145 end
1146 if gen_meta[k] then
1147 v = gen_meta[k](task)
1148 rawset(self, k, v)
1149 end
1150 return v
1151 end,
1152 }
1153
1154 local lazy_meta = {}
1155 setmetatable(lazy_meta, md_mt)
1156 return lazy_meta
1157
1158 end
1159
1160 -- Performs async call to redis hiding all complexity inside function
1161 -- task - rspamd_task
1162 -- redis_params - valid params returned by rspamd_parse_redis_server
1163 -- key - key to select upstream or nil to select round-robin/master-slave
1164 -- is_write - true if need to write to redis server
1165 -- callback - function to be called upon request is completed
1166 -- command - redis command
1167 -- args - table of arguments
1168 -- extra_opts - table of optional request arguments
1169 local function rspamd_redis_make_request(task, redis_params, key, is_write,
1170 callback, command, args, extra_opts)
1171 local addr
1172 local function rspamd_redis_make_request_cb(err, data)
1173 if err then
1174 addr:fail()
1175 else
1176 addr:ok()
1177 end
1178 if callback then
1179 callback(err, data, addr)
1180 end
1181 end
1182 if not task or not redis_params or not command then
1183 return false, nil, nil
1184 end
1185
1186 local rspamd_redis = require "rspamd_redis"
1187
1188 if key then
1189 if is_write then
1190 addr = redis_params['write_servers']:get_upstream_by_hash(key)
1191 else
1192 addr = redis_params['read_servers']:get_upstream_by_hash(key)
1193 end
1194 else
1195 if is_write then
1196 addr = redis_params['write_servers']:get_upstream_master_slave(key)
1197 else
1198 addr = redis_params['read_servers']:get_upstream_round_robin(key)
1199 end
1200 end
1201
1202 if not addr then
1203 logger.errx(task, 'cannot select server to make redis request')
1204 end
1205
1206 if redis_params['expand_keys'] then
1207 local m = get_key_expansion_metadata(task)
1208 local indexes = get_key_indexes(command, args)
1209 for _, i in ipairs(indexes) do
1210 args[i] = lutil.template(args[i], m)
1211 end
1212 end
1213
1214 local ip_addr = addr:get_addr()
1215 local options = {
1216 task = task,
1217 callback = rspamd_redis_make_request_cb,
1218 host = ip_addr,
1219 timeout = redis_params['timeout'],
1220 cmd = command,
1221 args = args
1222 }
1223
1224 if extra_opts then
1225 for k, v in pairs(extra_opts) do
1226 options[k] = v
1227 end
1228 end
1229
1230 if redis_params['username'] then
1231 options['username'] = redis_params['username']
1232 end
1233
1234 if redis_params['password'] then
1235 options['password'] = redis_params['password']
1236 end
1237
1238 if redis_params['db'] then
1239 options['dbname'] = redis_params['db']
1240 end
1241
1242 -- TLS options
1243 if redis_params.ssl ~= nil then options.ssl = redis_params.ssl end
1244 if redis_params.no_ssl_verify ~= nil then options.no_ssl_verify = redis_params.no_ssl_verify end
1245 if redis_params.ssl_ca then options.ssl_ca = redis_params.ssl_ca end
1246 if redis_params.ssl_ca_dir then options.ssl_ca_dir = redis_params.ssl_ca_dir end
1247 if redis_params.ssl_cert then options.ssl_cert = redis_params.ssl_cert end
1248 if redis_params.ssl_key then options.ssl_key = redis_params.ssl_key end
1249 if redis_params.sni then options.sni = redis_params.sni end
1250
1251 lutil.debugm(N, task, 'perform request to redis server' ..
1252 ' (host=%s, timeout=%s): cmd: %s', ip_addr,
1253 options.timeout, options.cmd)
1254
1255 local ret, conn = rspamd_redis.make_request(options)
1256
1257 if not ret then
1258 addr:fail()
1259 logger.warnx(task, "cannot make redis request to: %s", tostring(ip_addr))
1260 end
1261
1262 return ret, conn, addr
1263 end
1264
1265 --[[[
1266 -- @function lua_redis.redis_make_request(task, redis_params, key, is_write, callback, command, args)
1267 -- Sends a request to Redis
1268 -- @param {rspamd_task} task task object
1269 -- @param {table} redis_params redis configuration in format returned by lua_redis.parse_redis_server()
1270 -- @param {string} key key to use for sharding
1271 -- @param {boolean} is_write should be `true` if we are performing a write operating
1272 -- @param {function} callback callback function (first parameter is error if applicable, second is a 2D array (table))
1273 -- @param {string} command Redis command to run
1274 -- @param {table} args Numerically indexed table containing arguments for command
1275 --]]
1276
1277 exports.rspamd_redis_make_request = rspamd_redis_make_request
1278 exports.redis_make_request = rspamd_redis_make_request
1279
1280 local function redis_make_request_taskless(ev_base, cfg, redis_params, key,
1281 is_write, callback, command, args, extra_opts)
1282 if not ev_base or not redis_params or not command then
1283 return false, nil, nil
1284 end
1285
1286 local addr
1287 local function rspamd_redis_make_request_cb(err, data)
1288 if err then
1289 addr:fail()
1290 else
1291 addr:ok()
1292 end
1293 if callback then
1294 callback(err, data, addr)
1295 end
1296 end
1297
1298 local rspamd_redis = require "rspamd_redis"
1299
1300 if key then
1301 if is_write then
1302 addr = redis_params['write_servers']:get_upstream_by_hash(key)
1303 else
1304 addr = redis_params['read_servers']:get_upstream_by_hash(key)
1305 end
1306 else
1307 if is_write then
1308 addr = redis_params['write_servers']:get_upstream_master_slave(key)
1309 else
1310 addr = redis_params['read_servers']:get_upstream_round_robin(key)
1311 end
1312 end
1313
1314 if not addr then
1315 logger.errx(cfg, 'cannot select server to make redis request')
1316 end
1317
1318 local options = {
1319 ev_base = ev_base,
1320 config = cfg,
1321 callback = rspamd_redis_make_request_cb,
1322 host = addr:get_addr(),
1323 timeout = redis_params['timeout'],
1324 cmd = command,
1325 args = args
1326 }
1327 if extra_opts then
1328 for k, v in pairs(extra_opts) do
1329 options[k] = v
1330 end
1331 end
1332
1333 if redis_params['username'] then
1334 options['username'] = redis_params['username']
1335 end
1336
1337 if redis_params['password'] then
1338 options['password'] = redis_params['password']
1339 end
1340
1341 if redis_params['db'] then
1342 options['dbname'] = redis_params['db']
1343 end
1344
1345 -- TLS options
1346 if redis_params.ssl ~= nil then options.ssl = redis_params.ssl end
1347 if redis_params.no_ssl_verify ~= nil then options.no_ssl_verify = redis_params.no_ssl_verify end
1348 if redis_params.ssl_ca then options.ssl_ca = redis_params.ssl_ca end
1349 if redis_params.ssl_ca_dir then options.ssl_ca_dir = redis_params.ssl_ca_dir end
1350 if redis_params.ssl_cert then options.ssl_cert = redis_params.ssl_cert end
1351 if redis_params.ssl_key then options.ssl_key = redis_params.ssl_key end
1352 if redis_params.sni then options.sni = redis_params.sni end
1353
1354 lutil.debugm(N, cfg, 'perform taskless request to redis server' ..
1355 ' (host=%s, timeout=%s): cmd: %s', options.host:tostring(true),
1356 options.timeout, options.cmd)
1357 local ret, conn = rspamd_redis.make_request(options)
1358 if not ret then
1359 logger.errx('cannot execute redis request')
1360 addr:fail()
1361 end
1362
1363 return ret, conn, addr
1364 end
1365
1366 --[[[
1367 -- @function lua_redis.redis_make_request_taskless(ev_base, cfg, redis_params, key, is_write, callback, command, args)
1368 -- Sends a request to Redis in context where `task` is not available for some specific use-cases
1369 -- Identical to redis_make_request() except in that first parameter is an `event base` object and the second one is the 'config' object
1370 --]]
1371
1372 exports.rspamd_redis_make_request_taskless = redis_make_request_taskless
1373 exports.redis_make_request_taskless = redis_make_request_taskless
1374
1375 local redis_scripts = {
1376 }
1377
1378 local function script_set_loaded(script)
1379 if script.sha then
1380 script.loaded = true
1381 end
1382
1383 script.pending_upload = false -- Allow further reload
1384
1385 local wait_table = {}
1386 for _, s in ipairs(script.waitq) do
1387 table.insert(wait_table, s)
1388 end
1389
1390 script.waitq = {}
1391
1392 for _, s in ipairs(wait_table) do
1393 s(script.loaded)
1394 end
1395 end
1396
1397 local function prepare_redis_call(script)
1398 local servers = {}
1399 local options = {}
1400
1401 if script.redis_params.read_servers then
1402 servers = lutil.table_merge(servers, script.redis_params.read_servers:all_upstreams())
1403 end
1404 if script.redis_params.write_servers then
1405 servers = lutil.table_merge(servers, script.redis_params.write_servers:all_upstreams())
1406 end
1407
1408 -- Call load script on each server, set loaded flag
1409 if not script.servers_ready then
1410 script.servers_ready = { } -- possible values for each server: unsent, tempfail, failed, done
1411 end
1412
1413 for idx, s in ipairs(servers) do
1414 local cur_opts = {
1415 host = s:get_addr(),
1416 timeout = script.redis_params['timeout'],
1417 cmd = 'SCRIPT',
1418 args = { 'LOAD', script.script },
1419 upstream = s
1420 }
1421
1422 -- By default we start from unsent status
1423 if not script.servers_ready[idx] then
1424 script.servers_ready[idx] = "unsent"
1425 end
1426
1427 if script.redis_params['username'] then
1428 cur_opts['username'] = script.redis_params['username']
1429 end
1430
1431 if script.redis_params['password'] then
1432 cur_opts['password'] = script.redis_params['password']
1433 end
1434
1435 if script.redis_params['db'] then
1436 cur_opts['dbname'] = script.redis_params['db']
1437 end
1438
1439 table.insert(options, cur_opts)
1440 end
1441
1442 return options
1443 end
1444
1445 local function is_any_server_ready(script)
1446 for _, s in ipairs(script.servers_ready) do
1447 if s == "done" then
1448 return true
1449 end
1450 end
1451 return false
1452 end
1453
1454 local function is_all_servers_failed(script)
1455 for _, s in ipairs(script.servers_ready) do
1456 if s == "unsent" or s == "tempfail" or s == "done" then
1457 return false
1458 end
1459 end
1460
1461 return true
1462 end
1463
1464 local function script_description(script)
1465 return script.filename and ("from file: " .. script.filename)
1466 or ("with id: " .. script.id)
1467 end
1468
1469 local function load_script_task(script, task, is_write)
1470 local rspamd_redis = require "rspamd_redis"
1471 local opts = prepare_redis_call(script)
1472
1473 for idx, opt in ipairs(opts) do
1474 if script.servers_ready[idx] ~= 'done' then
1475 opt.task = task
1476 opt.is_write = is_write
1477 opt.callback = function(err, data)
1478 if err then
1479 if string.match(err, 'ERR') then
1480 logger.errx(task, 'cannot upload script %s to %s: %s; registered from: %s:%s',
1481 script_description(script),
1482 opt.upstream:get_addr():to_string(true),
1483 err, script.caller.short_src, script.caller.currentline)
1484 opt.upstream:fail()
1485 script.servers_ready[idx] = "failed"
1486 return
1487 else
1488 -- Assume temporary error
1489 logger.infox(task, 'temporary error uploading script %s to %s: %s; registered from: %s:%s',
1490 script_description(script),
1491 opt.upstream:get_addr():to_string(true),
1492 err, script.caller.short_src, script.caller.currentline)
1493 script.servers_ready[idx] = "tempfail"
1494 return
1495 end
1496 else
1497 opt.upstream:ok()
1498 logger.infox(task,
1499 "uploaded redis script to %s %s, sha: %s",
1500 opt.upstream:get_addr():to_string(true),
1501 script_description(script), data)
1502 script.sha = data -- We assume that sha is the same on all servers
1503 script.servers_ready[idx] = "done"
1504 end
1505 if is_any_server_ready(script) then
1506 script_set_loaded(script)
1507 elseif is_all_servers_failed(script) then
1508 script.pending_upload = false
1509 script.fatal_error = "cannot upload script to any server"
1510 end
1511 end -- callback
1512
1513 local ret = rspamd_redis.make_request(opt)
1514
1515 if not ret then
1516 logger.errx('cannot execute redis request to load script on %s',
1517 opt.upstream:get_addr())
1518 script.servers_ready[idx] = "failed"
1519 opt.upstream:fail()
1520 end
1521 end
1522
1523 if is_any_server_ready(script) then
1524 script_set_loaded(script)
1525 elseif is_all_servers_failed(script) then
1526 script.pending_upload = false
1527 script.fatal_error = "cannot upload script to any server"
1528 end
1529 end
1530 end
1531
1532 local function load_script_taskless(script, cfg, ev_base, is_write)
1533 local rspamd_redis = require "rspamd_redis"
1534 local opts = prepare_redis_call(script)
1535
1536 for idx, opt in ipairs(opts) do
1537 if script.servers_ready[idx] ~= 'done' then
1538 opt.config = cfg
1539 opt.ev_base = ev_base
1540 opt.is_write = is_write
1541 opt.callback = function(err, data)
1542 if err then
1543 if string.match(err, 'ERR') then
1544 logger.errx(cfg, 'cannot upload script %s to %s: %s; registered from: %s:%s',
1545 script_description(script),
1546 opt.upstream:get_addr():to_string(true),
1547 err, script.caller.short_src, script.caller.currentline)
1548 opt.upstream:fail()
1549 script.servers_ready[idx] = "failed"
1550 else
1551 -- Assume temporary error
1552 logger.infox(cfg, 'temporary error uploading script %s to %s: %s; registered from: %s:%s',
1553 script_description(script),
1554 opt.upstream:get_addr():to_string(true),
1555 err, script.caller.short_src, script.caller.currentline)
1556 script.servers_ready[idx] = "tempfail"
1557 end
1558 else
1559 opt.upstream:ok()
1560 logger.infox(cfg,
1561 "uploaded redis script %s to %s, sha: %s",
1562 script_description(script),
1563 opt.upstream:get_addr():to_string(true),
1564 data)
1565 script.sha = data -- We assume that sha is the same on all servers
1566 script.servers_ready[idx] = "done"
1567 end
1568
1569 if is_any_server_ready(script) then
1570 script_set_loaded(script)
1571 elseif is_all_servers_failed(script) then
1572 script.pending_upload = false
1573 script.fatal_error = "cannot upload script to any server"
1574 end
1575 end
1576 local ret = rspamd_redis.make_request(opt)
1577
1578 if not ret then
1579 logger.errx('cannot execute redis request to load script %s on %s',
1580 script_description(script),
1581 opt.upstream:get_addr())
1582 script.servers_ready[idx] = "failed"
1583 opt.upstream:fail()
1584 end
1585 end
1586
1587 if is_any_server_ready(script) then
1588 script_set_loaded(script)
1589 elseif is_all_servers_failed(script) then
1590 script.pending_upload = false
1591 script.fatal_error = "cannot upload script " .. script_description(script) .. " to any server"
1592 end
1593 end
1594 end
1595
1596 local function load_redis_script(script, cfg, ev_base, _)
1597 if script.redis_params then
1598 load_script_taskless(script, cfg, ev_base)
1599 end
1600 end
1601
1602 local function add_redis_script(script, redis_params, caller_level, maybe_filename)
1603 if not caller_level then
1604 caller_level = 2
1605 end
1606 local caller = debug.getinfo(caller_level) or debug.getinfo(caller_level - 1) or E
1607
1608 local new_script = {
1609 caller = caller,
1610 loaded = false,
1611 redis_params = redis_params,
1612 script = script,
1613 waitq = {}, -- callbacks pending for script being loaded
1614 id = #redis_scripts + 1,
1615 filename = maybe_filename,
1616 }
1617
1618 -- Register on load function
1619 rspamd_config:add_on_load(function(cfg, ev_base, worker)
1620 local mult = 0.0
1621 rspamd_config:add_periodic(ev_base, 0.0, function()
1622 if not new_script.sha and not new_script.fatal_error then
1623 load_redis_script(new_script, cfg, ev_base, worker)
1624
1625 -- Do not wait for more than 10 seconds to retry
1626 if mult < 10 then
1627 mult = mult + 1
1628 end
1629 return 1.0 * mult -- Check one more time in one second
1630 end
1631
1632 -- All loaded, we are good to go
1633 return false
1634 end, false)
1635 end)
1636
1637 table.insert(redis_scripts, new_script)
1638
1639 return #redis_scripts
1640 end
1641 exports.add_redis_script = add_redis_script
1642
1643 -- Loads a Redis script from a file, strips comments, and passes the content to
1644 -- `add_redis_script` function.
1645 --
1646 -- @param filename The name of the file containing the Redis script.
1647 -- @param redis_params The Redis parameters to use for this script.
1648 -- @return The ID of the newly added Redis script.
1649 --
1650 local function load_redis_script_from_file(filename, redis_params, dir)
1651 local lua_util = require "lua_util"
1652
1653 if not dir then
1654 dir = rspamd_paths.LUALIBDIR
1655 end
1656 local path = filename
1657 if filename:sub(1, 1) ~= package.config:sub(1, 1) then
1658 -- Relative path
1659 path = lua_util.join_path(dir, "redis_scripts", filename)
1660 end
1661 -- Read file contents
1662 local file = io.open(path, "r")
1663 if not file then
1664 return nil, string.format("failed to open Redis script file: %s", path)
1665 end
1666 local script = file:read("*all")
1667 if not script then
1668 file:close()
1669 return nil, string.format("failed to read Redis script file: %s", path)
1670 end
1671 file:close()
1672 script = lua_util.strip_lua_comments(script)
1673
1674 return add_redis_script(script, redis_params, 3, filename)
1675 end
1676
1677 exports.load_redis_script_from_file = load_redis_script_from_file
1678
1679 local function exec_redis_script(id, params, callback, keys, args)
1680 local redis_args = {}
1681
1682 if not redis_scripts[id] then
1683 logger.errx("cannot find registered script with id %s", id)
1684 return false
1685 end
1686
1687 local script = redis_scripts[id]
1688
1689 if script.fatal_error then
1690 callback(script.fatal_error, nil)
1691 return true
1692 end
1693
1694 if not script.redis_params then
1695 callback('no redis servers defined', nil)
1696 return true
1697 end
1698
1699 local function do_call(can_reload)
1700 local function redis_cb(err, data)
1701 if not err then
1702 callback(err, data)
1703 elseif string.match(err, 'NOSCRIPT') then
1704 -- Schedule restart if possible
1705 if can_reload then
1706 table.insert(script.waitq, do_call)
1707 if not script.pending_upload then
1708 logger.infox(params.task or rspamd_config,
1709 'redis script %s is not loaded (NOSCRIPT returned), scheduling reload',
1710 script_description(script))
1711 script.sha = nil
1712 script.loaded = nil
1713 script.pending_upload = true
1714 -- We must initialize all servers as we don't know here which one failed
1715 for i, _ in ipairs(script.servers_ready) do
1716 script.servers_ready[i] = "unsent"
1717 end
1718 -- Reload scripts if this has not been initiated yet
1719 if params.task then
1720 load_script_task(script, params.task)
1721 else
1722 load_script_taskless(script, rspamd_config, params.ev_base)
1723 end
1724 else
1725 logger.infox(params.task or rspamd_config,
1726 'redis script %s is not ready (NOSCRIPT returned), waiting it to be loaded',
1727 script_description(script))
1728 end
1729 else
1730 callback(err, data)
1731 end
1732 else
1733 callback(err, data)
1734 end
1735 end
1736
1737 if #redis_args == 0 then
1738 table.insert(redis_args, script.sha)
1739 table.insert(redis_args, tostring(#keys))
1740 for _, k in ipairs(keys) do
1741 table.insert(redis_args, k)
1742 end
1743
1744 if type(args) == 'table' then
1745 for _, a in ipairs(args) do
1746 table.insert(redis_args, a)
1747 end
1748 end
1749 end
1750
1751 local redis_command = 'EVALSHA'
1752 if not params.is_write and script.redis_params.redis_version and
1753 script.redis_params.redis_version >= 7 then
1754 redis_command = 'EVALSHA_RO'
1755 end
1756 if params.task then
1757 if not rspamd_redis_make_request(params.task, script.redis_params,
1758 params.key, params.is_write, redis_cb, redis_command, redis_args) then
1759 callback('Cannot make redis request', nil)
1760 end
1761 else
1762 if not redis_make_request_taskless(params.ev_base, rspamd_config,
1763 script.redis_params,
1764 params.key, params.is_write, redis_cb, redis_command, redis_args) then
1765 callback('Cannot make redis request', nil)
1766 end
1767 end
1768 end
1769
1770 if script.loaded and script.sha then
1771 do_call(true)
1772 else
1773 table.insert(script.waitq, do_call)
1774 if not script.servers_ready then
1775 -- Delayed until scripts are loaded
1776 logger.infox(params.task or rspamd_config, 'redis script %s is not loaded, trying to load',
1777 script_description(script))
1778 -- Reload scripts if this has not been initiated yet
1779 if params.task then
1780 load_script_task(script, params.task)
1781 else
1782 load_script_taskless(script, rspamd_config, params.ev_base)
1783 end
1784 else
1785 -- Already loaded, but not yet ready
1786 logger.infox(params.task or rspamd_config, 'redis script %s is not ready, waiting it to be loaded',
1787 script_description(script))
1788 end
1789 end
1790
1791 return true
1792 end
1793
1794 exports.exec_redis_script = exec_redis_script
1795
1796 local function redis_connect_sync(redis_params, is_write, key, cfg, ev_base)
1797 if not redis_params then
1798 return false, nil
1799 end
1800
1801 local rspamd_redis = require "rspamd_redis"
1802 local addr
1803
1804 if key then
1805 if is_write then
1806 addr = redis_params['write_servers']:get_upstream_by_hash(key)
1807 else
1808 addr = redis_params['read_servers']:get_upstream_by_hash(key)
1809 end
1810 else
1811 if is_write then
1812 addr = redis_params['write_servers']:get_upstream_master_slave(key)
1813 else
1814 addr = redis_params['read_servers']:get_upstream_round_robin(key)
1815 end
1816 end
1817
1818 if not addr then
1819 logger.errx(cfg, 'cannot select server to make redis request')
1820 end
1821
1822 local options = {
1823 host = addr:get_addr(),
1824 timeout = redis_params['timeout'],
1825 config = cfg or rspamd_config,
1826 ev_base = ev_base or rspamadm_ev_base,
1827 session = redis_params.session or rspamadm_session
1828 }
1829
1830 for k, v in pairs(redis_params) do
1831 options[k] = v
1832 end
1833
1834 if not options.config then
1835 logger.errx('config is not set')
1836 return false, nil, addr
1837 end
1838
1839 if not options.ev_base then
1840 logger.errx('ev_base is not set')
1841 return false, nil, addr
1842 end
1843
1844 if not options.session then
1845 logger.errx('session is not set')
1846 return false, nil, addr
1847 end
1848
1849 local ret, conn = rspamd_redis.connect_sync(options)
1850 if not ret then
1851 logger.errx('cannot create redis connection: %s', conn)
1852 addr:fail()
1853
1854 return false, nil, addr
1855 end
1856
1857 if conn then
1858 local need_exec = false
1859 if redis_params['username'] then
1860 if redis_params['password'] then
1861 conn:add_cmd('AUTH', { redis_params['username'], redis_params['password'] })
1862 need_exec = true
1863 else
1864 logger.warnx('Redis requires a password when username is supplied')
1865 return false, nil, addr
1866 end
1867 elseif redis_params['password'] then
1868 conn:add_cmd('AUTH', { redis_params['password'] })
1869 need_exec = true
1870 end
1871
1872 if redis_params['db'] then
1873 conn:add_cmd('SELECT', { tostring(redis_params['db']) })
1874 need_exec = true
1875 elseif redis_params['dbname'] then
1876 conn:add_cmd('SELECT', { tostring(redis_params['dbname']) })
1877 need_exec = true
1878 end
1879
1880 if need_exec then
1881 local exec_ret, res = conn:exec()
1882
1883 if not exec_ret then
1884 logger.errx('cannot prepare redis connection (authentication or db selection failure): %s',
1885 res)
1886 addr:fail()
1887 return false, nil, addr
1888 end
1889 end
1890 end
1891
1892 return ret, conn, addr
1893 end
1894
1895 exports.redis_connect_sync = redis_connect_sync
1896
1897 --[[[
1898 -- @function lua_redis.request(redis_params, attrs, req)
1899 -- Sends a request to Redis synchronously with coroutines or asynchronously using
1900 -- a callback (modern API)
1901 -- @param redis_params a table of redis server parameters
1902 -- @param attrs a table of redis request attributes (e.g. task, or ev_base + cfg + session)
1903 -- @param req a table of request: a command + command options
1904 -- @return {result,data/connection,address} boolean result, connection object in case of async request and results if using coroutines, redis server address
1905 --]]
1906
1907 exports.request = function(redis_params, attrs, req)
1908 local lua_util = require "lua_util"
1909
1910 if not attrs or not redis_params or not req then
1911 logger.errx('invalid arguments for redis request')
1912 return false, nil, nil
1913 end
1914
1915 if not (attrs.task or (attrs.config and attrs.ev_base)) then
1916 logger.errx('invalid attributes for redis request')
1917 return false, nil, nil
1918 end
1919
1920 local opts = lua_util.shallowcopy(attrs)
1921
1922 local log_obj = opts.task or opts.config
1923
1924 local addr
1925
1926 if opts.callback then
1927 -- Wrap callback
1928 local callback = opts.callback
1929 local function rspamd_redis_make_request_cb(err, data)
1930 if err then
1931 addr:fail()
1932 else
1933 addr:ok()
1934 end
1935 callback(err, data, addr)
1936 end
1937 opts.callback = rspamd_redis_make_request_cb
1938 end
1939
1940 local rspamd_redis = require "rspamd_redis"
1941 local is_write = opts.is_write
1942
1943 if opts.key then
1944 if is_write then
1945 addr = redis_params['write_servers']:get_upstream_by_hash(attrs.key)
1946 else
1947 addr = redis_params['read_servers']:get_upstream_by_hash(attrs.key)
1948 end
1949 else
1950 if is_write then
1951 addr = redis_params['write_servers']:get_upstream_master_slave(attrs.key)
1952 else
1953 addr = redis_params['read_servers']:get_upstream_round_robin(attrs.key)
1954 end
1955 end
1956
1957 if not addr then
1958 logger.errx(log_obj, 'cannot select server to make redis request')
1959 end
1960
1961 opts.host = addr:get_addr()
1962 opts.timeout = redis_params.timeout
1963
1964 if type(req) == 'string' then
1965 opts.cmd = req
1966 else
1967 -- XXX: modifies the input table
1968 opts.cmd = table.remove(req, 1);
1969 opts.args = req
1970 end
1971
1972 if redis_params.username then
1973 opts.username = redis_params.username
1974 end
1975
1976 if redis_params.password then
1977 opts.password = redis_params.password
1978 end
1979
1980 if redis_params.db then
1981 opts.dbname = redis_params.db
1982 end
1983
1984 -- TLS options
1985 if redis_params.ssl ~= nil then opts.ssl = redis_params.ssl end
1986 if redis_params.no_ssl_verify ~= nil then opts.no_ssl_verify = redis_params.no_ssl_verify end
1987 if redis_params.ssl_ca then opts.ssl_ca = redis_params.ssl_ca end
1988 if redis_params.ssl_ca_dir then opts.ssl_ca_dir = redis_params.ssl_ca_dir end
1989 if redis_params.ssl_cert then opts.ssl_cert = redis_params.ssl_cert end
1990 if redis_params.ssl_key then opts.ssl_key = redis_params.ssl_key end
1991 if redis_params.sni then opts.sni = redis_params.sni end
1992
1993 if opts.callback then
1994 lutil.debugm(N, 'perform generic async request to redis server' ..
1995 ' (host=%s, timeout=%s): cmd: %s, arguments: %s', addr,
1996 opts.timeout, opts.cmd, opts.args)
1997 local ret, conn = rspamd_redis.make_request(opts)
1998 if not ret then
1999 logger.errx(log_obj, 'cannot execute redis request')
2000 addr:fail()
2001 end
2002
2003 return ret, conn, addr
2004 else
2005 -- Coroutines version
2006 lutil.debugm(N, 'perform generic coroutine request to redis server' ..
2007 ' (host=%s, timeout=%s): cmd: %s, arguments: %s', addr,
2008 opts.timeout, opts.cmd, opts.args)
2009 local ret, conn = rspamd_redis.connect_sync(opts)
2010 if not ret then
2011 logger.errx(log_obj, 'cannot execute redis request')
2012 addr:fail()
2013 else
2014 conn:add_cmd(opts.cmd, opts.args)
2015 return conn:exec()
2016 end
2017 return false, nil, addr
2018 end
2019 end
2020
2021 --[[[
2022 -- @function lua_redis.connect(redis_params, attrs)
2023 -- Connects to Redis synchronously with coroutines or asynchronously using a callback (modern API)
2024 -- @param redis_params a table of redis server parameters
2025 -- @param attrs a table of redis request attributes (e.g. task, or ev_base + cfg + session)
2026 -- @return {result,connection,address} boolean result, connection object, redis server address
2027 --]]
2028
2029 exports.connect = function(redis_params, attrs)
2030 local lua_util = require "lua_util"
2031
2032 if not attrs or not redis_params then
2033 logger.errx('invalid arguments for redis connect')
2034 return false, nil, nil
2035 end
2036
2037 if not (attrs.task or (attrs.config and attrs.ev_base)) then
2038 logger.errx('invalid attributes for redis connect')
2039 return false, nil, nil
2040 end
2041
2042 local opts = lua_util.shallowcopy(attrs)
2043
2044 local log_obj = opts.task or opts.config
2045
2046 local addr
2047
2048 if opts.callback then
2049 -- Wrap callback
2050 local callback = opts.callback
2051 local function rspamd_redis_make_request_cb(err, data)
2052 if err then
2053 addr:fail()
2054 else
2055 addr:ok()
2056 end
2057 callback(err, data, addr)
2058 end
2059 opts.callback = rspamd_redis_make_request_cb
2060 end
2061
2062 local rspamd_redis = require "rspamd_redis"
2063 local is_write = opts.is_write
2064
2065 if opts.key then
2066 if is_write then
2067 addr = redis_params['write_servers']:get_upstream_by_hash(attrs.key)
2068 else
2069 addr = redis_params['read_servers']:get_upstream_by_hash(attrs.key)
2070 end
2071 else
2072 if is_write then
2073 addr = redis_params['write_servers']:get_upstream_master_slave(attrs.key)
2074 else
2075 addr = redis_params['read_servers']:get_upstream_round_robin(attrs.key)
2076 end
2077 end
2078
2079 if not addr then
2080 logger.errx(log_obj, 'cannot select server to make redis connect')
2081 end
2082
2083 opts.host = addr:get_addr()
2084 opts.timeout = redis_params.timeout
2085
2086 if redis_params.username then
2087 opts.username = redis_params.username
2088 end
2089
2090 if redis_params.password then
2091 opts.password = redis_params.password
2092 end
2093
2094 if redis_params.db then
2095 opts.dbname = redis_params.db
2096 end
2097
2098 -- TLS options
2099 if redis_params.ssl ~= nil then opts.ssl = redis_params.ssl end
2100 if redis_params.no_ssl_verify ~= nil then opts.no_ssl_verify = redis_params.no_ssl_verify end
2101 if redis_params.ssl_ca then opts.ssl_ca = redis_params.ssl_ca end
2102 if redis_params.ssl_ca_dir then opts.ssl_ca_dir = redis_params.ssl_ca_dir end
2103 if redis_params.ssl_cert then opts.ssl_cert = redis_params.ssl_cert end
2104 if redis_params.ssl_key then opts.ssl_key = redis_params.ssl_key end
2105 if redis_params.sni then opts.sni = redis_params.sni end
2106
2107 if opts.callback then
2108 local ret, conn = rspamd_redis.connect(opts)
2109 if not ret then
2110 logger.errx(log_obj, 'cannot execute redis connect')
2111 addr:fail()
2112 end
2113
2114 return ret, conn, addr
2115 else
2116 -- Coroutines version
2117 local ret, conn = rspamd_redis.connect_sync(opts)
2118 if not ret then
2119 logger.errx(log_obj, 'cannot execute redis connect')
2120 addr:fail()
2121 else
2122 return true, conn, addr
2123 end
2124
2125 return false, nil, addr
2126 end
2127 end
2128
2129 local redis_prefixes = {}
2130
2131 --[[[
2132 -- @function lua_redis.register_prefix(prefix, module, description[, optional])
2133 -- Register new redis prefix for documentation purposes
2134 -- @param {string} prefix string prefix
2135 -- @param {string} module module name
2136 -- @param {string} description prefix description
2137 -- @param {table} optional optional kv pairs (e.g. pattern)
2138 --]]
2139 local function register_prefix(prefix, module, description, optional)
2140 local pr = {
2141 module = module,
2142 description = description
2143 }
2144
2145 if optional and type(optional) == 'table' then
2146 for k, v in pairs(optional) do
2147 pr[k] = v
2148 end
2149 end
2150
2151 redis_prefixes[prefix] = pr
2152 end
2153
2154 exports.register_prefix = register_prefix
2155
2156 --[[[
2157 -- @function lua_redis.prefixes([mname])
2158 -- Returns prefixes for specific module (or all prefixes). Returns a table prefix -> table
2159 --]]
2160 exports.prefixes = function(mname)
2161 if not mname then
2162 return redis_prefixes
2163 else
2164 local fun = require "fun"
2165
2166 return fun.totable(fun.filter(function(_, data)
2167 return data.module == mname
2168 end,
2169 redis_prefixes))
2170 end
2171 end
2172
2173 return exports