Debian Codegraph @

lualib/lua_maps.lua

raw
1 --[[[
2 -- @module lua_maps
3 -- This module contains helper functions for managing rspamd maps
4 --]]
5
6 --[[
7 Copyright (c) 2022, Vsevolod Stakhov <[email protected]>
8
9 Licensed under the Apache License, Version 2.0 (the "License");
10 you may not use this file except in compliance with the License.
11 You may obtain a copy of the License at
12
13 http://www.apache.org/licenses/LICENSE-2.0
14
15 Unless required by applicable law or agreed to in writing, software
16 distributed under the License is distributed on an "AS IS" BASIS,
17 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18 See the License for the specific language governing permissions and
19 limitations under the License.
20 ]]--
21
22 local rspamd_logger = require "rspamd_logger"
23 local T = require "lua_shape.core"
24 local lua_util = require "lua_util"
25
26 local exports = {}
27
28 local maps_cache = {}
29
30 local function map_hash_key(data, mtype)
31 local hash = require "rspamd_cryptobox_hash"
32 local st = hash.create_specific('xxh64')
33 st:update(data)
34 st:update(mtype)
35
36 return st:hex()
37 end
38
39 local function starts(where, st)
40 return string.sub(where, 1, string.len(st)) == st
41 end
42
43 local function cut_prefix(where, st)
44 return string.sub(where, #st + 1)
45 end
46
47 local function maybe_adjust_type(data, mtype)
48 local function check_prefix(prefix, t)
49 if starts(data, prefix) then
50 data = cut_prefix(data, prefix)
51 mtype = t
52
53 return true
54 end
55
56 return false
57 end
58
59 local known_types = {
60 { 'regexp;', 'regexp' },
61 { 're;', 'regexp' },
62 { 'regexp_multi;', 'regexp_multi' },
63 { 're_multi;', 'regexp_multi' },
64 { 'glob;', 'glob' },
65 { 'glob_multi;', 'glob_multi' },
66 { 'radix;', 'radix' },
67 { 'ipnet;', 'radix' },
68 { 'set;', 'set' },
69 { 'hash;', 'hash' },
70 { 'plain;', 'hash' },
71 { 'cdb;', 'cdb' },
72 { 'cdb:/', 'cdb' },
73 }
74
75 if mtype == 'callback' then
76 return data, mtype
77 end
78
79 for _, t in ipairs(known_types) do
80 if check_prefix(t[1], t[2]) then
81 return data, mtype
82 end
83 end
84
85 -- No change
86 return data, mtype
87 end
88
89 local external_map_schema = T.table({
90 external = T.literal(true), -- must be true
91 backend = T.string():optional(), -- where to get data, required for HTTP
92 cdb = T.string():optional(), -- path to CDB file, required for CDB
93 method = T.enum({ "body", "header", "query" }):optional(), -- how to pass input
94 encode = T.enum({ "json", "messagepack" }):optional(), -- how to encode input (if relevant)
95 timeout = T.one_of({
96 T.number({ min = 0 }),
97 T.transform(T.string(), lua_util.parse_time_interval)
98 }):optional(),
99 })
100
101 -- Storage for CDB instances
102 local cdb_maps = {}
103 local cdb_finisher_set = false
104
105 local rspamd_http = require "rspamd_http"
106 local ucl = require "ucl"
107
108 -- Function to handle CDB maps
109 local function handle_cdb_map(map_config, key, callback, task)
110 local rspamd_cdb = require "rspamd_cdb"
111 local hash_key = map_config.cdb
112
113 -- Check if we need to open the CDB file
114 if not cdb_maps[hash_key] then
115 local cdb_file = map_config.cdb
116 -- Provide ev_base to monitor changes
117 local cdb_handle = rspamd_cdb.open(cdb_file, task:get_ev_base())
118
119 if not cdb_handle then
120 local err_msg = string.format("Failed to open CDB file: %s", cdb_file)
121 rspamd_logger.errx(task, err_msg)
122 if callback then
123 callback(false, err_msg, 500, task)
124 end
125 return nil
126 else
127 cdb_maps[hash_key] = cdb_handle
128 end
129 end
130
131 -- Look up the key in CDB
132 local result = cdb_maps[hash_key]:find(key)
133
134 if callback then
135 if result then
136 callback(true, result, 200, task)
137 else
138 callback(false, 'not found', 404, task)
139 end
140 return nil
141 end
142
143 return result
144 end
145
146 -- Query external map using HTTP or CDB
147 -- task_or_ctx can be either a task object or a context table with:
148 -- { config, ev_base, session, resolver } for rspamadm usage
149 -- If callback is nil and task_or_ctx is a context table (rspamadm), performs synchronous request
150 local function query_external_map(map_config, upstreams, key, callback, task_or_ctx)
151 -- Check if this is a CDB map
152 if map_config.cdb then
153 return handle_cdb_map(map_config, key, callback, task_or_ctx)
154 end
155 -- Fallback to HTTP
156 local http_method = (map_config.method == 'body' or map_config.method == 'form') and 'POST' or 'GET'
157 local upstream = upstreams:get_upstream_round_robin()
158 local http_headers = {
159 ['Accept'] = '*/*'
160 }
161 local http_body = nil
162 local url = map_config.backend
163
164 -- Determine logging target (task or config)
165 local log_obj = task_or_ctx
166 if type(task_or_ctx) == 'table' and task_or_ctx.config then
167 log_obj = task_or_ctx.config
168 end
169
170 if type(key) == 'string' or type(key) == 'userdata' then
171 if map_config.method == 'body' then
172 http_body = key
173 http_headers['Content-Type'] = 'text/plain'
174 elseif map_config.method == 'header' then
175 http_headers = {
176 key = key
177 }
178 elseif map_config.method == 'query' then
179 url = string.format('%s?key=%s', url, lua_util.url_encode_string(tostring(key)))
180 end
181 elseif type(key) == 'table' then
182 if map_config.method == 'body' then
183 if map_config.encode == 'json' then
184 http_body = ucl.to_format(key, 'json-compact', true)
185 http_headers['Content-Type'] = 'application/json'
186 elseif map_config.encode == 'messagepack' then
187 http_body = ucl.to_format(key, 'messagepack', true)
188 http_headers['Content-Type'] = 'application/msgpack'
189 else
190 local caller = debug.getinfo(2) or {}
191 rspamd_logger.errx(log_obj,
192 "requested external map key with a wrong combination body method and missing encode; caller: %s:%s",
193 caller.short_src, caller.currentline)
194 if callback then
195 callback(false, 'invalid map usage', 500, task_or_ctx)
196 end
197 return nil
198 end
199 else
200 -- query/header and no encode
201 if map_config.method == 'query' then
202 local params_table = {}
203 for k, v in pairs(key) do
204 if type(v) == 'string' then
205 table.insert(params_table,
206 string.format('%s=%s', lua_util.url_encode_string(k), lua_util.url_encode_string(v)))
207 end
208 end
209 url = string.format('%s?%s', url, table.concat(params_table, '&'))
210 elseif map_config.method == 'header' then
211 http_headers = key
212 else
213 local caller = debug.getinfo(2) or {}
214 rspamd_logger.errx(log_obj,
215 "requested external map key with a wrong combination of encode and input; caller: %s:%s",
216 caller.short_src, caller.currentline)
217 if callback then
218 callback(false, 'invalid map usage', 500, task_or_ctx)
219 end
220 return nil
221 end
222 end
223 end
224
225 -- Build HTTP request options - support both task and rspamadm context
226 local http_opts = {
227 url = url,
228 timeout = map_config.timeout or 1.0,
229 keepalive = true,
230 upstream = upstream,
231 method = http_method,
232 headers = http_headers,
233 body = http_body,
234 }
235
236 -- Check if task_or_ctx is a context table (rspamadm) or a task (userdata)
237 -- rspamadm context is a Lua table with ev_base, config, session, resolver fields
238 -- task is userdata (C object), so type(task) ~= 'table'
239 local is_rspamadm_ctx = type(task_or_ctx) == 'table' and task_or_ctx.ev_base and task_or_ctx.config
240 if is_rspamadm_ctx then
241 -- rspamadm context
242 http_opts.config = task_or_ctx.config
243 http_opts.ev_base = task_or_ctx.ev_base
244 http_opts.session = task_or_ctx.session
245 http_opts.resolver = task_or_ctx.resolver
246 elseif task_or_ctx then
247 -- Regular task (userdata)
248 http_opts.task = task_or_ctx
249 end
250
251 -- If no callback and rspamadm context, use coroutine-based synchronous mode
252 if not callback and is_rspamadm_ctx then
253 local err, response = rspamd_http.request(http_opts)
254 if err then
255 return nil
256 elseif response and response.code == 200 then
257 return response.content
258 else
259 return nil
260 end
261 end
262
263 -- Async mode with callback
264 local function map_callback(err, code, body, _)
265 if err then
266 callback(false, err, code, task_or_ctx)
267 elseif code == 200 then
268 callback(true, body, 200, task_or_ctx)
269 else
270 callback(false, err, code, task_or_ctx)
271 end
272 end
273
274 http_opts.callback = map_callback
275 local ret = rspamd_http.request(http_opts)
276
277 if not ret then
278 callback(false, 'http request error', 500, task_or_ctx)
279 end
280 end
281
282 --[[[
283 -- @function lua_maps.map_add_from_ucl(opt, mtype, description)
284 -- Creates a map from static data
285 -- Returns true if map was added or nil
286 -- @param {string or table} opt data for map (or URL)
287 -- @param {string} mtype type of map (`set`, `map`, `radix`, `regexp`)
288 -- @param {string} description human-readable description of map
289 -- @param {function} callback optional callback that will be called on map match (required for external maps)
290 -- @return {bool} true on success, or `nil`
291 --]]
292 local function rspamd_map_add_from_ucl(opt, mtype, description, callback)
293 local ret = {
294 -- get_key supports both task (userdata) and rspamadm context (table with ev_base, config, session, resolver)
295 -- For external maps with rspamadm context (no callback), uses coroutine-based synchronous request
296 get_key = function(t, k, key_callback, task_or_ctx)
297 if t.__data then
298 local cb = key_callback or callback
299 if t.__external then
300 -- Check if this is rspamadm context with no callback - use sync mode
301 -- rspamadm context is a Lua table; task is userdata (C object)
302 local is_rspamadm_ctx = type(task_or_ctx) == 'table' and task_or_ctx.ev_base and task_or_ctx.config
303 if not cb and is_rspamadm_ctx then
304 -- Coroutine-based synchronous external map query for rspamadm
305 return query_external_map(t.__data, t.__upstreams, k, nil, task_or_ctx)
306 elseif not cb or not task_or_ctx then
307 local caller = debug.getinfo(2) or {}
308 rspamd_logger.errx(rspamd_config, "requested external map key without callback or task/context; caller: %s:%s",
309 caller.short_src, caller.currentline)
310 return nil
311 else
312 query_external_map(t.__data, t.__upstreams, k, cb, task_or_ctx)
313 end
314 else
315 local result = t.__data:get_key(k)
316 if cb then
317 if result then
318 cb(true, result, 200, task_or_ctx)
319 else
320 cb(false, 'not found', 404, task_or_ctx)
321 end
322 else
323 return result
324 end
325 end
326 end
327
328 return nil
329 end,
330 foreach = function(t, cb)
331 return t.__data:foreach(cb)
332 end,
333 on_load = function(t, cb)
334 t.__data:on_load(cb)
335 end
336 }
337 local ret_mt = {
338 __index = function(t, k, key_callback, task_or_ctx)
339 if t.__data then
340 return t.get_key(k, key_callback, task_or_ctx)
341 end
342
343 return nil
344 end
345 }
346
347 if not opt then
348 return nil
349 end
350
351 local function maybe_register_selector()
352 if opt.selector_alias then
353 local lua_selectors = require "lua_selectors"
354 lua_selectors.add_map(opt.selector_alias, ret)
355 end
356 end
357
358 if type(opt) == 'string' then
359 opt, mtype = maybe_adjust_type(opt, mtype)
360 if not opt or opt == '' then
361 rspamd_logger.errx(rspamd_config, 'invalid map url (nil or empty): %s', description)
362 return nil
363 end
364 local cache_key = map_hash_key(opt, mtype)
365 if not callback and maps_cache[cache_key] then
366 rspamd_logger.infox(rspamd_config, 'reuse url for %s(%s)',
367 opt, mtype)
368
369 return maps_cache[cache_key]
370 end
371 -- We have a single string, so we treat it as a map
372 local map = rspamd_config:add_map {
373 type = mtype,
374 description = description,
375 url = opt,
376 }
377
378 if map then
379 ret.__data = map
380 ret.hash = cache_key
381 setmetatable(ret, ret_mt)
382 maps_cache[cache_key] = ret
383 return ret
384 end
385 elseif type(opt) == 'table' then
386 local cache_key = lua_util.unordered_table_digest(opt)
387 if not callback and maps_cache[cache_key] then
388 rspamd_logger.infox(rspamd_config, 'reuse url for complex map definition %s: %s',
389 cache_key:sub(1, 8), description)
390
391 return maps_cache[cache_key]
392 end
393
394 if opt[1] then
395 local function check_plain_map(line)
396 return lua_util.str_startswith(line, 'http')
397 or lua_util.str_startswith(line, 'file:')
398 or lua_util.str_startswith(line, '/')
399 end
400 -- Adjust each element if needed
401 local adjusted
402 for i, source in ipairs(opt) do
403 local nsrc, ntype = maybe_adjust_type(source, mtype)
404
405 if mtype ~= ntype then
406 if not adjusted then
407 mtype = ntype
408 end
409 adjusted = true
410 end
411 opt[i] = nsrc
412 end
413
414 if mtype == 'radix' then
415
416 if string.find(opt[1], '^%d') then
417 -- List of numeric stuff (hope it's ipnets definitions)
418 local map = rspamd_config:radix_from_ucl(opt, description)
419
420 if map then
421 ret.__data = map
422 setmetatable(ret, ret_mt)
423 maps_cache[cache_key] = ret
424 maybe_register_selector()
425
426 return ret
427 end
428 else
429 -- Plain table
430 local map = rspamd_config:add_map {
431 type = mtype,
432 description = description,
433 url = opt,
434 }
435 if map then
436 ret.__data = map
437 setmetatable(ret, ret_mt)
438 maps_cache[cache_key] = ret
439 maybe_register_selector()
440
441 return ret
442 end
443 end
444 elseif mtype == 'regexp' or mtype == 'glob' then
445 if check_plain_map(opt[1]) then
446 -- Plain table
447 local map = rspamd_config:add_map {
448 type = mtype,
449 description = description,
450 url = opt,
451 }
452 if map then
453 ret.__data = map
454 setmetatable(ret, ret_mt)
455 maps_cache[cache_key] = ret
456 maybe_register_selector()
457
458 return ret
459 end
460 else
461 local map = rspamd_config:add_map {
462 type = mtype,
463 description = description,
464 url = {
465 url = 'static',
466 data = opt,
467 }
468 }
469 if map then
470 ret.__data = map
471 setmetatable(ret, ret_mt)
472 maps_cache[cache_key] = ret
473 maybe_register_selector()
474
475 return ret
476 end
477 end
478 else
479 -- Not regexp/glob
480 if check_plain_map(opt[1]) then
481 -- Plain table
482 local map = rspamd_config:add_map {
483 type = mtype,
484 description = description,
485 url = opt,
486 }
487 if map then
488 ret.__data = map
489 setmetatable(ret, ret_mt)
490 maps_cache[cache_key] = ret
491 maybe_register_selector()
492
493 return ret
494 end
495 else
496 local data = {}
497 local nelts = 0
498 -- Plain array of keys, count merely numeric elts
499 for _, elt in ipairs(opt) do
500 if type(elt) == 'string' then
501 -- Numeric table
502 if mtype == 'hash' then
503 -- Treat as KV pair
504 local pieces = lua_util.str_split(elt, ' ')
505 if #pieces > 1 then
506 local key = table.remove(pieces, 1)
507 data[key] = table.concat(pieces, ' ')
508 else
509 data[elt] = true
510 end
511 else
512 data[elt] = true
513 end
514
515 nelts = nelts + 1
516 end
517 end
518
519 if nelts > 0 then
520 -- Plain Lua table that is used as a map
521 ret.__data = data
522 ret.get_key = function(t, k)
523 if k ~= '__data' then
524 return t.__data[k]
525 end
526
527 return nil
528 end
529 ret.foreach = function(_, func)
530 for k, v in pairs(ret.__data) do
531 if not func(k, v) then
532 return false
533 end
534 end
535
536 return true
537 end
538 ret.on_load = function(_, cb)
539 rspamd_config:add_on_load(function(_, _, _)
540 cb()
541 end)
542 end
543
544 maps_cache[cache_key] = ret
545 maybe_register_selector()
546
547 return ret
548 else
549 -- Empty map, huh?
550 rspamd_logger.errx(rspamd_config, 'invalid map element: %s',
551 opt)
552 end
553 end
554 end
555 else
556 if opt.external then
557 -- External map definition, missing fields are handled by schema
558 local parse_res, parse_err = external_map_schema:transform(opt)
559
560 if parse_res then
561 if opt.cdb then
562 ret.__data = opt
563 ret.__external = true
564 setmetatable(ret, ret_mt)
565 maybe_register_selector()
566
567 if not cdb_finisher_set then
568 -- Register a finalize script to close all CDB handles when Rspamd stops
569 rspamd_config:register_finish_script(function()
570 for path, _ in pairs(cdb_maps) do
571 rspamd_logger.infox(rspamd_config, 'closing CDB map: %s', path)
572 cdb_maps[path] = nil
573 end
574 end)
575 cdb_finisher_set = true
576 end
577
578 return ret
579 elseif opt.backend then
580 ret.__upstreams = lua_util.http_upstreams_by_url(rspamd_config:get_mempool(), opt.backend)
581 if ret.__upstreams then
582 ret.__data = opt
583 ret.__external = true
584 setmetatable(ret, ret_mt)
585 maybe_register_selector()
586
587 return ret
588 else
589 rspamd_logger.errx(rspamd_config, 'cannot parse external map upstreams: %s',
590 opt.backend)
591 end
592 else
593 rspamd_logger.errx(rspamd_config, 'external map requires either "cdb" or "backend" parameter')
594 end
595 else
596 rspamd_logger.errx(rspamd_config, 'cannot parse external map: %s',
597 parse_err)
598 end
599 else
600 -- Adjust lua specific augmentations in a trivial case
601 if type(opt.url) == 'string' then
602 local nsrc, ntype = maybe_adjust_type(opt.url, mtype)
603 if nsrc and ntype then
604 opt.url = nsrc
605 mtype = ntype
606 end
607 end
608 -- We have some non-trivial object so let C code to deal with it somehow...
609 local map = rspamd_config:add_map {
610 type = mtype,
611 description = description,
612 url = opt,
613 }
614 if map then
615 ret.__data = map
616 setmetatable(ret, ret_mt)
617 maps_cache[cache_key] = ret
618 maybe_register_selector()
619
620 return ret
621 end
622 end
623 end -- opt[1]
624 end
625
626 return nil
627 end
628
629 --[[[
630 -- @function lua_maps.map_add(mname, optname, mtype, description)
631 -- Creates a map from configuration elements (static data or URL)
632 -- Returns true if map was added or nil
633 -- @param {string} mname config section to use
634 -- @param {string} optname option name to use
635 -- @param {string} mtype type of map ('set', 'hash', 'radix', 'regexp', 'glob')
636 -- @param {string} description human-readable description of map
637 -- @param {function} callback optional callback that will be called on map match (required for external maps)
638 -- @return {bool} true on success, or `nil`
639 --]]
640
641 local function rspamd_map_add(mname, optname, mtype, description, callback)
642 local opt = rspamd_config:get_module_opt(mname, optname)
643
644 return rspamd_map_add_from_ucl(opt, mtype, description, callback)
645 end
646
647 exports.rspamd_map_add = rspamd_map_add
648 exports.map_add = rspamd_map_add
649 exports.rspamd_map_add_from_ucl = rspamd_map_add_from_ucl
650 exports.map_add_from_ucl = rspamd_map_add_from_ucl
651
652 -- Check `what` for being lua_map name, otherwise just compares key with what
653 local function rspamd_maybe_check_map(key, what)
654 local fun = require "fun"
655
656 if type(what) == "table" then
657 return fun.any(function(elt)
658 return rspamd_maybe_check_map(key, elt)
659 end, what)
660 end
661 if type(rspamd_maps) == "table" and starts(key, "map:") then
662 local mn = string.sub(key, 5)
663 if starts(mn, "//") then
664 mn = string.sub(mn, 3)
665 end
666 if rspamd_maps[mn] then
667 return rspamd_maps[mn]:get_key(what)
668 end
669 end
670
671 return what:lower() == key
672 end
673
674 exports.rspamd_maybe_check_map = rspamd_maybe_check_map
675
676 --[[[
677 -- @function lua_maps.fill_config_maps(mname, options, defs)
678 -- Fill maps that could be defined in defs, from the config in the options
679 -- Defs is a table indexed by a map's parameter name and defining it's config,
680 -- @example
681 -- defs = {
682 -- my_map = {
683 -- type = 'map',
684 -- description = 'my cool map',
685 -- optional = true,
686 -- }
687 -- }
688 -- -- Then this function will look for opts.my_map parameter and try to replace it with
689 -- -- a map with the specific type, description but not failing if it was empty.
690 -- -- It will also set options.my_map_orig to the original value defined in the map.
691 --]]
692 exports.fill_config_maps = function(mname, opts, map_defs)
693 assert(type(opts) == 'table')
694 assert(type(map_defs) == 'table')
695 for k, v in pairs(map_defs) do
696 if opts[k] then
697 local map = rspamd_map_add_from_ucl(opts[k], v.type or 'map', v.description)
698 if not map then
699 rspamd_logger.errx(rspamd_config, 'map add error %s for module %s', k, mname)
700 return false
701 end
702 opts[k .. '_orig'] = opts[k]
703 opts[k] = map
704 elseif not v.optional then
705 rspamd_logger.errx(rspamd_config, 'cannot find non optional map %s for module %s', k, mname)
706 return false
707 end
708 end
709
710 return true
711 end
712
713 local direct_map_schema = T.table({ -- complex object
714 name = T.string():optional(),
715 description = T.string():optional(),
716 selector_alias = T.string():optional(), -- an optional alias for the selectors framework
717 timeout = T.number(),
718 data = T.array(T.string()):optional(),
719 -- Can specify either upstreams or url
720 upstreams = T.one_of({
721 T.string(),
722 T.array(T.string()),
723 }):optional(),
724 url = T.one_of({
725 T.string(),
726 T.array(T.string()),
727 }):optional(),
728 })
729
730 exports.map_schema = T.one_of({
731 T.string(), -- 'http://some_map'
732 T.array(T.string()), -- ['foo', 'bar']
733 direct_map_schema, -- complex object with url/upstreams
734 external_map_schema -- external backend (HTTP API, etc)
735 })
736
737 return exports