Debian Codegraph @

lualib/lua_selectors/init.lua

raw
1 --[[
2 Copyright (c) 2022, Vsevolod Stakhov <[email protected]>
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 ]]--
16
17 -- This module contains 'selectors' implementation: code to extract data
18 -- from Rspamd tasks and compose those together
19 --
20 -- Read more at https://rspamd.com/doc/configuration/selectors.html
21
22 --[[[
23 -- @module lua_selectors
24 -- This module contains 'selectors' implementation: code to extract data
25 -- from Rspamd tasks and compose those together.
26 -- Typical selector looks like this: header(User).lower.substring(1, 2):ip
27 --]]
28
29 local exports = {
30 maps = require "lua_selectors/maps"
31 }
32
33 local logger = require 'rspamd_logger'
34 local fun = require 'fun'
35 local lua_util = require "lua_util"
36 local M = "selectors"
37 local rspamd_text = require "rspamd_text"
38 local unpack_function = table.unpack or unpack
39 local E = {}
40
41 local extractors = require "lua_selectors/extractors"
42 local transform_function = require "lua_selectors/transforms"
43
44 local text_cookie = rspamd_text.cookie
45
46 local function pure_type(ltype)
47 return ltype:match('^(.*)_list$')
48 end
49
50 local function implicit_tostring(t, ud_or_table)
51 if t == 'table' then
52 -- Table (very special)
53 if ud_or_table.value then
54 return ud_or_table.value, 'string'
55 elseif ud_or_table.addr then
56 return ud_or_table.addr, 'string'
57 end
58
59 return logger.slog("%s", ud_or_table), 'string'
60 elseif (t == 'string' or t == 'text') and type(ud_or_table) == 'userdata' then
61 if ud_or_table.cookie and ud_or_table.cookie == text_cookie then
62 -- Preserve opaque
63 return ud_or_table, 'string'
64 else
65 return tostring(ud_or_table), 'string'
66 end
67 elseif t ~= 'nil' then
68 return tostring(ud_or_table), 'string'
69 end
70
71 return nil
72 end
73
74 local function process_selector(task, sel)
75 local function allowed_type(t)
76 if t == 'string' or t == 'string_list' then
77 return true
78 end
79
80 return false
81 end
82
83 local function list_type(t)
84 return pure_type(t)
85 end
86
87 local input, etype = sel.selector.get_value(task, sel.selector.args)
88
89 if not input then
90 lua_util.debugm(M, task, 'no value extracted for %s', sel.selector.name)
91 return nil
92 end
93
94 lua_util.debugm(M, task, 'extracted %s, type %s',
95 sel.selector.name, etype)
96
97 local pipe = sel.processor_pipe or E
98 local first_elt = pipe[1]
99
100 if first_elt and (first_elt.method or
101 fun.any(function(t)
102 return t == 'userdata' or t == 'table'
103 end, first_elt.types)) then
104 -- Explicit conversion
105 local meth = first_elt
106
107 if meth.types[etype] then
108 lua_util.debugm(M, task, 'apply method `%s` to %s',
109 meth.name, etype)
110 input, etype = meth.process(input, etype, meth.args)
111 else
112 local pt = pure_type(etype)
113
114 if meth.types[pt] then
115 lua_util.debugm(M, task, 'map method `%s` to list of %s',
116 meth.name, pt)
117 -- Map method to a list of inputs, excluding empty elements
118 -- We need to fold it down here to get a proper type resolution
119 input = fun.totable(fun.filter(function(map_elt, _)
120 return map_elt
121 end,
122 fun.map(function(list_elt)
123 local ret, ty = meth.process(list_elt, pt, meth.args)
124 if ret then
125 etype = ty
126 end
127 return ret
128 end, input)))
129 if input and etype then
130 etype = etype .. "_list"
131 else
132 input = nil
133 end
134 end
135 end
136 -- Remove method from the pipeline
137 pipe = fun.drop_n(1, pipe)
138 elseif etype:match('^userdata') or etype:match('^table') then
139 -- Implicit conversion
140 local pt = pure_type(etype)
141
142 if not pt then
143 lua_util.debugm(M, task, 'apply implicit conversion %s->string', etype)
144 input = implicit_tostring(etype, input)
145 etype = 'string'
146 else
147 lua_util.debugm(M, task, 'apply implicit map %s->string', pt)
148 input = fun.filter(function(map_elt)
149 return map_elt
150 end,
151 fun.map(function(list_elt)
152 local ret = implicit_tostring(pt, list_elt)
153 return ret
154 end, input))
155 etype = 'string_list'
156 end
157 else
158 lua_util.debugm(M, task, 'avoid implicit conversion as the transformer accepts complex input')
159 end
160
161 -- Now we fold elements using left fold
162 local function fold_function(acc, x)
163 if acc == nil or acc[1] == nil then
164 lua_util.debugm(M, task, 'do not apply %s, accumulator is nil', x.name)
165 return nil
166 end
167
168 local value = acc[1]
169 local t = acc[2]
170
171 if not x.types[t] then
172 local pt = pure_type(t)
173
174 if pt and x.types['list'] then
175 -- Generic list processor
176 lua_util.debugm(M, task, 'apply list function `%s` to %s', x.name, t)
177 return { x.process(value, t, x.args) }
178 elseif pt and x.map_type and x.types[pt] then
179 local map_type = x.map_type .. '_list'
180 lua_util.debugm(M, task, 'map `%s` to list of %s resulting %s',
181 x.name, pt, map_type)
182 -- Apply map, filtering empty values
183 return {
184 fun.filter(function(map_elt)
185 return map_elt
186 end,
187 fun.map(function(list_elt)
188 if not list_elt then
189 return nil
190 end
191 local ret, _ = x.process(list_elt, pt, x.args)
192 return ret
193 end, value)),
194 map_type -- Returned type
195 }
196 end
197 logger.errx(task, 'cannot apply transform %s for type %s', x.name, t)
198 return nil
199 end
200
201 lua_util.debugm(M, task, 'apply %s to %s', x.name, t)
202 return { x.process(value, t, x.args) }
203 end
204
205 local res = fun.foldl(fold_function,
206 { input, etype },
207 pipe)
208
209 if not res or not res[1] then
210 return nil
211 end -- Pipeline failed
212
213 if not allowed_type(res[2]) then
214 -- Search for implicit conversion
215 local pt = pure_type(res[2])
216
217 if pt then
218 lua_util.debugm(M, task, 'apply implicit map %s->string_list', pt)
219 res[1] = fun.map(function(e)
220 return implicit_tostring(pt, e)
221 end, res[1])
222 res[2] = 'string_list'
223 else
224 res[1] = implicit_tostring(res[2], res[1])
225 res[2] = 'string'
226 end
227 end
228
229 if list_type(res[2]) then
230 -- Convert to table as it might have a functional form
231 res[1] = fun.totable(res[1])
232 end
233
234 lua_util.debugm(M, task, 'final selector type: %s, value: %s', res[2], res[1])
235
236 return res[1]
237 end
238
239 local function make_grammar()
240 local l = require "lpeg"
241 local spc = l.S(" \t\n") ^ 0
242 local cont = l.R("\128\191") -- continuation byte
243 local utf8_high = l.R("\194\223") * cont
244 + l.R("\224\239") * cont * cont
245 + l.R("\240\244") * cont * cont * cont
246 local atom_start = (l.R("az") + l.R("AZ") + l.R("09") + utf8_high + l.S "-") ^ 1
247 local atom_end = (l.R("az") + l.R("AZ") + l.R("09") + l.S "-_" + utf8_high) ^ 1
248 local atom_mid = (1 - l.S("'\r\n\f\\,)(}{= " .. '"')) ^ 1
249 local atom_argument = l.C(atom_start * atom_mid ^ 0 * atom_end ^ 0) -- We allow more characters for the arguments
250 local atom = l.C(atom_start * atom_end ^ 0) -- We are more strict about selector names itself
251 local singlequoted_string = l.P "'" * l.C(((1 - l.S "'\r\n\f\\") + (l.P '\\' * 1)) ^ 0) * "'"
252 local doublequoted_string = l.P '"' * l.C(((1 - l.S '"\r\n\f\\') + (l.P '\\' * 1)) ^ 0) * '"'
253 local argument = atom_argument + singlequoted_string + doublequoted_string
254 local dot = l.P(".")
255 local semicolon = l.P(":")
256 local obrace = "(" * spc
257 local tbl_obrace = "{" * spc
258 local eqsign = spc * "=" * spc
259 local tbl_ebrace = spc * "}"
260 local ebrace = spc * ")"
261 local comma = spc * "," * spc
262 local sel_separator = spc * l.S ";*" * spc
263
264 return l.P {
265 "LIST";
266 LIST = l.Ct(l.V("EXPR")) * (sel_separator * l.Ct(l.V("EXPR"))) ^ 0,
267 EXPR = l.V("FUNCTION") * (semicolon * l.V("METHOD")) ^ -1 * (dot * l.V("PROCESSOR")) ^ 0,
268 PROCESSOR = l.Ct(atom * spc * (obrace * l.V("ARG_LIST") * ebrace) ^ 0),
269 FUNCTION = l.Ct(atom * spc * (obrace * l.V("ARG_LIST") * ebrace) ^ 0),
270 METHOD = l.Ct(atom / function(e)
271 return '__' .. e
272 end * spc * (obrace * l.V("ARG_LIST") * ebrace) ^ 0),
273 ARG_LIST = l.Ct((l.V("ARG") * comma ^ 0) ^ 0),
274 ARG = l.Cf(tbl_obrace * l.V("NAMED_ARG") * tbl_ebrace, rawset) + argument + l.V("LIST_ARGS"),
275 NAMED_ARG = (l.Ct("") * l.Cg(argument * eqsign * (argument + l.V("LIST_ARGS")) * comma ^ 0) ^ 0),
276 LIST_ARGS = l.Ct(tbl_obrace * l.V("LIST_ARG") * tbl_ebrace),
277 LIST_ARG = l.Cg(argument * comma ^ 0) ^ 0,
278 }
279 end
280
281 local parser = make_grammar()
282
283 --[[[
284 -- @function lua_selectors.parse_selector(cfg, str)
285 --]]
286 exports.parse_selector = function(cfg, str)
287 local parsed = { parser:match(str) }
288 local output = {}
289
290 if not parsed or not parsed[1] then
291 return nil
292 end
293
294 local function check_args(name, schema, args)
295 if schema then
296 if getmetatable(schema) then
297 -- Schema covers all arguments
298 local res, err = schema:transform(args)
299 if not res then
300 logger.errx(rspamd_config, 'invalid arguments for %s: %s', name, err)
301 return false
302 else
303 for i, elt in ipairs(res) do
304 args[i] = elt
305 end
306 end
307 else
308 for i, selt in ipairs(schema) do
309 local res, err = selt:transform(args[i])
310
311 if err then
312 logger.errx(rspamd_config, 'invalid arguments for %s: argument number: %s, error: %s', name, i, err)
313 return false
314 else
315 args[i] = res
316 end
317 end
318 end
319 end
320
321 return true
322 end
323
324 -- Output AST format is the following:
325 -- table of individual selectors
326 -- each selector: list of functions
327 -- each function: function name + optional list of arguments
328 for _, sel in ipairs(parsed) do
329 local res = {
330 selector = {},
331 processor_pipe = {},
332 }
333
334 local selector_tbl = sel[1]
335 if not selector_tbl then
336 logger.errx(cfg, 'no selector represented')
337 return nil
338 end
339 if not extractors[selector_tbl[1]] then
340 logger.errx(cfg, 'selector %s is unknown', selector_tbl[1])
341 return nil
342 end
343
344 res.selector = lua_util.shallowcopy(extractors[selector_tbl[1]])
345 res.selector.name = selector_tbl[1]
346 res.selector.args = selector_tbl[2] or E
347
348 if not check_args(res.selector.name,
349 res.selector.args_schema,
350 res.selector.args) then
351 return nil
352 end
353
354 lua_util.debugm(M, cfg, 'processed selector %s, args: %s',
355 res.selector.name, res.selector.args)
356
357 local pipeline_error = false
358 -- Now process processors pipe
359 fun.each(function(proc_tbl)
360 local proc_name = proc_tbl[1]
361
362 if proc_name:match('^__') then
363 -- Special case - method
364 local method_name = proc_name:match('^__(.*)$')
365 -- Check array indexing...
366 if tonumber(method_name) then
367 method_name = tonumber(method_name)
368 end
369 local processor = {
370 name = tostring(method_name),
371 method = true,
372 args = proc_tbl[2] or E,
373 types = {
374 userdata = true,
375 table = true,
376 string = true,
377 },
378 map_type = 'string',
379 process = function(inp, t, args)
380 local ret
381 if t == 'table' then
382 -- Plain table field
383 ret = inp[method_name]
384 else
385 -- We call method unpacking arguments and dropping all but the first result returned
386 ret = (inp[method_name](inp, unpack_function(args or E)))
387 end
388
389 local ret_type = type(ret)
390
391 if ret_type == 'nil' then
392 return nil
393 end
394 -- Now apply types heuristic
395 if ret_type == 'string' then
396 return ret, 'string'
397 elseif ret_type == 'table' then
398 -- TODO: we need to ensure that 1) table is numeric 2) table has merely strings
399 return ret, 'string_list'
400 else
401 return implicit_tostring(ret_type, ret)
402 end
403 end,
404 }
405 lua_util.debugm(M, cfg, 'attached method %s to selector %s, args: %s',
406 proc_name, res.selector.name, processor.args)
407 table.insert(res.processor_pipe, processor)
408 else
409
410 if not transform_function[proc_name] then
411 logger.errx(cfg, 'processor %s is unknown', proc_name)
412 pipeline_error = proc_name
413 return nil
414 end
415 local processor = lua_util.shallowcopy(transform_function[proc_name])
416 processor.name = proc_name
417 processor.args = proc_tbl[2] or E
418
419 if not check_args(processor.name, processor.args_schema, processor.args) then
420 pipeline_error = 'args schema for ' .. proc_name
421 return nil
422 end
423
424 lua_util.debugm(M, cfg, 'attached processor %s to selector %s, args: %s',
425 proc_name, res.selector.name, processor.args)
426 table.insert(res.processor_pipe, processor)
427 end
428 end, fun.tail(sel))
429
430 if pipeline_error then
431 logger.errx(cfg, 'unknown or invalid processor used: "%s", exiting', pipeline_error)
432 return nil
433 end
434
435 table.insert(output, res)
436 end
437
438 return output
439 end
440
441 --[[[
442 -- @function lua_selectors.register_extractor(cfg, name, selector)
443 --]]
444 exports.register_extractor = function(cfg, name, selector)
445 if selector.get_value then
446 if extractors[name] then
447 logger.warnx(cfg, 'redefining selector %s', name)
448 end
449 extractors[name] = selector
450
451 return true
452 end
453
454 logger.errx(cfg, 'bad selector %s', name)
455 return false
456 end
457
458 --[[[
459 -- @function lua_selectors.register_transform(cfg, name, transform)
460 --]]
461 exports.register_transform = function(cfg, name, transform)
462 if transform.process and transform.types then
463 if transform_function[name] then
464 logger.warnx(cfg, 'redefining transform function %s', name)
465 end
466 transform_function[name] = transform
467
468 return true
469 end
470
471 logger.errx(cfg, 'bad transform function %s', name)
472 return false
473 end
474
475 --[[[
476 -- @function lua_selectors.process_selectors(task, selectors_pipe)
477 --]]
478 exports.process_selectors = function(task, selectors_pipe)
479 local ret = {}
480
481 for _, sel in ipairs(selectors_pipe) do
482 local r = process_selector(task, sel)
483
484 -- If any element is nil, then the whole selector is nil
485 if not r then
486 return nil
487 end
488 table.insert(ret, r)
489 end
490
491 return ret
492 end
493
494 --[[[
495 -- @function lua_selectors.combine_selectors(task, selectors, delimiter)
496 --]]
497 exports.combine_selectors = function(_, selectors, delimiter)
498 if not delimiter then
499 delimiter = ''
500 end
501
502 if not selectors then
503 return nil
504 end
505
506 local have_tables, have_userdata
507
508 for _, s in ipairs(selectors) do
509 if type(s) == 'table' then
510 have_tables = true
511 elseif type(s) == 'userdata' then
512 have_userdata = true
513 end
514 end
515
516 if not have_tables then
517 if not have_userdata then
518 return table.concat(selectors, delimiter)
519 else
520 return rspamd_text.fromtable(selectors, delimiter)
521 end
522 else
523 -- We need to do a spill on each table selector and make a cortesian product
524 -- e.g. s:tbl:s -> s:telt1:s + s:telt2:s ...
525 local tbl = {}
526 local res = {}
527
528 for i, s in ipairs(selectors) do
529 if type(s) == 'string' then
530 rawset(tbl, i, fun.duplicate(s))
531 elseif type(s) == 'userdata' then
532 rawset(tbl, i, fun.duplicate(tostring(s)))
533 else
534 -- Raw table
535 rawset(tbl, i, fun.map(tostring, s))
536 end
537 end
538
539 fun.each(function(...)
540 table.insert(res, table.concat({ ... }, delimiter))
541 end, fun.zip(lua_util.unpack(tbl)))
542
543 return res
544 end
545 end
546
547 --[[[
548 -- @function lua_selectors.flatten_selectors(selectors)
549 -- Convert selectors to a flat table of elements
550 --]]
551 exports.flatten_selectors = function(_, selectors, _)
552 local res = {}
553
554 local function fill(tbl)
555 for _, s in ipairs(tbl) do
556 if type(s) == 'string' then
557 rawset(res, #res + 1, s)
558 elseif type(s) == 'userdata' then
559 rawset(res, #res + 1, tostring(s))
560 else
561 fill(s)
562 end
563 end
564 end
565
566 fill(selectors)
567
568 return res
569 end
570
571 --[[[
572 -- @function lua_selectors.kv_table_from_pairs(selectors)
573 -- Convert selectors to a table where the odd elements are keys and even are elements
574 -- Similarly to make a map from (k, v) pairs list
575 -- To specify the concrete constant keys, one can use the `id` extractor
576 --]]
577 exports.kv_table_from_pairs = function(log_obj, selectors, _)
578 local res = {}
579 local rspamd_logger = require "rspamd_logger"
580
581 local function fill(tbl)
582 local tbl_len = #tbl
583 if tbl_len % 2 ~= 0 or tbl_len == 0 then
584 rspamd_logger.errx(log_obj, "invalid invocation of the `kv_table_from_pairs`: table length is invalid %s",
585 tbl_len)
586 return
587 end
588 for i = 1, tbl_len, 2 do
589 local k = tostring(tbl[i])
590 local v = tbl[i + 1]
591 if type(v) == 'string' then
592 res[k] = v
593 elseif type(v) == 'userdata' then
594 res[k] = tostring(v)
595 else
596 res[k] = fun.totable(fun.map(function(elt)
597 return tostring(elt)
598 end, v))
599 end
600 end
601 end
602
603 fill(selectors)
604
605 return res
606 end
607
608
609 --[[[
610 -- @function lua_selectors.create_closure(log_obj, cfg, selector_str, delimiter, fn)
611 -- Creates a closure from a string selector, using the specific combinator function
612 --]]
613 exports.create_selector_closure_fn = function(log_obj, cfg, selector_str, delimiter, fn)
614 local selector = exports.parse_selector(cfg, selector_str)
615
616 if not selector then
617 return nil
618 end
619
620 return function(task)
621 local res = exports.process_selectors(task, selector)
622
623 if res then
624 return fn(log_obj, res, delimiter)
625 end
626
627 return nil
628 end
629 end
630
631 --[[[
632 -- @function lua_selectors.create_closure(cfg, selector_str, delimiter='', flatten=false)
633 -- Creates a closure from a string selector
634 --]]
635 exports.create_selector_closure = function(cfg, selector_str, delimiter, flatten)
636 local combinator_fn = flatten and exports.flatten_selectors or exports.combine_selectors
637
638 return exports.create_selector_closure_fn(nil, cfg, selector_str, delimiter, combinator_fn)
639 end
640
641 local function display_selectors(tbl)
642 return fun.tomap(fun.map(function(k, v)
643 return k, fun.tomap(fun.filter(function(kk, vv)
644 return type(vv) ~= 'function'
645 end, v))
646 end, tbl))
647 end
648
649 exports.list_extractors = function()
650 return display_selectors(extractors)
651 end
652
653 exports.list_transforms = function()
654 return display_selectors(transform_function)
655 end
656
657 exports.add_map = function(name, map)
658 if not exports.maps[name] then
659 exports.maps[name] = map
660 else
661 logger.errx(rspamd_config, "duplicate map redefinition for the selectors: %s", name)
662 end
663 end
664
665 -- Combinator functions registry for use with external maps and other consumers
666 -- that need structured data instead of plain strings
667 local combinators = {
668 -- Default: concatenate all selector results into a single string
669 string = exports.combine_selectors,
670 -- Flatten all results into a flat array
671 array = exports.flatten_selectors,
672 -- Convert pairs of selectors into key-value object (use `id` extractor for keys)
673 object = exports.kv_table_from_pairs,
674 }
675
676 --[[[
677 -- @function lua_selectors.get_combinator(name)
678 -- Returns a combinator function by name
679 -- @param {string} name combinator name: 'string', 'array', or 'object'
680 -- @return {function} combinator function or nil if not found
681 --]]
682 exports.get_combinator = function(name)
683 return combinators[name]
684 end
685
686 --[[[
687 -- @function lua_selectors.list_combinators()
688 -- Returns list of available combinator names
689 -- @return {table} array of combinator names
690 --]]
691 exports.list_combinators = function()
692 local res = {}
693 for k, _ in pairs(combinators) do
694 table.insert(res, k)
695 end
696 return res
697 end
698
699 --[[[
700 -- @function lua_selectors.create_selector_closure_with_combinator(cfg, selector_str, delimiter, combinator_name)
701 -- Creates a closure from a string selector using named combinator
702 -- @param {rspamd_config} cfg rspamd config object
703 -- @param {string} selector_str selector string to parse
704 -- @param {string} delimiter delimiter for combining results (used by some combinators)
705 -- @param {string} combinator_name name of combinator: 'string', 'array', or 'object'
706 -- @return {function} closure that processes selector on task, or nil on error
707 --]]
708 exports.create_selector_closure_with_combinator = function(cfg, selector_str, delimiter, combinator_name)
709 local combinator_fn = combinators[combinator_name or 'string']
710
711 if not combinator_fn then
712 logger.errx(cfg, 'unknown combinator: %s, available: %s',
713 combinator_name, table.concat(exports.list_combinators(), ', '))
714 return nil
715 end
716
717 return exports.create_selector_closure_fn(cfg, cfg, selector_str, delimiter, combinator_fn)
718 end
719
720 -- Publish log target
721 exports.M = M
722
723 return exports