Debian Codegraph @

lualib/lua_selectors/transforms.lua

raw
1 --[[
2 Copyright (c) 2022, Vsevolod Stakhov <[email protected]>
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 ]] --
16
17 local fun = require 'fun'
18 local lua_util = require "lua_util"
19 local rspamd_util = require "rspamd_util"
20 local T = require "lua_shape.core"
21 local logger = require 'rspamd_logger'
22 local common = require "lua_selectors/common"
23 local M = "selectors"
24
25 local maps = require "lua_selectors/maps"
26
27 local function pure_type(ltype)
28 return ltype:match('^(.*)_list$')
29 end
30
31 local transform_function = {
32 -- Returns the lowercased string
33 ['lower'] = {
34 ['types'] = {
35 ['string'] = true,
36 },
37 ['map_type'] = 'string',
38 ['process'] = function(inp, _)
39 return inp:lower(), 'string'
40 end,
41 ['description'] = 'Returns the lowercased string',
42 },
43 -- Returns the lowercased utf8 string
44 ['lower_utf8'] = {
45 ['types'] = {
46 ['string'] = true,
47 },
48 ['map_type'] = 'string',
49 ['process'] = function(inp, t)
50 return rspamd_util.lower_utf8(inp), t
51 end,
52 ['description'] = 'Returns the lowercased utf8 string',
53 },
54 -- Returns the first element
55 ['first'] = {
56 ['types'] = {
57 ['list'] = true,
58 },
59 ['process'] = function(inp, t)
60 return fun.head(inp), pure_type(t)
61 end,
62 ['description'] = 'Returns the first element',
63 },
64 -- Returns the last element
65 ['last'] = {
66 ['types'] = {
67 ['list'] = true,
68 },
69 ['process'] = function(inp, t)
70 return fun.nth(fun.length(inp), inp), pure_type(t)
71 end,
72 ['description'] = 'Returns the last element',
73 },
74 -- Returns the nth element
75 ['nth'] = {
76 ['types'] = {
77 ['list'] = true,
78 },
79 ['process'] = function(inp, t, args)
80 return fun.nth(args[1] or 1, inp), pure_type(t)
81 end,
82 ['description'] = 'Returns the nth element',
83 ['args_schema'] = { T.one_of({ T.number(), T.transform(T.string(), tonumber) }) }
84 },
85 ['take_n'] = {
86 ['types'] = {
87 ['list'] = true,
88 },
89 ['process'] = function(inp, t, args)
90 return fun.take_n(args[1] or 1, inp), t
91 end,
92 ['description'] = 'Returns the n first elements',
93 ['args_schema'] = { T.one_of({ T.number(), T.transform(T.string(), tonumber) }) }
94 },
95 ['drop_n'] = {
96 ['types'] = {
97 ['list'] = true,
98 },
99 ['process'] = function(inp, t, args)
100 return fun.drop_n(args[1] or 1, inp), t
101 end,
102 ['description'] = 'Returns list without the first n elements',
103 ['args_schema'] = { T.one_of({ T.number(), T.transform(T.string(), tonumber) }) }
104 },
105 -- Joins strings into a single string using separator in the argument
106 ['join'] = {
107 ['types'] = {
108 ['string_list'] = true
109 },
110 ['process'] = function(inp, _, args)
111 return table.concat(fun.totable(inp), args[1] or ''), 'string'
112 end,
113 ['description'] = 'Joins strings into a single string using separator in the argument',
114 ['args_schema'] = { T.string():optional() }
115 },
116 -- Joins strings into a set of strings using N elements and a separator in the argument
117 ['join_nth'] = {
118 ['types'] = {
119 ['string_list'] = true
120 },
121 ['process'] = function(inp, _, args)
122 local step = args[1]
123 local sep = args[2] or ''
124 local inp_t = fun.totable(inp)
125 local res = {}
126
127 for i = 1, #inp_t, step do
128 table.insert(res, table.concat(inp_t, sep, i, i + step))
129 end
130 return res, 'string_list'
131 end,
132 ['description'] = 'Joins strings into a set of strings using N elements and a separator in the argument',
133 ['args_schema'] = { T.one_of({ T.number(), T.transform(T.string(), tonumber) }), T.string():optional() }
134 },
135 -- Joins tables into a table of strings
136 ['join_tables'] = {
137 ['types'] = {
138 ['list'] = true
139 },
140 ['process'] = function(inp, _, args)
141 local sep = args[1] or ''
142 return fun.map(function(t)
143 return table.concat(t, sep)
144 end, inp), 'string_list'
145 end,
146 ['description'] = 'Joins tables into a table of strings',
147 ['args_schema'] = { T.string():optional() }
148 },
149 -- Sort strings
150 ['sort'] = {
151 ['types'] = {
152 ['list'] = true
153 },
154 ['process'] = function(inp, t, _)
155 table.sort(inp)
156 return inp, t
157 end,
158 ['description'] = 'Sort strings lexicographically',
159 },
160 -- Return unique elements based on hashing (can work without sorting)
161 ['uniq'] = {
162 ['types'] = {
163 ['list'] = true
164 },
165 ['process'] = function(inp, t, _)
166 local tmp = {}
167 fun.each(function(val)
168 tmp[val] = true
169 end, inp)
170
171 return fun.map(function(k, _)
172 return k
173 end, tmp), t
174 end,
175 ['description'] = 'Returns a list of unique elements (using a hash table)',
176 },
177 -- Create a digest from string or a list of strings
178 ['digest'] = {
179 ['types'] = {
180 ['string'] = true
181 },
182 ['map_type'] = 'string',
183 ['process'] = function(inp, _, args)
184 return common.create_digest(inp, args), 'string'
185 end,
186 ['description'] = [[Create a digest from a string.
187 The first argument is encoding (`hex`, `base32` (and forms `bleach32`, `rbase32`), `base64`),
188 the second argument is optional hash type (`blake2`, `sha256`, `sha1`, `sha512`, `md5`)]],
189 ['args_schema'] = common.digest_schema()
190 },
191 -- Extracts substring
192 ['substring'] = {
193 ['types'] = {
194 ['string'] = true
195 },
196 ['map_type'] = 'string',
197 ['process'] = function(inp, _, args)
198 local start_pos = args[1] or 1
199 local end_pos = args[2] or -1
200
201 return inp:sub(start_pos, end_pos), 'string'
202 end,
203 ['description'] = 'Extracts substring; the first argument is start, the second is the last (like in Lua)',
204 ['args_schema'] = { (T.one_of({ T.number(), T.transform(T.string(), tonumber) })):optional(),
205 (T.one_of({ T.number(), T.transform(T.string(), tonumber) })):optional() }
206 },
207 -- Prepends a string or a strings list
208 ['prepend'] = {
209 ['types'] = {
210 ['string'] = true
211 },
212 ['map_type'] = 'string',
213 ['process'] = function(inp, _, args)
214 local prepend = table.concat(args, '')
215
216 return prepend .. inp, 'string'
217 end,
218 ['description'] = 'Prepends a string or a strings list',
219 },
220 -- Appends a string or a strings list
221 ['append'] = {
222 ['types'] = {
223 ['string'] = true
224 },
225 ['map_type'] = 'string',
226 ['process'] = function(inp, _, args)
227 local append = table.concat(args, '')
228
229 return inp .. append, 'string'
230 end,
231 ['description'] = 'Appends a string or a strings list',
232 },
233 -- Regexp matching
234 ['regexp'] = {
235 ['types'] = {
236 ['string'] = true
237 },
238 ['map_type'] = 'string',
239 ['process'] = function(inp, _, args)
240 local rspamd_regexp = require "rspamd_regexp"
241
242 local re = rspamd_regexp.create_cached(args[1])
243
244 if not re then
245 logger.errx('invalid regexp: %s', args[1])
246 return nil
247 end
248
249 local res = re:search(inp, false, true)
250
251 if res then
252 -- Map all results in a single list
253 local flattened_table = {}
254 local function flatten_table(tbl)
255 for _, v in ipairs(tbl) do
256 if type(v) == 'table' then
257 flatten_table(v)
258 else
259 table.insert(flattened_table, v)
260 end
261 end
262 end
263 flatten_table(res)
264 return flattened_table, 'string_list'
265 end
266
267 return nil
268 end,
269 ['description'] = 'Regexp matching, returns all matches flattened in a single list',
270 ['args_schema'] = { T.string() }
271 },
272 -- Returns a value if it exists in some map (or acts like a `filter` function)
273 ['filter_map'] = {
274 ['types'] = {
275 ['string'] = true
276 },
277 ['map_type'] = 'string',
278 ['process'] = function(inp, t, args)
279 local map = maps[args[1]]
280
281 if not map then
282 logger.errx('invalid map name: %s', args[1])
283 return nil
284 end
285
286 local res = map:get_key(inp)
287
288 if res then
289 return inp, t
290 end
291
292 return nil
293 end,
294 ['description'] = 'Returns a value if it exists in some map (or acts like a `filter` function)',
295 ['args_schema'] = { T.string() }
296 },
297 -- Returns a value if it exists in some map (or acts like a `filter` function)
298 ['except_map'] = {
299 ['types'] = {
300 ['string'] = true
301 },
302 ['map_type'] = 'string',
303 ['process'] = function(inp, t, args)
304 local map = maps[args[1]]
305
306 if not map then
307 logger.errx('invalid map name: %s', args[1])
308 return nil
309 end
310
311 local res = map:get_key(inp)
312
313 if not res then
314 return inp, t
315 end
316
317 return nil
318 end,
319 ['description'] = 'Returns a value if it does not exists in some map (or acts like a `except` function)',
320 ['args_schema'] = { T.string() }
321 },
322 -- Returns a value from some map corresponding to some key (or acts like a `map` function)
323 ['apply_map'] = {
324 ['types'] = {
325 ['string'] = true
326 },
327 ['map_type'] = 'string',
328 ['process'] = function(inp, t, args)
329 local map = maps[args[1]]
330
331 if not map then
332 logger.errx('invalid map name: %s', args[1])
333 return nil
334 end
335
336 local res = map:get_key(inp)
337
338 if res then
339 return res, t
340 end
341
342 return nil
343 end,
344 ['description'] = 'Returns a value from some map corresponding to some key (or acts like a `map` function)',
345 ['args_schema'] = { T.string() }
346 },
347 -- Drops input value and return values from function's arguments or an empty string
348 ['id'] = {
349 ['types'] = {
350 ['string'] = true,
351 ['list'] = true,
352 },
353 ['map_type'] = 'string',
354 ['process'] = function(_, _, args)
355 if args[1] and args[2] then
356 return fun.map(tostring, args), 'string_list'
357 elseif args[1] then
358 return args[1], 'string'
359 end
360
361 return '', 'string'
362 end,
363 ['description'] = 'Drops input value and return values from function\'s arguments or an empty string',
364 ['args_schema'] = T.one_of({ T.string(), T.array(T.string()) }):optional()
365 },
366 ['equal'] = {
367 ['types'] = {
368 ['string'] = true,
369 },
370 ['map_type'] = 'string',
371 ['process'] = function(inp, _, args)
372 if inp == args[1] then
373 return inp, 'string'
374 end
375
376 return nil
377 end,
378 ['description'] = [[Boolean function equal.
379 Returns either nil or its argument if input is equal to argument]],
380 ['args_schema'] = { T.string() }
381 },
382 -- Boolean function in, returns either nil or its input if input is in args list
383 ['in'] = {
384 ['types'] = {
385 ['string'] = true,
386 },
387 ['map_type'] = 'string',
388 ['process'] = function(inp, t, args)
389 for _, a in ipairs(args) do
390 if a == inp then
391 return inp, t
392 end
393 end
394 return nil
395 end,
396 ['description'] = [[Boolean function in.
397 Returns either nil or its input if input is in args list]],
398 ['args_schema'] = T.array(T.string())
399 },
400 ['not_in'] = {
401 ['types'] = {
402 ['string'] = true,
403 },
404 ['map_type'] = 'string',
405 ['process'] = function(inp, t, args)
406 for _, a in ipairs(args) do
407 if a == inp then
408 return nil
409 end
410 end
411 return inp, t
412 end,
413 ['description'] = [[Boolean function not in.
414 Returns either nil or its input if input is not in args list]],
415 ['args_schema'] = T.array(T.string())
416 },
417 ['inverse'] = {
418 ['types'] = {
419 ['string'] = true,
420 },
421 ['map_type'] = 'string',
422 ['process'] = function(inp, _, args)
423 if inp then
424 return nil
425 else
426 return (args[1] or 'true'), 'string'
427 end
428 end,
429 ['description'] = [[Inverses input.
430 Empty string comes the first argument or 'true', non-empty string comes nil]],
431 ['args_schema'] = { T.string():optional() }
432 },
433 ['ipmask'] = {
434 ['types'] = {
435 ['string'] = true,
436 },
437 ['map_type'] = 'string',
438 ['process'] = function(inp, _, args)
439 local rspamd_ip = require "rspamd_ip"
440 -- Non optimal: convert string to an IP address
441 local ip = rspamd_ip.from_string(inp)
442
443 if not ip or not ip:is_valid() then
444 lua_util.debugm(M, "cannot convert %s to IP", inp)
445 return nil
446 end
447
448 if ip:get_version() == 4 then
449 local mask = tonumber(args[1])
450
451 return ip:apply_mask(mask):to_string(), 'string'
452 else
453 -- IPv6 takes the second argument or the first one...
454 local mask_str = args[2] or args[1]
455 local mask = tonumber(mask_str)
456
457 return ip:apply_mask(mask):to_string(), 'string'
458 end
459 end,
460 ['description'] = 'Applies mask to IP address.' ..
461 ' The first argument is the mask for IPv4 addresses, the second is the mask for IPv6 addresses.',
462 ['args_schema'] = { (T.one_of({ T.number(), T.transform(T.string(), tonumber) })),
463 (T.one_of({ T.number(), T.transform(T.string(), tonumber) })):optional() }
464 },
465 -- Returns the string(s) with all non ascii chars replaced
466 ['to_ascii'] = {
467 ['types'] = {
468 ['string'] = true,
469 ['list'] = true,
470 },
471 ['map_type'] = 'string',
472 ['process'] = function(inp, _, args)
473 if type(inp) == 'table' then
474 return fun.map(
475 function(s)
476 return string.gsub(tostring(s), '[\128-\255]', args[1] or '?')
477 end, inp), 'string_list'
478 else
479 return string.gsub(tostring(inp), '[\128-\255]', '?'), 'string'
480 end
481 end,
482 ['description'] = 'Returns the string with all non-ascii bytes replaced with the character ' ..
483 'given as second argument or `?`',
484 ['args_schema'] = { T.string():optional() }
485 },
486 -- Extracts tld from a hostname
487 ['get_tld'] = {
488 ['types'] = {
489 ['string'] = true
490 },
491 ['map_type'] = 'string',
492 ['process'] = function(inp, _, _)
493 return rspamd_util.get_tld(inp), 'string'
494 end,
495 ['description'] = 'Returns effective second-level domain (eSLD) using the Public Suffix List',
496 ['args_schema'] = {}
497 },
498 -- Converts list of strings to numbers and returns a packed string
499 ['pack_numbers'] = {
500 ['types'] = {
501 ['string_list'] = true
502 },
503 ['map_type'] = 'string',
504 ['process'] = function(inp, _, args)
505 local fmt = args[1] or 'f'
506 local res = {}
507 for _, s in ipairs(inp) do
508 table.insert(res, tonumber(s))
509 end
510 return rspamd_util.pack(string.rep(fmt, #res), lua_util.unpack(res)), 'string'
511 end,
512 ['description'] = 'Converts a list of strings to numbers & returns a packed string',
513 ['args_schema'] = { T.string():optional() }
514 },
515 -- Filter nils from a list
516 ['filter_string_nils'] = {
517 ['types'] = {
518 ['string_list'] = true
519 },
520 ['process'] = function(inp, _, _)
521 return fun.filter(function(val)
522 return type(val) == 'string' and val ~= 'nil'
523 end, inp), 'string_list'
524 end,
525 ['description'] = 'Removes all nils from a list of strings (when converted implicitly)',
526 ['args_schema'] = {}
527 },
528 -- Call a set of methods on a userdata object
529 ['apply_methods'] = {
530 ['types'] = {
531 ['userdata'] = true,
532 },
533 ['process'] = function(inp, _, args)
534 local res = {}
535 for _, arg in ipairs(args) do
536 local meth = inp[arg]
537 local ret = meth(inp)
538 if ret then
539 table.insert(res, tostring(ret))
540 end
541 end
542 return res, 'string_list'
543 end,
544 ['description'] = 'Apply a list of method calls to the userdata object',
545 },
546 -- Apply method to list of userdata and use it as a filter, excluding elements for which method returns false/nil
547 ['filter_method'] = {
548 ['types'] = {
549 ['userdata_list'] = true
550 },
551 ['process'] = function(inp, t, args)
552 local meth = args[1]
553
554 if not meth then
555 logger.errx('invalid method name: %s', args[1])
556 return nil
557 end
558
559 return fun.filter(function(val)
560 return val[meth](val)
561 end, inp), 'userdata_list'
562 end,
563 ['description'] = 'Apply method to list of userdata and use it as a filter,' ..
564 ' excluding elements for which method returns false/nil',
565 ['args_schema'] = { T.string() }
566 },
567 }
568
569 transform_function.match = transform_function.regexp
570
571 return transform_function