| 1 |
--[[ |
| 2 |
Copyright (c) 2022, Vsevolod Stakhov <[email protected]> |
| 3 |
|
| 4 |
Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 |
you may not use this file except in compliance with the License. |
| 6 |
You may obtain a copy of the License at |
| 7 |
|
| 8 |
http://www.apache.org/licenses/LICENSE-2.0 |
| 9 |
|
| 10 |
Unless required by applicable law or agreed to in writing, software |
| 11 |
distributed under the License is distributed on an "AS IS" BASIS, |
| 12 |
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 |
See the License for the specific language governing permissions and |
| 14 |
limitations under the License. |
| 15 |
]] -- |
| 16 |
|
| 17 |
local fun = require 'fun' |
| 18 |
local lua_util = require "lua_util" |
| 19 |
local rspamd_util = require "rspamd_util" |
| 20 |
local T = require "lua_shape.core" |
| 21 |
local logger = require 'rspamd_logger' |
| 22 |
local common = require "lua_selectors/common" |
| 23 |
local M = "selectors" |
| 24 |
|
| 25 |
local maps = require "lua_selectors/maps" |
| 26 |
|
| 27 |
local function pure_type(ltype) |
| 28 |
return ltype:match('^(.*)_list$') |
| 29 |
end |
| 30 |
|
| 31 |
local transform_function = { |
| 32 |
-- Returns the lowercased string |
| 33 |
['lower'] = { |
| 34 |
['types'] = { |
| 35 |
['string'] = true, |
| 36 |
}, |
| 37 |
['map_type'] = 'string', |
| 38 |
['process'] = function(inp, _) |
| 39 |
return inp:lower(), 'string' |
| 40 |
end, |
| 41 |
['description'] = 'Returns the lowercased string', |
| 42 |
}, |
| 43 |
-- Returns the lowercased utf8 string |
| 44 |
['lower_utf8'] = { |
| 45 |
['types'] = { |
| 46 |
['string'] = true, |
| 47 |
}, |
| 48 |
['map_type'] = 'string', |
| 49 |
['process'] = function(inp, t) |
| 50 |
return rspamd_util.lower_utf8(inp), t |
| 51 |
end, |
| 52 |
['description'] = 'Returns the lowercased utf8 string', |
| 53 |
}, |
| 54 |
-- Returns the first element |
| 55 |
['first'] = { |
| 56 |
['types'] = { |
| 57 |
['list'] = true, |
| 58 |
}, |
| 59 |
['process'] = function(inp, t) |
| 60 |
return fun.head(inp), pure_type(t) |
| 61 |
end, |
| 62 |
['description'] = 'Returns the first element', |
| 63 |
}, |
| 64 |
-- Returns the last element |
| 65 |
['last'] = { |
| 66 |
['types'] = { |
| 67 |
['list'] = true, |
| 68 |
}, |
| 69 |
['process'] = function(inp, t) |
| 70 |
return fun.nth(fun.length(inp), inp), pure_type(t) |
| 71 |
end, |
| 72 |
['description'] = 'Returns the last element', |
| 73 |
}, |
| 74 |
-- Returns the nth element |
| 75 |
['nth'] = { |
| 76 |
['types'] = { |
| 77 |
['list'] = true, |
| 78 |
}, |
| 79 |
['process'] = function(inp, t, args) |
| 80 |
return fun.nth(args[1] or 1, inp), pure_type(t) |
| 81 |
end, |
| 82 |
['description'] = 'Returns the nth element', |
| 83 |
['args_schema'] = { T.one_of({ T.number(), T.transform(T.string(), tonumber) }) } |
| 84 |
}, |
| 85 |
['take_n'] = { |
| 86 |
['types'] = { |
| 87 |
['list'] = true, |
| 88 |
}, |
| 89 |
['process'] = function(inp, t, args) |
| 90 |
return fun.take_n(args[1] or 1, inp), t |
| 91 |
end, |
| 92 |
['description'] = 'Returns the n first elements', |
| 93 |
['args_schema'] = { T.one_of({ T.number(), T.transform(T.string(), tonumber) }) } |
| 94 |
}, |
| 95 |
['drop_n'] = { |
| 96 |
['types'] = { |
| 97 |
['list'] = true, |
| 98 |
}, |
| 99 |
['process'] = function(inp, t, args) |
| 100 |
return fun.drop_n(args[1] or 1, inp), t |
| 101 |
end, |
| 102 |
['description'] = 'Returns list without the first n elements', |
| 103 |
['args_schema'] = { T.one_of({ T.number(), T.transform(T.string(), tonumber) }) } |
| 104 |
}, |
| 105 |
-- Joins strings into a single string using separator in the argument |
| 106 |
['join'] = { |
| 107 |
['types'] = { |
| 108 |
['string_list'] = true |
| 109 |
}, |
| 110 |
['process'] = function(inp, _, args) |
| 111 |
return table.concat(fun.totable(inp), args[1] or ''), 'string' |
| 112 |
end, |
| 113 |
['description'] = 'Joins strings into a single string using separator in the argument', |
| 114 |
['args_schema'] = { T.string():optional() } |
| 115 |
}, |
| 116 |
-- Joins strings into a set of strings using N elements and a separator in the argument |
| 117 |
['join_nth'] = { |
| 118 |
['types'] = { |
| 119 |
['string_list'] = true |
| 120 |
}, |
| 121 |
['process'] = function(inp, _, args) |
| 122 |
local step = args[1] |
| 123 |
local sep = args[2] or '' |
| 124 |
local inp_t = fun.totable(inp) |
| 125 |
local res = {} |
| 126 |
|
| 127 |
for i = 1, #inp_t, step do |
| 128 |
table.insert(res, table.concat(inp_t, sep, i, i + step)) |
| 129 |
end |
| 130 |
return res, 'string_list' |
| 131 |
end, |
| 132 |
['description'] = 'Joins strings into a set of strings using N elements and a separator in the argument', |
| 133 |
['args_schema'] = { T.one_of({ T.number(), T.transform(T.string(), tonumber) }), T.string():optional() } |
| 134 |
}, |
| 135 |
-- Joins tables into a table of strings |
| 136 |
['join_tables'] = { |
| 137 |
['types'] = { |
| 138 |
['list'] = true |
| 139 |
}, |
| 140 |
['process'] = function(inp, _, args) |
| 141 |
local sep = args[1] or '' |
| 142 |
return fun.map(function(t) |
| 143 |
return table.concat(t, sep) |
| 144 |
end, inp), 'string_list' |
| 145 |
end, |
| 146 |
['description'] = 'Joins tables into a table of strings', |
| 147 |
['args_schema'] = { T.string():optional() } |
| 148 |
}, |
| 149 |
-- Sort strings |
| 150 |
['sort'] = { |
| 151 |
['types'] = { |
| 152 |
['list'] = true |
| 153 |
}, |
| 154 |
['process'] = function(inp, t, _) |
| 155 |
table.sort(inp) |
| 156 |
return inp, t |
| 157 |
end, |
| 158 |
['description'] = 'Sort strings lexicographically', |
| 159 |
}, |
| 160 |
-- Return unique elements based on hashing (can work without sorting) |
| 161 |
['uniq'] = { |
| 162 |
['types'] = { |
| 163 |
['list'] = true |
| 164 |
}, |
| 165 |
['process'] = function(inp, t, _) |
| 166 |
local tmp = {} |
| 167 |
fun.each(function(val) |
| 168 |
tmp[val] = true |
| 169 |
end, inp) |
| 170 |
|
| 171 |
return fun.map(function(k, _) |
| 172 |
return k |
| 173 |
end, tmp), t |
| 174 |
end, |
| 175 |
['description'] = 'Returns a list of unique elements (using a hash table)', |
| 176 |
}, |
| 177 |
-- Create a digest from string or a list of strings |
| 178 |
['digest'] = { |
| 179 |
['types'] = { |
| 180 |
['string'] = true |
| 181 |
}, |
| 182 |
['map_type'] = 'string', |
| 183 |
['process'] = function(inp, _, args) |
| 184 |
return common.create_digest(inp, args), 'string' |
| 185 |
end, |
| 186 |
['description'] = [[Create a digest from a string. |
| 187 |
The first argument is encoding (`hex`, `base32` (and forms `bleach32`, `rbase32`), `base64`), |
| 188 |
the second argument is optional hash type (`blake2`, `sha256`, `sha1`, `sha512`, `md5`)]], |
| 189 |
['args_schema'] = common.digest_schema() |
| 190 |
}, |
| 191 |
-- Extracts substring |
| 192 |
['substring'] = { |
| 193 |
['types'] = { |
| 194 |
['string'] = true |
| 195 |
}, |
| 196 |
['map_type'] = 'string', |
| 197 |
['process'] = function(inp, _, args) |
| 198 |
local start_pos = args[1] or 1 |
| 199 |
local end_pos = args[2] or -1 |
| 200 |
|
| 201 |
return inp:sub(start_pos, end_pos), 'string' |
| 202 |
end, |
| 203 |
['description'] = 'Extracts substring; the first argument is start, the second is the last (like in Lua)', |
| 204 |
['args_schema'] = { (T.one_of({ T.number(), T.transform(T.string(), tonumber) })):optional(), |
| 205 |
(T.one_of({ T.number(), T.transform(T.string(), tonumber) })):optional() } |
| 206 |
}, |
| 207 |
-- Prepends a string or a strings list |
| 208 |
['prepend'] = { |
| 209 |
['types'] = { |
| 210 |
['string'] = true |
| 211 |
}, |
| 212 |
['map_type'] = 'string', |
| 213 |
['process'] = function(inp, _, args) |
| 214 |
local prepend = table.concat(args, '') |
| 215 |
|
| 216 |
return prepend .. inp, 'string' |
| 217 |
end, |
| 218 |
['description'] = 'Prepends a string or a strings list', |
| 219 |
}, |
| 220 |
-- Appends a string or a strings list |
| 221 |
['append'] = { |
| 222 |
['types'] = { |
| 223 |
['string'] = true |
| 224 |
}, |
| 225 |
['map_type'] = 'string', |
| 226 |
['process'] = function(inp, _, args) |
| 227 |
local append = table.concat(args, '') |
| 228 |
|
| 229 |
return inp .. append, 'string' |
| 230 |
end, |
| 231 |
['description'] = 'Appends a string or a strings list', |
| 232 |
}, |
| 233 |
-- Regexp matching |
| 234 |
['regexp'] = { |
| 235 |
['types'] = { |
| 236 |
['string'] = true |
| 237 |
}, |
| 238 |
['map_type'] = 'string', |
| 239 |
['process'] = function(inp, _, args) |
| 240 |
local rspamd_regexp = require "rspamd_regexp" |
| 241 |
|
| 242 |
local re = rspamd_regexp.create_cached(args[1]) |
| 243 |
|
| 244 |
if not re then |
| 245 |
logger.errx('invalid regexp: %s', args[1]) |
| 246 |
return nil |
| 247 |
end |
| 248 |
|
| 249 |
local res = re:search(inp, false, true) |
| 250 |
|
| 251 |
if res then |
| 252 |
-- Map all results in a single list |
| 253 |
local flattened_table = {} |
| 254 |
local function flatten_table(tbl) |
| 255 |
for _, v in ipairs(tbl) do |
| 256 |
if type(v) == 'table' then |
| 257 |
flatten_table(v) |
| 258 |
else |
| 259 |
table.insert(flattened_table, v) |
| 260 |
end |
| 261 |
end |
| 262 |
end |
| 263 |
flatten_table(res) |
| 264 |
return flattened_table, 'string_list' |
| 265 |
end |
| 266 |
|
| 267 |
return nil |
| 268 |
end, |
| 269 |
['description'] = 'Regexp matching, returns all matches flattened in a single list', |
| 270 |
['args_schema'] = { T.string() } |
| 271 |
}, |
| 272 |
-- Returns a value if it exists in some map (or acts like a `filter` function) |
| 273 |
['filter_map'] = { |
| 274 |
['types'] = { |
| 275 |
['string'] = true |
| 276 |
}, |
| 277 |
['map_type'] = 'string', |
| 278 |
['process'] = function(inp, t, args) |
| 279 |
local map = maps[args[1]] |
| 280 |
|
| 281 |
if not map then |
| 282 |
logger.errx('invalid map name: %s', args[1]) |
| 283 |
return nil |
| 284 |
end |
| 285 |
|
| 286 |
local res = map:get_key(inp) |
| 287 |
|
| 288 |
if res then |
| 289 |
return inp, t |
| 290 |
end |
| 291 |
|
| 292 |
return nil |
| 293 |
end, |
| 294 |
['description'] = 'Returns a value if it exists in some map (or acts like a `filter` function)', |
| 295 |
['args_schema'] = { T.string() } |
| 296 |
}, |
| 297 |
-- Returns a value if it exists in some map (or acts like a `filter` function) |
| 298 |
['except_map'] = { |
| 299 |
['types'] = { |
| 300 |
['string'] = true |
| 301 |
}, |
| 302 |
['map_type'] = 'string', |
| 303 |
['process'] = function(inp, t, args) |
| 304 |
local map = maps[args[1]] |
| 305 |
|
| 306 |
if not map then |
| 307 |
logger.errx('invalid map name: %s', args[1]) |
| 308 |
return nil |
| 309 |
end |
| 310 |
|
| 311 |
local res = map:get_key(inp) |
| 312 |
|
| 313 |
if not res then |
| 314 |
return inp, t |
| 315 |
end |
| 316 |
|
| 317 |
return nil |
| 318 |
end, |
| 319 |
['description'] = 'Returns a value if it does not exists in some map (or acts like a `except` function)', |
| 320 |
['args_schema'] = { T.string() } |
| 321 |
}, |
| 322 |
-- Returns a value from some map corresponding to some key (or acts like a `map` function) |
| 323 |
['apply_map'] = { |
| 324 |
['types'] = { |
| 325 |
['string'] = true |
| 326 |
}, |
| 327 |
['map_type'] = 'string', |
| 328 |
['process'] = function(inp, t, args) |
| 329 |
local map = maps[args[1]] |
| 330 |
|
| 331 |
if not map then |
| 332 |
logger.errx('invalid map name: %s', args[1]) |
| 333 |
return nil |
| 334 |
end |
| 335 |
|
| 336 |
local res = map:get_key(inp) |
| 337 |
|
| 338 |
if res then |
| 339 |
return res, t |
| 340 |
end |
| 341 |
|
| 342 |
return nil |
| 343 |
end, |
| 344 |
['description'] = 'Returns a value from some map corresponding to some key (or acts like a `map` function)', |
| 345 |
['args_schema'] = { T.string() } |
| 346 |
}, |
| 347 |
-- Drops input value and return values from function's arguments or an empty string |
| 348 |
['id'] = { |
| 349 |
['types'] = { |
| 350 |
['string'] = true, |
| 351 |
['list'] = true, |
| 352 |
}, |
| 353 |
['map_type'] = 'string', |
| 354 |
['process'] = function(_, _, args) |
| 355 |
if args[1] and args[2] then |
| 356 |
return fun.map(tostring, args), 'string_list' |
| 357 |
elseif args[1] then |
| 358 |
return args[1], 'string' |
| 359 |
end |
| 360 |
|
| 361 |
return '', 'string' |
| 362 |
end, |
| 363 |
['description'] = 'Drops input value and return values from function\'s arguments or an empty string', |
| 364 |
['args_schema'] = T.one_of({ T.string(), T.array(T.string()) }):optional() |
| 365 |
}, |
| 366 |
['equal'] = { |
| 367 |
['types'] = { |
| 368 |
['string'] = true, |
| 369 |
}, |
| 370 |
['map_type'] = 'string', |
| 371 |
['process'] = function(inp, _, args) |
| 372 |
if inp == args[1] then |
| 373 |
return inp, 'string' |
| 374 |
end |
| 375 |
|
| 376 |
return nil |
| 377 |
end, |
| 378 |
['description'] = [[Boolean function equal. |
| 379 |
Returns either nil or its argument if input is equal to argument]], |
| 380 |
['args_schema'] = { T.string() } |
| 381 |
}, |
| 382 |
-- Boolean function in, returns either nil or its input if input is in args list |
| 383 |
['in'] = { |
| 384 |
['types'] = { |
| 385 |
['string'] = true, |
| 386 |
}, |
| 387 |
['map_type'] = 'string', |
| 388 |
['process'] = function(inp, t, args) |
| 389 |
for _, a in ipairs(args) do |
| 390 |
if a == inp then |
| 391 |
return inp, t |
| 392 |
end |
| 393 |
end |
| 394 |
return nil |
| 395 |
end, |
| 396 |
['description'] = [[Boolean function in. |
| 397 |
Returns either nil or its input if input is in args list]], |
| 398 |
['args_schema'] = T.array(T.string()) |
| 399 |
}, |
| 400 |
['not_in'] = { |
| 401 |
['types'] = { |
| 402 |
['string'] = true, |
| 403 |
}, |
| 404 |
['map_type'] = 'string', |
| 405 |
['process'] = function(inp, t, args) |
| 406 |
for _, a in ipairs(args) do |
| 407 |
if a == inp then |
| 408 |
return nil |
| 409 |
end |
| 410 |
end |
| 411 |
return inp, t |
| 412 |
end, |
| 413 |
['description'] = [[Boolean function not in. |
| 414 |
Returns either nil or its input if input is not in args list]], |
| 415 |
['args_schema'] = T.array(T.string()) |
| 416 |
}, |
| 417 |
['inverse'] = { |
| 418 |
['types'] = { |
| 419 |
['string'] = true, |
| 420 |
}, |
| 421 |
['map_type'] = 'string', |
| 422 |
['process'] = function(inp, _, args) |
| 423 |
if inp then |
| 424 |
return nil |
| 425 |
else |
| 426 |
return (args[1] or 'true'), 'string' |
| 427 |
end |
| 428 |
end, |
| 429 |
['description'] = [[Inverses input. |
| 430 |
Empty string comes the first argument or 'true', non-empty string comes nil]], |
| 431 |
['args_schema'] = { T.string():optional() } |
| 432 |
}, |
| 433 |
['ipmask'] = { |
| 434 |
['types'] = { |
| 435 |
['string'] = true, |
| 436 |
}, |
| 437 |
['map_type'] = 'string', |
| 438 |
['process'] = function(inp, _, args) |
| 439 |
local rspamd_ip = require "rspamd_ip" |
| 440 |
-- Non optimal: convert string to an IP address |
| 441 |
local ip = rspamd_ip.from_string(inp) |
| 442 |
|
| 443 |
if not ip or not ip:is_valid() then |
| 444 |
lua_util.debugm(M, "cannot convert %s to IP", inp) |
| 445 |
return nil |
| 446 |
end |
| 447 |
|
| 448 |
if ip:get_version() == 4 then |
| 449 |
local mask = tonumber(args[1]) |
| 450 |
|
| 451 |
return ip:apply_mask(mask):to_string(), 'string' |
| 452 |
else |
| 453 |
-- IPv6 takes the second argument or the first one... |
| 454 |
local mask_str = args[2] or args[1] |
| 455 |
local mask = tonumber(mask_str) |
| 456 |
|
| 457 |
return ip:apply_mask(mask):to_string(), 'string' |
| 458 |
end |
| 459 |
end, |
| 460 |
['description'] = 'Applies mask to IP address.' .. |
| 461 |
' The first argument is the mask for IPv4 addresses, the second is the mask for IPv6 addresses.', |
| 462 |
['args_schema'] = { (T.one_of({ T.number(), T.transform(T.string(), tonumber) })), |
| 463 |
(T.one_of({ T.number(), T.transform(T.string(), tonumber) })):optional() } |
| 464 |
}, |
| 465 |
-- Returns the string(s) with all non ascii chars replaced |
| 466 |
['to_ascii'] = { |
| 467 |
['types'] = { |
| 468 |
['string'] = true, |
| 469 |
['list'] = true, |
| 470 |
}, |
| 471 |
['map_type'] = 'string', |
| 472 |
['process'] = function(inp, _, args) |
| 473 |
if type(inp) == 'table' then |
| 474 |
return fun.map( |
| 475 |
function(s) |
| 476 |
return string.gsub(tostring(s), '[\128-\255]', args[1] or '?') |
| 477 |
end, inp), 'string_list' |
| 478 |
else |
| 479 |
return string.gsub(tostring(inp), '[\128-\255]', '?'), 'string' |
| 480 |
end |
| 481 |
end, |
| 482 |
['description'] = 'Returns the string with all non-ascii bytes replaced with the character ' .. |
| 483 |
'given as second argument or `?`', |
| 484 |
['args_schema'] = { T.string():optional() } |
| 485 |
}, |
| 486 |
-- Extracts tld from a hostname |
| 487 |
['get_tld'] = { |
| 488 |
['types'] = { |
| 489 |
['string'] = true |
| 490 |
}, |
| 491 |
['map_type'] = 'string', |
| 492 |
['process'] = function(inp, _, _) |
| 493 |
return rspamd_util.get_tld(inp), 'string' |
| 494 |
end, |
| 495 |
['description'] = 'Returns effective second-level domain (eSLD) using the Public Suffix List', |
| 496 |
['args_schema'] = {} |
| 497 |
}, |
| 498 |
-- Converts list of strings to numbers and returns a packed string |
| 499 |
['pack_numbers'] = { |
| 500 |
['types'] = { |
| 501 |
['string_list'] = true |
| 502 |
}, |
| 503 |
['map_type'] = 'string', |
| 504 |
['process'] = function(inp, _, args) |
| 505 |
local fmt = args[1] or 'f' |
| 506 |
local res = {} |
| 507 |
for _, s in ipairs(inp) do |
| 508 |
table.insert(res, tonumber(s)) |
| 509 |
end |
| 510 |
return rspamd_util.pack(string.rep(fmt, #res), lua_util.unpack(res)), 'string' |
| 511 |
end, |
| 512 |
['description'] = 'Converts a list of strings to numbers & returns a packed string', |
| 513 |
['args_schema'] = { T.string():optional() } |
| 514 |
}, |
| 515 |
-- Filter nils from a list |
| 516 |
['filter_string_nils'] = { |
| 517 |
['types'] = { |
| 518 |
['string_list'] = true |
| 519 |
}, |
| 520 |
['process'] = function(inp, _, _) |
| 521 |
return fun.filter(function(val) |
| 522 |
return type(val) == 'string' and val ~= 'nil' |
| 523 |
end, inp), 'string_list' |
| 524 |
end, |
| 525 |
['description'] = 'Removes all nils from a list of strings (when converted implicitly)', |
| 526 |
['args_schema'] = {} |
| 527 |
}, |
| 528 |
-- Call a set of methods on a userdata object |
| 529 |
['apply_methods'] = { |
| 530 |
['types'] = { |
| 531 |
['userdata'] = true, |
| 532 |
}, |
| 533 |
['process'] = function(inp, _, args) |
| 534 |
local res = {} |
| 535 |
for _, arg in ipairs(args) do |
| 536 |
local meth = inp[arg] |
| 537 |
local ret = meth(inp) |
| 538 |
if ret then |
| 539 |
table.insert(res, tostring(ret)) |
| 540 |
end |
| 541 |
end |
| 542 |
return res, 'string_list' |
| 543 |
end, |
| 544 |
['description'] = 'Apply a list of method calls to the userdata object', |
| 545 |
}, |
| 546 |
-- Apply method to list of userdata and use it as a filter, excluding elements for which method returns false/nil |
| 547 |
['filter_method'] = { |
| 548 |
['types'] = { |
| 549 |
['userdata_list'] = true |
| 550 |
}, |
| 551 |
['process'] = function(inp, t, args) |
| 552 |
local meth = args[1] |
| 553 |
|
| 554 |
if not meth then |
| 555 |
logger.errx('invalid method name: %s', args[1]) |
| 556 |
return nil |
| 557 |
end |
| 558 |
|
| 559 |
return fun.filter(function(val) |
| 560 |
return val[meth](val) |
| 561 |
end, inp), 'userdata_list' |
| 562 |
end, |
| 563 |
['description'] = 'Apply method to list of userdata and use it as a filter,' .. |
| 564 |
' excluding elements for which method returns false/nil', |
| 565 |
['args_schema'] = { T.string() } |
| 566 |
}, |
| 567 |
} |
| 568 |
|
| 569 |
transform_function.match = transform_function.regexp |
| 570 |
|
| 571 |
return transform_function |