Debian Codegraph @

lualib/lua_selectors/extractors.lua

raw
1 --[[
2 Copyright (c) 2022, Vsevolod Stakhov <[email protected]>
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 ]]--
16
17 local fun = require 'fun'
18 local meta_functions = require "lua_meta"
19 local lua_util = require "lua_util"
20 local rspamd_util = require "rspamd_util"
21 local rspamd_url = require "rspamd_url"
22 local common = require "lua_selectors/common"
23 local T = require "lua_shape.core"
24 local maps = require "lua_selectors/maps"
25 local E = {}
26 local M = "selectors"
27
28 local HOSTNAME = rspamd_util.get_hostname()
29
30 local url_flags_ts = T.array(T.enum(lua_util.keys(rspamd_url.flags))):optional()
31
32 local function gen_exclude_flags_filter(exclude_flags)
33 return function(u)
34 local got_flags = u:get_flags()
35 for _, flag in ipairs(exclude_flags) do
36 if got_flags[flag] then
37 return false
38 end
39 end
40 return true
41 end
42 end
43
44 local extractors = {
45 -- Plain id function
46 ['id'] = {
47 ['get_value'] = function(_, args)
48 if args[1] then
49 return args[1], 'string'
50 end
51
52 return '', 'string'
53 end,
54 ['description'] = [[Return value from function's argument or an empty string,
55 For example, `id('Something')` returns a string 'Something']],
56 ['args_schema'] = { T.string():optional() }
57 },
58 -- Similar but for making lists
59 ['list'] = {
60 ['get_value'] = function(_, args)
61 if args[1] then
62 return fun.map(tostring, args), 'string_list'
63 end
64
65 return {}, 'string_list'
66 end,
67 ['description'] = [[Return a list from function's arguments or an empty list,
68 For example, `list('foo', 'bar')` returns a list {'foo', 'bar'}]],
69 },
70 -- Get source IP address
71 ['ip'] = {
72 ['get_value'] = function(task)
73 local ip = task:get_ip()
74 if ip and ip:is_valid() then
75 return ip, 'userdata'
76 end
77 return nil
78 end,
79 ['description'] = [[Get source IP address]],
80 },
81 -- Get MIME from
82 ['from'] = {
83 ['get_value'] = function(task, args)
84 local from
85 if type(args) == 'table' then
86 from = task:get_from(args)
87 else
88 from = task:get_from(0)
89 end
90 if ((from or E)[1] or E).addr then
91 return from[1], 'table'
92 end
93 return nil
94 end,
95 ['description'] = [[Get MIME or SMTP from (e.g. `from('smtp')` or `from('mime')`,
96 uses any type by default)]],
97 },
98 ['rcpts'] = {
99 ['get_value'] = function(task, args)
100 local rcpts
101 if type(args) == 'table' then
102 rcpts = task:get_recipients(args)
103 else
104 rcpts = task:get_recipients(0)
105 end
106 if ((rcpts or E)[1] or E).addr then
107 return rcpts, 'table_list'
108 end
109 return nil
110 end,
111 ['description'] = [[Get MIME or SMTP rcpts (e.g. `rcpts('smtp')` or `rcpts('mime')`,
112 uses any type by default)]],
113 },
114 -- Get country (ASN module must be executed first)
115 ['country'] = {
116 ['get_value'] = function(task)
117 local country = task:get_mempool():get_variable('country')
118 if not country then
119 return nil
120 else
121 return country, 'string'
122 end
123 end,
124 ['description'] = [[Get country (ASN module must be executed first)]],
125 },
126 -- Get ASN number
127 ['asn'] = {
128 ['type'] = 'string',
129 ['get_value'] = function(task)
130 local asn = task:get_mempool():get_variable('asn')
131 if not asn then
132 return nil
133 else
134 return asn, 'string'
135 end
136 end,
137 ['description'] = [[Get AS number (ASN module must be executed first)]],
138 },
139 -- Get authenticated username
140 ['user'] = {
141 ['get_value'] = function(task)
142 local auser = task:get_user()
143 if not auser then
144 return nil
145 else
146 return auser, 'string'
147 end
148 end,
149 ['description'] = 'Get authenticated user name',
150 },
151 -- Get principal recipient
152 ['to'] = {
153 ['get_value'] = function(task)
154 return task:get_principal_recipient(), 'string'
155 end,
156 ['description'] = 'Get principal recipient',
157 },
158 -- Get content digest
159 ['digest'] = {
160 ['get_value'] = function(task)
161 return task:get_digest(), 'string'
162 end,
163 ['description'] = 'Get content digest',
164 },
165 -- Get list of all attachments digests
166 ['attachments'] = {
167 ['get_value'] = function(task, args)
168 local parts = task:get_parts() or E
169 local digests = {}
170 for i, p in ipairs(parts) do
171 if p:is_attachment() then
172 table.insert(digests, common.get_cached_or_raw_digest(task, i, p, args))
173 end
174 end
175
176 if #digests > 0 then
177 return digests, 'string_list'
178 end
179
180 return nil
181 end,
182 ['description'] = [[Get list of all attachments digests.
183 The first optional argument is encoding (`hex`, `base32` (and forms `bleach32`, `rbase32`), `base64`),
184 the second optional argument is optional hash type (`blake2`, `sha256`, `sha1`, `sha512`, `md5`)]],
185 ['args_schema'] = common.digest_schema()
186
187 },
188 -- Get all attachments files
189 ['files'] = {
190 ['get_value'] = function(task)
191 local parts = task:get_parts() or E
192 local files = {}
193
194 for _, p in ipairs(parts) do
195 local fname = p:get_filename()
196 if fname then
197 table.insert(files, fname)
198 end
199 end
200
201 if #files > 0 then
202 return files, 'string_list'
203 end
204
205 return nil
206 end,
207 ['description'] = 'Get all attachments files',
208 },
209 -- Get languages for text parts
210 ['languages'] = {
211 ['get_value'] = function(task)
212 local text_parts = task:get_text_parts() or E
213 local languages = {}
214
215 for _, p in ipairs(text_parts) do
216 local lang = p:get_language()
217 if lang then
218 table.insert(languages, lang)
219 end
220 end
221
222 if #languages > 0 then
223 return languages, 'string_list'
224 end
225
226 return nil
227 end,
228 ['description'] = 'Get languages for text parts',
229 },
230 -- Get helo value
231 ['helo'] = {
232 ['get_value'] = function(task)
233 return task:get_helo(), 'string'
234 end,
235 ['description'] = 'Get helo value',
236 },
237 -- Get header with the name that is expected as an argument. Returns list of
238 -- headers with this name
239 ['header'] = {
240 ['get_value'] = function(task, args)
241 local strong = false
242 if args[2] then
243 if args[2]:match('strong') then
244 strong = true
245 end
246
247 if args[2]:match('full') then
248 return task:get_header_full(args[1], strong), 'table_list'
249 end
250
251 return task:get_header(args[1], strong), 'string'
252 else
253 return task:get_header(args[1]), 'string'
254 end
255 end,
256 ['description'] = [[Get header with the name that is expected as an argument.
257 The optional second argument accepts list of flags:
258 - `full`: returns all headers with this name with all data (like task:get_header_full())
259 - `strong`: use case sensitive match when matching header's name]],
260 ['args_schema'] = { T.string(),
261 T.string():optional() }
262 },
263 -- Get list of received headers (returns list of tables)
264 ['received'] = {
265 ['get_value'] = function(task, args)
266 local rh = task:get_received_headers()
267 if not rh[1] then
268 return nil
269 end
270 if args[1] then
271 return fun.map(function(r)
272 return r[args[1]]
273 end, rh), 'string_list'
274 end
275
276 return rh, 'table_list'
277 end,
278 ['description'] = [[Get list of received headers.
279 If no arguments specified, returns list of tables. Otherwise, selects a specific element,
280 e.g. `by_hostname`]],
281 },
282 -- Get all urls
283 ['urls'] = {
284 ['get_value'] = function(task, args)
285 local urls = task:get_urls()
286 if not urls[1] then
287 return nil
288 end
289 if args[1] then
290 return fun.map(function(r)
291 return r[args[1]](r)
292 end, urls), 'string_list'
293 end
294 return urls, 'userdata_list'
295 end,
296 ['description'] = [[Get list of all urls.
297 If no arguments specified, returns list of url objects. Otherwise, calls a specific method,
298 e.g. `get_tld`]],
299 },
300 -- Get specific urls
301 ['specific_urls'] = {
302 ['get_value'] = function(task, args)
303 local params = args[1] or {}
304 params.task = task
305 params.no_cache = true
306 if params.exclude_flags then
307 params.filter = gen_exclude_flags_filter(params.exclude_flags)
308 end
309 local urls = lua_util.extract_specific_urls(params)
310 if not urls[1] then
311 return nil
312 end
313 return urls, 'userdata_list'
314 end,
315 ['description'] = [[Get most specific urls. Arguments are equal to the Lua API function]],
316 ['args_schema'] = { T.table {
317 limit = T.one_of({ T.number(), T.transform(T.string(), tonumber) }),
318 esld_limit = T.one_of({ T.number(), T.transform(T.string(), tonumber) }):optional(),
319 exclude_flags = url_flags_ts,
320 flags = url_flags_ts,
321 flags_mode = T.enum { 'explicit' }:optional(),
322 prefix = T.string():optional(),
323 need_content = T.one_of({
324 T.boolean(),
325 T.transform(T.string(), lua_util.toboolean)
326 }):optional(),
327 need_emails = T.one_of({
328 T.boolean(),
329 T.transform(T.string(), lua_util.toboolean)
330 }):optional(),
331 need_images = T.one_of({
332 T.boolean(),
333 T.transform(T.string(), lua_util.toboolean)
334 }):optional(),
335 ignore_redirected = T.one_of({
336 T.boolean(),
337 T.transform(T.string(), lua_util.toboolean)
338 }):optional(),
339 } }
340 },
341 ['specific_urls_filter_map'] = {
342 ['get_value'] = function(task, args)
343 local map = maps[args[1]]
344 if not map then
345 lua_util.debugm(M, "invalid/unknown map: %s", args[1])
346 end
347 local params = args[2] or {}
348 params.task = task
349 params.no_cache = true
350 if params.exclude_flags then
351 params.filter = gen_exclude_flags_filter(params.exclude_flags)
352 end
353 local urls = lua_util.extract_specific_urls(params)
354 if not urls[1] then
355 return nil
356 end
357 return fun.filter(function(u)
358 return map:get_key(tostring(u))
359 end, urls), 'userdata_list'
360 end,
361 ['description'] = [[Get most specific urls, filtered by some map. Arguments are equal to the Lua API function]],
362 ['args_schema'] = { T.string(), T.table {
363 limit = T.one_of({ T.number(), T.transform(T.string(), tonumber) }),
364 esld_limit = T.one_of({ T.number(), T.transform(T.string(), tonumber) }):optional(),
365 exclude_flags = url_flags_ts,
366 flags = url_flags_ts,
367 flags_mode = T.enum { 'explicit' }:optional(),
368 prefix = T.string():optional(),
369 need_content = T.one_of({
370 T.boolean(),
371 T.transform(T.string(), lua_util.toboolean)
372 }):optional(),
373 need_emails = T.one_of({
374 T.boolean(),
375 T.transform(T.string(), lua_util.toboolean)
376 }):optional(),
377 need_images = T.one_of({
378 T.boolean(),
379 T.transform(T.string(), lua_util.toboolean)
380 }):optional(),
381 ignore_redirected = T.one_of({
382 T.boolean(),
383 T.transform(T.string(), lua_util.toboolean)
384 }):optional(),
385 } }
386 },
387 -- URLs filtered by flags
388 ['urls_filtered'] = {
389 ['get_value'] = function(task, args)
390 local urls = task:get_urls_filtered(args[1], args[2])
391 if not urls[1] then
392 return nil
393 end
394 return urls, 'userdata_list'
395 end,
396 ['description'] = [[Get list of all urls filtered by flags_include/exclude
397 (see rspamd_task:get_urls_filtered for description)]],
398 ['args_schema'] = { T.array {
399 url_flags_ts:optional(), url_flags_ts:optional()
400 } }
401 },
402 -- Get all emails
403 ['emails'] = {
404 ['get_value'] = function(task, args)
405 local urls = task:get_emails()
406 if not urls[1] then
407 return nil
408 end
409 if args[1] then
410 return fun.map(function(r)
411 return r[args[1]](r)
412 end, urls), 'string_list'
413 end
414 return urls, 'userdata_list'
415 end,
416 ['description'] = [[Get list of all emails.
417 If no arguments specified, returns list of url objects. Otherwise, calls a specific method,
418 e.g. `get_user`]],
419 },
420 -- Get specific pool var. The first argument must be variable name,
421 -- the second argument is optional and defines the type (string by default)
422 ['pool_var'] = {
423 ['get_value'] = function(task, args)
424 local type = args[2] or 'string'
425 return task:get_mempool():get_variable(args[1], type), (type)
426 end,
427 ['description'] = [[Get specific pool var. The first argument must be variable name,
428 the second argument is optional and defines the type (string by default)]],
429 ['args_schema'] = { T.string(), T.string():optional() }
430 },
431 -- Get value of specific key from task cache
432 ['task_cache'] = {
433 ['get_value'] = function(task, args)
434 local val = task:cache_get(args[1])
435 if not val then
436 return
437 end
438 if type(val) == 'table' then
439 if not val[1] then
440 return
441 end
442 return val, 'string_list'
443 end
444 return val, 'string'
445 end,
446 ['description'] = [[Get value of specific key from task cache. The first argument must be
447 the key name]],
448 ['args_schema'] = { T.string() }
449 },
450 -- Get specific HTTP request header. The first argument must be header name.
451 ['request_header'] = {
452 ['get_value'] = function(task, args)
453 local hdr = task:get_request_header(args[1])
454 if hdr then
455 return tostring(hdr), 'string'
456 end
457
458 return nil
459 end,
460 ['description'] = [[Get specific HTTP request header.
461 The first argument must be header name.]],
462 ['args_schema'] = { T.string() }
463 },
464 -- Get task date, optionally formatted
465 ['time'] = {
466 ['get_value'] = function(task, args)
467 local what = args[1] or 'message'
468 local dt = task:get_date { format = what, gmt = true }
469
470 if dt then
471 if args[2] then
472 -- Should be in format !xxx, as dt is in GMT
473 return os.date(args[2], dt), 'string'
474 end
475
476 return tostring(dt), 'string'
477 end
478
479 return nil
480 end,
481 ['description'] = [[Get task timestamp. The first argument is type:
482 - `connect`: connection timestamp (default)
483 - `message`: timestamp as defined by `Date` header
484
485 The second argument is optional time format, see [os.date](http://pgl.yoyo.org/luai/i/os.date) description]],
486 ['args_schema'] = { T.enum { 'connect', 'message' }:optional(),
487 T.string():optional() }
488 },
489 -- Get text words from a message
490 ['words'] = {
491 ['get_value'] = function(task, args)
492 local how = args[1] or 'stem'
493 local tp = task:get_text_parts()
494
495 if tp then
496 local rtype = 'string_list'
497 if how == 'full' then
498 rtype = 'table_list'
499 end
500
501 return lua_util.flatten(
502 fun.map(function(p)
503 return p:get_words(how)
504 end, tp)), rtype
505 end
506
507 return nil
508 end,
509 ['description'] = [[Get words from text parts
510 - `stem`: stemmed words (default)
511 - `raw`: raw words
512 - `norm`: normalised words (lowercased)
513 - `full`: list of tables
514 ]],
515 ['args_schema'] = { T.enum { 'stem', 'raw', 'norm', 'full' }:optional() },
516 },
517 -- Get queue ID
518 ['queueid'] = {
519 ['get_value'] = function(task)
520 local queueid = task:get_queue_id()
521 if queueid then
522 return queueid, 'string'
523 end
524 return nil
525 end,
526 ['description'] = [[Get queue ID]],
527 },
528 -- Get ID of the task being processed
529 ['uid'] = {
530 ['get_value'] = function(task)
531 local uid = task:get_uid()
532 if uid then
533 return uid, 'string'
534 end
535 return nil
536 end,
537 ['description'] = [[Get ID of the task being processed]],
538 },
539 -- Get message ID of the task being processed
540 ['messageid'] = {
541 ['get_value'] = function(task)
542 local mid = task:get_message_id()
543 if mid then
544 return mid, 'string'
545 end
546 return nil
547 end,
548 ['description'] = [[Get message ID]],
549 },
550 -- Get specific symbol
551 ['symbol'] = {
552 ['get_value'] = function(task, args)
553 local symbol = task:get_symbol(args[1], args[2])
554 if symbol then
555 return symbol[1], 'table'
556 end
557 end,
558 ['description'] = 'Get specific symbol. The first argument must be the symbol name. ' ..
559 'The second argument is an optional shadow result name. ' ..
560 'Returns the symbol table. See task:get_symbol()',
561 ['args_schema'] = { T.string(), T.string():optional() }
562 },
563 -- Get full scan result
564 ['scan_result'] = {
565 ['get_value'] = function(task, args)
566 local res = task:get_metric_result(args[1])
567 if res then
568 return res, 'table'
569 end
570 end,
571 ['description'] = 'Get full scan result (either default or shadow if shadow result name is specified)' ..
572 'Returns the result table. See task:get_metric_result()',
573 ['args_schema'] = { T.string():optional() }
574 },
575 -- Get list of metatokens as strings
576 ['metatokens'] = {
577 ['get_value'] = function(task)
578 local tokens = meta_functions.gen_metatokens(task)
579 if not tokens[1] then
580 return nil
581 end
582 local res = {}
583 for _, t in ipairs(tokens) do
584 table.insert(res, tostring(t))
585 end
586 return res, 'string_list'
587 end,
588 ['description'] = 'Get metatokens for a message as strings',
589 },
590 ['rspamd_hostname'] = {
591 ['get_value'] = function(task)
592 return HOSTNAME, 'string'
593 end,
594 ['description'] = 'Get hostname of the filter server',
595 },
596 }
597
598 return extractors