Debian Codegraph @

lualib/lua_util.lua

raw
1 --[[
2 Copyright (c) 2023, Vsevolod Stakhov <[email protected]>
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 ]] --
16
17 --[[[
18 -- @module lua_util
19 -- This module contains utility functions for working with Lua and/or Rspamd
20 --]]
21
22 local exports = {}
23 local lpeg = require 'lpeg'
24 local rspamd_util = require "rspamd_util"
25 local fun = require "fun"
26 local lupa = require "lupa"
27
28 local split_grammar = {}
29 local spaces_split_grammar
30 local space = lpeg.S ' \t\n\v\f\r'
31 local nospace = 1 - space
32 local ptrim = space ^ 0 * lpeg.C((space ^ 0 * nospace ^ 1) ^ 0)
33 local match = lpeg.match
34
35 local function shallowcopy(orig)
36 local orig_type = type(orig)
37 local copy
38 if orig_type == 'table' then
39 copy = {}
40 for orig_key, orig_value in pairs(orig) do
41 copy[orig_key] = orig_value
42 end
43 else
44 copy = orig
45 end
46 return copy
47 end
48 local function deepcopy(orig)
49 local orig_type = type(orig)
50 local copy
51 if orig_type == 'table' then
52 copy = {}
53 for orig_key, orig_value in next, orig, nil do
54 copy[deepcopy(orig_key)] = deepcopy(orig_value)
55 end
56 if getmetatable(orig) then
57 setmetatable(copy, deepcopy(getmetatable(orig)))
58 end
59 else
60 -- number, string, boolean, etc
61 copy = orig
62 end
63 return copy
64 end
65
66 lupa.configure('{%', '%}', '{=', '=}', '{#', '#}', {
67 keep_trailing_newline = true,
68 autoescape = false,
69 })
70
71 lupa.filters.pbkdf = function(s)
72 local cr = require "rspamd_cryptobox"
73 return cr.pbkdf(s)
74 end
75
76 -- Dirty hacks to avoid shared state
77 package.loaded['lupa'] = nil
78 local lupa_orig = require "lupa"
79
80 local function rspamd_str_split(s, sep)
81 local gr
82 if not sep then
83 if not spaces_split_grammar then
84 local _sep = space
85 local elem = lpeg.C((1 - _sep) ^ 0)
86 local p = lpeg.Ct(elem * (_sep * elem) ^ 0)
87 spaces_split_grammar = p
88 end
89
90 gr = spaces_split_grammar
91 else
92 gr = split_grammar[sep]
93
94 if not gr then
95 local _sep
96 if type(sep) == 'string' then
97 _sep = lpeg.S(sep) -- Assume set
98 else
99 _sep = sep -- Assume lpeg object
100 end
101 local elem = lpeg.C((1 - _sep) ^ 0)
102 local p = lpeg.Ct(elem * (_sep * elem) ^ 0)
103 gr = p
104 split_grammar[sep] = gr
105 end
106 end
107
108 return gr:match(s)
109 end
110
111 --[[[
112 -- @function lua_util.str_split(text, delimiter)
113 -- Splits text into a numeric table by delimiter
114 -- @param {string} text delimited text
115 -- @param {string} delimiter the delimiter
116 -- @return {table} numeric table containing string parts
117 --]]
118
119 exports.rspamd_str_split = rspamd_str_split
120 exports.str_split = rspamd_str_split
121
122 local function rspamd_str_trim(s)
123 return match(ptrim, s)
124 end
125 exports.rspamd_str_trim = rspamd_str_trim
126 --[[[
127 -- @function lua_util.str_trim(text)
128 -- Returns a string with no trailing and leading spaces
129 -- @param {string} text input text
130 -- @return {string} string with no trailing and leading spaces
131 --]]
132 exports.str_trim = rspamd_str_trim
133
134 --[[[
135 -- @function lua_util.str_startswith(text, prefix)
136 -- @param {string} text
137 -- @param {string} prefix
138 -- @return {boolean} true if text starts with the specified prefix, false otherwise
139 --]]
140 exports.str_startswith = function(s, prefix)
141 return s:sub(1, prefix:len()) == prefix
142 end
143
144 --[[[
145 -- @function lua_util.str_endswith(text, suffix)
146 -- @param {string} text
147 -- @param {string} suffix
148 -- @return {boolean} true if text ends with the specified suffix, false otherwise
149 --]]
150 exports.str_endswith = function(s, suffix)
151 return s:find(suffix, -suffix:len(), true) ~= nil
152 end
153
154 --[[[
155 -- @function lua_util.round(number, decimalPlaces)
156 -- Round number to fixed number of decimal points
157 -- @param {number} number number to round
158 -- @param {number} decimalPlaces number of decimal points
159 -- @return {number} rounded number
160 --]]
161
162 -- modified version from Robert Jay Gould http://lua-users.org/wiki/SimpleRound
163 exports.round = function(num, numDecimalPlaces)
164 local mult = 10 ^ (numDecimalPlaces or 0)
165 if num >= 0 then
166 return math.floor(num * mult + 0.5) / mult
167 else
168 return math.ceil(num * mult - 0.5) / mult
169 end
170 end
171
172 --[[[
173 -- @function lua_util.template(text, replacements)
174 -- Replaces values in a text template
175 -- Variable names can contain letters, numbers and underscores, are prefixed with `$` and may or not use curly braces.
176 -- @param {string} text text containing variables
177 -- @param {table} replacements key/value pairs for replacements
178 -- @return {string} string containing replaced values
179 -- @example
180 -- local goop = lua_util.template("HELLO $FOO ${BAR}!", {['FOO'] = 'LUA', ['BAR'] = 'WORLD'})
181 -- -- goop contains "HELLO LUA WORLD!"
182 --]]
183
184 exports.template = function(tmpl, keys)
185 local var_lit = lpeg.P { lpeg.R("az") + lpeg.R("AZ") + lpeg.R("09") + "_" }
186 local var = lpeg.P { (lpeg.P("$") / "") * ((var_lit ^ 1) / keys) }
187 local var_braced = lpeg.P { (lpeg.P("${") / "") * ((var_lit ^ 1) / keys) * (lpeg.P("}") / "") }
188
189 local template_grammar = lpeg.Cs((var + var_braced + 1) ^ 0)
190
191 return lpeg.match(template_grammar, tmpl)
192 end
193
194 local function enrich_template_with_globals(env)
195 local newenv = shallowcopy(env)
196 newenv.paths = rspamd_paths
197 newenv.env = rspamd_env
198
199 return newenv
200 end
201 --[[[
202 -- @function lua_util.jinja_template(text, env[, skip_global_env][, is_orig][, custom_filters])
203 -- Replaces values in a text template according to jinja2 syntax
204 -- @param {string} text text containing variables
205 -- @param {table} replacements key/value pairs for replacements
206 -- @param {boolean} skip_global_env don't export Rspamd superglobals
207 -- @param {boolean} is_orig use the original lupa configuration with {% raw %}`{{`{% endraw %} for variables
208 -- @param {table} custom_filters custom filters to use (or nil if not needed)
209 -- @return {string} string containing replaced values
210 -- @example
211 -- lua_util.jinja_template("HELLO {=FOO=} {=BAR=}!", {['FOO'] = 'LUA', ['BAR'] = 'WORLD'})
212 -- "HELLO LUA WORLD!"
213 --]]
214 exports.jinja_template = function(text, env, skip_global_env, is_orig, custom_filters)
215 local lupa_to_use = is_orig and lupa_orig or lupa
216 if not skip_global_env then
217 env = enrich_template_with_globals(env)
218 end
219
220 local orig_filters = {}
221 if type(custom_filters) == 'table' then
222 for k, v in pairs(custom_filters) do
223 orig_filters[k] = lupa_to_use.filters[k]
224 lupa_to_use.filters[k] = v
225 end
226 end
227
228 local result = lupa_to_use.expand(text, env)
229
230 -- Restore custom filters
231 if type(custom_filters) == 'table' then
232 for k, _ in pairs(custom_filters) do
233 lupa_to_use.filters[k] = orig_filters[k]
234 end
235 end
236
237 return result
238 end
239
240 --[[[
241 -- @function lua_util.jinja_file(filename, env[, skip_global_env][, is_orig][, custom_filters])
242 -- Replaces values in a text template according to jinja2 syntax
243 -- @param {string} filename name of file to expand
244 -- @param {table} replacements key/value pairs for replacements
245 -- @param {boolean} skip_global_env don't export Rspamd superglobals
246 -- @param {boolean} is_orig use the original lupa configuration with {% raw %}`{{`{% endraw %} for variables
247 -- @param {table} custom_filters custom filters to use (or nil if not needed)
248 -- @return {string} string containing replaced values
249 -- @example
250 -- lua_util.jinja_template("HELLO {=FOO=} {=BAR=}!", {['FOO'] = 'LUA', ['BAR'] = 'WORLD'})
251 -- "HELLO LUA WORLD!"
252 --]]
253 exports.jinja_template_file = function(filename, env, skip_global_env, is_orig, custom_filters)
254 local lupa_to_use = is_orig and lupa_orig or lupa
255 if not skip_global_env then
256 env = enrich_template_with_globals(env)
257 end
258
259 local orig_filters = {}
260 if type(custom_filters) == 'table' then
261 for k, v in pairs(custom_filters) do
262 orig_filters[k] = lupa_to_use.filters[k]
263 lupa_to_use.filters[k] = v
264 end
265 end
266
267 local result = lupa_to_use.expand_file(filename, env)
268
269 -- Restore custom filters
270 if type(custom_filters) == 'table' then
271 for k, _ in pairs(custom_filters) do
272 lupa_to_use.filters[k] = orig_filters[k]
273 end
274 end
275
276 return result
277 end
278
279 exports.remove_email_aliases = function(email_addr)
280 local function check_gmail_user(addr)
281 -- Remove all points
282 local no_dots_user = string.gsub(addr.user, '%.', '')
283 local cap, pluses = string.match(no_dots_user, '^([^%+][^%+]*)(%+.*)$')
284 if cap then
285 return cap, rspamd_str_split(pluses, '+'), nil
286 elseif no_dots_user ~= addr.user then
287 return no_dots_user, {}, nil
288 end
289
290 return nil
291 end
292
293 local function check_address(addr)
294 if addr.user then
295 local cap, pluses = string.match(addr.user, '^([^%+][^%+]*)(%+.*)$')
296 if cap then
297 return cap, rspamd_str_split(pluses, '+'), nil
298 end
299 end
300
301 return nil
302 end
303
304 local function set_addr(addr, new_user, new_domain)
305 if new_user then
306 addr.user = new_user
307 end
308 if new_domain then
309 addr.domain = new_domain
310 end
311
312 if addr.domain then
313 addr.addr = string.format('%s@%s', addr.user, addr.domain)
314 else
315 addr.addr = string.format('%s@', addr.user)
316 end
317
318 if addr.name and #addr.name > 0 then
319 addr.raw = string.format('"%s" <%s>', addr.name, addr.addr)
320 else
321 addr.raw = string.format('<%s>', addr.addr)
322 end
323 end
324
325 local function check_gmail(addr)
326 local nu, tags, nd = check_gmail_user(addr)
327
328 if nu then
329 return nu, tags, nd
330 end
331
332 return nil
333 end
334
335 local function check_googlemail(addr)
336 local nd = 'gmail.com'
337 local nu, tags = check_gmail_user(addr)
338
339 if nu then
340 return nu, tags, nd
341 end
342
343 return nil, nil, nd
344 end
345
346 local specific_domains = {
347 ['gmail.com'] = check_gmail,
348 ['googlemail.com'] = check_googlemail,
349 }
350
351 if email_addr then
352 if email_addr.domain and specific_domains[email_addr.domain] then
353 local nu, tags, nd = specific_domains[email_addr.domain](email_addr)
354 if nu or nd then
355 set_addr(email_addr, nu, nd)
356
357 return nu, tags
358 end
359 else
360 local nu, tags, nd = check_address(email_addr)
361 if nu or nd then
362 set_addr(email_addr, nu, nd)
363
364 return nu, tags
365 end
366 end
367
368 return nil
369 end
370 end
371
372 exports.is_rspamc_or_controller = function(task)
373 local ua = task:get_request_header('User-Agent') or ''
374 local pwd = task:get_request_header('Password')
375 local is_rspamc = false
376 if tostring(ua) == 'rspamc' or pwd then
377 is_rspamc = true
378 end
379
380 return is_rspamc
381 end
382
383 --[[[
384 -- @function lua_util.unpack(table)
385 -- Converts numeric table to varargs
386 -- This is `unpack` on Lua 5.1/5.2/LuaJIT and `table.unpack` on Lua 5.3
387 -- @param {table} table numerically indexed table to unpack
388 -- @return {varargs} unpacked table elements
389 --]]
390
391 local unpack_function = table.unpack or unpack
392 exports.unpack = function(...)
393 return unpack_function(...)
394 end
395
396 --[[[
397 -- @function lua_util.flatten(table)
398 -- Flatten underlying tables in a single table
399 -- @param {table} table table of tables
400 -- @return {table} flattened table
401 --]]
402 exports.flatten = function(t)
403 local res = {}
404 for _, e in fun.iter(t) do
405 for _, v in fun.iter(e) do
406 res[#res + 1] = v
407 end
408 end
409
410 return res
411 end
412
413 --[[[
414 -- @function lua_util.spairs(table)
415 -- Like `pairs` but keys are sorted lexicographically
416 -- @param {table} table table containing key/value pairs
417 -- @return {function} generator function returning key/value pairs
418 --]]
419
420 -- Sorted iteration:
421 -- for k,v in spairs(t) do ... end
422 --
423 -- or with custom comparison:
424 -- for k, v in spairs(t, function(t, a, b) return t[a] < t[b] end)
425 --
426 -- optional limit is also available (e.g. return top X elements)
427 local function spairs(t, order, lim)
428 -- collect the keys
429 local keys = {}
430 for k in pairs(t) do
431 keys[#keys + 1] = k
432 end
433
434 -- if order function given, sort by it by passing the table and keys a, b,
435 -- otherwise just sort the keys
436 if order then
437 table.sort(keys, function(a, b)
438 return order(t, a, b)
439 end)
440 else
441 table.sort(keys)
442 end
443
444 -- return the iterator function
445 local i = 0
446 return function()
447 i = i + 1
448 if not lim or i <= lim then
449 if keys[i] then
450 return keys[i], t[keys[i]]
451 end
452 end
453 end
454 end
455
456 exports.spairs = spairs
457
458 local lua_cfg_utils = require "lua_cfg_utils"
459
460 exports.config_utils = lua_cfg_utils
461 exports.disable_module = lua_cfg_utils.disable_module
462
463 --[[[
464 -- @function lua_util.disable_module(modname)
465 -- Checks experimental plugins state and disable if needed
466 -- @param {string} modname name of plugin to check
467 -- @return {boolean} true if plugin should be enabled, false otherwise
468 --]]
469 local function check_experimental(modname)
470 if rspamd_config:experimental_enabled() then
471 return true
472 else
473 lua_cfg_utils.disable_module(modname, 'experimental')
474 end
475
476 return false
477 end
478
479 exports.check_experimental = check_experimental
480
481 --[[[
482 -- @function lua_util.list_to_hash(list)
483 -- Converts numerically-indexed table to table indexed by values
484 -- @param {table} list numerically-indexed table or string, which is treated as a one-element list
485 -- @return {table} table indexed by values
486 -- @example
487 -- local h = lua_util.list_to_hash({"a", "b"})
488 -- -- h contains {a = true, b = true}
489 --]]
490 local function list_to_hash(list)
491 if type(list) == 'table' then
492 if list[1] then
493 local h = {}
494 for _, e in ipairs(list) do
495 h[e] = true
496 end
497 return h
498 else
499 return list
500 end
501 elseif type(list) == 'string' then
502 local h = {}
503 h[list] = true
504 return h
505 end
506 end
507
508 exports.list_to_hash = list_to_hash
509
510 --[[[
511 -- @function lua_util.nkeys(table|gen, param, state)
512 -- Returns number of keys in a table (i.e. from both the array and hash parts combined)
513 -- @param {table} list numerically-indexed table or string, which is treated as a one-element list
514 -- @return {number} number of keys
515 -- @example
516 -- print(lua_util.nkeys({})) -- 0
517 -- print(lua_util.nkeys({ "a", nil, "b" })) -- 2
518 -- print(lua_util.nkeys({ dog = 3, cat = 4, bird = nil })) -- 2
519 -- print(lua_util.nkeys({ "a", dog = 3, cat = 4 })) -- 3
520 --
521 --]]
522 local function nkeys(gen, param, state)
523 local n = 0
524 if not param then
525 for _, _ in pairs(gen) do
526 n = n + 1
527 end
528 else
529 for _, _ in fun.iter(gen, param, state) do
530 n = n + 1
531 end
532 end
533 return n
534 end
535
536 exports.nkeys = nkeys
537
538 --[[[
539 -- @function lua_util.parse_time_interval(str)
540 -- Parses human readable time interval
541 -- Accepts 's' for seconds, 'm' for minutes, 'h' for hours, 'd' for days,
542 -- 'w' for weeks, 'y' for years
543 -- @param {string} str input string
544 -- @return {number|nil} parsed interval as seconds (might be fractional)
545 --]]
546 local function parse_time_interval(str)
547 local function parse_time_suffix(s)
548 if s == 's' then
549 return 1
550 elseif s == 'm' then
551 return 60
552 elseif s == 'h' then
553 return 3600
554 elseif s == 'd' then
555 return 86400
556 elseif s == 'w' then
557 return 86400 * 7
558 elseif s == 'y' then
559 return 365 * 86400;
560 end
561 end
562
563 local digit = lpeg.R("09")
564 local parser = {}
565 parser.integer = (lpeg.S("+-") ^ -1) *
566 (digit ^ 1)
567 parser.fractional = (lpeg.P(".")) *
568 (digit ^ 1)
569 parser.number = (parser.integer *
570 (parser.fractional ^ -1)) +
571 (lpeg.S("+-") * parser.fractional)
572 parser.time = lpeg.Cf(lpeg.Cc(1) *
573 (parser.number / tonumber) *
574 ((lpeg.S("smhdwy") / parse_time_suffix) ^ -1),
575 function(acc, val)
576 return acc * val
577 end)
578
579 local t = lpeg.match(parser.time, str)
580
581 return t
582 end
583
584 exports.parse_time_interval = parse_time_interval
585
586 --[[[
587 -- @function lua_util.dehumanize_number(str)
588 -- Parses human readable number
589 -- Accepts 'k' for thousands, 'm' for millions, 'g' for billions, 'b' suffix for 1024 multiplier,
590 -- e.g. `10mb` equal to `10 * 1024 * 1024`
591 -- @param {string} str input string
592 -- @return {number|nil} parsed number
593 --]]
594 local function dehumanize_number(str)
595 local function parse_suffix(s)
596 if s == 'k' then
597 return 1000
598 elseif s == 'm' then
599 return 1000000
600 elseif s == 'g' then
601 return 1e9
602 elseif s == 'kb' then
603 return 1024
604 elseif s == 'mb' then
605 return 1024 * 1024
606 elseif s == 'gb' then
607 return 1024 * 1024;
608 end
609 end
610
611 local digit = lpeg.R("09")
612 local parser = {}
613 parser.integer = (lpeg.S("+-") ^ -1) *
614 (digit ^ 1)
615 parser.fractional = (lpeg.P(".")) *
616 (digit ^ 1)
617 parser.number = (parser.integer *
618 (parser.fractional ^ -1)) +
619 (lpeg.S("+-") * parser.fractional)
620 parser.humanized_number = lpeg.Cf(lpeg.Cc(1) *
621 (parser.number / tonumber) *
622 (((lpeg.S("kmg") * (lpeg.P("b") ^ -1)) / parse_suffix) ^ -1),
623 function(acc, val)
624 return acc * val
625 end)
626
627 local t = lpeg.match(parser.humanized_number, str)
628
629 return t
630 end
631
632 exports.dehumanize_number = dehumanize_number
633
634 --[[[
635 -- @function lua_util.table_cmp(t1, t2)
636 -- Compare two tables deeply
637 --]]
638 local function table_cmp(table1, table2)
639 local avoid_loops = {}
640 local function recurse(t1, t2)
641 if type(t1) ~= type(t2) then
642 return false
643 end
644 if type(t1) ~= "table" then
645 return t1 == t2
646 end
647
648 if avoid_loops[t1] then
649 return avoid_loops[t1] == t2
650 end
651 avoid_loops[t1] = t2
652 -- Copy keys from t2
653 local t2keys = {}
654 local t2tablekeys = {}
655 for k, _ in pairs(t2) do
656 if type(k) == "table" then
657 table.insert(t2tablekeys, k)
658 end
659 t2keys[k] = true
660 end
661 -- Let's iterate keys from t1
662 for k1, v1 in pairs(t1) do
663 local v2 = t2[k1]
664 if type(k1) == "table" then
665 -- if key is a table, we need to find an equivalent one.
666 local ok = false
667 for i, tk in ipairs(t2tablekeys) do
668 if table_cmp(k1, tk) and recurse(v1, t2[tk]) then
669 table.remove(t2tablekeys, i)
670 t2keys[tk] = nil
671 ok = true
672 break
673 end
674 end
675 if not ok then
676 return false
677 end
678 else
679 -- t1 has a key which t2 doesn't have, fail.
680 if v2 == nil then
681 return false
682 end
683 t2keys[k1] = nil
684 if not recurse(v1, v2) then
685 return false
686 end
687 end
688 end
689 -- if t2 has a key which t1 doesn't have, fail.
690 if next(t2keys) then
691 return false
692 end
693 return true
694 end
695 return recurse(table1, table2)
696 end
697
698 exports.table_cmp = table_cmp
699
700 --[[[
701 -- @function lua_util.table_merge(t1, t2)
702 -- Merge two tables
703 --]]
704 local function table_merge(t1, t2)
705 local res = {}
706 local nidx = 1 -- for numeric indicies
707 local it_func = function(k, v)
708 if type(k) == 'number' then
709 res[nidx] = v
710 nidx = nidx + 1
711 else
712 res[k] = v
713 end
714 end
715 for k, v in pairs(t1) do
716 it_func(k, v)
717 end
718 for k, v in pairs(t2) do
719 it_func(k, v)
720 end
721 return res
722 end
723
724 exports.table_merge = table_merge
725
726 --[[[
727 -- @function lua_util.table_cmp(task, name, value, stop_chars)
728 -- Performs header folding
729 --]]
730 exports.fold_header = function(task, name, value, stop_chars)
731 local how
732
733 if task:has_flag("milter") then
734 how = "lf"
735 else
736 how = task:get_newlines_type()
737 end
738
739 return rspamd_util.fold_header(name, value, how, stop_chars)
740 end
741
742 --[[[
743 -- @function lua_util.fold_header_with_encoding(task, name, value[, opts])
744 -- Folds header value using name and optionally encodes the result.
745 -- Encoding policy defaults to 'auto':
746 -- - If MIME UTF-8 is enabled, encode only when the folded value is not valid UTF-8
747 -- - Otherwise, always encode
748 -- `opts` table fields:
749 -- * stop_chars: optional string with extra fold-on characters
750 -- * encode: true|false|'auto' (default: 'auto')
751 -- * structured: boolean, pass as `is_structured` to mime_header_encode (default: false)
752 -- @return {string} folded (and possibly encoded) header value
753 --]]
754 exports.fold_header_with_encoding = function(task, name, value, opts)
755 opts = opts or {}
756 local stop_chars = opts.stop_chars
757 local encode = opts.encode
758 local structured = opts.structured or false
759
760 local folded = exports.fold_header(task, name, value, stop_chars)
761
762 if encode == nil or encode == 'auto' then
763 if rspamd_config:is_mime_utf8() then
764 if not rspamd_util.is_valid_utf8(folded) then
765 folded = rspamd_util.mime_header_encode(folded, structured)
766 end
767 else
768 folded = rspamd_util.mime_header_encode(folded, structured)
769 end
770 elseif encode == true then
771 folded = rspamd_util.mime_header_encode(folded, structured)
772 end
773
774 return folded
775 end
776
777 --[[[
778 -- @function lua_util.override_defaults(defaults, override)
779 -- Overrides values from defaults with override
780 --]]
781 local function override_defaults(def, override)
782 -- Corner cases
783 if not override or type(override) ~= 'table' then
784 return def
785 end
786 if not def or type(def) ~= 'table' then
787 return override
788 end
789
790 local res = {}
791
792 -- Allow transparent ucl
793 local pairs_func, def_pairs_func = pairs, pairs
794 local type_func, def_type_func = type, type
795 if type(override[0]) == 'userdata' then
796 pairs_func = function(t)
797 return t:pairs()
798 end
799 type_func = function(t)
800 return t:type()
801 end
802 end
803 if type(def[0]) == 'userdata' then
804 def_pairs_func = function(t)
805 return t:pairs()
806 end
807 def_type_func = function(t)
808 return t:type()
809 end
810 end
811
812 for k, v in pairs_func(override) do
813 if type_func(v) == 'table' then
814 if def[k] and def_type_func(def[k]) == 'table' then
815 -- Recursively override elements
816 res[k] = override_defaults(def[k], v)
817 else
818 res[k] = v
819 end
820 else
821 res[k] = v
822 end
823 end
824
825 for k, v in def_pairs_func(def) do
826 if type(res[k]) == 'nil' then
827 res[k] = v
828 end
829 end
830
831 return res
832 end
833
834 exports.override_defaults = override_defaults
835
836 --[[[
837 -- @function lua_util.filter_specific_urls(urls, params)
838 -- params: {
839 - - task - if needed to save in the cache
840 - - limit <int> (default = 9999)
841 - - esld_limit <int> (default = 9999) n domains per eSLD (effective second level domain)
842 works only if number of unique eSLD less than `limit`
843 - - need_emails <bool> (default = false)
844 - - filter <callback> (default = nil)
845 - - prefix <string> cache prefix (default = nil)
846 -- }
847 -- Apply heuristic in extracting of urls from `urls` table, this function
848 -- tries its best to extract specific number of urls from a task based on
849 -- their characteristics
850 --]]
851 exports.filter_specific_urls = function(urls, params)
852 local cache_key
853
854 if params.task and not params.no_cache then
855 if params.prefix then
856 cache_key = params.prefix
857 else
858 cache_key = string.format('sp_urls_%d%s%s%s', params.limit,
859 tostring(params.need_emails or false),
860 tostring(params.need_images or false),
861 tostring(params.need_content or false))
862 end
863 local cached = params.task:cache_get(cache_key)
864
865 if cached then
866 return cached
867 end
868 end
869
870 if not urls then
871 return {}
872 end
873
874 if params.filter then
875 urls = fun.totable(fun.filter(params.filter, urls))
876 end
877
878 -- Memory bounds protection against DoS: absolute maximum of URLs to process
879 -- Even if we have 100k URLs of 1KB each in input, we process at most first 50k
880 -- This prevents excessive memory usage and CPU time for malicious messages
881 local max_urls_to_process = 50000
882 if #urls > max_urls_to_process then
883 local logger = require "rspamd_logger"
884 logger.warnx(params.task, 'too many URLs to process: %d, limiting to %d',
885 #urls, max_urls_to_process)
886 -- Truncate the urls table
887 local truncated = {}
888 for i = 1, max_urls_to_process do
889 truncated[i] = urls[i]
890 end
891 urls = truncated
892 end
893
894 -- Filter by tld:
895 local tlds = {}
896 local eslds = {}
897 local ntlds, neslds = 0, 0
898
899 -- Use two structures: hash set for deduplication, array for results
900 local res = {} -- array of URLs (maintains order)
901 local seen = {} -- hash set for deduplication (hash -> true)
902 local nres = 0
903
904 local cta_priority_map
905
906 if params.task and params.task.get_text_parts then
907 local text_parts = params.task:get_text_parts()
908 if text_parts then
909 cta_priority_map = {}
910 for _, part in ipairs(text_parts) do
911 if part.is_html and part:is_html() and part.get_cta_urls then
912 local entries = part:get_cta_urls({original = true, with_weights = true})
913 if type(entries) == 'table' then
914 for _, entry in ipairs(entries) do
915 if entry and entry.url then
916 local url = entry.url
917 -- Use hash instead of tostring to avoid string interning
918 local hash = url:get_hash()
919 local weight = entry.weight or 0
920 local score = 6 + math.floor(weight * 10 + 0.5)
921 if not cta_priority_map[hash] or score > cta_priority_map[hash] then
922 cta_priority_map[hash] = score
923 end
924 local redir = url:get_redirected()
925 if redir then
926 -- Skip display-only URLs (phishing bait text) - only include real redirects
927 local redir_flags = redir:get_flags()
928 if not redir_flags.html_displayed then
929 local rhash = redir:get_hash()
930 if not cta_priority_map[rhash] or score > cta_priority_map[rhash] then
931 cta_priority_map[rhash] = score
932 end
933 end
934 end
935 end
936 end
937 end
938 end
939 end
940
941 if next(cta_priority_map) == nil then
942 cta_priority_map = nil
943 end
944 end
945 end
946
947 local function insert_url(hash, u)
948 if not seen[hash] then
949 seen[hash] = true
950 table.insert(res, u)
951 nres = nres + 1
952
953 return true
954 end
955
956 return false
957 end
958
959 local url_index = 0 -- Track URL processing order for stable sorting
960 local function process_single_url(u, default_priority)
961 url_index = url_index + 1
962 local priority = default_priority or 1 -- Normal priority
963 local flags = u:get_flags()
964 if params.ignore_ip and flags.numeric then
965 return
966 end
967
968 if flags.redirected then
969 local redir = u:get_redirected() -- get the real url
970
971 if params.ignore_redirected then
972 -- Replace `u` with redir
973 u = redir
974 priority = 2
975 else
976 -- Process both redirected url and the original one
977 process_single_url(redir, 2)
978 end
979 end
980
981 if flags.image then
982 if not params.need_images then
983 -- Ignore url
984 return
985 else
986 -- Penalise images in urls
987 priority = 0
988 end
989 end
990
991 local esld = u:get_tld()
992 -- Use fast hash instead of tostring to avoid string interning
993 -- This is critical for DoS protection with 100k+ URLs
994 local url_hash = u:get_hash()
995
996 if cta_priority_map then
997 local cta_pr = cta_priority_map[url_hash]
998 if not cta_pr and flags.redirected then
999 local redir_url = u:get_redirected()
1000 if redir_url then
1001 cta_pr = cta_priority_map[redir_url:get_hash()]
1002 end
1003 end
1004
1005 if cta_pr then
1006 -- Use CTA priority as base, but add phished/subject bonuses on top
1007 priority = cta_pr
1008 if flags.phished then
1009 priority = priority + 5 -- Phished URLs get significant extra priority (security critical)
1010 end
1011 if flags.subject then
1012 priority = priority + 2 -- Subject URLs get extra priority
1013 end
1014 end
1015 end
1016
1017 if esld then
1018 -- Special cases (only apply if CTA priority wasn't set)
1019 -- CTA priority takes precedence as it's explicitly calculated with bonuses
1020 local had_cta = priority ~= (default_priority or 1)
1021
1022 if not had_cta then
1023 if (u:get_protocol() ~= 'mailto') and (not flags.html_displayed) then
1024 if flags.obscured then
1025 priority = 3
1026 else
1027 if (flags.has_user or flags.has_port) then
1028 priority = 2
1029 elseif (flags.subject or flags.phished) then
1030 priority = 2
1031 end
1032 end
1033 elseif flags.html_displayed then
1034 priority = 0
1035 end
1036 end
1037
1038 if not eslds[esld] then
1039 eslds[esld] = { { url_hash, u, priority, url_index } }
1040 neslds = neslds + 1
1041 else
1042 if #eslds[esld] < params.esld_limit then
1043 table.insert(eslds[esld], { url_hash, u, priority, url_index })
1044 end
1045 end
1046
1047
1048 -- eSLD - 1 part => tld
1049 local parts = rspamd_str_split(esld, '.')
1050 local tld = table.concat(fun.totable(fun.tail(parts)), '.')
1051
1052 if not tlds[tld] then
1053 tlds[tld] = { { url_hash, u, priority, url_index } }
1054 ntlds = ntlds + 1
1055 else
1056 table.insert(tlds[tld], { url_hash, u, priority, url_index })
1057 end
1058 end
1059 end
1060
1061 for _, u in ipairs(urls) do
1062 process_single_url(u)
1063 end
1064
1065 local limit = params.limit
1066 limit = limit - nres
1067 if limit < 0 then
1068 limit = 0
1069 end
1070
1071 if limit == 0 then
1072 if params.task and not params.no_cache then
1073 params.task:cache_set(cache_key, res)
1074 end
1075 return res
1076 end
1077
1078 -- Sort eSLDs and tlds
1079 local function sort_stuff(tbl)
1080 -- Sort according to max priority
1081 table.sort(tbl, function(e1, e2)
1082 -- Sort by priority (desc) then by url_index (asc) for stable sorting
1083 table.sort(e1, function(tr1, tr2)
1084 if tr1[3] ~= tr2[3] then
1085 return tr1[3] < tr2[3] -- Lower priority first
1086 else
1087 return tr1[4] < tr2[4] -- Earlier URL index first (stable)
1088 end
1089 end)
1090 table.sort(e2, function(tr1, tr2)
1091 if tr1[3] ~= tr2[3] then
1092 return tr1[3] < tr2[3]
1093 else
1094 return tr1[4] < tr2[4]
1095 end
1096 end)
1097
1098 if e1[#e1][3] ~= e2[#e2][3] then
1099 -- Sort by priority so max priority is at the beginning
1100 return e1[#e1][3] > e2[#e2][3]
1101 else
1102 -- Prefer less urls to more urls per esld, then earlier index
1103 if #e1 ~= #e2 then
1104 return #e1 < #e2
1105 else
1106 return e1[#e1][4] < e2[#e2][4] -- Use index as final tiebreaker
1107 end
1108 end
1109 end)
1110
1111 return tbl
1112 end
1113
1114 eslds = sort_stuff(exports.values(eslds))
1115 neslds = #eslds
1116
1117 if neslds <= limit then
1118 -- Number of eslds < limit
1119 repeat
1120 local item_found = false
1121
1122 for _, lurls in ipairs(eslds) do
1123 if #lurls > 0 then
1124 local last = table.remove(lurls)
1125 insert_url(last[1], last[2])
1126 limit = limit - 1
1127 item_found = true
1128 end
1129 end
1130 until limit <= 0 or not item_found
1131
1132 if params.task and not params.no_cache then
1133 params.task:cache_set(cache_key, res)
1134 end
1135 return res
1136 end
1137
1138 tlds = sort_stuff(exports.values(tlds))
1139 ntlds = #tlds
1140
1141 -- Number of tlds < limit
1142 while limit > 0 do
1143 for _, lurls in ipairs(tlds) do
1144 if #lurls > 0 then
1145 local last = table.remove(lurls)
1146 insert_url(last[1], last[2])
1147 limit = limit - 1
1148 end
1149 if limit == 0 then
1150 break
1151 end
1152 end
1153 end
1154
1155 if params.task and not params.no_cache then
1156 params.task:cache_set(cache_key, res)
1157 end
1158 return res
1159 end
1160
1161 --[[[
1162 -- @function lua_util.extract_specific_urls(params)
1163 -- params: {
1164 - - task
1165 - - limit <int> (default = 9999)
1166 - - esld_limit <int> (default = 9999) n domains per eSLD (effective second level domain)
1167 works only if number of unique eSLD less than `limit`
1168 - - need_emails <bool> (default = false)
1169 - - filter <callback> (default = nil)
1170 - - prefix <string> cache prefix (default = nil)
1171 - - ignore_redirected <bool> (default = false)
1172 - - need_images <bool> (default = false)
1173 - - need_content <bool> (default = nil, uses global include_content_urls config which defaults to true)
1174 -- }
1175 -- Apply heuristic in extracting of urls from task, this function
1176 -- tries its best to extract specific number of urls from a task based on
1177 -- their characteristics
1178 --]]
1179 -- exports.extract_specific_urls = function(params_or_task, limit, need_emails, filter, prefix)
1180 exports.extract_specific_urls = function(params_or_task, lim, need_emails, filter, prefix)
1181 local default_params = {
1182 limit = 9999,
1183 esld_limit = 9999,
1184 need_emails = false,
1185 need_images = false,
1186 need_content = nil, -- nil means use global include_content_urls config (default: true)
1187 filter = nil,
1188 prefix = nil,
1189 ignore_ip = false,
1190 ignore_redirected = false,
1191 no_cache = false,
1192 }
1193
1194 local params
1195 if type(params_or_task) == 'table' and type(lim) == 'nil' then
1196 params = params_or_task
1197 else
1198 -- Deprecated call
1199 params = {
1200 task = params_or_task,
1201 limit = lim,
1202 need_emails = need_emails,
1203 filter = filter,
1204 prefix = prefix
1205 }
1206 end
1207 for k, v in pairs(default_params) do
1208 if type(params[k]) == 'nil' and v ~= nil then
1209 params[k] = v
1210 end
1211 end
1212 local url_params = {
1213 emails = params.need_emails,
1214 images = params.need_images,
1215 content = params.need_content,
1216 flags = params.flags, -- maybe nil
1217 flags_mode = params.flags_mode, -- maybe nil
1218 }
1219
1220 -- Shortcut for cached stuff
1221 if params.task and not params.no_cache then
1222 local cache_key
1223 if params.prefix then
1224 cache_key = params.prefix
1225 else
1226 local cache_key_suffix
1227 if params.flags then
1228 cache_key_suffix = table.concat(params.flags) .. (params.flags_mode or '')
1229 else
1230 -- Use tostring directly to distinguish nil (config default) from false (explicit exclude)
1231 cache_key_suffix = string.format('%s%s%s',
1232 tostring(params.need_emails or false),
1233 tostring(params.need_images or false),
1234 tostring(params.need_content)) -- nil = config default, false = explicit exclude
1235 end
1236 cache_key = string.format('sp_urls_%d%s', params.limit, cache_key_suffix)
1237 end
1238 local cached = params.task:cache_get(cache_key)
1239
1240 if cached then
1241 return cached
1242 end
1243 end
1244
1245 -- No cache version
1246 local urls = params.task:get_urls(url_params)
1247
1248 return exports.filter_specific_urls(urls, params)
1249 end
1250
1251
1252 --[[[
1253 -- @function lua_util.deepcopy(table)
1254 -- params: {
1255 - - table
1256 -- }
1257 -- Performs deep copy of the table. Including metatables
1258 --]]
1259 exports.deepcopy = deepcopy
1260
1261 --[[[
1262 -- @function lua_util.deepsort(table)
1263 -- params: {
1264 - - table
1265 -- }
1266 -- Performs recursive in-place sort of a table
1267 --]]
1268 local function default_sort_cmp(e1, e2)
1269 if type(e1) == type(e2) then
1270 return e1 < e2
1271 else
1272 return type(e1) < type(e2)
1273 end
1274 end
1275
1276 local function deepsort(tbl, sort_func)
1277 local orig_type = type(tbl)
1278 if orig_type == 'table' then
1279 table.sort(tbl, sort_func or default_sort_cmp)
1280 for _, orig_value in next, tbl, nil do
1281 deepsort(orig_value)
1282 end
1283 end
1284 end
1285
1286 exports.deepsort = deepsort
1287
1288 --[[[
1289 -- @function lua_util.shallowcopy(tbl)
1290 -- Performs shallow (and fast) copy of a table or another Lua type
1291 --]]
1292 exports.shallowcopy = shallowcopy
1293
1294 -- Debugging support
1295 local logger = require "rspamd_logger"
1296 local unconditional_debug = logger.log_level() == 'debug'
1297 local debug_modules = {}
1298 local debug_aliases = {}
1299 local log_level = 384 -- debug + forced (1 << 7 | 1 << 8)
1300
1301
1302 exports.init_debug_logging = function(config)
1303 -- Fill debug modules from the config
1304 if not unconditional_debug then
1305 local log_config = config:get_all_opt('logging')
1306 if log_config then
1307 local log_level_str = log_config.level
1308 if log_level_str then
1309 if log_level_str == 'debug' then
1310 unconditional_debug = true
1311 end
1312 end
1313 if log_config.debug_modules then
1314 for _, m in ipairs(log_config.debug_modules) do
1315 debug_modules[m] = true
1316 logger.infox(config, 'enable debug for Lua module %s', m)
1317 end
1318 end
1319
1320 if #debug_aliases > 0 then
1321 for alias, mod in pairs(debug_aliases) do
1322 if debug_modules[mod] then
1323 debug_modules[alias] = true
1324 logger.infox(config, 'enable debug for Lua module %s (%s aliased)',
1325 alias, mod)
1326 end
1327 end
1328 end
1329 end
1330 end
1331 end
1332
1333 exports.enable_debug_logging = function()
1334 unconditional_debug = true
1335 end
1336
1337 exports.enable_debug_modules = function(...)
1338 for _, m in ipairs({ ... }) do
1339 debug_modules[m] = true
1340 end
1341 end
1342
1343 exports.disable_debug_logging = function()
1344 unconditional_debug = false
1345 end
1346
1347 --[[[
1348 -- @function lua_util.debugm(module, [log_object], format, ...)
1349 -- Performs fast debug log for a specific module
1350 --]]
1351 exports.debugm = function(mod, obj_or_fmt, fmt_or_something, ...)
1352 if unconditional_debug or debug_modules[mod] then
1353 if type(obj_or_fmt) == 'string' then
1354 logger.logx(log_level, mod, '', 2, obj_or_fmt, fmt_or_something, ...)
1355 else
1356 logger.logx(log_level, mod, obj_or_fmt, 2, fmt_or_something, ...)
1357 end
1358 end
1359 end
1360
1361 --[[[
1362 -- @function lua_util.add_debug_alias(mod, alias)
1363 -- Add debugging alias so logging to `alias` will be treated as logging to `mod`
1364 --]]
1365 exports.add_debug_alias = function(mod, alias)
1366 debug_aliases[alias] = mod
1367
1368 if debug_modules[mod] then
1369 debug_modules[alias] = true
1370 logger.infox(rspamd_config, 'enable debug for Lua module %s (%s aliased)',
1371 alias, mod)
1372 end
1373 end
1374 ---[[[
1375 -- @function lua_util.get_task_verdict(task)
1376 -- Returns verdict for a task + score if certain, must be called from idempotent filters only
1377 -- Returns string:
1378 -- * `spam`: if message have over reject threshold and has more than one positive rule
1379 -- * `junk`: if a message has between score between [add_header/rewrite subject] to reject thresholds and has more than two positive rules
1380 -- * `passthrough`: if a message has been passed through some short-circuit rule
1381 -- * `ham`: if a message has overall score below junk level **and** more than three negative rule, or negative total score
1382 -- * `uncertain`: all other cases
1383 --]]
1384 exports.get_task_verdict = function(task)
1385 local lua_verdict = require "lua_verdict"
1386
1387 return lua_verdict.get_default_verdict(task)
1388 end
1389
1390 ---[[[
1391 -- @function lua_util.maybe_obfuscate_string(subject, settings, prefix)
1392 -- Obfuscate string if enabled in settings. Also checks utf8 validity - if
1393 -- string is not valid utf8 then '???' is returned. Empty string returned as is.
1394 -- Supported settings:
1395 -- * <prefix>_privacy = false - subject privacy is off
1396 -- * <prefix>_privacy_alg = 'blake2' - default hash-algorithm to obfuscate subject
1397 -- * <prefix>_privacy_prefix = 'obf' - prefix to show it's obfuscated
1398 -- * <prefix>_privacy_length = 16 - cut the length of the hash; if 0 or fasle full hash is returned
1399 -- @return obfuscated or validated subject
1400 --]]
1401
1402 exports.maybe_obfuscate_string = function(subject, settings, prefix)
1403 local hash = require 'rspamd_cryptobox_hash'
1404 if not subject or subject == '' then
1405 return subject
1406 elseif not rspamd_util.is_valid_utf8(subject) then
1407 subject = '???'
1408 elseif settings[prefix .. '_privacy'] then
1409 local hash_alg = settings[prefix .. '_privacy_alg'] or 'blake2'
1410 local subject_hash = hash.create_specific(hash_alg, subject)
1411
1412 local strip_len = settings[prefix .. '_privacy_length']
1413 if strip_len and strip_len > 0 then
1414 subject = subject_hash:hex():sub(1, strip_len)
1415 else
1416 subject = subject_hash:hex()
1417 end
1418
1419 local privacy_prefix = settings[prefix .. '_privacy_prefix']
1420 if privacy_prefix and #privacy_prefix > 0 then
1421 subject = privacy_prefix .. ':' .. subject
1422 end
1423 end
1424
1425 return subject
1426 end
1427
1428 ---[[[
1429 -- @function lua_util.maybe_encrypt_header(header, settings, prefix)
1430 -- Encode header with configured public key if enabled in settings.
1431 -- If header is not set then nil is returned. If pub_key is empty then header is returned.
1432 -- Supported settings:
1433 -- * <prefix>_encrypt = false - no need for encryption of a header
1434 -- * <prefix>_key = 'key' - key that is used encrypt header
1435 -- * <prefix>_nonce = 'nonce' - nonce to encrypt header(optional)
1436 -- @return encrypted header
1437 ---]]]
1438 exports.maybe_encrypt_header = function(header, settings, prefix)
1439 local rspamd_secretbox = require "rspamd_cryptobox_secretbox"
1440
1441 if not header or header == '' then
1442 logger.errx(rspamd_config, "Header is empty or nil. Header: %s", header)
1443 return nil
1444 elseif settings[prefix .. '_encrypt'] then
1445 local key = settings[prefix .. '_key']
1446 if not key or key == '' then
1447 logger.errx(rspamd_config, "Key is empty or nil. Key: %s", key)
1448 return header
1449 end
1450 local cryptobox = rspamd_secretbox.create(key)
1451
1452 local nonce = settings[prefix .. '_nonce']
1453 local encrypted_header = ''
1454 if not nonce or nonce == '' then
1455 encrypted_header, nonce = cryptobox:encrypt(header)
1456 else
1457 encrypted_header = cryptobox:encrypt(header, nonce)
1458 end
1459 encrypted_header = nonce .. encrypted_header
1460 return encrypted_header
1461 end
1462 end
1463
1464 ---[[[
1465 -- @function lua_util.maybe_decrypt_header(header, settings, prefix, nonce)
1466 -- Decode enoced with configured public_key header if enabled in settings.
1467 -- If encoded header is not set then nil is returned. If pub_key is empty then encoded header is returned.
1468 -- Supported settings:
1469 -- * <prefix>_encrypt = false - no need for decryption of a header
1470 -- * <prefix>_key = 'key' - key that is used decrypt header
1471 -- * <prefix>_nonce = 'nonce' - nonce used to encrypt header(optional)
1472 -- Nonce is an optional argument if <prefix>_nonce is provided, otherwise it is an required argument
1473 -- and <prefix>_nonce is an optional
1474 -- @return decrypted header
1475 ---]]]
1476 exports.maybe_decrypt_header = function(encrypted_header, settings, prefix, nonce)
1477 local rspamd_secretbox = require "rspamd_cryptobox_secretbox"
1478
1479 if not encrypted_header or encrypted_header == '' then
1480 logger.errx(rspamd_config, "Encrypted header is empty or nil. Encrypted header: %s", encrypted_header)
1481 return nil
1482 elseif settings[prefix .. '_encrypt'] then
1483 local key = settings[prefix .. '_key']
1484 if not key or key == '' then
1485 logger.errx(rspamd_config, "Key is empty or nil. Key: %s", key)
1486 return encrypted_header
1487 end
1488 local cryptobox = rspamd_secretbox.create(key)
1489
1490 local result = false
1491 local header = ''
1492 if not nonce then
1493 result, header = cryptobox:decrypt(encrypted_header, settings[prefix .. '_nonce'])
1494 else
1495 result, header = cryptobox:decrypt(encrypted_header, nonce)
1496 end
1497
1498 if not result then
1499 logger.infox(rspamd_config, "Decryption is failed with result: %s and decrypted header: %s", result, header)
1500 return nil
1501 end
1502
1503 return header
1504 end
1505 end
1506
1507 ---[[[
1508 -- @function lua_util.callback_from_string(str)
1509 -- Converts a string like `return function(...) end` to lua function and return true and this function
1510 -- or returns false + error message
1511 -- @return status code and function object or an error message
1512 --]]]
1513 exports.callback_from_string = function(s)
1514 local loadstring = loadstring or load
1515
1516 if not s or #s == 0 then
1517 return false, 'invalid or empty string'
1518 end
1519
1520 s = exports.rspamd_str_trim(s)
1521 local inp
1522
1523 if s:match('^return%s*function') then
1524 -- 'return function', can be evaluated directly
1525 inp = s
1526 elseif s:match('^function%s*%(') then
1527 inp = 'return ' .. s
1528 else
1529 -- Just a plain sequence
1530 inp = 'return function(...)\n' .. s .. '; end'
1531 end
1532
1533 local chunk, err = loadstring(inp)
1534 if not chunk then
1535 return false, err
1536 end
1537
1538 local ret, res_or_err = pcall(chunk)
1539
1540 if not ret or type(res_or_err) ~= 'function' then
1541 return false, res_or_err
1542 end
1543
1544 return ret, res_or_err
1545 end
1546
1547 ---[[[
1548 -- @function lua_util.keys(t)
1549 -- Returns all keys from a specific table
1550 -- @param {table} t input table (or iterator triplet)
1551 -- @return array of keys
1552 --]]]
1553 exports.keys = function(gen, param, state)
1554 local keys = {}
1555 local i = 1
1556
1557 if param then
1558 for k, _ in fun.iter(gen, param, state) do
1559 rawset(keys, i, k)
1560 i = i + 1
1561 end
1562 else
1563 for k, _ in pairs(gen) do
1564 rawset(keys, i, k)
1565 i = i + 1
1566 end
1567 end
1568
1569 return keys
1570 end
1571
1572 ---[[[
1573 -- @function lua_util.values(t)
1574 -- Returns all values from a specific table
1575 -- @param {table} t input table
1576 -- @return array of values
1577 --]]]
1578 exports.values = function(gen, param, state)
1579 local values = {}
1580 local i = 1
1581
1582 if param then
1583 for _, v in fun.iter(gen, param, state) do
1584 rawset(values, i, v)
1585 i = i + 1
1586 end
1587 else
1588 for _, v in pairs(gen) do
1589 rawset(values, i, v)
1590 i = i + 1
1591 end
1592 end
1593
1594 return values
1595 end
1596
1597 ---[[[
1598 -- @function lua_util.distance_sorted(t1, t2)
1599 -- Returns distance between two sorted tables t1 and t2
1600 -- @param {table} t1 input table
1601 -- @param {table} t2 input table
1602 -- @return distance between `t1` and `t2`
1603 --]]]
1604 exports.distance_sorted = function(t1, t2)
1605 local ncomp = #t1
1606 local ndiff = 0
1607 local i, j = 1, 1
1608
1609 if ncomp < #t2 then
1610 ncomp = #t2
1611 end
1612
1613 for _ = 1, ncomp do
1614 if j > #t2 then
1615 ndiff = ndiff + ncomp - #t2
1616 if i > j then
1617 ndiff = ndiff - (i - j)
1618 end
1619 break
1620 elseif i > #t1 then
1621 ndiff = ndiff + ncomp - #t1
1622 if j > i then
1623 ndiff = ndiff - (j - i)
1624 end
1625 break
1626 end
1627
1628 if t1[i] == t2[j] then
1629 i = i + 1
1630 j = j + 1
1631 elseif t1[i] < t2[j] then
1632 i = i + 1
1633 ndiff = ndiff + 1
1634 else
1635 j = j + 1
1636 ndiff = ndiff + 1
1637 end
1638 end
1639
1640 return ndiff
1641 end
1642
1643 ---[[[
1644 -- @function lua_util.table_digest(t)
1645 -- Returns hash of all values if t[1] is string or all keys/values otherwise
1646 -- @param {table} t input array or map
1647 -- @return {string} base32 representation of blake2b hash of all strings
1648 --]]]
1649 local function table_digest(t)
1650 local cr = require "rspamd_cryptobox_hash"
1651 local h = cr.create()
1652
1653 if t[1] then
1654 for _, e in ipairs(t) do
1655 if type(e) == 'table' then
1656 h:update(table_digest(e))
1657 else
1658 h:update(tostring(e))
1659 end
1660 end
1661 else
1662 for k, v in pairs(t) do
1663 h:update(tostring(k))
1664
1665 if type(v) == 'string' then
1666 h:update(v)
1667 elseif type(v) == 'table' then
1668 h:update(table_digest(v))
1669 end
1670 end
1671 end
1672 return h:base32()
1673 end
1674
1675 exports.table_digest = table_digest
1676
1677 ---[[[
1678 -- @function lua_util.unordered_table_digest(t)
1679 -- Returns a hash of table contents that is independent of iteration order.
1680 -- Uses XXH3 fast hash with XOR accumulation for O(n) performance.
1681 -- All value types (string, number, boolean, table) are included in the hash.
1682 -- @param {table} t input array or map
1683 -- @return {string} hex representation of the 64-bit hash
1684 --]]]
1685 local function unordered_table_digest(t)
1686 local cr = require "rspamd_cryptobox"
1687 local bit = require "bit"
1688
1689 -- Internal function that returns high/low 32-bit parts
1690 local function digest_impl(tbl)
1691 local acc_hi, acc_lo = 0, 0
1692
1693 if tbl[1] ~= nil then
1694 -- Array: order matters, so include index in hash
1695 for i, e in ipairs(tbl) do
1696 local str
1697 if type(e) == 'table' then
1698 -- Recursively compute digest for nested table
1699 str = tostring(i) .. '\0' .. digest_impl(e)
1700 else
1701 str = tostring(i) .. '\0' .. tostring(e)
1702 end
1703 local hi, lo = cr.fast_hash64(str)
1704 acc_hi = bit.bxor(acc_hi, hi)
1705 acc_lo = bit.bxor(acc_lo, lo)
1706 end
1707 else
1708 -- Map: order doesn't matter, XOR all k-v hashes
1709 for k, v in pairs(tbl) do
1710 local str
1711 if type(v) == 'table' then
1712 -- Recursively compute digest for nested table
1713 str = tostring(k) .. '\0' .. digest_impl(v)
1714 else
1715 str = tostring(k) .. '\0' .. tostring(v)
1716 end
1717 local hi, lo = cr.fast_hash64(str)
1718 acc_hi = bit.bxor(acc_hi, hi)
1719 acc_lo = bit.bxor(acc_lo, lo)
1720 end
1721 end
1722
1723 -- Return as hex string for nested calls
1724 -- Use bit.tohex() which properly handles signed 32-bit values
1725 return bit.tohex(acc_hi) .. bit.tohex(acc_lo)
1726 end
1727
1728 return digest_impl(t)
1729 end
1730
1731 exports.unordered_table_digest = unordered_table_digest
1732
1733 ---[[[
1734 -- @function lua_util.toboolean(v)
1735 -- Converts a string or a number to boolean
1736 -- @param {string|number} v
1737 -- @return {boolean} v converted to boolean
1738 --]]]
1739 exports.toboolean = function(v)
1740 local true_t = {
1741 ['1'] = true,
1742 ['true'] = true,
1743 ['TRUE'] = true,
1744 ['True'] = true,
1745 };
1746 local false_t = {
1747 ['0'] = false,
1748 ['false'] = false,
1749 ['FALSE'] = false,
1750 ['False'] = false,
1751 };
1752
1753 if type(v) == 'boolean' then
1754 return v
1755 elseif type(v) == 'string' then
1756 if true_t[v] == true then
1757 return true;
1758 elseif false_t[v] == false then
1759 return false;
1760 else
1761 return false, string.format('cannot convert %q to boolean', v);
1762 end
1763 elseif type(v) == 'number' then
1764 return v ~= 0
1765 else
1766 return false, string.format('cannot convert %q to boolean', v);
1767 end
1768 end
1769
1770 ---[[[
1771 -- @function lua_util.config_check_local_or_authed(config, modname)
1772 -- Reads check_local and check_authed from the config as this is used in many modules
1773 -- @param {rspamd_config} config `rspamd_config` global
1774 -- @param {name} module name
1775 -- @return {boolean} v converted to boolean
1776 --]]]
1777 exports.config_check_local_or_authed = function(rspamd_config, modname, def_local, def_authed)
1778 local check_local = def_local or false
1779 local check_authed = def_authed or false
1780
1781 local function try_section(where)
1782 local ret = false
1783 local opts = rspamd_config:get_all_opt(where)
1784 if type(opts) == 'table' then
1785 if type(opts['check_local']) == 'boolean' then
1786 check_local = opts['check_local']
1787 ret = true
1788 end
1789 if type(opts['check_authed']) == 'boolean' then
1790 check_authed = opts['check_authed']
1791 ret = true
1792 end
1793 end
1794
1795 return ret
1796 end
1797
1798 if not try_section(modname) then
1799 try_section('options')
1800 end
1801
1802 return { check_local, check_authed }
1803 end
1804
1805 ---[[[
1806 -- @function lua_util.is_skip_local_or_authed(task, conf[, ip])
1807 -- Returns `true` if local or authenticated task should be skipped for this module
1808 -- @param {rspamd_task} task
1809 -- @param {table} conf table returned from `config_check_local_or_authed`
1810 -- @param {rspamd_ip} ip optional ip address (can be obtained from a task)
1811 -- @return {boolean} true if check should be skipped
1812 --]]]
1813 exports.is_skip_local_or_authed = function(task, conf, ip)
1814 if not ip then
1815 ip = task:get_from_ip()
1816 end
1817 if not conf then
1818 conf = { false, false }
1819 end
1820 if ((not conf[2] and task:get_user()) or
1821 (not conf[1] and type(ip) == 'userdata' and ip:is_local())) then
1822 return true
1823 end
1824
1825 return false
1826 end
1827
1828 ---[[[
1829 -- @function lua_util.maybe_smtp_quote_value(str)
1830 -- Checks string for the forbidden elements (tspecials in RFC and quote string if needed)
1831 -- @param {string} str input string
1832 -- @return {string} original or quoted string
1833 --]]]
1834 local tspecial = lpeg.S "()<>,;:\\\"/[]?= \t\v"
1835 local special_match = lpeg.P((1 - tspecial) ^ 0 * tspecial ^ 1)
1836 exports.maybe_smtp_quote_value = function(str)
1837 if special_match:match(str) then
1838 return string.format('"%s"', str:gsub('"', '\\"'))
1839 end
1840
1841 return str
1842 end
1843
1844 ---[[[
1845 -- @function lua_util.shuffle(table)
1846 -- Performs in-place shuffling of a table
1847 -- @param {table} tbl table to shuffle
1848 -- @return {table} same table
1849 --]]]
1850 exports.shuffle = function(tbl)
1851 local size = #tbl
1852 for i = size, 1, -1 do
1853 local rand = math.random(size)
1854 tbl[i], tbl[rand] = tbl[rand], tbl[i]
1855 end
1856 return tbl
1857 end
1858
1859 --
1860 local hex_table = {}
1861 for idx = 0, 255 do
1862 hex_table[("%02X"):format(idx)] = string.char(idx)
1863 hex_table[("%02x"):format(idx)] = string.char(idx)
1864 end
1865
1866 ---[[[
1867 -- @function lua_util.unhex(str)
1868 -- Decode hex encoded string
1869 -- @param {string} str string to decode
1870 -- @return {string} hex decoded string (valid hex pairs are decoded, everything else is printed as is)
1871 --]]]
1872 exports.unhex = function(str)
1873 return str:gsub('(..)', hex_table)
1874 end
1875
1876 local http_upstream_lists = {}
1877 local function http_upstreams_by_url(pool, url)
1878 local rspamd_url = require "rspamd_url"
1879
1880 local cached = http_upstream_lists[url]
1881 if cached then
1882 return cached
1883 end
1884
1885 local real_url = rspamd_url.create(pool, url)
1886
1887 if not real_url then
1888 return nil
1889 end
1890
1891 local host = real_url:get_host()
1892 local proto = real_url:get_protocol() or 'http'
1893 local port = real_url:get_port() or (proto == 'https' and 443 or 80)
1894 local upstream_list = require "rspamd_upstream_list"
1895 local upstreams = upstream_list.create(host, port)
1896
1897 if upstreams then
1898 http_upstream_lists[url] = upstreams
1899 return upstreams
1900 end
1901
1902 return nil
1903 end
1904 ---[[[
1905 -- @function lua_util.http_upstreams_by_url(pool, url)
1906 -- Returns a cached or new upstreams list that corresponds to the specific url
1907 -- @param {mempool} pool memory pool to use (typically static pool from rspamd_config)
1908 -- @param {string} url full url
1909 -- @return {upstreams_list} object to get upstream from an url
1910 --]]]
1911 exports.http_upstreams_by_url = http_upstreams_by_url
1912
1913 ---[[[
1914 -- @function lua_util.dns_timeout_augmentation(cfg)
1915 -- Returns an augmentation suitable to define DNS timeout for a module
1916 -- @return {string} a string in format 'timeout=x' where `x` is a number of seconds for DNS timeout
1917 --]]]
1918 local function dns_timeout_augmentation(cfg)
1919 return string.format('timeout=%f', cfg:get_dns_timeout() or 0.0)
1920 end
1921
1922 exports.dns_timeout_augmentation = dns_timeout_augmentation
1923
1924 ---[[[
1925 --- @function lua_util.strip_lua_comments(lua_code)
1926 -- Strips single-line and multi-line comments from a given Lua code string and removes
1927 -- any extra spaces or newlines.
1928 --
1929 -- @param lua_code The Lua code string to strip comments from.
1930 -- @return The resulting Lua code string with comments and extra spaces removed.
1931 --
1932 ---]]]
1933 local function strip_lua_comments(lua_code)
1934 -- Remove single-line comments
1935 lua_code = lua_code:gsub("%-%-[^\r\n]*", "")
1936
1937 -- Remove multi-line comments
1938 lua_code = lua_code:gsub("%-%-%[%[.-%]%]", "")
1939
1940 -- Remove extra spaces and newlines
1941 lua_code = lua_code:gsub("%s+", " ")
1942
1943 return lua_code
1944 end
1945
1946 exports.strip_lua_comments = strip_lua_comments
1947
1948 ---[[[
1949 -- @function lua_util.join_path(...)
1950 -- Joins path components into a single path string using the appropriate separator
1951 -- for the current operating system.
1952 --
1953 -- @param ... Any number of path components to join together.
1954 -- @return A single path string, with components separated by the appropriate separator.
1955 --
1956 ---]]]
1957 local path_sep = package.config:sub(1, 1) or '/'
1958 local function join_path(...)
1959 local components = { ... }
1960
1961 -- Join components using separator
1962 return table.concat(components, path_sep)
1963 end
1964 exports.join_path = join_path
1965
1966 ---[[[
1967 -- @function lua_util.url_encode_string(str)
1968 -- URL encodes a string
1969 --
1970 -- @param {string} str string to encode
1971 -- @return {string} URL encoded string
1972 --
1973 ---]]]
1974 local function url_encode_string(str)
1975 if str == nil then
1976 return ''
1977 end
1978 -- Use explicit ASCII ranges instead of %w which is locale-dependent
1979 -- and may match non-ASCII bytes in UTF-8 locales
1980 str = string.gsub(str, "([^A-Za-z0-9 _%%%-%.~])",
1981 function(c)
1982 return string.format("%%%02X", string.byte(c))
1983 end)
1984 str = string.gsub(str, " ", "+")
1985 return str
1986 end
1987 exports.url_encode_string = url_encode_string
1988
1989 assert(url_encode_string('上海+中國') == '%E4%B8%8A%E6%B5%B7%2B%E4%B8%AD%E5%9C%8B')
1990 assert(url_encode_string('? and the Mysterians') == '%3F+and+the+Mysterians')
1991
1992 -- Short unit test for sanity
1993 if path_sep == '/' then
1994 assert(join_path('/path', 'to', 'file') == '/path/to/file')
1995 else
1996 assert(join_path('C:', 'path', 'to', 'file') == 'C:\\path\\to\\file')
1997 end
1998
1999 -- Defines symbols priorities for common usage in prefilters/postfilters
2000 exports.symbols_priorities = {
2001 top = 10, -- Symbols must be executed first (or last), such as settings
2002 high = 9, -- Example: asn
2003 medium = 5, -- Everything should use this as default
2004 low = 0,
2005 }
2006
2007 ---[[[
2008 -- @function lua_util.table_to_multipart_body(tbl, boundary)
2009 -- Converts a key-value map to the table representing multipart body, with the following values:
2010 -- `data`: data of the part
2011 -- `filename`: optional filename
2012 -- `content-type`: content type of the element (optional)
2013 -- `content-transfer-encoding`: optional CTE header
2014 local function table_to_multipart_body(tbl, boundary)
2015 local seen_data = false
2016 local out = {}
2017
2018 for k, v in pairs(tbl) do
2019 if v.data then
2020 seen_data = true
2021 table.insert(out, string.format('--%s\r\n', boundary))
2022 if v.filename then
2023 table.insert(out,
2024 string.format('Content-Disposition: form-data; name="%s"; filename="%s"\r\n',
2025 k, v.filename))
2026 else
2027 table.insert(out,
2028 string.format('Content-Disposition: form-data; name="%s"\r\n', k))
2029 end
2030 if v['content-type'] then
2031 table.insert(out,
2032 string.format('Content-Type: %s\r\n', v['content-type']))
2033 else
2034 table.insert(out, 'Content-Type: text/plain\r\n')
2035 end
2036 if v['content-transfer-encoding'] then
2037 table.insert(out,
2038 string.format('Content-Transfer-Encoding: %s\r\n',
2039 v['content-transfer-encoding']))
2040 else
2041 table.insert(out, 'Content-Transfer-Encoding: binary\r\n')
2042 end
2043 table.insert(out, '\r\n')
2044 table.insert(out, v.data)
2045 table.insert(out, '\r\n')
2046 end
2047 end
2048
2049 if seen_data then
2050 table.insert(out, string.format('--%s--\r\n', boundary))
2051 end
2052
2053 return out
2054 end
2055
2056 exports.table_to_multipart_body = table_to_multipart_body
2057
2058 return exports