| 1 |
--[[ |
| 2 |
Copyright (c) 2023, Vsevolod Stakhov <[email protected]> |
| 3 |
|
| 4 |
Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 |
you may not use this file except in compliance with the License. |
| 6 |
You may obtain a copy of the License at |
| 7 |
|
| 8 |
http://www.apache.org/licenses/LICENSE-2.0 |
| 9 |
|
| 10 |
Unless required by applicable law or agreed to in writing, software |
| 11 |
distributed under the License is distributed on an "AS IS" BASIS, |
| 12 |
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 |
See the License for the specific language governing permissions and |
| 14 |
limitations under the License. |
| 15 |
]] -- |
| 16 |
|
| 17 |
--[[[ |
| 18 |
-- @module lua_util |
| 19 |
-- This module contains utility functions for working with Lua and/or Rspamd |
| 20 |
--]] |
| 21 |
|
| 22 |
local exports = {} |
| 23 |
local lpeg = require 'lpeg' |
| 24 |
local rspamd_util = require "rspamd_util" |
| 25 |
local fun = require "fun" |
| 26 |
local lupa = require "lupa" |
| 27 |
|
| 28 |
local split_grammar = {} |
| 29 |
local spaces_split_grammar |
| 30 |
local space = lpeg.S ' \t\n\v\f\r' |
| 31 |
local nospace = 1 - space |
| 32 |
local ptrim = space ^ 0 * lpeg.C((space ^ 0 * nospace ^ 1) ^ 0) |
| 33 |
local match = lpeg.match |
| 34 |
|
| 35 |
local function shallowcopy(orig) |
| 36 |
local orig_type = type(orig) |
| 37 |
local copy |
| 38 |
if orig_type == 'table' then |
| 39 |
copy = {} |
| 40 |
for orig_key, orig_value in pairs(orig) do |
| 41 |
copy[orig_key] = orig_value |
| 42 |
end |
| 43 |
else |
| 44 |
copy = orig |
| 45 |
end |
| 46 |
return copy |
| 47 |
end |
| 48 |
local function deepcopy(orig) |
| 49 |
local orig_type = type(orig) |
| 50 |
local copy |
| 51 |
if orig_type == 'table' then |
| 52 |
copy = {} |
| 53 |
for orig_key, orig_value in next, orig, nil do |
| 54 |
copy[deepcopy(orig_key)] = deepcopy(orig_value) |
| 55 |
end |
| 56 |
if getmetatable(orig) then |
| 57 |
setmetatable(copy, deepcopy(getmetatable(orig))) |
| 58 |
end |
| 59 |
else |
| 60 |
-- number, string, boolean, etc |
| 61 |
copy = orig |
| 62 |
end |
| 63 |
return copy |
| 64 |
end |
| 65 |
|
| 66 |
lupa.configure('{%', '%}', '{=', '=}', '{#', '#}', { |
| 67 |
keep_trailing_newline = true, |
| 68 |
autoescape = false, |
| 69 |
}) |
| 70 |
|
| 71 |
lupa.filters.pbkdf = function(s) |
| 72 |
local cr = require "rspamd_cryptobox" |
| 73 |
return cr.pbkdf(s) |
| 74 |
end |
| 75 |
|
| 76 |
-- Dirty hacks to avoid shared state |
| 77 |
package.loaded['lupa'] = nil |
| 78 |
local lupa_orig = require "lupa" |
| 79 |
|
| 80 |
local function rspamd_str_split(s, sep) |
| 81 |
local gr |
| 82 |
if not sep then |
| 83 |
if not spaces_split_grammar then |
| 84 |
local _sep = space |
| 85 |
local elem = lpeg.C((1 - _sep) ^ 0) |
| 86 |
local p = lpeg.Ct(elem * (_sep * elem) ^ 0) |
| 87 |
spaces_split_grammar = p |
| 88 |
end |
| 89 |
|
| 90 |
gr = spaces_split_grammar |
| 91 |
else |
| 92 |
gr = split_grammar[sep] |
| 93 |
|
| 94 |
if not gr then |
| 95 |
local _sep |
| 96 |
if type(sep) == 'string' then |
| 97 |
_sep = lpeg.S(sep) -- Assume set |
| 98 |
else |
| 99 |
_sep = sep -- Assume lpeg object |
| 100 |
end |
| 101 |
local elem = lpeg.C((1 - _sep) ^ 0) |
| 102 |
local p = lpeg.Ct(elem * (_sep * elem) ^ 0) |
| 103 |
gr = p |
| 104 |
split_grammar[sep] = gr |
| 105 |
end |
| 106 |
end |
| 107 |
|
| 108 |
return gr:match(s) |
| 109 |
end |
| 110 |
|
| 111 |
--[[[ |
| 112 |
-- @function lua_util.str_split(text, delimiter) |
| 113 |
-- Splits text into a numeric table by delimiter |
| 114 |
-- @param {string} text delimited text |
| 115 |
-- @param {string} delimiter the delimiter |
| 116 |
-- @return {table} numeric table containing string parts |
| 117 |
--]] |
| 118 |
|
| 119 |
exports.rspamd_str_split = rspamd_str_split |
| 120 |
exports.str_split = rspamd_str_split |
| 121 |
|
| 122 |
local function rspamd_str_trim(s) |
| 123 |
return match(ptrim, s) |
| 124 |
end |
| 125 |
exports.rspamd_str_trim = rspamd_str_trim |
| 126 |
--[[[ |
| 127 |
-- @function lua_util.str_trim(text) |
| 128 |
-- Returns a string with no trailing and leading spaces |
| 129 |
-- @param {string} text input text |
| 130 |
-- @return {string} string with no trailing and leading spaces |
| 131 |
--]] |
| 132 |
exports.str_trim = rspamd_str_trim |
| 133 |
|
| 134 |
--[[[ |
| 135 |
-- @function lua_util.str_startswith(text, prefix) |
| 136 |
-- @param {string} text |
| 137 |
-- @param {string} prefix |
| 138 |
-- @return {boolean} true if text starts with the specified prefix, false otherwise |
| 139 |
--]] |
| 140 |
exports.str_startswith = function(s, prefix) |
| 141 |
return s:sub(1, prefix:len()) == prefix |
| 142 |
end |
| 143 |
|
| 144 |
--[[[ |
| 145 |
-- @function lua_util.str_endswith(text, suffix) |
| 146 |
-- @param {string} text |
| 147 |
-- @param {string} suffix |
| 148 |
-- @return {boolean} true if text ends with the specified suffix, false otherwise |
| 149 |
--]] |
| 150 |
exports.str_endswith = function(s, suffix) |
| 151 |
return s:find(suffix, -suffix:len(), true) ~= nil |
| 152 |
end |
| 153 |
|
| 154 |
--[[[ |
| 155 |
-- @function lua_util.round(number, decimalPlaces) |
| 156 |
-- Round number to fixed number of decimal points |
| 157 |
-- @param {number} number number to round |
| 158 |
-- @param {number} decimalPlaces number of decimal points |
| 159 |
-- @return {number} rounded number |
| 160 |
--]] |
| 161 |
|
| 162 |
-- modified version from Robert Jay Gould http://lua-users.org/wiki/SimpleRound |
| 163 |
exports.round = function(num, numDecimalPlaces) |
| 164 |
local mult = 10 ^ (numDecimalPlaces or 0) |
| 165 |
if num >= 0 then |
| 166 |
return math.floor(num * mult + 0.5) / mult |
| 167 |
else |
| 168 |
return math.ceil(num * mult - 0.5) / mult |
| 169 |
end |
| 170 |
end |
| 171 |
|
| 172 |
--[[[ |
| 173 |
-- @function lua_util.template(text, replacements) |
| 174 |
-- Replaces values in a text template |
| 175 |
-- Variable names can contain letters, numbers and underscores, are prefixed with `$` and may or not use curly braces. |
| 176 |
-- @param {string} text text containing variables |
| 177 |
-- @param {table} replacements key/value pairs for replacements |
| 178 |
-- @return {string} string containing replaced values |
| 179 |
-- @example |
| 180 |
-- local goop = lua_util.template("HELLO $FOO ${BAR}!", {['FOO'] = 'LUA', ['BAR'] = 'WORLD'}) |
| 181 |
-- -- goop contains "HELLO LUA WORLD!" |
| 182 |
--]] |
| 183 |
|
| 184 |
exports.template = function(tmpl, keys) |
| 185 |
local var_lit = lpeg.P { lpeg.R("az") + lpeg.R("AZ") + lpeg.R("09") + "_" } |
| 186 |
local var = lpeg.P { (lpeg.P("$") / "") * ((var_lit ^ 1) / keys) } |
| 187 |
local var_braced = lpeg.P { (lpeg.P("${") / "") * ((var_lit ^ 1) / keys) * (lpeg.P("}") / "") } |
| 188 |
|
| 189 |
local template_grammar = lpeg.Cs((var + var_braced + 1) ^ 0) |
| 190 |
|
| 191 |
return lpeg.match(template_grammar, tmpl) |
| 192 |
end |
| 193 |
|
| 194 |
local function enrich_template_with_globals(env) |
| 195 |
local newenv = shallowcopy(env) |
| 196 |
newenv.paths = rspamd_paths |
| 197 |
newenv.env = rspamd_env |
| 198 |
|
| 199 |
return newenv |
| 200 |
end |
| 201 |
--[[[ |
| 202 |
-- @function lua_util.jinja_template(text, env[, skip_global_env][, is_orig][, custom_filters]) |
| 203 |
-- Replaces values in a text template according to jinja2 syntax |
| 204 |
-- @param {string} text text containing variables |
| 205 |
-- @param {table} replacements key/value pairs for replacements |
| 206 |
-- @param {boolean} skip_global_env don't export Rspamd superglobals |
| 207 |
-- @param {boolean} is_orig use the original lupa configuration with {% raw %}`{{`{% endraw %} for variables |
| 208 |
-- @param {table} custom_filters custom filters to use (or nil if not needed) |
| 209 |
-- @return {string} string containing replaced values |
| 210 |
-- @example |
| 211 |
-- lua_util.jinja_template("HELLO {=FOO=} {=BAR=}!", {['FOO'] = 'LUA', ['BAR'] = 'WORLD'}) |
| 212 |
-- "HELLO LUA WORLD!" |
| 213 |
--]] |
| 214 |
exports.jinja_template = function(text, env, skip_global_env, is_orig, custom_filters) |
| 215 |
local lupa_to_use = is_orig and lupa_orig or lupa |
| 216 |
if not skip_global_env then |
| 217 |
env = enrich_template_with_globals(env) |
| 218 |
end |
| 219 |
|
| 220 |
local orig_filters = {} |
| 221 |
if type(custom_filters) == 'table' then |
| 222 |
for k, v in pairs(custom_filters) do |
| 223 |
orig_filters[k] = lupa_to_use.filters[k] |
| 224 |
lupa_to_use.filters[k] = v |
| 225 |
end |
| 226 |
end |
| 227 |
|
| 228 |
local result = lupa_to_use.expand(text, env) |
| 229 |
|
| 230 |
-- Restore custom filters |
| 231 |
if type(custom_filters) == 'table' then |
| 232 |
for k, _ in pairs(custom_filters) do |
| 233 |
lupa_to_use.filters[k] = orig_filters[k] |
| 234 |
end |
| 235 |
end |
| 236 |
|
| 237 |
return result |
| 238 |
end |
| 239 |
|
| 240 |
--[[[ |
| 241 |
-- @function lua_util.jinja_file(filename, env[, skip_global_env][, is_orig][, custom_filters]) |
| 242 |
-- Replaces values in a text template according to jinja2 syntax |
| 243 |
-- @param {string} filename name of file to expand |
| 244 |
-- @param {table} replacements key/value pairs for replacements |
| 245 |
-- @param {boolean} skip_global_env don't export Rspamd superglobals |
| 246 |
-- @param {boolean} is_orig use the original lupa configuration with {% raw %}`{{`{% endraw %} for variables |
| 247 |
-- @param {table} custom_filters custom filters to use (or nil if not needed) |
| 248 |
-- @return {string} string containing replaced values |
| 249 |
-- @example |
| 250 |
-- lua_util.jinja_template("HELLO {=FOO=} {=BAR=}!", {['FOO'] = 'LUA', ['BAR'] = 'WORLD'}) |
| 251 |
-- "HELLO LUA WORLD!" |
| 252 |
--]] |
| 253 |
exports.jinja_template_file = function(filename, env, skip_global_env, is_orig, custom_filters) |
| 254 |
local lupa_to_use = is_orig and lupa_orig or lupa |
| 255 |
if not skip_global_env then |
| 256 |
env = enrich_template_with_globals(env) |
| 257 |
end |
| 258 |
|
| 259 |
local orig_filters = {} |
| 260 |
if type(custom_filters) == 'table' then |
| 261 |
for k, v in pairs(custom_filters) do |
| 262 |
orig_filters[k] = lupa_to_use.filters[k] |
| 263 |
lupa_to_use.filters[k] = v |
| 264 |
end |
| 265 |
end |
| 266 |
|
| 267 |
local result = lupa_to_use.expand_file(filename, env) |
| 268 |
|
| 269 |
-- Restore custom filters |
| 270 |
if type(custom_filters) == 'table' then |
| 271 |
for k, _ in pairs(custom_filters) do |
| 272 |
lupa_to_use.filters[k] = orig_filters[k] |
| 273 |
end |
| 274 |
end |
| 275 |
|
| 276 |
return result |
| 277 |
end |
| 278 |
|
| 279 |
exports.remove_email_aliases = function(email_addr) |
| 280 |
local function check_gmail_user(addr) |
| 281 |
-- Remove all points |
| 282 |
local no_dots_user = string.gsub(addr.user, '%.', '') |
| 283 |
local cap, pluses = string.match(no_dots_user, '^([^%+][^%+]*)(%+.*)$') |
| 284 |
if cap then |
| 285 |
return cap, rspamd_str_split(pluses, '+'), nil |
| 286 |
elseif no_dots_user ~= addr.user then |
| 287 |
return no_dots_user, {}, nil |
| 288 |
end |
| 289 |
|
| 290 |
return nil |
| 291 |
end |
| 292 |
|
| 293 |
local function check_address(addr) |
| 294 |
if addr.user then |
| 295 |
local cap, pluses = string.match(addr.user, '^([^%+][^%+]*)(%+.*)$') |
| 296 |
if cap then |
| 297 |
return cap, rspamd_str_split(pluses, '+'), nil |
| 298 |
end |
| 299 |
end |
| 300 |
|
| 301 |
return nil |
| 302 |
end |
| 303 |
|
| 304 |
local function set_addr(addr, new_user, new_domain) |
| 305 |
if new_user then |
| 306 |
addr.user = new_user |
| 307 |
end |
| 308 |
if new_domain then |
| 309 |
addr.domain = new_domain |
| 310 |
end |
| 311 |
|
| 312 |
if addr.domain then |
| 313 |
addr.addr = string.format('%s@%s', addr.user, addr.domain) |
| 314 |
else |
| 315 |
addr.addr = string.format('%s@', addr.user) |
| 316 |
end |
| 317 |
|
| 318 |
if addr.name and #addr.name > 0 then |
| 319 |
addr.raw = string.format('"%s" <%s>', addr.name, addr.addr) |
| 320 |
else |
| 321 |
addr.raw = string.format('<%s>', addr.addr) |
| 322 |
end |
| 323 |
end |
| 324 |
|
| 325 |
local function check_gmail(addr) |
| 326 |
local nu, tags, nd = check_gmail_user(addr) |
| 327 |
|
| 328 |
if nu then |
| 329 |
return nu, tags, nd |
| 330 |
end |
| 331 |
|
| 332 |
return nil |
| 333 |
end |
| 334 |
|
| 335 |
local function check_googlemail(addr) |
| 336 |
local nd = 'gmail.com' |
| 337 |
local nu, tags = check_gmail_user(addr) |
| 338 |
|
| 339 |
if nu then |
| 340 |
return nu, tags, nd |
| 341 |
end |
| 342 |
|
| 343 |
return nil, nil, nd |
| 344 |
end |
| 345 |
|
| 346 |
local specific_domains = { |
| 347 |
['gmail.com'] = check_gmail, |
| 348 |
['googlemail.com'] = check_googlemail, |
| 349 |
} |
| 350 |
|
| 351 |
if email_addr then |
| 352 |
if email_addr.domain and specific_domains[email_addr.domain] then |
| 353 |
local nu, tags, nd = specific_domains[email_addr.domain](email_addr) |
| 354 |
if nu or nd then |
| 355 |
set_addr(email_addr, nu, nd) |
| 356 |
|
| 357 |
return nu, tags |
| 358 |
end |
| 359 |
else |
| 360 |
local nu, tags, nd = check_address(email_addr) |
| 361 |
if nu or nd then |
| 362 |
set_addr(email_addr, nu, nd) |
| 363 |
|
| 364 |
return nu, tags |
| 365 |
end |
| 366 |
end |
| 367 |
|
| 368 |
return nil |
| 369 |
end |
| 370 |
end |
| 371 |
|
| 372 |
exports.is_rspamc_or_controller = function(task) |
| 373 |
local ua = task:get_request_header('User-Agent') or '' |
| 374 |
local pwd = task:get_request_header('Password') |
| 375 |
local is_rspamc = false |
| 376 |
if tostring(ua) == 'rspamc' or pwd then |
| 377 |
is_rspamc = true |
| 378 |
end |
| 379 |
|
| 380 |
return is_rspamc |
| 381 |
end |
| 382 |
|
| 383 |
--[[[ |
| 384 |
-- @function lua_util.unpack(table) |
| 385 |
-- Converts numeric table to varargs |
| 386 |
-- This is `unpack` on Lua 5.1/5.2/LuaJIT and `table.unpack` on Lua 5.3 |
| 387 |
-- @param {table} table numerically indexed table to unpack |
| 388 |
-- @return {varargs} unpacked table elements |
| 389 |
--]] |
| 390 |
|
| 391 |
local unpack_function = table.unpack or unpack |
| 392 |
exports.unpack = function(...) |
| 393 |
return unpack_function(...) |
| 394 |
end |
| 395 |
|
| 396 |
--[[[ |
| 397 |
-- @function lua_util.flatten(table) |
| 398 |
-- Flatten underlying tables in a single table |
| 399 |
-- @param {table} table table of tables |
| 400 |
-- @return {table} flattened table |
| 401 |
--]] |
| 402 |
exports.flatten = function(t) |
| 403 |
local res = {} |
| 404 |
for _, e in fun.iter(t) do |
| 405 |
for _, v in fun.iter(e) do |
| 406 |
res[#res + 1] = v |
| 407 |
end |
| 408 |
end |
| 409 |
|
| 410 |
return res |
| 411 |
end |
| 412 |
|
| 413 |
--[[[ |
| 414 |
-- @function lua_util.spairs(table) |
| 415 |
-- Like `pairs` but keys are sorted lexicographically |
| 416 |
-- @param {table} table table containing key/value pairs |
| 417 |
-- @return {function} generator function returning key/value pairs |
| 418 |
--]] |
| 419 |
|
| 420 |
-- Sorted iteration: |
| 421 |
-- for k,v in spairs(t) do ... end |
| 422 |
-- |
| 423 |
-- or with custom comparison: |
| 424 |
-- for k, v in spairs(t, function(t, a, b) return t[a] < t[b] end) |
| 425 |
-- |
| 426 |
-- optional limit is also available (e.g. return top X elements) |
| 427 |
local function spairs(t, order, lim) |
| 428 |
-- collect the keys |
| 429 |
local keys = {} |
| 430 |
for k in pairs(t) do |
| 431 |
keys[#keys + 1] = k |
| 432 |
end |
| 433 |
|
| 434 |
-- if order function given, sort by it by passing the table and keys a, b, |
| 435 |
-- otherwise just sort the keys |
| 436 |
if order then |
| 437 |
table.sort(keys, function(a, b) |
| 438 |
return order(t, a, b) |
| 439 |
end) |
| 440 |
else |
| 441 |
table.sort(keys) |
| 442 |
end |
| 443 |
|
| 444 |
-- return the iterator function |
| 445 |
local i = 0 |
| 446 |
return function() |
| 447 |
i = i + 1 |
| 448 |
if not lim or i <= lim then |
| 449 |
if keys[i] then |
| 450 |
return keys[i], t[keys[i]] |
| 451 |
end |
| 452 |
end |
| 453 |
end |
| 454 |
end |
| 455 |
|
| 456 |
exports.spairs = spairs |
| 457 |
|
| 458 |
local lua_cfg_utils = require "lua_cfg_utils" |
| 459 |
|
| 460 |
exports.config_utils = lua_cfg_utils |
| 461 |
exports.disable_module = lua_cfg_utils.disable_module |
| 462 |
|
| 463 |
--[[[ |
| 464 |
-- @function lua_util.disable_module(modname) |
| 465 |
-- Checks experimental plugins state and disable if needed |
| 466 |
-- @param {string} modname name of plugin to check |
| 467 |
-- @return {boolean} true if plugin should be enabled, false otherwise |
| 468 |
--]] |
| 469 |
local function check_experimental(modname) |
| 470 |
if rspamd_config:experimental_enabled() then |
| 471 |
return true |
| 472 |
else |
| 473 |
lua_cfg_utils.disable_module(modname, 'experimental') |
| 474 |
end |
| 475 |
|
| 476 |
return false |
| 477 |
end |
| 478 |
|
| 479 |
exports.check_experimental = check_experimental |
| 480 |
|
| 481 |
--[[[ |
| 482 |
-- @function lua_util.list_to_hash(list) |
| 483 |
-- Converts numerically-indexed table to table indexed by values |
| 484 |
-- @param {table} list numerically-indexed table or string, which is treated as a one-element list |
| 485 |
-- @return {table} table indexed by values |
| 486 |
-- @example |
| 487 |
-- local h = lua_util.list_to_hash({"a", "b"}) |
| 488 |
-- -- h contains {a = true, b = true} |
| 489 |
--]] |
| 490 |
local function list_to_hash(list) |
| 491 |
if type(list) == 'table' then |
| 492 |
if list[1] then |
| 493 |
local h = {} |
| 494 |
for _, e in ipairs(list) do |
| 495 |
h[e] = true |
| 496 |
end |
| 497 |
return h |
| 498 |
else |
| 499 |
return list |
| 500 |
end |
| 501 |
elseif type(list) == 'string' then |
| 502 |
local h = {} |
| 503 |
h[list] = true |
| 504 |
return h |
| 505 |
end |
| 506 |
end |
| 507 |
|
| 508 |
exports.list_to_hash = list_to_hash |
| 509 |
|
| 510 |
--[[[ |
| 511 |
-- @function lua_util.nkeys(table|gen, param, state) |
| 512 |
-- Returns number of keys in a table (i.e. from both the array and hash parts combined) |
| 513 |
-- @param {table} list numerically-indexed table or string, which is treated as a one-element list |
| 514 |
-- @return {number} number of keys |
| 515 |
-- @example |
| 516 |
-- print(lua_util.nkeys({})) -- 0 |
| 517 |
-- print(lua_util.nkeys({ "a", nil, "b" })) -- 2 |
| 518 |
-- print(lua_util.nkeys({ dog = 3, cat = 4, bird = nil })) -- 2 |
| 519 |
-- print(lua_util.nkeys({ "a", dog = 3, cat = 4 })) -- 3 |
| 520 |
-- |
| 521 |
--]] |
| 522 |
local function nkeys(gen, param, state) |
| 523 |
local n = 0 |
| 524 |
if not param then |
| 525 |
for _, _ in pairs(gen) do |
| 526 |
n = n + 1 |
| 527 |
end |
| 528 |
else |
| 529 |
for _, _ in fun.iter(gen, param, state) do |
| 530 |
n = n + 1 |
| 531 |
end |
| 532 |
end |
| 533 |
return n |
| 534 |
end |
| 535 |
|
| 536 |
exports.nkeys = nkeys |
| 537 |
|
| 538 |
--[[[ |
| 539 |
-- @function lua_util.parse_time_interval(str) |
| 540 |
-- Parses human readable time interval |
| 541 |
-- Accepts 's' for seconds, 'm' for minutes, 'h' for hours, 'd' for days, |
| 542 |
-- 'w' for weeks, 'y' for years |
| 543 |
-- @param {string} str input string |
| 544 |
-- @return {number|nil} parsed interval as seconds (might be fractional) |
| 545 |
--]] |
| 546 |
local function parse_time_interval(str) |
| 547 |
local function parse_time_suffix(s) |
| 548 |
if s == 's' then |
| 549 |
return 1 |
| 550 |
elseif s == 'm' then |
| 551 |
return 60 |
| 552 |
elseif s == 'h' then |
| 553 |
return 3600 |
| 554 |
elseif s == 'd' then |
| 555 |
return 86400 |
| 556 |
elseif s == 'w' then |
| 557 |
return 86400 * 7 |
| 558 |
elseif s == 'y' then |
| 559 |
return 365 * 86400; |
| 560 |
end |
| 561 |
end |
| 562 |
|
| 563 |
local digit = lpeg.R("09") |
| 564 |
local parser = {} |
| 565 |
parser.integer = (lpeg.S("+-") ^ -1) * |
| 566 |
(digit ^ 1) |
| 567 |
parser.fractional = (lpeg.P(".")) * |
| 568 |
(digit ^ 1) |
| 569 |
parser.number = (parser.integer * |
| 570 |
(parser.fractional ^ -1)) + |
| 571 |
(lpeg.S("+-") * parser.fractional) |
| 572 |
parser.time = lpeg.Cf(lpeg.Cc(1) * |
| 573 |
(parser.number / tonumber) * |
| 574 |
((lpeg.S("smhdwy") / parse_time_suffix) ^ -1), |
| 575 |
function(acc, val) |
| 576 |
return acc * val |
| 577 |
end) |
| 578 |
|
| 579 |
local t = lpeg.match(parser.time, str) |
| 580 |
|
| 581 |
return t |
| 582 |
end |
| 583 |
|
| 584 |
exports.parse_time_interval = parse_time_interval |
| 585 |
|
| 586 |
--[[[ |
| 587 |
-- @function lua_util.dehumanize_number(str) |
| 588 |
-- Parses human readable number |
| 589 |
-- Accepts 'k' for thousands, 'm' for millions, 'g' for billions, 'b' suffix for 1024 multiplier, |
| 590 |
-- e.g. `10mb` equal to `10 * 1024 * 1024` |
| 591 |
-- @param {string} str input string |
| 592 |
-- @return {number|nil} parsed number |
| 593 |
--]] |
| 594 |
local function dehumanize_number(str) |
| 595 |
local function parse_suffix(s) |
| 596 |
if s == 'k' then |
| 597 |
return 1000 |
| 598 |
elseif s == 'm' then |
| 599 |
return 1000000 |
| 600 |
elseif s == 'g' then |
| 601 |
return 1e9 |
| 602 |
elseif s == 'kb' then |
| 603 |
return 1024 |
| 604 |
elseif s == 'mb' then |
| 605 |
return 1024 * 1024 |
| 606 |
elseif s == 'gb' then |
| 607 |
return 1024 * 1024; |
| 608 |
end |
| 609 |
end |
| 610 |
|
| 611 |
local digit = lpeg.R("09") |
| 612 |
local parser = {} |
| 613 |
parser.integer = (lpeg.S("+-") ^ -1) * |
| 614 |
(digit ^ 1) |
| 615 |
parser.fractional = (lpeg.P(".")) * |
| 616 |
(digit ^ 1) |
| 617 |
parser.number = (parser.integer * |
| 618 |
(parser.fractional ^ -1)) + |
| 619 |
(lpeg.S("+-") * parser.fractional) |
| 620 |
parser.humanized_number = lpeg.Cf(lpeg.Cc(1) * |
| 621 |
(parser.number / tonumber) * |
| 622 |
(((lpeg.S("kmg") * (lpeg.P("b") ^ -1)) / parse_suffix) ^ -1), |
| 623 |
function(acc, val) |
| 624 |
return acc * val |
| 625 |
end) |
| 626 |
|
| 627 |
local t = lpeg.match(parser.humanized_number, str) |
| 628 |
|
| 629 |
return t |
| 630 |
end |
| 631 |
|
| 632 |
exports.dehumanize_number = dehumanize_number |
| 633 |
|
| 634 |
--[[[ |
| 635 |
-- @function lua_util.table_cmp(t1, t2) |
| 636 |
-- Compare two tables deeply |
| 637 |
--]] |
| 638 |
local function table_cmp(table1, table2) |
| 639 |
local avoid_loops = {} |
| 640 |
local function recurse(t1, t2) |
| 641 |
if type(t1) ~= type(t2) then |
| 642 |
return false |
| 643 |
end |
| 644 |
if type(t1) ~= "table" then |
| 645 |
return t1 == t2 |
| 646 |
end |
| 647 |
|
| 648 |
if avoid_loops[t1] then |
| 649 |
return avoid_loops[t1] == t2 |
| 650 |
end |
| 651 |
avoid_loops[t1] = t2 |
| 652 |
-- Copy keys from t2 |
| 653 |
local t2keys = {} |
| 654 |
local t2tablekeys = {} |
| 655 |
for k, _ in pairs(t2) do |
| 656 |
if type(k) == "table" then |
| 657 |
table.insert(t2tablekeys, k) |
| 658 |
end |
| 659 |
t2keys[k] = true |
| 660 |
end |
| 661 |
-- Let's iterate keys from t1 |
| 662 |
for k1, v1 in pairs(t1) do |
| 663 |
local v2 = t2[k1] |
| 664 |
if type(k1) == "table" then |
| 665 |
-- if key is a table, we need to find an equivalent one. |
| 666 |
local ok = false |
| 667 |
for i, tk in ipairs(t2tablekeys) do |
| 668 |
if table_cmp(k1, tk) and recurse(v1, t2[tk]) then |
| 669 |
table.remove(t2tablekeys, i) |
| 670 |
t2keys[tk] = nil |
| 671 |
ok = true |
| 672 |
break |
| 673 |
end |
| 674 |
end |
| 675 |
if not ok then |
| 676 |
return false |
| 677 |
end |
| 678 |
else |
| 679 |
-- t1 has a key which t2 doesn't have, fail. |
| 680 |
if v2 == nil then |
| 681 |
return false |
| 682 |
end |
| 683 |
t2keys[k1] = nil |
| 684 |
if not recurse(v1, v2) then |
| 685 |
return false |
| 686 |
end |
| 687 |
end |
| 688 |
end |
| 689 |
-- if t2 has a key which t1 doesn't have, fail. |
| 690 |
if next(t2keys) then |
| 691 |
return false |
| 692 |
end |
| 693 |
return true |
| 694 |
end |
| 695 |
return recurse(table1, table2) |
| 696 |
end |
| 697 |
|
| 698 |
exports.table_cmp = table_cmp |
| 699 |
|
| 700 |
--[[[ |
| 701 |
-- @function lua_util.table_merge(t1, t2) |
| 702 |
-- Merge two tables |
| 703 |
--]] |
| 704 |
local function table_merge(t1, t2) |
| 705 |
local res = {} |
| 706 |
local nidx = 1 -- for numeric indicies |
| 707 |
local it_func = function(k, v) |
| 708 |
if type(k) == 'number' then |
| 709 |
res[nidx] = v |
| 710 |
nidx = nidx + 1 |
| 711 |
else |
| 712 |
res[k] = v |
| 713 |
end |
| 714 |
end |
| 715 |
for k, v in pairs(t1) do |
| 716 |
it_func(k, v) |
| 717 |
end |
| 718 |
for k, v in pairs(t2) do |
| 719 |
it_func(k, v) |
| 720 |
end |
| 721 |
return res |
| 722 |
end |
| 723 |
|
| 724 |
exports.table_merge = table_merge |
| 725 |
|
| 726 |
--[[[ |
| 727 |
-- @function lua_util.table_cmp(task, name, value, stop_chars) |
| 728 |
-- Performs header folding |
| 729 |
--]] |
| 730 |
exports.fold_header = function(task, name, value, stop_chars) |
| 731 |
local how |
| 732 |
|
| 733 |
if task:has_flag("milter") then |
| 734 |
how = "lf" |
| 735 |
else |
| 736 |
how = task:get_newlines_type() |
| 737 |
end |
| 738 |
|
| 739 |
return rspamd_util.fold_header(name, value, how, stop_chars) |
| 740 |
end |
| 741 |
|
| 742 |
--[[[ |
| 743 |
-- @function lua_util.fold_header_with_encoding(task, name, value[, opts]) |
| 744 |
-- Folds header value using name and optionally encodes the result. |
| 745 |
-- Encoding policy defaults to 'auto': |
| 746 |
-- - If MIME UTF-8 is enabled, encode only when the folded value is not valid UTF-8 |
| 747 |
-- - Otherwise, always encode |
| 748 |
-- `opts` table fields: |
| 749 |
-- * stop_chars: optional string with extra fold-on characters |
| 750 |
-- * encode: true|false|'auto' (default: 'auto') |
| 751 |
-- * structured: boolean, pass as `is_structured` to mime_header_encode (default: false) |
| 752 |
-- @return {string} folded (and possibly encoded) header value |
| 753 |
--]] |
| 754 |
exports.fold_header_with_encoding = function(task, name, value, opts) |
| 755 |
opts = opts or {} |
| 756 |
local stop_chars = opts.stop_chars |
| 757 |
local encode = opts.encode |
| 758 |
local structured = opts.structured or false |
| 759 |
|
| 760 |
local folded = exports.fold_header(task, name, value, stop_chars) |
| 761 |
|
| 762 |
if encode == nil or encode == 'auto' then |
| 763 |
if rspamd_config:is_mime_utf8() then |
| 764 |
if not rspamd_util.is_valid_utf8(folded) then |
| 765 |
folded = rspamd_util.mime_header_encode(folded, structured) |
| 766 |
end |
| 767 |
else |
| 768 |
folded = rspamd_util.mime_header_encode(folded, structured) |
| 769 |
end |
| 770 |
elseif encode == true then |
| 771 |
folded = rspamd_util.mime_header_encode(folded, structured) |
| 772 |
end |
| 773 |
|
| 774 |
return folded |
| 775 |
end |
| 776 |
|
| 777 |
--[[[ |
| 778 |
-- @function lua_util.override_defaults(defaults, override) |
| 779 |
-- Overrides values from defaults with override |
| 780 |
--]] |
| 781 |
local function override_defaults(def, override) |
| 782 |
-- Corner cases |
| 783 |
if not override or type(override) ~= 'table' then |
| 784 |
return def |
| 785 |
end |
| 786 |
if not def or type(def) ~= 'table' then |
| 787 |
return override |
| 788 |
end |
| 789 |
|
| 790 |
local res = {} |
| 791 |
|
| 792 |
-- Allow transparent ucl |
| 793 |
local pairs_func, def_pairs_func = pairs, pairs |
| 794 |
local type_func, def_type_func = type, type |
| 795 |
if type(override[0]) == 'userdata' then |
| 796 |
pairs_func = function(t) |
| 797 |
return t:pairs() |
| 798 |
end |
| 799 |
type_func = function(t) |
| 800 |
return t:type() |
| 801 |
end |
| 802 |
end |
| 803 |
if type(def[0]) == 'userdata' then |
| 804 |
def_pairs_func = function(t) |
| 805 |
return t:pairs() |
| 806 |
end |
| 807 |
def_type_func = function(t) |
| 808 |
return t:type() |
| 809 |
end |
| 810 |
end |
| 811 |
|
| 812 |
for k, v in pairs_func(override) do |
| 813 |
if type_func(v) == 'table' then |
| 814 |
if def[k] and def_type_func(def[k]) == 'table' then |
| 815 |
-- Recursively override elements |
| 816 |
res[k] = override_defaults(def[k], v) |
| 817 |
else |
| 818 |
res[k] = v |
| 819 |
end |
| 820 |
else |
| 821 |
res[k] = v |
| 822 |
end |
| 823 |
end |
| 824 |
|
| 825 |
for k, v in def_pairs_func(def) do |
| 826 |
if type(res[k]) == 'nil' then |
| 827 |
res[k] = v |
| 828 |
end |
| 829 |
end |
| 830 |
|
| 831 |
return res |
| 832 |
end |
| 833 |
|
| 834 |
exports.override_defaults = override_defaults |
| 835 |
|
| 836 |
--[[[ |
| 837 |
-- @function lua_util.filter_specific_urls(urls, params) |
| 838 |
-- params: { |
| 839 |
- - task - if needed to save in the cache |
| 840 |
- - limit <int> (default = 9999) |
| 841 |
- - esld_limit <int> (default = 9999) n domains per eSLD (effective second level domain) |
| 842 |
works only if number of unique eSLD less than `limit` |
| 843 |
- - need_emails <bool> (default = false) |
| 844 |
- - filter <callback> (default = nil) |
| 845 |
- - prefix <string> cache prefix (default = nil) |
| 846 |
-- } |
| 847 |
-- Apply heuristic in extracting of urls from `urls` table, this function |
| 848 |
-- tries its best to extract specific number of urls from a task based on |
| 849 |
-- their characteristics |
| 850 |
--]] |
| 851 |
exports.filter_specific_urls = function(urls, params) |
| 852 |
local cache_key |
| 853 |
|
| 854 |
if params.task and not params.no_cache then |
| 855 |
if params.prefix then |
| 856 |
cache_key = params.prefix |
| 857 |
else |
| 858 |
cache_key = string.format('sp_urls_%d%s%s%s', params.limit, |
| 859 |
tostring(params.need_emails or false), |
| 860 |
tostring(params.need_images or false), |
| 861 |
tostring(params.need_content or false)) |
| 862 |
end |
| 863 |
local cached = params.task:cache_get(cache_key) |
| 864 |
|
| 865 |
if cached then |
| 866 |
return cached |
| 867 |
end |
| 868 |
end |
| 869 |
|
| 870 |
if not urls then |
| 871 |
return {} |
| 872 |
end |
| 873 |
|
| 874 |
if params.filter then |
| 875 |
urls = fun.totable(fun.filter(params.filter, urls)) |
| 876 |
end |
| 877 |
|
| 878 |
-- Memory bounds protection against DoS: absolute maximum of URLs to process |
| 879 |
-- Even if we have 100k URLs of 1KB each in input, we process at most first 50k |
| 880 |
-- This prevents excessive memory usage and CPU time for malicious messages |
| 881 |
local max_urls_to_process = 50000 |
| 882 |
if #urls > max_urls_to_process then |
| 883 |
local logger = require "rspamd_logger" |
| 884 |
logger.warnx(params.task, 'too many URLs to process: %d, limiting to %d', |
| 885 |
#urls, max_urls_to_process) |
| 886 |
-- Truncate the urls table |
| 887 |
local truncated = {} |
| 888 |
for i = 1, max_urls_to_process do |
| 889 |
truncated[i] = urls[i] |
| 890 |
end |
| 891 |
urls = truncated |
| 892 |
end |
| 893 |
|
| 894 |
-- Filter by tld: |
| 895 |
local tlds = {} |
| 896 |
local eslds = {} |
| 897 |
local ntlds, neslds = 0, 0 |
| 898 |
|
| 899 |
-- Use two structures: hash set for deduplication, array for results |
| 900 |
local res = {} -- array of URLs (maintains order) |
| 901 |
local seen = {} -- hash set for deduplication (hash -> true) |
| 902 |
local nres = 0 |
| 903 |
|
| 904 |
local cta_priority_map |
| 905 |
|
| 906 |
if params.task and params.task.get_text_parts then |
| 907 |
local text_parts = params.task:get_text_parts() |
| 908 |
if text_parts then |
| 909 |
cta_priority_map = {} |
| 910 |
for _, part in ipairs(text_parts) do |
| 911 |
if part.is_html and part:is_html() and part.get_cta_urls then |
| 912 |
local entries = part:get_cta_urls({original = true, with_weights = true}) |
| 913 |
if type(entries) == 'table' then |
| 914 |
for _, entry in ipairs(entries) do |
| 915 |
if entry and entry.url then |
| 916 |
local url = entry.url |
| 917 |
-- Use hash instead of tostring to avoid string interning |
| 918 |
local hash = url:get_hash() |
| 919 |
local weight = entry.weight or 0 |
| 920 |
local score = 6 + math.floor(weight * 10 + 0.5) |
| 921 |
if not cta_priority_map[hash] or score > cta_priority_map[hash] then |
| 922 |
cta_priority_map[hash] = score |
| 923 |
end |
| 924 |
local redir = url:get_redirected() |
| 925 |
if redir then |
| 926 |
-- Skip display-only URLs (phishing bait text) - only include real redirects |
| 927 |
local redir_flags = redir:get_flags() |
| 928 |
if not redir_flags.html_displayed then |
| 929 |
local rhash = redir:get_hash() |
| 930 |
if not cta_priority_map[rhash] or score > cta_priority_map[rhash] then |
| 931 |
cta_priority_map[rhash] = score |
| 932 |
end |
| 933 |
end |
| 934 |
end |
| 935 |
end |
| 936 |
end |
| 937 |
end |
| 938 |
end |
| 939 |
end |
| 940 |
|
| 941 |
if next(cta_priority_map) == nil then |
| 942 |
cta_priority_map = nil |
| 943 |
end |
| 944 |
end |
| 945 |
end |
| 946 |
|
| 947 |
local function insert_url(hash, u) |
| 948 |
if not seen[hash] then |
| 949 |
seen[hash] = true |
| 950 |
table.insert(res, u) |
| 951 |
nres = nres + 1 |
| 952 |
|
| 953 |
return true |
| 954 |
end |
| 955 |
|
| 956 |
return false |
| 957 |
end |
| 958 |
|
| 959 |
local url_index = 0 -- Track URL processing order for stable sorting |
| 960 |
local function process_single_url(u, default_priority) |
| 961 |
url_index = url_index + 1 |
| 962 |
local priority = default_priority or 1 -- Normal priority |
| 963 |
local flags = u:get_flags() |
| 964 |
if params.ignore_ip and flags.numeric then |
| 965 |
return |
| 966 |
end |
| 967 |
|
| 968 |
if flags.redirected then |
| 969 |
local redir = u:get_redirected() -- get the real url |
| 970 |
|
| 971 |
if params.ignore_redirected then |
| 972 |
-- Replace `u` with redir |
| 973 |
u = redir |
| 974 |
priority = 2 |
| 975 |
else |
| 976 |
-- Process both redirected url and the original one |
| 977 |
process_single_url(redir, 2) |
| 978 |
end |
| 979 |
end |
| 980 |
|
| 981 |
if flags.image then |
| 982 |
if not params.need_images then |
| 983 |
-- Ignore url |
| 984 |
return |
| 985 |
else |
| 986 |
-- Penalise images in urls |
| 987 |
priority = 0 |
| 988 |
end |
| 989 |
end |
| 990 |
|
| 991 |
local esld = u:get_tld() |
| 992 |
-- Use fast hash instead of tostring to avoid string interning |
| 993 |
-- This is critical for DoS protection with 100k+ URLs |
| 994 |
local url_hash = u:get_hash() |
| 995 |
|
| 996 |
if cta_priority_map then |
| 997 |
local cta_pr = cta_priority_map[url_hash] |
| 998 |
if not cta_pr and flags.redirected then |
| 999 |
local redir_url = u:get_redirected() |
| 1000 |
if redir_url then |
| 1001 |
cta_pr = cta_priority_map[redir_url:get_hash()] |
| 1002 |
end |
| 1003 |
end |
| 1004 |
|
| 1005 |
if cta_pr then |
| 1006 |
-- Use CTA priority as base, but add phished/subject bonuses on top |
| 1007 |
priority = cta_pr |
| 1008 |
if flags.phished then |
| 1009 |
priority = priority + 5 -- Phished URLs get significant extra priority (security critical) |
| 1010 |
end |
| 1011 |
if flags.subject then |
| 1012 |
priority = priority + 2 -- Subject URLs get extra priority |
| 1013 |
end |
| 1014 |
end |
| 1015 |
end |
| 1016 |
|
| 1017 |
if esld then |
| 1018 |
-- Special cases (only apply if CTA priority wasn't set) |
| 1019 |
-- CTA priority takes precedence as it's explicitly calculated with bonuses |
| 1020 |
local had_cta = priority ~= (default_priority or 1) |
| 1021 |
|
| 1022 |
if not had_cta then |
| 1023 |
if (u:get_protocol() ~= 'mailto') and (not flags.html_displayed) then |
| 1024 |
if flags.obscured then |
| 1025 |
priority = 3 |
| 1026 |
else |
| 1027 |
if (flags.has_user or flags.has_port) then |
| 1028 |
priority = 2 |
| 1029 |
elseif (flags.subject or flags.phished) then |
| 1030 |
priority = 2 |
| 1031 |
end |
| 1032 |
end |
| 1033 |
elseif flags.html_displayed then |
| 1034 |
priority = 0 |
| 1035 |
end |
| 1036 |
end |
| 1037 |
|
| 1038 |
if not eslds[esld] then |
| 1039 |
eslds[esld] = { { url_hash, u, priority, url_index } } |
| 1040 |
neslds = neslds + 1 |
| 1041 |
else |
| 1042 |
if #eslds[esld] < params.esld_limit then |
| 1043 |
table.insert(eslds[esld], { url_hash, u, priority, url_index }) |
| 1044 |
end |
| 1045 |
end |
| 1046 |
|
| 1047 |
|
| 1048 |
-- eSLD - 1 part => tld |
| 1049 |
local parts = rspamd_str_split(esld, '.') |
| 1050 |
local tld = table.concat(fun.totable(fun.tail(parts)), '.') |
| 1051 |
|
| 1052 |
if not tlds[tld] then |
| 1053 |
tlds[tld] = { { url_hash, u, priority, url_index } } |
| 1054 |
ntlds = ntlds + 1 |
| 1055 |
else |
| 1056 |
table.insert(tlds[tld], { url_hash, u, priority, url_index }) |
| 1057 |
end |
| 1058 |
end |
| 1059 |
end |
| 1060 |
|
| 1061 |
for _, u in ipairs(urls) do |
| 1062 |
process_single_url(u) |
| 1063 |
end |
| 1064 |
|
| 1065 |
local limit = params.limit |
| 1066 |
limit = limit - nres |
| 1067 |
if limit < 0 then |
| 1068 |
limit = 0 |
| 1069 |
end |
| 1070 |
|
| 1071 |
if limit == 0 then |
| 1072 |
if params.task and not params.no_cache then |
| 1073 |
params.task:cache_set(cache_key, res) |
| 1074 |
end |
| 1075 |
return res |
| 1076 |
end |
| 1077 |
|
| 1078 |
-- Sort eSLDs and tlds |
| 1079 |
local function sort_stuff(tbl) |
| 1080 |
-- Sort according to max priority |
| 1081 |
table.sort(tbl, function(e1, e2) |
| 1082 |
-- Sort by priority (desc) then by url_index (asc) for stable sorting |
| 1083 |
table.sort(e1, function(tr1, tr2) |
| 1084 |
if tr1[3] ~= tr2[3] then |
| 1085 |
return tr1[3] < tr2[3] -- Lower priority first |
| 1086 |
else |
| 1087 |
return tr1[4] < tr2[4] -- Earlier URL index first (stable) |
| 1088 |
end |
| 1089 |
end) |
| 1090 |
table.sort(e2, function(tr1, tr2) |
| 1091 |
if tr1[3] ~= tr2[3] then |
| 1092 |
return tr1[3] < tr2[3] |
| 1093 |
else |
| 1094 |
return tr1[4] < tr2[4] |
| 1095 |
end |
| 1096 |
end) |
| 1097 |
|
| 1098 |
if e1[#e1][3] ~= e2[#e2][3] then |
| 1099 |
-- Sort by priority so max priority is at the beginning |
| 1100 |
return e1[#e1][3] > e2[#e2][3] |
| 1101 |
else |
| 1102 |
-- Prefer less urls to more urls per esld, then earlier index |
| 1103 |
if #e1 ~= #e2 then |
| 1104 |
return #e1 < #e2 |
| 1105 |
else |
| 1106 |
return e1[#e1][4] < e2[#e2][4] -- Use index as final tiebreaker |
| 1107 |
end |
| 1108 |
end |
| 1109 |
end) |
| 1110 |
|
| 1111 |
return tbl |
| 1112 |
end |
| 1113 |
|
| 1114 |
eslds = sort_stuff(exports.values(eslds)) |
| 1115 |
neslds = #eslds |
| 1116 |
|
| 1117 |
if neslds <= limit then |
| 1118 |
-- Number of eslds < limit |
| 1119 |
repeat |
| 1120 |
local item_found = false |
| 1121 |
|
| 1122 |
for _, lurls in ipairs(eslds) do |
| 1123 |
if #lurls > 0 then |
| 1124 |
local last = table.remove(lurls) |
| 1125 |
insert_url(last[1], last[2]) |
| 1126 |
limit = limit - 1 |
| 1127 |
item_found = true |
| 1128 |
end |
| 1129 |
end |
| 1130 |
until limit <= 0 or not item_found |
| 1131 |
|
| 1132 |
if params.task and not params.no_cache then |
| 1133 |
params.task:cache_set(cache_key, res) |
| 1134 |
end |
| 1135 |
return res |
| 1136 |
end |
| 1137 |
|
| 1138 |
tlds = sort_stuff(exports.values(tlds)) |
| 1139 |
ntlds = #tlds |
| 1140 |
|
| 1141 |
-- Number of tlds < limit |
| 1142 |
while limit > 0 do |
| 1143 |
for _, lurls in ipairs(tlds) do |
| 1144 |
if #lurls > 0 then |
| 1145 |
local last = table.remove(lurls) |
| 1146 |
insert_url(last[1], last[2]) |
| 1147 |
limit = limit - 1 |
| 1148 |
end |
| 1149 |
if limit == 0 then |
| 1150 |
break |
| 1151 |
end |
| 1152 |
end |
| 1153 |
end |
| 1154 |
|
| 1155 |
if params.task and not params.no_cache then |
| 1156 |
params.task:cache_set(cache_key, res) |
| 1157 |
end |
| 1158 |
return res |
| 1159 |
end |
| 1160 |
|
| 1161 |
--[[[ |
| 1162 |
-- @function lua_util.extract_specific_urls(params) |
| 1163 |
-- params: { |
| 1164 |
- - task |
| 1165 |
- - limit <int> (default = 9999) |
| 1166 |
- - esld_limit <int> (default = 9999) n domains per eSLD (effective second level domain) |
| 1167 |
works only if number of unique eSLD less than `limit` |
| 1168 |
- - need_emails <bool> (default = false) |
| 1169 |
- - filter <callback> (default = nil) |
| 1170 |
- - prefix <string> cache prefix (default = nil) |
| 1171 |
- - ignore_redirected <bool> (default = false) |
| 1172 |
- - need_images <bool> (default = false) |
| 1173 |
- - need_content <bool> (default = nil, uses global include_content_urls config which defaults to true) |
| 1174 |
-- } |
| 1175 |
-- Apply heuristic in extracting of urls from task, this function |
| 1176 |
-- tries its best to extract specific number of urls from a task based on |
| 1177 |
-- their characteristics |
| 1178 |
--]] |
| 1179 |
-- exports.extract_specific_urls = function(params_or_task, limit, need_emails, filter, prefix) |
| 1180 |
exports.extract_specific_urls = function(params_or_task, lim, need_emails, filter, prefix) |
| 1181 |
local default_params = { |
| 1182 |
limit = 9999, |
| 1183 |
esld_limit = 9999, |
| 1184 |
need_emails = false, |
| 1185 |
need_images = false, |
| 1186 |
need_content = nil, -- nil means use global include_content_urls config (default: true) |
| 1187 |
filter = nil, |
| 1188 |
prefix = nil, |
| 1189 |
ignore_ip = false, |
| 1190 |
ignore_redirected = false, |
| 1191 |
no_cache = false, |
| 1192 |
} |
| 1193 |
|
| 1194 |
local params |
| 1195 |
if type(params_or_task) == 'table' and type(lim) == 'nil' then |
| 1196 |
params = params_or_task |
| 1197 |
else |
| 1198 |
-- Deprecated call |
| 1199 |
params = { |
| 1200 |
task = params_or_task, |
| 1201 |
limit = lim, |
| 1202 |
need_emails = need_emails, |
| 1203 |
filter = filter, |
| 1204 |
prefix = prefix |
| 1205 |
} |
| 1206 |
end |
| 1207 |
for k, v in pairs(default_params) do |
| 1208 |
if type(params[k]) == 'nil' and v ~= nil then |
| 1209 |
params[k] = v |
| 1210 |
end |
| 1211 |
end |
| 1212 |
local url_params = { |
| 1213 |
emails = params.need_emails, |
| 1214 |
images = params.need_images, |
| 1215 |
content = params.need_content, |
| 1216 |
flags = params.flags, -- maybe nil |
| 1217 |
flags_mode = params.flags_mode, -- maybe nil |
| 1218 |
} |
| 1219 |
|
| 1220 |
-- Shortcut for cached stuff |
| 1221 |
if params.task and not params.no_cache then |
| 1222 |
local cache_key |
| 1223 |
if params.prefix then |
| 1224 |
cache_key = params.prefix |
| 1225 |
else |
| 1226 |
local cache_key_suffix |
| 1227 |
if params.flags then |
| 1228 |
cache_key_suffix = table.concat(params.flags) .. (params.flags_mode or '') |
| 1229 |
else |
| 1230 |
-- Use tostring directly to distinguish nil (config default) from false (explicit exclude) |
| 1231 |
cache_key_suffix = string.format('%s%s%s', |
| 1232 |
tostring(params.need_emails or false), |
| 1233 |
tostring(params.need_images or false), |
| 1234 |
tostring(params.need_content)) -- nil = config default, false = explicit exclude |
| 1235 |
end |
| 1236 |
cache_key = string.format('sp_urls_%d%s', params.limit, cache_key_suffix) |
| 1237 |
end |
| 1238 |
local cached = params.task:cache_get(cache_key) |
| 1239 |
|
| 1240 |
if cached then |
| 1241 |
return cached |
| 1242 |
end |
| 1243 |
end |
| 1244 |
|
| 1245 |
-- No cache version |
| 1246 |
local urls = params.task:get_urls(url_params) |
| 1247 |
|
| 1248 |
return exports.filter_specific_urls(urls, params) |
| 1249 |
end |
| 1250 |
|
| 1251 |
|
| 1252 |
--[[[ |
| 1253 |
-- @function lua_util.deepcopy(table) |
| 1254 |
-- params: { |
| 1255 |
- - table |
| 1256 |
-- } |
| 1257 |
-- Performs deep copy of the table. Including metatables |
| 1258 |
--]] |
| 1259 |
exports.deepcopy = deepcopy |
| 1260 |
|
| 1261 |
--[[[ |
| 1262 |
-- @function lua_util.deepsort(table) |
| 1263 |
-- params: { |
| 1264 |
- - table |
| 1265 |
-- } |
| 1266 |
-- Performs recursive in-place sort of a table |
| 1267 |
--]] |
| 1268 |
local function default_sort_cmp(e1, e2) |
| 1269 |
if type(e1) == type(e2) then |
| 1270 |
return e1 < e2 |
| 1271 |
else |
| 1272 |
return type(e1) < type(e2) |
| 1273 |
end |
| 1274 |
end |
| 1275 |
|
| 1276 |
local function deepsort(tbl, sort_func) |
| 1277 |
local orig_type = type(tbl) |
| 1278 |
if orig_type == 'table' then |
| 1279 |
table.sort(tbl, sort_func or default_sort_cmp) |
| 1280 |
for _, orig_value in next, tbl, nil do |
| 1281 |
deepsort(orig_value) |
| 1282 |
end |
| 1283 |
end |
| 1284 |
end |
| 1285 |
|
| 1286 |
exports.deepsort = deepsort |
| 1287 |
|
| 1288 |
--[[[ |
| 1289 |
-- @function lua_util.shallowcopy(tbl) |
| 1290 |
-- Performs shallow (and fast) copy of a table or another Lua type |
| 1291 |
--]] |
| 1292 |
exports.shallowcopy = shallowcopy |
| 1293 |
|
| 1294 |
-- Debugging support |
| 1295 |
local logger = require "rspamd_logger" |
| 1296 |
local unconditional_debug = logger.log_level() == 'debug' |
| 1297 |
local debug_modules = {} |
| 1298 |
local debug_aliases = {} |
| 1299 |
local log_level = 384 -- debug + forced (1 << 7 | 1 << 8) |
| 1300 |
|
| 1301 |
|
| 1302 |
exports.init_debug_logging = function(config) |
| 1303 |
-- Fill debug modules from the config |
| 1304 |
if not unconditional_debug then |
| 1305 |
local log_config = config:get_all_opt('logging') |
| 1306 |
if log_config then |
| 1307 |
local log_level_str = log_config.level |
| 1308 |
if log_level_str then |
| 1309 |
if log_level_str == 'debug' then |
| 1310 |
unconditional_debug = true |
| 1311 |
end |
| 1312 |
end |
| 1313 |
if log_config.debug_modules then |
| 1314 |
for _, m in ipairs(log_config.debug_modules) do |
| 1315 |
debug_modules[m] = true |
| 1316 |
logger.infox(config, 'enable debug for Lua module %s', m) |
| 1317 |
end |
| 1318 |
end |
| 1319 |
|
| 1320 |
if #debug_aliases > 0 then |
| 1321 |
for alias, mod in pairs(debug_aliases) do |
| 1322 |
if debug_modules[mod] then |
| 1323 |
debug_modules[alias] = true |
| 1324 |
logger.infox(config, 'enable debug for Lua module %s (%s aliased)', |
| 1325 |
alias, mod) |
| 1326 |
end |
| 1327 |
end |
| 1328 |
end |
| 1329 |
end |
| 1330 |
end |
| 1331 |
end |
| 1332 |
|
| 1333 |
exports.enable_debug_logging = function() |
| 1334 |
unconditional_debug = true |
| 1335 |
end |
| 1336 |
|
| 1337 |
exports.enable_debug_modules = function(...) |
| 1338 |
for _, m in ipairs({ ... }) do |
| 1339 |
debug_modules[m] = true |
| 1340 |
end |
| 1341 |
end |
| 1342 |
|
| 1343 |
exports.disable_debug_logging = function() |
| 1344 |
unconditional_debug = false |
| 1345 |
end |
| 1346 |
|
| 1347 |
--[[[ |
| 1348 |
-- @function lua_util.debugm(module, [log_object], format, ...) |
| 1349 |
-- Performs fast debug log for a specific module |
| 1350 |
--]] |
| 1351 |
exports.debugm = function(mod, obj_or_fmt, fmt_or_something, ...) |
| 1352 |
if unconditional_debug or debug_modules[mod] then |
| 1353 |
if type(obj_or_fmt) == 'string' then |
| 1354 |
logger.logx(log_level, mod, '', 2, obj_or_fmt, fmt_or_something, ...) |
| 1355 |
else |
| 1356 |
logger.logx(log_level, mod, obj_or_fmt, 2, fmt_or_something, ...) |
| 1357 |
end |
| 1358 |
end |
| 1359 |
end |
| 1360 |
|
| 1361 |
--[[[ |
| 1362 |
-- @function lua_util.add_debug_alias(mod, alias) |
| 1363 |
-- Add debugging alias so logging to `alias` will be treated as logging to `mod` |
| 1364 |
--]] |
| 1365 |
exports.add_debug_alias = function(mod, alias) |
| 1366 |
debug_aliases[alias] = mod |
| 1367 |
|
| 1368 |
if debug_modules[mod] then |
| 1369 |
debug_modules[alias] = true |
| 1370 |
logger.infox(rspamd_config, 'enable debug for Lua module %s (%s aliased)', |
| 1371 |
alias, mod) |
| 1372 |
end |
| 1373 |
end |
| 1374 |
---[[[ |
| 1375 |
-- @function lua_util.get_task_verdict(task) |
| 1376 |
-- Returns verdict for a task + score if certain, must be called from idempotent filters only |
| 1377 |
-- Returns string: |
| 1378 |
-- * `spam`: if message have over reject threshold and has more than one positive rule |
| 1379 |
-- * `junk`: if a message has between score between [add_header/rewrite subject] to reject thresholds and has more than two positive rules |
| 1380 |
-- * `passthrough`: if a message has been passed through some short-circuit rule |
| 1381 |
-- * `ham`: if a message has overall score below junk level **and** more than three negative rule, or negative total score |
| 1382 |
-- * `uncertain`: all other cases |
| 1383 |
--]] |
| 1384 |
exports.get_task_verdict = function(task) |
| 1385 |
local lua_verdict = require "lua_verdict" |
| 1386 |
|
| 1387 |
return lua_verdict.get_default_verdict(task) |
| 1388 |
end |
| 1389 |
|
| 1390 |
---[[[ |
| 1391 |
-- @function lua_util.maybe_obfuscate_string(subject, settings, prefix) |
| 1392 |
-- Obfuscate string if enabled in settings. Also checks utf8 validity - if |
| 1393 |
-- string is not valid utf8 then '???' is returned. Empty string returned as is. |
| 1394 |
-- Supported settings: |
| 1395 |
-- * <prefix>_privacy = false - subject privacy is off |
| 1396 |
-- * <prefix>_privacy_alg = 'blake2' - default hash-algorithm to obfuscate subject |
| 1397 |
-- * <prefix>_privacy_prefix = 'obf' - prefix to show it's obfuscated |
| 1398 |
-- * <prefix>_privacy_length = 16 - cut the length of the hash; if 0 or fasle full hash is returned |
| 1399 |
-- @return obfuscated or validated subject |
| 1400 |
--]] |
| 1401 |
|
| 1402 |
exports.maybe_obfuscate_string = function(subject, settings, prefix) |
| 1403 |
local hash = require 'rspamd_cryptobox_hash' |
| 1404 |
if not subject or subject == '' then |
| 1405 |
return subject |
| 1406 |
elseif not rspamd_util.is_valid_utf8(subject) then |
| 1407 |
subject = '???' |
| 1408 |
elseif settings[prefix .. '_privacy'] then |
| 1409 |
local hash_alg = settings[prefix .. '_privacy_alg'] or 'blake2' |
| 1410 |
local subject_hash = hash.create_specific(hash_alg, subject) |
| 1411 |
|
| 1412 |
local strip_len = settings[prefix .. '_privacy_length'] |
| 1413 |
if strip_len and strip_len > 0 then |
| 1414 |
subject = subject_hash:hex():sub(1, strip_len) |
| 1415 |
else |
| 1416 |
subject = subject_hash:hex() |
| 1417 |
end |
| 1418 |
|
| 1419 |
local privacy_prefix = settings[prefix .. '_privacy_prefix'] |
| 1420 |
if privacy_prefix and #privacy_prefix > 0 then |
| 1421 |
subject = privacy_prefix .. ':' .. subject |
| 1422 |
end |
| 1423 |
end |
| 1424 |
|
| 1425 |
return subject |
| 1426 |
end |
| 1427 |
|
| 1428 |
---[[[ |
| 1429 |
-- @function lua_util.maybe_encrypt_header(header, settings, prefix) |
| 1430 |
-- Encode header with configured public key if enabled in settings. |
| 1431 |
-- If header is not set then nil is returned. If pub_key is empty then header is returned. |
| 1432 |
-- Supported settings: |
| 1433 |
-- * <prefix>_encrypt = false - no need for encryption of a header |
| 1434 |
-- * <prefix>_key = 'key' - key that is used encrypt header |
| 1435 |
-- * <prefix>_nonce = 'nonce' - nonce to encrypt header(optional) |
| 1436 |
-- @return encrypted header |
| 1437 |
---]]] |
| 1438 |
exports.maybe_encrypt_header = function(header, settings, prefix) |
| 1439 |
local rspamd_secretbox = require "rspamd_cryptobox_secretbox" |
| 1440 |
|
| 1441 |
if not header or header == '' then |
| 1442 |
logger.errx(rspamd_config, "Header is empty or nil. Header: %s", header) |
| 1443 |
return nil |
| 1444 |
elseif settings[prefix .. '_encrypt'] then |
| 1445 |
local key = settings[prefix .. '_key'] |
| 1446 |
if not key or key == '' then |
| 1447 |
logger.errx(rspamd_config, "Key is empty or nil. Key: %s", key) |
| 1448 |
return header |
| 1449 |
end |
| 1450 |
local cryptobox = rspamd_secretbox.create(key) |
| 1451 |
|
| 1452 |
local nonce = settings[prefix .. '_nonce'] |
| 1453 |
local encrypted_header = '' |
| 1454 |
if not nonce or nonce == '' then |
| 1455 |
encrypted_header, nonce = cryptobox:encrypt(header) |
| 1456 |
else |
| 1457 |
encrypted_header = cryptobox:encrypt(header, nonce) |
| 1458 |
end |
| 1459 |
encrypted_header = nonce .. encrypted_header |
| 1460 |
return encrypted_header |
| 1461 |
end |
| 1462 |
end |
| 1463 |
|
| 1464 |
---[[[ |
| 1465 |
-- @function lua_util.maybe_decrypt_header(header, settings, prefix, nonce) |
| 1466 |
-- Decode enoced with configured public_key header if enabled in settings. |
| 1467 |
-- If encoded header is not set then nil is returned. If pub_key is empty then encoded header is returned. |
| 1468 |
-- Supported settings: |
| 1469 |
-- * <prefix>_encrypt = false - no need for decryption of a header |
| 1470 |
-- * <prefix>_key = 'key' - key that is used decrypt header |
| 1471 |
-- * <prefix>_nonce = 'nonce' - nonce used to encrypt header(optional) |
| 1472 |
-- Nonce is an optional argument if <prefix>_nonce is provided, otherwise it is an required argument |
| 1473 |
-- and <prefix>_nonce is an optional |
| 1474 |
-- @return decrypted header |
| 1475 |
---]]] |
| 1476 |
exports.maybe_decrypt_header = function(encrypted_header, settings, prefix, nonce) |
| 1477 |
local rspamd_secretbox = require "rspamd_cryptobox_secretbox" |
| 1478 |
|
| 1479 |
if not encrypted_header or encrypted_header == '' then |
| 1480 |
logger.errx(rspamd_config, "Encrypted header is empty or nil. Encrypted header: %s", encrypted_header) |
| 1481 |
return nil |
| 1482 |
elseif settings[prefix .. '_encrypt'] then |
| 1483 |
local key = settings[prefix .. '_key'] |
| 1484 |
if not key or key == '' then |
| 1485 |
logger.errx(rspamd_config, "Key is empty or nil. Key: %s", key) |
| 1486 |
return encrypted_header |
| 1487 |
end |
| 1488 |
local cryptobox = rspamd_secretbox.create(key) |
| 1489 |
|
| 1490 |
local result = false |
| 1491 |
local header = '' |
| 1492 |
if not nonce then |
| 1493 |
result, header = cryptobox:decrypt(encrypted_header, settings[prefix .. '_nonce']) |
| 1494 |
else |
| 1495 |
result, header = cryptobox:decrypt(encrypted_header, nonce) |
| 1496 |
end |
| 1497 |
|
| 1498 |
if not result then |
| 1499 |
logger.infox(rspamd_config, "Decryption is failed with result: %s and decrypted header: %s", result, header) |
| 1500 |
return nil |
| 1501 |
end |
| 1502 |
|
| 1503 |
return header |
| 1504 |
end |
| 1505 |
end |
| 1506 |
|
| 1507 |
---[[[ |
| 1508 |
-- @function lua_util.callback_from_string(str) |
| 1509 |
-- Converts a string like `return function(...) end` to lua function and return true and this function |
| 1510 |
-- or returns false + error message |
| 1511 |
-- @return status code and function object or an error message |
| 1512 |
--]]] |
| 1513 |
exports.callback_from_string = function(s) |
| 1514 |
local loadstring = loadstring or load |
| 1515 |
|
| 1516 |
if not s or #s == 0 then |
| 1517 |
return false, 'invalid or empty string' |
| 1518 |
end |
| 1519 |
|
| 1520 |
s = exports.rspamd_str_trim(s) |
| 1521 |
local inp |
| 1522 |
|
| 1523 |
if s:match('^return%s*function') then |
| 1524 |
-- 'return function', can be evaluated directly |
| 1525 |
inp = s |
| 1526 |
elseif s:match('^function%s*%(') then |
| 1527 |
inp = 'return ' .. s |
| 1528 |
else |
| 1529 |
-- Just a plain sequence |
| 1530 |
inp = 'return function(...)\n' .. s .. '; end' |
| 1531 |
end |
| 1532 |
|
| 1533 |
local chunk, err = loadstring(inp) |
| 1534 |
if not chunk then |
| 1535 |
return false, err |
| 1536 |
end |
| 1537 |
|
| 1538 |
local ret, res_or_err = pcall(chunk) |
| 1539 |
|
| 1540 |
if not ret or type(res_or_err) ~= 'function' then |
| 1541 |
return false, res_or_err |
| 1542 |
end |
| 1543 |
|
| 1544 |
return ret, res_or_err |
| 1545 |
end |
| 1546 |
|
| 1547 |
---[[[ |
| 1548 |
-- @function lua_util.keys(t) |
| 1549 |
-- Returns all keys from a specific table |
| 1550 |
-- @param {table} t input table (or iterator triplet) |
| 1551 |
-- @return array of keys |
| 1552 |
--]]] |
| 1553 |
exports.keys = function(gen, param, state) |
| 1554 |
local keys = {} |
| 1555 |
local i = 1 |
| 1556 |
|
| 1557 |
if param then |
| 1558 |
for k, _ in fun.iter(gen, param, state) do |
| 1559 |
rawset(keys, i, k) |
| 1560 |
i = i + 1 |
| 1561 |
end |
| 1562 |
else |
| 1563 |
for k, _ in pairs(gen) do |
| 1564 |
rawset(keys, i, k) |
| 1565 |
i = i + 1 |
| 1566 |
end |
| 1567 |
end |
| 1568 |
|
| 1569 |
return keys |
| 1570 |
end |
| 1571 |
|
| 1572 |
---[[[ |
| 1573 |
-- @function lua_util.values(t) |
| 1574 |
-- Returns all values from a specific table |
| 1575 |
-- @param {table} t input table |
| 1576 |
-- @return array of values |
| 1577 |
--]]] |
| 1578 |
exports.values = function(gen, param, state) |
| 1579 |
local values = {} |
| 1580 |
local i = 1 |
| 1581 |
|
| 1582 |
if param then |
| 1583 |
for _, v in fun.iter(gen, param, state) do |
| 1584 |
rawset(values, i, v) |
| 1585 |
i = i + 1 |
| 1586 |
end |
| 1587 |
else |
| 1588 |
for _, v in pairs(gen) do |
| 1589 |
rawset(values, i, v) |
| 1590 |
i = i + 1 |
| 1591 |
end |
| 1592 |
end |
| 1593 |
|
| 1594 |
return values |
| 1595 |
end |
| 1596 |
|
| 1597 |
---[[[ |
| 1598 |
-- @function lua_util.distance_sorted(t1, t2) |
| 1599 |
-- Returns distance between two sorted tables t1 and t2 |
| 1600 |
-- @param {table} t1 input table |
| 1601 |
-- @param {table} t2 input table |
| 1602 |
-- @return distance between `t1` and `t2` |
| 1603 |
--]]] |
| 1604 |
exports.distance_sorted = function(t1, t2) |
| 1605 |
local ncomp = #t1 |
| 1606 |
local ndiff = 0 |
| 1607 |
local i, j = 1, 1 |
| 1608 |
|
| 1609 |
if ncomp < #t2 then |
| 1610 |
ncomp = #t2 |
| 1611 |
end |
| 1612 |
|
| 1613 |
for _ = 1, ncomp do |
| 1614 |
if j > #t2 then |
| 1615 |
ndiff = ndiff + ncomp - #t2 |
| 1616 |
if i > j then |
| 1617 |
ndiff = ndiff - (i - j) |
| 1618 |
end |
| 1619 |
break |
| 1620 |
elseif i > #t1 then |
| 1621 |
ndiff = ndiff + ncomp - #t1 |
| 1622 |
if j > i then |
| 1623 |
ndiff = ndiff - (j - i) |
| 1624 |
end |
| 1625 |
break |
| 1626 |
end |
| 1627 |
|
| 1628 |
if t1[i] == t2[j] then |
| 1629 |
i = i + 1 |
| 1630 |
j = j + 1 |
| 1631 |
elseif t1[i] < t2[j] then |
| 1632 |
i = i + 1 |
| 1633 |
ndiff = ndiff + 1 |
| 1634 |
else |
| 1635 |
j = j + 1 |
| 1636 |
ndiff = ndiff + 1 |
| 1637 |
end |
| 1638 |
end |
| 1639 |
|
| 1640 |
return ndiff |
| 1641 |
end |
| 1642 |
|
| 1643 |
---[[[ |
| 1644 |
-- @function lua_util.table_digest(t) |
| 1645 |
-- Returns hash of all values if t[1] is string or all keys/values otherwise |
| 1646 |
-- @param {table} t input array or map |
| 1647 |
-- @return {string} base32 representation of blake2b hash of all strings |
| 1648 |
--]]] |
| 1649 |
local function table_digest(t) |
| 1650 |
local cr = require "rspamd_cryptobox_hash" |
| 1651 |
local h = cr.create() |
| 1652 |
|
| 1653 |
if t[1] then |
| 1654 |
for _, e in ipairs(t) do |
| 1655 |
if type(e) == 'table' then |
| 1656 |
h:update(table_digest(e)) |
| 1657 |
else |
| 1658 |
h:update(tostring(e)) |
| 1659 |
end |
| 1660 |
end |
| 1661 |
else |
| 1662 |
for k, v in pairs(t) do |
| 1663 |
h:update(tostring(k)) |
| 1664 |
|
| 1665 |
if type(v) == 'string' then |
| 1666 |
h:update(v) |
| 1667 |
elseif type(v) == 'table' then |
| 1668 |
h:update(table_digest(v)) |
| 1669 |
end |
| 1670 |
end |
| 1671 |
end |
| 1672 |
return h:base32() |
| 1673 |
end |
| 1674 |
|
| 1675 |
exports.table_digest = table_digest |
| 1676 |
|
| 1677 |
---[[[ |
| 1678 |
-- @function lua_util.unordered_table_digest(t) |
| 1679 |
-- Returns a hash of table contents that is independent of iteration order. |
| 1680 |
-- Uses XXH3 fast hash with XOR accumulation for O(n) performance. |
| 1681 |
-- All value types (string, number, boolean, table) are included in the hash. |
| 1682 |
-- @param {table} t input array or map |
| 1683 |
-- @return {string} hex representation of the 64-bit hash |
| 1684 |
--]]] |
| 1685 |
local function unordered_table_digest(t) |
| 1686 |
local cr = require "rspamd_cryptobox" |
| 1687 |
local bit = require "bit" |
| 1688 |
|
| 1689 |
-- Internal function that returns high/low 32-bit parts |
| 1690 |
local function digest_impl(tbl) |
| 1691 |
local acc_hi, acc_lo = 0, 0 |
| 1692 |
|
| 1693 |
if tbl[1] ~= nil then |
| 1694 |
-- Array: order matters, so include index in hash |
| 1695 |
for i, e in ipairs(tbl) do |
| 1696 |
local str |
| 1697 |
if type(e) == 'table' then |
| 1698 |
-- Recursively compute digest for nested table |
| 1699 |
str = tostring(i) .. '\0' .. digest_impl(e) |
| 1700 |
else |
| 1701 |
str = tostring(i) .. '\0' .. tostring(e) |
| 1702 |
end |
| 1703 |
local hi, lo = cr.fast_hash64(str) |
| 1704 |
acc_hi = bit.bxor(acc_hi, hi) |
| 1705 |
acc_lo = bit.bxor(acc_lo, lo) |
| 1706 |
end |
| 1707 |
else |
| 1708 |
-- Map: order doesn't matter, XOR all k-v hashes |
| 1709 |
for k, v in pairs(tbl) do |
| 1710 |
local str |
| 1711 |
if type(v) == 'table' then |
| 1712 |
-- Recursively compute digest for nested table |
| 1713 |
str = tostring(k) .. '\0' .. digest_impl(v) |
| 1714 |
else |
| 1715 |
str = tostring(k) .. '\0' .. tostring(v) |
| 1716 |
end |
| 1717 |
local hi, lo = cr.fast_hash64(str) |
| 1718 |
acc_hi = bit.bxor(acc_hi, hi) |
| 1719 |
acc_lo = bit.bxor(acc_lo, lo) |
| 1720 |
end |
| 1721 |
end |
| 1722 |
|
| 1723 |
-- Return as hex string for nested calls |
| 1724 |
-- Use bit.tohex() which properly handles signed 32-bit values |
| 1725 |
return bit.tohex(acc_hi) .. bit.tohex(acc_lo) |
| 1726 |
end |
| 1727 |
|
| 1728 |
return digest_impl(t) |
| 1729 |
end |
| 1730 |
|
| 1731 |
exports.unordered_table_digest = unordered_table_digest |
| 1732 |
|
| 1733 |
---[[[ |
| 1734 |
-- @function lua_util.toboolean(v) |
| 1735 |
-- Converts a string or a number to boolean |
| 1736 |
-- @param {string|number} v |
| 1737 |
-- @return {boolean} v converted to boolean |
| 1738 |
--]]] |
| 1739 |
exports.toboolean = function(v) |
| 1740 |
local true_t = { |
| 1741 |
['1'] = true, |
| 1742 |
['true'] = true, |
| 1743 |
['TRUE'] = true, |
| 1744 |
['True'] = true, |
| 1745 |
}; |
| 1746 |
local false_t = { |
| 1747 |
['0'] = false, |
| 1748 |
['false'] = false, |
| 1749 |
['FALSE'] = false, |
| 1750 |
['False'] = false, |
| 1751 |
}; |
| 1752 |
|
| 1753 |
if type(v) == 'boolean' then |
| 1754 |
return v |
| 1755 |
elseif type(v) == 'string' then |
| 1756 |
if true_t[v] == true then |
| 1757 |
return true; |
| 1758 |
elseif false_t[v] == false then |
| 1759 |
return false; |
| 1760 |
else |
| 1761 |
return false, string.format('cannot convert %q to boolean', v); |
| 1762 |
end |
| 1763 |
elseif type(v) == 'number' then |
| 1764 |
return v ~= 0 |
| 1765 |
else |
| 1766 |
return false, string.format('cannot convert %q to boolean', v); |
| 1767 |
end |
| 1768 |
end |
| 1769 |
|
| 1770 |
---[[[ |
| 1771 |
-- @function lua_util.config_check_local_or_authed(config, modname) |
| 1772 |
-- Reads check_local and check_authed from the config as this is used in many modules |
| 1773 |
-- @param {rspamd_config} config `rspamd_config` global |
| 1774 |
-- @param {name} module name |
| 1775 |
-- @return {boolean} v converted to boolean |
| 1776 |
--]]] |
| 1777 |
exports.config_check_local_or_authed = function(rspamd_config, modname, def_local, def_authed) |
| 1778 |
local check_local = def_local or false |
| 1779 |
local check_authed = def_authed or false |
| 1780 |
|
| 1781 |
local function try_section(where) |
| 1782 |
local ret = false |
| 1783 |
local opts = rspamd_config:get_all_opt(where) |
| 1784 |
if type(opts) == 'table' then |
| 1785 |
if type(opts['check_local']) == 'boolean' then |
| 1786 |
check_local = opts['check_local'] |
| 1787 |
ret = true |
| 1788 |
end |
| 1789 |
if type(opts['check_authed']) == 'boolean' then |
| 1790 |
check_authed = opts['check_authed'] |
| 1791 |
ret = true |
| 1792 |
end |
| 1793 |
end |
| 1794 |
|
| 1795 |
return ret |
| 1796 |
end |
| 1797 |
|
| 1798 |
if not try_section(modname) then |
| 1799 |
try_section('options') |
| 1800 |
end |
| 1801 |
|
| 1802 |
return { check_local, check_authed } |
| 1803 |
end |
| 1804 |
|
| 1805 |
---[[[ |
| 1806 |
-- @function lua_util.is_skip_local_or_authed(task, conf[, ip]) |
| 1807 |
-- Returns `true` if local or authenticated task should be skipped for this module |
| 1808 |
-- @param {rspamd_task} task |
| 1809 |
-- @param {table} conf table returned from `config_check_local_or_authed` |
| 1810 |
-- @param {rspamd_ip} ip optional ip address (can be obtained from a task) |
| 1811 |
-- @return {boolean} true if check should be skipped |
| 1812 |
--]]] |
| 1813 |
exports.is_skip_local_or_authed = function(task, conf, ip) |
| 1814 |
if not ip then |
| 1815 |
ip = task:get_from_ip() |
| 1816 |
end |
| 1817 |
if not conf then |
| 1818 |
conf = { false, false } |
| 1819 |
end |
| 1820 |
if ((not conf[2] and task:get_user()) or |
| 1821 |
(not conf[1] and type(ip) == 'userdata' and ip:is_local())) then |
| 1822 |
return true |
| 1823 |
end |
| 1824 |
|
| 1825 |
return false |
| 1826 |
end |
| 1827 |
|
| 1828 |
---[[[ |
| 1829 |
-- @function lua_util.maybe_smtp_quote_value(str) |
| 1830 |
-- Checks string for the forbidden elements (tspecials in RFC and quote string if needed) |
| 1831 |
-- @param {string} str input string |
| 1832 |
-- @return {string} original or quoted string |
| 1833 |
--]]] |
| 1834 |
local tspecial = lpeg.S "()<>,;:\\\"/[]?= \t\v" |
| 1835 |
local special_match = lpeg.P((1 - tspecial) ^ 0 * tspecial ^ 1) |
| 1836 |
exports.maybe_smtp_quote_value = function(str) |
| 1837 |
if special_match:match(str) then |
| 1838 |
return string.format('"%s"', str:gsub('"', '\\"')) |
| 1839 |
end |
| 1840 |
|
| 1841 |
return str |
| 1842 |
end |
| 1843 |
|
| 1844 |
---[[[ |
| 1845 |
-- @function lua_util.shuffle(table) |
| 1846 |
-- Performs in-place shuffling of a table |
| 1847 |
-- @param {table} tbl table to shuffle |
| 1848 |
-- @return {table} same table |
| 1849 |
--]]] |
| 1850 |
exports.shuffle = function(tbl) |
| 1851 |
local size = #tbl |
| 1852 |
for i = size, 1, -1 do |
| 1853 |
local rand = math.random(size) |
| 1854 |
tbl[i], tbl[rand] = tbl[rand], tbl[i] |
| 1855 |
end |
| 1856 |
return tbl |
| 1857 |
end |
| 1858 |
|
| 1859 |
-- |
| 1860 |
local hex_table = {} |
| 1861 |
for idx = 0, 255 do |
| 1862 |
hex_table[("%02X"):format(idx)] = string.char(idx) |
| 1863 |
hex_table[("%02x"):format(idx)] = string.char(idx) |
| 1864 |
end |
| 1865 |
|
| 1866 |
---[[[ |
| 1867 |
-- @function lua_util.unhex(str) |
| 1868 |
-- Decode hex encoded string |
| 1869 |
-- @param {string} str string to decode |
| 1870 |
-- @return {string} hex decoded string (valid hex pairs are decoded, everything else is printed as is) |
| 1871 |
--]]] |
| 1872 |
exports.unhex = function(str) |
| 1873 |
return str:gsub('(..)', hex_table) |
| 1874 |
end |
| 1875 |
|
| 1876 |
local http_upstream_lists = {} |
| 1877 |
local function http_upstreams_by_url(pool, url) |
| 1878 |
local rspamd_url = require "rspamd_url" |
| 1879 |
|
| 1880 |
local cached = http_upstream_lists[url] |
| 1881 |
if cached then |
| 1882 |
return cached |
| 1883 |
end |
| 1884 |
|
| 1885 |
local real_url = rspamd_url.create(pool, url) |
| 1886 |
|
| 1887 |
if not real_url then |
| 1888 |
return nil |
| 1889 |
end |
| 1890 |
|
| 1891 |
local host = real_url:get_host() |
| 1892 |
local proto = real_url:get_protocol() or 'http' |
| 1893 |
local port = real_url:get_port() or (proto == 'https' and 443 or 80) |
| 1894 |
local upstream_list = require "rspamd_upstream_list" |
| 1895 |
local upstreams = upstream_list.create(host, port) |
| 1896 |
|
| 1897 |
if upstreams then |
| 1898 |
http_upstream_lists[url] = upstreams |
| 1899 |
return upstreams |
| 1900 |
end |
| 1901 |
|
| 1902 |
return nil |
| 1903 |
end |
| 1904 |
---[[[ |
| 1905 |
-- @function lua_util.http_upstreams_by_url(pool, url) |
| 1906 |
-- Returns a cached or new upstreams list that corresponds to the specific url |
| 1907 |
-- @param {mempool} pool memory pool to use (typically static pool from rspamd_config) |
| 1908 |
-- @param {string} url full url |
| 1909 |
-- @return {upstreams_list} object to get upstream from an url |
| 1910 |
--]]] |
| 1911 |
exports.http_upstreams_by_url = http_upstreams_by_url |
| 1912 |
|
| 1913 |
---[[[ |
| 1914 |
-- @function lua_util.dns_timeout_augmentation(cfg) |
| 1915 |
-- Returns an augmentation suitable to define DNS timeout for a module |
| 1916 |
-- @return {string} a string in format 'timeout=x' where `x` is a number of seconds for DNS timeout |
| 1917 |
--]]] |
| 1918 |
local function dns_timeout_augmentation(cfg) |
| 1919 |
return string.format('timeout=%f', cfg:get_dns_timeout() or 0.0) |
| 1920 |
end |
| 1921 |
|
| 1922 |
exports.dns_timeout_augmentation = dns_timeout_augmentation |
| 1923 |
|
| 1924 |
---[[[ |
| 1925 |
--- @function lua_util.strip_lua_comments(lua_code) |
| 1926 |
-- Strips single-line and multi-line comments from a given Lua code string and removes |
| 1927 |
-- any extra spaces or newlines. |
| 1928 |
-- |
| 1929 |
-- @param lua_code The Lua code string to strip comments from. |
| 1930 |
-- @return The resulting Lua code string with comments and extra spaces removed. |
| 1931 |
-- |
| 1932 |
---]]] |
| 1933 |
local function strip_lua_comments(lua_code) |
| 1934 |
-- Remove single-line comments |
| 1935 |
lua_code = lua_code:gsub("%-%-[^\r\n]*", "") |
| 1936 |
|
| 1937 |
-- Remove multi-line comments |
| 1938 |
lua_code = lua_code:gsub("%-%-%[%[.-%]%]", "") |
| 1939 |
|
| 1940 |
-- Remove extra spaces and newlines |
| 1941 |
lua_code = lua_code:gsub("%s+", " ") |
| 1942 |
|
| 1943 |
return lua_code |
| 1944 |
end |
| 1945 |
|
| 1946 |
exports.strip_lua_comments = strip_lua_comments |
| 1947 |
|
| 1948 |
---[[[ |
| 1949 |
-- @function lua_util.join_path(...) |
| 1950 |
-- Joins path components into a single path string using the appropriate separator |
| 1951 |
-- for the current operating system. |
| 1952 |
-- |
| 1953 |
-- @param ... Any number of path components to join together. |
| 1954 |
-- @return A single path string, with components separated by the appropriate separator. |
| 1955 |
-- |
| 1956 |
---]]] |
| 1957 |
local path_sep = package.config:sub(1, 1) or '/' |
| 1958 |
local function join_path(...) |
| 1959 |
local components = { ... } |
| 1960 |
|
| 1961 |
-- Join components using separator |
| 1962 |
return table.concat(components, path_sep) |
| 1963 |
end |
| 1964 |
exports.join_path = join_path |
| 1965 |
|
| 1966 |
---[[[ |
| 1967 |
-- @function lua_util.url_encode_string(str) |
| 1968 |
-- URL encodes a string |
| 1969 |
-- |
| 1970 |
-- @param {string} str string to encode |
| 1971 |
-- @return {string} URL encoded string |
| 1972 |
-- |
| 1973 |
---]]] |
| 1974 |
local function url_encode_string(str) |
| 1975 |
if str == nil then |
| 1976 |
return '' |
| 1977 |
end |
| 1978 |
-- Use explicit ASCII ranges instead of %w which is locale-dependent |
| 1979 |
-- and may match non-ASCII bytes in UTF-8 locales |
| 1980 |
str = string.gsub(str, "([^A-Za-z0-9 _%%%-%.~])", |
| 1981 |
function(c) |
| 1982 |
return string.format("%%%02X", string.byte(c)) |
| 1983 |
end) |
| 1984 |
str = string.gsub(str, " ", "+") |
| 1985 |
return str |
| 1986 |
end |
| 1987 |
exports.url_encode_string = url_encode_string |
| 1988 |
|
| 1989 |
assert(url_encode_string('上海+中國') == '%E4%B8%8A%E6%B5%B7%2B%E4%B8%AD%E5%9C%8B') |
| 1990 |
assert(url_encode_string('? and the Mysterians') == '%3F+and+the+Mysterians') |
| 1991 |
|
| 1992 |
-- Short unit test for sanity |
| 1993 |
if path_sep == '/' then |
| 1994 |
assert(join_path('/path', 'to', 'file') == '/path/to/file') |
| 1995 |
else |
| 1996 |
assert(join_path('C:', 'path', 'to', 'file') == 'C:\\path\\to\\file') |
| 1997 |
end |
| 1998 |
|
| 1999 |
-- Defines symbols priorities for common usage in prefilters/postfilters |
| 2000 |
exports.symbols_priorities = { |
| 2001 |
top = 10, -- Symbols must be executed first (or last), such as settings |
| 2002 |
high = 9, -- Example: asn |
| 2003 |
medium = 5, -- Everything should use this as default |
| 2004 |
low = 0, |
| 2005 |
} |
| 2006 |
|
| 2007 |
---[[[ |
| 2008 |
-- @function lua_util.table_to_multipart_body(tbl, boundary) |
| 2009 |
-- Converts a key-value map to the table representing multipart body, with the following values: |
| 2010 |
-- `data`: data of the part |
| 2011 |
-- `filename`: optional filename |
| 2012 |
-- `content-type`: content type of the element (optional) |
| 2013 |
-- `content-transfer-encoding`: optional CTE header |
| 2014 |
local function table_to_multipart_body(tbl, boundary) |
| 2015 |
local seen_data = false |
| 2016 |
local out = {} |
| 2017 |
|
| 2018 |
for k, v in pairs(tbl) do |
| 2019 |
if v.data then |
| 2020 |
seen_data = true |
| 2021 |
table.insert(out, string.format('--%s\r\n', boundary)) |
| 2022 |
if v.filename then |
| 2023 |
table.insert(out, |
| 2024 |
string.format('Content-Disposition: form-data; name="%s"; filename="%s"\r\n', |
| 2025 |
k, v.filename)) |
| 2026 |
else |
| 2027 |
table.insert(out, |
| 2028 |
string.format('Content-Disposition: form-data; name="%s"\r\n', k)) |
| 2029 |
end |
| 2030 |
if v['content-type'] then |
| 2031 |
table.insert(out, |
| 2032 |
string.format('Content-Type: %s\r\n', v['content-type'])) |
| 2033 |
else |
| 2034 |
table.insert(out, 'Content-Type: text/plain\r\n') |
| 2035 |
end |
| 2036 |
if v['content-transfer-encoding'] then |
| 2037 |
table.insert(out, |
| 2038 |
string.format('Content-Transfer-Encoding: %s\r\n', |
| 2039 |
v['content-transfer-encoding'])) |
| 2040 |
else |
| 2041 |
table.insert(out, 'Content-Transfer-Encoding: binary\r\n') |
| 2042 |
end |
| 2043 |
table.insert(out, '\r\n') |
| 2044 |
table.insert(out, v.data) |
| 2045 |
table.insert(out, '\r\n') |
| 2046 |
end |
| 2047 |
end |
| 2048 |
|
| 2049 |
if seen_data then |
| 2050 |
table.insert(out, string.format('--%s--\r\n', boundary)) |
| 2051 |
end |
| 2052 |
|
| 2053 |
return out |
| 2054 |
end |
| 2055 |
|
| 2056 |
exports.table_to_multipart_body = table_to_multipart_body |
| 2057 |
|
| 2058 |
return exports |