| 1 |
--[[ |
| 2 |
Copyright (c) 2022, Vsevolod Stakhov <[email protected]> |
| 3 |
|
| 4 |
Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 |
you may not use this file except in compliance with the License. |
| 6 |
You may obtain a copy of the License at |
| 7 |
|
| 8 |
http://www.apache.org/licenses/LICENSE-2.0 |
| 9 |
|
| 10 |
Unless required by applicable law or agreed to in writing, software |
| 11 |
distributed under the License is distributed on an "AS IS" BASIS, |
| 12 |
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 |
See the License for the specific language governing permissions and |
| 14 |
limitations under the License. |
| 15 |
]]-- |
| 16 |
|
| 17 |
--[[[ |
| 18 |
-- @module lua_fuzzy |
| 19 |
-- This module contains helper functions for supporting fuzzy check module |
| 20 |
--]] |
| 21 |
|
| 22 |
|
| 23 |
local N = "lua_fuzzy" |
| 24 |
local lua_util = require "lua_util" |
| 25 |
local rspamd_regexp = require "rspamd_regexp" |
| 26 |
local fun = require "fun" |
| 27 |
local rspamd_logger = require "rspamd_logger" |
| 28 |
local T = require "lua_shape.core" |
| 29 |
|
| 30 |
-- Filled by C code, indexed by number in this table |
| 31 |
local rules = {} |
| 32 |
|
| 33 |
-- Pre-defined rules options |
| 34 |
local policies = { |
| 35 |
recommended = { |
| 36 |
min_bytes = 1024, |
| 37 |
min_height = 500, |
| 38 |
min_width = 500, |
| 39 |
min_length = 64, |
| 40 |
text_multiplier = 4.0, -- divide min_bytes by 4 for texts |
| 41 |
mime_types = { "application/*" }, |
| 42 |
scan_archives = true, |
| 43 |
short_text_direct_hash = true, |
| 44 |
text_shingles = true, |
| 45 |
text_hashes = true, |
| 46 |
skip_images = false, |
| 47 |
} |
| 48 |
} |
| 49 |
|
| 50 |
local default_policy = policies.recommended |
| 51 |
|
| 52 |
local schema_fields = { |
| 53 |
min_bytes = T.transform( |
| 54 |
T.one_of({T.number(), T.string()}), |
| 55 |
tonumber |
| 56 |
), |
| 57 |
min_height = T.transform( |
| 58 |
T.one_of({T.number(), T.string()}), |
| 59 |
tonumber |
| 60 |
), |
| 61 |
min_width = T.transform( |
| 62 |
T.one_of({T.number(), T.string()}), |
| 63 |
tonumber |
| 64 |
), |
| 65 |
min_length = T.transform( |
| 66 |
T.one_of({T.number(), T.string()}), |
| 67 |
tonumber |
| 68 |
), |
| 69 |
text_multiplier = T.number(), |
| 70 |
mime_types = T.array(T.string()), |
| 71 |
scan_archives = T.boolean(), |
| 72 |
short_text_direct_hash = T.boolean(), |
| 73 |
text_shingles = T.boolean(), |
| 74 |
text_hashes = T.boolean(), |
| 75 |
skip_images = T.boolean(), |
| 76 |
} |
| 77 |
local policy_schema = T.table(schema_fields) |
| 78 |
|
| 79 |
local policy_schema_open = T.table(schema_fields, { |
| 80 |
open = true, |
| 81 |
}) |
| 82 |
|
| 83 |
local exports = {} |
| 84 |
|
| 85 |
local function apply_checks_overrides(rule) |
| 86 |
local checks = rule.checks |
| 87 |
|
| 88 |
if type(checks) ~= 'table' then |
| 89 |
return |
| 90 |
end |
| 91 |
|
| 92 |
local function find_section(name) |
| 93 |
local lname = name:lower() |
| 94 |
|
| 95 |
for k, v in pairs(checks) do |
| 96 |
if type(k) == 'string' and k:lower() == lname then |
| 97 |
return v |
| 98 |
end |
| 99 |
end |
| 100 |
|
| 101 |
return nil |
| 102 |
end |
| 103 |
|
| 104 |
local function bool_opt(section, key) |
| 105 |
if type(section) ~= 'table' then |
| 106 |
return nil |
| 107 |
end |
| 108 |
|
| 109 |
if section[key] == nil then |
| 110 |
return nil |
| 111 |
end |
| 112 |
|
| 113 |
return lua_util.toboolean(section[key]) |
| 114 |
end |
| 115 |
|
| 116 |
local function number_opt(section, key) |
| 117 |
if type(section) ~= 'table' then |
| 118 |
return nil |
| 119 |
end |
| 120 |
|
| 121 |
if section[key] == nil then |
| 122 |
return nil |
| 123 |
end |
| 124 |
|
| 125 |
return tonumber(section[key]) |
| 126 |
end |
| 127 |
|
| 128 |
local text_section = find_section('text') |
| 129 |
|
| 130 |
if text_section then |
| 131 |
local enabled = bool_opt(text_section, 'enabled') |
| 132 |
|
| 133 |
if enabled == nil then |
| 134 |
enabled = true |
| 135 |
end |
| 136 |
|
| 137 |
rule.text_hashes = enabled |
| 138 |
|
| 139 |
local opt = bool_opt(text_section, 'no_subject') |
| 140 |
|
| 141 |
if opt ~= nil then |
| 142 |
rule.no_subject = opt |
| 143 |
end |
| 144 |
|
| 145 |
opt = bool_opt(text_section, 'short_text_direct_hash') |
| 146 |
|
| 147 |
if opt ~= nil then |
| 148 |
rule.short_text_direct_hash = opt |
| 149 |
end |
| 150 |
|
| 151 |
local num = number_opt(text_section, 'min_length') |
| 152 |
|
| 153 |
if num ~= nil then |
| 154 |
rule.min_length = num |
| 155 |
end |
| 156 |
|
| 157 |
num = number_opt(text_section, 'text_multiplier') |
| 158 |
|
| 159 |
if num ~= nil then |
| 160 |
rule.text_multiplier = num |
| 161 |
end |
| 162 |
end |
| 163 |
|
| 164 |
local html_section = find_section('html') |
| 165 |
|
| 166 |
if html_section then |
| 167 |
local enabled = bool_opt(html_section, 'enabled') |
| 168 |
|
| 169 |
if enabled == nil then |
| 170 |
enabled = true |
| 171 |
end |
| 172 |
|
| 173 |
rule.html_shingles = enabled |
| 174 |
|
| 175 |
local num = number_opt(html_section, 'min_html_tags') |
| 176 |
|
| 177 |
if num == nil then |
| 178 |
num = number_opt(html_section, 'min_tags') |
| 179 |
end |
| 180 |
|
| 181 |
if num ~= nil then |
| 182 |
rule.min_html_tags = num |
| 183 |
end |
| 184 |
|
| 185 |
num = number_opt(html_section, 'html_weight') |
| 186 |
|
| 187 |
if num == nil then |
| 188 |
num = number_opt(html_section, 'weight') |
| 189 |
end |
| 190 |
|
| 191 |
if num ~= nil then |
| 192 |
rule.html_weight = num |
| 193 |
end |
| 194 |
end |
| 195 |
|
| 196 |
local image_section = find_section('image') or find_section('images') |
| 197 |
|
| 198 |
if image_section then |
| 199 |
local enabled = bool_opt(image_section, 'enabled') |
| 200 |
|
| 201 |
if enabled == nil then |
| 202 |
enabled = true |
| 203 |
end |
| 204 |
|
| 205 |
rule.skip_images = not enabled |
| 206 |
|
| 207 |
local num = number_opt(image_section, 'min_height') |
| 208 |
|
| 209 |
if num ~= nil then |
| 210 |
rule.min_height = num |
| 211 |
end |
| 212 |
|
| 213 |
num = number_opt(image_section, 'min_width') |
| 214 |
|
| 215 |
if num ~= nil then |
| 216 |
rule.min_width = num |
| 217 |
end |
| 218 |
end |
| 219 |
|
| 220 |
local archive_section = find_section('archive') or find_section('archives') |
| 221 |
|
| 222 |
if archive_section then |
| 223 |
local enabled = bool_opt(archive_section, 'enabled') |
| 224 |
|
| 225 |
if enabled == nil then |
| 226 |
enabled = true |
| 227 |
end |
| 228 |
|
| 229 |
rule.scan_archives = enabled |
| 230 |
end |
| 231 |
|
| 232 |
rule.checks = nil |
| 233 |
end |
| 234 |
|
| 235 |
--[[[ |
| 236 |
-- @function lua_fuzzy.register_policy(name, policy) |
| 237 |
-- Adds a new policy with name `name`. Must be valid, checked using policy_schema |
| 238 |
--]] |
| 239 |
exports.register_policy = function(name, policy) |
| 240 |
if policies[name] then |
| 241 |
rspamd_logger.warnx(rspamd_config, "overriding policy %s", name) |
| 242 |
end |
| 243 |
|
| 244 |
local parsed_policy, err = policy_schema:transform(policy) |
| 245 |
|
| 246 |
if not parsed_policy then |
| 247 |
rspamd_logger.errx(rspamd_config, 'invalid fuzzy rule policy %s: %s', |
| 248 |
name, err) |
| 249 |
|
| 250 |
return |
| 251 |
else |
| 252 |
policies.name = parsed_policy |
| 253 |
end |
| 254 |
end |
| 255 |
|
| 256 |
--[[[ |
| 257 |
-- @function lua_fuzzy.process_rule(rule) |
| 258 |
-- Processes fuzzy rule (applying policies or defaults if needed). Returns policy id |
| 259 |
--]] |
| 260 |
exports.process_rule = function(rule) |
| 261 |
local processed_rule = lua_util.shallowcopy(rule) |
| 262 |
local policy = default_policy |
| 263 |
|
| 264 |
if processed_rule.policy then |
| 265 |
policy = policies[processed_rule.policy] |
| 266 |
end |
| 267 |
|
| 268 |
if policy then |
| 269 |
processed_rule = lua_util.override_defaults(policy, processed_rule) |
| 270 |
|
| 271 |
apply_checks_overrides(processed_rule) |
| 272 |
|
| 273 |
local parsed_policy, err = policy_schema_open:transform(processed_rule) |
| 274 |
|
| 275 |
if not parsed_policy then |
| 276 |
rspamd_logger.errx(rspamd_config, 'invalid fuzzy rule default fields: %s', err) |
| 277 |
else |
| 278 |
processed_rule = parsed_policy |
| 279 |
end |
| 280 |
else |
| 281 |
rspamd_logger.warnx(rspamd_config, "unknown policy %s", processed_rule.policy) |
| 282 |
apply_checks_overrides(processed_rule) |
| 283 |
end |
| 284 |
|
| 285 |
if processed_rule.mime_types then |
| 286 |
processed_rule.mime_types = fun.totable(fun.map(function(gl) |
| 287 |
return rspamd_regexp.import_glob(gl, 'i') |
| 288 |
end, processed_rule.mime_types)) |
| 289 |
end |
| 290 |
|
| 291 |
table.insert(rules, processed_rule) |
| 292 |
return #rules |
| 293 |
end |
| 294 |
|
| 295 |
-- CJK languages use multi-byte characters (3 bytes per char in UTF-8) and |
| 296 |
-- carry more semantic content per token than Latin languages. |
| 297 |
local function is_cjk_language(lang) |
| 298 |
return lang and (lang == 'ja' or lang == 'zh' or lang == 'ko') |
| 299 |
end |
| 300 |
|
| 301 |
local function check_length(task, part, rule) |
| 302 |
local bytes = part:get_length() |
| 303 |
local length_ok = bytes > 0 |
| 304 |
|
| 305 |
local id = part:get_id() |
| 306 |
lua_util.debugm(N, task, 'check size of part %s', id) |
| 307 |
|
| 308 |
if length_ok and rule.min_bytes > 0 then |
| 309 |
|
| 310 |
local adjusted_bytes = bytes |
| 311 |
|
| 312 |
if part:is_text() then |
| 313 |
-- Fuzzy plugin uses stripped utf content to get an exact hash, that |
| 314 |
-- corresponds to `get_content_oneline()` |
| 315 |
-- However, in the case of empty parts this method returns `nil`, so extra |
| 316 |
-- sanity check is required. |
| 317 |
bytes = #(part:get_text():get_content_oneline() or '') |
| 318 |
|
| 319 |
-- Short hashing algorithm also use subject unless explicitly denied |
| 320 |
if not rule.no_subject then |
| 321 |
local subject = task:get_subject() or '' |
| 322 |
bytes = bytes + #subject |
| 323 |
end |
| 324 |
|
| 325 |
if rule.text_multiplier then |
| 326 |
local multiplier = rule.text_multiplier |
| 327 |
|
| 328 |
-- CJK characters are 3 bytes in UTF-8, so the same semantic content |
| 329 |
-- takes ~3x more bytes than Latin text; boost the multiplier to compensate |
| 330 |
local lang = part:get_text():get_language() |
| 331 |
if is_cjk_language(lang) then |
| 332 |
multiplier = multiplier * 3.0 |
| 333 |
lua_util.debugm(N, task, 'CJK language %s: boosted text_multiplier to %s', |
| 334 |
lang, multiplier) |
| 335 |
end |
| 336 |
|
| 337 |
adjusted_bytes = bytes * multiplier |
| 338 |
end |
| 339 |
end |
| 340 |
|
| 341 |
if rule.min_bytes > adjusted_bytes then |
| 342 |
lua_util.debugm(N, task, 'skip part of length %s (%s adjusted) ' .. |
| 343 |
'as it has less than %s bytes', |
| 344 |
bytes, adjusted_bytes, rule.min_bytes) |
| 345 |
length_ok = false |
| 346 |
else |
| 347 |
lua_util.debugm(N, task, 'allow part of length %s (%s adjusted)', |
| 348 |
bytes, adjusted_bytes, rule.min_bytes) |
| 349 |
end |
| 350 |
else |
| 351 |
lua_util.debugm(N, task, 'allow part %s, no length limits', id) |
| 352 |
end |
| 353 |
|
| 354 |
return length_ok |
| 355 |
end |
| 356 |
|
| 357 |
local function check_text_part(task, part, rule, text) |
| 358 |
local allow_direct, allow_shingles = false, false |
| 359 |
|
| 360 |
local id = part:get_id() |
| 361 |
lua_util.debugm(N, task, 'check text part %s', id) |
| 362 |
|
| 363 |
if rule.text_hashes == false then |
| 364 |
lua_util.debugm(N, task, 'text hashes disabled, relying on HTML for part %s', id) |
| 365 |
return rule.html_shingles == true, false |
| 366 |
end |
| 367 |
|
| 368 |
local wcnt = text:get_words_count() |
| 369 |
|
| 370 |
if rule.text_shingles then |
| 371 |
-- Check number of words |
| 372 |
local min_words = rule.min_length or 0 |
| 373 |
local min_floor = 32 |
| 374 |
|
| 375 |
-- CJK morphemes carry higher semantic density per token, so fewer words |
| 376 |
-- are needed for meaningful shingle generation (3-word window still works |
| 377 |
-- well with as few as 12 tokens producing 10 windows) |
| 378 |
local lang = text:get_language() |
| 379 |
if is_cjk_language(lang) then |
| 380 |
min_words = math.floor(min_words / 3) |
| 381 |
min_floor = 12 |
| 382 |
lua_util.debugm(N, task, 'CJK language %s: adjusted min_words to %s (floor %s)', |
| 383 |
lang, min_words, min_floor) |
| 384 |
end |
| 385 |
|
| 386 |
if min_words < min_floor then |
| 387 |
min_words = min_floor |
| 388 |
end |
| 389 |
if wcnt < min_words then |
| 390 |
lua_util.debugm(N, task, 'text has less than %s words: %s; disable shingles', |
| 391 |
min_words, wcnt) |
| 392 |
allow_shingles = false |
| 393 |
else |
| 394 |
lua_util.debugm(N, task, 'allow shingles in text %s, %s words', |
| 395 |
id, wcnt) |
| 396 |
allow_shingles = true |
| 397 |
end |
| 398 |
|
| 399 |
if not rule.short_text_direct_hash and not allow_shingles then |
| 400 |
allow_direct = false |
| 401 |
else |
| 402 |
if not allow_shingles then |
| 403 |
lua_util.debugm(N, task, |
| 404 |
'allow direct hash for short text %s, %s words', |
| 405 |
id, wcnt) |
| 406 |
allow_direct = check_length(task, part, rule) |
| 407 |
else |
| 408 |
allow_direct = wcnt > 0 |
| 409 |
end |
| 410 |
end |
| 411 |
else |
| 412 |
lua_util.debugm(N, task, |
| 413 |
'disable shingles in text %s', id) |
| 414 |
allow_direct = check_length(task, part, rule) |
| 415 |
end |
| 416 |
|
| 417 |
return allow_direct, allow_shingles |
| 418 |
end |
| 419 |
|
| 420 |
--local function has_sane_text_parts(task) |
| 421 |
-- local text_parts = task:get_text_parts() or {} |
| 422 |
-- return fun.any(function(tp) return tp:get_words_count() > 32 end, text_parts) |
| 423 |
--end |
| 424 |
|
| 425 |
local function check_image_part(task, part, rule, image) |
| 426 |
if rule.skip_images then |
| 427 |
lua_util.debugm(N, task, 'skip image part as images are disabled') |
| 428 |
return false, false |
| 429 |
end |
| 430 |
|
| 431 |
local id = part:get_id() |
| 432 |
lua_util.debugm(N, task, 'check image part %s', id) |
| 433 |
|
| 434 |
if rule.min_width > 0 or rule.min_height > 0 then |
| 435 |
-- Check dimensions |
| 436 |
local min_width = rule.min_width or rule.min_height |
| 437 |
local min_height = rule.min_height or rule.min_width |
| 438 |
local height = image:get_height() |
| 439 |
local width = image:get_width() |
| 440 |
|
| 441 |
if height and width then |
| 442 |
if height < min_height or width < min_width then |
| 443 |
lua_util.debugm(N, task, 'skip image part %s as it does not meet minimum sizes: %sx%s < %sx%s', |
| 444 |
id, width, height, min_width, min_height) |
| 445 |
return false, false |
| 446 |
else |
| 447 |
lua_util.debugm(N, task, 'allow image part %s: %sx%s', |
| 448 |
id, width, height) |
| 449 |
end |
| 450 |
end |
| 451 |
end |
| 452 |
|
| 453 |
return check_length(task, part, rule), false |
| 454 |
end |
| 455 |
|
| 456 |
local function mime_types_check(task, part, rule) |
| 457 |
local t, st = part:get_type() |
| 458 |
|
| 459 |
if not t then |
| 460 |
return false, false |
| 461 |
end |
| 462 |
|
| 463 |
local ct = string.format('%s/%s', t, st) |
| 464 |
|
| 465 |
local detected_ct |
| 466 |
t, st = part:get_detected_type() |
| 467 |
if t then |
| 468 |
detected_ct = string.format('%s/%s', t, st) |
| 469 |
else |
| 470 |
detected_ct = ct |
| 471 |
end |
| 472 |
|
| 473 |
local id = part:get_id() |
| 474 |
lua_util.debugm(N, task, 'check binary part %s: %s', id, ct) |
| 475 |
|
| 476 |
-- For bad mime parts we implicitly enable fuzzy check |
| 477 |
local mime_trace = (task:get_symbol('MIME_TRACE') or {})[1] |
| 478 |
local opts = {} |
| 479 |
|
| 480 |
if mime_trace then |
| 481 |
opts = mime_trace.options or opts |
| 482 |
end |
| 483 |
opts = fun.tomap(fun.map(function(opt) |
| 484 |
local elts = lua_util.str_split(opt, ':') |
| 485 |
return elts[1], elts[2] |
| 486 |
end, opts)) |
| 487 |
|
| 488 |
if opts[id] and opts[id] == '-' then |
| 489 |
lua_util.debugm(N, task, 'explicitly check binary part %s: bad mime type %s', id, ct) |
| 490 |
return check_length(task, part, rule), false |
| 491 |
end |
| 492 |
|
| 493 |
if rule.mime_types then |
| 494 |
|
| 495 |
if fun.any(function(gl_re) |
| 496 |
if gl_re:match(ct) or (detected_ct and gl_re:match(detected_ct)) then |
| 497 |
return true |
| 498 |
else |
| 499 |
return false |
| 500 |
end |
| 501 |
end, rule.mime_types) then |
| 502 |
lua_util.debugm(N, task, 'found mime type match for part %s: %s (%s detected)', |
| 503 |
id, ct, detected_ct) |
| 504 |
return check_length(task, part, rule), false |
| 505 |
end |
| 506 |
|
| 507 |
return false, false |
| 508 |
end |
| 509 |
|
| 510 |
return false, false |
| 511 |
end |
| 512 |
|
| 513 |
exports.check_mime_part = function(task, part, rule_id) |
| 514 |
local rule = rules[rule_id] |
| 515 |
|
| 516 |
if not rule then |
| 517 |
rspamd_logger.errx(task, 'cannot find rule with id %s', rule_id) |
| 518 |
|
| 519 |
return false, false |
| 520 |
end |
| 521 |
|
| 522 |
if part:is_text() then |
| 523 |
return check_text_part(task, part, rule, part:get_text()) |
| 524 |
end |
| 525 |
|
| 526 |
if part:is_image() then |
| 527 |
return check_image_part(task, part, rule, part:get_image()) |
| 528 |
end |
| 529 |
|
| 530 |
if part:is_archive() and rule.scan_archives then |
| 531 |
-- Always send archives |
| 532 |
lua_util.debugm(N, task, 'check archive part %s', part:get_id()) |
| 533 |
|
| 534 |
return true, false |
| 535 |
end |
| 536 |
|
| 537 |
if part:is_specific() then |
| 538 |
local sp = part:get_specific() |
| 539 |
|
| 540 |
if type(sp) == 'table' and sp.fuzzy_hashes then |
| 541 |
lua_util.debugm(N, task, 'check specific part %s', part:get_id()) |
| 542 |
return true, false |
| 543 |
end |
| 544 |
end |
| 545 |
|
| 546 |
if part:is_attachment() then |
| 547 |
return mime_types_check(task, part, rule) |
| 548 |
end |
| 549 |
|
| 550 |
return false, false |
| 551 |
end |
| 552 |
|
| 553 |
exports.cleanup_rules = function() |
| 554 |
rules = {} |
| 555 |
end |
| 556 |
|
| 557 |
return exports |