| 1 |
--[[ |
| 2 |
Copyright (c) 2022, Vsevolod Stakhov <[email protected]> |
| 3 |
|
| 4 |
Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 |
you may not use this file except in compliance with the License. |
| 6 |
You may obtain a copy of the License at |
| 7 |
|
| 8 |
http://www.apache.org/licenses/LICENSE-2.0 |
| 9 |
|
| 10 |
Unless required by applicable law or agreed to in writing, software |
| 11 |
distributed under the License is distributed on an "AS IS" BASIS, |
| 12 |
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 |
See the License for the specific language governing permissions and |
| 14 |
limitations under the License. |
| 15 |
]] -- |
| 16 |
|
| 17 |
--[[[ |
| 18 |
-- @module lua_magic/heuristics |
| 19 |
-- This module contains heuristics for some specific cases |
| 20 |
--]] |
| 21 |
|
| 22 |
local rspamd_trie = require "rspamd_trie" |
| 23 |
local rspamd_util = require "rspamd_util" |
| 24 |
local lua_util = require "lua_util" |
| 25 |
local bit = require "bit" |
| 26 |
local fun = require "fun" |
| 27 |
|
| 28 |
local N = "lua_magic" |
| 29 |
local msoffice_trie |
| 30 |
local msoffice_patterns = { |
| 31 |
doc = { [[WordDocument]] }, |
| 32 |
xls = { [[Workbook]], [[Book]] }, |
| 33 |
ppt = { [[PowerPoint Document]], [[Current User]] }, |
| 34 |
vsd = { [[VisioDocument]] }, |
| 35 |
} |
| 36 |
local msoffice_trie_clsid |
| 37 |
local msoffice_clsids = { |
| 38 |
doc = { [[0609020000000000c000000000000046]] }, |
| 39 |
xls = { [[1008020000000000c000000000000046]], [[2008020000000000c000000000000046]] }, |
| 40 |
ppt = { [[108d81649b4fcf1186ea00aa00b929e8]] }, |
| 41 |
msg = { [[46f0060000000000c000000000000046]], [[0b0d020000000000c000000000000046]] }, |
| 42 |
msi = { [[84100c0000000000c000000000000046]] }, |
| 43 |
} |
| 44 |
local zip_trie |
| 45 |
local zip_patterns = { |
| 46 |
-- https://lists.oasis-open.org/archives/office/200505/msg00006.html |
| 47 |
odt = { |
| 48 |
[[mimetypeapplication/vnd\.oasis\.opendocument\.text]], |
| 49 |
[[mimetypeapplication/vnd\.oasis\.opendocument\.image]], |
| 50 |
[[mimetypeapplication/vnd\.oasis\.opendocument\.graphic]] |
| 51 |
}, |
| 52 |
ods = { |
| 53 |
[[mimetypeapplication/vnd\.oasis\.opendocument\.spreadsheet]], |
| 54 |
[[mimetypeapplication/vnd\.oasis\.opendocument\.formula]], |
| 55 |
[[mimetypeapplication/vnd\.oasis\.opendocument\.chart]] |
| 56 |
}, |
| 57 |
odp = { [[mimetypeapplication/vnd\.oasis\.opendocument\.presentation]] }, |
| 58 |
epub = { [[epub\+zip]] }, |
| 59 |
asice = { [[mimetypeapplication/vnd\.etsi\.asic-e\+zipPK]] }, |
| 60 |
asics = { [[mimetypeapplication/vnd\.etsi\.asic-s\+zipPK]] }, |
| 61 |
} |
| 62 |
|
| 63 |
local txt_trie |
| 64 |
local txt_patterns = { |
| 65 |
html = { |
| 66 |
{ [=[(?i)<html[\s>]]=], 32 }, |
| 67 |
{ [[(?i)<script\b]], 20 }, -- Commonly used by spammers |
| 68 |
{ [[<script\s+type="text\/javascript">]], 31 }, -- Another spammy pattern |
| 69 |
{ [[(?i)<\!DOCTYPE HTML\b]], 33 }, |
| 70 |
{ [[(?i)<body\b]], 20 }, |
| 71 |
{ [[(?i)<table\b]], 20 }, |
| 72 |
{ [[(?i)<a\s]], 10 }, |
| 73 |
{ [[(?i)<p\b]], 10 }, |
| 74 |
{ [[(?i)<div\b]], 10 }, |
| 75 |
{ [[(?i)<span\b]], 10 }, |
| 76 |
}, |
| 77 |
csv = { |
| 78 |
{ [[(?:[-a-zA-Z0-9_]+\s*,){2,}(?:[-a-zA-Z0-9_]+,?[ ]*[\r\n])]], 20 } |
| 79 |
}, |
| 80 |
ics = { |
| 81 |
{ [[^BEGIN:VCALENDAR\r?\n]], 40 }, |
| 82 |
}, |
| 83 |
vcf = { |
| 84 |
{ [[^BEGIN:VCARD\r?\n]], 40 }, |
| 85 |
}, |
| 86 |
xml = { |
| 87 |
{ [[<\?xml\b.+\?>]], 40 }, |
| 88 |
} |
| 89 |
} |
| 90 |
|
| 91 |
-- Used to match pattern index and extension |
| 92 |
local msoffice_clsid_indexes = {} |
| 93 |
local msoffice_patterns_indexes = {} |
| 94 |
local zip_patterns_indexes = {} |
| 95 |
local txt_patterns_indexes = {} |
| 96 |
|
| 97 |
local exports = {} |
| 98 |
|
| 99 |
local function compile_tries() |
| 100 |
local default_compile_flags = bit.bor(rspamd_trie.flags.re, |
| 101 |
rspamd_trie.flags.dot_all, |
| 102 |
rspamd_trie.flags.single_match, |
| 103 |
rspamd_trie.flags.no_start) |
| 104 |
local function compile_pats(patterns, indexes, transform_func, compile_flags) |
| 105 |
local strs = {} |
| 106 |
for ext, pats in pairs(patterns) do |
| 107 |
for _, pat in ipairs(pats) do |
| 108 |
-- These are utf16 strings in fact... |
| 109 |
strs[#strs + 1] = transform_func(pat) |
| 110 |
indexes[#indexes + 1] = { ext, pat } |
| 111 |
end |
| 112 |
end |
| 113 |
|
| 114 |
return rspamd_trie.create(strs, compile_flags or default_compile_flags) |
| 115 |
end |
| 116 |
|
| 117 |
if not msoffice_trie then |
| 118 |
-- Directory names |
| 119 |
local function msoffice_pattern_transform(pat) |
| 120 |
return '^' .. |
| 121 |
table.concat( |
| 122 |
fun.totable( |
| 123 |
fun.map(function(c) |
| 124 |
return c .. [[\x{00}]] |
| 125 |
end, |
| 126 |
fun.iter(pat)))) |
| 127 |
end |
| 128 |
local function msoffice_clsid_transform(pat) |
| 129 |
local hex_table = {} |
| 130 |
for i = 1, #pat, 2 do |
| 131 |
local subc = pat:sub(i, i + 1) |
| 132 |
hex_table[#hex_table + 1] = string.format('\\x{%s}', subc) |
| 133 |
end |
| 134 |
|
| 135 |
return '^' .. table.concat(hex_table) .. '$' |
| 136 |
end |
| 137 |
-- Directory entries |
| 138 |
msoffice_trie = compile_pats(msoffice_patterns, msoffice_patterns_indexes, |
| 139 |
msoffice_pattern_transform) |
| 140 |
-- Clsids |
| 141 |
msoffice_trie_clsid = compile_pats(msoffice_clsids, msoffice_clsid_indexes, |
| 142 |
msoffice_clsid_transform) |
| 143 |
-- Misc zip patterns at the initial fragment |
| 144 |
zip_trie = compile_pats(zip_patterns, zip_patterns_indexes, |
| 145 |
function(pat) |
| 146 |
return pat |
| 147 |
end) |
| 148 |
-- Text patterns at the initial fragment |
| 149 |
txt_trie = compile_pats(txt_patterns, txt_patterns_indexes, |
| 150 |
function(pat_tbl) |
| 151 |
return pat_tbl[1] |
| 152 |
end, |
| 153 |
bit.bor(rspamd_trie.flags.re, |
| 154 |
rspamd_trie.flags.dot_all, |
| 155 |
rspamd_trie.flags.no_start)) |
| 156 |
end |
| 157 |
end |
| 158 |
|
| 159 |
-- Call immediately on require |
| 160 |
compile_tries() |
| 161 |
|
| 162 |
local function detect_ole_format(input, log_obj, _, part) |
| 163 |
local inplen = #input |
| 164 |
if inplen < 0x31 + 4 then |
| 165 |
lua_util.debugm(N, log_obj, "short length: %s", inplen) |
| 166 |
return nil |
| 167 |
end |
| 168 |
|
| 169 |
local bom, sec_size = rspamd_util.unpack('<I2<I2', input:span(29, 4)) |
| 170 |
if bom == 0xFFFE then |
| 171 |
bom = '<' |
| 172 |
else |
| 173 |
lua_util.debugm(N, log_obj, "bom file!: %s", bom) |
| 174 |
bom = '>'; |
| 175 |
sec_size = bit.bswap(sec_size) |
| 176 |
end |
| 177 |
|
| 178 |
if sec_size < 7 or sec_size > 31 then |
| 179 |
lua_util.debugm(N, log_obj, "bad sec_size: %s", sec_size) |
| 180 |
return nil |
| 181 |
end |
| 182 |
|
| 183 |
sec_size = 2 ^ sec_size |
| 184 |
|
| 185 |
-- SecID of first sector of the directory stream |
| 186 |
local directory_offset = (rspamd_util.unpack(bom .. 'I4', input:span(0x31, 4))) |
| 187 |
* sec_size + 512 + 1 |
| 188 |
lua_util.debugm(N, log_obj, "directory: %s", directory_offset) |
| 189 |
|
| 190 |
if inplen < directory_offset then |
| 191 |
lua_util.debugm(N, log_obj, "short length: %s", inplen) |
| 192 |
return nil |
| 193 |
end |
| 194 |
|
| 195 |
local function process_dir_entry(offset) |
| 196 |
local dtype = input:byte(offset + 66) |
| 197 |
lua_util.debugm(N, log_obj, "dtype: %s, offset: %s", dtype, offset) |
| 198 |
|
| 199 |
if dtype then |
| 200 |
if dtype == 5 then |
| 201 |
-- Extract clsid |
| 202 |
local matches = msoffice_trie_clsid:match(input:span(offset + 80, 16)) |
| 203 |
if matches then |
| 204 |
for n, _ in pairs(matches) do |
| 205 |
if msoffice_clsid_indexes[n] then |
| 206 |
lua_util.debugm(N, log_obj, "found valid clsid for %s", |
| 207 |
msoffice_clsid_indexes[n][1]) |
| 208 |
return true, msoffice_clsid_indexes[n][1] |
| 209 |
end |
| 210 |
end |
| 211 |
end |
| 212 |
return true, nil |
| 213 |
elseif dtype == 2 then |
| 214 |
local matches = msoffice_trie:match(input:span(offset, 64)) |
| 215 |
if matches then |
| 216 |
for n, _ in pairs(matches) do |
| 217 |
if msoffice_patterns_indexes[n] then |
| 218 |
return true, msoffice_patterns_indexes[n][1] |
| 219 |
end |
| 220 |
end |
| 221 |
end |
| 222 |
return true, nil |
| 223 |
elseif dtype >= 0 and dtype < 5 then |
| 224 |
-- Bad type |
| 225 |
return true, nil |
| 226 |
end |
| 227 |
end |
| 228 |
|
| 229 |
return false, nil |
| 230 |
end |
| 231 |
|
| 232 |
repeat |
| 233 |
local res, ext = process_dir_entry(directory_offset) |
| 234 |
|
| 235 |
if res and ext then |
| 236 |
return ext, 60 |
| 237 |
end |
| 238 |
|
| 239 |
if not res then |
| 240 |
break |
| 241 |
end |
| 242 |
|
| 243 |
directory_offset = directory_offset + 128 |
| 244 |
until directory_offset >= inplen |
| 245 |
end |
| 246 |
|
| 247 |
exports.ole_format_heuristic = detect_ole_format |
| 248 |
|
| 249 |
local function process_top_detected(res) |
| 250 |
local extensions = lua_util.keys(res) |
| 251 |
|
| 252 |
if #extensions > 0 then |
| 253 |
table.sort(extensions, function(ex1, ex2) |
| 254 |
return res[ex1] > res[ex2] |
| 255 |
end) |
| 256 |
|
| 257 |
return extensions[1], res[extensions[1]] |
| 258 |
end |
| 259 |
|
| 260 |
return nil |
| 261 |
end |
| 262 |
|
| 263 |
local function detect_archive_flaw(part, arch, log_obj, _) |
| 264 |
local arch_type = arch:get_type() |
| 265 |
local res = { |
| 266 |
docx = 0, |
| 267 |
xlsx = 0, |
| 268 |
pptx = 0, |
| 269 |
jar = 0, |
| 270 |
odt = 0, |
| 271 |
odp = 0, |
| 272 |
ods = 0, |
| 273 |
apk = 0, |
| 274 |
} -- ext + confidence pairs |
| 275 |
|
| 276 |
local function has_control_or_zw(fname) |
| 277 |
-- control ASCII |
| 278 |
if fname:find("[%z\1-\31]") then return true end |
| 279 |
-- common zero-width UTF-8: U+200B..U+200D, U+FEFF |
| 280 |
if fname:find("\226\128[\139-\141]") then return true end -- U+200B..U+200D |
| 281 |
if fname:find("\239\187\191") then return true end -- U+FEFF |
| 282 |
return false |
| 283 |
end |
| 284 |
|
| 285 |
-- General msoffice patterns |
| 286 |
local function add_msoffice_confidence(incr) |
| 287 |
res.docx = res.docx + incr |
| 288 |
res.xlsx = res.xlsx + incr |
| 289 |
res.pptx = res.pptx + incr |
| 290 |
end |
| 291 |
|
| 292 |
if arch_type == 'zip' then |
| 293 |
-- Find specific files/folders in zip file |
| 294 |
local files = arch:get_files(100) or {} |
| 295 |
for _, file in ipairs(files) do |
| 296 |
if has_control_or_zw(file) then |
| 297 |
lua_util.debugm(N, log_obj, "archive filename has control/zw chars: %s", file) |
| 298 |
end |
| 299 |
if file == '[Content_Types].xml' then |
| 300 |
add_msoffice_confidence(10) |
| 301 |
elseif file:sub(1, 3) == 'xl/' then |
| 302 |
res.xlsx = res.xlsx + 30 |
| 303 |
elseif file:sub(1, 5) == 'word/' then |
| 304 |
res.docx = res.docx + 30 |
| 305 |
elseif file:sub(1, 4) == 'ppt/' then |
| 306 |
res.pptx = res.pptx + 30 |
| 307 |
elseif file == 'META-INF/MANIFEST.MF' then |
| 308 |
res.jar = res.jar + 40 |
| 309 |
elseif file == 'AndroidManifest.xml' then |
| 310 |
res.apk = res.apk + 60 |
| 311 |
end |
| 312 |
end |
| 313 |
|
| 314 |
local ext, weight = process_top_detected(res) |
| 315 |
|
| 316 |
if weight >= 40 then |
| 317 |
return ext, weight |
| 318 |
end |
| 319 |
|
| 320 |
-- Apply misc Zip detection logic |
| 321 |
local content = part:get_content() |
| 322 |
|
| 323 |
if #content > 128 then |
| 324 |
local start_span = content:span(1, 128) |
| 325 |
|
| 326 |
local matches = zip_trie:match(start_span) |
| 327 |
if matches then |
| 328 |
for n, _ in pairs(matches) do |
| 329 |
if zip_patterns_indexes[n] then |
| 330 |
lua_util.debugm(N, log_obj, "found zip pattern for %s", |
| 331 |
zip_patterns_indexes[n][1]) |
| 332 |
return zip_patterns_indexes[n][1], 40 |
| 333 |
end |
| 334 |
end |
| 335 |
end |
| 336 |
end |
| 337 |
end |
| 338 |
|
| 339 |
return arch_type:lower(), 40 |
| 340 |
end |
| 341 |
|
| 342 |
local csv_grammar |
| 343 |
-- Returns a grammar that will count commas |
| 344 |
local function get_csv_grammar() |
| 345 |
if not csv_grammar then |
| 346 |
local lpeg = require 'lpeg' |
| 347 |
|
| 348 |
local field = '"' * lpeg.Cs(((lpeg.P(1) - '"') + lpeg.P '""' / '"') ^ 0) * '"' + |
| 349 |
lpeg.C((1 - lpeg.S ',\n"') ^ 0) |
| 350 |
|
| 351 |
csv_grammar = lpeg.Cf(lpeg.Cc(0) * field * lpeg.P((lpeg.P(',') + |
| 352 |
lpeg.P('\t')) * field) ^ 1 * (lpeg.S '\r\n' + -1), |
| 353 |
function(acc) |
| 354 |
return acc + 1 |
| 355 |
end) |
| 356 |
end |
| 357 |
|
| 358 |
return csv_grammar |
| 359 |
end |
| 360 |
local function validate_csv(part, content, log_obj) |
| 361 |
local max_chunk = 32768 |
| 362 |
local chunk = content:sub(1, max_chunk) |
| 363 |
|
| 364 |
local expected_commas |
| 365 |
local matched_lines = 0 |
| 366 |
local max_matched_lines = 10 |
| 367 |
|
| 368 |
lua_util.debugm(N, log_obj, "check for csv pattern") |
| 369 |
|
| 370 |
for s in chunk:lines() do |
| 371 |
local ncommas = get_csv_grammar():match(s) |
| 372 |
|
| 373 |
if not ncommas then |
| 374 |
lua_util.debugm(N, log_obj, "not a csv line at line number %s", |
| 375 |
matched_lines) |
| 376 |
return false |
| 377 |
end |
| 378 |
|
| 379 |
if expected_commas and ncommas ~= expected_commas then |
| 380 |
-- Mismatched commas |
| 381 |
lua_util.debugm(N, log_obj, "missmatched commas on line %s: %s != %s", |
| 382 |
matched_lines, ncommas, expected_commas) |
| 383 |
return false |
| 384 |
elseif not expected_commas then |
| 385 |
if ncommas == 0 then |
| 386 |
lua_util.debugm(N, log_obj, "no commas in the first line") |
| 387 |
return false |
| 388 |
end |
| 389 |
expected_commas = ncommas |
| 390 |
end |
| 391 |
|
| 392 |
matched_lines = matched_lines + 1 |
| 393 |
|
| 394 |
if matched_lines > max_matched_lines then |
| 395 |
break |
| 396 |
end |
| 397 |
end |
| 398 |
|
| 399 |
lua_util.debugm(N, log_obj, "csv content is sane: %s fields; %s lines checked", |
| 400 |
expected_commas, matched_lines) |
| 401 |
|
| 402 |
return true |
| 403 |
end |
| 404 |
|
| 405 |
exports.mime_part_heuristic = function(part, log_obj, _) |
| 406 |
if part:is_archive() then |
| 407 |
local arch = part:get_archive() |
| 408 |
return detect_archive_flaw(part, arch, log_obj) |
| 409 |
end |
| 410 |
|
| 411 |
return nil |
| 412 |
end |
| 413 |
|
| 414 |
exports.text_part_heuristic = function(part, log_obj, _) |
| 415 |
-- We get some span of data and check it |
| 416 |
local function is_span_text(span) |
| 417 |
-- We examine 8 bit content, and we assume it might be localized text |
| 418 |
-- if it has more than 3 subsequent 8 bit characters |
| 419 |
local function rough_8bit_check(bytes, idx, remain, len) |
| 420 |
local b = bytes[idx] |
| 421 |
local n8bit = 0 |
| 422 |
|
| 423 |
while b >= 127 and idx < len do |
| 424 |
-- utf8 part |
| 425 |
if bit.band(b, 0xe0) == 0xc0 and remain > 1 and |
| 426 |
bit.band(bytes[idx + 1], 0xc0) == 0x80 then |
| 427 |
return true, 1 |
| 428 |
elseif bit.band(b, 0xf0) == 0xe0 and remain > 2 and |
| 429 |
bit.band(bytes[idx + 1], 0xc0) == 0x80 and |
| 430 |
bit.band(bytes[idx + 2], 0xc0) == 0x80 then |
| 431 |
return true, 2 |
| 432 |
elseif bit.band(b, 0xf8) == 0xf0 and remain > 3 and |
| 433 |
bit.band(bytes[idx + 1], 0xc0) == 0x80 and |
| 434 |
bit.band(bytes[idx + 2], 0xc0) == 0x80 and |
| 435 |
bit.band(bytes[idx + 3], 0xc0) == 0x80 then |
| 436 |
return true, 3 |
| 437 |
end |
| 438 |
|
| 439 |
n8bit = n8bit + 1 |
| 440 |
idx = idx + 1 |
| 441 |
b = bytes[idx] |
| 442 |
remain = remain - 1 |
| 443 |
end |
| 444 |
|
| 445 |
if n8bit >= 3 then |
| 446 |
return true, n8bit |
| 447 |
end |
| 448 |
|
| 449 |
return false, 0 |
| 450 |
end |
| 451 |
|
| 452 |
-- Convert to string as LuaJIT can optimise string.sub (and fun.iter) but not C calls |
| 453 |
local tlen = #span |
| 454 |
local non_printable = 0 |
| 455 |
local bytes = span:bytes() |
| 456 |
local i = 1 |
| 457 |
repeat |
| 458 |
local b = bytes[i] |
| 459 |
|
| 460 |
if (b < 0x20) and not (b == 0x0d or b == 0x0a or b == 0x09) then |
| 461 |
non_printable = non_printable + 1 |
| 462 |
elseif b >= 127 then |
| 463 |
local c, nskip = rough_8bit_check(bytes, i, tlen - i, tlen) |
| 464 |
|
| 465 |
if not c then |
| 466 |
non_printable = non_printable + 1 |
| 467 |
else |
| 468 |
i = i + nskip |
| 469 |
end |
| 470 |
end |
| 471 |
i = i + 1 |
| 472 |
until i > tlen |
| 473 |
|
| 474 |
lua_util.debugm(N, log_obj, "text part check: %s printable, %s non-printable, %s total", |
| 475 |
tlen - non_printable, non_printable, tlen) |
| 476 |
if non_printable / tlen > 0.0078125 then |
| 477 |
return false |
| 478 |
end |
| 479 |
|
| 480 |
return true |
| 481 |
end |
| 482 |
|
| 483 |
local parent = part:get_parent() |
| 484 |
|
| 485 |
if parent then |
| 486 |
local parent_type, parent_subtype = parent:get_type() |
| 487 |
|
| 488 |
if parent_type == 'multipart' and parent_subtype == 'encrypted' then |
| 489 |
-- Skip text heuristics for encrypted parts |
| 490 |
lua_util.debugm(N, log_obj, "text part check: parent is encrypted, not a text part") |
| 491 |
|
| 492 |
return false |
| 493 |
end |
| 494 |
end |
| 495 |
|
| 496 |
local content = part:get_content() |
| 497 |
local mtype, msubtype = part:get_type() |
| 498 |
local clen = #content |
| 499 |
local is_text |
| 500 |
|
| 501 |
if clen > 0 then |
| 502 |
if clen > 80 * 3 then |
| 503 |
-- Use chunks |
| 504 |
is_text = is_span_text(content:span(1, 160)) and is_span_text(content:span(clen - 80, 80)) |
| 505 |
else |
| 506 |
is_text = is_span_text(content) |
| 507 |
end |
| 508 |
|
| 509 |
if is_text and mtype ~= 'message' then |
| 510 |
-- Try patterns |
| 511 |
local span_len = math.min(4096, clen) |
| 512 |
local start_span = content:span(1, span_len) |
| 513 |
local matches = txt_trie:match(start_span) |
| 514 |
local res = {} |
| 515 |
local fname = part:get_filename() |
| 516 |
|
| 517 |
if matches then |
| 518 |
-- Require at least 2 occurrences of those patterns |
| 519 |
for n, positions in pairs(matches) do |
| 520 |
local ext, weight = txt_patterns_indexes[n][1], txt_patterns_indexes[n][2][2] |
| 521 |
if ext then |
| 522 |
res[ext] = (res[ext] or 0) + weight * #positions |
| 523 |
lua_util.debugm(N, log_obj, "found txt pattern for %s: %s, total: %s; %s/%s announced", |
| 524 |
ext, weight * #positions, res[ext], mtype, msubtype) |
| 525 |
end |
| 526 |
end |
| 527 |
|
| 528 |
if res.html and res.html >= 40 then |
| 529 |
-- HTML has priority over something like js... |
| 530 |
return 'html', res.html |
| 531 |
end |
| 532 |
|
| 533 |
-- XML prolog can appear inside HTML; do not let xml override html |
| 534 |
if res.xml and res.html then |
| 535 |
res.xml = nil |
| 536 |
end |
| 537 |
|
| 538 |
local ext, weight = process_top_detected(res) |
| 539 |
|
| 540 |
if weight then |
| 541 |
if weight >= 40 then |
| 542 |
-- Extra validation for csv extension |
| 543 |
if ext ~= 'csv' or validate_csv(part, content, log_obj) then |
| 544 |
return ext, weight |
| 545 |
end |
| 546 |
elseif fname and weight >= 20 then |
| 547 |
return ext, weight |
| 548 |
end |
| 549 |
end |
| 550 |
end |
| 551 |
|
| 552 |
-- Content type stuff |
| 553 |
if (mtype == 'text' or mtype == 'application') and |
| 554 |
(msubtype == 'html' or msubtype == 'xhtml+xml') then |
| 555 |
return 'html', 21 |
| 556 |
end |
| 557 |
|
| 558 |
if msubtype:lower() == 'csv' then |
| 559 |
if validate_csv(part, content, log_obj) then |
| 560 |
return 'csv', 40 |
| 561 |
end |
| 562 |
end |
| 563 |
|
| 564 |
-- Extension stuff |
| 565 |
local function has_extension(file, ext) |
| 566 |
local ext_len = ext:len() |
| 567 |
return file:len() > ext_len + 1 |
| 568 |
and file:sub(-ext_len):lower() == ext |
| 569 |
and file:sub(-ext_len - 1, -ext_len - 1) == '.' |
| 570 |
end |
| 571 |
|
| 572 |
if fname and (has_extension(fname, 'htm') or has_extension(fname, 'html')) then |
| 573 |
return 'html', 21 |
| 574 |
end |
| 575 |
|
| 576 |
if mtype ~= 'text' then |
| 577 |
-- Do not treat non text patterns as text |
| 578 |
return nil |
| 579 |
end |
| 580 |
|
| 581 |
return 'txt', 40 |
| 582 |
end |
| 583 |
end |
| 584 |
end |
| 585 |
|
| 586 |
exports.pdf_format_heuristic = function(input, log_obj, pos, part) |
| 587 |
local weight = 10 |
| 588 |
local ext = string.match(part:get_filename() or '', '%.([^.]+)$') |
| 589 |
-- If we found a pattern at the beginning |
| 590 |
if pos <= 10 then |
| 591 |
weight = weight + 30 |
| 592 |
end |
| 593 |
-- If the announced extension is `pdf` |
| 594 |
if ext and ext:lower() == 'pdf' then |
| 595 |
weight = weight + 30 |
| 596 |
end |
| 597 |
|
| 598 |
return 'pdf', weight |
| 599 |
end |
| 600 |
|
| 601 |
exports.pe_part_heuristic = function(input, log_obj, pos, part) |
| 602 |
if not input then |
| 603 |
return |
| 604 |
end |
| 605 |
|
| 606 |
-- pe header should start at the offset that is placed in msdos header at position 60..64 |
| 607 |
local pe_ptr_bin = input:sub(60, 64) |
| 608 |
if #pe_ptr_bin ~= 4 then |
| 609 |
return |
| 610 |
end |
| 611 |
|
| 612 |
-- it is an LE 32 bit integer |
| 613 |
local pe_ptr = rspamd_util.unpack("<I4", pe_ptr_bin) |
| 614 |
-- if pe header magic matches the offset, it is definitely a PE file |
| 615 |
if pe_ptr ~= pos then |
| 616 |
return |
| 617 |
end |
| 618 |
|
| 619 |
return 'exe', 30 |
| 620 |
end |
| 621 |
|
| 622 |
-- SVG heuristic: check if this is actually HTML with embedded SVG |
| 623 |
exports.svg_format_heuristic = function(input, log_obj, pos, part) |
| 624 |
if not input then |
| 625 |
return |
| 626 |
end |
| 627 |
|
| 628 |
-- Only check content before the <svg> tag position |
| 629 |
local check_len = math.min(pos, 4096) |
| 630 |
if check_len < 5 then |
| 631 |
-- <svg> is at the very beginning, likely a real SVG |
| 632 |
return 'svg', 40 |
| 633 |
end |
| 634 |
|
| 635 |
local head = tostring(input:span(1, check_len)):lower() |
| 636 |
|
| 637 |
-- Check for HTML markers that would appear before <svg> in an HTML document |
| 638 |
-- If we find these, it's HTML with embedded SVG, not a standalone SVG |
| 639 |
if head:find('<!doctype%s+html') or |
| 640 |
head:find('<html[%s>]') or |
| 641 |
head:find('<head[%s>]') or |
| 642 |
head:find('<body[%s>]') or |
| 643 |
head:find('<meta[%s>]') then |
| 644 |
lua_util.debugm(N, log_obj, 'svg pattern found at %s but HTML markers present, skipping svg detection', |
| 645 |
pos) |
| 646 |
return nil |
| 647 |
end |
| 648 |
|
| 649 |
return 'svg', 40 |
| 650 |
end |
| 651 |
|
| 652 |
return exports |