Debian Codegraph @

lualib/lua_magic/heuristics.lua

raw
1 --[[
2 Copyright (c) 2022, Vsevolod Stakhov <[email protected]>
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 ]] --
16
17 --[[[
18 -- @module lua_magic/heuristics
19 -- This module contains heuristics for some specific cases
20 --]]
21
22 local rspamd_trie = require "rspamd_trie"
23 local rspamd_util = require "rspamd_util"
24 local lua_util = require "lua_util"
25 local bit = require "bit"
26 local fun = require "fun"
27
28 local N = "lua_magic"
29 local msoffice_trie
30 local msoffice_patterns = {
31 doc = { [[WordDocument]] },
32 xls = { [[Workbook]], [[Book]] },
33 ppt = { [[PowerPoint Document]], [[Current User]] },
34 vsd = { [[VisioDocument]] },
35 }
36 local msoffice_trie_clsid
37 local msoffice_clsids = {
38 doc = { [[0609020000000000c000000000000046]] },
39 xls = { [[1008020000000000c000000000000046]], [[2008020000000000c000000000000046]] },
40 ppt = { [[108d81649b4fcf1186ea00aa00b929e8]] },
41 msg = { [[46f0060000000000c000000000000046]], [[0b0d020000000000c000000000000046]] },
42 msi = { [[84100c0000000000c000000000000046]] },
43 }
44 local zip_trie
45 local zip_patterns = {
46 -- https://lists.oasis-open.org/archives/office/200505/msg00006.html
47 odt = {
48 [[mimetypeapplication/vnd\.oasis\.opendocument\.text]],
49 [[mimetypeapplication/vnd\.oasis\.opendocument\.image]],
50 [[mimetypeapplication/vnd\.oasis\.opendocument\.graphic]]
51 },
52 ods = {
53 [[mimetypeapplication/vnd\.oasis\.opendocument\.spreadsheet]],
54 [[mimetypeapplication/vnd\.oasis\.opendocument\.formula]],
55 [[mimetypeapplication/vnd\.oasis\.opendocument\.chart]]
56 },
57 odp = { [[mimetypeapplication/vnd\.oasis\.opendocument\.presentation]] },
58 epub = { [[epub\+zip]] },
59 asice = { [[mimetypeapplication/vnd\.etsi\.asic-e\+zipPK]] },
60 asics = { [[mimetypeapplication/vnd\.etsi\.asic-s\+zipPK]] },
61 }
62
63 local txt_trie
64 local txt_patterns = {
65 html = {
66 { [=[(?i)<html[\s>]]=], 32 },
67 { [[(?i)<script\b]], 20 }, -- Commonly used by spammers
68 { [[<script\s+type="text\/javascript">]], 31 }, -- Another spammy pattern
69 { [[(?i)<\!DOCTYPE HTML\b]], 33 },
70 { [[(?i)<body\b]], 20 },
71 { [[(?i)<table\b]], 20 },
72 { [[(?i)<a\s]], 10 },
73 { [[(?i)<p\b]], 10 },
74 { [[(?i)<div\b]], 10 },
75 { [[(?i)<span\b]], 10 },
76 },
77 csv = {
78 { [[(?:[-a-zA-Z0-9_]+\s*,){2,}(?:[-a-zA-Z0-9_]+,?[ ]*[\r\n])]], 20 }
79 },
80 ics = {
81 { [[^BEGIN:VCALENDAR\r?\n]], 40 },
82 },
83 vcf = {
84 { [[^BEGIN:VCARD\r?\n]], 40 },
85 },
86 xml = {
87 { [[<\?xml\b.+\?>]], 40 },
88 }
89 }
90
91 -- Used to match pattern index and extension
92 local msoffice_clsid_indexes = {}
93 local msoffice_patterns_indexes = {}
94 local zip_patterns_indexes = {}
95 local txt_patterns_indexes = {}
96
97 local exports = {}
98
99 local function compile_tries()
100 local default_compile_flags = bit.bor(rspamd_trie.flags.re,
101 rspamd_trie.flags.dot_all,
102 rspamd_trie.flags.single_match,
103 rspamd_trie.flags.no_start)
104 local function compile_pats(patterns, indexes, transform_func, compile_flags)
105 local strs = {}
106 for ext, pats in pairs(patterns) do
107 for _, pat in ipairs(pats) do
108 -- These are utf16 strings in fact...
109 strs[#strs + 1] = transform_func(pat)
110 indexes[#indexes + 1] = { ext, pat }
111 end
112 end
113
114 return rspamd_trie.create(strs, compile_flags or default_compile_flags)
115 end
116
117 if not msoffice_trie then
118 -- Directory names
119 local function msoffice_pattern_transform(pat)
120 return '^' ..
121 table.concat(
122 fun.totable(
123 fun.map(function(c)
124 return c .. [[\x{00}]]
125 end,
126 fun.iter(pat))))
127 end
128 local function msoffice_clsid_transform(pat)
129 local hex_table = {}
130 for i = 1, #pat, 2 do
131 local subc = pat:sub(i, i + 1)
132 hex_table[#hex_table + 1] = string.format('\\x{%s}', subc)
133 end
134
135 return '^' .. table.concat(hex_table) .. '$'
136 end
137 -- Directory entries
138 msoffice_trie = compile_pats(msoffice_patterns, msoffice_patterns_indexes,
139 msoffice_pattern_transform)
140 -- Clsids
141 msoffice_trie_clsid = compile_pats(msoffice_clsids, msoffice_clsid_indexes,
142 msoffice_clsid_transform)
143 -- Misc zip patterns at the initial fragment
144 zip_trie = compile_pats(zip_patterns, zip_patterns_indexes,
145 function(pat)
146 return pat
147 end)
148 -- Text patterns at the initial fragment
149 txt_trie = compile_pats(txt_patterns, txt_patterns_indexes,
150 function(pat_tbl)
151 return pat_tbl[1]
152 end,
153 bit.bor(rspamd_trie.flags.re,
154 rspamd_trie.flags.dot_all,
155 rspamd_trie.flags.no_start))
156 end
157 end
158
159 -- Call immediately on require
160 compile_tries()
161
162 local function detect_ole_format(input, log_obj, _, part)
163 local inplen = #input
164 if inplen < 0x31 + 4 then
165 lua_util.debugm(N, log_obj, "short length: %s", inplen)
166 return nil
167 end
168
169 local bom, sec_size = rspamd_util.unpack('<I2<I2', input:span(29, 4))
170 if bom == 0xFFFE then
171 bom = '<'
172 else
173 lua_util.debugm(N, log_obj, "bom file!: %s", bom)
174 bom = '>';
175 sec_size = bit.bswap(sec_size)
176 end
177
178 if sec_size < 7 or sec_size > 31 then
179 lua_util.debugm(N, log_obj, "bad sec_size: %s", sec_size)
180 return nil
181 end
182
183 sec_size = 2 ^ sec_size
184
185 -- SecID of first sector of the directory stream
186 local directory_offset = (rspamd_util.unpack(bom .. 'I4', input:span(0x31, 4)))
187 * sec_size + 512 + 1
188 lua_util.debugm(N, log_obj, "directory: %s", directory_offset)
189
190 if inplen < directory_offset then
191 lua_util.debugm(N, log_obj, "short length: %s", inplen)
192 return nil
193 end
194
195 local function process_dir_entry(offset)
196 local dtype = input:byte(offset + 66)
197 lua_util.debugm(N, log_obj, "dtype: %s, offset: %s", dtype, offset)
198
199 if dtype then
200 if dtype == 5 then
201 -- Extract clsid
202 local matches = msoffice_trie_clsid:match(input:span(offset + 80, 16))
203 if matches then
204 for n, _ in pairs(matches) do
205 if msoffice_clsid_indexes[n] then
206 lua_util.debugm(N, log_obj, "found valid clsid for %s",
207 msoffice_clsid_indexes[n][1])
208 return true, msoffice_clsid_indexes[n][1]
209 end
210 end
211 end
212 return true, nil
213 elseif dtype == 2 then
214 local matches = msoffice_trie:match(input:span(offset, 64))
215 if matches then
216 for n, _ in pairs(matches) do
217 if msoffice_patterns_indexes[n] then
218 return true, msoffice_patterns_indexes[n][1]
219 end
220 end
221 end
222 return true, nil
223 elseif dtype >= 0 and dtype < 5 then
224 -- Bad type
225 return true, nil
226 end
227 end
228
229 return false, nil
230 end
231
232 repeat
233 local res, ext = process_dir_entry(directory_offset)
234
235 if res and ext then
236 return ext, 60
237 end
238
239 if not res then
240 break
241 end
242
243 directory_offset = directory_offset + 128
244 until directory_offset >= inplen
245 end
246
247 exports.ole_format_heuristic = detect_ole_format
248
249 local function process_top_detected(res)
250 local extensions = lua_util.keys(res)
251
252 if #extensions > 0 then
253 table.sort(extensions, function(ex1, ex2)
254 return res[ex1] > res[ex2]
255 end)
256
257 return extensions[1], res[extensions[1]]
258 end
259
260 return nil
261 end
262
263 local function detect_archive_flaw(part, arch, log_obj, _)
264 local arch_type = arch:get_type()
265 local res = {
266 docx = 0,
267 xlsx = 0,
268 pptx = 0,
269 jar = 0,
270 odt = 0,
271 odp = 0,
272 ods = 0,
273 apk = 0,
274 } -- ext + confidence pairs
275
276 local function has_control_or_zw(fname)
277 -- control ASCII
278 if fname:find("[%z\1-\31]") then return true end
279 -- common zero-width UTF-8: U+200B..U+200D, U+FEFF
280 if fname:find("\226\128[\139-\141]") then return true end -- U+200B..U+200D
281 if fname:find("\239\187\191") then return true end -- U+FEFF
282 return false
283 end
284
285 -- General msoffice patterns
286 local function add_msoffice_confidence(incr)
287 res.docx = res.docx + incr
288 res.xlsx = res.xlsx + incr
289 res.pptx = res.pptx + incr
290 end
291
292 if arch_type == 'zip' then
293 -- Find specific files/folders in zip file
294 local files = arch:get_files(100) or {}
295 for _, file in ipairs(files) do
296 if has_control_or_zw(file) then
297 lua_util.debugm(N, log_obj, "archive filename has control/zw chars: %s", file)
298 end
299 if file == '[Content_Types].xml' then
300 add_msoffice_confidence(10)
301 elseif file:sub(1, 3) == 'xl/' then
302 res.xlsx = res.xlsx + 30
303 elseif file:sub(1, 5) == 'word/' then
304 res.docx = res.docx + 30
305 elseif file:sub(1, 4) == 'ppt/' then
306 res.pptx = res.pptx + 30
307 elseif file == 'META-INF/MANIFEST.MF' then
308 res.jar = res.jar + 40
309 elseif file == 'AndroidManifest.xml' then
310 res.apk = res.apk + 60
311 end
312 end
313
314 local ext, weight = process_top_detected(res)
315
316 if weight >= 40 then
317 return ext, weight
318 end
319
320 -- Apply misc Zip detection logic
321 local content = part:get_content()
322
323 if #content > 128 then
324 local start_span = content:span(1, 128)
325
326 local matches = zip_trie:match(start_span)
327 if matches then
328 for n, _ in pairs(matches) do
329 if zip_patterns_indexes[n] then
330 lua_util.debugm(N, log_obj, "found zip pattern for %s",
331 zip_patterns_indexes[n][1])
332 return zip_patterns_indexes[n][1], 40
333 end
334 end
335 end
336 end
337 end
338
339 return arch_type:lower(), 40
340 end
341
342 local csv_grammar
343 -- Returns a grammar that will count commas
344 local function get_csv_grammar()
345 if not csv_grammar then
346 local lpeg = require 'lpeg'
347
348 local field = '"' * lpeg.Cs(((lpeg.P(1) - '"') + lpeg.P '""' / '"') ^ 0) * '"' +
349 lpeg.C((1 - lpeg.S ',\n"') ^ 0)
350
351 csv_grammar = lpeg.Cf(lpeg.Cc(0) * field * lpeg.P((lpeg.P(',') +
352 lpeg.P('\t')) * field) ^ 1 * (lpeg.S '\r\n' + -1),
353 function(acc)
354 return acc + 1
355 end)
356 end
357
358 return csv_grammar
359 end
360 local function validate_csv(part, content, log_obj)
361 local max_chunk = 32768
362 local chunk = content:sub(1, max_chunk)
363
364 local expected_commas
365 local matched_lines = 0
366 local max_matched_lines = 10
367
368 lua_util.debugm(N, log_obj, "check for csv pattern")
369
370 for s in chunk:lines() do
371 local ncommas = get_csv_grammar():match(s)
372
373 if not ncommas then
374 lua_util.debugm(N, log_obj, "not a csv line at line number %s",
375 matched_lines)
376 return false
377 end
378
379 if expected_commas and ncommas ~= expected_commas then
380 -- Mismatched commas
381 lua_util.debugm(N, log_obj, "missmatched commas on line %s: %s != %s",
382 matched_lines, ncommas, expected_commas)
383 return false
384 elseif not expected_commas then
385 if ncommas == 0 then
386 lua_util.debugm(N, log_obj, "no commas in the first line")
387 return false
388 end
389 expected_commas = ncommas
390 end
391
392 matched_lines = matched_lines + 1
393
394 if matched_lines > max_matched_lines then
395 break
396 end
397 end
398
399 lua_util.debugm(N, log_obj, "csv content is sane: %s fields; %s lines checked",
400 expected_commas, matched_lines)
401
402 return true
403 end
404
405 exports.mime_part_heuristic = function(part, log_obj, _)
406 if part:is_archive() then
407 local arch = part:get_archive()
408 return detect_archive_flaw(part, arch, log_obj)
409 end
410
411 return nil
412 end
413
414 exports.text_part_heuristic = function(part, log_obj, _)
415 -- We get some span of data and check it
416 local function is_span_text(span)
417 -- We examine 8 bit content, and we assume it might be localized text
418 -- if it has more than 3 subsequent 8 bit characters
419 local function rough_8bit_check(bytes, idx, remain, len)
420 local b = bytes[idx]
421 local n8bit = 0
422
423 while b >= 127 and idx < len do
424 -- utf8 part
425 if bit.band(b, 0xe0) == 0xc0 and remain > 1 and
426 bit.band(bytes[idx + 1], 0xc0) == 0x80 then
427 return true, 1
428 elseif bit.band(b, 0xf0) == 0xe0 and remain > 2 and
429 bit.band(bytes[idx + 1], 0xc0) == 0x80 and
430 bit.band(bytes[idx + 2], 0xc0) == 0x80 then
431 return true, 2
432 elseif bit.band(b, 0xf8) == 0xf0 and remain > 3 and
433 bit.band(bytes[idx + 1], 0xc0) == 0x80 and
434 bit.band(bytes[idx + 2], 0xc0) == 0x80 and
435 bit.band(bytes[idx + 3], 0xc0) == 0x80 then
436 return true, 3
437 end
438
439 n8bit = n8bit + 1
440 idx = idx + 1
441 b = bytes[idx]
442 remain = remain - 1
443 end
444
445 if n8bit >= 3 then
446 return true, n8bit
447 end
448
449 return false, 0
450 end
451
452 -- Convert to string as LuaJIT can optimise string.sub (and fun.iter) but not C calls
453 local tlen = #span
454 local non_printable = 0
455 local bytes = span:bytes()
456 local i = 1
457 repeat
458 local b = bytes[i]
459
460 if (b < 0x20) and not (b == 0x0d or b == 0x0a or b == 0x09) then
461 non_printable = non_printable + 1
462 elseif b >= 127 then
463 local c, nskip = rough_8bit_check(bytes, i, tlen - i, tlen)
464
465 if not c then
466 non_printable = non_printable + 1
467 else
468 i = i + nskip
469 end
470 end
471 i = i + 1
472 until i > tlen
473
474 lua_util.debugm(N, log_obj, "text part check: %s printable, %s non-printable, %s total",
475 tlen - non_printable, non_printable, tlen)
476 if non_printable / tlen > 0.0078125 then
477 return false
478 end
479
480 return true
481 end
482
483 local parent = part:get_parent()
484
485 if parent then
486 local parent_type, parent_subtype = parent:get_type()
487
488 if parent_type == 'multipart' and parent_subtype == 'encrypted' then
489 -- Skip text heuristics for encrypted parts
490 lua_util.debugm(N, log_obj, "text part check: parent is encrypted, not a text part")
491
492 return false
493 end
494 end
495
496 local content = part:get_content()
497 local mtype, msubtype = part:get_type()
498 local clen = #content
499 local is_text
500
501 if clen > 0 then
502 if clen > 80 * 3 then
503 -- Use chunks
504 is_text = is_span_text(content:span(1, 160)) and is_span_text(content:span(clen - 80, 80))
505 else
506 is_text = is_span_text(content)
507 end
508
509 if is_text and mtype ~= 'message' then
510 -- Try patterns
511 local span_len = math.min(4096, clen)
512 local start_span = content:span(1, span_len)
513 local matches = txt_trie:match(start_span)
514 local res = {}
515 local fname = part:get_filename()
516
517 if matches then
518 -- Require at least 2 occurrences of those patterns
519 for n, positions in pairs(matches) do
520 local ext, weight = txt_patterns_indexes[n][1], txt_patterns_indexes[n][2][2]
521 if ext then
522 res[ext] = (res[ext] or 0) + weight * #positions
523 lua_util.debugm(N, log_obj, "found txt pattern for %s: %s, total: %s; %s/%s announced",
524 ext, weight * #positions, res[ext], mtype, msubtype)
525 end
526 end
527
528 if res.html and res.html >= 40 then
529 -- HTML has priority over something like js...
530 return 'html', res.html
531 end
532
533 -- XML prolog can appear inside HTML; do not let xml override html
534 if res.xml and res.html then
535 res.xml = nil
536 end
537
538 local ext, weight = process_top_detected(res)
539
540 if weight then
541 if weight >= 40 then
542 -- Extra validation for csv extension
543 if ext ~= 'csv' or validate_csv(part, content, log_obj) then
544 return ext, weight
545 end
546 elseif fname and weight >= 20 then
547 return ext, weight
548 end
549 end
550 end
551
552 -- Content type stuff
553 if (mtype == 'text' or mtype == 'application') and
554 (msubtype == 'html' or msubtype == 'xhtml+xml') then
555 return 'html', 21
556 end
557
558 if msubtype:lower() == 'csv' then
559 if validate_csv(part, content, log_obj) then
560 return 'csv', 40
561 end
562 end
563
564 -- Extension stuff
565 local function has_extension(file, ext)
566 local ext_len = ext:len()
567 return file:len() > ext_len + 1
568 and file:sub(-ext_len):lower() == ext
569 and file:sub(-ext_len - 1, -ext_len - 1) == '.'
570 end
571
572 if fname and (has_extension(fname, 'htm') or has_extension(fname, 'html')) then
573 return 'html', 21
574 end
575
576 if mtype ~= 'text' then
577 -- Do not treat non text patterns as text
578 return nil
579 end
580
581 return 'txt', 40
582 end
583 end
584 end
585
586 exports.pdf_format_heuristic = function(input, log_obj, pos, part)
587 local weight = 10
588 local ext = string.match(part:get_filename() or '', '%.([^.]+)$')
589 -- If we found a pattern at the beginning
590 if pos <= 10 then
591 weight = weight + 30
592 end
593 -- If the announced extension is `pdf`
594 if ext and ext:lower() == 'pdf' then
595 weight = weight + 30
596 end
597
598 return 'pdf', weight
599 end
600
601 exports.pe_part_heuristic = function(input, log_obj, pos, part)
602 if not input then
603 return
604 end
605
606 -- pe header should start at the offset that is placed in msdos header at position 60..64
607 local pe_ptr_bin = input:sub(60, 64)
608 if #pe_ptr_bin ~= 4 then
609 return
610 end
611
612 -- it is an LE 32 bit integer
613 local pe_ptr = rspamd_util.unpack("<I4", pe_ptr_bin)
614 -- if pe header magic matches the offset, it is definitely a PE file
615 if pe_ptr ~= pos then
616 return
617 end
618
619 return 'exe', 30
620 end
621
622 -- SVG heuristic: check if this is actually HTML with embedded SVG
623 exports.svg_format_heuristic = function(input, log_obj, pos, part)
624 if not input then
625 return
626 end
627
628 -- Only check content before the <svg> tag position
629 local check_len = math.min(pos, 4096)
630 if check_len < 5 then
631 -- <svg> is at the very beginning, likely a real SVG
632 return 'svg', 40
633 end
634
635 local head = tostring(input:span(1, check_len)):lower()
636
637 -- Check for HTML markers that would appear before <svg> in an HTML document
638 -- If we find these, it's HTML with embedded SVG, not a standalone SVG
639 if head:find('<!doctype%s+html') or
640 head:find('<html[%s>]') or
641 head:find('<head[%s>]') or
642 head:find('<body[%s>]') or
643 head:find('<meta[%s>]') then
644 lua_util.debugm(N, log_obj, 'svg pattern found at %s but HTML markers present, skipping svg detection',
645 pos)
646 return nil
647 end
648
649 return 'svg', 40
650 end
651
652 return exports