Debian Codegraph @

lualib/lua_content/pdf.lua

raw
1 --[[
2 Copyright (c) 2022, Vsevolod Stakhov <[email protected]>
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 ]]--
16
17 --[[[
18 -- @module lua_content/pdf
19 -- This module contains some heuristics for PDF files
20 --]]
21
22 local rspamd_trie = require "rspamd_trie"
23 local rspamd_util = require "rspamd_util"
24 local rspamd_text = require "rspamd_text"
25 local rspamd_url = require "rspamd_url"
26 local bit = require "bit"
27 local N = "lua_content"
28 local lua_util = require "lua_util"
29 local rspamd_regexp = require "rspamd_regexp"
30 local lpeg = require "lpeg"
31 local pdf_patterns = {
32 trailer = {
33 patterns = {
34 [[\ntrailer\r?\n]]
35 }
36 },
37 suspicious = {
38 patterns = {
39 [[netsh\s]],
40 [[echo\s]],
41 [=[\/[A-Za-z]*#\d\d[#A-Za-z<>/\s]]=], -- Hex encode obfuscation
42 }
43 },
44 start_object = {
45 patterns = {
46 [=[[\r\n\0]\s*\d+\s+\d+\s+obj[\s<]]=]
47 }
48 },
49 end_object = {
50 patterns = {
51 [=[endobj[\r\n]]=]
52 }
53 },
54 start_stream = {
55 patterns = {
56 [=[>\s*stream[\r\n]]=],
57 }
58 },
59 end_stream = {
60 patterns = {
61 [=[endstream[\r\n]]=]
62 }
63 }
64 }
65
66 local pdf_cmap_patterns = {
67 start = {
68 patterns = {
69 [[\d\s+beginbfchar\s]],
70 [[\d\s+beginbfrange\s]]
71 }
72 },
73 stop = {
74 patterns = {
75 [[\sendbfrange\b]],
76 [[\sendbchar\b]]
77 }
78 }
79 }
80
81 -- index[n] ->
82 -- t[1] - pattern,
83 -- t[2] - key in patterns table,
84 -- t[3] - value in patterns table
85 -- t[4] - local pattern index
86 local pdf_indexes = {}
87 local pdf_cmap_indexes = {}
88
89 local pdf_trie
90 local pdf_text_trie
91 local pdf_cmap_trie
92
93 local exports = {}
94
95 local config = {
96 max_extraction_size = 512 * 1024,
97 max_processing_size = 32 * 1024,
98 text_extraction = true,
99 url_extraction = true,
100 enabled = true,
101 js_fuzzy = true, -- Generate fuzzy hashes from PDF javascripts
102 min_js_fuzzy = 256, -- Minimum size of js to be considered as a fuzzy
103 openaction_fuzzy_only = false, -- Generate fuzzy from all scripts
104 max_pdf_objects = 10000, -- Maximum number of objects to be considered
105 min_obj_content_size = 32, -- Skip objects smaller than this (evasion padding)
106 max_pdf_trailer = 10 * 1024 * 1024, -- Maximum trailer size (to avoid abuse)
107 max_pdf_trailer_lines = 100, -- Maximum number of lines in pdf trailer
108 pdf_process_timeout = 2.0, -- Timeout in seconds for processing
109 text_quality_threshold = 0.4, -- Minimum confidence to accept extracted text
110 text_quality_min_length = 10, -- Minimum text length to apply quality filtering
111 text_quality_enabled = true, -- Enable text quality filtering
112 }
113
114 -- Used to process patterns found in PDF
115 -- positions for functional processors should be a iter/table from trie matcher in form
116 ---- [{n1, pat_idx1}, ... {nn, pat_idxn}] where
117 ---- pat_idxn is pattern index and n1 ... nn are match positions
118 local processors = {}
119 -- PDF objects outer grammar in LPEG style (performing table captures)
120 local pdf_outer_grammar
121 local pdf_text_grammar
122
123 -- Used to match objects
124 local object_re = rspamd_regexp.create_cached([=[/(\d+)\s+(\d+)\s+obj\s*/]=])
125
126 local function config_module()
127 local opts = rspamd_config:get_all_opt('lua_content')
128
129 if opts and opts.pdf then
130 config = lua_util.override_defaults(config, opts.pdf)
131 end
132 end
133
134 local function compile_tries()
135 local default_compile_flags = bit.bor(rspamd_trie.flags.re,
136 rspamd_trie.flags.dot_all,
137 rspamd_trie.flags.no_start)
138 local function compile_pats(patterns, indexes, compile_flags)
139 local strs = {}
140 for what, data in pairs(patterns) do
141 for i, pat in ipairs(data.patterns) do
142 strs[#strs + 1] = pat
143 indexes[#indexes + 1] = { what, data, pat, i }
144 end
145 end
146
147 return rspamd_trie.create(strs, compile_flags or default_compile_flags)
148 end
149
150 if not pdf_trie then
151 pdf_trie = compile_pats(pdf_patterns, pdf_indexes)
152 end
153 if not pdf_text_trie then
154 pdf_text_trie = rspamd_trie.create({
155 [[\sBT\s]],
156 [[\sET\b]]
157 }, default_compile_flags)
158 end
159 if not pdf_cmap_trie then
160 pdf_cmap_trie = compile_pats(pdf_cmap_patterns, pdf_cmap_indexes)
161 end
162 end
163
164 -- StandardEncoding/MacRomanEncoding ligature substitutions
165 -- Applied only to rendered text, NOT to dictionary strings (URI values etc.)
166 -- to avoid corrupting soft hyphens (U+00AD = byte 0xAD = \173) in URLs
167 local function apply_ligature_substitutions(s)
168 if not s then return s end
169 s = s:gsub('\171', 'ff')
170 s = s:gsub('\172', 'ffi')
171 s = s:gsub('\173', 'ffl')
172 s = s:gsub('\174', 'fi')
173 s = s:gsub('\175', 'fl')
174 s = s:gsub('\222', 'fi')
175 s = s:gsub('\223', 'fl')
176 return s
177 end
178
179 -- Returns a table with generic grammar elements for PDF
180 local function generic_grammar_elts()
181 local P = lpeg.P
182 local R = lpeg.R
183 local S = lpeg.S
184 local V = lpeg.V
185 local C = lpeg.C
186 local D = R '09' -- Digits
187
188 local grammar_elts = {}
189
190 -- Helper functions
191 local function pdf_hexstring_unescape(s)
192 local res
193 if #s % 2 == 0 then
194 -- Sane hex string
195 res = lua_util.unhex(s)
196 else
197 -- WTF hex string
198 -- Append '0' to it and unescape...
199 res = lua_util.unhex(s:sub(1, #s - 1)) .. lua_util.unhex((s:sub(#s) .. '0'))
200 end
201
202 return res
203 end
204
205 local function pdf_string_unescape(s)
206 local function ue_single(cc)
207 if cc == '\\r' then
208 return '\r'
209 elseif cc == '\\n' then
210 return '\n'
211 else
212 return cc:gsub(2, 2)
213 end
214 end
215 -- simple unescape \char
216 s = s:gsub('\\[^%d]', ue_single)
217 -- unescape octal
218 local function ue_octal(cc)
219 -- Replace unknown stuff with '?'
220 return string.char(tonumber(cc:sub(2), 8) or 63)
221 end
222 s = s:gsub('\\%d%d?%d?', ue_octal)
223
224 return s
225 end
226
227 local function pdf_id_unescape(s)
228 return (s:gsub('#%d%d', function(cc)
229 return string.char(tonumber(cc:sub(2), 16))
230 end))
231 end
232
233 local delim = S '()<>[]{}/%'
234 grammar_elts.ws = S '\0 \r\n\t\f'
235 local hex = R 'af' + R 'AF' + D
236 -- Comments.
237 local eol = P '\r\n' + '\n'
238 local line = (1 - S '\r\n\f') ^ 0 * eol ^ -1
239 grammar_elts.comment = P '%' * line
240
241 -- Numbers.
242 local sign = S '+-' ^ -1
243 local decimal = D ^ 1
244 local float = D ^ 1 * P '.' * D ^ 0 + P '.' * D ^ 1
245 grammar_elts.number = C(sign * (float + decimal)) / tonumber
246
247 -- String
248 grammar_elts.str = P { "(" * C(((1 - S "()\\") + (P '\\' * 1) + V(1)) ^ 0) / pdf_string_unescape * ")" }
249 grammar_elts.hexstr = P { "<" * C(hex ^ 0) / pdf_hexstring_unescape * ">" }
250
251 -- Identifier
252 grammar_elts.id = P { '/' * C((1 - (delim + grammar_elts.ws)) ^ 1) / pdf_id_unescape }
253
254 -- Booleans (who care about them?)
255 grammar_elts.boolean = C(P("true") + P("false"))
256
257 -- Stupid references
258 grammar_elts.ref = lpeg.Ct { lpeg.Cc("%REF%") * C(D ^ 1) * " " * C(D ^ 1) * " " * "R" }
259
260 return grammar_elts
261 end
262
263
264 -- Generates a grammar to parse outer elements (external objects in PDF notation)
265 local function gen_outer_grammar()
266 local V = lpeg.V
267 local gen = generic_grammar_elts()
268
269 return lpeg.P {
270 "EXPR";
271 EXPR = gen.ws ^ 0 * V("ELT") ^ 0 * gen.ws ^ 0,
272 ELT = V("ARRAY") + V("DICT") + V("ATOM"),
273 ATOM = gen.ws ^ 0 * (gen.comment + gen.boolean + gen.ref +
274 gen.number + V("STRING") + gen.id) * gen.ws ^ 0,
275 DICT = "<<" * gen.ws ^ 0 * lpeg.Cf(lpeg.Ct("") * V("KV_PAIR") ^ 0, rawset) * gen.ws ^ 0 * ">>",
276 KV_PAIR = lpeg.Cg(gen.id * gen.ws ^ 0 * V("ELT") * gen.ws ^ 0),
277 ARRAY = "[" * gen.ws ^ 0 * lpeg.Ct(V("ELT") ^ 0) * gen.ws ^ 0 * "]",
278 STRING = lpeg.P { gen.str + gen.hexstr },
279 }
280 end
281
282 -- Graphic state in PDF
283 local function gen_graphics_unary()
284 local P = lpeg.P
285 local S = lpeg.S
286
287 return P("q") + P("Q") + P("h")
288 + S("WSsFfBb") * P("*") ^ 0 + P("n")
289
290 end
291 local function gen_graphics_binary()
292 local P = lpeg.P
293 local S = lpeg.S
294
295 return S("gGwJjMi") +
296 P("M") + P("ri") + P("gs") +
297 P("CS") + P("cs") + P("sh")
298 end
299 local function gen_graphics_ternary()
300 local P = lpeg.P
301 local S = lpeg.S
302
303 return P("d") + P("m") + S("lm")
304 end
305 local function gen_graphics_nary()
306 local P = lpeg.P
307 local S = lpeg.S
308
309 return P("SC") + P("sc") + P("SCN") + P("scn") + P("k") + P("K") + P("re") + S("cvy") +
310 P("RG") + P("rg")
311 end
312
313 -- Calculate text quality confidence score using UTF-8 aware analysis
314 -- Returns a score between 0.0 (garbage) and 1.0 (high quality text)
315 local function calculate_text_confidence(text)
316 if not text or #text < config.text_quality_min_length then
317 return 1.0 -- Don't filter short text
318 end
319
320 local stats = rspamd_util.get_text_quality(text)
321 if not stats or stats.total == 0 then
322 return 0.0
323 end
324
325 local score = 0.0
326 local non_ws = stats.total - stats.spaces
327
328 -- Printable ratio (weight: 0.25) - target > 0.95
329 local printable_ratio = stats.printable / stats.total
330 score = score + math.min(printable_ratio / 0.95, 1.0) * 0.25
331
332 -- Letter ratio (weight: 0.20) - target > 0.6
333 local letter_ratio = 0
334 if non_ws > 0 then
335 letter_ratio = stats.letters / non_ws
336 end
337 score = score + math.min(letter_ratio / 0.6, 1.0) * 0.20
338
339 -- Word ratio (weight: 0.25) - target > 0.7
340 local word_ratio = 0
341 if non_ws > 0 then
342 word_ratio = stats.word_chars / non_ws
343 end
344 score = score + math.min(word_ratio / 0.7, 1.0) * 0.25
345
346 -- Average word length (weight: 0.15) - ideal: 3-10
347 local avg_word_len = 0
348 if stats.words > 0 then
349 avg_word_len = stats.word_chars / stats.words
350 end
351 local word_len_score = 0
352 if avg_word_len >= 3 and avg_word_len <= 10 then
353 word_len_score = 1.0
354 elseif avg_word_len >= 2 and avg_word_len < 3 then
355 word_len_score = 0.7
356 elseif avg_word_len > 10 and avg_word_len <= 15 then
357 word_len_score = 0.5
358 else
359 word_len_score = 0.2
360 end
361 score = score + word_len_score * 0.15
362
363 -- Space ratio (weight: 0.15) - ideal: 0.08-0.25
364 local space_ratio = stats.spaces / stats.total
365 local space_score = 0
366 if space_ratio >= 0.08 and space_ratio <= 0.25 then
367 space_score = 1.0
368 elseif space_ratio > 0.25 and space_ratio <= 0.4 then
369 space_score = 0.6
370 elseif space_ratio > 0 and space_ratio < 0.08 then
371 space_score = 0.5
372 else
373 space_score = 0.2
374 end
375 score = score + space_score * 0.15
376
377 return score
378 end
379
380 -- Generates a grammar to parse text blocks (between BT and ET)
381 local function gen_text_grammar()
382 local V = lpeg.V
383 local P = lpeg.P
384 local C = lpeg.C
385 local gen = generic_grammar_elts()
386
387 local function sanitize_pdf_text(s)
388 if not s or #s < 4 then return s end
389
390 local nulls_odd = 0
391 local nulls_even = 0
392 local len = #s
393
394 local limit = math.min(len, 16)
395 for i = 1, limit do
396 local b = string.byte(s, i)
397 if b == 0 then
398 if i % 2 == 1 then
399 nulls_odd = nulls_odd + 1
400 else
401 nulls_even = nulls_even + 1
402 end
403 end
404 end
405
406 if len > 32 then
407 for i = len - 15, len do
408 local b = string.byte(s, i)
409 if b == 0 then
410 if i % 2 == 1 then
411 nulls_odd = nulls_odd + 1
412 else
413 nulls_even = nulls_even + 1
414 end
415 end
416 end
417 elseif len > 16 then
418 for i = 17, len do
419 local b = string.byte(s, i)
420 if b == 0 then
421 if i % 2 == 1 then
422 nulls_odd = nulls_odd + 1
423 else
424 nulls_even = nulls_even + 1
425 end
426 end
427 end
428 end
429
430 local total_checked = (len > 32) and 32 or len
431 local total_odd = math.ceil(total_checked / 2)
432 local total_even = math.floor(total_checked / 2)
433
434 if len > 32 then
435 total_odd = 16
436 total_even = 16
437 end
438
439 local ratio_odd = nulls_odd / total_odd
440 local ratio_even = nulls_even / total_even
441 local charset
442
443 if ratio_odd > 0.8 and ratio_even < 0.2 then
444 charset = 'UTF-16BE'
445 elseif ratio_even > 0.8 and ratio_odd < 0.2 then
446 charset = 'UTF-16LE'
447 end
448
449 if charset and rspamd_util.to_utf8 then
450 local conv = rspamd_util.to_utf8(s, charset)
451 if conv then
452 local garbage_limit = 0
453 local clen = #conv
454 for i = 1, clen do
455 local b = conv:byte(i)
456 if b < 32 and b ~= 9 and b ~= 10 and b ~= 13 then
457 garbage_limit = garbage_limit + 1
458 end
459 end
460
461 if garbage_limit > 0 then
462 return ''
463 end
464
465 return conv
466 end
467 end
468
469 -- Strip null bytes and control characters from non-UTF-16 text.
470 -- PDF hex strings like <0041> produce raw bytes including \x00 which
471 -- are not meaningful in extracted text and cause false positives in
472 -- sa_raw_body rules matching \x00 patterns.
473 local has_control = false
474 for i = 1, len do
475 local b = s:byte(i)
476 if b == 0 or (b < 32 and b ~= 9 and b ~= 10 and b ~= 13) then
477 has_control = true
478 break
479 end
480 end
481
482 if has_control then
483 -- Remove null bytes and other control characters (keep tab, newline, carriage return)
484 s = s:gsub('[%z\1-\8\11\12\14-\31]', '')
485 if #s == 0 then
486 return ''
487 end
488 end
489
490 return s
491 end
492
493 local function text_op_handler(...)
494 local args = { ... }
495 local op = args[#args]
496 local t = args[#args - 1]
497
498 local res = t
499 if type(t) == 'table' then
500 local tres = {}
501 for _, chunk in ipairs(t) do
502 if type(chunk) == 'string' then
503 table.insert(tres, chunk)
504 elseif type(chunk) == 'number' and chunk < -200 then
505 table.insert(tres, ' ')
506 end
507 end
508 res = table.concat(tres)
509 end
510
511 -- Apply ligature substitutions for rendered text only (not dictionary strings like /URI)
512 if type(res) == 'string' then
513 res = apply_ligature_substitutions(res)
514 end
515
516 res = sanitize_pdf_text(res)
517
518 -- Apply text quality filtering to reject garbage chunks
519 if config.text_quality_enabled and res and #res >= config.text_quality_min_length then
520 local confidence = calculate_text_confidence(res)
521 if confidence < config.text_quality_threshold then
522 lua_util.debugm(N, nil, 'rejected low confidence text chunk (%.2f): %s',
523 confidence, res:sub(1, 50))
524 return ''
525 end
526 end
527
528 if op == "'" or op == '"' then
529 return '\n' .. res
530 end
531
532 return res
533 end
534
535 local function nary_op_handler(...)
536 local args = { ... }
537 local op = args[#args]
538
539 if op == 'Tm' then
540 return '\n'
541 end
542
543 return ''
544 end
545
546 local function ternary_op_handler(...)
547 local args = { ... }
548 local op = args[#args]
549 local a2 = args[#args - 2]
550
551 if (op == 'Td' or op == 'TD') and type(a2) == 'number' and a2 ~= 0 then
552 return '\n'
553 end
554
555 return ''
556 end
557
558 local empty = ""
559 local unary_ops = C("T*") / "\n" +
560 C(gen_graphics_unary()) / empty
561 local binary_ops = P("Tc") + P("Tw") + P("Tz") + P("TL") + P("Tr") + P("Ts") +
562 gen_graphics_binary()
563 local ternary_ops = P("TD") + P("Td") + gen_graphics_ternary()
564 local nary_op = P("Tm") + gen_graphics_nary()
565 local text_binary_op = C(P("Tj") + P("TJ") + P("'"))
566 local text_quote_op = C(P('"'))
567 local font_op = P("Tf")
568
569 return lpeg.P {
570 "EXPR";
571 EXPR = gen.ws ^ 0 * lpeg.Ct(V("COMMAND") ^ 0),
572 COMMAND = (V("UNARY") + V("BINARY") + V("TERNARY") + V("NARY") + V("TEXT") +
573 V("FONT") + gen.comment) * gen.ws ^ 0,
574 UNARY = unary_ops,
575 BINARY = V("ARG") / empty * gen.ws ^ 1 * binary_ops,
576 TERNARY = (V("ARG") * gen.ws ^ 1 * V("ARG") * gen.ws ^ 1 * ternary_ops) / ternary_op_handler,
577 NARY = lpeg.Ct((V("ARG") * gen.ws ^ 1) ^ 1) * (gen.id / empty * gen.ws ^ 0) ^ -1 * nary_op / nary_op_handler,
578 ARG = V("ARRAY") + V("DICT") + V("ATOM"),
579 ATOM = (gen.comment + gen.boolean + gen.ref +
580 gen.number + V("STRING") + gen.id),
581 DICT = "<<" * gen.ws ^ 0 * lpeg.Cf(lpeg.Ct("") * V("KV_PAIR") ^ 0, rawset) * gen.ws ^ 0 * ">>",
582 KV_PAIR = lpeg.Cg(gen.id * gen.ws ^ 0 * V("ARG") * gen.ws ^ 0),
583 ARRAY = "[" * gen.ws ^ 0 * lpeg.Ct(V("ARG") ^ 0) * gen.ws ^ 0 * "]",
584 STRING = lpeg.P { gen.str + gen.hexstr },
585 TEXT = ((V("TEXT_ARG") * gen.ws ^ 0 * text_binary_op) / text_op_handler) +
586 ((V("ARG") / empty * gen.ws ^ 1 * V("ARG") / empty * gen.ws ^ 1 * V("TEXT_ARG") * gen.ws ^ 0 * text_quote_op) / text_op_handler),
587 FONT = (V("FONT_ARG") * gen.ws ^ 1 * (gen.number / empty) * gen.ws ^ 1 * font_op) / empty,
588 FONT_ARG = lpeg.Ct(lpeg.Cc("%font%") * gen.id),
589 TEXT_ARG = lpeg.Ct(V("STRING")) + V("TEXT_ARRAY"),
590 TEXT_ARRAY = "[" * gen.ws ^ 0 * lpeg.Ct((V("TEXT_ARRAY_ELT") * gen.ws ^ 0) ^ 0) * "]",
591 TEXT_ARRAY_ELT = gen.number + gen.str + gen.hexstr,
592 }
593 end
594
595
596 -- Call immediately on require
597 compile_tries()
598 config_module()
599 pdf_outer_grammar = gen_outer_grammar()
600 pdf_text_grammar = gen_text_grammar()
601
602 local function extract_text_data(specific)
603 local res = {}
604 if specific.objects then
605 for _, obj in ipairs(specific.objects) do
606 if obj.text then
607 if type(obj.text) == 'userdata' then
608 res[#res + 1] = tostring(obj.text)
609 else
610 res[#res + 1] = obj.text
611 end
612 end
613 end
614 end
615
616 return res
617 end
618
619 -- Generates index for major/minor pair
620 local function obj_ref(major, minor)
621 return major * 10.0 + 1.0 / (minor + 1.0)
622 end
623
624 -- Return indirect object reference (if needed)
625 local function maybe_dereference_object(elt, pdf, task)
626 if type(elt) == 'table' and elt[1] == '%REF%' then
627 local ref = obj_ref(elt[2], elt[3])
628
629 if pdf.ref[ref] then
630 -- No recursion!
631 return pdf.ref[ref]
632 else
633 lua_util.debugm(N, task, 'cannot dereference %s:%s -> %s, no object',
634 elt[2], elt[3], obj_ref(elt[2], elt[3]))
635 return nil
636 end
637 end
638
639 return elt
640 end
641
642 -- Apply PDF stream filter
643 local function apply_pdf_filter(input, filt)
644 -- Validate input before processing
645 if not input or (type(input) == 'string' and #input == 0) then
646 return nil
647 end
648
649 if filt == 'FlateDecode' or filt == 'Fl' then
650 return rspamd_util.inflate(input, config.max_extraction_size)
651 elseif filt == 'ASCIIHexDecode' or filt == 'AHx' then
652 -- Strip > at the end if present (should be stripped by parser but safety check)
653 -- Also strip whitespaces
654 local to_decode = input:gsub('%s', '')
655 if to_decode:sub(-1) == '>' then
656 to_decode = to_decode:sub(1, -2)
657 end
658 if #to_decode == 0 then
659 return nil
660 end
661 return lua_util.unhex(to_decode)
662 elseif filt == 'ASCII85Decode' or filt == 'A85' then
663 return rspamd_util.decode_ascii85(input)
664 end
665
666 return nil
667 end
668
669 -- Conditionally apply a pipeline of stream filters and return uncompressed data
670 local function maybe_apply_filter(dict, data, pdf, task)
671 local uncompressed = data
672
673 if dict.Filter then
674 local filt = dict.Filter
675 local filts = {}
676
677 if type(filt) == 'string' then
678 filts = { filt }
679 elseif type(filt) == 'table' then
680 filts = filt
681 end
682
683 if dict.DecodeParms then
684 local decode_params = maybe_dereference_object(dict.DecodeParms, pdf, task)
685
686 if type(decode_params) == 'table' then
687 if decode_params.Predictor then
688 local predictor = tonumber(decode_params.Predictor) or 1
689 if predictor > 1 then
690 return nil, 'predictor exists: ' .. tostring(predictor)
691 end
692 end
693 end
694 end
695
696 for _, f in ipairs(filts) do
697 local next_uncompressed = apply_pdf_filter(uncompressed, f)
698
699 if next_uncompressed then
700 uncompressed = next_uncompressed
701 else
702 return nil, 'filter failed: ' .. tostring(f)
703 end
704 end
705 end
706
707 return uncompressed, nil
708 end
709
710 -- Conditionally extract stream data from object and attach it as obj.uncompressed
711 local function maybe_extract_object_stream(obj, pdf, task)
712 if pdf.encrypted then
713 -- TODO add decryption some day
714 return nil
715 end
716 if not obj.stream then
717 return nil
718 end
719 -- Defensive checks for stream structure
720 if not obj.stream.data or not obj.stream.len then
721 lua_util.debugm(N, task, 'malformed stream in object %s:%s',
722 obj.major, obj.minor)
723 return nil
724 end
725 local dict = obj.dict or {}
726 local len = obj.stream.len
727 if dict.Length then
728 local decl_len = maybe_dereference_object(dict.Length, pdf, task)
729 if decl_len then
730 local nlen = tonumber(decl_len)
731 if nlen then
732 len = math.min(len, nlen)
733 end
734 end
735 end
736
737 if len > 0 then
738 -- Wrap stream extraction in pcall to handle malformed data
739 local ret, real_stream = pcall(function()
740 return obj.stream.data:span(1, len)
741 end)
742 if not ret or not real_stream then
743 lua_util.debugm(N, task, 'cannot extract stream span from object %s:%s: %s',
744 obj.major, obj.minor, real_stream)
745 return nil
746 end
747
748 local uncompressed, filter_err = maybe_apply_filter(dict, real_stream, pdf, task)
749
750 if uncompressed then
751 obj.uncompressed = uncompressed
752 lua_util.debugm(N, task, 'extracted object %s:%s: (%s -> %s)',
753 obj.major, obj.minor, len, #uncompressed)
754 return obj.uncompressed
755 else
756 lua_util.debugm(N, task, 'cannot extract object %s:%s; len = %s; filter = %s: %s',
757 obj.major, obj.minor, len, dict.Filter, filter_err)
758 end
759 else
760 lua_util.debugm(N, task, 'cannot extract object %s:%s; len = %s',
761 obj.major, obj.minor, len)
762 end
763 end
764
765 local function parse_object_grammar(obj, task, pdf)
766 -- Parse grammar
767 local obj_dict_span
768 if obj.stream then
769 obj_dict_span = obj.data:span(1, obj.stream.start - obj.start)
770 else
771 obj_dict_span = obj.data
772 end
773
774 if obj_dict_span:len() < config.max_processing_size then
775 local ret, obj_or_err = pcall(pdf_outer_grammar.match, pdf_outer_grammar, obj_dict_span)
776
777 if ret then
778 if obj.stream then
779 if type(obj_or_err) == 'table' then
780 obj.dict = obj_or_err
781 else
782 obj.dict = {}
783 end
784
785 lua_util.debugm(N, task, 'stream object %s:%s is parsed to: %s',
786 obj.major, obj.minor, obj_or_err)
787 else
788 -- Direct object
789 if type(obj_or_err) == 'table' then
790 obj.dict = obj_or_err
791 obj.uncompressed = obj_or_err
792 lua_util.debugm(N, task, 'direct object %s:%s is parsed to: %s',
793 obj.major, obj.minor, obj_or_err)
794 pdf.ref[obj_ref(obj.major, obj.minor)] = obj
795 else
796 lua_util.debugm(N, task, 'direct object %s:%s is parsed to raw data: %s',
797 obj.major, obj.minor, obj_or_err)
798 pdf.ref[obj_ref(obj.major, obj.minor)] = obj_or_err
799 obj.dict = {}
800 obj.uncompressed = obj_or_err
801 end
802 end
803 else
804 lua_util.debugm(N, task, 'object %s:%s cannot be parsed: %s',
805 obj.major, obj.minor, obj_or_err)
806 end
807 else
808 lua_util.debugm(N, task, 'object %s:%s cannot be parsed: too large %s',
809 obj.major, obj.minor, obj_dict_span:len())
810 end
811 end
812
813 -- Extracts font data and process /ToUnicode mappings
814 -- NYI in fact as cmap is ridiculously stupid and complicated
815 --[[
816 local function process_font(task, pdf, font, fname)
817 local dict = font
818 if font.dict then
819 dict = font.dict
820 end
821
822 if type(dict) == 'table' and dict.ToUnicode then
823 local cmap = maybe_dereference_object(dict.ToUnicode, pdf, task)
824
825 if cmap and cmap.dict then
826 maybe_extract_object_stream(cmap, pdf, task)
827 lua_util.debugm(N, task, 'found cmap for font %s: %s',
828 fname, cmap.uncompressed)
829 end
830 end
831 end
832 --]]
833
834 -- Forward declaration
835 local process_dict
836
837 -- This function processes javascript string and returns JS hash and JS rspamd_text
838 local function process_javascript(task, pdf, js, obj)
839 local rspamd_cryptobox_hash = require "rspamd_cryptobox_hash"
840 if type(js) == 'string' then
841 js = rspamd_text.fromstring(js):oneline()
842 elseif type(js) == 'userdata' then
843 js = js:oneline()
844 else
845 return nil
846 end
847
848 local hash = rspamd_cryptobox_hash.create(js)
849 local bin_hash = hash:bin()
850
851 if not pdf.scripts then
852 pdf.scripts = {}
853 end
854
855 if pdf.scripts[bin_hash] then
856 -- Duplicate
857 return pdf.scripts[bin_hash]
858 end
859
860 local njs = {
861 data = js,
862 hash = hash:hex(),
863 bin_hash = bin_hash,
864 object = obj,
865 }
866 pdf.scripts[bin_hash] = njs
867 return njs
868 end
869
870 -- Extract interesting stuff from /Action, e.g. javascript
871 local function process_action(task, pdf, obj)
872 if not (obj.js or obj.launch) and (obj.dict and obj.dict.JS) then
873 local js = maybe_dereference_object(obj.dict.JS, pdf, task)
874
875 if js then
876 if type(js) == 'table' then
877 local extracted_js = maybe_extract_object_stream(js, pdf, task)
878
879 if not extracted_js then
880 lua_util.debugm(N, task, 'invalid type for JavaScript from %s:%s: %s',
881 obj.major, obj.minor, js)
882 else
883 js = extracted_js
884 end
885 end
886
887 js = process_javascript(task, pdf, js, obj)
888 if js then
889 obj.js = js
890 lua_util.debugm(N, task, 'extracted javascript from %s:%s: %s',
891 obj.major, obj.minor, obj.js.data)
892 else
893 lua_util.debugm(N, task, 'invalid type for JavaScript from %s:%s: %s',
894 obj.major, obj.minor, js)
895 end
896 elseif obj.dict.F then
897 local launch = maybe_dereference_object(obj.dict.F, pdf, task)
898
899 if launch then
900 if type(launch) == 'string' then
901 obj.launch = rspamd_text.fromstring(launch):exclude_chars('%n%c')
902 lua_util.debugm(N, task, 'extracted launch from %s:%s: %s',
903 obj.major, obj.minor, obj.launch)
904 elseif type(launch) == 'userdata' then
905 obj.launch = launch:exclude_chars('%n%c')
906 lua_util.debugm(N, task, 'extracted launch from %s:%s: %s',
907 obj.major, obj.minor, obj.launch)
908 else
909 lua_util.debugm(N, task, 'invalid type for launch from %s:%s: %s',
910 obj.major, obj.minor, launch)
911 end
912 end
913 else
914
915 lua_util.debugm(N, task, 'no JS attribute in action %s:%s',
916 obj.major, obj.minor)
917 end
918 end
919 end
920
921 -- Extract interesting stuff from /Catalog, e.g. javascript in /OpenAction
922 local function process_catalog(task, pdf, obj)
923 if obj.dict then
924 if obj.dict.OpenAction then
925 local action = maybe_dereference_object(obj.dict.OpenAction, pdf, task)
926
927 if action and type(action) == 'table' then
928 -- This also processes action js (if not already processed)
929 process_dict(task, pdf, action, action.dict)
930 if action.js then
931 lua_util.debugm(N, task, 'found openaction JS in %s:%s: %s',
932 obj.major, obj.minor, action.js)
933 pdf.openaction = action.js
934 action.js.object = obj
935 elseif action.launch then
936 lua_util.debugm(N, task, 'found openaction launch in %s:%s: %s',
937 obj.major, obj.minor, action.launch)
938 pdf.launch = action.launch
939 else
940 lua_util.debugm(N, task, 'no JS in openaction %s:%s: %s',
941 obj.major, obj.minor, action)
942 end
943 else
944 lua_util.debugm(N, task, 'cannot find openaction %s:%s: %s -> %s',
945 obj.major, obj.minor, obj.dict.OpenAction, action)
946 end
947 else
948 lua_util.debugm(N, task, 'no openaction in catalog %s:%s',
949 obj.major, obj.minor)
950 end
951 end
952 end
953
954 local function process_xref(task, pdf, obj)
955 if obj.dict then
956 if obj.dict.Encrypt then
957 local encrypt = maybe_dereference_object(obj.dict.Encrypt, pdf, task)
958 lua_util.debugm(N, task, 'found encrypt: %s in xref object %s:%s',
959 encrypt, obj.major, obj.minor)
960 pdf.encrypted = true
961 end
962 end
963 end
964
965 process_dict = function(task, pdf, obj, dict)
966 if not obj.type and type(dict) == 'table' then
967 if dict.Type and type(dict.Type) == 'string' then
968 -- Common stuff
969 obj.type = dict.Type
970 end
971
972 if not obj.type then
973 -- Defensive: check obj.dict exists before accessing its fields
974 if obj.dict and type(obj.dict) == 'table' and obj.dict.S and obj.dict.JS then
975 obj.type = 'Javascript'
976 lua_util.debugm(N, task, 'implicit type for JavaScript object %s:%s',
977 obj.major, obj.minor)
978 else
979 lua_util.debugm(N, task, 'no type for %s:%s',
980 obj.major, obj.minor)
981 return
982 end
983 end
984
985 lua_util.debugm(N, task, 'processed stream dictionary for object %s:%s -> %s',
986 obj.major, obj.minor, obj.type)
987 local contents = dict.Contents
988 if contents and type(contents) == 'table' then
989 if contents[1] == '%REF%' then
990 -- Single reference
991 contents = { contents }
992 end
993 obj.contents = {}
994
995 for _, c in ipairs(contents) do
996 local cobj = maybe_dereference_object(c, pdf, task)
997 if cobj and type(cobj) == 'table' then
998 obj.contents[#obj.contents + 1] = cobj
999 cobj.parent = obj
1000 cobj.type = 'content'
1001 end
1002 end
1003
1004 lua_util.debugm(N, task, 'found content objects for %s:%s -> %s',
1005 obj.major, obj.minor, #obj.contents)
1006 end
1007
1008 local resources = dict.Resources
1009 if resources and type(resources) == 'table' then
1010 local res_ref = maybe_dereference_object(resources, pdf, task)
1011
1012 if type(res_ref) ~= 'table' then
1013 lua_util.debugm(N, task, 'cannot parse resources from pdf: %s',
1014 resources)
1015 obj.resources = {}
1016 elseif res_ref.dict then
1017 obj.resources = res_ref.dict
1018 else
1019 obj.resources = {}
1020 end
1021 else
1022 -- Fucking pdf: we need to inherit from parent
1023 resources = {}
1024 if dict.Parent then
1025 local parent = maybe_dereference_object(dict.Parent, pdf, task)
1026
1027 if parent and type(parent) == 'table' and parent.dict then
1028 if parent.resources then
1029 lua_util.debugm(N, task, 'propagated resources from %s:%s to %s:%s',
1030 parent.major, parent.minor, obj.major, obj.minor)
1031 resources = parent.resources
1032 end
1033 end
1034 end
1035
1036 obj.resources = resources
1037 end
1038
1039
1040
1041 --[[Disabled fonts extraction
1042 local fonts = obj.resources.Font
1043 if fonts and type(fonts) == 'table' then
1044 obj.fonts = {}
1045 for k,v in pairs(fonts) do
1046 obj.fonts[k] = maybe_dereference_object(v, pdf, task)
1047
1048 if obj.fonts[k] then
1049 local font = obj.fonts[k]
1050
1051 if config.text_extraction then
1052 process_font(task, pdf, font, k)
1053 lua_util.debugm(N, task, 'found font "%s" for object %s:%s -> %s',
1054 k, obj.major, obj.minor, font)
1055 end
1056 end
1057 end
1058 end
1059 ]]
1060
1061 lua_util.debugm(N, task, 'found resources for object %s:%s (%s): %s',
1062 obj.major, obj.minor, obj.type, obj.resources)
1063
1064 if obj.type == 'Action' then
1065 process_action(task, pdf, obj)
1066 elseif obj.type == 'Catalog' then
1067 process_catalog(task, pdf, obj)
1068 elseif obj.type == 'XRef' then
1069 -- XRef stream instead of trailer from PDF 1.5 (thanks Adobe)
1070 process_xref(task, pdf, obj)
1071 elseif obj.type == 'Javascript' then
1072 local js = maybe_dereference_object(obj.dict.JS, pdf, task)
1073
1074 if js then
1075 if type(js) == 'table' then
1076 local extracted_js = maybe_extract_object_stream(js, pdf, task)
1077
1078 if not extracted_js then
1079 lua_util.debugm(N, task, 'invalid type for JavaScript from %s:%s: %s',
1080 obj.major, obj.minor, js)
1081 else
1082 js = extracted_js
1083 end
1084 end
1085
1086 js = process_javascript(task, pdf, js, obj)
1087 if js then
1088 obj.js = js
1089 lua_util.debugm(N, task, 'extracted javascript from %s:%s: %s',
1090 obj.major, obj.minor, obj.js.data)
1091 else
1092 lua_util.debugm(N, task, 'invalid type for JavaScript from %s:%s: %s',
1093 obj.major, obj.minor, js)
1094 end
1095 end
1096 end
1097 end -- Already processed dict (obj.type is not empty)
1098 end
1099
1100 -- This function is intended to unpack objects from ObjStm crappy structure
1101 local compound_obj_grammar
1102 local function compound_obj_grammar_gen()
1103 if not compound_obj_grammar then
1104 local gen = generic_grammar_elts()
1105 compound_obj_grammar = gen.ws ^ 0 * (gen.comment * gen.ws ^ 1) ^ 0 *
1106 lpeg.Ct(lpeg.Ct(gen.number * gen.ws ^ 1 * gen.number * gen.ws ^ 0) ^ 1)
1107 end
1108
1109 return compound_obj_grammar
1110 end
1111 local function pdf_compound_object_unpack(_, uncompressed, pdf, task, first)
1112 -- First, we need to parse data line by line likely to find a line
1113 -- that consists of pairs of numbers
1114 compound_obj_grammar_gen()
1115 -- Wrap grammar match in pcall for safety
1116 local match_ok, elts = pcall(compound_obj_grammar.match, compound_obj_grammar, uncompressed)
1117 if not match_ok then
1118 lua_util.debugm(N, task, 'compound object grammar match failed: %s', elts)
1119 return
1120 end
1121 if elts and type(elts) == 'table' and #elts > 0 then
1122 lua_util.debugm(N, task, 'compound elts (chunk length %s): %s',
1123 #uncompressed, elts)
1124
1125 for i, pair in ipairs(elts) do
1126 -- Defensive: check pair is a valid table
1127 if type(pair) ~= 'table' or not pair[1] or not pair[2] then
1128 lua_util.debugm(N, task, 'invalid pair in compound object at index %s', i)
1129 else
1130 local obj_number, offset = pair[1], pair[2]
1131
1132 offset = offset + first
1133 if offset < #uncompressed then
1134 local span_len
1135 if i == #elts then
1136 span_len = #uncompressed - offset
1137 else
1138 local next_pair = elts[i + 1]
1139 if type(next_pair) == 'table' and next_pair[2] then
1140 span_len = (next_pair[2] + first) - offset
1141 else
1142 span_len = #uncompressed - offset
1143 end
1144 end
1145
1146 if span_len > 0 and offset + span_len <= #uncompressed then
1147 -- Wrap span extraction in pcall
1148 local span_ok, span_data = pcall(function()
1149 return uncompressed:span(offset + 1, span_len)
1150 end)
1151 if not span_ok or not span_data then
1152 lua_util.debugm(N, task, 'cannot extract span for compound object %s', obj_number)
1153 else
1154 local obj = {
1155 major = obj_number,
1156 minor = 0, -- Implicit
1157 data = span_data,
1158 ref = obj_ref(obj_number, 0)
1159 }
1160 parse_object_grammar(obj, task, pdf)
1161
1162 if obj.dict then
1163 pdf.objects[#pdf.objects + 1] = obj
1164 end
1165 end
1166 else
1167 lua_util.debugm(N, task, 'invalid span_len for compound object %s:%s; offset = %s, len = %s',
1168 pair[1], pair[2], offset + span_len, #uncompressed)
1169 end
1170 end
1171 end
1172 end
1173 end
1174 end
1175
1176 -- PDF 1.5 ObjStmt
1177 local function extract_pdf_compound_objects(task, pdf)
1178 for i, obj in ipairs(pdf.objects or {}) do
1179 if i > 0 and i % 100 == 0 then
1180 local now = rspamd_util.get_ticks()
1181
1182 if now >= pdf.end_timestamp then
1183 pdf.timeout_processing = now - pdf.start_timestamp
1184
1185 lua_util.debugm(N, task, 'pdf: timeout processing compound objects after spending %s seconds, ' ..
1186 '%s elements processed',
1187 pdf.timeout_processing, i)
1188 break
1189 end
1190 end
1191 if obj.stream and obj.dict and type(obj.dict) == 'table' then
1192 local t = obj.dict.Type
1193 if t and t == 'ObjStm' then
1194 -- We are in troubles sir...
1195 local nobjs = tonumber(maybe_dereference_object(obj.dict.N, pdf, task))
1196 local first = tonumber(maybe_dereference_object(obj.dict.First, pdf, task))
1197
1198 if nobjs and first then
1199 --local extend = maybe_dereference_object(obj.dict.Extends, pdf, task)
1200 lua_util.debugm(N, task, 'extract ObjStm with %s objects (%s first) %s extend',
1201 nobjs, first, obj.dict.Extends)
1202
1203 local uncompressed = maybe_extract_object_stream(obj, pdf, task)
1204
1205 if uncompressed then
1206 pdf_compound_object_unpack(obj, uncompressed, pdf, task, first)
1207 end
1208 else
1209 lua_util.debugm(N, task, 'ObjStm object %s:%s has bad dict: %s',
1210 obj.major, obj.minor, obj.dict)
1211 end
1212 end
1213 end
1214 end
1215 end
1216
1217 -- This function arranges starts and ends of all objects and process them into initial
1218 -- set of objects
1219 local function extract_outer_objects(task, input, pdf)
1220 local start_pos, end_pos = 1, 1
1221 local obj_count = 0
1222 local stored = 0
1223 local total_start = #pdf.start_objects
1224 local total_end = #pdf.end_objects
1225
1226 lua_util.debugm(N, task, "pdf: extract objects from %s start positions and %s end positions",
1227 total_start, total_end)
1228
1229 while start_pos <= total_start and end_pos <= total_end do
1230 -- Timeout check every 500 iterations
1231 if start_pos % 500 == 0 then
1232 local now = rspamd_util.get_ticks()
1233 if now >= pdf.end_timestamp then
1234 pdf.timeout_processing = now - pdf.start_timestamp
1235 lua_util.debugm(N, task, 'pdf: timeout extracting objects after %s seconds, ' ..
1236 '%s stored, %s/%s positions',
1237 pdf.timeout_processing, stored, start_pos, total_start)
1238 break
1239 end
1240 end
1241
1242 local first = pdf.start_objects[start_pos]
1243 local last = pdf.end_objects[end_pos]
1244
1245 -- 7 is length of `endobj\n`
1246 if first + 6 < last then
1247 local len = last - first - 6
1248
1249 -- Only count non-tiny objects toward the limit; small objects (e.g. padding)
1250 -- are still stored but don't consume the budget
1251 if len >= config.min_obj_content_size then
1252 if obj_count >= config.max_pdf_objects then
1253 break
1254 end
1255 obj_count = obj_count + 1
1256 end
1257
1258 -- Also get the starting span and try to match it versus obj re to get numbers
1259 local obj_line_potential = first - 32
1260 if obj_line_potential < 1 then
1261 obj_line_potential = 1
1262 end
1263 local prev_obj_end = pdf.end_objects[end_pos - 1]
1264 if end_pos > 1 and prev_obj_end >= obj_line_potential and prev_obj_end < first then
1265 obj_line_potential = prev_obj_end + 1
1266 end
1267
1268 local obj_line_span = input:span(obj_line_potential, first - obj_line_potential + 1)
1269 local matches = object_re:search(obj_line_span, true, true)
1270
1271 if matches and matches[1] then
1272 stored = stored + 1
1273 local nobj = {
1274 start = first,
1275 len = len,
1276 data = input:span(first, len),
1277 major = tonumber(matches[1][2]),
1278 minor = tonumber(matches[1][3]),
1279 }
1280 pdf.objects[stored] = nobj
1281 if nobj.major and nobj.minor then
1282 -- Add reference
1283 local ref = obj_ref(nobj.major, nobj.minor)
1284 nobj.ref = ref -- Our internal reference
1285 pdf.ref[ref] = nobj
1286 end
1287 end
1288
1289 start_pos = start_pos + 1
1290 end_pos = end_pos + 1
1291 elseif first > last then
1292 end_pos = end_pos + 1
1293 else
1294 start_pos = start_pos + 1
1295 end_pos = end_pos + 1
1296 end
1297 end
1298
1299 lua_util.debugm(N, task, 'pdf: stored %s objects (%s non-tiny toward limit) from %s positions',
1300 stored, obj_count, total_start)
1301 end
1302
1303 -- This function attaches streams to objects and processes outer pdf grammar
1304 local function attach_pdf_streams(task, input, pdf)
1305 if pdf.start_streams and pdf.end_streams then
1306 local start_pos, end_pos = 1, 1
1307 local total_start = #pdf.start_streams
1308 local total_end = #pdf.end_streams
1309 local iter_count = 0
1310
1311 for _, obj in ipairs(pdf.objects) do
1312 while start_pos <= total_start and end_pos <= total_end do
1313 -- Timeout check every 500 iterations
1314 iter_count = iter_count + 1
1315 if iter_count % 500 == 0 then
1316 local now = rspamd_util.get_ticks()
1317 if now >= pdf.end_timestamp then
1318 pdf.timeout_processing = now - pdf.start_timestamp
1319 lua_util.debugm(N, task, 'pdf: timeout attaching streams after %s seconds',
1320 pdf.timeout_processing)
1321 return
1322 end
1323 end
1324
1325 local first = pdf.start_streams[start_pos]
1326 local last = pdf.end_streams[end_pos]
1327 last = last - 10 -- Exclude endstream\n pattern
1328 lua_util.debugm(N, task, "start: %s, end: %s; obj: %s-%s",
1329 first, last, obj.start, obj.start + obj.len)
1330 if first > obj.start and last < obj.start + obj.len and last > first then
1331 -- In case if we have fake endstream :(
1332 while pdf.end_streams[end_pos + 1] and pdf.end_streams[end_pos + 1] < obj.start + obj.len do
1333 end_pos = end_pos + 1
1334 last = pdf.end_streams[end_pos]
1335 end
1336 -- Strip the first \n
1337 while first < last do
1338 local chr = input:byte(first)
1339 if chr ~= 13 and chr ~= 10 then
1340 break
1341 end
1342 first = first + 1
1343 end
1344 local len = last - first
1345 obj.stream = {
1346 start = first,
1347 len = len,
1348 data = input:span(first, len)
1349 }
1350 start_pos = start_pos + 1
1351 end_pos = end_pos + 1
1352 break
1353 elseif first < obj.start then
1354 start_pos = start_pos + 1
1355 elseif last > obj.start + obj.len then
1356 -- Not this object
1357 break
1358 else
1359 start_pos = start_pos + 1
1360 end_pos = end_pos + 1
1361 end
1362 end
1363 if obj.stream then
1364 lua_util.debugm(N, task, 'found object %s:%s %s start %s len, %s stream start, %s stream length',
1365 obj.major, obj.minor, obj.start, obj.len, obj.stream.start, obj.stream.len)
1366 else
1367 lua_util.debugm(N, task, 'found object %s:%s %s start %s len, no stream',
1368 obj.major, obj.minor, obj.start, obj.len)
1369 end
1370 end
1371 end
1372 end
1373
1374 -- Processes PDF objects: extracts streams, object numbers, process outer grammar,
1375 -- augment object types
1376 local function postprocess_pdf_objects(task, input, pdf)
1377 pdf.objects = {} -- objects table
1378 pdf.ref = {} -- references table
1379 extract_outer_objects(task, input, pdf)
1380
1381 -- Now we have objects and we need to attach streams that are in bounds
1382 attach_pdf_streams(task, input, pdf)
1383 -- Parse grammar for outer objects
1384 for i, obj in ipairs(pdf.objects) do
1385 if i > 0 and i % 100 == 0 then
1386 local now = rspamd_util.get_ticks()
1387
1388 if now >= pdf.end_timestamp then
1389 pdf.timeout_processing = now - pdf.start_timestamp
1390 lua_util.debugm(N, task, 'pdf: timeout processing grammars after spending %s seconds, ' ..
1391 '%s elements processed',
1392 pdf.timeout_processing, i)
1393 break
1394 end
1395 end
1396 if obj.ref then
1397 parse_object_grammar(obj, task, pdf)
1398
1399 -- Special early handling
1400 if obj.dict and obj.dict.Type and obj.dict.Type == 'XRef' then
1401 process_xref(task, pdf, obj)
1402 end
1403 end
1404 end
1405
1406 if not pdf.timeout_processing then
1407 extract_pdf_compound_objects(task, pdf)
1408 else
1409 -- ENOTIME
1410 return
1411 end
1412
1413 -- Now we might probably have all objects being processed
1414 for i, obj in ipairs(pdf.objects) do
1415 if obj.dict then
1416 -- Types processing
1417 if i > 0 and i % 100 == 0 then
1418 local now = rspamd_util.get_ticks()
1419
1420 if now >= pdf.end_timestamp then
1421 pdf.timeout_processing = now - pdf.start_timestamp
1422
1423 lua_util.debugm(N, task, 'pdf: timeout processing dicts after spending %s seconds, ' ..
1424 '%s elements processed',
1425 pdf.timeout_processing, i)
1426 break
1427 end
1428 end
1429 process_dict(task, pdf, obj, obj.dict)
1430 end
1431 end
1432 end
1433
1434 local function offsets_to_blocks(starts, ends, out)
1435 local start_pos, end_pos = 1, 1
1436
1437 while start_pos <= #starts and end_pos <= #ends do
1438 local first = starts[start_pos]
1439 local last = ends[end_pos]
1440
1441 if first < last then
1442 local len = last - first
1443 out[#out + 1] = {
1444 start = first,
1445 len = len,
1446 }
1447 start_pos = start_pos + 1
1448 end_pos = end_pos + 1
1449 elseif first > last then
1450 end_pos = end_pos + 1
1451 else
1452 -- Not ordered properly!
1453 break
1454 end
1455 end
1456 end
1457
1458 local function search_text(task, pdf, mpart)
1459 for _, obj in ipairs(pdf.objects) do
1460 if obj.type == 'Page' and obj.contents then
1461 local text = {}
1462 for _, tobj in ipairs(obj.contents) do
1463 maybe_extract_object_stream(tobj, pdf, task)
1464 -- Defensive: ensure uncompressed data is usable
1465 local uncompressed = tobj.uncompressed
1466 if not uncompressed then
1467 uncompressed = ''
1468 end
1469 -- Wrap trie match in pcall to handle unexpected input
1470 local match_ok, matches = pcall(pdf_text_trie.match, pdf_text_trie, uncompressed)
1471 if not match_ok then
1472 lua_util.debugm(N, task, 'trie match failed for object %s:%s: %s',
1473 tobj.major, tobj.minor, matches)
1474 matches = nil
1475 end
1476 if matches and type(matches) == 'table' then
1477 local text_blocks = {}
1478 local starts = {}
1479 local ends = {}
1480
1481 for npat, matched_positions in pairs(matches) do
1482 if type(matched_positions) ~= 'table' then
1483 -- Skip malformed match results
1484 lua_util.debugm(N, task, 'skipping malformed trie match result: %s', type(matched_positions))
1485 elseif npat == 1 then
1486 for _, pos in ipairs(matched_positions) do
1487 starts[#starts + 1] = pos
1488 end
1489 else
1490 for _, pos in ipairs(matched_positions) do
1491 ends[#ends + 1] = pos
1492 end
1493 end
1494 end
1495
1496 table.sort(starts)
1497 table.sort(ends)
1498
1499 offsets_to_blocks(starts, ends, text_blocks)
1500 for _, bl in ipairs(text_blocks) do
1501 if bl.len and bl.len > 2 then
1502 -- To remove \s+ET\b pattern (it can leave trailing space or not but it doesn't matter)
1503 bl.len = bl.len - 2
1504 end
1505 -- Defensive: wrap span extraction in pcall
1506 local span_ok, span_data = pcall(function()
1507 if type(uncompressed) == 'userdata' or type(uncompressed) == 'string' then
1508 return uncompressed:span(bl.start, bl.len)
1509 end
1510 return nil
1511 end)
1512 if not span_ok or not span_data then
1513 lua_util.debugm(N, task, 'cannot extract text span from object %s:%s',
1514 tobj.major, tobj.minor)
1515 else
1516 bl.data = span_data
1517 if bl.len <= 256 then
1518 lua_util.debugm(N, task, 'extracted text from object %s:%s: %s',
1519 tobj.major, tobj.minor, bl.data)
1520 else
1521 lua_util.debugm(N, task, 'extracted text from object %s:%s (%d bytes)',
1522 tobj.major, tobj.minor, bl.len)
1523 end
1524
1525 if bl.len < config.max_processing_size then
1526 local ret, obj_or_err = pcall(pdf_text_grammar.match, pdf_text_grammar,
1527 bl.data)
1528
1529 if ret and type(obj_or_err) == 'table' then
1530 if #obj_or_err == 0 then
1531 lua_util.debugm(N, task, 'empty text match from block: %s', bl.data)
1532 end
1533 for _, chunk in ipairs(obj_or_err) do
1534 text[#text + 1] = chunk
1535 end
1536 text[#text + 1] = '\n'
1537 lua_util.debugm(N, task, 'attached %s from content object %s:%s to %s:%s',
1538 obj_or_err, tobj.major, tobj.minor, obj.major, obj.minor)
1539 else
1540 lua_util.debugm(N, task, 'object %s:%s cannot be parsed: %s',
1541 obj.major, obj.minor, obj_or_err)
1542 end
1543 end
1544 end
1545 end
1546 end
1547 end
1548
1549 -- Join all text data together
1550 if #text > 0 then
1551 for i, chunk in ipairs(text) do
1552 if type(chunk) == 'userdata' then
1553 text[i] = tostring(chunk)
1554 elseif type(chunk) == 'table' then
1555 local function flatten(t)
1556 local res = {}
1557 local stack = { { tbl = t, idx = 1 } }
1558 local max_depth = 100
1559
1560 while #stack > 0 and #stack <= max_depth do
1561 local frame = stack[#stack]
1562 local tbl, idx = frame.tbl, frame.idx
1563
1564 if idx > #tbl then
1565 stack[#stack] = nil
1566 else
1567 local v = tbl[idx]
1568 frame.idx = idx + 1
1569
1570 if type(v) == 'userdata' then
1571 res[#res + 1] = tostring(v)
1572 elseif type(v) == 'table' then
1573 stack[#stack + 1] = { tbl = v, idx = 1 }
1574 elseif v ~= nil then
1575 res[#res + 1] = tostring(v)
1576 end
1577 end
1578 end
1579
1580 return table.concat(res, '')
1581 end
1582 text[i] = flatten(chunk)
1583 end
1584 end
1585 local res = table.concat(text, '')
1586
1587 -- Page-level confidence check before storing text
1588 if config.text_quality_enabled and #res >= config.text_quality_min_length then
1589 local page_confidence = calculate_text_confidence(res)
1590 if page_confidence < config.text_quality_threshold then
1591 lua_util.debugm(N, task, 'skipping low confidence page text for %s:%s (%.2f)',
1592 obj.major, obj.minor, page_confidence)
1593 -- Don't store this page's text
1594 else
1595 obj.text = rspamd_text.fromstring(res)
1596 lua_util.debugm(N, task, 'object %s:%s is parsed (confidence: %.2f): %s',
1597 obj.major, obj.minor, page_confidence, obj.text)
1598 end
1599 else
1600 obj.text = rspamd_text.fromstring(res)
1601 lua_util.debugm(N, task, 'object %s:%s is parsed to: %s',
1602 obj.major, obj.minor, obj.text)
1603 end
1604 end
1605 end
1606 end
1607
1608 if task.inject_part then
1609 local all_text = {}
1610
1611 for _, obj in ipairs(pdf.objects) do
1612 if obj.text and obj.text:len() > 0 then
1613 table.insert(all_text, obj.text)
1614 end
1615 end
1616
1617 if #all_text > 0 then
1618 task:inject_part('text', all_text, mpart)
1619 end
1620 end
1621 end
1622
1623 -- This function searches objects for `/URI` key and parses it's content
1624 local function search_urls(task, pdf, mpart)
1625 local function recursive_object_traverse(obj, dict, rec)
1626 if rec > 10 then
1627 lua_util.debugm(N, task, 'object %s:%s recurses too much',
1628 obj.major, obj.minor)
1629 return
1630 end
1631 -- Defensive: ensure dict is actually a table we can iterate
1632 if type(dict) ~= 'table' then
1633 return
1634 end
1635
1636 for k, v in pairs(dict) do
1637 if type(v) == 'table' then
1638 recursive_object_traverse(obj, v, rec + 1)
1639 elseif k == 'URI' then
1640 v = maybe_dereference_object(v, pdf, task)
1641 if type(v) == 'string' then
1642 -- Wrap URL creation in pcall to handle malformed URLs
1643 local url_ok, url = pcall(rspamd_url.create, task:get_mempool(), v, { 'content' })
1644
1645 if url_ok and url then
1646 lua_util.debugm(N, task, 'found url %s in object %s:%s',
1647 v, obj.major, obj.minor)
1648 task:inject_url(url, mpart)
1649 end
1650 elseif type(v) == 'userdata' then
1651 -- Handle rspamd_text objects
1652 local str_ok, str_v = pcall(tostring, v)
1653 if str_ok and str_v then
1654 local url_ok, url = pcall(rspamd_url.create, task:get_mempool(), str_v, { 'content' })
1655 if url_ok and url then
1656 lua_util.debugm(N, task, 'found url %s in object %s:%s',
1657 str_v, obj.major, obj.minor)
1658 task:inject_url(url, mpart)
1659 end
1660 end
1661 end
1662 end
1663 end
1664 end
1665
1666 for _, obj in ipairs(pdf.objects) do
1667 if obj.dict and type(obj.dict) == 'table' then
1668 -- Wrap the traversal in pcall to handle any unexpected errors
1669 local ok, err = pcall(recursive_object_traverse, obj, obj.dict, 0)
1670 if not ok then
1671 lua_util.debugm(N, task, 'error traversing object %s:%s for URLs: %s',
1672 obj.major, obj.minor, err)
1673 end
1674 end
1675 end
1676 end
1677
1678 local function process_pdf(input, mpart, task)
1679 if not config.enabled then
1680 -- Skip processing
1681 return {}
1682 end
1683
1684 local matches = pdf_trie:match(input)
1685
1686 if matches then
1687 local start_ts = rspamd_util.get_ticks()
1688 -- Temp object used to share data between pdf extraction methods
1689 local pdf_object = {
1690 tag = 'pdf',
1691 extract_text = extract_text_data,
1692 start_timestamp = start_ts,
1693 end_timestamp = start_ts + config.pdf_process_timeout,
1694 }
1695 -- Output object that excludes all internal stuff
1696 local pdf_output = lua_util.shallowcopy(pdf_object)
1697 local grouped_processors = {}
1698 for npat, matched_positions in pairs(matches) do
1699 if type(matched_positions) ~= 'table' then
1700 -- Skip malformed match results
1701 lua_util.debugm(N, task, 'skipping malformed trie match result: %s', type(matched_positions))
1702 else
1703 local index = pdf_indexes[npat]
1704
1705 local proc_key, loc_npat = index[1], index[4]
1706
1707 if not grouped_processors[proc_key] then
1708 grouped_processors[proc_key] = {
1709 processor_func = processors[proc_key],
1710 offsets = {},
1711 }
1712 end
1713 local proc = grouped_processors[proc_key]
1714 -- Fill offsets
1715 for _, pos in ipairs(matched_positions) do
1716 proc.offsets[#proc.offsets + 1] = { pos, loc_npat }
1717 end
1718 end
1719 end
1720
1721 for name, processor in pairs(grouped_processors) do
1722 -- Sort by offset
1723 lua_util.debugm(N, task, "pdf: process group %s with %s matches",
1724 name, #processor.offsets)
1725 table.sort(processor.offsets, function(e1, e2)
1726 return e1[1] < e2[1]
1727 end)
1728 -- Wrap processor call in pcall to handle any errors gracefully
1729 local proc_ok, proc_err = pcall(processor.processor_func, input, task, processor.offsets, pdf_object, pdf_output)
1730 if not proc_ok then
1731 lua_util.debugm(N, task, "pdf: processor %s failed: %s", name, proc_err)
1732 end
1733 end
1734
1735 pdf_output.flags = {}
1736
1737 if pdf_object.start_objects and pdf_object.end_objects then
1738 if #pdf_object.start_objects > config.max_pdf_objects then
1739 pdf_output.many_objects = #pdf_object.start_objects
1740 -- Trim
1741 end
1742
1743 -- Postprocess objects - wrap in pcall for safety
1744 local pp_ok, pp_err = pcall(postprocess_pdf_objects, task, input, pdf_object)
1745 if not pp_ok then
1746 lua_util.debugm(N, task, "pdf: postprocess_pdf_objects failed: %s", pp_err)
1747 end
1748 pdf_output.objects = pdf_object.objects
1749 -- Skip text extraction if timeout occurred - partial results would be incorrect
1750 if config.text_extraction and not pdf_object.timeout_processing then
1751 -- Wrap in pcall for safety
1752 local st_ok, st_err = pcall(search_text, task, pdf_object, mpart)
1753 if not st_ok then
1754 lua_util.debugm(N, task, "pdf: search_text failed: %s", st_err)
1755 end
1756 end
1757 if config.url_extraction then
1758 -- Wrap in pcall for safety
1759 local su_ok, su_err = pcall(search_urls, task, pdf_object, mpart, pdf_output)
1760 if not su_ok then
1761 lua_util.debugm(N, task, "pdf: search_urls failed: %s", su_err)
1762 end
1763 end
1764
1765 if config.js_fuzzy and pdf_object.scripts then
1766 pdf_output.fuzzy_hashes = {}
1767 if config.openaction_fuzzy_only then
1768 -- OpenAction only
1769 if pdf_object.openaction and pdf_object.openaction.bin_hash then
1770 if config.min_js_fuzzy and #pdf_object.openaction.data >= config.min_js_fuzzy then
1771 lua_util.debugm(N, task, "pdf: add fuzzy hash from openaction: %s; size = %s; object: %s:%s",
1772 pdf_object.openaction.hash,
1773 #pdf_object.openaction.data,
1774 pdf_object.openaction.object.major, pdf_object.openaction.object.minor)
1775 table.insert(pdf_output.fuzzy_hashes, pdf_object.openaction.bin_hash)
1776 else
1777 lua_util.debugm(N, task, "pdf: skip fuzzy hash from JavaScript: %s, too short: %s",
1778 pdf_object.openaction.hash, #pdf_object.openaction.data)
1779 end
1780 end
1781 else
1782 -- All hashes
1783 for h, sc in pairs(pdf_object.scripts) do
1784 if config.min_js_fuzzy and #sc.data >= config.min_js_fuzzy then
1785 lua_util.debugm(N, task, "pdf: add fuzzy hash from JavaScript: %s; size = %s; object: %s:%s",
1786 sc.hash,
1787 #sc.data,
1788 sc.object.major, sc.object.minor)
1789 table.insert(pdf_output.fuzzy_hashes, h)
1790 else
1791 lua_util.debugm(N, task, "pdf: skip fuzzy hash from JavaScript: %s, too short: %s",
1792 sc.hash, #sc.data)
1793 end
1794 end
1795
1796 end
1797 end
1798 else
1799 pdf_output.flags.no_objects = true
1800 end
1801
1802 -- Propagate from object to output
1803 if pdf_object.encrypted then
1804 pdf_output.encrypted = true
1805 end
1806 if pdf_object.scripts then
1807 pdf_output.scripts = true
1808 end
1809
1810 return pdf_output
1811 end
1812 end
1813
1814 -- Processes the PDF trailer
1815 processors.trailer = function(input, task, positions, pdf_object, pdf_output)
1816 -- Defensive checks
1817 if not positions or #positions == 0 then
1818 return
1819 end
1820 local last_pos = positions[#positions]
1821 if not last_pos or type(last_pos) ~= 'table' or not last_pos[1] then
1822 return
1823 end
1824
1825 lua_util.debugm(N, task, 'pdf: process trailer at position %s (%s total length)',
1826 last_pos, #input)
1827
1828 if last_pos[1] > config.max_pdf_trailer then
1829 pdf_output.long_trailer = #input - last_pos[1]
1830 return
1831 end
1832
1833 -- Wrap span extraction in pcall
1834 local span_ok, last_span = pcall(input.span, input, last_pos[1])
1835 if not span_ok or not last_span then
1836 lua_util.debugm(N, task, 'pdf: cannot extract trailer span')
1837 return
1838 end
1839
1840 local lines_checked = 0
1841 -- Wrap lines iteration in pcall
1842 local iter_ok, iter_err = pcall(function()
1843 for line in last_span:lines(true) do
1844 if line:find('/Encrypt ') then
1845 lua_util.debugm(N, task, "pdf: found encrypted line in trailer: %s",
1846 line)
1847 pdf_output.encrypted = true
1848 pdf_object.encrypted = true
1849 break
1850 end
1851 lines_checked = lines_checked + 1
1852
1853 if lines_checked > config.max_pdf_trailer_lines then
1854 lua_util.debugm(N, task, "pdf: trailer has too many lines, stop checking")
1855 pdf_output.long_trailer = #input - last_pos[1]
1856 break
1857 end
1858 end
1859 end)
1860 if not iter_ok then
1861 lua_util.debugm(N, task, 'pdf: error iterating trailer lines: %s', iter_err)
1862 end
1863 end
1864
1865 processors.suspicious = function(input, task, positions, pdf_object, pdf_output)
1866 -- Defensive check for positions
1867 if not positions or type(positions) ~= 'table' then
1868 return
1869 end
1870
1871 local suspicious_factor = 0.0
1872 local nexec = 0
1873 local nencoded = 0
1874 local close_encoded = 0
1875 local last_encoded
1876 for _, match in ipairs(positions) do
1877 -- Defensive check for match structure
1878 if type(match) ~= 'table' or not match[1] or not match[2] then
1879 goto continue_suspicious
1880 end
1881
1882 if match[2] == 1 then
1883 -- netsh
1884 suspicious_factor = suspicious_factor + 0.5
1885 elseif match[2] == 2 then
1886 nexec = nexec + 1
1887 elseif match[2] == 3 then
1888 -- Wrap input:sub in pcall for safety
1889 local sub_ok, enc_data = pcall(input.sub, input, match[1] - 2, match[1] - 1)
1890 local legal_escape = false
1891
1892 if sub_ok and enc_data then
1893 local strtoul_ok, strtoul_result = pcall(enc_data.strtoul, enc_data)
1894 enc_data = strtoul_ok and strtoul_result or nil
1895
1896 if enc_data then
1897 -- Legit encode cases are non printable characters (e.g. spaces)
1898 if enc_data < 0x21 or enc_data >= 0x7f then
1899 legal_escape = true
1900 end
1901 end
1902 end
1903
1904 if not legal_escape then
1905 nencoded = nencoded + 1
1906
1907 if last_encoded then
1908 if match[1] - last_encoded < 8 then
1909 -- likely consecutive encoded chars, increase factor
1910 close_encoded = close_encoded + 1
1911 end
1912 end
1913 last_encoded = match[1]
1914
1915 end
1916 end
1917 ::continue_suspicious::
1918 end
1919
1920 if nencoded > 10 then
1921 suspicious_factor = suspicious_factor + nencoded / 10
1922 end
1923 if nexec > 1 then
1924 suspicious_factor = suspicious_factor + nexec / 2.0
1925 end
1926 if close_encoded > 4 and nencoded - close_encoded < 5 then
1927 -- Too many close encoded comparing to the total number of encoded characters
1928 suspicious_factor = suspicious_factor + 0.5
1929 end
1930
1931 lua_util.debugm(N, task, 'pdf: found a suspicious patterns: %s exec, %s encoded (%s close), ' ..
1932 '%s final factor',
1933 nexec, nencoded, close_encoded, suspicious_factor)
1934
1935 if suspicious_factor > 1.0 then
1936 suspicious_factor = 1.0
1937 end
1938
1939 pdf_output.suspicious = suspicious_factor
1940 end
1941
1942 local function generic_table_inserter(positions, pdf_object, output_key)
1943 -- Defensive checks
1944 if not positions or type(positions) ~= 'table' then
1945 return
1946 end
1947 if not pdf_object or type(pdf_object) ~= 'table' then
1948 return
1949 end
1950 if not pdf_object[output_key] then
1951 pdf_object[output_key] = {}
1952 end
1953 local shift = #pdf_object[output_key]
1954 for i, pos in ipairs(positions) do
1955 -- Check pos is a table with valid first element
1956 if type(pos) == 'table' and pos[1] then
1957 pdf_object[output_key][i + shift] = pos[1]
1958 end
1959 end
1960 end
1961
1962 processors.start_object = function(_, task, positions, pdf_object)
1963 generic_table_inserter(positions, pdf_object, 'start_objects')
1964 end
1965
1966 processors.end_object = function(_, task, positions, pdf_object)
1967 generic_table_inserter(positions, pdf_object, 'end_objects')
1968 end
1969
1970 processors.start_stream = function(_, task, positions, pdf_object)
1971 generic_table_inserter(positions, pdf_object, 'start_streams')
1972 end
1973
1974 processors.end_stream = function(_, task, positions, pdf_object)
1975 generic_table_inserter(positions, pdf_object, 'end_streams')
1976 end
1977
1978 exports.process = process_pdf
1979
1980 return exports