| 1 |
--[[ |
| 2 |
Copyright (c) 2022, Vsevolod Stakhov <[email protected]> |
| 3 |
|
| 4 |
Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 |
you may not use this file except in compliance with the License. |
| 6 |
You may obtain a copy of the License at |
| 7 |
|
| 8 |
http://www.apache.org/licenses/LICENSE-2.0 |
| 9 |
|
| 10 |
Unless required by applicable law or agreed to in writing, software |
| 11 |
distributed under the License is distributed on an "AS IS" BASIS, |
| 12 |
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 |
See the License for the specific language governing permissions and |
| 14 |
limitations under the License. |
| 15 |
]]-- |
| 16 |
|
| 17 |
--[[[ |
| 18 |
-- @module lua_content/pdf |
| 19 |
-- This module contains some heuristics for PDF files |
| 20 |
--]] |
| 21 |
|
| 22 |
local rspamd_trie = require "rspamd_trie" |
| 23 |
local rspamd_util = require "rspamd_util" |
| 24 |
local rspamd_text = require "rspamd_text" |
| 25 |
local rspamd_url = require "rspamd_url" |
| 26 |
local bit = require "bit" |
| 27 |
local N = "lua_content" |
| 28 |
local lua_util = require "lua_util" |
| 29 |
local rspamd_regexp = require "rspamd_regexp" |
| 30 |
local lpeg = require "lpeg" |
| 31 |
local pdf_patterns = { |
| 32 |
trailer = { |
| 33 |
patterns = { |
| 34 |
[[\ntrailer\r?\n]] |
| 35 |
} |
| 36 |
}, |
| 37 |
suspicious = { |
| 38 |
patterns = { |
| 39 |
[[netsh\s]], |
| 40 |
[[echo\s]], |
| 41 |
[=[\/[A-Za-z]*#\d\d[#A-Za-z<>/\s]]=], -- Hex encode obfuscation |
| 42 |
} |
| 43 |
}, |
| 44 |
start_object = { |
| 45 |
patterns = { |
| 46 |
[=[[\r\n\0]\s*\d+\s+\d+\s+obj[\s<]]=] |
| 47 |
} |
| 48 |
}, |
| 49 |
end_object = { |
| 50 |
patterns = { |
| 51 |
[=[endobj[\r\n]]=] |
| 52 |
} |
| 53 |
}, |
| 54 |
start_stream = { |
| 55 |
patterns = { |
| 56 |
[=[>\s*stream[\r\n]]=], |
| 57 |
} |
| 58 |
}, |
| 59 |
end_stream = { |
| 60 |
patterns = { |
| 61 |
[=[endstream[\r\n]]=] |
| 62 |
} |
| 63 |
} |
| 64 |
} |
| 65 |
|
| 66 |
local pdf_cmap_patterns = { |
| 67 |
start = { |
| 68 |
patterns = { |
| 69 |
[[\d\s+beginbfchar\s]], |
| 70 |
[[\d\s+beginbfrange\s]] |
| 71 |
} |
| 72 |
}, |
| 73 |
stop = { |
| 74 |
patterns = { |
| 75 |
[[\sendbfrange\b]], |
| 76 |
[[\sendbchar\b]] |
| 77 |
} |
| 78 |
} |
| 79 |
} |
| 80 |
|
| 81 |
-- index[n] -> |
| 82 |
-- t[1] - pattern, |
| 83 |
-- t[2] - key in patterns table, |
| 84 |
-- t[3] - value in patterns table |
| 85 |
-- t[4] - local pattern index |
| 86 |
local pdf_indexes = {} |
| 87 |
local pdf_cmap_indexes = {} |
| 88 |
|
| 89 |
local pdf_trie |
| 90 |
local pdf_text_trie |
| 91 |
local pdf_cmap_trie |
| 92 |
|
| 93 |
local exports = {} |
| 94 |
|
| 95 |
local config = { |
| 96 |
max_extraction_size = 512 * 1024, |
| 97 |
max_processing_size = 32 * 1024, |
| 98 |
text_extraction = true, |
| 99 |
url_extraction = true, |
| 100 |
enabled = true, |
| 101 |
js_fuzzy = true, -- Generate fuzzy hashes from PDF javascripts |
| 102 |
min_js_fuzzy = 256, -- Minimum size of js to be considered as a fuzzy |
| 103 |
openaction_fuzzy_only = false, -- Generate fuzzy from all scripts |
| 104 |
max_pdf_objects = 10000, -- Maximum number of objects to be considered |
| 105 |
min_obj_content_size = 32, -- Skip objects smaller than this (evasion padding) |
| 106 |
max_pdf_trailer = 10 * 1024 * 1024, -- Maximum trailer size (to avoid abuse) |
| 107 |
max_pdf_trailer_lines = 100, -- Maximum number of lines in pdf trailer |
| 108 |
pdf_process_timeout = 2.0, -- Timeout in seconds for processing |
| 109 |
text_quality_threshold = 0.4, -- Minimum confidence to accept extracted text |
| 110 |
text_quality_min_length = 10, -- Minimum text length to apply quality filtering |
| 111 |
text_quality_enabled = true, -- Enable text quality filtering |
| 112 |
} |
| 113 |
|
| 114 |
-- Used to process patterns found in PDF |
| 115 |
-- positions for functional processors should be a iter/table from trie matcher in form |
| 116 |
---- [{n1, pat_idx1}, ... {nn, pat_idxn}] where |
| 117 |
---- pat_idxn is pattern index and n1 ... nn are match positions |
| 118 |
local processors = {} |
| 119 |
-- PDF objects outer grammar in LPEG style (performing table captures) |
| 120 |
local pdf_outer_grammar |
| 121 |
local pdf_text_grammar |
| 122 |
|
| 123 |
-- Used to match objects |
| 124 |
local object_re = rspamd_regexp.create_cached([=[/(\d+)\s+(\d+)\s+obj\s*/]=]) |
| 125 |
|
| 126 |
local function config_module() |
| 127 |
local opts = rspamd_config:get_all_opt('lua_content') |
| 128 |
|
| 129 |
if opts and opts.pdf then |
| 130 |
config = lua_util.override_defaults(config, opts.pdf) |
| 131 |
end |
| 132 |
end |
| 133 |
|
| 134 |
local function compile_tries() |
| 135 |
local default_compile_flags = bit.bor(rspamd_trie.flags.re, |
| 136 |
rspamd_trie.flags.dot_all, |
| 137 |
rspamd_trie.flags.no_start) |
| 138 |
local function compile_pats(patterns, indexes, compile_flags) |
| 139 |
local strs = {} |
| 140 |
for what, data in pairs(patterns) do |
| 141 |
for i, pat in ipairs(data.patterns) do |
| 142 |
strs[#strs + 1] = pat |
| 143 |
indexes[#indexes + 1] = { what, data, pat, i } |
| 144 |
end |
| 145 |
end |
| 146 |
|
| 147 |
return rspamd_trie.create(strs, compile_flags or default_compile_flags) |
| 148 |
end |
| 149 |
|
| 150 |
if not pdf_trie then |
| 151 |
pdf_trie = compile_pats(pdf_patterns, pdf_indexes) |
| 152 |
end |
| 153 |
if not pdf_text_trie then |
| 154 |
pdf_text_trie = rspamd_trie.create({ |
| 155 |
[[\sBT\s]], |
| 156 |
[[\sET\b]] |
| 157 |
}, default_compile_flags) |
| 158 |
end |
| 159 |
if not pdf_cmap_trie then |
| 160 |
pdf_cmap_trie = compile_pats(pdf_cmap_patterns, pdf_cmap_indexes) |
| 161 |
end |
| 162 |
end |
| 163 |
|
| 164 |
-- StandardEncoding/MacRomanEncoding ligature substitutions |
| 165 |
-- Applied only to rendered text, NOT to dictionary strings (URI values etc.) |
| 166 |
-- to avoid corrupting soft hyphens (U+00AD = byte 0xAD = \173) in URLs |
| 167 |
local function apply_ligature_substitutions(s) |
| 168 |
if not s then return s end |
| 169 |
s = s:gsub('\171', 'ff') |
| 170 |
s = s:gsub('\172', 'ffi') |
| 171 |
s = s:gsub('\173', 'ffl') |
| 172 |
s = s:gsub('\174', 'fi') |
| 173 |
s = s:gsub('\175', 'fl') |
| 174 |
s = s:gsub('\222', 'fi') |
| 175 |
s = s:gsub('\223', 'fl') |
| 176 |
return s |
| 177 |
end |
| 178 |
|
| 179 |
-- Returns a table with generic grammar elements for PDF |
| 180 |
local function generic_grammar_elts() |
| 181 |
local P = lpeg.P |
| 182 |
local R = lpeg.R |
| 183 |
local S = lpeg.S |
| 184 |
local V = lpeg.V |
| 185 |
local C = lpeg.C |
| 186 |
local D = R '09' -- Digits |
| 187 |
|
| 188 |
local grammar_elts = {} |
| 189 |
|
| 190 |
-- Helper functions |
| 191 |
local function pdf_hexstring_unescape(s) |
| 192 |
local res |
| 193 |
if #s % 2 == 0 then |
| 194 |
-- Sane hex string |
| 195 |
res = lua_util.unhex(s) |
| 196 |
else |
| 197 |
-- WTF hex string |
| 198 |
-- Append '0' to it and unescape... |
| 199 |
res = lua_util.unhex(s:sub(1, #s - 1)) .. lua_util.unhex((s:sub(#s) .. '0')) |
| 200 |
end |
| 201 |
|
| 202 |
return res |
| 203 |
end |
| 204 |
|
| 205 |
local function pdf_string_unescape(s) |
| 206 |
local function ue_single(cc) |
| 207 |
if cc == '\\r' then |
| 208 |
return '\r' |
| 209 |
elseif cc == '\\n' then |
| 210 |
return '\n' |
| 211 |
else |
| 212 |
return cc:gsub(2, 2) |
| 213 |
end |
| 214 |
end |
| 215 |
-- simple unescape \char |
| 216 |
s = s:gsub('\\[^%d]', ue_single) |
| 217 |
-- unescape octal |
| 218 |
local function ue_octal(cc) |
| 219 |
-- Replace unknown stuff with '?' |
| 220 |
return string.char(tonumber(cc:sub(2), 8) or 63) |
| 221 |
end |
| 222 |
s = s:gsub('\\%d%d?%d?', ue_octal) |
| 223 |
|
| 224 |
return s |
| 225 |
end |
| 226 |
|
| 227 |
local function pdf_id_unescape(s) |
| 228 |
return (s:gsub('#%d%d', function(cc) |
| 229 |
return string.char(tonumber(cc:sub(2), 16)) |
| 230 |
end)) |
| 231 |
end |
| 232 |
|
| 233 |
local delim = S '()<>[]{}/%' |
| 234 |
grammar_elts.ws = S '\0 \r\n\t\f' |
| 235 |
local hex = R 'af' + R 'AF' + D |
| 236 |
-- Comments. |
| 237 |
local eol = P '\r\n' + '\n' |
| 238 |
local line = (1 - S '\r\n\f') ^ 0 * eol ^ -1 |
| 239 |
grammar_elts.comment = P '%' * line |
| 240 |
|
| 241 |
-- Numbers. |
| 242 |
local sign = S '+-' ^ -1 |
| 243 |
local decimal = D ^ 1 |
| 244 |
local float = D ^ 1 * P '.' * D ^ 0 + P '.' * D ^ 1 |
| 245 |
grammar_elts.number = C(sign * (float + decimal)) / tonumber |
| 246 |
|
| 247 |
-- String |
| 248 |
grammar_elts.str = P { "(" * C(((1 - S "()\\") + (P '\\' * 1) + V(1)) ^ 0) / pdf_string_unescape * ")" } |
| 249 |
grammar_elts.hexstr = P { "<" * C(hex ^ 0) / pdf_hexstring_unescape * ">" } |
| 250 |
|
| 251 |
-- Identifier |
| 252 |
grammar_elts.id = P { '/' * C((1 - (delim + grammar_elts.ws)) ^ 1) / pdf_id_unescape } |
| 253 |
|
| 254 |
-- Booleans (who care about them?) |
| 255 |
grammar_elts.boolean = C(P("true") + P("false")) |
| 256 |
|
| 257 |
-- Stupid references |
| 258 |
grammar_elts.ref = lpeg.Ct { lpeg.Cc("%REF%") * C(D ^ 1) * " " * C(D ^ 1) * " " * "R" } |
| 259 |
|
| 260 |
return grammar_elts |
| 261 |
end |
| 262 |
|
| 263 |
|
| 264 |
-- Generates a grammar to parse outer elements (external objects in PDF notation) |
| 265 |
local function gen_outer_grammar() |
| 266 |
local V = lpeg.V |
| 267 |
local gen = generic_grammar_elts() |
| 268 |
|
| 269 |
return lpeg.P { |
| 270 |
"EXPR"; |
| 271 |
EXPR = gen.ws ^ 0 * V("ELT") ^ 0 * gen.ws ^ 0, |
| 272 |
ELT = V("ARRAY") + V("DICT") + V("ATOM"), |
| 273 |
ATOM = gen.ws ^ 0 * (gen.comment + gen.boolean + gen.ref + |
| 274 |
gen.number + V("STRING") + gen.id) * gen.ws ^ 0, |
| 275 |
DICT = "<<" * gen.ws ^ 0 * lpeg.Cf(lpeg.Ct("") * V("KV_PAIR") ^ 0, rawset) * gen.ws ^ 0 * ">>", |
| 276 |
KV_PAIR = lpeg.Cg(gen.id * gen.ws ^ 0 * V("ELT") * gen.ws ^ 0), |
| 277 |
ARRAY = "[" * gen.ws ^ 0 * lpeg.Ct(V("ELT") ^ 0) * gen.ws ^ 0 * "]", |
| 278 |
STRING = lpeg.P { gen.str + gen.hexstr }, |
| 279 |
} |
| 280 |
end |
| 281 |
|
| 282 |
-- Graphic state in PDF |
| 283 |
local function gen_graphics_unary() |
| 284 |
local P = lpeg.P |
| 285 |
local S = lpeg.S |
| 286 |
|
| 287 |
return P("q") + P("Q") + P("h") |
| 288 |
+ S("WSsFfBb") * P("*") ^ 0 + P("n") |
| 289 |
|
| 290 |
end |
| 291 |
local function gen_graphics_binary() |
| 292 |
local P = lpeg.P |
| 293 |
local S = lpeg.S |
| 294 |
|
| 295 |
return S("gGwJjMi") + |
| 296 |
P("M") + P("ri") + P("gs") + |
| 297 |
P("CS") + P("cs") + P("sh") |
| 298 |
end |
| 299 |
local function gen_graphics_ternary() |
| 300 |
local P = lpeg.P |
| 301 |
local S = lpeg.S |
| 302 |
|
| 303 |
return P("d") + P("m") + S("lm") |
| 304 |
end |
| 305 |
local function gen_graphics_nary() |
| 306 |
local P = lpeg.P |
| 307 |
local S = lpeg.S |
| 308 |
|
| 309 |
return P("SC") + P("sc") + P("SCN") + P("scn") + P("k") + P("K") + P("re") + S("cvy") + |
| 310 |
P("RG") + P("rg") |
| 311 |
end |
| 312 |
|
| 313 |
-- Calculate text quality confidence score using UTF-8 aware analysis |
| 314 |
-- Returns a score between 0.0 (garbage) and 1.0 (high quality text) |
| 315 |
local function calculate_text_confidence(text) |
| 316 |
if not text or #text < config.text_quality_min_length then |
| 317 |
return 1.0 -- Don't filter short text |
| 318 |
end |
| 319 |
|
| 320 |
local stats = rspamd_util.get_text_quality(text) |
| 321 |
if not stats or stats.total == 0 then |
| 322 |
return 0.0 |
| 323 |
end |
| 324 |
|
| 325 |
local score = 0.0 |
| 326 |
local non_ws = stats.total - stats.spaces |
| 327 |
|
| 328 |
-- Printable ratio (weight: 0.25) - target > 0.95 |
| 329 |
local printable_ratio = stats.printable / stats.total |
| 330 |
score = score + math.min(printable_ratio / 0.95, 1.0) * 0.25 |
| 331 |
|
| 332 |
-- Letter ratio (weight: 0.20) - target > 0.6 |
| 333 |
local letter_ratio = 0 |
| 334 |
if non_ws > 0 then |
| 335 |
letter_ratio = stats.letters / non_ws |
| 336 |
end |
| 337 |
score = score + math.min(letter_ratio / 0.6, 1.0) * 0.20 |
| 338 |
|
| 339 |
-- Word ratio (weight: 0.25) - target > 0.7 |
| 340 |
local word_ratio = 0 |
| 341 |
if non_ws > 0 then |
| 342 |
word_ratio = stats.word_chars / non_ws |
| 343 |
end |
| 344 |
score = score + math.min(word_ratio / 0.7, 1.0) * 0.25 |
| 345 |
|
| 346 |
-- Average word length (weight: 0.15) - ideal: 3-10 |
| 347 |
local avg_word_len = 0 |
| 348 |
if stats.words > 0 then |
| 349 |
avg_word_len = stats.word_chars / stats.words |
| 350 |
end |
| 351 |
local word_len_score = 0 |
| 352 |
if avg_word_len >= 3 and avg_word_len <= 10 then |
| 353 |
word_len_score = 1.0 |
| 354 |
elseif avg_word_len >= 2 and avg_word_len < 3 then |
| 355 |
word_len_score = 0.7 |
| 356 |
elseif avg_word_len > 10 and avg_word_len <= 15 then |
| 357 |
word_len_score = 0.5 |
| 358 |
else |
| 359 |
word_len_score = 0.2 |
| 360 |
end |
| 361 |
score = score + word_len_score * 0.15 |
| 362 |
|
| 363 |
-- Space ratio (weight: 0.15) - ideal: 0.08-0.25 |
| 364 |
local space_ratio = stats.spaces / stats.total |
| 365 |
local space_score = 0 |
| 366 |
if space_ratio >= 0.08 and space_ratio <= 0.25 then |
| 367 |
space_score = 1.0 |
| 368 |
elseif space_ratio > 0.25 and space_ratio <= 0.4 then |
| 369 |
space_score = 0.6 |
| 370 |
elseif space_ratio > 0 and space_ratio < 0.08 then |
| 371 |
space_score = 0.5 |
| 372 |
else |
| 373 |
space_score = 0.2 |
| 374 |
end |
| 375 |
score = score + space_score * 0.15 |
| 376 |
|
| 377 |
return score |
| 378 |
end |
| 379 |
|
| 380 |
-- Generates a grammar to parse text blocks (between BT and ET) |
| 381 |
local function gen_text_grammar() |
| 382 |
local V = lpeg.V |
| 383 |
local P = lpeg.P |
| 384 |
local C = lpeg.C |
| 385 |
local gen = generic_grammar_elts() |
| 386 |
|
| 387 |
local function sanitize_pdf_text(s) |
| 388 |
if not s or #s < 4 then return s end |
| 389 |
|
| 390 |
local nulls_odd = 0 |
| 391 |
local nulls_even = 0 |
| 392 |
local len = #s |
| 393 |
|
| 394 |
local limit = math.min(len, 16) |
| 395 |
for i = 1, limit do |
| 396 |
local b = string.byte(s, i) |
| 397 |
if b == 0 then |
| 398 |
if i % 2 == 1 then |
| 399 |
nulls_odd = nulls_odd + 1 |
| 400 |
else |
| 401 |
nulls_even = nulls_even + 1 |
| 402 |
end |
| 403 |
end |
| 404 |
end |
| 405 |
|
| 406 |
if len > 32 then |
| 407 |
for i = len - 15, len do |
| 408 |
local b = string.byte(s, i) |
| 409 |
if b == 0 then |
| 410 |
if i % 2 == 1 then |
| 411 |
nulls_odd = nulls_odd + 1 |
| 412 |
else |
| 413 |
nulls_even = nulls_even + 1 |
| 414 |
end |
| 415 |
end |
| 416 |
end |
| 417 |
elseif len > 16 then |
| 418 |
for i = 17, len do |
| 419 |
local b = string.byte(s, i) |
| 420 |
if b == 0 then |
| 421 |
if i % 2 == 1 then |
| 422 |
nulls_odd = nulls_odd + 1 |
| 423 |
else |
| 424 |
nulls_even = nulls_even + 1 |
| 425 |
end |
| 426 |
end |
| 427 |
end |
| 428 |
end |
| 429 |
|
| 430 |
local total_checked = (len > 32) and 32 or len |
| 431 |
local total_odd = math.ceil(total_checked / 2) |
| 432 |
local total_even = math.floor(total_checked / 2) |
| 433 |
|
| 434 |
if len > 32 then |
| 435 |
total_odd = 16 |
| 436 |
total_even = 16 |
| 437 |
end |
| 438 |
|
| 439 |
local ratio_odd = nulls_odd / total_odd |
| 440 |
local ratio_even = nulls_even / total_even |
| 441 |
local charset |
| 442 |
|
| 443 |
if ratio_odd > 0.8 and ratio_even < 0.2 then |
| 444 |
charset = 'UTF-16BE' |
| 445 |
elseif ratio_even > 0.8 and ratio_odd < 0.2 then |
| 446 |
charset = 'UTF-16LE' |
| 447 |
end |
| 448 |
|
| 449 |
if charset and rspamd_util.to_utf8 then |
| 450 |
local conv = rspamd_util.to_utf8(s, charset) |
| 451 |
if conv then |
| 452 |
local garbage_limit = 0 |
| 453 |
local clen = #conv |
| 454 |
for i = 1, clen do |
| 455 |
local b = conv:byte(i) |
| 456 |
if b < 32 and b ~= 9 and b ~= 10 and b ~= 13 then |
| 457 |
garbage_limit = garbage_limit + 1 |
| 458 |
end |
| 459 |
end |
| 460 |
|
| 461 |
if garbage_limit > 0 then |
| 462 |
return '' |
| 463 |
end |
| 464 |
|
| 465 |
return conv |
| 466 |
end |
| 467 |
end |
| 468 |
|
| 469 |
-- Strip null bytes and control characters from non-UTF-16 text. |
| 470 |
-- PDF hex strings like <0041> produce raw bytes including \x00 which |
| 471 |
-- are not meaningful in extracted text and cause false positives in |
| 472 |
-- sa_raw_body rules matching \x00 patterns. |
| 473 |
local has_control = false |
| 474 |
for i = 1, len do |
| 475 |
local b = s:byte(i) |
| 476 |
if b == 0 or (b < 32 and b ~= 9 and b ~= 10 and b ~= 13) then |
| 477 |
has_control = true |
| 478 |
break |
| 479 |
end |
| 480 |
end |
| 481 |
|
| 482 |
if has_control then |
| 483 |
-- Remove null bytes and other control characters (keep tab, newline, carriage return) |
| 484 |
s = s:gsub('[%z\1-\8\11\12\14-\31]', '') |
| 485 |
if #s == 0 then |
| 486 |
return '' |
| 487 |
end |
| 488 |
end |
| 489 |
|
| 490 |
return s |
| 491 |
end |
| 492 |
|
| 493 |
local function text_op_handler(...) |
| 494 |
local args = { ... } |
| 495 |
local op = args[#args] |
| 496 |
local t = args[#args - 1] |
| 497 |
|
| 498 |
local res = t |
| 499 |
if type(t) == 'table' then |
| 500 |
local tres = {} |
| 501 |
for _, chunk in ipairs(t) do |
| 502 |
if type(chunk) == 'string' then |
| 503 |
table.insert(tres, chunk) |
| 504 |
elseif type(chunk) == 'number' and chunk < -200 then |
| 505 |
table.insert(tres, ' ') |
| 506 |
end |
| 507 |
end |
| 508 |
res = table.concat(tres) |
| 509 |
end |
| 510 |
|
| 511 |
-- Apply ligature substitutions for rendered text only (not dictionary strings like /URI) |
| 512 |
if type(res) == 'string' then |
| 513 |
res = apply_ligature_substitutions(res) |
| 514 |
end |
| 515 |
|
| 516 |
res = sanitize_pdf_text(res) |
| 517 |
|
| 518 |
-- Apply text quality filtering to reject garbage chunks |
| 519 |
if config.text_quality_enabled and res and #res >= config.text_quality_min_length then |
| 520 |
local confidence = calculate_text_confidence(res) |
| 521 |
if confidence < config.text_quality_threshold then |
| 522 |
lua_util.debugm(N, nil, 'rejected low confidence text chunk (%.2f): %s', |
| 523 |
confidence, res:sub(1, 50)) |
| 524 |
return '' |
| 525 |
end |
| 526 |
end |
| 527 |
|
| 528 |
if op == "'" or op == '"' then |
| 529 |
return '\n' .. res |
| 530 |
end |
| 531 |
|
| 532 |
return res |
| 533 |
end |
| 534 |
|
| 535 |
local function nary_op_handler(...) |
| 536 |
local args = { ... } |
| 537 |
local op = args[#args] |
| 538 |
|
| 539 |
if op == 'Tm' then |
| 540 |
return '\n' |
| 541 |
end |
| 542 |
|
| 543 |
return '' |
| 544 |
end |
| 545 |
|
| 546 |
local function ternary_op_handler(...) |
| 547 |
local args = { ... } |
| 548 |
local op = args[#args] |
| 549 |
local a2 = args[#args - 2] |
| 550 |
|
| 551 |
if (op == 'Td' or op == 'TD') and type(a2) == 'number' and a2 ~= 0 then |
| 552 |
return '\n' |
| 553 |
end |
| 554 |
|
| 555 |
return '' |
| 556 |
end |
| 557 |
|
| 558 |
local empty = "" |
| 559 |
local unary_ops = C("T*") / "\n" + |
| 560 |
C(gen_graphics_unary()) / empty |
| 561 |
local binary_ops = P("Tc") + P("Tw") + P("Tz") + P("TL") + P("Tr") + P("Ts") + |
| 562 |
gen_graphics_binary() |
| 563 |
local ternary_ops = P("TD") + P("Td") + gen_graphics_ternary() |
| 564 |
local nary_op = P("Tm") + gen_graphics_nary() |
| 565 |
local text_binary_op = C(P("Tj") + P("TJ") + P("'")) |
| 566 |
local text_quote_op = C(P('"')) |
| 567 |
local font_op = P("Tf") |
| 568 |
|
| 569 |
return lpeg.P { |
| 570 |
"EXPR"; |
| 571 |
EXPR = gen.ws ^ 0 * lpeg.Ct(V("COMMAND") ^ 0), |
| 572 |
COMMAND = (V("UNARY") + V("BINARY") + V("TERNARY") + V("NARY") + V("TEXT") + |
| 573 |
V("FONT") + gen.comment) * gen.ws ^ 0, |
| 574 |
UNARY = unary_ops, |
| 575 |
BINARY = V("ARG") / empty * gen.ws ^ 1 * binary_ops, |
| 576 |
TERNARY = (V("ARG") * gen.ws ^ 1 * V("ARG") * gen.ws ^ 1 * ternary_ops) / ternary_op_handler, |
| 577 |
NARY = lpeg.Ct((V("ARG") * gen.ws ^ 1) ^ 1) * (gen.id / empty * gen.ws ^ 0) ^ -1 * nary_op / nary_op_handler, |
| 578 |
ARG = V("ARRAY") + V("DICT") + V("ATOM"), |
| 579 |
ATOM = (gen.comment + gen.boolean + gen.ref + |
| 580 |
gen.number + V("STRING") + gen.id), |
| 581 |
DICT = "<<" * gen.ws ^ 0 * lpeg.Cf(lpeg.Ct("") * V("KV_PAIR") ^ 0, rawset) * gen.ws ^ 0 * ">>", |
| 582 |
KV_PAIR = lpeg.Cg(gen.id * gen.ws ^ 0 * V("ARG") * gen.ws ^ 0), |
| 583 |
ARRAY = "[" * gen.ws ^ 0 * lpeg.Ct(V("ARG") ^ 0) * gen.ws ^ 0 * "]", |
| 584 |
STRING = lpeg.P { gen.str + gen.hexstr }, |
| 585 |
TEXT = ((V("TEXT_ARG") * gen.ws ^ 0 * text_binary_op) / text_op_handler) + |
| 586 |
((V("ARG") / empty * gen.ws ^ 1 * V("ARG") / empty * gen.ws ^ 1 * V("TEXT_ARG") * gen.ws ^ 0 * text_quote_op) / text_op_handler), |
| 587 |
FONT = (V("FONT_ARG") * gen.ws ^ 1 * (gen.number / empty) * gen.ws ^ 1 * font_op) / empty, |
| 588 |
FONT_ARG = lpeg.Ct(lpeg.Cc("%font%") * gen.id), |
| 589 |
TEXT_ARG = lpeg.Ct(V("STRING")) + V("TEXT_ARRAY"), |
| 590 |
TEXT_ARRAY = "[" * gen.ws ^ 0 * lpeg.Ct((V("TEXT_ARRAY_ELT") * gen.ws ^ 0) ^ 0) * "]", |
| 591 |
TEXT_ARRAY_ELT = gen.number + gen.str + gen.hexstr, |
| 592 |
} |
| 593 |
end |
| 594 |
|
| 595 |
|
| 596 |
-- Call immediately on require |
| 597 |
compile_tries() |
| 598 |
config_module() |
| 599 |
pdf_outer_grammar = gen_outer_grammar() |
| 600 |
pdf_text_grammar = gen_text_grammar() |
| 601 |
|
| 602 |
local function extract_text_data(specific) |
| 603 |
local res = {} |
| 604 |
if specific.objects then |
| 605 |
for _, obj in ipairs(specific.objects) do |
| 606 |
if obj.text then |
| 607 |
if type(obj.text) == 'userdata' then |
| 608 |
res[#res + 1] = tostring(obj.text) |
| 609 |
else |
| 610 |
res[#res + 1] = obj.text |
| 611 |
end |
| 612 |
end |
| 613 |
end |
| 614 |
end |
| 615 |
|
| 616 |
return res |
| 617 |
end |
| 618 |
|
| 619 |
-- Generates index for major/minor pair |
| 620 |
local function obj_ref(major, minor) |
| 621 |
return major * 10.0 + 1.0 / (minor + 1.0) |
| 622 |
end |
| 623 |
|
| 624 |
-- Return indirect object reference (if needed) |
| 625 |
local function maybe_dereference_object(elt, pdf, task) |
| 626 |
if type(elt) == 'table' and elt[1] == '%REF%' then |
| 627 |
local ref = obj_ref(elt[2], elt[3]) |
| 628 |
|
| 629 |
if pdf.ref[ref] then |
| 630 |
-- No recursion! |
| 631 |
return pdf.ref[ref] |
| 632 |
else |
| 633 |
lua_util.debugm(N, task, 'cannot dereference %s:%s -> %s, no object', |
| 634 |
elt[2], elt[3], obj_ref(elt[2], elt[3])) |
| 635 |
return nil |
| 636 |
end |
| 637 |
end |
| 638 |
|
| 639 |
return elt |
| 640 |
end |
| 641 |
|
| 642 |
-- Apply PDF stream filter |
| 643 |
local function apply_pdf_filter(input, filt) |
| 644 |
-- Validate input before processing |
| 645 |
if not input or (type(input) == 'string' and #input == 0) then |
| 646 |
return nil |
| 647 |
end |
| 648 |
|
| 649 |
if filt == 'FlateDecode' or filt == 'Fl' then |
| 650 |
return rspamd_util.inflate(input, config.max_extraction_size) |
| 651 |
elseif filt == 'ASCIIHexDecode' or filt == 'AHx' then |
| 652 |
-- Strip > at the end if present (should be stripped by parser but safety check) |
| 653 |
-- Also strip whitespaces |
| 654 |
local to_decode = input:gsub('%s', '') |
| 655 |
if to_decode:sub(-1) == '>' then |
| 656 |
to_decode = to_decode:sub(1, -2) |
| 657 |
end |
| 658 |
if #to_decode == 0 then |
| 659 |
return nil |
| 660 |
end |
| 661 |
return lua_util.unhex(to_decode) |
| 662 |
elseif filt == 'ASCII85Decode' or filt == 'A85' then |
| 663 |
return rspamd_util.decode_ascii85(input) |
| 664 |
end |
| 665 |
|
| 666 |
return nil |
| 667 |
end |
| 668 |
|
| 669 |
-- Conditionally apply a pipeline of stream filters and return uncompressed data |
| 670 |
local function maybe_apply_filter(dict, data, pdf, task) |
| 671 |
local uncompressed = data |
| 672 |
|
| 673 |
if dict.Filter then |
| 674 |
local filt = dict.Filter |
| 675 |
local filts = {} |
| 676 |
|
| 677 |
if type(filt) == 'string' then |
| 678 |
filts = { filt } |
| 679 |
elseif type(filt) == 'table' then |
| 680 |
filts = filt |
| 681 |
end |
| 682 |
|
| 683 |
if dict.DecodeParms then |
| 684 |
local decode_params = maybe_dereference_object(dict.DecodeParms, pdf, task) |
| 685 |
|
| 686 |
if type(decode_params) == 'table' then |
| 687 |
if decode_params.Predictor then |
| 688 |
local predictor = tonumber(decode_params.Predictor) or 1 |
| 689 |
if predictor > 1 then |
| 690 |
return nil, 'predictor exists: ' .. tostring(predictor) |
| 691 |
end |
| 692 |
end |
| 693 |
end |
| 694 |
end |
| 695 |
|
| 696 |
for _, f in ipairs(filts) do |
| 697 |
local next_uncompressed = apply_pdf_filter(uncompressed, f) |
| 698 |
|
| 699 |
if next_uncompressed then |
| 700 |
uncompressed = next_uncompressed |
| 701 |
else |
| 702 |
return nil, 'filter failed: ' .. tostring(f) |
| 703 |
end |
| 704 |
end |
| 705 |
end |
| 706 |
|
| 707 |
return uncompressed, nil |
| 708 |
end |
| 709 |
|
| 710 |
-- Conditionally extract stream data from object and attach it as obj.uncompressed |
| 711 |
local function maybe_extract_object_stream(obj, pdf, task) |
| 712 |
if pdf.encrypted then |
| 713 |
-- TODO add decryption some day |
| 714 |
return nil |
| 715 |
end |
| 716 |
if not obj.stream then |
| 717 |
return nil |
| 718 |
end |
| 719 |
-- Defensive checks for stream structure |
| 720 |
if not obj.stream.data or not obj.stream.len then |
| 721 |
lua_util.debugm(N, task, 'malformed stream in object %s:%s', |
| 722 |
obj.major, obj.minor) |
| 723 |
return nil |
| 724 |
end |
| 725 |
local dict = obj.dict or {} |
| 726 |
local len = obj.stream.len |
| 727 |
if dict.Length then |
| 728 |
local decl_len = maybe_dereference_object(dict.Length, pdf, task) |
| 729 |
if decl_len then |
| 730 |
local nlen = tonumber(decl_len) |
| 731 |
if nlen then |
| 732 |
len = math.min(len, nlen) |
| 733 |
end |
| 734 |
end |
| 735 |
end |
| 736 |
|
| 737 |
if len > 0 then |
| 738 |
-- Wrap stream extraction in pcall to handle malformed data |
| 739 |
local ret, real_stream = pcall(function() |
| 740 |
return obj.stream.data:span(1, len) |
| 741 |
end) |
| 742 |
if not ret or not real_stream then |
| 743 |
lua_util.debugm(N, task, 'cannot extract stream span from object %s:%s: %s', |
| 744 |
obj.major, obj.minor, real_stream) |
| 745 |
return nil |
| 746 |
end |
| 747 |
|
| 748 |
local uncompressed, filter_err = maybe_apply_filter(dict, real_stream, pdf, task) |
| 749 |
|
| 750 |
if uncompressed then |
| 751 |
obj.uncompressed = uncompressed |
| 752 |
lua_util.debugm(N, task, 'extracted object %s:%s: (%s -> %s)', |
| 753 |
obj.major, obj.minor, len, #uncompressed) |
| 754 |
return obj.uncompressed |
| 755 |
else |
| 756 |
lua_util.debugm(N, task, 'cannot extract object %s:%s; len = %s; filter = %s: %s', |
| 757 |
obj.major, obj.minor, len, dict.Filter, filter_err) |
| 758 |
end |
| 759 |
else |
| 760 |
lua_util.debugm(N, task, 'cannot extract object %s:%s; len = %s', |
| 761 |
obj.major, obj.minor, len) |
| 762 |
end |
| 763 |
end |
| 764 |
|
| 765 |
local function parse_object_grammar(obj, task, pdf) |
| 766 |
-- Parse grammar |
| 767 |
local obj_dict_span |
| 768 |
if obj.stream then |
| 769 |
obj_dict_span = obj.data:span(1, obj.stream.start - obj.start) |
| 770 |
else |
| 771 |
obj_dict_span = obj.data |
| 772 |
end |
| 773 |
|
| 774 |
if obj_dict_span:len() < config.max_processing_size then |
| 775 |
local ret, obj_or_err = pcall(pdf_outer_grammar.match, pdf_outer_grammar, obj_dict_span) |
| 776 |
|
| 777 |
if ret then |
| 778 |
if obj.stream then |
| 779 |
if type(obj_or_err) == 'table' then |
| 780 |
obj.dict = obj_or_err |
| 781 |
else |
| 782 |
obj.dict = {} |
| 783 |
end |
| 784 |
|
| 785 |
lua_util.debugm(N, task, 'stream object %s:%s is parsed to: %s', |
| 786 |
obj.major, obj.minor, obj_or_err) |
| 787 |
else |
| 788 |
-- Direct object |
| 789 |
if type(obj_or_err) == 'table' then |
| 790 |
obj.dict = obj_or_err |
| 791 |
obj.uncompressed = obj_or_err |
| 792 |
lua_util.debugm(N, task, 'direct object %s:%s is parsed to: %s', |
| 793 |
obj.major, obj.minor, obj_or_err) |
| 794 |
pdf.ref[obj_ref(obj.major, obj.minor)] = obj |
| 795 |
else |
| 796 |
lua_util.debugm(N, task, 'direct object %s:%s is parsed to raw data: %s', |
| 797 |
obj.major, obj.minor, obj_or_err) |
| 798 |
pdf.ref[obj_ref(obj.major, obj.minor)] = obj_or_err |
| 799 |
obj.dict = {} |
| 800 |
obj.uncompressed = obj_or_err |
| 801 |
end |
| 802 |
end |
| 803 |
else |
| 804 |
lua_util.debugm(N, task, 'object %s:%s cannot be parsed: %s', |
| 805 |
obj.major, obj.minor, obj_or_err) |
| 806 |
end |
| 807 |
else |
| 808 |
lua_util.debugm(N, task, 'object %s:%s cannot be parsed: too large %s', |
| 809 |
obj.major, obj.minor, obj_dict_span:len()) |
| 810 |
end |
| 811 |
end |
| 812 |
|
| 813 |
-- Extracts font data and process /ToUnicode mappings |
| 814 |
-- NYI in fact as cmap is ridiculously stupid and complicated |
| 815 |
--[[ |
| 816 |
local function process_font(task, pdf, font, fname) |
| 817 |
local dict = font |
| 818 |
if font.dict then |
| 819 |
dict = font.dict |
| 820 |
end |
| 821 |
|
| 822 |
if type(dict) == 'table' and dict.ToUnicode then |
| 823 |
local cmap = maybe_dereference_object(dict.ToUnicode, pdf, task) |
| 824 |
|
| 825 |
if cmap and cmap.dict then |
| 826 |
maybe_extract_object_stream(cmap, pdf, task) |
| 827 |
lua_util.debugm(N, task, 'found cmap for font %s: %s', |
| 828 |
fname, cmap.uncompressed) |
| 829 |
end |
| 830 |
end |
| 831 |
end |
| 832 |
--]] |
| 833 |
|
| 834 |
-- Forward declaration |
| 835 |
local process_dict |
| 836 |
|
| 837 |
-- This function processes javascript string and returns JS hash and JS rspamd_text |
| 838 |
local function process_javascript(task, pdf, js, obj) |
| 839 |
local rspamd_cryptobox_hash = require "rspamd_cryptobox_hash" |
| 840 |
if type(js) == 'string' then |
| 841 |
js = rspamd_text.fromstring(js):oneline() |
| 842 |
elseif type(js) == 'userdata' then |
| 843 |
js = js:oneline() |
| 844 |
else |
| 845 |
return nil |
| 846 |
end |
| 847 |
|
| 848 |
local hash = rspamd_cryptobox_hash.create(js) |
| 849 |
local bin_hash = hash:bin() |
| 850 |
|
| 851 |
if not pdf.scripts then |
| 852 |
pdf.scripts = {} |
| 853 |
end |
| 854 |
|
| 855 |
if pdf.scripts[bin_hash] then |
| 856 |
-- Duplicate |
| 857 |
return pdf.scripts[bin_hash] |
| 858 |
end |
| 859 |
|
| 860 |
local njs = { |
| 861 |
data = js, |
| 862 |
hash = hash:hex(), |
| 863 |
bin_hash = bin_hash, |
| 864 |
object = obj, |
| 865 |
} |
| 866 |
pdf.scripts[bin_hash] = njs |
| 867 |
return njs |
| 868 |
end |
| 869 |
|
| 870 |
-- Extract interesting stuff from /Action, e.g. javascript |
| 871 |
local function process_action(task, pdf, obj) |
| 872 |
if not (obj.js or obj.launch) and (obj.dict and obj.dict.JS) then |
| 873 |
local js = maybe_dereference_object(obj.dict.JS, pdf, task) |
| 874 |
|
| 875 |
if js then |
| 876 |
if type(js) == 'table' then |
| 877 |
local extracted_js = maybe_extract_object_stream(js, pdf, task) |
| 878 |
|
| 879 |
if not extracted_js then |
| 880 |
lua_util.debugm(N, task, 'invalid type for JavaScript from %s:%s: %s', |
| 881 |
obj.major, obj.minor, js) |
| 882 |
else |
| 883 |
js = extracted_js |
| 884 |
end |
| 885 |
end |
| 886 |
|
| 887 |
js = process_javascript(task, pdf, js, obj) |
| 888 |
if js then |
| 889 |
obj.js = js |
| 890 |
lua_util.debugm(N, task, 'extracted javascript from %s:%s: %s', |
| 891 |
obj.major, obj.minor, obj.js.data) |
| 892 |
else |
| 893 |
lua_util.debugm(N, task, 'invalid type for JavaScript from %s:%s: %s', |
| 894 |
obj.major, obj.minor, js) |
| 895 |
end |
| 896 |
elseif obj.dict.F then |
| 897 |
local launch = maybe_dereference_object(obj.dict.F, pdf, task) |
| 898 |
|
| 899 |
if launch then |
| 900 |
if type(launch) == 'string' then |
| 901 |
obj.launch = rspamd_text.fromstring(launch):exclude_chars('%n%c') |
| 902 |
lua_util.debugm(N, task, 'extracted launch from %s:%s: %s', |
| 903 |
obj.major, obj.minor, obj.launch) |
| 904 |
elseif type(launch) == 'userdata' then |
| 905 |
obj.launch = launch:exclude_chars('%n%c') |
| 906 |
lua_util.debugm(N, task, 'extracted launch from %s:%s: %s', |
| 907 |
obj.major, obj.minor, obj.launch) |
| 908 |
else |
| 909 |
lua_util.debugm(N, task, 'invalid type for launch from %s:%s: %s', |
| 910 |
obj.major, obj.minor, launch) |
| 911 |
end |
| 912 |
end |
| 913 |
else |
| 914 |
|
| 915 |
lua_util.debugm(N, task, 'no JS attribute in action %s:%s', |
| 916 |
obj.major, obj.minor) |
| 917 |
end |
| 918 |
end |
| 919 |
end |
| 920 |
|
| 921 |
-- Extract interesting stuff from /Catalog, e.g. javascript in /OpenAction |
| 922 |
local function process_catalog(task, pdf, obj) |
| 923 |
if obj.dict then |
| 924 |
if obj.dict.OpenAction then |
| 925 |
local action = maybe_dereference_object(obj.dict.OpenAction, pdf, task) |
| 926 |
|
| 927 |
if action and type(action) == 'table' then |
| 928 |
-- This also processes action js (if not already processed) |
| 929 |
process_dict(task, pdf, action, action.dict) |
| 930 |
if action.js then |
| 931 |
lua_util.debugm(N, task, 'found openaction JS in %s:%s: %s', |
| 932 |
obj.major, obj.minor, action.js) |
| 933 |
pdf.openaction = action.js |
| 934 |
action.js.object = obj |
| 935 |
elseif action.launch then |
| 936 |
lua_util.debugm(N, task, 'found openaction launch in %s:%s: %s', |
| 937 |
obj.major, obj.minor, action.launch) |
| 938 |
pdf.launch = action.launch |
| 939 |
else |
| 940 |
lua_util.debugm(N, task, 'no JS in openaction %s:%s: %s', |
| 941 |
obj.major, obj.minor, action) |
| 942 |
end |
| 943 |
else |
| 944 |
lua_util.debugm(N, task, 'cannot find openaction %s:%s: %s -> %s', |
| 945 |
obj.major, obj.minor, obj.dict.OpenAction, action) |
| 946 |
end |
| 947 |
else |
| 948 |
lua_util.debugm(N, task, 'no openaction in catalog %s:%s', |
| 949 |
obj.major, obj.minor) |
| 950 |
end |
| 951 |
end |
| 952 |
end |
| 953 |
|
| 954 |
local function process_xref(task, pdf, obj) |
| 955 |
if obj.dict then |
| 956 |
if obj.dict.Encrypt then |
| 957 |
local encrypt = maybe_dereference_object(obj.dict.Encrypt, pdf, task) |
| 958 |
lua_util.debugm(N, task, 'found encrypt: %s in xref object %s:%s', |
| 959 |
encrypt, obj.major, obj.minor) |
| 960 |
pdf.encrypted = true |
| 961 |
end |
| 962 |
end |
| 963 |
end |
| 964 |
|
| 965 |
process_dict = function(task, pdf, obj, dict) |
| 966 |
if not obj.type and type(dict) == 'table' then |
| 967 |
if dict.Type and type(dict.Type) == 'string' then |
| 968 |
-- Common stuff |
| 969 |
obj.type = dict.Type |
| 970 |
end |
| 971 |
|
| 972 |
if not obj.type then |
| 973 |
-- Defensive: check obj.dict exists before accessing its fields |
| 974 |
if obj.dict and type(obj.dict) == 'table' and obj.dict.S and obj.dict.JS then |
| 975 |
obj.type = 'Javascript' |
| 976 |
lua_util.debugm(N, task, 'implicit type for JavaScript object %s:%s', |
| 977 |
obj.major, obj.minor) |
| 978 |
else |
| 979 |
lua_util.debugm(N, task, 'no type for %s:%s', |
| 980 |
obj.major, obj.minor) |
| 981 |
return |
| 982 |
end |
| 983 |
end |
| 984 |
|
| 985 |
lua_util.debugm(N, task, 'processed stream dictionary for object %s:%s -> %s', |
| 986 |
obj.major, obj.minor, obj.type) |
| 987 |
local contents = dict.Contents |
| 988 |
if contents and type(contents) == 'table' then |
| 989 |
if contents[1] == '%REF%' then |
| 990 |
-- Single reference |
| 991 |
contents = { contents } |
| 992 |
end |
| 993 |
obj.contents = {} |
| 994 |
|
| 995 |
for _, c in ipairs(contents) do |
| 996 |
local cobj = maybe_dereference_object(c, pdf, task) |
| 997 |
if cobj and type(cobj) == 'table' then |
| 998 |
obj.contents[#obj.contents + 1] = cobj |
| 999 |
cobj.parent = obj |
| 1000 |
cobj.type = 'content' |
| 1001 |
end |
| 1002 |
end |
| 1003 |
|
| 1004 |
lua_util.debugm(N, task, 'found content objects for %s:%s -> %s', |
| 1005 |
obj.major, obj.minor, #obj.contents) |
| 1006 |
end |
| 1007 |
|
| 1008 |
local resources = dict.Resources |
| 1009 |
if resources and type(resources) == 'table' then |
| 1010 |
local res_ref = maybe_dereference_object(resources, pdf, task) |
| 1011 |
|
| 1012 |
if type(res_ref) ~= 'table' then |
| 1013 |
lua_util.debugm(N, task, 'cannot parse resources from pdf: %s', |
| 1014 |
resources) |
| 1015 |
obj.resources = {} |
| 1016 |
elseif res_ref.dict then |
| 1017 |
obj.resources = res_ref.dict |
| 1018 |
else |
| 1019 |
obj.resources = {} |
| 1020 |
end |
| 1021 |
else |
| 1022 |
-- Fucking pdf: we need to inherit from parent |
| 1023 |
resources = {} |
| 1024 |
if dict.Parent then |
| 1025 |
local parent = maybe_dereference_object(dict.Parent, pdf, task) |
| 1026 |
|
| 1027 |
if parent and type(parent) == 'table' and parent.dict then |
| 1028 |
if parent.resources then |
| 1029 |
lua_util.debugm(N, task, 'propagated resources from %s:%s to %s:%s', |
| 1030 |
parent.major, parent.minor, obj.major, obj.minor) |
| 1031 |
resources = parent.resources |
| 1032 |
end |
| 1033 |
end |
| 1034 |
end |
| 1035 |
|
| 1036 |
obj.resources = resources |
| 1037 |
end |
| 1038 |
|
| 1039 |
|
| 1040 |
|
| 1041 |
--[[Disabled fonts extraction |
| 1042 |
local fonts = obj.resources.Font |
| 1043 |
if fonts and type(fonts) == 'table' then |
| 1044 |
obj.fonts = {} |
| 1045 |
for k,v in pairs(fonts) do |
| 1046 |
obj.fonts[k] = maybe_dereference_object(v, pdf, task) |
| 1047 |
|
| 1048 |
if obj.fonts[k] then |
| 1049 |
local font = obj.fonts[k] |
| 1050 |
|
| 1051 |
if config.text_extraction then |
| 1052 |
process_font(task, pdf, font, k) |
| 1053 |
lua_util.debugm(N, task, 'found font "%s" for object %s:%s -> %s', |
| 1054 |
k, obj.major, obj.minor, font) |
| 1055 |
end |
| 1056 |
end |
| 1057 |
end |
| 1058 |
end |
| 1059 |
]] |
| 1060 |
|
| 1061 |
lua_util.debugm(N, task, 'found resources for object %s:%s (%s): %s', |
| 1062 |
obj.major, obj.minor, obj.type, obj.resources) |
| 1063 |
|
| 1064 |
if obj.type == 'Action' then |
| 1065 |
process_action(task, pdf, obj) |
| 1066 |
elseif obj.type == 'Catalog' then |
| 1067 |
process_catalog(task, pdf, obj) |
| 1068 |
elseif obj.type == 'XRef' then |
| 1069 |
-- XRef stream instead of trailer from PDF 1.5 (thanks Adobe) |
| 1070 |
process_xref(task, pdf, obj) |
| 1071 |
elseif obj.type == 'Javascript' then |
| 1072 |
local js = maybe_dereference_object(obj.dict.JS, pdf, task) |
| 1073 |
|
| 1074 |
if js then |
| 1075 |
if type(js) == 'table' then |
| 1076 |
local extracted_js = maybe_extract_object_stream(js, pdf, task) |
| 1077 |
|
| 1078 |
if not extracted_js then |
| 1079 |
lua_util.debugm(N, task, 'invalid type for JavaScript from %s:%s: %s', |
| 1080 |
obj.major, obj.minor, js) |
| 1081 |
else |
| 1082 |
js = extracted_js |
| 1083 |
end |
| 1084 |
end |
| 1085 |
|
| 1086 |
js = process_javascript(task, pdf, js, obj) |
| 1087 |
if js then |
| 1088 |
obj.js = js |
| 1089 |
lua_util.debugm(N, task, 'extracted javascript from %s:%s: %s', |
| 1090 |
obj.major, obj.minor, obj.js.data) |
| 1091 |
else |
| 1092 |
lua_util.debugm(N, task, 'invalid type for JavaScript from %s:%s: %s', |
| 1093 |
obj.major, obj.minor, js) |
| 1094 |
end |
| 1095 |
end |
| 1096 |
end |
| 1097 |
end -- Already processed dict (obj.type is not empty) |
| 1098 |
end |
| 1099 |
|
| 1100 |
-- This function is intended to unpack objects from ObjStm crappy structure |
| 1101 |
local compound_obj_grammar |
| 1102 |
local function compound_obj_grammar_gen() |
| 1103 |
if not compound_obj_grammar then |
| 1104 |
local gen = generic_grammar_elts() |
| 1105 |
compound_obj_grammar = gen.ws ^ 0 * (gen.comment * gen.ws ^ 1) ^ 0 * |
| 1106 |
lpeg.Ct(lpeg.Ct(gen.number * gen.ws ^ 1 * gen.number * gen.ws ^ 0) ^ 1) |
| 1107 |
end |
| 1108 |
|
| 1109 |
return compound_obj_grammar |
| 1110 |
end |
| 1111 |
local function pdf_compound_object_unpack(_, uncompressed, pdf, task, first) |
| 1112 |
-- First, we need to parse data line by line likely to find a line |
| 1113 |
-- that consists of pairs of numbers |
| 1114 |
compound_obj_grammar_gen() |
| 1115 |
-- Wrap grammar match in pcall for safety |
| 1116 |
local match_ok, elts = pcall(compound_obj_grammar.match, compound_obj_grammar, uncompressed) |
| 1117 |
if not match_ok then |
| 1118 |
lua_util.debugm(N, task, 'compound object grammar match failed: %s', elts) |
| 1119 |
return |
| 1120 |
end |
| 1121 |
if elts and type(elts) == 'table' and #elts > 0 then |
| 1122 |
lua_util.debugm(N, task, 'compound elts (chunk length %s): %s', |
| 1123 |
#uncompressed, elts) |
| 1124 |
|
| 1125 |
for i, pair in ipairs(elts) do |
| 1126 |
-- Defensive: check pair is a valid table |
| 1127 |
if type(pair) ~= 'table' or not pair[1] or not pair[2] then |
| 1128 |
lua_util.debugm(N, task, 'invalid pair in compound object at index %s', i) |
| 1129 |
else |
| 1130 |
local obj_number, offset = pair[1], pair[2] |
| 1131 |
|
| 1132 |
offset = offset + first |
| 1133 |
if offset < #uncompressed then |
| 1134 |
local span_len |
| 1135 |
if i == #elts then |
| 1136 |
span_len = #uncompressed - offset |
| 1137 |
else |
| 1138 |
local next_pair = elts[i + 1] |
| 1139 |
if type(next_pair) == 'table' and next_pair[2] then |
| 1140 |
span_len = (next_pair[2] + first) - offset |
| 1141 |
else |
| 1142 |
span_len = #uncompressed - offset |
| 1143 |
end |
| 1144 |
end |
| 1145 |
|
| 1146 |
if span_len > 0 and offset + span_len <= #uncompressed then |
| 1147 |
-- Wrap span extraction in pcall |
| 1148 |
local span_ok, span_data = pcall(function() |
| 1149 |
return uncompressed:span(offset + 1, span_len) |
| 1150 |
end) |
| 1151 |
if not span_ok or not span_data then |
| 1152 |
lua_util.debugm(N, task, 'cannot extract span for compound object %s', obj_number) |
| 1153 |
else |
| 1154 |
local obj = { |
| 1155 |
major = obj_number, |
| 1156 |
minor = 0, -- Implicit |
| 1157 |
data = span_data, |
| 1158 |
ref = obj_ref(obj_number, 0) |
| 1159 |
} |
| 1160 |
parse_object_grammar(obj, task, pdf) |
| 1161 |
|
| 1162 |
if obj.dict then |
| 1163 |
pdf.objects[#pdf.objects + 1] = obj |
| 1164 |
end |
| 1165 |
end |
| 1166 |
else |
| 1167 |
lua_util.debugm(N, task, 'invalid span_len for compound object %s:%s; offset = %s, len = %s', |
| 1168 |
pair[1], pair[2], offset + span_len, #uncompressed) |
| 1169 |
end |
| 1170 |
end |
| 1171 |
end |
| 1172 |
end |
| 1173 |
end |
| 1174 |
end |
| 1175 |
|
| 1176 |
-- PDF 1.5 ObjStmt |
| 1177 |
local function extract_pdf_compound_objects(task, pdf) |
| 1178 |
for i, obj in ipairs(pdf.objects or {}) do |
| 1179 |
if i > 0 and i % 100 == 0 then |
| 1180 |
local now = rspamd_util.get_ticks() |
| 1181 |
|
| 1182 |
if now >= pdf.end_timestamp then |
| 1183 |
pdf.timeout_processing = now - pdf.start_timestamp |
| 1184 |
|
| 1185 |
lua_util.debugm(N, task, 'pdf: timeout processing compound objects after spending %s seconds, ' .. |
| 1186 |
'%s elements processed', |
| 1187 |
pdf.timeout_processing, i) |
| 1188 |
break |
| 1189 |
end |
| 1190 |
end |
| 1191 |
if obj.stream and obj.dict and type(obj.dict) == 'table' then |
| 1192 |
local t = obj.dict.Type |
| 1193 |
if t and t == 'ObjStm' then |
| 1194 |
-- We are in troubles sir... |
| 1195 |
local nobjs = tonumber(maybe_dereference_object(obj.dict.N, pdf, task)) |
| 1196 |
local first = tonumber(maybe_dereference_object(obj.dict.First, pdf, task)) |
| 1197 |
|
| 1198 |
if nobjs and first then |
| 1199 |
--local extend = maybe_dereference_object(obj.dict.Extends, pdf, task) |
| 1200 |
lua_util.debugm(N, task, 'extract ObjStm with %s objects (%s first) %s extend', |
| 1201 |
nobjs, first, obj.dict.Extends) |
| 1202 |
|
| 1203 |
local uncompressed = maybe_extract_object_stream(obj, pdf, task) |
| 1204 |
|
| 1205 |
if uncompressed then |
| 1206 |
pdf_compound_object_unpack(obj, uncompressed, pdf, task, first) |
| 1207 |
end |
| 1208 |
else |
| 1209 |
lua_util.debugm(N, task, 'ObjStm object %s:%s has bad dict: %s', |
| 1210 |
obj.major, obj.minor, obj.dict) |
| 1211 |
end |
| 1212 |
end |
| 1213 |
end |
| 1214 |
end |
| 1215 |
end |
| 1216 |
|
| 1217 |
-- This function arranges starts and ends of all objects and process them into initial |
| 1218 |
-- set of objects |
| 1219 |
local function extract_outer_objects(task, input, pdf) |
| 1220 |
local start_pos, end_pos = 1, 1 |
| 1221 |
local obj_count = 0 |
| 1222 |
local stored = 0 |
| 1223 |
local total_start = #pdf.start_objects |
| 1224 |
local total_end = #pdf.end_objects |
| 1225 |
|
| 1226 |
lua_util.debugm(N, task, "pdf: extract objects from %s start positions and %s end positions", |
| 1227 |
total_start, total_end) |
| 1228 |
|
| 1229 |
while start_pos <= total_start and end_pos <= total_end do |
| 1230 |
-- Timeout check every 500 iterations |
| 1231 |
if start_pos % 500 == 0 then |
| 1232 |
local now = rspamd_util.get_ticks() |
| 1233 |
if now >= pdf.end_timestamp then |
| 1234 |
pdf.timeout_processing = now - pdf.start_timestamp |
| 1235 |
lua_util.debugm(N, task, 'pdf: timeout extracting objects after %s seconds, ' .. |
| 1236 |
'%s stored, %s/%s positions', |
| 1237 |
pdf.timeout_processing, stored, start_pos, total_start) |
| 1238 |
break |
| 1239 |
end |
| 1240 |
end |
| 1241 |
|
| 1242 |
local first = pdf.start_objects[start_pos] |
| 1243 |
local last = pdf.end_objects[end_pos] |
| 1244 |
|
| 1245 |
-- 7 is length of `endobj\n` |
| 1246 |
if first + 6 < last then |
| 1247 |
local len = last - first - 6 |
| 1248 |
|
| 1249 |
-- Only count non-tiny objects toward the limit; small objects (e.g. padding) |
| 1250 |
-- are still stored but don't consume the budget |
| 1251 |
if len >= config.min_obj_content_size then |
| 1252 |
if obj_count >= config.max_pdf_objects then |
| 1253 |
break |
| 1254 |
end |
| 1255 |
obj_count = obj_count + 1 |
| 1256 |
end |
| 1257 |
|
| 1258 |
-- Also get the starting span and try to match it versus obj re to get numbers |
| 1259 |
local obj_line_potential = first - 32 |
| 1260 |
if obj_line_potential < 1 then |
| 1261 |
obj_line_potential = 1 |
| 1262 |
end |
| 1263 |
local prev_obj_end = pdf.end_objects[end_pos - 1] |
| 1264 |
if end_pos > 1 and prev_obj_end >= obj_line_potential and prev_obj_end < first then |
| 1265 |
obj_line_potential = prev_obj_end + 1 |
| 1266 |
end |
| 1267 |
|
| 1268 |
local obj_line_span = input:span(obj_line_potential, first - obj_line_potential + 1) |
| 1269 |
local matches = object_re:search(obj_line_span, true, true) |
| 1270 |
|
| 1271 |
if matches and matches[1] then |
| 1272 |
stored = stored + 1 |
| 1273 |
local nobj = { |
| 1274 |
start = first, |
| 1275 |
len = len, |
| 1276 |
data = input:span(first, len), |
| 1277 |
major = tonumber(matches[1][2]), |
| 1278 |
minor = tonumber(matches[1][3]), |
| 1279 |
} |
| 1280 |
pdf.objects[stored] = nobj |
| 1281 |
if nobj.major and nobj.minor then |
| 1282 |
-- Add reference |
| 1283 |
local ref = obj_ref(nobj.major, nobj.minor) |
| 1284 |
nobj.ref = ref -- Our internal reference |
| 1285 |
pdf.ref[ref] = nobj |
| 1286 |
end |
| 1287 |
end |
| 1288 |
|
| 1289 |
start_pos = start_pos + 1 |
| 1290 |
end_pos = end_pos + 1 |
| 1291 |
elseif first > last then |
| 1292 |
end_pos = end_pos + 1 |
| 1293 |
else |
| 1294 |
start_pos = start_pos + 1 |
| 1295 |
end_pos = end_pos + 1 |
| 1296 |
end |
| 1297 |
end |
| 1298 |
|
| 1299 |
lua_util.debugm(N, task, 'pdf: stored %s objects (%s non-tiny toward limit) from %s positions', |
| 1300 |
stored, obj_count, total_start) |
| 1301 |
end |
| 1302 |
|
| 1303 |
-- This function attaches streams to objects and processes outer pdf grammar |
| 1304 |
local function attach_pdf_streams(task, input, pdf) |
| 1305 |
if pdf.start_streams and pdf.end_streams then |
| 1306 |
local start_pos, end_pos = 1, 1 |
| 1307 |
local total_start = #pdf.start_streams |
| 1308 |
local total_end = #pdf.end_streams |
| 1309 |
local iter_count = 0 |
| 1310 |
|
| 1311 |
for _, obj in ipairs(pdf.objects) do |
| 1312 |
while start_pos <= total_start and end_pos <= total_end do |
| 1313 |
-- Timeout check every 500 iterations |
| 1314 |
iter_count = iter_count + 1 |
| 1315 |
if iter_count % 500 == 0 then |
| 1316 |
local now = rspamd_util.get_ticks() |
| 1317 |
if now >= pdf.end_timestamp then |
| 1318 |
pdf.timeout_processing = now - pdf.start_timestamp |
| 1319 |
lua_util.debugm(N, task, 'pdf: timeout attaching streams after %s seconds', |
| 1320 |
pdf.timeout_processing) |
| 1321 |
return |
| 1322 |
end |
| 1323 |
end |
| 1324 |
|
| 1325 |
local first = pdf.start_streams[start_pos] |
| 1326 |
local last = pdf.end_streams[end_pos] |
| 1327 |
last = last - 10 -- Exclude endstream\n pattern |
| 1328 |
lua_util.debugm(N, task, "start: %s, end: %s; obj: %s-%s", |
| 1329 |
first, last, obj.start, obj.start + obj.len) |
| 1330 |
if first > obj.start and last < obj.start + obj.len and last > first then |
| 1331 |
-- In case if we have fake endstream :( |
| 1332 |
while pdf.end_streams[end_pos + 1] and pdf.end_streams[end_pos + 1] < obj.start + obj.len do |
| 1333 |
end_pos = end_pos + 1 |
| 1334 |
last = pdf.end_streams[end_pos] |
| 1335 |
end |
| 1336 |
-- Strip the first \n |
| 1337 |
while first < last do |
| 1338 |
local chr = input:byte(first) |
| 1339 |
if chr ~= 13 and chr ~= 10 then |
| 1340 |
break |
| 1341 |
end |
| 1342 |
first = first + 1 |
| 1343 |
end |
| 1344 |
local len = last - first |
| 1345 |
obj.stream = { |
| 1346 |
start = first, |
| 1347 |
len = len, |
| 1348 |
data = input:span(first, len) |
| 1349 |
} |
| 1350 |
start_pos = start_pos + 1 |
| 1351 |
end_pos = end_pos + 1 |
| 1352 |
break |
| 1353 |
elseif first < obj.start then |
| 1354 |
start_pos = start_pos + 1 |
| 1355 |
elseif last > obj.start + obj.len then |
| 1356 |
-- Not this object |
| 1357 |
break |
| 1358 |
else |
| 1359 |
start_pos = start_pos + 1 |
| 1360 |
end_pos = end_pos + 1 |
| 1361 |
end |
| 1362 |
end |
| 1363 |
if obj.stream then |
| 1364 |
lua_util.debugm(N, task, 'found object %s:%s %s start %s len, %s stream start, %s stream length', |
| 1365 |
obj.major, obj.minor, obj.start, obj.len, obj.stream.start, obj.stream.len) |
| 1366 |
else |
| 1367 |
lua_util.debugm(N, task, 'found object %s:%s %s start %s len, no stream', |
| 1368 |
obj.major, obj.minor, obj.start, obj.len) |
| 1369 |
end |
| 1370 |
end |
| 1371 |
end |
| 1372 |
end |
| 1373 |
|
| 1374 |
-- Processes PDF objects: extracts streams, object numbers, process outer grammar, |
| 1375 |
-- augment object types |
| 1376 |
local function postprocess_pdf_objects(task, input, pdf) |
| 1377 |
pdf.objects = {} -- objects table |
| 1378 |
pdf.ref = {} -- references table |
| 1379 |
extract_outer_objects(task, input, pdf) |
| 1380 |
|
| 1381 |
-- Now we have objects and we need to attach streams that are in bounds |
| 1382 |
attach_pdf_streams(task, input, pdf) |
| 1383 |
-- Parse grammar for outer objects |
| 1384 |
for i, obj in ipairs(pdf.objects) do |
| 1385 |
if i > 0 and i % 100 == 0 then |
| 1386 |
local now = rspamd_util.get_ticks() |
| 1387 |
|
| 1388 |
if now >= pdf.end_timestamp then |
| 1389 |
pdf.timeout_processing = now - pdf.start_timestamp |
| 1390 |
lua_util.debugm(N, task, 'pdf: timeout processing grammars after spending %s seconds, ' .. |
| 1391 |
'%s elements processed', |
| 1392 |
pdf.timeout_processing, i) |
| 1393 |
break |
| 1394 |
end |
| 1395 |
end |
| 1396 |
if obj.ref then |
| 1397 |
parse_object_grammar(obj, task, pdf) |
| 1398 |
|
| 1399 |
-- Special early handling |
| 1400 |
if obj.dict and obj.dict.Type and obj.dict.Type == 'XRef' then |
| 1401 |
process_xref(task, pdf, obj) |
| 1402 |
end |
| 1403 |
end |
| 1404 |
end |
| 1405 |
|
| 1406 |
if not pdf.timeout_processing then |
| 1407 |
extract_pdf_compound_objects(task, pdf) |
| 1408 |
else |
| 1409 |
-- ENOTIME |
| 1410 |
return |
| 1411 |
end |
| 1412 |
|
| 1413 |
-- Now we might probably have all objects being processed |
| 1414 |
for i, obj in ipairs(pdf.objects) do |
| 1415 |
if obj.dict then |
| 1416 |
-- Types processing |
| 1417 |
if i > 0 and i % 100 == 0 then |
| 1418 |
local now = rspamd_util.get_ticks() |
| 1419 |
|
| 1420 |
if now >= pdf.end_timestamp then |
| 1421 |
pdf.timeout_processing = now - pdf.start_timestamp |
| 1422 |
|
| 1423 |
lua_util.debugm(N, task, 'pdf: timeout processing dicts after spending %s seconds, ' .. |
| 1424 |
'%s elements processed', |
| 1425 |
pdf.timeout_processing, i) |
| 1426 |
break |
| 1427 |
end |
| 1428 |
end |
| 1429 |
process_dict(task, pdf, obj, obj.dict) |
| 1430 |
end |
| 1431 |
end |
| 1432 |
end |
| 1433 |
|
| 1434 |
local function offsets_to_blocks(starts, ends, out) |
| 1435 |
local start_pos, end_pos = 1, 1 |
| 1436 |
|
| 1437 |
while start_pos <= #starts and end_pos <= #ends do |
| 1438 |
local first = starts[start_pos] |
| 1439 |
local last = ends[end_pos] |
| 1440 |
|
| 1441 |
if first < last then |
| 1442 |
local len = last - first |
| 1443 |
out[#out + 1] = { |
| 1444 |
start = first, |
| 1445 |
len = len, |
| 1446 |
} |
| 1447 |
start_pos = start_pos + 1 |
| 1448 |
end_pos = end_pos + 1 |
| 1449 |
elseif first > last then |
| 1450 |
end_pos = end_pos + 1 |
| 1451 |
else |
| 1452 |
-- Not ordered properly! |
| 1453 |
break |
| 1454 |
end |
| 1455 |
end |
| 1456 |
end |
| 1457 |
|
| 1458 |
local function search_text(task, pdf, mpart) |
| 1459 |
for _, obj in ipairs(pdf.objects) do |
| 1460 |
if obj.type == 'Page' and obj.contents then |
| 1461 |
local text = {} |
| 1462 |
for _, tobj in ipairs(obj.contents) do |
| 1463 |
maybe_extract_object_stream(tobj, pdf, task) |
| 1464 |
-- Defensive: ensure uncompressed data is usable |
| 1465 |
local uncompressed = tobj.uncompressed |
| 1466 |
if not uncompressed then |
| 1467 |
uncompressed = '' |
| 1468 |
end |
| 1469 |
-- Wrap trie match in pcall to handle unexpected input |
| 1470 |
local match_ok, matches = pcall(pdf_text_trie.match, pdf_text_trie, uncompressed) |
| 1471 |
if not match_ok then |
| 1472 |
lua_util.debugm(N, task, 'trie match failed for object %s:%s: %s', |
| 1473 |
tobj.major, tobj.minor, matches) |
| 1474 |
matches = nil |
| 1475 |
end |
| 1476 |
if matches and type(matches) == 'table' then |
| 1477 |
local text_blocks = {} |
| 1478 |
local starts = {} |
| 1479 |
local ends = {} |
| 1480 |
|
| 1481 |
for npat, matched_positions in pairs(matches) do |
| 1482 |
if type(matched_positions) ~= 'table' then |
| 1483 |
-- Skip malformed match results |
| 1484 |
lua_util.debugm(N, task, 'skipping malformed trie match result: %s', type(matched_positions)) |
| 1485 |
elseif npat == 1 then |
| 1486 |
for _, pos in ipairs(matched_positions) do |
| 1487 |
starts[#starts + 1] = pos |
| 1488 |
end |
| 1489 |
else |
| 1490 |
for _, pos in ipairs(matched_positions) do |
| 1491 |
ends[#ends + 1] = pos |
| 1492 |
end |
| 1493 |
end |
| 1494 |
end |
| 1495 |
|
| 1496 |
table.sort(starts) |
| 1497 |
table.sort(ends) |
| 1498 |
|
| 1499 |
offsets_to_blocks(starts, ends, text_blocks) |
| 1500 |
for _, bl in ipairs(text_blocks) do |
| 1501 |
if bl.len and bl.len > 2 then |
| 1502 |
-- To remove \s+ET\b pattern (it can leave trailing space or not but it doesn't matter) |
| 1503 |
bl.len = bl.len - 2 |
| 1504 |
end |
| 1505 |
-- Defensive: wrap span extraction in pcall |
| 1506 |
local span_ok, span_data = pcall(function() |
| 1507 |
if type(uncompressed) == 'userdata' or type(uncompressed) == 'string' then |
| 1508 |
return uncompressed:span(bl.start, bl.len) |
| 1509 |
end |
| 1510 |
return nil |
| 1511 |
end) |
| 1512 |
if not span_ok or not span_data then |
| 1513 |
lua_util.debugm(N, task, 'cannot extract text span from object %s:%s', |
| 1514 |
tobj.major, tobj.minor) |
| 1515 |
else |
| 1516 |
bl.data = span_data |
| 1517 |
if bl.len <= 256 then |
| 1518 |
lua_util.debugm(N, task, 'extracted text from object %s:%s: %s', |
| 1519 |
tobj.major, tobj.minor, bl.data) |
| 1520 |
else |
| 1521 |
lua_util.debugm(N, task, 'extracted text from object %s:%s (%d bytes)', |
| 1522 |
tobj.major, tobj.minor, bl.len) |
| 1523 |
end |
| 1524 |
|
| 1525 |
if bl.len < config.max_processing_size then |
| 1526 |
local ret, obj_or_err = pcall(pdf_text_grammar.match, pdf_text_grammar, |
| 1527 |
bl.data) |
| 1528 |
|
| 1529 |
if ret and type(obj_or_err) == 'table' then |
| 1530 |
if #obj_or_err == 0 then |
| 1531 |
lua_util.debugm(N, task, 'empty text match from block: %s', bl.data) |
| 1532 |
end |
| 1533 |
for _, chunk in ipairs(obj_or_err) do |
| 1534 |
text[#text + 1] = chunk |
| 1535 |
end |
| 1536 |
text[#text + 1] = '\n' |
| 1537 |
lua_util.debugm(N, task, 'attached %s from content object %s:%s to %s:%s', |
| 1538 |
obj_or_err, tobj.major, tobj.minor, obj.major, obj.minor) |
| 1539 |
else |
| 1540 |
lua_util.debugm(N, task, 'object %s:%s cannot be parsed: %s', |
| 1541 |
obj.major, obj.minor, obj_or_err) |
| 1542 |
end |
| 1543 |
end |
| 1544 |
end |
| 1545 |
end |
| 1546 |
end |
| 1547 |
end |
| 1548 |
|
| 1549 |
-- Join all text data together |
| 1550 |
if #text > 0 then |
| 1551 |
for i, chunk in ipairs(text) do |
| 1552 |
if type(chunk) == 'userdata' then |
| 1553 |
text[i] = tostring(chunk) |
| 1554 |
elseif type(chunk) == 'table' then |
| 1555 |
local function flatten(t) |
| 1556 |
local res = {} |
| 1557 |
local stack = { { tbl = t, idx = 1 } } |
| 1558 |
local max_depth = 100 |
| 1559 |
|
| 1560 |
while #stack > 0 and #stack <= max_depth do |
| 1561 |
local frame = stack[#stack] |
| 1562 |
local tbl, idx = frame.tbl, frame.idx |
| 1563 |
|
| 1564 |
if idx > #tbl then |
| 1565 |
stack[#stack] = nil |
| 1566 |
else |
| 1567 |
local v = tbl[idx] |
| 1568 |
frame.idx = idx + 1 |
| 1569 |
|
| 1570 |
if type(v) == 'userdata' then |
| 1571 |
res[#res + 1] = tostring(v) |
| 1572 |
elseif type(v) == 'table' then |
| 1573 |
stack[#stack + 1] = { tbl = v, idx = 1 } |
| 1574 |
elseif v ~= nil then |
| 1575 |
res[#res + 1] = tostring(v) |
| 1576 |
end |
| 1577 |
end |
| 1578 |
end |
| 1579 |
|
| 1580 |
return table.concat(res, '') |
| 1581 |
end |
| 1582 |
text[i] = flatten(chunk) |
| 1583 |
end |
| 1584 |
end |
| 1585 |
local res = table.concat(text, '') |
| 1586 |
|
| 1587 |
-- Page-level confidence check before storing text |
| 1588 |
if config.text_quality_enabled and #res >= config.text_quality_min_length then |
| 1589 |
local page_confidence = calculate_text_confidence(res) |
| 1590 |
if page_confidence < config.text_quality_threshold then |
| 1591 |
lua_util.debugm(N, task, 'skipping low confidence page text for %s:%s (%.2f)', |
| 1592 |
obj.major, obj.minor, page_confidence) |
| 1593 |
-- Don't store this page's text |
| 1594 |
else |
| 1595 |
obj.text = rspamd_text.fromstring(res) |
| 1596 |
lua_util.debugm(N, task, 'object %s:%s is parsed (confidence: %.2f): %s', |
| 1597 |
obj.major, obj.minor, page_confidence, obj.text) |
| 1598 |
end |
| 1599 |
else |
| 1600 |
obj.text = rspamd_text.fromstring(res) |
| 1601 |
lua_util.debugm(N, task, 'object %s:%s is parsed to: %s', |
| 1602 |
obj.major, obj.minor, obj.text) |
| 1603 |
end |
| 1604 |
end |
| 1605 |
end |
| 1606 |
end |
| 1607 |
|
| 1608 |
if task.inject_part then |
| 1609 |
local all_text = {} |
| 1610 |
|
| 1611 |
for _, obj in ipairs(pdf.objects) do |
| 1612 |
if obj.text and obj.text:len() > 0 then |
| 1613 |
table.insert(all_text, obj.text) |
| 1614 |
end |
| 1615 |
end |
| 1616 |
|
| 1617 |
if #all_text > 0 then |
| 1618 |
task:inject_part('text', all_text, mpart) |
| 1619 |
end |
| 1620 |
end |
| 1621 |
end |
| 1622 |
|
| 1623 |
-- This function searches objects for `/URI` key and parses it's content |
| 1624 |
local function search_urls(task, pdf, mpart) |
| 1625 |
local function recursive_object_traverse(obj, dict, rec) |
| 1626 |
if rec > 10 then |
| 1627 |
lua_util.debugm(N, task, 'object %s:%s recurses too much', |
| 1628 |
obj.major, obj.minor) |
| 1629 |
return |
| 1630 |
end |
| 1631 |
-- Defensive: ensure dict is actually a table we can iterate |
| 1632 |
if type(dict) ~= 'table' then |
| 1633 |
return |
| 1634 |
end |
| 1635 |
|
| 1636 |
for k, v in pairs(dict) do |
| 1637 |
if type(v) == 'table' then |
| 1638 |
recursive_object_traverse(obj, v, rec + 1) |
| 1639 |
elseif k == 'URI' then |
| 1640 |
v = maybe_dereference_object(v, pdf, task) |
| 1641 |
if type(v) == 'string' then |
| 1642 |
-- Wrap URL creation in pcall to handle malformed URLs |
| 1643 |
local url_ok, url = pcall(rspamd_url.create, task:get_mempool(), v, { 'content' }) |
| 1644 |
|
| 1645 |
if url_ok and url then |
| 1646 |
lua_util.debugm(N, task, 'found url %s in object %s:%s', |
| 1647 |
v, obj.major, obj.minor) |
| 1648 |
task:inject_url(url, mpart) |
| 1649 |
end |
| 1650 |
elseif type(v) == 'userdata' then |
| 1651 |
-- Handle rspamd_text objects |
| 1652 |
local str_ok, str_v = pcall(tostring, v) |
| 1653 |
if str_ok and str_v then |
| 1654 |
local url_ok, url = pcall(rspamd_url.create, task:get_mempool(), str_v, { 'content' }) |
| 1655 |
if url_ok and url then |
| 1656 |
lua_util.debugm(N, task, 'found url %s in object %s:%s', |
| 1657 |
str_v, obj.major, obj.minor) |
| 1658 |
task:inject_url(url, mpart) |
| 1659 |
end |
| 1660 |
end |
| 1661 |
end |
| 1662 |
end |
| 1663 |
end |
| 1664 |
end |
| 1665 |
|
| 1666 |
for _, obj in ipairs(pdf.objects) do |
| 1667 |
if obj.dict and type(obj.dict) == 'table' then |
| 1668 |
-- Wrap the traversal in pcall to handle any unexpected errors |
| 1669 |
local ok, err = pcall(recursive_object_traverse, obj, obj.dict, 0) |
| 1670 |
if not ok then |
| 1671 |
lua_util.debugm(N, task, 'error traversing object %s:%s for URLs: %s', |
| 1672 |
obj.major, obj.minor, err) |
| 1673 |
end |
| 1674 |
end |
| 1675 |
end |
| 1676 |
end |
| 1677 |
|
| 1678 |
local function process_pdf(input, mpart, task) |
| 1679 |
if not config.enabled then |
| 1680 |
-- Skip processing |
| 1681 |
return {} |
| 1682 |
end |
| 1683 |
|
| 1684 |
local matches = pdf_trie:match(input) |
| 1685 |
|
| 1686 |
if matches then |
| 1687 |
local start_ts = rspamd_util.get_ticks() |
| 1688 |
-- Temp object used to share data between pdf extraction methods |
| 1689 |
local pdf_object = { |
| 1690 |
tag = 'pdf', |
| 1691 |
extract_text = extract_text_data, |
| 1692 |
start_timestamp = start_ts, |
| 1693 |
end_timestamp = start_ts + config.pdf_process_timeout, |
| 1694 |
} |
| 1695 |
-- Output object that excludes all internal stuff |
| 1696 |
local pdf_output = lua_util.shallowcopy(pdf_object) |
| 1697 |
local grouped_processors = {} |
| 1698 |
for npat, matched_positions in pairs(matches) do |
| 1699 |
if type(matched_positions) ~= 'table' then |
| 1700 |
-- Skip malformed match results |
| 1701 |
lua_util.debugm(N, task, 'skipping malformed trie match result: %s', type(matched_positions)) |
| 1702 |
else |
| 1703 |
local index = pdf_indexes[npat] |
| 1704 |
|
| 1705 |
local proc_key, loc_npat = index[1], index[4] |
| 1706 |
|
| 1707 |
if not grouped_processors[proc_key] then |
| 1708 |
grouped_processors[proc_key] = { |
| 1709 |
processor_func = processors[proc_key], |
| 1710 |
offsets = {}, |
| 1711 |
} |
| 1712 |
end |
| 1713 |
local proc = grouped_processors[proc_key] |
| 1714 |
-- Fill offsets |
| 1715 |
for _, pos in ipairs(matched_positions) do |
| 1716 |
proc.offsets[#proc.offsets + 1] = { pos, loc_npat } |
| 1717 |
end |
| 1718 |
end |
| 1719 |
end |
| 1720 |
|
| 1721 |
for name, processor in pairs(grouped_processors) do |
| 1722 |
-- Sort by offset |
| 1723 |
lua_util.debugm(N, task, "pdf: process group %s with %s matches", |
| 1724 |
name, #processor.offsets) |
| 1725 |
table.sort(processor.offsets, function(e1, e2) |
| 1726 |
return e1[1] < e2[1] |
| 1727 |
end) |
| 1728 |
-- Wrap processor call in pcall to handle any errors gracefully |
| 1729 |
local proc_ok, proc_err = pcall(processor.processor_func, input, task, processor.offsets, pdf_object, pdf_output) |
| 1730 |
if not proc_ok then |
| 1731 |
lua_util.debugm(N, task, "pdf: processor %s failed: %s", name, proc_err) |
| 1732 |
end |
| 1733 |
end |
| 1734 |
|
| 1735 |
pdf_output.flags = {} |
| 1736 |
|
| 1737 |
if pdf_object.start_objects and pdf_object.end_objects then |
| 1738 |
if #pdf_object.start_objects > config.max_pdf_objects then |
| 1739 |
pdf_output.many_objects = #pdf_object.start_objects |
| 1740 |
-- Trim |
| 1741 |
end |
| 1742 |
|
| 1743 |
-- Postprocess objects - wrap in pcall for safety |
| 1744 |
local pp_ok, pp_err = pcall(postprocess_pdf_objects, task, input, pdf_object) |
| 1745 |
if not pp_ok then |
| 1746 |
lua_util.debugm(N, task, "pdf: postprocess_pdf_objects failed: %s", pp_err) |
| 1747 |
end |
| 1748 |
pdf_output.objects = pdf_object.objects |
| 1749 |
-- Skip text extraction if timeout occurred - partial results would be incorrect |
| 1750 |
if config.text_extraction and not pdf_object.timeout_processing then |
| 1751 |
-- Wrap in pcall for safety |
| 1752 |
local st_ok, st_err = pcall(search_text, task, pdf_object, mpart) |
| 1753 |
if not st_ok then |
| 1754 |
lua_util.debugm(N, task, "pdf: search_text failed: %s", st_err) |
| 1755 |
end |
| 1756 |
end |
| 1757 |
if config.url_extraction then |
| 1758 |
-- Wrap in pcall for safety |
| 1759 |
local su_ok, su_err = pcall(search_urls, task, pdf_object, mpart, pdf_output) |
| 1760 |
if not su_ok then |
| 1761 |
lua_util.debugm(N, task, "pdf: search_urls failed: %s", su_err) |
| 1762 |
end |
| 1763 |
end |
| 1764 |
|
| 1765 |
if config.js_fuzzy and pdf_object.scripts then |
| 1766 |
pdf_output.fuzzy_hashes = {} |
| 1767 |
if config.openaction_fuzzy_only then |
| 1768 |
-- OpenAction only |
| 1769 |
if pdf_object.openaction and pdf_object.openaction.bin_hash then |
| 1770 |
if config.min_js_fuzzy and #pdf_object.openaction.data >= config.min_js_fuzzy then |
| 1771 |
lua_util.debugm(N, task, "pdf: add fuzzy hash from openaction: %s; size = %s; object: %s:%s", |
| 1772 |
pdf_object.openaction.hash, |
| 1773 |
#pdf_object.openaction.data, |
| 1774 |
pdf_object.openaction.object.major, pdf_object.openaction.object.minor) |
| 1775 |
table.insert(pdf_output.fuzzy_hashes, pdf_object.openaction.bin_hash) |
| 1776 |
else |
| 1777 |
lua_util.debugm(N, task, "pdf: skip fuzzy hash from JavaScript: %s, too short: %s", |
| 1778 |
pdf_object.openaction.hash, #pdf_object.openaction.data) |
| 1779 |
end |
| 1780 |
end |
| 1781 |
else |
| 1782 |
-- All hashes |
| 1783 |
for h, sc in pairs(pdf_object.scripts) do |
| 1784 |
if config.min_js_fuzzy and #sc.data >= config.min_js_fuzzy then |
| 1785 |
lua_util.debugm(N, task, "pdf: add fuzzy hash from JavaScript: %s; size = %s; object: %s:%s", |
| 1786 |
sc.hash, |
| 1787 |
#sc.data, |
| 1788 |
sc.object.major, sc.object.minor) |
| 1789 |
table.insert(pdf_output.fuzzy_hashes, h) |
| 1790 |
else |
| 1791 |
lua_util.debugm(N, task, "pdf: skip fuzzy hash from JavaScript: %s, too short: %s", |
| 1792 |
sc.hash, #sc.data) |
| 1793 |
end |
| 1794 |
end |
| 1795 |
|
| 1796 |
end |
| 1797 |
end |
| 1798 |
else |
| 1799 |
pdf_output.flags.no_objects = true |
| 1800 |
end |
| 1801 |
|
| 1802 |
-- Propagate from object to output |
| 1803 |
if pdf_object.encrypted then |
| 1804 |
pdf_output.encrypted = true |
| 1805 |
end |
| 1806 |
if pdf_object.scripts then |
| 1807 |
pdf_output.scripts = true |
| 1808 |
end |
| 1809 |
|
| 1810 |
return pdf_output |
| 1811 |
end |
| 1812 |
end |
| 1813 |
|
| 1814 |
-- Processes the PDF trailer |
| 1815 |
processors.trailer = function(input, task, positions, pdf_object, pdf_output) |
| 1816 |
-- Defensive checks |
| 1817 |
if not positions or #positions == 0 then |
| 1818 |
return |
| 1819 |
end |
| 1820 |
local last_pos = positions[#positions] |
| 1821 |
if not last_pos or type(last_pos) ~= 'table' or not last_pos[1] then |
| 1822 |
return |
| 1823 |
end |
| 1824 |
|
| 1825 |
lua_util.debugm(N, task, 'pdf: process trailer at position %s (%s total length)', |
| 1826 |
last_pos, #input) |
| 1827 |
|
| 1828 |
if last_pos[1] > config.max_pdf_trailer then |
| 1829 |
pdf_output.long_trailer = #input - last_pos[1] |
| 1830 |
return |
| 1831 |
end |
| 1832 |
|
| 1833 |
-- Wrap span extraction in pcall |
| 1834 |
local span_ok, last_span = pcall(input.span, input, last_pos[1]) |
| 1835 |
if not span_ok or not last_span then |
| 1836 |
lua_util.debugm(N, task, 'pdf: cannot extract trailer span') |
| 1837 |
return |
| 1838 |
end |
| 1839 |
|
| 1840 |
local lines_checked = 0 |
| 1841 |
-- Wrap lines iteration in pcall |
| 1842 |
local iter_ok, iter_err = pcall(function() |
| 1843 |
for line in last_span:lines(true) do |
| 1844 |
if line:find('/Encrypt ') then |
| 1845 |
lua_util.debugm(N, task, "pdf: found encrypted line in trailer: %s", |
| 1846 |
line) |
| 1847 |
pdf_output.encrypted = true |
| 1848 |
pdf_object.encrypted = true |
| 1849 |
break |
| 1850 |
end |
| 1851 |
lines_checked = lines_checked + 1 |
| 1852 |
|
| 1853 |
if lines_checked > config.max_pdf_trailer_lines then |
| 1854 |
lua_util.debugm(N, task, "pdf: trailer has too many lines, stop checking") |
| 1855 |
pdf_output.long_trailer = #input - last_pos[1] |
| 1856 |
break |
| 1857 |
end |
| 1858 |
end |
| 1859 |
end) |
| 1860 |
if not iter_ok then |
| 1861 |
lua_util.debugm(N, task, 'pdf: error iterating trailer lines: %s', iter_err) |
| 1862 |
end |
| 1863 |
end |
| 1864 |
|
| 1865 |
processors.suspicious = function(input, task, positions, pdf_object, pdf_output) |
| 1866 |
-- Defensive check for positions |
| 1867 |
if not positions or type(positions) ~= 'table' then |
| 1868 |
return |
| 1869 |
end |
| 1870 |
|
| 1871 |
local suspicious_factor = 0.0 |
| 1872 |
local nexec = 0 |
| 1873 |
local nencoded = 0 |
| 1874 |
local close_encoded = 0 |
| 1875 |
local last_encoded |
| 1876 |
for _, match in ipairs(positions) do |
| 1877 |
-- Defensive check for match structure |
| 1878 |
if type(match) ~= 'table' or not match[1] or not match[2] then |
| 1879 |
goto continue_suspicious |
| 1880 |
end |
| 1881 |
|
| 1882 |
if match[2] == 1 then |
| 1883 |
-- netsh |
| 1884 |
suspicious_factor = suspicious_factor + 0.5 |
| 1885 |
elseif match[2] == 2 then |
| 1886 |
nexec = nexec + 1 |
| 1887 |
elseif match[2] == 3 then |
| 1888 |
-- Wrap input:sub in pcall for safety |
| 1889 |
local sub_ok, enc_data = pcall(input.sub, input, match[1] - 2, match[1] - 1) |
| 1890 |
local legal_escape = false |
| 1891 |
|
| 1892 |
if sub_ok and enc_data then |
| 1893 |
local strtoul_ok, strtoul_result = pcall(enc_data.strtoul, enc_data) |
| 1894 |
enc_data = strtoul_ok and strtoul_result or nil |
| 1895 |
|
| 1896 |
if enc_data then |
| 1897 |
-- Legit encode cases are non printable characters (e.g. spaces) |
| 1898 |
if enc_data < 0x21 or enc_data >= 0x7f then |
| 1899 |
legal_escape = true |
| 1900 |
end |
| 1901 |
end |
| 1902 |
end |
| 1903 |
|
| 1904 |
if not legal_escape then |
| 1905 |
nencoded = nencoded + 1 |
| 1906 |
|
| 1907 |
if last_encoded then |
| 1908 |
if match[1] - last_encoded < 8 then |
| 1909 |
-- likely consecutive encoded chars, increase factor |
| 1910 |
close_encoded = close_encoded + 1 |
| 1911 |
end |
| 1912 |
end |
| 1913 |
last_encoded = match[1] |
| 1914 |
|
| 1915 |
end |
| 1916 |
end |
| 1917 |
::continue_suspicious:: |
| 1918 |
end |
| 1919 |
|
| 1920 |
if nencoded > 10 then |
| 1921 |
suspicious_factor = suspicious_factor + nencoded / 10 |
| 1922 |
end |
| 1923 |
if nexec > 1 then |
| 1924 |
suspicious_factor = suspicious_factor + nexec / 2.0 |
| 1925 |
end |
| 1926 |
if close_encoded > 4 and nencoded - close_encoded < 5 then |
| 1927 |
-- Too many close encoded comparing to the total number of encoded characters |
| 1928 |
suspicious_factor = suspicious_factor + 0.5 |
| 1929 |
end |
| 1930 |
|
| 1931 |
lua_util.debugm(N, task, 'pdf: found a suspicious patterns: %s exec, %s encoded (%s close), ' .. |
| 1932 |
'%s final factor', |
| 1933 |
nexec, nencoded, close_encoded, suspicious_factor) |
| 1934 |
|
| 1935 |
if suspicious_factor > 1.0 then |
| 1936 |
suspicious_factor = 1.0 |
| 1937 |
end |
| 1938 |
|
| 1939 |
pdf_output.suspicious = suspicious_factor |
| 1940 |
end |
| 1941 |
|
| 1942 |
local function generic_table_inserter(positions, pdf_object, output_key) |
| 1943 |
-- Defensive checks |
| 1944 |
if not positions or type(positions) ~= 'table' then |
| 1945 |
return |
| 1946 |
end |
| 1947 |
if not pdf_object or type(pdf_object) ~= 'table' then |
| 1948 |
return |
| 1949 |
end |
| 1950 |
if not pdf_object[output_key] then |
| 1951 |
pdf_object[output_key] = {} |
| 1952 |
end |
| 1953 |
local shift = #pdf_object[output_key] |
| 1954 |
for i, pos in ipairs(positions) do |
| 1955 |
-- Check pos is a table with valid first element |
| 1956 |
if type(pos) == 'table' and pos[1] then |
| 1957 |
pdf_object[output_key][i + shift] = pos[1] |
| 1958 |
end |
| 1959 |
end |
| 1960 |
end |
| 1961 |
|
| 1962 |
processors.start_object = function(_, task, positions, pdf_object) |
| 1963 |
generic_table_inserter(positions, pdf_object, 'start_objects') |
| 1964 |
end |
| 1965 |
|
| 1966 |
processors.end_object = function(_, task, positions, pdf_object) |
| 1967 |
generic_table_inserter(positions, pdf_object, 'end_objects') |
| 1968 |
end |
| 1969 |
|
| 1970 |
processors.start_stream = function(_, task, positions, pdf_object) |
| 1971 |
generic_table_inserter(positions, pdf_object, 'start_streams') |
| 1972 |
end |
| 1973 |
|
| 1974 |
processors.end_stream = function(_, task, positions, pdf_object) |
| 1975 |
generic_table_inserter(positions, pdf_object, 'end_streams') |
| 1976 |
end |
| 1977 |
|
| 1978 |
exports.process = process_pdf |
| 1979 |
|
| 1980 |
return exports |