| 1 |
--[[ |
| 2 |
Copyright (c) 2022, Vsevolod Stakhov <[email protected]> |
| 3 |
|
| 4 |
Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 |
you may not use this file except in compliance with the License. |
| 6 |
You may obtain a copy of the License at |
| 7 |
|
| 8 |
http://www.apache.org/licenses/LICENSE-2.0 |
| 9 |
|
| 10 |
Unless required by applicable law or agreed to in writing, software |
| 11 |
distributed under the License is distributed on an "AS IS" BASIS, |
| 12 |
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 |
See the License for the specific language governing permissions and |
| 14 |
limitations under the License. |
| 15 |
]] -- |
| 16 |
|
| 17 |
local argparse = require "argparse" |
| 18 |
local ansicolors = require "ansicolors" |
| 19 |
local rspamd_util = require "rspamd_util" |
| 20 |
local rspamd_task = require "rspamd_task" |
| 21 |
local rspamd_text = require "rspamd_text" |
| 22 |
local rspamd_logger = require "rspamd_logger" |
| 23 |
local lua_meta = require "lua_meta" |
| 24 |
local rspamd_url = require "rspamd_url" |
| 25 |
local lua_util = require "lua_util" |
| 26 |
local lua_mime = require "lua_mime" |
| 27 |
local ucl = require "ucl" |
| 28 |
|
| 29 |
-- Define command line options |
| 30 |
local parser = argparse() |
| 31 |
:name "rspamadm mime" |
| 32 |
:description "Mime manipulations provided by Rspamd" |
| 33 |
:help_description_margin(30) |
| 34 |
:command_target("command") |
| 35 |
:require_command(true) |
| 36 |
|
| 37 |
parser:option "-c --config" |
| 38 |
:description "Path to config file" |
| 39 |
:argname("<cfg>") |
| 40 |
:default(rspamd_paths["CONFDIR"] .. "/" .. "rspamd.conf") |
| 41 |
parser:mutex( |
| 42 |
parser:flag "-j --json" |
| 43 |
:description "JSON output", |
| 44 |
parser:flag "-U --ucl" |
| 45 |
:description "UCL output", |
| 46 |
parser:flag "-M --messagepack" |
| 47 |
:description "MessagePack output" |
| 48 |
) |
| 49 |
parser:flag "-C --compact" |
| 50 |
:description "Use compact format" |
| 51 |
parser:flag "--no-file" |
| 52 |
:description "Do not print filename" |
| 53 |
|
| 54 |
-- Extract subcommand |
| 55 |
local extract = parser:command "extract ex e" |
| 56 |
:description "Extracts data from MIME messages" |
| 57 |
extract:argument "file" |
| 58 |
:description "File to process" |
| 59 |
:argname "<file>" |
| 60 |
:args "+" |
| 61 |
|
| 62 |
extract:flag "-t --text" |
| 63 |
:description "Extracts plain text data from a message" |
| 64 |
extract:flag "-r --raw" |
| 65 |
:description "Load as raw file" |
| 66 |
extract:flag "-H --html" |
| 67 |
:description "Extracts htm data from a message" |
| 68 |
extract:option "-o --output" |
| 69 |
:description "Output format ('raw', 'content', 'oneline', 'decoded', 'decoded_utf')" |
| 70 |
:argname("<type>") |
| 71 |
:convert { |
| 72 |
raw = "raw", |
| 73 |
content = "content", |
| 74 |
oneline = "content_oneline", |
| 75 |
decoded = "raw_parsed", |
| 76 |
decoded_utf = "raw_utf" |
| 77 |
} |
| 78 |
:default "content" |
| 79 |
extract:flag "-w --words" |
| 80 |
:description "Extracts words" |
| 81 |
extract:flag "-p --part" |
| 82 |
:description "Show part info" |
| 83 |
extract:flag "-s --structure" |
| 84 |
:description "Show structure info (e.g. HTML tags)" |
| 85 |
extract:flag "-i --invisible" |
| 86 |
:description "Show invisible content for HTML parts" |
| 87 |
extract:option "-F --words-format" |
| 88 |
:description "Words format ('stem', 'norm', 'raw', 'full')" |
| 89 |
:argname("<type>") |
| 90 |
:convert { |
| 91 |
stem = "stem", |
| 92 |
norm = "norm", |
| 93 |
raw = "raw", |
| 94 |
full = "full", |
| 95 |
} |
| 96 |
:default "stem" |
| 97 |
extract:option "-L --limit" |
| 98 |
:description "Maximum output size in bytes" |
| 99 |
:argname("<bytes>") |
| 100 |
:convert(tonumber) |
| 101 |
extract:flag "-Q --strip-quotes" |
| 102 |
:description "Remove quoted content (lines starting with >)" |
| 103 |
extract:flag "-S --strip-signatures" |
| 104 |
:description "Remove email signatures" |
| 105 |
extract:flag "-R --strip-reply-headers" |
| 106 |
:description "Remove reply headers (On X wrote:, From: Sent:)" |
| 107 |
extract:flag "-T --smart-trim" |
| 108 |
:description "Enable all text trimming heuristics" |
| 109 |
|
| 110 |
local stat = parser:command "stat st s" |
| 111 |
:description "Extracts statistical data from MIME messages" |
| 112 |
stat:argument "file" |
| 113 |
:description "File to process" |
| 114 |
:argname "<file>" |
| 115 |
:args "+" |
| 116 |
stat:mutex( |
| 117 |
stat:flag "-m --meta" |
| 118 |
:description "Lua metatokens", |
| 119 |
stat:flag "-b --bayes" |
| 120 |
:description "Bayes tokens", |
| 121 |
stat:flag "-F --fuzzy" |
| 122 |
:description "Fuzzy hashes" |
| 123 |
) |
| 124 |
stat:flag "-s --shingles" |
| 125 |
:description "Show shingles for fuzzy hashes" |
| 126 |
|
| 127 |
local urls = parser:command "urls url u" |
| 128 |
:description "Extracts URLs from MIME messages" |
| 129 |
urls:argument "file" |
| 130 |
:description "File to process" |
| 131 |
:argname "<file>" |
| 132 |
:args "+" |
| 133 |
urls:mutex( |
| 134 |
urls:flag "-t --tld" |
| 135 |
:description "Get TLDs only", |
| 136 |
urls:flag "-H --host" |
| 137 |
:description "Get hosts only", |
| 138 |
urls:flag "-f --full" |
| 139 |
:description "Show piecewise urls as processed by Rspamd" |
| 140 |
) |
| 141 |
|
| 142 |
urls:flag "-u --unique" |
| 143 |
:description "Print only unique urls" |
| 144 |
urls:flag "-s --sort" |
| 145 |
:description "Sort output" |
| 146 |
urls:flag "--count" |
| 147 |
:description "Print count of each printed element" |
| 148 |
urls:flag "-r --reverse" |
| 149 |
:description "Reverse sort order" |
| 150 |
urls:flag "--raw" |
| 151 |
:description "Load as raw file (for PDFs and other non-email files)" |
| 152 |
|
| 153 |
local modify = parser:command "modify mod m" |
| 154 |
:description "Modifies MIME message" |
| 155 |
modify:argument "file" |
| 156 |
:description "File to process" |
| 157 |
:argname "<file>" |
| 158 |
:args "+" |
| 159 |
|
| 160 |
modify:option "-a --add-header" |
| 161 |
:description "Adds specific header" |
| 162 |
:argname "<header=value>" |
| 163 |
:count "*" |
| 164 |
modify:option "-r --remove-header" |
| 165 |
:description "Removes specific header (all occurrences)" |
| 166 |
:argname "<header>" |
| 167 |
:count "*" |
| 168 |
modify:option "-R --rewrite-header" |
| 169 |
:description "Rewrites specific header, uses Lua string.format pattern" |
| 170 |
:argname "<header=pattern>" |
| 171 |
:count "*" |
| 172 |
modify:option "-t --text-footer" |
| 173 |
:description "Adds footer to text/plain parts from a specific file" |
| 174 |
:argname "<file>" |
| 175 |
modify:option "-H --html-footer" |
| 176 |
:description "Adds footer to text/html parts from a specific file" |
| 177 |
:argname "<file>" |
| 178 |
|
| 179 |
local strip = parser:command "strip" |
| 180 |
:description "Strip attachments from a message" |
| 181 |
strip:argument "file" |
| 182 |
:description "File to process" |
| 183 |
:argname "<file>" |
| 184 |
:args "+" |
| 185 |
strip:flag "-i --keep-images" |
| 186 |
:description "Keep images" |
| 187 |
strip:option "--min-text-size" |
| 188 |
:description "Minimal text size to keep" |
| 189 |
:argname "<size>" |
| 190 |
:convert(tonumber) |
| 191 |
:default(0) |
| 192 |
strip:option "--max-text-size" |
| 193 |
:description "Max text size to keep" |
| 194 |
:argname "<size>" |
| 195 |
:convert(tonumber) |
| 196 |
:default(math.huge) |
| 197 |
|
| 198 |
local anonymize = parser:command "anonymize" |
| 199 |
:description "Try to remove sensitive information from a message" |
| 200 |
anonymize:argument "file" |
| 201 |
:description "File to process" |
| 202 |
:argname "<file>" |
| 203 |
:args "+" |
| 204 |
anonymize:option "--exclude-header -X" |
| 205 |
:description "Exclude specific headers from anonymization" |
| 206 |
:argname "<header>" |
| 207 |
:count "*" |
| 208 |
anonymize:option "--include-header -I" |
| 209 |
:description "Include specific headers from anonymization" |
| 210 |
:argname "<header>" |
| 211 |
:count "*" |
| 212 |
anonymize:flag "--gpt" |
| 213 |
:description "Use LLM model for anonymization (requires GPT plugin to be configured)" |
| 214 |
anonymize:option "--model" |
| 215 |
:description "Model to use for anonymization" |
| 216 |
:argname "<model>" |
| 217 |
anonymize:option "--prompt" |
| 218 |
:description "Prompt to use for anonymization" |
| 219 |
:argname "<prompt>" |
| 220 |
|
| 221 |
local sign = parser:command "sign" |
| 222 |
:description "Performs DKIM signing" |
| 223 |
sign:argument "file" |
| 224 |
:description "File to process" |
| 225 |
:argname "<file>" |
| 226 |
:args "+" |
| 227 |
|
| 228 |
sign:option "-d --domain" |
| 229 |
:description "Use specific domain" |
| 230 |
:argname "<domain>" |
| 231 |
:count "1" |
| 232 |
sign:option "-s --selector" |
| 233 |
:description "Use specific selector" |
| 234 |
:argname "<selector>" |
| 235 |
:count "1" |
| 236 |
sign:option "-k --key" |
| 237 |
:description "Use specific key of file" |
| 238 |
:argname "<key>" |
| 239 |
:count "1" |
| 240 |
sign:option "-t --type" |
| 241 |
:description "ARC or DKIM signing" |
| 242 |
:argname("<arc|dkim>") |
| 243 |
:convert { |
| 244 |
['arc'] = 'arc', |
| 245 |
['dkim'] = 'dkim', |
| 246 |
} |
| 247 |
:default 'dkim' |
| 248 |
sign:option "-o --output" |
| 249 |
:description "Output format" |
| 250 |
:argname("<message|signature>") |
| 251 |
:convert { |
| 252 |
['message'] = 'message', |
| 253 |
['signature'] = 'signature', |
| 254 |
} |
| 255 |
:default 'message' |
| 256 |
|
| 257 |
local dump = parser:command "dump" |
| 258 |
:description "Dumps a raw message in different formats" |
| 259 |
dump:argument "file" |
| 260 |
:description "File to process" |
| 261 |
:argname "<file>" |
| 262 |
:args "+" |
| 263 |
-- Duplicate format for convenience |
| 264 |
dump:mutex( |
| 265 |
parser:flag "-j --json" |
| 266 |
:description "JSON output", |
| 267 |
parser:flag "-U --ucl" |
| 268 |
:description "UCL output", |
| 269 |
parser:flag "-M --messagepack" |
| 270 |
:description "MessagePack output" |
| 271 |
) |
| 272 |
dump:flag "-s --split" |
| 273 |
:description "Split the output file contents such that no content is embedded" |
| 274 |
|
| 275 |
dump:option "-o --outdir" |
| 276 |
:description "Output directory" |
| 277 |
:argname("<directory>") |
| 278 |
|
| 279 |
local function load_config(opts, load_tokenizers) |
| 280 |
local _r, err = rspamd_config:load_ucl(opts['config']) |
| 281 |
|
| 282 |
if not _r then |
| 283 |
rspamd_logger.errx('cannot parse %s: %s', opts['config'], err) |
| 284 |
os.exit(1) |
| 285 |
end |
| 286 |
|
| 287 |
_r, err = rspamd_config:parse_rcl({ 'logging', 'worker' }) |
| 288 |
if not _r then |
| 289 |
rspamd_logger.errx('cannot process %s: %s', opts['config'], err) |
| 290 |
os.exit(1) |
| 291 |
end |
| 292 |
|
| 293 |
-- Load custom tokenizers if requested |
| 294 |
if load_tokenizers then |
| 295 |
local success, tokenizer_err = rspamd_config:load_custom_tokenizers() |
| 296 |
if not success then |
| 297 |
rspamd_logger.errx('cannot load custom tokenizers: %s', tokenizer_err or 'unknown error') |
| 298 |
-- Don't exit here as custom tokenizers are optional |
| 299 |
rspamd_logger.warnx('proceeding without custom tokenizers') |
| 300 |
end |
| 301 |
end |
| 302 |
end |
| 303 |
|
| 304 |
-- Helper function to ensure proper cleanup of tokenizers |
| 305 |
local function cleanup_tokenizers() |
| 306 |
if rspamd_config then |
| 307 |
rspamd_config:unload_custom_tokenizers() |
| 308 |
end |
| 309 |
end |
| 310 |
|
| 311 |
local function load_task(opts, fname) |
| 312 |
if not fname then |
| 313 |
fname = '-' |
| 314 |
end |
| 315 |
|
| 316 |
local task = rspamd_task.create(rspamd_config, rspamadm_ev_base) |
| 317 |
task:set_session(rspamadm_session) |
| 318 |
task:set_resolver(rspamadm_dns_resolver) |
| 319 |
|
| 320 |
if opts.raw then |
| 321 |
local f |
| 322 |
if fname == '-' then |
| 323 |
f = io.stdin |
| 324 |
else |
| 325 |
f = io.open(fname, "rb") |
| 326 |
end |
| 327 |
|
| 328 |
if not f then |
| 329 |
parser:error("cannot open file " .. fname) |
| 330 |
end |
| 331 |
|
| 332 |
local content = f:read("*a") |
| 333 |
if fname ~= '-' then |
| 334 |
f:close() |
| 335 |
end |
| 336 |
|
| 337 |
local lua_magic = require "lua_magic" |
| 338 |
local dummy_part = { |
| 339 |
get_content = function() |
| 340 |
return content |
| 341 |
end, |
| 342 |
get_filename = function() |
| 343 |
return fname |
| 344 |
end, |
| 345 |
} |
| 346 |
local _, type_data = lua_magic.detect(dummy_part, rspamd_config) |
| 347 |
local ct = "application/octet-stream" |
| 348 |
if type_data and type_data.type then |
| 349 |
ct = type_data.type |
| 350 |
end |
| 351 |
|
| 352 |
if fname:match('%.pdf$') and (not ct or ct == 'application/octet-stream' or ct == 'binary') then |
| 353 |
ct = 'application/pdf' |
| 354 |
end |
| 355 |
|
| 356 |
-- Construct message |
| 357 |
local msg = string.format("Content-Type: %s\r\nContent-Transfer-Encoding: 8bit\r\n\r\n", ct) |
| 358 |
task:load_from_string(msg .. content) |
| 359 |
else |
| 360 |
local res = task:load_from_file(fname) |
| 361 |
|
| 362 |
if not res then |
| 363 |
parser:error(string.format('cannot read message from %s: %s', fname, |
| 364 |
task)) |
| 365 |
return nil |
| 366 |
end |
| 367 |
end |
| 368 |
|
| 369 |
if not task:process_message() then |
| 370 |
parser:error(string.format('cannot read message from %s: %s', fname, |
| 371 |
'failed to parse')) |
| 372 |
return nil |
| 373 |
end |
| 374 |
|
| 375 |
return task |
| 376 |
end |
| 377 |
|
| 378 |
local function highlight(fmt, ...) |
| 379 |
return ansicolors.white .. string.format(fmt, ...) .. ansicolors.reset |
| 380 |
end |
| 381 |
|
| 382 |
local function maybe_print_fname(opts, fname) |
| 383 |
if not opts.json and not opts['no-file'] then |
| 384 |
rspamd_logger.messagex(highlight('File: %s', fname)) |
| 385 |
end |
| 386 |
end |
| 387 |
|
| 388 |
local function output_fmt(opts) |
| 389 |
local fmt = 'json' |
| 390 |
if opts.compact then |
| 391 |
fmt = 'json-compact' |
| 392 |
end |
| 393 |
if opts.ucl then |
| 394 |
fmt = 'ucl' |
| 395 |
end |
| 396 |
if opts.messagepack then |
| 397 |
fmt = 'msgpack' |
| 398 |
end |
| 399 |
|
| 400 |
return fmt |
| 401 |
end |
| 402 |
|
| 403 |
-- Print elements in form |
| 404 |
-- filename -> table of elements |
| 405 |
local function print_elts(elts, opts, func) |
| 406 |
local fun = require "fun" |
| 407 |
|
| 408 |
if opts.json or opts.ucl then |
| 409 |
io.write(ucl.to_format(elts, output_fmt(opts))) |
| 410 |
else |
| 411 |
fun.each(function(fname, elt) |
| 412 |
if not opts.json and not opts.ucl then |
| 413 |
if func then |
| 414 |
elt = fun.map(func, elt) |
| 415 |
end |
| 416 |
maybe_print_fname(opts, fname) |
| 417 |
fun.each(function(e) |
| 418 |
io.write(e) |
| 419 |
io.write("\n") |
| 420 |
end, elt) |
| 421 |
end |
| 422 |
end, elts) |
| 423 |
end |
| 424 |
end |
| 425 |
|
| 426 |
local function extract_handler(opts) |
| 427 |
local out_elts = {} |
| 428 |
local tasks = {} |
| 429 |
local process_func |
| 430 |
|
| 431 |
if opts.words then |
| 432 |
-- Enable stemming and urls detection |
| 433 |
load_config(opts, true) -- Load with custom tokenizers |
| 434 |
rspamd_url.init(rspamd_config:get_tld_path()) |
| 435 |
rspamd_config:init_subsystem('langdet') |
| 436 |
end |
| 437 |
|
| 438 |
local function maybe_print_text_part_info(part, out) |
| 439 |
local fun = require "fun" |
| 440 |
if opts.part then |
| 441 |
local t = 'plain text' |
| 442 |
if part:is_html() then |
| 443 |
t = 'html' |
| 444 |
end |
| 445 |
|
| 446 |
if not opts.json and not opts.ucl then |
| 447 |
table.insert(out, |
| 448 |
rspamd_logger.slog('Part: %s: %s, language: %s, size: %s (%s raw), words: %s', |
| 449 |
part:get_mimepart():get_digest():sub(1, 8), |
| 450 |
t, |
| 451 |
part:get_language(), |
| 452 |
part:get_length(), part:get_raw_length(), |
| 453 |
part:get_words_count())) |
| 454 |
table.insert(out, |
| 455 |
rspamd_logger.slog('Stats: %s', |
| 456 |
fun.foldl(function(acc, k, v) |
| 457 |
if acc ~= '' then |
| 458 |
return string.format('%s, %s:%s', acc, k, v) |
| 459 |
else |
| 460 |
return string.format('%s:%s', k, v) |
| 461 |
end |
| 462 |
end, '', part:get_stats()))) |
| 463 |
end |
| 464 |
end |
| 465 |
end |
| 466 |
|
| 467 |
local function maybe_print_mime_part_info(part, out) |
| 468 |
if opts.part then |
| 469 |
if not opts.json and not opts.ucl then |
| 470 |
local mtype, msubtype = part:get_type() |
| 471 |
local det_mtype, det_msubtype = part:get_detected_type() |
| 472 |
table.insert(out, |
| 473 |
rspamd_logger.slog('Mime Part: %s: %s/%s (%s/%s detected), filename: %s (%s detected ext), size: %s', |
| 474 |
part:get_digest():sub(1, 8), |
| 475 |
mtype, msubtype, |
| 476 |
det_mtype, det_msubtype, |
| 477 |
part:get_filename(), |
| 478 |
part:get_detected_ext(), |
| 479 |
part:get_length())) |
| 480 |
end |
| 481 |
end |
| 482 |
end |
| 483 |
|
| 484 |
local function print_words(words, full) |
| 485 |
local fun = require "fun" |
| 486 |
|
| 487 |
if not full then |
| 488 |
return table.concat(words, ' ') |
| 489 |
else |
| 490 |
return table.concat( |
| 491 |
fun.totable( |
| 492 |
fun.map(function(w) |
| 493 |
-- [1] - stemmed word |
| 494 |
-- [2] - normalised word |
| 495 |
-- [3] - raw word |
| 496 |
-- [4] - flags (table of strings) |
| 497 |
return string.format('%s|%s|%s(%s)', |
| 498 |
w[3], w[2], w[1], table.concat(w[4], ',')) |
| 499 |
end, words) |
| 500 |
), |
| 501 |
' ' |
| 502 |
) |
| 503 |
end |
| 504 |
end |
| 505 |
|
| 506 |
for _, fname in ipairs(opts.file) do |
| 507 |
local task = load_task(opts, fname) |
| 508 |
out_elts[fname] = {} |
| 509 |
|
| 510 |
if not opts.text and not opts.html then |
| 511 |
opts.text = true |
| 512 |
opts.html = true |
| 513 |
end |
| 514 |
|
| 515 |
-- Check if we use extract_text_limited options |
| 516 |
if opts.limit or opts['strip_quotes'] or opts['strip_signatures'] or opts['smart_trim'] or opts['strip_reply_headers'] then |
| 517 |
local res = lua_mime.extract_text_limited(task, { |
| 518 |
max_bytes = opts.limit, |
| 519 |
strip_quotes = opts['strip_quotes'], |
| 520 |
strip_signatures = opts['strip_signatures'], |
| 521 |
strip_reply_headers = opts['strip_reply_headers'], |
| 522 |
smart_trim = opts['smart_trim'] |
| 523 |
}) |
| 524 |
|
| 525 |
if opts.json or opts.ucl then |
| 526 |
table.insert(out_elts[fname], res) |
| 527 |
else |
| 528 |
if res.truncated then |
| 529 |
table.insert(out_elts[fname], string.format("[Truncated] %s", res.text)) |
| 530 |
else |
| 531 |
table.insert(out_elts[fname], res.text) |
| 532 |
end |
| 533 |
|
| 534 |
if opts.part then |
| 535 |
table.insert(out_elts[fname], rspamd_logger.slog("Stats: %s", res.stats)) |
| 536 |
end |
| 537 |
end |
| 538 |
|
| 539 |
table.insert(out_elts[fname], "") |
| 540 |
table.insert(tasks, task) |
| 541 |
|
| 542 |
-- Skip normal processing |
| 543 |
goto continue |
| 544 |
end |
| 545 |
|
| 546 |
if opts.words then |
| 547 |
local how_words = opts['words_format'] or 'stem' |
| 548 |
table.insert(out_elts[fname], 'meta_words: ' .. |
| 549 |
print_words(task:get_meta_words(how_words), how_words == 'full')) |
| 550 |
end |
| 551 |
|
| 552 |
if opts.text or opts.html then |
| 553 |
local mp_all = task:get_parts(true) or {} |
| 554 |
|
| 555 |
-- Build map: parent_part -> injected_text_part |
| 556 |
local injected_map = {} |
| 557 |
for _, p in ipairs(mp_all) do |
| 558 |
if p:is_injected() and p:is_text() then |
| 559 |
|
| 560 |
local parent = p:get_parent() |
| 561 |
if parent then |
| 562 |
injected_map[parent:get_digest()] = p:get_text() |
| 563 |
end |
| 564 |
end |
| 565 |
end |
| 566 |
|
| 567 |
-- Build table: {{part, injected_text or nil}, ...} |
| 568 |
local parts_to_process = {} |
| 569 |
for _, p in ipairs(mp_all) do |
| 570 |
if not p:is_injected() then |
| 571 |
table.insert(parts_to_process, { p, injected_map[p:get_digest()] }) |
| 572 |
end |
| 573 |
end |
| 574 |
|
| 575 |
-- Process the parts |
| 576 |
for _, entry in ipairs(parts_to_process) do |
| 577 |
local mime_part = entry[1] |
| 578 |
local injected_part = entry[2] |
| 579 |
local how = opts.output |
| 580 |
local part |
| 581 |
|
| 582 |
if mime_part:is_text() then |
| 583 |
part = mime_part:get_text() |
| 584 |
end |
| 585 |
|
| 586 |
if part and opts.text and not part:is_html() then |
| 587 |
maybe_print_text_part_info(part, out_elts[fname]) |
| 588 |
maybe_print_mime_part_info(mime_part, out_elts[fname]) |
| 589 |
if not opts.json and not opts.ucl then |
| 590 |
table.insert(out_elts[fname], '\n') |
| 591 |
end |
| 592 |
|
| 593 |
if opts.words then |
| 594 |
local how_words = opts['words_format'] or 'stem' |
| 595 |
table.insert(out_elts[fname], print_words(part:get_words(how_words), |
| 596 |
how_words == 'full')) |
| 597 |
else |
| 598 |
table.insert(out_elts[fname], tostring(part:get_content(how))) |
| 599 |
end |
| 600 |
elseif injected_part and opts.text and not injected_part:is_html() then |
| 601 |
-- Show parent part info but content from injected child |
| 602 |
maybe_print_mime_part_info(mime_part, out_elts[fname]) |
| 603 |
if not opts.json and not opts.ucl then |
| 604 |
table.insert(out_elts[fname], string.format('[Extracted text from %s]', |
| 605 |
mime_part:get_filename() or 'attachment')) |
| 606 |
table.insert(out_elts[fname], '\n') |
| 607 |
end |
| 608 |
|
| 609 |
if opts.words then |
| 610 |
local how_words = opts['words_format'] or 'stem' |
| 611 |
table.insert(out_elts[fname], print_words(injected_part:get_words(how_words), |
| 612 |
how_words == 'full')) |
| 613 |
else |
| 614 |
table.insert(out_elts[fname], tostring(injected_part:get_content(how))) |
| 615 |
end |
| 616 |
elseif part and opts.html and part:is_html() then |
| 617 |
maybe_print_text_part_info(part, out_elts[fname]) |
| 618 |
maybe_print_mime_part_info(mime_part, out_elts[fname]) |
| 619 |
if not opts.json and not opts.ucl then |
| 620 |
table.insert(out_elts[fname], '\n') |
| 621 |
end |
| 622 |
|
| 623 |
if opts.words then |
| 624 |
local how_words = opts['words_format'] or 'stem' |
| 625 |
table.insert(out_elts[fname], print_words(part:get_words(how_words), |
| 626 |
how_words == 'full')) |
| 627 |
else |
| 628 |
if opts.structure then |
| 629 |
local hc = part:get_html() |
| 630 |
local res = {} |
| 631 |
process_func = function(elt) |
| 632 |
local fun = require "fun" |
| 633 |
if type(elt) == 'table' then |
| 634 |
return table.concat(fun.totable( |
| 635 |
fun.map( |
| 636 |
function(t) |
| 637 |
return rspamd_logger.slog("%s", t) |
| 638 |
end, |
| 639 |
elt)), '\n') |
| 640 |
else |
| 641 |
return rspamd_logger.slog("%s", elt) |
| 642 |
end |
| 643 |
end |
| 644 |
|
| 645 |
hc:foreach_tag('any', function(tag) |
| 646 |
local elt = {} |
| 647 |
local ex = tag:get_extra() |
| 648 |
elt.tag = tag:get_type() |
| 649 |
if ex then |
| 650 |
elt.extra = ex |
| 651 |
end |
| 652 |
local content = tag:get_content() |
| 653 |
if content then |
| 654 |
elt.content = tostring(content) |
| 655 |
end |
| 656 |
local style = tag:get_style() |
| 657 |
if style then |
| 658 |
elt.style = style |
| 659 |
end |
| 660 |
table.insert(res, elt) |
| 661 |
end) |
| 662 |
table.insert(out_elts[fname], res) |
| 663 |
else |
| 664 |
-- opts.structure |
| 665 |
table.insert(out_elts[fname], tostring(part:get_content(how))) |
| 666 |
end |
| 667 |
if opts.invisible then |
| 668 |
local hc = part:get_html() |
| 669 |
table.insert(out_elts[fname], string.format('invisible content: %s', |
| 670 |
tostring(hc:get_invisible()))) |
| 671 |
end |
| 672 |
end |
| 673 |
end |
| 674 |
|
| 675 |
if not part then |
| 676 |
maybe_print_mime_part_info(mime_part, out_elts[fname]) |
| 677 |
end |
| 678 |
end |
| 679 |
end |
| 680 |
|
| 681 |
table.insert(out_elts[fname], "") |
| 682 |
table.insert(tasks, task) |
| 683 |
|
| 684 |
::continue:: |
| 685 |
end |
| 686 |
|
| 687 |
print_elts(out_elts, opts, process_func) |
| 688 |
-- To avoid use after free we postpone tasks destruction |
| 689 |
for _, task in ipairs(tasks) do |
| 690 |
task:destroy() |
| 691 |
end |
| 692 |
|
| 693 |
-- Cleanup custom tokenizers if they were loaded |
| 694 |
if opts.words then |
| 695 |
cleanup_tokenizers() |
| 696 |
end |
| 697 |
end |
| 698 |
|
| 699 |
local function stat_handler(opts) |
| 700 |
local fun = require "fun" |
| 701 |
local out_elts = {} |
| 702 |
|
| 703 |
load_config(opts, true) -- Load with custom tokenizers for stat generation |
| 704 |
rspamd_url.init(rspamd_config:get_tld_path()) |
| 705 |
rspamd_config:init_subsystem('langdet,stat') -- Needed to gen stat tokens |
| 706 |
|
| 707 |
local process_func |
| 708 |
|
| 709 |
for _, fname in ipairs(opts.file) do |
| 710 |
local task = load_task(opts, fname) |
| 711 |
out_elts[fname] = {} |
| 712 |
|
| 713 |
if opts.meta then |
| 714 |
local mt = lua_meta.gen_metatokens_table(task) |
| 715 |
out_elts[fname] = mt |
| 716 |
process_func = function(k, v) |
| 717 |
return string.format("%s = %s", k, v) |
| 718 |
end |
| 719 |
elseif opts.bayes then |
| 720 |
local bt = task:get_stat_tokens() |
| 721 |
out_elts[fname] = bt |
| 722 |
process_func = function(e) |
| 723 |
return string.format('%s (%d): "%s"+"%s", [%s]', e.data, e.win, e.t1 or "", |
| 724 |
e.t2 or "", table.concat(fun.totable( |
| 725 |
fun.map(function(k) |
| 726 |
return k |
| 727 |
end, e.flags)), ",")) |
| 728 |
end |
| 729 |
elseif opts.fuzzy then |
| 730 |
local parts = task:get_parts() or {} |
| 731 |
out_elts[fname] = {} |
| 732 |
process_func = function(e) |
| 733 |
local ret = string.format('part: %s(%s): %s', e.type, e.file or "", e.digest) |
| 734 |
if opts.shingles and e.shingles then |
| 735 |
local sgl = {} |
| 736 |
for _, s in ipairs(e.shingles) do |
| 737 |
table.insert(sgl, string.format('%s: %s+%s+%s', s[1], s[2], s[3], s[4])) |
| 738 |
end |
| 739 |
|
| 740 |
ret = ret .. '\n' .. table.concat(sgl, '\n') |
| 741 |
end |
| 742 |
return ret |
| 743 |
end |
| 744 |
for _, part in ipairs(parts) do |
| 745 |
if not part:is_multipart() then |
| 746 |
local text = part:get_text() |
| 747 |
|
| 748 |
if text then |
| 749 |
local digest, shingles = text:get_fuzzy_hashes(task:get_mempool(), task:get_subject()) |
| 750 |
table.insert(out_elts[fname], { |
| 751 |
digest = digest, |
| 752 |
shingles = shingles, |
| 753 |
type = string.format('%s/%s', |
| 754 |
({ part:get_type() })[1], |
| 755 |
({ part:get_type() })[2]) |
| 756 |
}) |
| 757 |
else |
| 758 |
table.insert(out_elts[fname], { |
| 759 |
digest = part:get_digest(), |
| 760 |
file = part:get_filename(), |
| 761 |
type = string.format('%s/%s', |
| 762 |
({ part:get_type() })[1], |
| 763 |
({ part:get_type() })[2]) |
| 764 |
}) |
| 765 |
end |
| 766 |
end |
| 767 |
end |
| 768 |
end |
| 769 |
|
| 770 |
task:destroy() -- No automatic dtor |
| 771 |
end |
| 772 |
|
| 773 |
print_elts(out_elts, opts, process_func) |
| 774 |
|
| 775 |
-- Cleanup custom tokenizers |
| 776 |
cleanup_tokenizers() |
| 777 |
end |
| 778 |
|
| 779 |
local function urls_handler(opts) |
| 780 |
load_config(opts, false) -- URLs don't need custom tokenizers |
| 781 |
rspamd_url.init(rspamd_config:get_tld_path()) |
| 782 |
local out_elts = {} |
| 783 |
|
| 784 |
if opts.json then |
| 785 |
rspamd_logger.messagex('[') |
| 786 |
end |
| 787 |
|
| 788 |
for _, fname in ipairs(opts.file) do |
| 789 |
out_elts[fname] = {} |
| 790 |
local task = load_task(opts, fname) |
| 791 |
local elts = {} |
| 792 |
|
| 793 |
local function process_url(u) |
| 794 |
local s |
| 795 |
if opts.tld then |
| 796 |
s = u:get_tld() |
| 797 |
elseif opts.host then |
| 798 |
s = u:get_host() |
| 799 |
elseif opts.full then |
| 800 |
s = rspamd_logger.slog('%s: %s', u:get_text(), u:to_table()) |
| 801 |
else |
| 802 |
s = u:get_text() |
| 803 |
end |
| 804 |
|
| 805 |
if opts.unique then |
| 806 |
if elts[s] then |
| 807 |
elts[s].count = elts[s].count + 1 |
| 808 |
else |
| 809 |
elts[s] = { |
| 810 |
count = 1, |
| 811 |
url = u:to_table() |
| 812 |
} |
| 813 |
end |
| 814 |
else |
| 815 |
if opts.json then |
| 816 |
table.insert(elts, u) |
| 817 |
else |
| 818 |
table.insert(elts, s) |
| 819 |
end |
| 820 |
end |
| 821 |
end |
| 822 |
|
| 823 |
-- Use get_urls_filtered with nil params to get all URLs including content URLs |
| 824 |
for _, u in ipairs(task:get_urls_filtered()) do |
| 825 |
process_url(u) |
| 826 |
end |
| 827 |
|
| 828 |
local json_elts = {} |
| 829 |
|
| 830 |
local function process_elt(s, u) |
| 831 |
if opts.unique then |
| 832 |
-- s is string, u is {url = url, count = count } |
| 833 |
if not opts.json then |
| 834 |
if opts.count then |
| 835 |
table.insert(json_elts, string.format('%s : %s', s, u.count)) |
| 836 |
else |
| 837 |
table.insert(json_elts, s) |
| 838 |
end |
| 839 |
else |
| 840 |
local tb = u.url |
| 841 |
tb.count = u.count |
| 842 |
table.insert(json_elts, tb) |
| 843 |
end |
| 844 |
else |
| 845 |
-- s is index, u is url or string |
| 846 |
if opts.json then |
| 847 |
table.insert(json_elts, u) |
| 848 |
else |
| 849 |
table.insert(json_elts, u) |
| 850 |
end |
| 851 |
end |
| 852 |
end |
| 853 |
|
| 854 |
if opts.sort then |
| 855 |
local sfunc |
| 856 |
if opts.unique then |
| 857 |
sfunc = function(t, a, b) |
| 858 |
if t[a].count ~= t[b].count then |
| 859 |
if opts.reverse then |
| 860 |
return t[a].count > t[b].count |
| 861 |
else |
| 862 |
return t[a].count < t[b].count |
| 863 |
end |
| 864 |
else |
| 865 |
-- Sort lexicography |
| 866 |
if opts.reverse then |
| 867 |
return a > b |
| 868 |
else |
| 869 |
return a < b |
| 870 |
end |
| 871 |
end |
| 872 |
end |
| 873 |
else |
| 874 |
sfunc = function(t, a, b) |
| 875 |
local va, vb |
| 876 |
if opts.json then |
| 877 |
va = t[a]:get_text() |
| 878 |
vb = t[b]:get_text() |
| 879 |
else |
| 880 |
va = t[a] |
| 881 |
vb = t[b] |
| 882 |
end |
| 883 |
if opts.reverse then |
| 884 |
return va > vb |
| 885 |
else |
| 886 |
return va < vb |
| 887 |
end |
| 888 |
end |
| 889 |
end |
| 890 |
|
| 891 |
for s, u in lua_util.spairs(elts, sfunc) do |
| 892 |
process_elt(s, u) |
| 893 |
end |
| 894 |
else |
| 895 |
for s, u in pairs(elts) do |
| 896 |
process_elt(s, u) |
| 897 |
end |
| 898 |
end |
| 899 |
|
| 900 |
out_elts[fname] = json_elts |
| 901 |
|
| 902 |
task:destroy() -- No automatic dtor |
| 903 |
end |
| 904 |
|
| 905 |
print_elts(out_elts, opts) |
| 906 |
end |
| 907 |
|
| 908 |
local function newline(task) |
| 909 |
local t = task:get_newlines_type() |
| 910 |
|
| 911 |
if t == 'cr' then |
| 912 |
return '\r' |
| 913 |
elseif t == 'lf' then |
| 914 |
return '\n' |
| 915 |
end |
| 916 |
|
| 917 |
return '\r\n' |
| 918 |
end |
| 919 |
|
| 920 |
local function modify_handler(opts) |
| 921 |
load_config(opts, false) -- Modification doesn't need custom tokenizers |
| 922 |
rspamd_url.init(rspamd_config:get_tld_path()) |
| 923 |
|
| 924 |
local function read_file(file) |
| 925 |
local f = assert(io.open(file, "rb")) |
| 926 |
local content = f:read("*all") |
| 927 |
f:close() |
| 928 |
return content |
| 929 |
end |
| 930 |
|
| 931 |
local text_footer, html_footer |
| 932 |
|
| 933 |
if opts['text_footer'] then |
| 934 |
text_footer = read_file(opts['text_footer']) |
| 935 |
end |
| 936 |
|
| 937 |
if opts['html_footer'] then |
| 938 |
html_footer = read_file(opts['html_footer']) |
| 939 |
end |
| 940 |
|
| 941 |
for _, fname in ipairs(opts.file) do |
| 942 |
local task = load_task(opts, fname) |
| 943 |
local newline_s = newline(task) |
| 944 |
local seen_cte |
| 945 |
|
| 946 |
local rewrite = lua_mime.add_text_footer(task, html_footer, text_footer) or {} |
| 947 |
local out = {} -- Start with headers |
| 948 |
|
| 949 |
local function process_headers_cb(name, hdr) |
| 950 |
for _, h in ipairs(opts['remove_header']) do |
| 951 |
if name:match(h) then |
| 952 |
return |
| 953 |
end |
| 954 |
end |
| 955 |
|
| 956 |
for _, h in ipairs(opts['rewrite_header']) do |
| 957 |
local hname, hpattern = h:match('^([^=]+)=(.+)$') |
| 958 |
if hname == name then |
| 959 |
local new_value = string.format(hpattern, hdr.decoded) |
| 960 |
new_value = string.format('%s:%s%s', |
| 961 |
name, hdr.separator, |
| 962 |
rspamd_util.fold_header(name, |
| 963 |
rspamd_util.mime_header_encode(new_value), |
| 964 |
task:get_newlines_type())) |
| 965 |
out[#out + 1] = new_value |
| 966 |
return |
| 967 |
end |
| 968 |
end |
| 969 |
|
| 970 |
if rewrite.need_rewrite_ct then |
| 971 |
if name:lower() == 'content-type' then |
| 972 |
local nct = string.format('%s: %s/%s; charset=utf-8', |
| 973 |
'Content-Type', rewrite.new_ct.type, rewrite.new_ct.subtype) |
| 974 |
out[#out + 1] = nct |
| 975 |
return |
| 976 |
elseif name:lower() == 'content-transfer-encoding' then |
| 977 |
out[#out + 1] = string.format('%s: %s', |
| 978 |
'Content-Transfer-Encoding', rewrite.new_cte or 'quoted-printable') |
| 979 |
seen_cte = true |
| 980 |
return |
| 981 |
end |
| 982 |
end |
| 983 |
|
| 984 |
out[#out + 1] = hdr.raw:gsub('\r?\n?$', '') |
| 985 |
end |
| 986 |
|
| 987 |
task:headers_foreach(process_headers_cb, { full = true }) |
| 988 |
|
| 989 |
for _, h in ipairs(opts['add_header']) do |
| 990 |
local hname, hvalue = h:match('^([^=]+)=(.+)$') |
| 991 |
|
| 992 |
if hname and hvalue then |
| 993 |
out[#out + 1] = string.format('%s: %s', hname, |
| 994 |
rspamd_util.fold_header(hname, hvalue, task:get_newlines_type())) |
| 995 |
end |
| 996 |
end |
| 997 |
|
| 998 |
if not seen_cte and rewrite.need_rewrite_ct then |
| 999 |
out[#out + 1] = string.format('%s: %s', |
| 1000 |
'Content-Transfer-Encoding', rewrite.new_cte or 'quoted-printable') |
| 1001 |
end |
| 1002 |
|
| 1003 |
-- End of headers |
| 1004 |
out[#out + 1] = '' |
| 1005 |
|
| 1006 |
if rewrite.out then |
| 1007 |
for _, o in ipairs(rewrite.out) do |
| 1008 |
out[#out + 1] = o |
| 1009 |
end |
| 1010 |
else |
| 1011 |
out[#out + 1] = { task:get_rawbody(), false } |
| 1012 |
end |
| 1013 |
|
| 1014 |
for _, o in ipairs(out) do |
| 1015 |
if type(o) == 'string' then |
| 1016 |
io.write(o) |
| 1017 |
io.write(newline_s) |
| 1018 |
elseif type(o) == 'table' then |
| 1019 |
io.flush() |
| 1020 |
if type(o[1]) == 'string' then |
| 1021 |
io.write(o[1]) |
| 1022 |
else |
| 1023 |
o[1]:save_in_file(1) |
| 1024 |
end |
| 1025 |
|
| 1026 |
if o[2] then |
| 1027 |
io.write(newline_s) |
| 1028 |
end |
| 1029 |
else |
| 1030 |
o:save_in_file(1) |
| 1031 |
io.write(newline_s) |
| 1032 |
end |
| 1033 |
end |
| 1034 |
|
| 1035 |
task:destroy() -- No automatic dtor |
| 1036 |
end |
| 1037 |
end |
| 1038 |
|
| 1039 |
local function sign_handler(opts) |
| 1040 |
load_config(opts, false) -- Signing doesn't need custom tokenizers |
| 1041 |
rspamd_url.init(rspamd_config:get_tld_path()) |
| 1042 |
|
| 1043 |
local lua_dkim = require("lua_ffi").dkim |
| 1044 |
|
| 1045 |
if not lua_dkim then |
| 1046 |
io.stderr:write('FFI support is required: please use LuaJIT or install lua-ffi') |
| 1047 |
os.exit(1) |
| 1048 |
end |
| 1049 |
|
| 1050 |
local sign_key |
| 1051 |
if rspamd_util.file_exists(opts.key) then |
| 1052 |
sign_key = lua_dkim.load_sign_key(opts.key, 'file') |
| 1053 |
else |
| 1054 |
sign_key = lua_dkim.load_sign_key(opts.key, 'base64') |
| 1055 |
end |
| 1056 |
|
| 1057 |
if not sign_key then |
| 1058 |
io.stderr:write('Cannot load key: ' .. opts.key .. '\n') |
| 1059 |
os.exit(1) |
| 1060 |
end |
| 1061 |
|
| 1062 |
for _, fname in ipairs(opts.file) do |
| 1063 |
local task = load_task(opts, fname) |
| 1064 |
local ctx = lua_dkim.create_sign_context(task, sign_key, nil, opts.algorithm) |
| 1065 |
|
| 1066 |
if not ctx then |
| 1067 |
io.stderr:write('Cannot init signing\n') |
| 1068 |
os.exit(1) |
| 1069 |
end |
| 1070 |
|
| 1071 |
local sig = lua_dkim.do_sign(task, ctx, opts.selector, opts.domain) |
| 1072 |
|
| 1073 |
if not sig then |
| 1074 |
io.stderr:write('Cannot create signature\n') |
| 1075 |
os.exit(1) |
| 1076 |
end |
| 1077 |
|
| 1078 |
if opts.output == 'signature' then |
| 1079 |
io.write(sig) |
| 1080 |
io.write('\n') |
| 1081 |
io.flush() |
| 1082 |
else |
| 1083 |
local dkim_hdr = string.format('%s: %s%s', |
| 1084 |
'DKIM-Signature', |
| 1085 |
rspamd_util.fold_header('DKIM-Signature', |
| 1086 |
rspamd_util.mime_header_encode(sig), |
| 1087 |
task:get_newlines_type()), |
| 1088 |
newline(task)) |
| 1089 |
io.write(dkim_hdr) |
| 1090 |
io.flush() |
| 1091 |
task:get_content():save_in_file(1) |
| 1092 |
end |
| 1093 |
|
| 1094 |
task:destroy() -- No automatic dtor |
| 1095 |
end |
| 1096 |
end |
| 1097 |
|
| 1098 |
local function strip_handler(opts) |
| 1099 |
load_config(opts, false) -- Stripping doesn't need custom tokenizers |
| 1100 |
rspamd_url.init(rspamd_config:get_tld_path()) |
| 1101 |
|
| 1102 |
for _, fname in ipairs(opts.file) do |
| 1103 |
local task = load_task(opts, fname) |
| 1104 |
local newline_s = newline(task) |
| 1105 |
|
| 1106 |
local rewrite = lua_mime.remove_attachments(task, { |
| 1107 |
keep_images = opts.keep_images, |
| 1108 |
min_text_size = opts.min_text_size, |
| 1109 |
max_text_size = opts.max_text_size |
| 1110 |
}) or {} |
| 1111 |
local out = {} -- Start with headers |
| 1112 |
|
| 1113 |
local function process_headers_cb(name, hdr) |
| 1114 |
out[#out + 1] = hdr.raw:gsub('\r?\n?$', '') |
| 1115 |
end |
| 1116 |
|
| 1117 |
task:headers_foreach(process_headers_cb, { full = true }) |
| 1118 |
-- End of headers |
| 1119 |
out[#out + 1] = '' |
| 1120 |
|
| 1121 |
if rewrite.out then |
| 1122 |
for _, o in ipairs(rewrite.out) do |
| 1123 |
out[#out + 1] = o |
| 1124 |
end |
| 1125 |
else |
| 1126 |
out[#out + 1] = { task:get_rawbody(), false } |
| 1127 |
end |
| 1128 |
|
| 1129 |
for _, o in ipairs(out) do |
| 1130 |
if type(o) == 'string' then |
| 1131 |
io.write(o) |
| 1132 |
io.write(newline_s) |
| 1133 |
elseif type(o) == 'table' then |
| 1134 |
io.flush() |
| 1135 |
if type(o[1]) == 'string' then |
| 1136 |
io.write(o[1]) |
| 1137 |
else |
| 1138 |
o[1]:save_in_file(1) |
| 1139 |
end |
| 1140 |
|
| 1141 |
if o[2] then |
| 1142 |
io.write(newline_s) |
| 1143 |
end |
| 1144 |
else |
| 1145 |
o:save_in_file(1) |
| 1146 |
io.write(newline_s) |
| 1147 |
end |
| 1148 |
end |
| 1149 |
|
| 1150 |
task:destroy() -- No automatic dtor |
| 1151 |
end |
| 1152 |
end |
| 1153 |
|
| 1154 |
local function anonymize_handler(opts) |
| 1155 |
load_config(opts, false) -- Anonymization doesn't need custom tokenizers |
| 1156 |
rspamd_url.init(rspamd_config:get_tld_path()) |
| 1157 |
|
| 1158 |
for _, fname in ipairs(opts.file) do |
| 1159 |
local task = load_task(opts, fname) |
| 1160 |
local newline_s = newline(task) |
| 1161 |
|
| 1162 |
local rewrite = lua_mime.anonymize_message(task, opts) |
| 1163 |
|
| 1164 |
if not rewrite or not rewrite.out then |
| 1165 |
rspamd_logger.errx('cannot anonymize message from %s', fname) |
| 1166 |
task:destroy() |
| 1167 |
os.exit(1) |
| 1168 |
end |
| 1169 |
|
| 1170 |
for _, o in ipairs(rewrite.out) do |
| 1171 |
if type(o) == 'string' then |
| 1172 |
io.write(o) |
| 1173 |
io.write(newline_s) |
| 1174 |
elseif type(o) == 'table' then |
| 1175 |
io.flush() |
| 1176 |
if type(o[1]) == 'string' then |
| 1177 |
io.write(o[1]) |
| 1178 |
else |
| 1179 |
o[1]:save_in_file(1) |
| 1180 |
end |
| 1181 |
|
| 1182 |
if o[2] then |
| 1183 |
io.write(newline_s) |
| 1184 |
end |
| 1185 |
else |
| 1186 |
o:save_in_file(1) |
| 1187 |
io.write(newline_s) |
| 1188 |
end |
| 1189 |
end |
| 1190 |
|
| 1191 |
task:destroy() -- No automatic dtor |
| 1192 |
end |
| 1193 |
end |
| 1194 |
|
| 1195 |
-- Strips directories and .extensions (if present) from a filepath |
| 1196 |
local function filename_only(filepath) |
| 1197 |
local filename = filepath:match(".*%/([^%.]+)") |
| 1198 |
if not filename then |
| 1199 |
filename = filepath:match("([^%.]+)") |
| 1200 |
end |
| 1201 |
return filename |
| 1202 |
end |
| 1203 |
|
| 1204 |
assert(filename_only("very_simple") == "very_simple") |
| 1205 |
assert(filename_only("/home/very_simple.eml") == "very_simple") |
| 1206 |
assert(filename_only("very_simple.eml") == "very_simple") |
| 1207 |
assert(filename_only("very_simple.example.eml") == "very_simple") |
| 1208 |
assert(filename_only("/home/very_simple") == "very_simple") |
| 1209 |
assert(filename_only("home/very_simple") == "very_simple") |
| 1210 |
assert(filename_only("./home/very_simple") == "very_simple") |
| 1211 |
assert(filename_only("../home/very_simple.eml") == "very_simple") |
| 1212 |
assert(filename_only("/home/dir.with.dots/very_simple.eml") == "very_simple") |
| 1213 |
|
| 1214 |
--Write the dump content to file or standard out |
| 1215 |
local function write_dump_content(dump_content, fname, extension, outdir) |
| 1216 |
if type(dump_content) == "string" then |
| 1217 |
dump_content = rspamd_text.fromstring(dump_content) |
| 1218 |
end |
| 1219 |
|
| 1220 |
local wrote_filepath = nil |
| 1221 |
if outdir then |
| 1222 |
if outdir:sub(-1) ~= "/" then |
| 1223 |
outdir = outdir .. "/" |
| 1224 |
end |
| 1225 |
|
| 1226 |
local outpath = string.format("%s%s.%s", outdir, filename_only(fname), extension) |
| 1227 |
if rspamd_util.file_exists(outpath) then |
| 1228 |
os.remove(outpath) |
| 1229 |
end |
| 1230 |
if dump_content:save_in_file(outpath) then |
| 1231 |
wrote_filepath = outpath |
| 1232 |
io.write(wrote_filepath .. "\n") |
| 1233 |
else |
| 1234 |
io.stderr:write(string.format("Unable to save dump content to file: %s\n", outpath)) |
| 1235 |
end |
| 1236 |
else |
| 1237 |
dump_content:save_in_file(1) |
| 1238 |
end |
| 1239 |
return wrote_filepath |
| 1240 |
end |
| 1241 |
|
| 1242 |
-- Get the formatted ucl (split or unsplit) or the raw task content |
| 1243 |
local function get_dump_content(task, opts, fname) |
| 1244 |
if opts.ucl or opts.json or opts.messagepack then |
| 1245 |
local ucl_object = lua_mime.message_to_ucl(task) |
| 1246 |
-- Split out the content field into separate raws and update the ucl |
| 1247 |
if opts.split then |
| 1248 |
for i, part in ipairs(ucl_object.parts) do |
| 1249 |
if part.content then |
| 1250 |
local part_filename = string.format("%s-part%d", filename_only(fname), i) |
| 1251 |
local part_path = write_dump_content(part.content, part_filename, "raw", opts.outdir) |
| 1252 |
if part_path then |
| 1253 |
part.content = ucl.null |
| 1254 |
part.content_path = part_path |
| 1255 |
end |
| 1256 |
end |
| 1257 |
end |
| 1258 |
end |
| 1259 |
local extension = output_fmt(opts) |
| 1260 |
return ucl.to_format(ucl_object, extension), extension |
| 1261 |
end |
| 1262 |
return task:get_content(), "mime" |
| 1263 |
end |
| 1264 |
|
| 1265 |
local function dump_handler(opts) |
| 1266 |
load_config(opts, false) -- Dumping doesn't need custom tokenizers |
| 1267 |
rspamd_url.init(rspamd_config:get_tld_path()) |
| 1268 |
|
| 1269 |
for _, fname in ipairs(opts.file) do |
| 1270 |
local task = load_task(opts, fname) |
| 1271 |
local data, extension = get_dump_content(task, opts, fname) |
| 1272 |
write_dump_content(data, fname, extension, opts.outdir) |
| 1273 |
|
| 1274 |
task:destroy() -- No automatic dtor |
| 1275 |
end |
| 1276 |
end |
| 1277 |
|
| 1278 |
local function handler(args) |
| 1279 |
local opts = parser:parse(args) |
| 1280 |
|
| 1281 |
local command = opts.command |
| 1282 |
|
| 1283 |
if type(opts.file) == 'string' then |
| 1284 |
opts.file = { opts.file } |
| 1285 |
elseif type(opts.file) == 'none' then |
| 1286 |
opts.file = {} |
| 1287 |
end |
| 1288 |
|
| 1289 |
if command == 'extract' then |
| 1290 |
extract_handler(opts) |
| 1291 |
elseif command == 'stat' then |
| 1292 |
stat_handler(opts) |
| 1293 |
elseif command == 'urls' then |
| 1294 |
urls_handler(opts) |
| 1295 |
elseif command == 'modify' then |
| 1296 |
modify_handler(opts) |
| 1297 |
elseif command == 'strip' then |
| 1298 |
strip_handler(opts) |
| 1299 |
elseif command == 'sign' then |
| 1300 |
sign_handler(opts) |
| 1301 |
elseif command == 'dump' then |
| 1302 |
dump_handler(opts) |
| 1303 |
elseif command == 'anonymize' then |
| 1304 |
anonymize_handler(opts) |
| 1305 |
else |
| 1306 |
parser:error('command %s is not implemented', command) |
| 1307 |
end |
| 1308 |
end |
| 1309 |
|
| 1310 |
return { |
| 1311 |
name = 'mime', |
| 1312 |
aliases = { 'mime_tool' }, |
| 1313 |
handler = handler, |
| 1314 |
description = parser._description |
| 1315 |
} |