Debian Codegraph @

lualib/rspamadm/mime.lua

raw
1 --[[
2 Copyright (c) 2022, Vsevolod Stakhov <[email protected]>
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 ]] --
16
17 local argparse = require "argparse"
18 local ansicolors = require "ansicolors"
19 local rspamd_util = require "rspamd_util"
20 local rspamd_task = require "rspamd_task"
21 local rspamd_text = require "rspamd_text"
22 local rspamd_logger = require "rspamd_logger"
23 local lua_meta = require "lua_meta"
24 local rspamd_url = require "rspamd_url"
25 local lua_util = require "lua_util"
26 local lua_mime = require "lua_mime"
27 local ucl = require "ucl"
28
29 -- Define command line options
30 local parser = argparse()
31 :name "rspamadm mime"
32 :description "Mime manipulations provided by Rspamd"
33 :help_description_margin(30)
34 :command_target("command")
35 :require_command(true)
36
37 parser:option "-c --config"
38 :description "Path to config file"
39 :argname("<cfg>")
40 :default(rspamd_paths["CONFDIR"] .. "/" .. "rspamd.conf")
41 parser:mutex(
42 parser:flag "-j --json"
43 :description "JSON output",
44 parser:flag "-U --ucl"
45 :description "UCL output",
46 parser:flag "-M --messagepack"
47 :description "MessagePack output"
48 )
49 parser:flag "-C --compact"
50 :description "Use compact format"
51 parser:flag "--no-file"
52 :description "Do not print filename"
53
54 -- Extract subcommand
55 local extract = parser:command "extract ex e"
56 :description "Extracts data from MIME messages"
57 extract:argument "file"
58 :description "File to process"
59 :argname "<file>"
60 :args "+"
61
62 extract:flag "-t --text"
63 :description "Extracts plain text data from a message"
64 extract:flag "-r --raw"
65 :description "Load as raw file"
66 extract:flag "-H --html"
67 :description "Extracts htm data from a message"
68 extract:option "-o --output"
69 :description "Output format ('raw', 'content', 'oneline', 'decoded', 'decoded_utf')"
70 :argname("<type>")
71 :convert {
72 raw = "raw",
73 content = "content",
74 oneline = "content_oneline",
75 decoded = "raw_parsed",
76 decoded_utf = "raw_utf"
77 }
78 :default "content"
79 extract:flag "-w --words"
80 :description "Extracts words"
81 extract:flag "-p --part"
82 :description "Show part info"
83 extract:flag "-s --structure"
84 :description "Show structure info (e.g. HTML tags)"
85 extract:flag "-i --invisible"
86 :description "Show invisible content for HTML parts"
87 extract:option "-F --words-format"
88 :description "Words format ('stem', 'norm', 'raw', 'full')"
89 :argname("<type>")
90 :convert {
91 stem = "stem",
92 norm = "norm",
93 raw = "raw",
94 full = "full",
95 }
96 :default "stem"
97 extract:option "-L --limit"
98 :description "Maximum output size in bytes"
99 :argname("<bytes>")
100 :convert(tonumber)
101 extract:flag "-Q --strip-quotes"
102 :description "Remove quoted content (lines starting with >)"
103 extract:flag "-S --strip-signatures"
104 :description "Remove email signatures"
105 extract:flag "-R --strip-reply-headers"
106 :description "Remove reply headers (On X wrote:, From: Sent:)"
107 extract:flag "-T --smart-trim"
108 :description "Enable all text trimming heuristics"
109
110 local stat = parser:command "stat st s"
111 :description "Extracts statistical data from MIME messages"
112 stat:argument "file"
113 :description "File to process"
114 :argname "<file>"
115 :args "+"
116 stat:mutex(
117 stat:flag "-m --meta"
118 :description "Lua metatokens",
119 stat:flag "-b --bayes"
120 :description "Bayes tokens",
121 stat:flag "-F --fuzzy"
122 :description "Fuzzy hashes"
123 )
124 stat:flag "-s --shingles"
125 :description "Show shingles for fuzzy hashes"
126
127 local urls = parser:command "urls url u"
128 :description "Extracts URLs from MIME messages"
129 urls:argument "file"
130 :description "File to process"
131 :argname "<file>"
132 :args "+"
133 urls:mutex(
134 urls:flag "-t --tld"
135 :description "Get TLDs only",
136 urls:flag "-H --host"
137 :description "Get hosts only",
138 urls:flag "-f --full"
139 :description "Show piecewise urls as processed by Rspamd"
140 )
141
142 urls:flag "-u --unique"
143 :description "Print only unique urls"
144 urls:flag "-s --sort"
145 :description "Sort output"
146 urls:flag "--count"
147 :description "Print count of each printed element"
148 urls:flag "-r --reverse"
149 :description "Reverse sort order"
150 urls:flag "--raw"
151 :description "Load as raw file (for PDFs and other non-email files)"
152
153 local modify = parser:command "modify mod m"
154 :description "Modifies MIME message"
155 modify:argument "file"
156 :description "File to process"
157 :argname "<file>"
158 :args "+"
159
160 modify:option "-a --add-header"
161 :description "Adds specific header"
162 :argname "<header=value>"
163 :count "*"
164 modify:option "-r --remove-header"
165 :description "Removes specific header (all occurrences)"
166 :argname "<header>"
167 :count "*"
168 modify:option "-R --rewrite-header"
169 :description "Rewrites specific header, uses Lua string.format pattern"
170 :argname "<header=pattern>"
171 :count "*"
172 modify:option "-t --text-footer"
173 :description "Adds footer to text/plain parts from a specific file"
174 :argname "<file>"
175 modify:option "-H --html-footer"
176 :description "Adds footer to text/html parts from a specific file"
177 :argname "<file>"
178
179 local strip = parser:command "strip"
180 :description "Strip attachments from a message"
181 strip:argument "file"
182 :description "File to process"
183 :argname "<file>"
184 :args "+"
185 strip:flag "-i --keep-images"
186 :description "Keep images"
187 strip:option "--min-text-size"
188 :description "Minimal text size to keep"
189 :argname "<size>"
190 :convert(tonumber)
191 :default(0)
192 strip:option "--max-text-size"
193 :description "Max text size to keep"
194 :argname "<size>"
195 :convert(tonumber)
196 :default(math.huge)
197
198 local anonymize = parser:command "anonymize"
199 :description "Try to remove sensitive information from a message"
200 anonymize:argument "file"
201 :description "File to process"
202 :argname "<file>"
203 :args "+"
204 anonymize:option "--exclude-header -X"
205 :description "Exclude specific headers from anonymization"
206 :argname "<header>"
207 :count "*"
208 anonymize:option "--include-header -I"
209 :description "Include specific headers from anonymization"
210 :argname "<header>"
211 :count "*"
212 anonymize:flag "--gpt"
213 :description "Use LLM model for anonymization (requires GPT plugin to be configured)"
214 anonymize:option "--model"
215 :description "Model to use for anonymization"
216 :argname "<model>"
217 anonymize:option "--prompt"
218 :description "Prompt to use for anonymization"
219 :argname "<prompt>"
220
221 local sign = parser:command "sign"
222 :description "Performs DKIM signing"
223 sign:argument "file"
224 :description "File to process"
225 :argname "<file>"
226 :args "+"
227
228 sign:option "-d --domain"
229 :description "Use specific domain"
230 :argname "<domain>"
231 :count "1"
232 sign:option "-s --selector"
233 :description "Use specific selector"
234 :argname "<selector>"
235 :count "1"
236 sign:option "-k --key"
237 :description "Use specific key of file"
238 :argname "<key>"
239 :count "1"
240 sign:option "-t --type"
241 :description "ARC or DKIM signing"
242 :argname("<arc|dkim>")
243 :convert {
244 ['arc'] = 'arc',
245 ['dkim'] = 'dkim',
246 }
247 :default 'dkim'
248 sign:option "-o --output"
249 :description "Output format"
250 :argname("<message|signature>")
251 :convert {
252 ['message'] = 'message',
253 ['signature'] = 'signature',
254 }
255 :default 'message'
256
257 local dump = parser:command "dump"
258 :description "Dumps a raw message in different formats"
259 dump:argument "file"
260 :description "File to process"
261 :argname "<file>"
262 :args "+"
263 -- Duplicate format for convenience
264 dump:mutex(
265 parser:flag "-j --json"
266 :description "JSON output",
267 parser:flag "-U --ucl"
268 :description "UCL output",
269 parser:flag "-M --messagepack"
270 :description "MessagePack output"
271 )
272 dump:flag "-s --split"
273 :description "Split the output file contents such that no content is embedded"
274
275 dump:option "-o --outdir"
276 :description "Output directory"
277 :argname("<directory>")
278
279 local function load_config(opts, load_tokenizers)
280 local _r, err = rspamd_config:load_ucl(opts['config'])
281
282 if not _r then
283 rspamd_logger.errx('cannot parse %s: %s', opts['config'], err)
284 os.exit(1)
285 end
286
287 _r, err = rspamd_config:parse_rcl({ 'logging', 'worker' })
288 if not _r then
289 rspamd_logger.errx('cannot process %s: %s', opts['config'], err)
290 os.exit(1)
291 end
292
293 -- Load custom tokenizers if requested
294 if load_tokenizers then
295 local success, tokenizer_err = rspamd_config:load_custom_tokenizers()
296 if not success then
297 rspamd_logger.errx('cannot load custom tokenizers: %s', tokenizer_err or 'unknown error')
298 -- Don't exit here as custom tokenizers are optional
299 rspamd_logger.warnx('proceeding without custom tokenizers')
300 end
301 end
302 end
303
304 -- Helper function to ensure proper cleanup of tokenizers
305 local function cleanup_tokenizers()
306 if rspamd_config then
307 rspamd_config:unload_custom_tokenizers()
308 end
309 end
310
311 local function load_task(opts, fname)
312 if not fname then
313 fname = '-'
314 end
315
316 local task = rspamd_task.create(rspamd_config, rspamadm_ev_base)
317 task:set_session(rspamadm_session)
318 task:set_resolver(rspamadm_dns_resolver)
319
320 if opts.raw then
321 local f
322 if fname == '-' then
323 f = io.stdin
324 else
325 f = io.open(fname, "rb")
326 end
327
328 if not f then
329 parser:error("cannot open file " .. fname)
330 end
331
332 local content = f:read("*a")
333 if fname ~= '-' then
334 f:close()
335 end
336
337 local lua_magic = require "lua_magic"
338 local dummy_part = {
339 get_content = function()
340 return content
341 end,
342 get_filename = function()
343 return fname
344 end,
345 }
346 local _, type_data = lua_magic.detect(dummy_part, rspamd_config)
347 local ct = "application/octet-stream"
348 if type_data and type_data.type then
349 ct = type_data.type
350 end
351
352 if fname:match('%.pdf$') and (not ct or ct == 'application/octet-stream' or ct == 'binary') then
353 ct = 'application/pdf'
354 end
355
356 -- Construct message
357 local msg = string.format("Content-Type: %s\r\nContent-Transfer-Encoding: 8bit\r\n\r\n", ct)
358 task:load_from_string(msg .. content)
359 else
360 local res = task:load_from_file(fname)
361
362 if not res then
363 parser:error(string.format('cannot read message from %s: %s', fname,
364 task))
365 return nil
366 end
367 end
368
369 if not task:process_message() then
370 parser:error(string.format('cannot read message from %s: %s', fname,
371 'failed to parse'))
372 return nil
373 end
374
375 return task
376 end
377
378 local function highlight(fmt, ...)
379 return ansicolors.white .. string.format(fmt, ...) .. ansicolors.reset
380 end
381
382 local function maybe_print_fname(opts, fname)
383 if not opts.json and not opts['no-file'] then
384 rspamd_logger.messagex(highlight('File: %s', fname))
385 end
386 end
387
388 local function output_fmt(opts)
389 local fmt = 'json'
390 if opts.compact then
391 fmt = 'json-compact'
392 end
393 if opts.ucl then
394 fmt = 'ucl'
395 end
396 if opts.messagepack then
397 fmt = 'msgpack'
398 end
399
400 return fmt
401 end
402
403 -- Print elements in form
404 -- filename -> table of elements
405 local function print_elts(elts, opts, func)
406 local fun = require "fun"
407
408 if opts.json or opts.ucl then
409 io.write(ucl.to_format(elts, output_fmt(opts)))
410 else
411 fun.each(function(fname, elt)
412 if not opts.json and not opts.ucl then
413 if func then
414 elt = fun.map(func, elt)
415 end
416 maybe_print_fname(opts, fname)
417 fun.each(function(e)
418 io.write(e)
419 io.write("\n")
420 end, elt)
421 end
422 end, elts)
423 end
424 end
425
426 local function extract_handler(opts)
427 local out_elts = {}
428 local tasks = {}
429 local process_func
430
431 if opts.words then
432 -- Enable stemming and urls detection
433 load_config(opts, true) -- Load with custom tokenizers
434 rspamd_url.init(rspamd_config:get_tld_path())
435 rspamd_config:init_subsystem('langdet')
436 end
437
438 local function maybe_print_text_part_info(part, out)
439 local fun = require "fun"
440 if opts.part then
441 local t = 'plain text'
442 if part:is_html() then
443 t = 'html'
444 end
445
446 if not opts.json and not opts.ucl then
447 table.insert(out,
448 rspamd_logger.slog('Part: %s: %s, language: %s, size: %s (%s raw), words: %s',
449 part:get_mimepart():get_digest():sub(1, 8),
450 t,
451 part:get_language(),
452 part:get_length(), part:get_raw_length(),
453 part:get_words_count()))
454 table.insert(out,
455 rspamd_logger.slog('Stats: %s',
456 fun.foldl(function(acc, k, v)
457 if acc ~= '' then
458 return string.format('%s, %s:%s', acc, k, v)
459 else
460 return string.format('%s:%s', k, v)
461 end
462 end, '', part:get_stats())))
463 end
464 end
465 end
466
467 local function maybe_print_mime_part_info(part, out)
468 if opts.part then
469 if not opts.json and not opts.ucl then
470 local mtype, msubtype = part:get_type()
471 local det_mtype, det_msubtype = part:get_detected_type()
472 table.insert(out,
473 rspamd_logger.slog('Mime Part: %s: %s/%s (%s/%s detected), filename: %s (%s detected ext), size: %s',
474 part:get_digest():sub(1, 8),
475 mtype, msubtype,
476 det_mtype, det_msubtype,
477 part:get_filename(),
478 part:get_detected_ext(),
479 part:get_length()))
480 end
481 end
482 end
483
484 local function print_words(words, full)
485 local fun = require "fun"
486
487 if not full then
488 return table.concat(words, ' ')
489 else
490 return table.concat(
491 fun.totable(
492 fun.map(function(w)
493 -- [1] - stemmed word
494 -- [2] - normalised word
495 -- [3] - raw word
496 -- [4] - flags (table of strings)
497 return string.format('%s|%s|%s(%s)',
498 w[3], w[2], w[1], table.concat(w[4], ','))
499 end, words)
500 ),
501 ' '
502 )
503 end
504 end
505
506 for _, fname in ipairs(opts.file) do
507 local task = load_task(opts, fname)
508 out_elts[fname] = {}
509
510 if not opts.text and not opts.html then
511 opts.text = true
512 opts.html = true
513 end
514
515 -- Check if we use extract_text_limited options
516 if opts.limit or opts['strip_quotes'] or opts['strip_signatures'] or opts['smart_trim'] or opts['strip_reply_headers'] then
517 local res = lua_mime.extract_text_limited(task, {
518 max_bytes = opts.limit,
519 strip_quotes = opts['strip_quotes'],
520 strip_signatures = opts['strip_signatures'],
521 strip_reply_headers = opts['strip_reply_headers'],
522 smart_trim = opts['smart_trim']
523 })
524
525 if opts.json or opts.ucl then
526 table.insert(out_elts[fname], res)
527 else
528 if res.truncated then
529 table.insert(out_elts[fname], string.format("[Truncated] %s", res.text))
530 else
531 table.insert(out_elts[fname], res.text)
532 end
533
534 if opts.part then
535 table.insert(out_elts[fname], rspamd_logger.slog("Stats: %s", res.stats))
536 end
537 end
538
539 table.insert(out_elts[fname], "")
540 table.insert(tasks, task)
541
542 -- Skip normal processing
543 goto continue
544 end
545
546 if opts.words then
547 local how_words = opts['words_format'] or 'stem'
548 table.insert(out_elts[fname], 'meta_words: ' ..
549 print_words(task:get_meta_words(how_words), how_words == 'full'))
550 end
551
552 if opts.text or opts.html then
553 local mp_all = task:get_parts(true) or {}
554
555 -- Build map: parent_part -> injected_text_part
556 local injected_map = {}
557 for _, p in ipairs(mp_all) do
558 if p:is_injected() and p:is_text() then
559
560 local parent = p:get_parent()
561 if parent then
562 injected_map[parent:get_digest()] = p:get_text()
563 end
564 end
565 end
566
567 -- Build table: {{part, injected_text or nil}, ...}
568 local parts_to_process = {}
569 for _, p in ipairs(mp_all) do
570 if not p:is_injected() then
571 table.insert(parts_to_process, { p, injected_map[p:get_digest()] })
572 end
573 end
574
575 -- Process the parts
576 for _, entry in ipairs(parts_to_process) do
577 local mime_part = entry[1]
578 local injected_part = entry[2]
579 local how = opts.output
580 local part
581
582 if mime_part:is_text() then
583 part = mime_part:get_text()
584 end
585
586 if part and opts.text and not part:is_html() then
587 maybe_print_text_part_info(part, out_elts[fname])
588 maybe_print_mime_part_info(mime_part, out_elts[fname])
589 if not opts.json and not opts.ucl then
590 table.insert(out_elts[fname], '\n')
591 end
592
593 if opts.words then
594 local how_words = opts['words_format'] or 'stem'
595 table.insert(out_elts[fname], print_words(part:get_words(how_words),
596 how_words == 'full'))
597 else
598 table.insert(out_elts[fname], tostring(part:get_content(how)))
599 end
600 elseif injected_part and opts.text and not injected_part:is_html() then
601 -- Show parent part info but content from injected child
602 maybe_print_mime_part_info(mime_part, out_elts[fname])
603 if not opts.json and not opts.ucl then
604 table.insert(out_elts[fname], string.format('[Extracted text from %s]',
605 mime_part:get_filename() or 'attachment'))
606 table.insert(out_elts[fname], '\n')
607 end
608
609 if opts.words then
610 local how_words = opts['words_format'] or 'stem'
611 table.insert(out_elts[fname], print_words(injected_part:get_words(how_words),
612 how_words == 'full'))
613 else
614 table.insert(out_elts[fname], tostring(injected_part:get_content(how)))
615 end
616 elseif part and opts.html and part:is_html() then
617 maybe_print_text_part_info(part, out_elts[fname])
618 maybe_print_mime_part_info(mime_part, out_elts[fname])
619 if not opts.json and not opts.ucl then
620 table.insert(out_elts[fname], '\n')
621 end
622
623 if opts.words then
624 local how_words = opts['words_format'] or 'stem'
625 table.insert(out_elts[fname], print_words(part:get_words(how_words),
626 how_words == 'full'))
627 else
628 if opts.structure then
629 local hc = part:get_html()
630 local res = {}
631 process_func = function(elt)
632 local fun = require "fun"
633 if type(elt) == 'table' then
634 return table.concat(fun.totable(
635 fun.map(
636 function(t)
637 return rspamd_logger.slog("%s", t)
638 end,
639 elt)), '\n')
640 else
641 return rspamd_logger.slog("%s", elt)
642 end
643 end
644
645 hc:foreach_tag('any', function(tag)
646 local elt = {}
647 local ex = tag:get_extra()
648 elt.tag = tag:get_type()
649 if ex then
650 elt.extra = ex
651 end
652 local content = tag:get_content()
653 if content then
654 elt.content = tostring(content)
655 end
656 local style = tag:get_style()
657 if style then
658 elt.style = style
659 end
660 table.insert(res, elt)
661 end)
662 table.insert(out_elts[fname], res)
663 else
664 -- opts.structure
665 table.insert(out_elts[fname], tostring(part:get_content(how)))
666 end
667 if opts.invisible then
668 local hc = part:get_html()
669 table.insert(out_elts[fname], string.format('invisible content: %s',
670 tostring(hc:get_invisible())))
671 end
672 end
673 end
674
675 if not part then
676 maybe_print_mime_part_info(mime_part, out_elts[fname])
677 end
678 end
679 end
680
681 table.insert(out_elts[fname], "")
682 table.insert(tasks, task)
683
684 ::continue::
685 end
686
687 print_elts(out_elts, opts, process_func)
688 -- To avoid use after free we postpone tasks destruction
689 for _, task in ipairs(tasks) do
690 task:destroy()
691 end
692
693 -- Cleanup custom tokenizers if they were loaded
694 if opts.words then
695 cleanup_tokenizers()
696 end
697 end
698
699 local function stat_handler(opts)
700 local fun = require "fun"
701 local out_elts = {}
702
703 load_config(opts, true) -- Load with custom tokenizers for stat generation
704 rspamd_url.init(rspamd_config:get_tld_path())
705 rspamd_config:init_subsystem('langdet,stat') -- Needed to gen stat tokens
706
707 local process_func
708
709 for _, fname in ipairs(opts.file) do
710 local task = load_task(opts, fname)
711 out_elts[fname] = {}
712
713 if opts.meta then
714 local mt = lua_meta.gen_metatokens_table(task)
715 out_elts[fname] = mt
716 process_func = function(k, v)
717 return string.format("%s = %s", k, v)
718 end
719 elseif opts.bayes then
720 local bt = task:get_stat_tokens()
721 out_elts[fname] = bt
722 process_func = function(e)
723 return string.format('%s (%d): "%s"+"%s", [%s]', e.data, e.win, e.t1 or "",
724 e.t2 or "", table.concat(fun.totable(
725 fun.map(function(k)
726 return k
727 end, e.flags)), ","))
728 end
729 elseif opts.fuzzy then
730 local parts = task:get_parts() or {}
731 out_elts[fname] = {}
732 process_func = function(e)
733 local ret = string.format('part: %s(%s): %s', e.type, e.file or "", e.digest)
734 if opts.shingles and e.shingles then
735 local sgl = {}
736 for _, s in ipairs(e.shingles) do
737 table.insert(sgl, string.format('%s: %s+%s+%s', s[1], s[2], s[3], s[4]))
738 end
739
740 ret = ret .. '\n' .. table.concat(sgl, '\n')
741 end
742 return ret
743 end
744 for _, part in ipairs(parts) do
745 if not part:is_multipart() then
746 local text = part:get_text()
747
748 if text then
749 local digest, shingles = text:get_fuzzy_hashes(task:get_mempool(), task:get_subject())
750 table.insert(out_elts[fname], {
751 digest = digest,
752 shingles = shingles,
753 type = string.format('%s/%s',
754 ({ part:get_type() })[1],
755 ({ part:get_type() })[2])
756 })
757 else
758 table.insert(out_elts[fname], {
759 digest = part:get_digest(),
760 file = part:get_filename(),
761 type = string.format('%s/%s',
762 ({ part:get_type() })[1],
763 ({ part:get_type() })[2])
764 })
765 end
766 end
767 end
768 end
769
770 task:destroy() -- No automatic dtor
771 end
772
773 print_elts(out_elts, opts, process_func)
774
775 -- Cleanup custom tokenizers
776 cleanup_tokenizers()
777 end
778
779 local function urls_handler(opts)
780 load_config(opts, false) -- URLs don't need custom tokenizers
781 rspamd_url.init(rspamd_config:get_tld_path())
782 local out_elts = {}
783
784 if opts.json then
785 rspamd_logger.messagex('[')
786 end
787
788 for _, fname in ipairs(opts.file) do
789 out_elts[fname] = {}
790 local task = load_task(opts, fname)
791 local elts = {}
792
793 local function process_url(u)
794 local s
795 if opts.tld then
796 s = u:get_tld()
797 elseif opts.host then
798 s = u:get_host()
799 elseif opts.full then
800 s = rspamd_logger.slog('%s: %s', u:get_text(), u:to_table())
801 else
802 s = u:get_text()
803 end
804
805 if opts.unique then
806 if elts[s] then
807 elts[s].count = elts[s].count + 1
808 else
809 elts[s] = {
810 count = 1,
811 url = u:to_table()
812 }
813 end
814 else
815 if opts.json then
816 table.insert(elts, u)
817 else
818 table.insert(elts, s)
819 end
820 end
821 end
822
823 -- Use get_urls_filtered with nil params to get all URLs including content URLs
824 for _, u in ipairs(task:get_urls_filtered()) do
825 process_url(u)
826 end
827
828 local json_elts = {}
829
830 local function process_elt(s, u)
831 if opts.unique then
832 -- s is string, u is {url = url, count = count }
833 if not opts.json then
834 if opts.count then
835 table.insert(json_elts, string.format('%s : %s', s, u.count))
836 else
837 table.insert(json_elts, s)
838 end
839 else
840 local tb = u.url
841 tb.count = u.count
842 table.insert(json_elts, tb)
843 end
844 else
845 -- s is index, u is url or string
846 if opts.json then
847 table.insert(json_elts, u)
848 else
849 table.insert(json_elts, u)
850 end
851 end
852 end
853
854 if opts.sort then
855 local sfunc
856 if opts.unique then
857 sfunc = function(t, a, b)
858 if t[a].count ~= t[b].count then
859 if opts.reverse then
860 return t[a].count > t[b].count
861 else
862 return t[a].count < t[b].count
863 end
864 else
865 -- Sort lexicography
866 if opts.reverse then
867 return a > b
868 else
869 return a < b
870 end
871 end
872 end
873 else
874 sfunc = function(t, a, b)
875 local va, vb
876 if opts.json then
877 va = t[a]:get_text()
878 vb = t[b]:get_text()
879 else
880 va = t[a]
881 vb = t[b]
882 end
883 if opts.reverse then
884 return va > vb
885 else
886 return va < vb
887 end
888 end
889 end
890
891 for s, u in lua_util.spairs(elts, sfunc) do
892 process_elt(s, u)
893 end
894 else
895 for s, u in pairs(elts) do
896 process_elt(s, u)
897 end
898 end
899
900 out_elts[fname] = json_elts
901
902 task:destroy() -- No automatic dtor
903 end
904
905 print_elts(out_elts, opts)
906 end
907
908 local function newline(task)
909 local t = task:get_newlines_type()
910
911 if t == 'cr' then
912 return '\r'
913 elseif t == 'lf' then
914 return '\n'
915 end
916
917 return '\r\n'
918 end
919
920 local function modify_handler(opts)
921 load_config(opts, false) -- Modification doesn't need custom tokenizers
922 rspamd_url.init(rspamd_config:get_tld_path())
923
924 local function read_file(file)
925 local f = assert(io.open(file, "rb"))
926 local content = f:read("*all")
927 f:close()
928 return content
929 end
930
931 local text_footer, html_footer
932
933 if opts['text_footer'] then
934 text_footer = read_file(opts['text_footer'])
935 end
936
937 if opts['html_footer'] then
938 html_footer = read_file(opts['html_footer'])
939 end
940
941 for _, fname in ipairs(opts.file) do
942 local task = load_task(opts, fname)
943 local newline_s = newline(task)
944 local seen_cte
945
946 local rewrite = lua_mime.add_text_footer(task, html_footer, text_footer) or {}
947 local out = {} -- Start with headers
948
949 local function process_headers_cb(name, hdr)
950 for _, h in ipairs(opts['remove_header']) do
951 if name:match(h) then
952 return
953 end
954 end
955
956 for _, h in ipairs(opts['rewrite_header']) do
957 local hname, hpattern = h:match('^([^=]+)=(.+)$')
958 if hname == name then
959 local new_value = string.format(hpattern, hdr.decoded)
960 new_value = string.format('%s:%s%s',
961 name, hdr.separator,
962 rspamd_util.fold_header(name,
963 rspamd_util.mime_header_encode(new_value),
964 task:get_newlines_type()))
965 out[#out + 1] = new_value
966 return
967 end
968 end
969
970 if rewrite.need_rewrite_ct then
971 if name:lower() == 'content-type' then
972 local nct = string.format('%s: %s/%s; charset=utf-8',
973 'Content-Type', rewrite.new_ct.type, rewrite.new_ct.subtype)
974 out[#out + 1] = nct
975 return
976 elseif name:lower() == 'content-transfer-encoding' then
977 out[#out + 1] = string.format('%s: %s',
978 'Content-Transfer-Encoding', rewrite.new_cte or 'quoted-printable')
979 seen_cte = true
980 return
981 end
982 end
983
984 out[#out + 1] = hdr.raw:gsub('\r?\n?$', '')
985 end
986
987 task:headers_foreach(process_headers_cb, { full = true })
988
989 for _, h in ipairs(opts['add_header']) do
990 local hname, hvalue = h:match('^([^=]+)=(.+)$')
991
992 if hname and hvalue then
993 out[#out + 1] = string.format('%s: %s', hname,
994 rspamd_util.fold_header(hname, hvalue, task:get_newlines_type()))
995 end
996 end
997
998 if not seen_cte and rewrite.need_rewrite_ct then
999 out[#out + 1] = string.format('%s: %s',
1000 'Content-Transfer-Encoding', rewrite.new_cte or 'quoted-printable')
1001 end
1002
1003 -- End of headers
1004 out[#out + 1] = ''
1005
1006 if rewrite.out then
1007 for _, o in ipairs(rewrite.out) do
1008 out[#out + 1] = o
1009 end
1010 else
1011 out[#out + 1] = { task:get_rawbody(), false }
1012 end
1013
1014 for _, o in ipairs(out) do
1015 if type(o) == 'string' then
1016 io.write(o)
1017 io.write(newline_s)
1018 elseif type(o) == 'table' then
1019 io.flush()
1020 if type(o[1]) == 'string' then
1021 io.write(o[1])
1022 else
1023 o[1]:save_in_file(1)
1024 end
1025
1026 if o[2] then
1027 io.write(newline_s)
1028 end
1029 else
1030 o:save_in_file(1)
1031 io.write(newline_s)
1032 end
1033 end
1034
1035 task:destroy() -- No automatic dtor
1036 end
1037 end
1038
1039 local function sign_handler(opts)
1040 load_config(opts, false) -- Signing doesn't need custom tokenizers
1041 rspamd_url.init(rspamd_config:get_tld_path())
1042
1043 local lua_dkim = require("lua_ffi").dkim
1044
1045 if not lua_dkim then
1046 io.stderr:write('FFI support is required: please use LuaJIT or install lua-ffi')
1047 os.exit(1)
1048 end
1049
1050 local sign_key
1051 if rspamd_util.file_exists(opts.key) then
1052 sign_key = lua_dkim.load_sign_key(opts.key, 'file')
1053 else
1054 sign_key = lua_dkim.load_sign_key(opts.key, 'base64')
1055 end
1056
1057 if not sign_key then
1058 io.stderr:write('Cannot load key: ' .. opts.key .. '\n')
1059 os.exit(1)
1060 end
1061
1062 for _, fname in ipairs(opts.file) do
1063 local task = load_task(opts, fname)
1064 local ctx = lua_dkim.create_sign_context(task, sign_key, nil, opts.algorithm)
1065
1066 if not ctx then
1067 io.stderr:write('Cannot init signing\n')
1068 os.exit(1)
1069 end
1070
1071 local sig = lua_dkim.do_sign(task, ctx, opts.selector, opts.domain)
1072
1073 if not sig then
1074 io.stderr:write('Cannot create signature\n')
1075 os.exit(1)
1076 end
1077
1078 if opts.output == 'signature' then
1079 io.write(sig)
1080 io.write('\n')
1081 io.flush()
1082 else
1083 local dkim_hdr = string.format('%s: %s%s',
1084 'DKIM-Signature',
1085 rspamd_util.fold_header('DKIM-Signature',
1086 rspamd_util.mime_header_encode(sig),
1087 task:get_newlines_type()),
1088 newline(task))
1089 io.write(dkim_hdr)
1090 io.flush()
1091 task:get_content():save_in_file(1)
1092 end
1093
1094 task:destroy() -- No automatic dtor
1095 end
1096 end
1097
1098 local function strip_handler(opts)
1099 load_config(opts, false) -- Stripping doesn't need custom tokenizers
1100 rspamd_url.init(rspamd_config:get_tld_path())
1101
1102 for _, fname in ipairs(opts.file) do
1103 local task = load_task(opts, fname)
1104 local newline_s = newline(task)
1105
1106 local rewrite = lua_mime.remove_attachments(task, {
1107 keep_images = opts.keep_images,
1108 min_text_size = opts.min_text_size,
1109 max_text_size = opts.max_text_size
1110 }) or {}
1111 local out = {} -- Start with headers
1112
1113 local function process_headers_cb(name, hdr)
1114 out[#out + 1] = hdr.raw:gsub('\r?\n?$', '')
1115 end
1116
1117 task:headers_foreach(process_headers_cb, { full = true })
1118 -- End of headers
1119 out[#out + 1] = ''
1120
1121 if rewrite.out then
1122 for _, o in ipairs(rewrite.out) do
1123 out[#out + 1] = o
1124 end
1125 else
1126 out[#out + 1] = { task:get_rawbody(), false }
1127 end
1128
1129 for _, o in ipairs(out) do
1130 if type(o) == 'string' then
1131 io.write(o)
1132 io.write(newline_s)
1133 elseif type(o) == 'table' then
1134 io.flush()
1135 if type(o[1]) == 'string' then
1136 io.write(o[1])
1137 else
1138 o[1]:save_in_file(1)
1139 end
1140
1141 if o[2] then
1142 io.write(newline_s)
1143 end
1144 else
1145 o:save_in_file(1)
1146 io.write(newline_s)
1147 end
1148 end
1149
1150 task:destroy() -- No automatic dtor
1151 end
1152 end
1153
1154 local function anonymize_handler(opts)
1155 load_config(opts, false) -- Anonymization doesn't need custom tokenizers
1156 rspamd_url.init(rspamd_config:get_tld_path())
1157
1158 for _, fname in ipairs(opts.file) do
1159 local task = load_task(opts, fname)
1160 local newline_s = newline(task)
1161
1162 local rewrite = lua_mime.anonymize_message(task, opts)
1163
1164 if not rewrite or not rewrite.out then
1165 rspamd_logger.errx('cannot anonymize message from %s', fname)
1166 task:destroy()
1167 os.exit(1)
1168 end
1169
1170 for _, o in ipairs(rewrite.out) do
1171 if type(o) == 'string' then
1172 io.write(o)
1173 io.write(newline_s)
1174 elseif type(o) == 'table' then
1175 io.flush()
1176 if type(o[1]) == 'string' then
1177 io.write(o[1])
1178 else
1179 o[1]:save_in_file(1)
1180 end
1181
1182 if o[2] then
1183 io.write(newline_s)
1184 end
1185 else
1186 o:save_in_file(1)
1187 io.write(newline_s)
1188 end
1189 end
1190
1191 task:destroy() -- No automatic dtor
1192 end
1193 end
1194
1195 -- Strips directories and .extensions (if present) from a filepath
1196 local function filename_only(filepath)
1197 local filename = filepath:match(".*%/([^%.]+)")
1198 if not filename then
1199 filename = filepath:match("([^%.]+)")
1200 end
1201 return filename
1202 end
1203
1204 assert(filename_only("very_simple") == "very_simple")
1205 assert(filename_only("/home/very_simple.eml") == "very_simple")
1206 assert(filename_only("very_simple.eml") == "very_simple")
1207 assert(filename_only("very_simple.example.eml") == "very_simple")
1208 assert(filename_only("/home/very_simple") == "very_simple")
1209 assert(filename_only("home/very_simple") == "very_simple")
1210 assert(filename_only("./home/very_simple") == "very_simple")
1211 assert(filename_only("../home/very_simple.eml") == "very_simple")
1212 assert(filename_only("/home/dir.with.dots/very_simple.eml") == "very_simple")
1213
1214 --Write the dump content to file or standard out
1215 local function write_dump_content(dump_content, fname, extension, outdir)
1216 if type(dump_content) == "string" then
1217 dump_content = rspamd_text.fromstring(dump_content)
1218 end
1219
1220 local wrote_filepath = nil
1221 if outdir then
1222 if outdir:sub(-1) ~= "/" then
1223 outdir = outdir .. "/"
1224 end
1225
1226 local outpath = string.format("%s%s.%s", outdir, filename_only(fname), extension)
1227 if rspamd_util.file_exists(outpath) then
1228 os.remove(outpath)
1229 end
1230 if dump_content:save_in_file(outpath) then
1231 wrote_filepath = outpath
1232 io.write(wrote_filepath .. "\n")
1233 else
1234 io.stderr:write(string.format("Unable to save dump content to file: %s\n", outpath))
1235 end
1236 else
1237 dump_content:save_in_file(1)
1238 end
1239 return wrote_filepath
1240 end
1241
1242 -- Get the formatted ucl (split or unsplit) or the raw task content
1243 local function get_dump_content(task, opts, fname)
1244 if opts.ucl or opts.json or opts.messagepack then
1245 local ucl_object = lua_mime.message_to_ucl(task)
1246 -- Split out the content field into separate raws and update the ucl
1247 if opts.split then
1248 for i, part in ipairs(ucl_object.parts) do
1249 if part.content then
1250 local part_filename = string.format("%s-part%d", filename_only(fname), i)
1251 local part_path = write_dump_content(part.content, part_filename, "raw", opts.outdir)
1252 if part_path then
1253 part.content = ucl.null
1254 part.content_path = part_path
1255 end
1256 end
1257 end
1258 end
1259 local extension = output_fmt(opts)
1260 return ucl.to_format(ucl_object, extension), extension
1261 end
1262 return task:get_content(), "mime"
1263 end
1264
1265 local function dump_handler(opts)
1266 load_config(opts, false) -- Dumping doesn't need custom tokenizers
1267 rspamd_url.init(rspamd_config:get_tld_path())
1268
1269 for _, fname in ipairs(opts.file) do
1270 local task = load_task(opts, fname)
1271 local data, extension = get_dump_content(task, opts, fname)
1272 write_dump_content(data, fname, extension, opts.outdir)
1273
1274 task:destroy() -- No automatic dtor
1275 end
1276 end
1277
1278 local function handler(args)
1279 local opts = parser:parse(args)
1280
1281 local command = opts.command
1282
1283 if type(opts.file) == 'string' then
1284 opts.file = { opts.file }
1285 elseif type(opts.file) == 'none' then
1286 opts.file = {}
1287 end
1288
1289 if command == 'extract' then
1290 extract_handler(opts)
1291 elseif command == 'stat' then
1292 stat_handler(opts)
1293 elseif command == 'urls' then
1294 urls_handler(opts)
1295 elseif command == 'modify' then
1296 modify_handler(opts)
1297 elseif command == 'strip' then
1298 strip_handler(opts)
1299 elseif command == 'sign' then
1300 sign_handler(opts)
1301 elseif command == 'dump' then
1302 dump_handler(opts)
1303 elseif command == 'anonymize' then
1304 anonymize_handler(opts)
1305 else
1306 parser:error('command %s is not implemented', command)
1307 end
1308 end
1309
1310 return {
1311 name = 'mime',
1312 aliases = { 'mime_tool' },
1313 handler = handler,
1314 description = parser._description
1315 }