Debian Codegraph @

lualib/lua_mime.lua

raw
1 --[[
2 Copyright (c) 2022, Vsevolod Stakhov <[email protected]>
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 ]]--
16
17 --[[[
18 -- @module lua_mime
19 -- This module contains helper functions to modify mime parts
20 --]]
21
22 local logger = require "rspamd_logger"
23 local rspamd_util = require "rspamd_util"
24 local rspamd_text = require "rspamd_text"
25 local ucl = require "ucl"
26
27 local exports = {}
28
29 -- Default multilingual patterns for text extraction
30 -- These can be overridden via rspamd_config options
31 local default_signature_patterns = {
32 -- English
33 "Sent from my iPhone",
34 "Sent from my Android",
35 "Sent from my iPad",
36 "Sent from my mobile",
37 "Sent from Mail for Windows",
38 "Get Outlook for ",
39 "Sent from Samsung Mobile",
40 "Sent from Yahoo Mail",
41 "Sent from AOL Mobile Mail",
42 -- German
43 "Gesendet von meinem iPhone",
44 "Gesendet von meinem Android",
45 "Von meinem iPhone gesendet",
46 "Von meinem Samsung Galaxy gesendet",
47 "Mit freundlichen Grüßen",
48 "Mit freundlichem Gruß",
49 "Viele Grüße",
50 "Liebe Grüße",
51 "Herzliche Grüße",
52 -- French
53 "Envoyé de mon iPhone",
54 "Envoyé de mon Android",
55 "Envoyé depuis mon mobile",
56 "Cordialement",
57 "Bien cordialement",
58 "Salutations",
59 "Meilleures salutations",
60 "Sincères salutations",
61 -- Spanish
62 "Enviado desde mi iPhone",
63 "Enviado desde mi Android",
64 "Enviado desde mi móvil",
65 "Saludos cordiales",
66 "Un cordial saludo",
67 "Atentamente",
68 "Saludos",
69 -- Russian
70 "Отправлено с iPhone",
71 "Отправлено с Android",
72 "Отправлено с мобильного",
73 "С уважением",
74 "С наилучшими пожеланиями",
75 "С наилучшими",
76 "Всего наилучшего",
77 "Всего доброго",
78 -- Portuguese
79 "Enviado do meu iPhone",
80 "Enviado do meu Android",
81 "Atenciosamente",
82 "Abraços",
83 -- Italian
84 "Inviato da iPhone",
85 "Inviato da Android",
86 "Cordiali saluti",
87 "Distinti saluti",
88 -- Chinese (Simplified)
89 "发自我的 iPhone",
90 "发自我的 Android",
91 -- Japanese
92 "iPhoneから送信",
93 -- Polish
94 "Wysłano z iPhone'a",
95 "Wysłano z urządzenia Android",
96 "Z poważaniem",
97 "Pozdrawiam",
98 }
99
100 local default_reply_header_patterns = {
101 -- English
102 "From: ",
103 "----- Original Message -----",
104 "-------- Original Message --------",
105 "----- Forwarded message -----",
106 "Begin forwarded message:",
107 "_______________",
108 "________________________________",
109 -- German
110 "----- Ursprüngliche Nachricht -----",
111 "-------- Ursprüngliche Nachricht --------",
112 "----- Weitergeleitete Nachricht -----",
113 "Von: ",
114 "Gesendet: ",
115 "An: ",
116 "Betreff: ",
117 -- French
118 "----- Message d'origine -----",
119 "-------- Message original --------",
120 "----- Message transféré -----",
121 "De : ",
122 "Envoyé : ",
123 "À : ",
124 "Objet : ",
125 -- Spanish
126 "----- Mensaje original -----",
127 "-------- Mensaje original --------",
128 "----- Mensaje reenviado -----",
129 "De: ",
130 "Enviado: ",
131 "Para: ",
132 "Asunto: ",
133 -- Russian
134 "----- Исходное сообщение -----",
135 "-------- Исходное сообщение --------",
136 "----- Пересылаемое сообщение -----",
137 "От: ",
138 "Отправлено: ",
139 "Кому: ",
140 "Тема: ",
141 -- Portuguese
142 "----- Mensagem original -----",
143 -- Italian
144 "----- Messaggio originale -----",
145 }
146
147 local default_reply_wrote_patterns = {
148 -- English
149 "^On .*, .* wrote:$",
150 "wrote:$",
151 -- German
152 "schrieb:$",
153 -- French
154 "a écrit :$",
155 -- Spanish
156 "escribió:$",
157 -- Russian
158 "написал:$",
159 "написала:$",
160 }
161
162 -- Module-level cached maps (rebuilt when config changes)
163 local cached_sig_map = nil
164 local cached_reply_header_map = nil
165 local cached_reply_header_regexps = nil
166 local cached_reply_wrote_map = nil
167 local cached_reply_wrote_regexps = nil
168
169 local function merge_patterns(defaults, extra)
170 local out = {}
171 for _, pattern in ipairs(defaults) do
172 table.insert(out, pattern)
173 end
174 for _, pattern in ipairs(extra) do
175 table.insert(out, pattern)
176 end
177 return out
178 end
179
180 local function patterns_count(patterns)
181 if type(patterns) == 'table' and patterns[1] then
182 return #patterns
183 end
184 return nil
185 end
186
187 local function normalize_patterns(patterns)
188 if type(patterns) ~= 'table' or not patterns[1] then
189 return patterns
190 end
191
192 local out = {}
193 for _, pattern in ipairs(patterns) do
194 if type(pattern) == 'string' then
195 if pattern:match("%s") and not pattern:match('^["/]') then
196 pattern = '"' .. pattern .. '"'
197 end
198 table.insert(out, pattern)
199 end
200 end
201
202 return out
203 end
204
205 local function build_regexp_list(patterns)
206 if type(patterns) ~= 'table' or not patterns[1] then
207 return nil
208 end
209
210 local rspamd_regexp = require "rspamd_regexp"
211 local out = {}
212
213 for _, pattern in ipairs(patterns) do
214 if type(pattern) == 'string' then
215 if pattern:match('^".*"$') then
216 pattern = pattern:sub(2, -2)
217 end
218 local re = rspamd_regexp.create_cached(pattern)
219 if re then
220 out[#out + 1] = re
221 end
222 end
223 end
224
225 if #out == 0 then
226 return nil
227 end
228
229 return out
230 end
231
232 local function build_regexp_map(patterns, description, map_type)
233 local lua_maps = require "lua_maps"
234 map_type = map_type or 'regexp'
235
236 if type(patterns) == 'table' and type(patterns.get_key) == 'function' then
237 return patterns
238 end
239
240 if type(patterns) == 'table' and patterns[1] then
241 patterns = normalize_patterns(patterns)
242 if map_type == 'regexp_multi' or map_type == 'glob_multi' then
243 patterns = {
244 url = 'static',
245 data = patterns,
246 }
247 end
248 end
249
250 return lua_maps.map_add_from_ucl(patterns, map_type, description)
251 end
252
253 --[[[
254 -- @function lua_mime.configure_text_extraction(cfg)
255 -- Configures text extraction patterns from config
256 -- @param {table} cfg Configuration table with optional fields:
257 -- * signature_patterns: array of signature patterns or regexp map
258 -- * reply_header_patterns: array of reply header patterns or regexp map
259 -- * reply_header_wrote_patterns: array of reply header "wrote" patterns or regexp map
260 -- * extend_defaults: boolean - if true, adds to defaults instead of replacing
261 --]]
262 exports.configure_text_extraction = function(cfg)
263 cfg = cfg or {}
264
265 local sig_patterns = default_signature_patterns
266 local reply_patterns = default_reply_header_patterns
267 local reply_wrote_patterns = default_reply_wrote_patterns
268 local sig_count = #default_signature_patterns
269 local reply_count = #default_reply_header_patterns
270 local reply_wrote_count = #default_reply_wrote_patterns
271
272 if cfg.signature_patterns then
273 if cfg.extend_defaults and type(cfg.signature_patterns) == 'table' and cfg.signature_patterns[1] then
274 -- Merge with defaults
275 sig_patterns = merge_patterns(default_signature_patterns, cfg.signature_patterns)
276 sig_count = #sig_patterns
277 else
278 sig_patterns = cfg.signature_patterns
279 sig_count = patterns_count(sig_patterns)
280 if cfg.extend_defaults then
281 logger.infox(rspamd_config, 'text extraction: signature_patterns is a map definition, extend_defaults ignored')
282 end
283 end
284 end
285
286 if cfg.reply_header_patterns then
287 if cfg.extend_defaults and type(cfg.reply_header_patterns) == 'table' and cfg.reply_header_patterns[1] then
288 reply_patterns = merge_patterns(default_reply_header_patterns, cfg.reply_header_patterns)
289 reply_count = #reply_patterns
290 else
291 reply_patterns = cfg.reply_header_patterns
292 reply_count = patterns_count(reply_patterns)
293 if cfg.extend_defaults then
294 logger.infox(rspamd_config, 'text extraction: reply_header_patterns is a map definition, extend_defaults ignored')
295 end
296 end
297 end
298
299 if cfg.reply_header_wrote_patterns then
300 if cfg.extend_defaults and type(cfg.reply_header_wrote_patterns) == 'table' and cfg.reply_header_wrote_patterns[1] then
301 reply_wrote_patterns = merge_patterns(default_reply_wrote_patterns, cfg.reply_header_wrote_patterns)
302 reply_wrote_count = #reply_wrote_patterns
303 else
304 reply_wrote_patterns = cfg.reply_header_wrote_patterns
305 reply_wrote_count = patterns_count(reply_wrote_patterns)
306 if cfg.extend_defaults then
307 logger.infox(rspamd_config, 'text extraction: reply_header_wrote_patterns is a map definition, extend_defaults ignored')
308 end
309 end
310 end
311
312 cached_sig_map = build_regexp_map(sig_patterns, 'text extraction signature patterns', 'regexp')
313 cached_reply_header_map = build_regexp_map(reply_patterns, 'text extraction reply header patterns', 'regexp')
314 cached_reply_header_regexps = build_regexp_list(reply_patterns)
315 cached_reply_wrote_map = build_regexp_map(reply_wrote_patterns,
316 'text extraction reply header wrote patterns', 'regexp')
317 cached_reply_wrote_regexps = build_regexp_list(reply_wrote_patterns)
318
319 if not cached_sig_map then
320 logger.errx(rspamd_config, 'text extraction: cannot build signature map')
321 end
322
323 if not cached_reply_header_map then
324 logger.errx(rspamd_config, 'text extraction: cannot build reply header map')
325 end
326
327 if not cached_reply_wrote_map then
328 logger.errx(rspamd_config, 'text extraction: cannot build reply wrote map')
329 end
330
331 logger.infox(rspamd_config,
332 'text extraction configured: %s signature patterns, %s reply patterns, %s reply wrote patterns',
333 sig_count or 'map', reply_count or 'map', reply_wrote_count or 'map')
334 end
335
336 -- Get or create signature map
337 local function get_signature_map()
338 if cached_sig_map then
339 return cached_sig_map
340 end
341
342 cached_sig_map = build_regexp_map(default_signature_patterns,
343 'text extraction signature patterns (default)', 'regexp')
344 return cached_sig_map
345 end
346
347 -- Get or create reply header map
348 local function get_reply_header_map()
349 if cached_reply_header_map then
350 return cached_reply_header_map
351 end
352
353 cached_reply_header_map = build_regexp_map(default_reply_header_patterns,
354 'text extraction reply header patterns (default)', 'regexp')
355 cached_reply_header_regexps = build_regexp_list(default_reply_header_patterns)
356 return cached_reply_header_map
357 end
358
359 local function get_reply_wrote_map()
360 if cached_reply_wrote_map then
361 return cached_reply_wrote_map
362 end
363
364 cached_reply_wrote_map = build_regexp_map(default_reply_wrote_patterns,
365 'text extraction reply header wrote patterns (default)', 'regexp')
366 cached_reply_wrote_regexps = build_regexp_list(default_reply_wrote_patterns)
367 return cached_reply_wrote_map
368 end
369
370 local function newline(task)
371 local t = task:get_newlines_type()
372
373 if t == 'cr' then
374 return '\r'
375 elseif t == 'lf' then
376 return '\n'
377 end
378
379 return '\r\n'
380 end
381
382 local function do_append_footer(task, part, footer, is_multipart, out, state)
383 local tp = part:get_text()
384 local ct = 'text/plain'
385 local cte = 'quoted-printable'
386 local newline_s = state.newline_s
387
388 if tp:is_html() then
389 ct = 'text/html'
390 end
391
392 local encode_func = function(input)
393 return rspamd_util.encode_qp(input, 80, task:get_newlines_type())
394 end
395
396 if part:get_cte() == '7bit' then
397 cte = '7bit'
398 encode_func = function(input)
399 if type(input) == 'userdata' then
400 return input
401 else
402 return rspamd_text.fromstring(input)
403 end
404 end
405 end
406
407 if is_multipart then
408 out[#out + 1] = string.format('Content-Type: %s; charset=utf-8%s' ..
409 'Content-Transfer-Encoding: %s',
410 ct, newline_s, cte)
411 out[#out + 1] = ''
412 else
413 state.new_cte = cte
414 end
415
416 local content = tp:get_content('raw_utf') or ''
417 local double_nline = newline_s .. newline_s
418 local nlen = #double_nline
419 -- Hack, if part ends with 2 newline, then we append it after footer
420 if content:sub(-(nlen), nlen + 1) == double_nline then
421 -- content without last newline
422 content = content:sub(-(#newline_s), #newline_s + 1) .. footer
423 out[#out + 1] = { encode_func(content), true }
424 out[#out + 1] = ''
425 else
426 content = content .. footer
427 out[#out + 1] = { encode_func(content), true }
428 out[#out + 1] = ''
429 end
430
431 end
432
433 --[[[
434 -- @function lua_mime.add_text_footer(task, html_footer, text_footer)
435 -- Adds a footer to all text parts in a message. It returns a table with the following
436 -- fields:
437 -- * out: new content (body only)
438 -- * need_rewrite_ct: boolean field that means if we must rewrite content type
439 -- * new_ct: new content type (type => string, subtype => string)
440 -- * new_cte: new content-transfer encoding (string)
441 --]]
442 exports.add_text_footer = function(task, html_footer, text_footer)
443 local newline_s = newline(task)
444 local state = {
445 newline_s = newline_s
446 }
447 local out = {}
448 local text_parts = task:get_text_parts()
449
450 if not (html_footer or text_footer) or not (text_parts and #text_parts > 0) then
451 return false
452 end
453
454 if html_footer or text_footer then
455 -- We need to take extra care about content-type and cte
456 local ct = task:get_header('Content-Type')
457 if ct then
458 ct = rspamd_util.parse_content_type(ct, task:get_mempool())
459 end
460
461 if ct then
462 if ct.type and ct.type == 'text' then
463 if ct.subtype then
464 if html_footer and (ct.subtype == 'html' or ct.subtype == 'htm') then
465 state.need_rewrite_ct = true
466 elseif text_footer and ct.subtype == 'plain' then
467 state.need_rewrite_ct = true
468 end
469 else
470 if text_footer then
471 state.need_rewrite_ct = true
472 end
473 end
474
475 state.new_ct = ct
476 end
477 else
478
479 if text_parts then
480
481 if #text_parts == 1 then
482 state.need_rewrite_ct = true
483 state.new_ct = {
484 type = 'text',
485 subtype = 'plain'
486 }
487 elseif #text_parts > 1 then
488 -- XXX: in fact, it cannot be
489 state.new_ct = {
490 type = 'multipart',
491 subtype = 'mixed'
492 }
493 end
494 end
495 end
496 end
497
498 local boundaries = {}
499 local cur_boundary
500 for _, part in ipairs(task:get_parts()) do
501 local boundary = part:get_boundary()
502 local part_ct = part:get_header('Content-Type')
503 if part_ct then
504 part_ct = rspamd_util.parse_content_type(part_ct, task:get_mempool())
505 end
506 if part:is_multipart() then
507 if cur_boundary then
508 out[#out + 1] = string.format('--%s',
509 boundaries[#boundaries].boundary)
510 end
511
512 boundaries[#boundaries + 1] = {
513 boundary = boundary or '--XXX',
514 ct_type = part_ct.type or '',
515 ct_subtype = part_ct.subtype or '',
516 }
517 cur_boundary = boundary
518
519 local rh = part:get_raw_headers()
520 if #rh > 0 then
521 out[#out + 1] = { rh, true }
522 end
523 elseif part:is_message() then
524 if boundary then
525 if cur_boundary and boundary ~= cur_boundary then
526 -- Need to close boundary
527 out[#out + 1] = string.format('--%s--%s',
528 boundaries[#boundaries].boundary, newline_s)
529 table.remove(boundaries)
530 cur_boundary = nil
531 end
532 out[#out + 1] = string.format('--%s',
533 boundary)
534 end
535
536 out[#out + 1] = { part:get_raw_headers(), true }
537 else
538 local append_footer = false
539 local skip_footer = part:is_attachment()
540
541 local parent = part:get_parent()
542 if parent then
543 local t, st = parent:get_type()
544
545 if t == 'multipart' and st == 'signed' then
546 -- Do not modify signed parts
547 skip_footer = true
548 end
549 end
550 if text_footer and part:is_text() then
551 local tp = part:get_text()
552
553 if not tp:is_html() then
554 append_footer = text_footer
555 end
556 end
557
558 if html_footer and part:is_text() then
559 local tp = part:get_text()
560
561 if tp:is_html() then
562 append_footer = html_footer
563 end
564 end
565
566 if boundary then
567 if cur_boundary and boundary ~= cur_boundary then
568 -- Need to close boundary
569 out[#out + 1] = string.format('--%s--%s',
570 boundaries[#boundaries].boundary, newline_s)
571 -- Need to close previous boundary, if ct_subtype is related
572 if #boundaries > 1 and boundaries[#boundaries].ct_type == "multipart" and boundaries[#boundaries].ct_subtype == "related" then
573 out[#out + 1] = string.format('--%s--%s',
574 boundaries[#boundaries - 1].boundary, newline_s)
575 table.remove(boundaries)
576 end
577 table.remove(boundaries)
578 cur_boundary = boundary
579 end
580 out[#out + 1] = string.format('--%s',
581 boundary)
582 end
583
584 if append_footer and not skip_footer then
585 do_append_footer(task, part, append_footer,
586 parent and parent:is_multipart(), out, state)
587 else
588 out[#out + 1] = { part:get_raw_headers(), true }
589 out[#out + 1] = { part:get_raw_content(), false }
590 end
591 end
592 end
593
594 -- Close remaining
595 local b = table.remove(boundaries)
596 while b do
597 out[#out + 1] = string.format('--%s--', b.boundary)
598 if #boundaries > 0 then
599 out[#out + 1] = ''
600 end
601 b = table.remove(boundaries)
602 end
603
604 state.out = out
605
606 return state
607 end
608
609 local function do_replacement (task, part, mp, replacements,
610 is_multipart, out, state)
611
612 local tp = part:get_text()
613 local ct = 'text/plain'
614 local cte = 'quoted-printable'
615 local newline_s = state.newline_s
616
617 if tp:is_html() then
618 ct = 'text/html'
619 end
620
621 local encode_func = function(input)
622 return rspamd_util.encode_qp(input, 80, task:get_newlines_type())
623 end
624
625 if part:get_cte() == '7bit' then
626 cte = '7bit'
627 encode_func = function(input)
628 if type(input) == 'userdata' then
629 return input
630 else
631 return rspamd_text.fromstring(input)
632 end
633 end
634 end
635
636 local content = tp:get_content('raw_utf') or rspamd_text.fromstring('')
637 local match_pos = mp:match(content, true)
638
639 if match_pos then
640 -- sort matches and form the table:
641 -- start .. end for inclusion position
642 local matches_flattened = {}
643 for npat, matches in pairs(match_pos) do
644 for _, m in ipairs(matches) do
645 table.insert(matches_flattened, { m, npat })
646 end
647 end
648
649 -- Handle the case of empty match
650 if #matches_flattened == 0 then
651 out[#out + 1] = { part:get_raw_headers(), true }
652 out[#out + 1] = { part:get_raw_content(), false }
653
654 return
655 end
656
657 if is_multipart then
658 out[#out + 1] = { string.format('Content-Type: %s; charset="utf-8"%s' ..
659 'Content-Transfer-Encoding: %s',
660 ct, newline_s, cte), true }
661 out[#out + 1] = { '', true }
662 else
663 state.new_cte = cte
664 end
665
666 state.has_matches = true
667 -- now sort flattened by start of match and eliminate all overlaps
668 table.sort(matches_flattened, function(m1, m2)
669 return m1[1][1] < m2[1][1]
670 end)
671
672 for i = 1, #matches_flattened - 1 do
673 local st = matches_flattened[i][1][1] -- current start of match
674 local e = matches_flattened[i][1][2] -- current end of match
675 local max_npat = matches_flattened[i][2]
676 for j = i + 1, #matches_flattened do
677 if matches_flattened[j][1][1] == st then
678 -- overlap
679 if matches_flattened[j][1][2] > e then
680 -- larger exclusion and switch replacement
681 e = matches_flattened[j][1][2]
682 max_npat = matches_flattened[j][2]
683 end
684 else
685 break
686 end
687 end
688 -- Maximum overlap for all matches
689 for j = i, #matches_flattened do
690 if matches_flattened[j][1][1] == st then
691 if e > matches_flattened[j][1][2] then
692 matches_flattened[j][1][2] = e
693 matches_flattened[j][2] = max_npat
694 end
695 else
696 break
697 end
698 end
699 end
700 -- Off-by one: match returns 0 based positions while we use 1 based in Lua
701 for _, m in ipairs(matches_flattened) do
702 m[1][1] = m[1][1] + 1
703 m[1][2] = m[1][2] + 1
704 end
705
706 -- Now flattened match table is sorted by start pos and has the maximum overlapped pattern
707 -- Matches with the same start and end are covering the same replacement
708 -- e.g. we had something like [1 .. 2] -> replacement 1 and [1 .. 4] -> replacement 2
709 -- after flattening we should have [1 .. 4] -> 2 and [1 .. 4] -> 2
710 -- we can safely ignore those duplicates in the following code
711
712 local cur_start = 1
713 local fragments = {}
714 for _, m in ipairs(matches_flattened) do
715 if m[1][1] >= cur_start then
716 fragments[#fragments + 1] = content:sub(cur_start, m[1][1] - 1)
717 fragments[#fragments + 1] = replacements[m[2]]
718 cur_start = m[1][2] -- end of match
719 end
720 end
721
722 -- last part
723 if cur_start < #content then
724 fragments[#fragments + 1] = content:span(cur_start)
725 end
726
727 -- Final stuff
728 out[#out + 1] = { encode_func(rspamd_text.fromtable(fragments)), false }
729 else
730 -- No matches
731 out[#out + 1] = { part:get_raw_headers(), true }
732 out[#out + 1] = { part:get_raw_content(), false }
733 end
734 end
735
736 --[[[
737 -- @function lua_mime.multipattern_text_replace(task, mp, replacements)
738 -- Replaces text according to multipattern matches. It returns a table with the following
739 -- fields:
740 -- * out: new content (body only)
741 -- * need_rewrite_ct: boolean field that means if we must rewrite content type
742 -- * new_ct: new content type (type => string, subtype => string)
743 -- * new_cte: new content-transfer encoding (string)
744 --]]
745 exports.multipattern_text_replace = function(task, mp, replacements)
746 local newline_s = newline(task)
747 local state = {
748 newline_s = newline_s
749 }
750 local out = {}
751 local text_parts = task:get_text_parts()
752
753 if not mp or not (text_parts and #text_parts > 0) then
754 return false
755 end
756
757 -- We need to take extra care about content-type and cte
758 local ct = task:get_header('Content-Type')
759 if ct then
760 ct = rspamd_util.parse_content_type(ct, task:get_mempool())
761 end
762
763 if ct then
764 if ct.type and ct.type == 'text' then
765 state.need_rewrite_ct = true
766 state.new_ct = ct
767 end
768 else
769 -- No explicit CT, need to guess
770 if text_parts then
771 if #text_parts == 1 then
772 state.need_rewrite_ct = true
773 state.new_ct = {
774 type = 'text',
775 subtype = 'plain'
776 }
777 elseif #text_parts > 1 then
778 -- XXX: in fact, it cannot be
779 state.new_ct = {
780 type = 'multipart',
781 subtype = 'mixed'
782 }
783 end
784 end
785 end
786
787 local boundaries = {}
788 local cur_boundary
789 for _, part in ipairs(task:get_parts()) do
790 local boundary = part:get_boundary()
791 if part:is_multipart() then
792 if cur_boundary then
793 out[#out + 1] = { string.format('--%s',
794 boundaries[#boundaries]), true }
795 end
796
797 boundaries[#boundaries + 1] = boundary or '--XXX'
798 cur_boundary = boundary
799
800 local rh = part:get_raw_headers()
801 if #rh > 0 then
802 out[#out + 1] = { rh, true }
803 end
804 elseif part:is_message() then
805 if boundary then
806 if cur_boundary and boundary ~= cur_boundary then
807 -- Need to close boundary
808 out[#out + 1] = { string.format('--%s--',
809 boundaries[#boundaries]), true }
810 table.remove(boundaries)
811 cur_boundary = nil
812 end
813 out[#out + 1] = { string.format('--%s',
814 boundary), true }
815 end
816
817 out[#out + 1] = { part:get_raw_headers(), true }
818 else
819 local skip_replacement = part:is_attachment()
820
821 local parent = part:get_parent()
822 if parent then
823 local t, st = parent:get_type()
824
825 if t == 'multipart' and st == 'signed' then
826 -- Do not modify signed parts
827 skip_replacement = true
828 end
829 end
830 if not part:is_text() then
831 skip_replacement = true
832 end
833
834 if boundary then
835 if cur_boundary and boundary ~= cur_boundary then
836 -- Need to close boundary
837 out[#out + 1] = { string.format('--%s--',
838 boundaries[#boundaries]), true }
839 table.remove(boundaries)
840 cur_boundary = boundary
841 end
842 out[#out + 1] = { string.format('--%s',
843 boundary), true }
844 end
845
846 if not skip_replacement then
847 do_replacement(task, part, mp, replacements,
848 parent and parent:is_multipart(), out, state)
849 else
850 -- Append as is
851 out[#out + 1] = { part:get_raw_headers(), true }
852 out[#out + 1] = { part:get_raw_content(), false }
853 end
854 end
855 end
856
857 -- Close remaining
858 local b = table.remove(boundaries)
859 while b do
860 out[#out + 1] = { string.format('--%s--', b), true }
861 if #boundaries > 0 then
862 out[#out + 1] = { '', true }
863 end
864 b = table.remove(boundaries)
865 end
866
867 state.out = out
868
869 return state
870 end
871
872 --[[[
873 -- @function lua_mime.modify_headers(task, {add = {hname = {value = 'value', order = 1}}, remove = {hname = {1,2}}})
874 -- Adds/removes headers both internal and in the milter reply
875 -- Mode defines to be compatible with Rspamd <=3.2 and is the default (equal to 'compat')
876 --]]
877 exports.modify_headers = function(task, hdr_alterations, mode)
878 -- Assume default mode compatibility
879 if not mode then
880 mode = 'compat'
881 end
882 local add = hdr_alterations.add or {}
883 local remove = hdr_alterations.remove or {}
884
885 local add_headers = {} -- For Milter reply
886 local hdr_flattened = {} -- For C API
887
888 local function flatten_add_header(hname, hdr)
889 if not add_headers[hname] then
890 add_headers[hname] = {}
891 end
892 if not hdr_flattened[hname] then
893 hdr_flattened[hname] = { add = {} }
894 end
895 local add_tbl = hdr_flattened[hname].add
896 if hdr.value then
897 table.insert(add_headers[hname], {
898 order = (tonumber(hdr.order) or -1),
899 value = hdr.value,
900 })
901 table.insert(add_tbl, { tonumber(hdr.order) or -1, hdr.value })
902 elseif type(hdr) == 'table' then
903 for _, v in ipairs(hdr) do
904 flatten_add_header(hname, v)
905 end
906 elseif type(hdr) == 'string' then
907 table.insert(add_headers[hname], {
908 order = -1,
909 value = hdr,
910 })
911 table.insert(add_tbl, { -1, hdr })
912 else
913 logger.errx(task, 'invalid modification of header: %s', hdr)
914 end
915
916 if mode == 'compat' and #add_headers[hname] == 1 then
917 -- Switch to the compatibility mode
918 add_headers[hname] = add_headers[hname][1]
919 end
920 end
921 if hdr_alterations.order then
922 -- Get headers alterations ordered
923 for _, hname in ipairs(hdr_alterations.order) do
924 flatten_add_header(hname, add[hname])
925 end
926 else
927 for hname, hdr in pairs(add) do
928 flatten_add_header(hname, hdr)
929 end
930 end
931
932 for hname, hdr in pairs(remove) do
933 if not hdr_flattened[hname] then
934 hdr_flattened[hname] = { remove = {} }
935 end
936 if not hdr_flattened[hname].remove then
937 hdr_flattened[hname].remove = {}
938 end
939 local remove_tbl = hdr_flattened[hname].remove
940 local t_hdr = type(hdr)
941 if t_hdr == 'number' then
942 table.insert(remove_tbl, hdr)
943 elseif t_hdr == 'userdata' then
944 hdr_alterations.remove[hname] = nil
945 else
946 for _, num in ipairs(hdr) do
947 table.insert(remove_tbl, num)
948 end
949 end
950 end
951
952 if mode == 'compat' then
953 -- Clear empty alterations in the compat mode
954 if add_headers and not next(add_headers) then
955 add_headers = nil
956 end
957 if hdr_alterations.remove and not next(hdr_alterations.remove) then
958 hdr_alterations.remove = nil
959 end
960 end
961 task:set_milter_reply({
962 add_headers = add_headers,
963 remove_headers = hdr_alterations.remove
964 })
965
966 for hname, flat_rules in pairs(hdr_flattened) do
967 task:modify_header(hname, flat_rules)
968 end
969 end
970
971 --[[[
972 -- @function lua_mime.message_to_ucl(task, [stringify_content])
973 -- Exports a message to an ucl object
974 --]]
975 exports.message_to_ucl = function(task, stringify_content)
976 local E = {}
977
978 local maybe_stringify_f = stringify_content and
979 tostring or function(t)
980 return t
981 end
982 local result = {
983 size = task:get_size(),
984 digest = task:get_digest(),
985 newlines = task:get_newlines_type(),
986 headers = task:get_headers(true)
987 }
988
989 -- Utility to convert ip addr to a string or nil if invalid/absent
990 local function maybe_stringify_ip(addr)
991 if addr and addr:is_valid() then
992 return addr:to_string()
993 end
994
995 return nil
996 end
997
998 -- Envelope (smtp) information from email (nil if empty)
999 result.envelope = {
1000 from_smtp = (task:get_from('smtp') or E)[1],
1001 recipients_smtp = task:get_recipients('smtp'),
1002 helo = task:get_helo(),
1003 hostname = task:get_hostname(),
1004 client_ip = maybe_stringify_ip(task:get_client_ip()),
1005 from_ip = maybe_stringify_ip(task:get_from_ip()),
1006 }
1007 if not next(result.envelope) then
1008 result.envelope = ucl.null
1009 end
1010
1011 local parts = task:get_parts() or E
1012 result.parts = {}
1013 for _, part in ipairs(parts) do
1014 if not part:is_multipart() and not part:is_message() then
1015 local p = {
1016 size = part:get_length(),
1017 type = string.format('%s/%s', part:get_type()),
1018 detected_type = string.format('%s/%s', part:get_detected_type()),
1019 filename = part:get_filename(),
1020 content = maybe_stringify_f(part:get_content()),
1021 headers = part:get_headers(true) or E,
1022 boundary = part:get_enclosing_boundary(),
1023 }
1024 table.insert(result.parts, p)
1025 else
1026 -- Service part: multipart container or message/rfc822
1027 local p = {
1028 type = string.format('%s/%s', part:get_type()),
1029 headers = part:get_headers(true) or E,
1030 boundary = part:get_enclosing_boundary(),
1031 size = 0,
1032 }
1033
1034 if part:is_multipart() then
1035 p.multipart_boundary = part:get_boundary()
1036 end
1037
1038 table.insert(result.parts, p)
1039 end
1040 end
1041
1042 return result
1043 end
1044
1045 --[[[
1046 -- @function lua_mime.message_to_ucl_schema()
1047 -- Returns schema for a message to verify result/document fields
1048 --]]
1049 exports.message_to_ucl_schema = function()
1050 local T = require "lua_shape.core"
1051
1052 local function headers_schema()
1053 return T.table({
1054 order = T.integer():doc({ summary = 'Header order in a message' }),
1055 raw = T.string():doc({ summary = 'Raw header value' }):optional(),
1056 empty_separator = T.boolean():doc({ summary = 'Whether header has an empty separator' }),
1057 separator = T.string():doc({ summary = 'Separator between a header and a value' }),
1058 decoded = T.string():doc({ summary = 'Decoded value' }):optional(),
1059 value = T.string():doc({ summary = 'Decoded value' }):optional(),
1060 name = T.string():doc({ summary = 'Header name' }),
1061 tab_separated = T.boolean():doc({ summary = 'Whether header has tab as a separator' })
1062 })
1063 end
1064
1065 local function part_schema()
1066 return T.table({
1067 content = T.string():doc({ summary = 'Decoded content' }):optional(),
1068 multipart_boundary = T.string():doc({ summary = 'Multipart service boundary' }):optional(),
1069 size = T.integer():doc({ summary = 'Size of the part' }),
1070 type = T.string():doc({ summary = 'Announced type' }):optional(),
1071 detected_type = T.string():doc({ summary = 'Detected type' }):optional(),
1072 boundary = T.string():doc({ summary = 'Eclosing boundary' }):optional(),
1073 filename = T.string():doc({ summary = 'File name for attachments' }):optional(),
1074 headers = T.array(headers_schema()):doc({ summary = 'Part headers' }),
1075 })
1076 end
1077
1078 local function email_addr_schema()
1079 return T.table({
1080 addr = T.string():doc({ summary = 'Parsed address' }):optional(),
1081 raw = T.string():doc({ summary = 'Raw address' }),
1082 flags = T.table({
1083 valid = T.boolean():doc({ summary = 'Valid address' }):optional(),
1084 ip = T.boolean():doc({ summary = 'IP like address' }):optional(),
1085 braced = T.boolean():doc({ summary = 'Have braces around address' }):optional(),
1086 quoted = T.boolean():doc({ summary = 'Have quotes around address' }):optional(),
1087 empty = T.boolean():doc({ summary = 'Empty address' }):optional(),
1088 backslash = T.boolean():doc({ summary = 'Backslash in address' }):optional(),
1089 ['8bit'] = T.boolean():doc({ summary = '8 bit characters in address' }):optional(),
1090 }),
1091 user = T.string():doc({ summary = 'Parsed user part' }):optional(),
1092 name = T.string():doc({ summary = 'Displayed name' }):optional(),
1093 domain = T.string():doc({ summary = 'Parsed domain part' }):optional(),
1094 })
1095 end
1096 local function envelope_schema()
1097 return T.table({
1098 from_smtp = email_addr_schema():doc({ summary = 'SMTP from' }):optional(),
1099 recipients_smtp = T.array(email_addr_schema()):doc({ summary = 'SMTP recipients' }):optional(),
1100 helo = T.string():doc({ summary = 'SMTP Helo' }):optional(),
1101 hostname = T.string():doc({ summary = 'Sender hostname' }):optional(),
1102 client_ip = T.string():doc({ summary = 'Client ip' }):optional(),
1103 from_ip = T.string():doc({ summary = 'Sender ip' }):optional(),
1104 })
1105 end
1106
1107 return T.table({
1108 headers = T.array(headers_schema()),
1109 parts = T.array(part_schema()),
1110 digest = T.string({ pattern = string.format('^%s$', string.rep('%x', 32)) })
1111 :doc({ summary = 'Message digest' }),
1112 newlines = T.enum({ "cr", "lf", "crlf" }):doc({ summary = 'Newlines type' }),
1113 size = T.integer():doc({ summary = 'Size of the message in bytes' }),
1114 envelope = envelope_schema()
1115 })
1116 end
1117
1118 --[[[
1119 -- @function lua_mime.remove_attachments(task, settings)
1120 -- Removes all attachments from a message, keeping only text parts
1121 -- @param {task} task Rspamd task object
1122 -- @param {table} settings Table with the following fields:
1123 -- * keep_images: boolean, whether to keep inline images (default: false)
1124 -- * min_text_size: number, minimum text part size to keep (default: 0)
1125 -- * max_text_size: number, maximum text part size to keep (default: inf)
1126 -- @return {table} modified message state similar to other modification functions:
1127 -- * out: new content (body only)
1128 --]]
1129 exports.remove_attachments = function(task, settings)
1130 local newline_s = newline(task)
1131 local state = {
1132 newline_s = newline_s
1133 }
1134 local out = {}
1135
1136 settings = settings or {}
1137 local keep_images = settings.keep_images or false
1138 local min_text_size = settings.min_text_size or 0
1139 local max_text_size = settings.max_text_size or math.huge
1140
1141 -- Process message structure
1142 local boundaries = {}
1143 local cur_boundary
1144 local has_attachments = false
1145 local parts_to_keep = {}
1146 local parts_indexes_to_keep = {}
1147
1148 -- First pass: identify parts to keep
1149 for i, part in ipairs(task:get_parts()) do
1150 local keep_part = false
1151
1152 if part:is_text() and not part:is_attachment() then
1153 local length = part:get_length()
1154 if length >= min_text_size and length <= max_text_size then
1155 keep_part = true
1156 end
1157 elseif keep_images and part:is_image() then
1158 local cd = part:get_header('Content-Disposition')
1159 if cd and cd:lower():match('inline') then
1160 keep_part = true
1161 end
1162 end
1163
1164 if keep_part then
1165 table.insert(parts_to_keep, part)
1166 parts_indexes_to_keep[i] = true
1167 else
1168 has_attachments = true
1169 end
1170 end
1171
1172 -- If no attachments found, return false to indicate that no alterations are required
1173 if not has_attachments then
1174 return false
1175 end
1176
1177 -- Second pass: reconstruct message
1178 for i, part in ipairs(task:get_parts()) do
1179 local boundary = part:get_boundary()
1180 if part:is_multipart() then
1181 if cur_boundary then
1182 out[#out + 1] = { string.format('--%s',
1183 boundaries[#boundaries]), true }
1184 end
1185
1186 boundaries[#boundaries + 1] = boundary or '--XXX'
1187 cur_boundary = boundary
1188
1189 local rh = part:get_raw_headers()
1190 if #rh > 0 then
1191 out[#out + 1] = { rh, true }
1192 end
1193 elseif part:is_message() then
1194 if boundary then
1195 if cur_boundary and boundary ~= cur_boundary then
1196 -- Need to close boundary
1197 out[#out + 1] = { string.format('--%s--',
1198 boundaries[#boundaries]), true }
1199 table.remove(boundaries)
1200 cur_boundary = nil
1201 end
1202 out[#out + 1] = { string.format('--%s',
1203 boundary), true }
1204 end
1205
1206 out[#out + 1] = { part:get_raw_headers(), true }
1207 else
1208 if parts_indexes_to_keep[i] then
1209 if boundary then
1210 if cur_boundary and boundary ~= cur_boundary then
1211 -- Need to close previous boundary
1212 out[#out + 1] = { string.format('--%s--',
1213 boundaries[#boundaries]), true }
1214 table.remove(boundaries)
1215 cur_boundary = boundary
1216 end
1217 out[#out + 1] = { string.format('--%s',
1218 boundary), true }
1219 end
1220 -- Add part headers
1221 local headers = part:get_raw_headers()
1222
1223 if headers then
1224 out[#out + 1] = {
1225 headers,
1226 true
1227 }
1228 end
1229
1230 -- Add content
1231 out[#out + 1] = {
1232 part:get_raw_content(),
1233 false
1234 }
1235 end
1236
1237 end
1238 end
1239
1240 -- Close remaining boundaries
1241 local b = table.remove(boundaries)
1242 while b do
1243 out[#out + 1] = { string.format('--%s--', b), true }
1244 if #boundaries > 0 then
1245 out[#out + 1] = { '', true }
1246 end
1247 b = table.remove(boundaries)
1248 end
1249
1250 state.out = out
1251
1252 return state
1253 end
1254
1255 --[[[
1256 -- @function lua_mime.get_displayed_text_part(task[, min_words])
1257 -- Returns the most relevant displayed content from an email
1258 -- @param {task} task Rspamd task object
1259 -- @param {number} min_words optional minimum word count threshold (default 10)
1260 -- @return {text_part} a selected part
1261 --]]
1262 exports.get_displayed_text_part = function(task, min_words)
1263 min_words = min_words or 10
1264 local text_parts = task:get_text_parts()
1265 if not text_parts then
1266 return nil
1267 end
1268
1269 local html_part
1270 local text_part
1271 local html_attachment
1272 local text_attachment
1273
1274 -- First pass: categorize parts
1275 for _, part in ipairs(text_parts) do
1276 local mp = part:get_mimepart()
1277 if not mp:is_attachment() then
1278 if part:is_html() then
1279 html_part = part
1280 else
1281 text_part = text_part or part
1282 end
1283 else
1284 if part:is_html() and mp:get_length() < 102400 then
1285 -- 100KB limit, as long ones are likely not something that we should check
1286 html_attachment = part
1287 elseif not part:is_html() and mp:get_length() < 102400 then
1288 text_attachment = text_attachment or part
1289 end
1290 end
1291 end
1292
1293 -- Decision logic
1294 if html_part then
1295 local word_count = html_part:get_words_count() or 0
1296 if word_count >= min_words then
1297 return html_part
1298 end
1299 end
1300
1301 if text_part then
1302 local word_count = text_part:get_words_count() or 0
1303 if word_count >= min_words then
1304 return text_part
1305 end
1306 end
1307
1308 if html_attachment then
1309 return html_attachment
1310 end
1311
1312 if text_attachment then
1313 return text_attachment
1314 end
1315
1316 -- Only short parts, but still let's try our best
1317 return html_part or text_part
1318 end
1319
1320 --[[[
1321 -- @function lua_mime.get_distinct_text_parts(task)
1322 -- Returns the list of parts that are visible or have a distinct content
1323 -- @param {task} task Rspamd task object
1324 -- @return array of {text_part} a selected part
1325 --]]
1326 exports.get_distinct_text_parts = function(task)
1327 local text_parts = task:get_text_parts()
1328 if not text_parts then
1329 return {}
1330 end
1331
1332 local text_part_idx
1333
1334 local distance = task:get_mempool():get_variable('parts_distance', 'double')
1335 if not distance then
1336 return text_parts
1337 end
1338 distance = tonumber(distance)
1339
1340 if distance > 0.5 then
1341 -- Parts are distinct
1342 return text_parts
1343 end
1344
1345 -- First pass: categorize parts
1346 for i, part in ipairs(text_parts) do
1347 local mp = part:get_mimepart()
1348 if not mp:is_attachment() then
1349 if not part:is_html() then
1350 -- Found text part that is similar to html part
1351 text_part_idx = i
1352 end
1353 end
1354 end
1355
1356 if text_part_idx then
1357 table.remove(text_parts, text_part_idx)
1358 end
1359
1360 return text_parts
1361 end
1362
1363 --[[[
1364 -- @function lua_mime.anonymize_message(task, settings)
1365 -- Anonymizes message content by replacing sensitive data
1366 -- @param {task} task Rspamd task object
1367 -- @param {table} settings Table with the following fields:
1368 -- * strip_attachments: boolean, whether to strip all attachments
1369 -- * custom_header_process: table of header_name => function(orig_header) pairs
1370 -- @return {table} modified message state similar to other modification functions
1371 --]]
1372 exports.anonymize_message = function(task, settings)
1373 local rspamd_re = require "rspamd_regexp"
1374 local lua_util = require "lua_util"
1375
1376 logger.debugm('lua_mime', task, 'anonymize_message: starting, gpt mode: %s', settings.gpt or false)
1377
1378 -- We exclude words with digits, currency symbols and so on
1379 local exclude_words_re = rspamd_re.create_cached([=[/^(?:\d+|\d+\D{1,3}|\p{Sc}.*|(\+?\d{1,3}[\s\-]?)?)$/(:?^[[:alpha:]]*\d{4,}.*$)/u]=])
1380 local newline_s = newline(task)
1381 local state = {
1382 newline_s = newline_s
1383 }
1384 local out = {}
1385
1386 -- Default header processors
1387 local function anonymize_email_header(hdr)
1388 local addrs = rspamd_util.parse_mail_address(hdr.value, task:get_mempool())
1389 if addrs and addrs[1] then
1390 local modified = {}
1391 for _, addr in ipairs(addrs) do
1392 table.insert(modified, string.format('anonymous@%s', addr.domain or 'example.com'))
1393 end
1394
1395 return table.concat(modified, ',')
1396 end
1397 return '[email protected]'
1398 end
1399
1400 local function anonymize_received_header(hdr)
1401 local processed = string.gsub(hdr.value, '%d+%.%d+%.%d+%.%d+', 'x.x.x.x')
1402 processed = string.gsub(processed, '%x+:%x+:%x+:%x+:%x+:%x+:%x+:%x+', 'x:x:x:x:x:x:x:x')
1403 -- Anonymize email addresses in "for <[email protected]>" clauses
1404 processed = string.gsub(processed, 'for%s+<([^@>]+)@([^>]+)>', 'for <anonymous@%2>')
1405 -- Anonymize email addresses in "envelope-from <[email protected]>" clauses
1406 processed = string.gsub(processed, 'envelope%-from%s+<([^@>]+)@([^>]+)>', 'envelope-from <anonymous@%2>')
1407 return processed
1408 end
1409
1410 local function remove_header(hdr)
1411 -- Return nil to remove the header
1412 return nil
1413 end
1414
1415 local function anonymize_subject_header(hdr)
1416 -- Will be replaced by LLM anonymization if GPT mode is enabled
1417 -- Otherwise use generic subject
1418 return 'Email message'
1419 end
1420
1421 local default_header_process = {
1422 ['from'] = anonymize_email_header,
1423 ['to'] = anonymize_email_header,
1424 ['cc'] = anonymize_email_header,
1425 ['bcc'] = anonymize_email_header,
1426 ['return-path'] = anonymize_email_header,
1427 ['delivered-to'] = anonymize_email_header,
1428 ['received'] = anonymize_received_header,
1429 ['dkim-signature'] = remove_header,
1430 ['arc-seal'] = remove_header,
1431 ['arc-message-signature'] = remove_header,
1432 ['arc-authentication-results'] = remove_header,
1433 ['authentication-results'] = remove_header,
1434 ['x-spamd-result'] = remove_header,
1435 ['x-rspamd-server'] = remove_header,
1436 ['x-rspamd-queue-id'] = remove_header,
1437 ['subject'] = anonymize_subject_header,
1438 ['thread-topic'] = anonymize_subject_header,
1439 }
1440
1441 -- Merge with custom processors
1442 local header_processors = settings.custom_header_process or {}
1443 for k, v in pairs(default_header_process) do
1444 if not header_processors[k] then
1445 header_processors[k] = v
1446 end
1447 end
1448
1449 -- Process headers
1450 local all_include = true
1451 local all_exclude = false
1452
1453 -- Convert strings list to a list of globs where possible
1454 local function process_exceptions_list(list)
1455 if list and #list > 0 then
1456 for i, hdr in ipairs(list) do
1457 local gl = rspamd_re.import_glob(hdr, 'i')
1458 if gl then
1459 list[i] = gl
1460 end
1461 end
1462 return true
1463 end
1464 end
1465
1466 local function maybe_match_header(hdr, list)
1467 if not list then
1468 return false
1469 end
1470 for _, expr in ipairs(list) do
1471 if type(expr) == 'userdata' then
1472 if expr:match(hdr) then
1473 return true
1474 end
1475 else
1476 if expr:lower() == hdr:lower() then
1477 return true
1478 end
1479 end
1480 end
1481 return false
1482 end
1483
1484 if process_exceptions_list(settings.include_header) then
1485 all_include = false
1486 all_exclude = true
1487 end
1488 if process_exceptions_list(settings.exclude_header) then
1489 all_exclude = true
1490 end
1491
1492 local modified_headers = {}
1493 local function process_hdr(name, hdr)
1494 local include_hdr = (all_include and not maybe_match_header(name, settings.exclude_header)) or
1495 (all_exclude and maybe_match_header(name, settings.include_header))
1496 if include_hdr then
1497 local processor = header_processors[name:lower()]
1498 if processor then
1499 local new_value = processor(hdr)
1500 if new_value then
1501 table.insert(modified_headers, {
1502 name = name,
1503 value = new_value
1504 })
1505 end
1506 else
1507 table.insert(modified_headers, {
1508 name = name,
1509 value = hdr.value
1510 })
1511 end
1512 end
1513 end
1514
1515 task:headers_foreach(process_hdr, { full = true })
1516
1517 logger.debugm('lua_mime', task, 'anonymize_message: processed %s headers', #modified_headers)
1518
1519 -- Create new text content
1520 local text_content = {}
1521 local urls = {}
1522 local emails = {}
1523
1524 local sel_part = exports.get_displayed_text_part(task)
1525
1526 if not sel_part then
1527 logger.warnx(task, 'anonymize_message: no displayed text part found')
1528 return false
1529 end
1530
1531 logger.debugm('lua_mime', task, 'anonymize_message: selected text part, is_html: %s, length: %s',
1532 sel_part:is_html(), sel_part:get_length())
1533
1534 if settings.gpt then
1535 -- LLM version
1536 logger.debugm('lua_mime', task, 'anonymize_message: using GPT mode')
1537 local gpt_settings = rspamd_config:get_all_opt('gpt')
1538
1539 if not gpt_settings then
1540 logger.errx(task, 'anonymize_message: no gpt settings found in config')
1541 return false
1542 end
1543
1544 logger.debugm('lua_mime', task, 'anonymize_message: loaded gpt settings, type: %s', gpt_settings.type)
1545
1546 -- Get original Subject and Thread-Topic for anonymization
1547 local orig_subject = task:get_header('Subject') or ''
1548 local orig_thread_topic = task:get_header('Thread-Topic') or ''
1549
1550 -- Prepare the LLM request
1551 local function send_to_llm(input_content, subject, thread_topic)
1552 local rspamd_http = require 'rspamd_http'
1553
1554 logger.debugm('lua_mime', task, 'anonymize_message: preparing LLM request, content length: %s bytes',
1555 #tostring(input_content))
1556
1557 -- settings for LLM API
1558 local llm_settings = lua_util.override_defaults(gpt_settings, {
1559 api_key = settings.api_key,
1560 model = settings.model,
1561 timeout = settings.timeout,
1562 url = settings.url,
1563 })
1564
1565 -- Check for model-specific parameters
1566 if gpt_settings.model_parameters and llm_settings.model then
1567 local model_params = gpt_settings.model_parameters[llm_settings.model]
1568 if model_params then
1569 logger.debugm('lua_mime', task, 'anonymize_message: found model-specific parameters for %s',
1570 llm_settings.model)
1571 llm_settings = lua_util.override_defaults(llm_settings, model_params)
1572 end
1573 end
1574
1575 logger.debugm('lua_mime', task, 'anonymize_message: using LLM %s, model: %s, url: %s',
1576 llm_settings.type or 'unknown', llm_settings.model or 'default', llm_settings.url)
1577
1578 -- Build the system prompt with subject information
1579 local base_prompt = settings.prompt or [[You are a privacy-focused email anonymization assistant. Your task is to remove all personally identifiable information (PII) from emails while preserving their structure and meaning.
1580
1581 Remove or anonymize:
1582 - Real names (replace with "Person A", "Person B", etc.)
1583 - Email addresses (replace with "[email protected]" format)
1584 - Phone numbers (replace with "XXX-XXX-XXXX" format)
1585 - Physical addresses (replace with "City, Country" format)
1586 - Organization names (replace with generic terms like "Company A", "Organization B")
1587 - Account numbers, IDs, and credentials
1588 - IP addresses (replace with "X.X.X.X" format)
1589 - Dates that could identify individuals (keep year if relevant to context)
1590 - URLs (keep domain only if relevant, anonymize paths)
1591
1592 Preserve:
1593 - The overall message structure and flow
1594 - Technical terms and generic concepts
1595 - The general topic and context
1596 - Sentiment and tone
1597
1598 Response format:
1599 First line must be: "SUBJECT: <anonymized subject line>"
1600 Then a blank line
1601 Then the anonymized email content
1602
1603 The anonymized subject should preserve the general topic but remove all PII. Keep it concise and relevant.
1604
1605 Example:
1606 SUBJECT: Discussion about project timeline
1607
1608 <anonymized email content here>
1609
1610 Return ONLY the response in this format without any explanations, markdown formatting, or meta-commentary.]]
1611
1612 -- Add subject context to the prompt
1613 llm_settings.prompt = base_prompt .. string.format("\n\nThe original email subject is: %s",
1614 subject and subject ~= '' and subject or 'No subject')
1615
1616 logger.debugm('lua_mime', task, 'anonymize_message: prepared LLM prompt with subject: %s',
1617 subject and subject ~= '' and subject or 'No subject')
1618
1619 local request_body
1620 if llm_settings.type == 'anthropic' or llm_settings.type == 'claude' then
1621 -- Claude/Anthropic API format
1622 request_body = {
1623 model = llm_settings.model,
1624 max_tokens = llm_settings.max_tokens or llm_settings.max_completion_tokens or 4096,
1625 system = llm_settings.prompt,
1626 messages = {
1627 {
1628 role = 'user',
1629 content = input_content
1630 }
1631 }
1632 }
1633 -- Add temperature if configured
1634 if llm_settings.temperature then
1635 request_body.temperature = llm_settings.temperature
1636 end
1637 else
1638 -- OpenAI/Ollama API format
1639 request_body = {
1640 model = llm_settings.model,
1641 messages = {
1642 {
1643 role = 'system',
1644 content = llm_settings.prompt
1645 },
1646 {
1647 role = 'user',
1648 content = input_content
1649 }
1650 }
1651 }
1652 end
1653
1654 -- Add temperature if configured (only for OpenAI/Ollama, Claude handles it above)
1655 if llm_settings.temperature and llm_settings.type ~= 'anthropic' and llm_settings.type ~= 'claude' then
1656 request_body.temperature = llm_settings.temperature
1657 end
1658
1659 -- Add max tokens parameter - only for OpenAI/Ollama (Claude already has it)
1660 if not (llm_settings.type == 'anthropic' or llm_settings.type == 'claude') then
1661 if llm_settings.max_completion_tokens then
1662 -- Model-specific config uses new parameter name
1663 request_body.max_completion_tokens = llm_settings.max_completion_tokens
1664 elseif llm_settings.max_tokens then
1665 -- Use legacy parameter or convert based on API type
1666 if llm_settings.type == 'openai' then
1667 request_body.max_completion_tokens = llm_settings.max_tokens
1668 else
1669 request_body.max_tokens = llm_settings.max_tokens
1670 end
1671 end
1672 end
1673
1674 -- Ollama-specific settings
1675 if llm_settings.type == 'ollama' then
1676 request_body.stream = false
1677 logger.debugm('lua_mime', task, 'anonymize_message: disabled streaming for ollama')
1678 end
1679
1680 -- Prepare HTTP headers based on API type
1681 local headers
1682 if llm_settings.type == 'anthropic' or llm_settings.type == 'claude' then
1683 headers = {
1684 ['x-api-key'] = llm_settings.api_key,
1685 ['anthropic-version'] = llm_settings.anthropic_version or '2023-06-01',
1686 ['Content-Type'] = 'application/json'
1687 }
1688 else
1689 headers = {
1690 ['Authorization'] = 'Bearer ' .. llm_settings.api_key,
1691 ['Content-Type'] = 'application/json'
1692 }
1693 end
1694
1695 -- Make the HTTP request to the LLM API
1696 local http_params = {
1697 url = llm_settings.url,
1698 headers = headers,
1699 body = ucl.to_format(request_body, 'json-compact'),
1700 method = 'POST',
1701 task = task,
1702 timeout = llm_settings.timeout,
1703 }
1704
1705 logger.debugm('lua_mime', task, 'anonymize_message: sending HTTP request to LLM, timeout: %s',
1706 llm_settings.timeout or 'default')
1707
1708 local err, data = rspamd_http.request(http_params)
1709
1710 if err then
1711 logger.errx(task, 'anonymize_message: LLM request failed: %s', err)
1712 return false
1713 end
1714
1715 logger.debugm('lua_mime', task, 'anonymize_message: LLM response received, size: %s bytes',
1716 data.content and #data.content or 0)
1717
1718 local parser = ucl.parser()
1719 local res, parse_err = parser:parse_string(data.content)
1720 if not res then
1721 logger.errx(task, 'anonymize_message: cannot parse LLM response: %s', parse_err)
1722 return false
1723 end
1724
1725 local reply = parser:get_object()
1726 logger.debugm('lua_mime', task, 'anonymize_message: parsed LLM response successfully')
1727
1728 -- Log the response structure for debugging
1729 logger.debugm('lua_mime', task, 'anonymize_message: response structure: %s',
1730 logger.slog('%1', reply))
1731
1732 -- Check for API errors in response
1733 if reply.error then
1734 logger.errx(task, 'anonymize_message: LLM API returned error: %s (type: %s, code: %s)',
1735 reply.error.message or 'unknown', reply.error.type or 'unknown', reply.error.code or 'unknown')
1736 return false
1737 end
1738
1739 local anonymized_content
1740 local finish_reason
1741 if llm_settings.type == 'anthropic' or llm_settings.type == 'claude' then
1742 logger.debugm('lua_mime', task, 'anonymize_message: extracting content from Claude/Anthropic response')
1743 logger.debugm('lua_mime', task, 'anonymize_message: reply.content exists: %s, type: %s',
1744 reply.content ~= nil, type(reply.content))
1745 if reply.content and reply.content[1] then
1746 logger.debugm('lua_mime', task, 'anonymize_message: reply.content[1] exists, has text: %s',
1747 reply.content[1].text ~= nil)
1748 anonymized_content = reply.content[1].text
1749 finish_reason = reply.stop_reason
1750 end
1751 elseif llm_settings.type == 'openai' then
1752 logger.debugm('lua_mime', task, 'anonymize_message: extracting content from OpenAI response')
1753 logger.debugm('lua_mime', task, 'anonymize_message: reply.choices exists: %s, type: %s',
1754 reply.choices ~= nil, type(reply.choices))
1755 if reply.choices and reply.choices[1] then
1756 logger.debugm('lua_mime', task, 'anonymize_message: reply.choices[1] exists, has message: %s',
1757 reply.choices[1].message ~= nil)
1758 if reply.choices[1].message then
1759 logger.debugm('lua_mime', task, 'anonymize_message: reply.choices[1].message.content exists: %s',
1760 reply.choices[1].message.content ~= nil)
1761 end
1762 anonymized_content = reply.choices[1].message and reply.choices[1].message.content
1763 finish_reason = reply.choices[1].finish_reason
1764 end
1765 elseif llm_settings.type == 'ollama' then
1766 logger.debugm('lua_mime', task, 'anonymize_message: extracting content from Ollama response')
1767 logger.debugm('lua_mime', task, 'anonymize_message: reply.message exists: %s',
1768 reply.message ~= nil)
1769 anonymized_content = reply.message and reply.message.content
1770 finish_reason = reply.finish_reason
1771 else
1772 logger.warnx(task, 'anonymize_message: unknown LLM type: %s', llm_settings.type)
1773 end
1774
1775 if anonymized_content and #tostring(anonymized_content) > 0 then
1776 logger.debugm('lua_mime', task,
1777 'anonymize_message: successfully extracted anonymized content, length: %s bytes',
1778 #tostring(anonymized_content))
1779
1780 -- Parse the subject from the LLM response
1781 -- Expected format: "SUBJECT: <anonymized subject>\n\n<content>"
1782 local anonymized_subject = 'Email message' -- default fallback
1783 local body_content = anonymized_content
1784
1785 local subject_pattern = '^SUBJECT:%s*([^\n]+)\n\n(.*)$'
1786 local subj, content = string.match(tostring(anonymized_content), subject_pattern)
1787 if subj and content then
1788 anonymized_subject = subj
1789 body_content = content
1790 logger.debugm('lua_mime', task, 'anonymize_message: extracted anonymized subject: %s', anonymized_subject)
1791 else
1792 logger.debugm('lua_mime', task,
1793 'anonymize_message: could not extract subject from LLM response, using default')
1794 end
1795
1796 -- Update the subject header in modified_headers with LLM-anonymized value
1797 for i, hdr in ipairs(modified_headers) do
1798 if hdr.name:lower() == 'subject' or hdr.name:lower() == 'thread-topic' then
1799 modified_headers[i].value = anonymized_subject
1800 logger.debugm('lua_mime', task, 'anonymize_message: updated %s header with LLM-anonymized value',
1801 hdr.name)
1802 end
1803 end
1804
1805 -- Create new message with anonymized content
1806 local cur_boundary = '--XXX'
1807
1808 -- Add headers
1809 out[#out + 1] = {
1810 string.format('Content-Type: multipart/mixed; boundary="%s"', cur_boundary),
1811 true
1812 }
1813 for _, hdr in ipairs(modified_headers) do
1814 if hdr.name:lower() ~= 'content-type' then
1815 out[#out + 1] = {
1816 string.format('%s: %s', hdr.name, hdr.value),
1817 true
1818 }
1819 end
1820 end
1821 out[#out + 1] = { '', true }
1822
1823 -- Add text part with anonymized content
1824 out[#out + 1] = {
1825 string.format('--%s', cur_boundary),
1826 true
1827 }
1828 out[#out + 1] = {
1829 'Content-Type: text/plain; charset=utf-8\nContent-Transfer-Encoding: quoted-printable',
1830 true
1831 }
1832 out[#out + 1] = { '', true }
1833 out[#out + 1] = {
1834 rspamd_util.encode_qp(body_content, 76, task:get_newlines_type()),
1835 true
1836 }
1837
1838 -- Close boundaries
1839 out[#out + 1] = {
1840 string.format('--%s--', cur_boundary),
1841 true
1842 }
1843
1844 state.out = out
1845 state.need_rewrite_ct = true
1846 state.new_ct = {
1847 type = 'multipart',
1848 subtype = 'mixed'
1849 }
1850
1851 logger.debugm('lua_mime', task, 'anonymize_message: GPT anonymization complete, %s output parts', #out)
1852 return state
1853 else
1854 -- Provide helpful error message based on finish_reason/stop_reason
1855 if finish_reason == 'length' or finish_reason == 'max_tokens' then
1856 logger.errx(task,
1857 'anonymize_message: LLM response was truncated due to token limit (finish_reason: %s), increase max_tokens in GPT config',
1858 finish_reason)
1859 elseif finish_reason then
1860 logger.errx(task, 'anonymize_message: LLM returned empty content (finish_reason: %s)', finish_reason)
1861 else
1862 logger.errx(task, 'anonymize_message: no anonymized content extracted from LLM response')
1863 end
1864 end
1865
1866 return false
1867 end
1868
1869 -- Send content to LLM with subject
1870 logger.debugm('lua_mime', task, 'anonymize_message: sending content to LLM with subject: %s',
1871 orig_subject ~= '' and orig_subject or 'No subject')
1872 return send_to_llm(sel_part:get_content(), orig_subject, orig_thread_topic)
1873 else
1874 logger.debugm('lua_mime', task, 'anonymize_message: using regex-based anonymization')
1875
1876 if sel_part then
1877 text_content = sel_part:get_words('norm')
1878 for i, w in ipairs(text_content) do
1879 if exclude_words_re:match(w) then
1880 text_content[i] = string.rep('x', #w)
1881 end
1882 end
1883 end
1884
1885 -- Process URLs
1886 local function process_url(url)
1887 local clean_url = url:get_host()
1888 local path = url:get_path()
1889 if path and path ~= "/" then
1890 clean_url = string.format("%s/%s", clean_url, path)
1891 end
1892 return string.format('https://%s', clean_url)
1893 end
1894
1895 local url_list = task:get_urls(true) or {}
1896 logger.debugm('lua_mime', task, 'anonymize_message: processing %s URLs', #url_list)
1897 for _, url in ipairs(url_list) do
1898 urls[process_url(url)] = true
1899 end
1900
1901 -- Process emails
1902 local function process_email(email)
1903 return string.format('nobody@%s', email.domain or 'example.com')
1904 end
1905
1906 local email_list = task:get_emails() or {}
1907 logger.debugm('lua_mime', task, 'anonymize_message: processing %s emails', #email_list)
1908 for _, email in ipairs(email_list) do
1909 emails[process_email(email)] = true
1910 end
1911
1912 -- Construct new message
1913 table.insert(text_content, '\nurls:')
1914 table.insert(text_content, table.concat(lua_util.keys(urls), ', '))
1915 table.insert(text_content, '\nemails:')
1916 table.insert(text_content, table.concat(lua_util.keys(emails), ', '))
1917 local new_text = table.concat(text_content, ' ')
1918
1919 -- Create new message structure
1920 local cur_boundary = '--XXX'
1921
1922 -- Add headers
1923 out[#out + 1] = {
1924 string.format('Content-Type: multipart/mixed; boundary="%s"', cur_boundary),
1925 true
1926 }
1927 for _, hdr in ipairs(modified_headers) do
1928 if hdr.name ~= 'Content-Type' then
1929 out[#out + 1] = {
1930 string.format('%s: %s', hdr.name, hdr.value),
1931 true
1932 }
1933 end
1934 end
1935 out[#out + 1] = { '', true }
1936
1937 -- Add text part
1938 out[#out + 1] = {
1939 string.format('--%s', cur_boundary),
1940 true
1941 }
1942 out[#out + 1] = {
1943 'Content-Type: text/plain; charset=utf-8\nContent-Transfer-Encoding: quoted-printable',
1944 true
1945 }
1946 out[#out + 1] = { '', true }
1947 out[#out + 1] = {
1948 rspamd_util.encode_qp(new_text, 76, task:get_newlines_type()),
1949 true
1950 }
1951
1952 -- Close boundaries
1953 out[#out + 1] = {
1954 string.format('--%s--', cur_boundary),
1955 true
1956 }
1957
1958 state.out = out
1959 state.need_rewrite_ct = true
1960 state.new_ct = {
1961 type = 'multipart',
1962 subtype = 'mixed'
1963 }
1964
1965 logger.debugm('lua_mime', task,
1966 'anonymize_message: regex anonymization complete, %s output parts, %s unique URLs, %s unique emails',
1967 #out, lua_util.table_len(urls), lua_util.table_len(emails))
1968 return state
1969 end
1970 end
1971
1972 --[[[
1973 -- @function lua_mime.extract_text_limited(task, opts)
1974 -- Extracts text from a message with size limits and optional cleanup
1975 -- @param {task} task Rspamd task object
1976 -- @param {table} opts Options:
1977 -- * max_bytes: number - hard limit on output size (default: 32KB)
1978 -- * max_words: number - alternative limit by word count
1979 -- * preserve_first_part: boolean - prioritize newest content (top-post style)
1980 -- * strip_quotes: boolean - remove quoted replies
1981 -- * strip_reply_headers: boolean - remove "On X wrote:" patterns
1982 -- * strip_signatures: boolean - remove signature blocks
1983 -- * strip_footers: boolean - remove common email footers
1984 -- * smart_trim: boolean - enable all heuristics
1985 -- @return {table} Result table:
1986 -- * text: string - extracted text
1987 -- * truncated: boolean - whether text was truncated
1988 -- * stats: table - statistics about extraction (removed_quotes, removed_signatures, etc)
1989 --]]
1990 exports.extract_text_limited = function(task, opts)
1991 opts = opts or {}
1992 local max_bytes = opts.max_bytes or 32768
1993 local strip_quotes = opts.strip_quotes or opts.smart_trim
1994 local strip_reply_headers = opts.strip_reply_headers or opts.smart_trim
1995 local strip_signatures = opts.strip_signatures or opts.smart_trim
1996 -- strip_footers reserved for future use
1997 local _ = opts.strip_footers or opts.smart_trim
1998
1999 local stats = {
2000 removed_quotes = 0,
2001 removed_reply_headers = 0,
2002 removed_signatures = 0,
2003 removed_footers = 0
2004 }
2005
2006 -- Get the most relevant text part
2007 local part = exports.get_displayed_text_part(task, opts.min_words)
2008 if not part then
2009 return {
2010 text = "",
2011 truncated = false,
2012 stats = stats
2013 }
2014 end
2015
2016 -- Get the text content (parsed for HTML, raw for plain text)
2017 -- Use 'content' mode which preserves newlines (needed for line-based processing)
2018 -- Keep as rspamd_text userdata for efficient memory usage (no Lua string interning)
2019 local content = part:get_content('content') or rspamd_text.fromstring("")
2020
2021 local rspamd_regexp = require "rspamd_regexp"
2022
2023 -- Use rspamd_text:lines(true) iterator which returns strings without interning
2024 -- the entire content. This is more memory efficient for large emails.
2025 local line_iterator = content:lines(true)
2026 local result_lines = {}
2027 local truncated = false
2028 local current_bytes = 0
2029
2030 -- Regex patterns (pre-compiled for performance)
2031 local quote_re = rspamd_regexp.create_cached("^>+ ?")
2032
2033 -- Use cached multilingual maps (lazy initialization)
2034 local signature_map = get_signature_map()
2035 local reply_header_map = get_reply_header_map()
2036 local reply_wrote_map = get_reply_wrote_map()
2037 local reply_header_regexps = cached_reply_header_regexps
2038 local reply_wrote_regexps = cached_reply_wrote_regexps
2039
2040 local skip_rest = false
2041
2042 for line in line_iterator do
2043 if skip_rest then break end
2044
2045 local keep_line = true
2046 local trimmed_line = line:match("^%s*(.-)%s*$") or ""
2047
2048
2049 -- Check for standard signature separator (-- or --- with optional trailing whitespace)
2050 if strip_signatures and trimmed_line:match("^%-%-+%s*$") then
2051 skip_rest = true
2052 stats.removed_signatures = stats.removed_signatures + 1
2053 keep_line = false
2054 end
2055
2056 -- Check for mobile signature lines (these are usually at the very end)
2057 if keep_line and strip_signatures and signature_map then
2058 if signature_map:get_key(line) then
2059 skip_rest = true
2060 stats.removed_signatures = stats.removed_signatures + 1
2061 keep_line = false
2062 end
2063 end
2064
2065 -- Check for quoted lines (starting with >)
2066 if keep_line and strip_quotes then
2067 if quote_re:match(line) then
2068 keep_line = false
2069 stats.removed_quotes = stats.removed_quotes + 1
2070 end
2071 end
2072
2073 -- Check for reply headers using map first
2074 if keep_line and strip_reply_headers then
2075 local header_matches = reply_header_map and reply_header_map:get_key(trimmed_line)
2076 if not header_matches and reply_header_regexps then
2077 for _, re in ipairs(reply_header_regexps) do
2078 if re:match(trimmed_line) then
2079 header_matches = true
2080 break
2081 end
2082 end
2083 end
2084 if header_matches then
2085 if trimmed_line:match("^[-_]+") or trimmed_line:match("^%S+:%s*") then
2086 skip_rest = true
2087 stats.removed_reply_headers = stats.removed_reply_headers + 1
2088 keep_line = false
2089 end
2090 end
2091
2092 if keep_line and reply_wrote_map then
2093 local wrote_matches = reply_wrote_map:get_key(trimmed_line)
2094 if wrote_matches then
2095 skip_rest = true
2096 stats.removed_reply_headers = stats.removed_reply_headers + 1
2097 keep_line = false
2098 end
2099 end
2100
2101 if keep_line and reply_wrote_regexps then
2102 for _, re in ipairs(reply_wrote_regexps) do
2103 if re:match(trimmed_line) then
2104 skip_rest = true
2105 stats.removed_reply_headers = stats.removed_reply_headers + 1
2106 keep_line = false
2107 break
2108 end
2109 end
2110 end
2111 end
2112
2113 if keep_line then
2114 local line_len = #line + 1 -- +1 for newline
2115 if current_bytes + line_len > max_bytes then
2116 truncated = true
2117 break
2118 end
2119 table.insert(result_lines, line)
2120 current_bytes = current_bytes + line_len
2121 end
2122
2123 end
2124
2125 local text = table.concat(result_lines, "\n")
2126
2127 -- Handle max_words limit by counting words in the result
2128 if opts.max_words and opts.max_words > 0 then
2129 -- Simple word counting by splitting on whitespace
2130 local word_count = 0
2131 local last_word_end = 0
2132 for _, word_end in text:gmatch("()%S+()") do
2133 word_count = word_count + 1
2134 if word_count <= opts.max_words then
2135 last_word_end = word_end - 1
2136 else
2137 truncated = true
2138 text = text:sub(1, last_word_end)
2139 break
2140 end
2141 end
2142 end
2143
2144 return {
2145 text = text,
2146 truncated = truncated,
2147 stats = stats
2148 }
2149 end
2150
2151 return exports
2152