Debian Codegraph @

lualib/rspamadm/dmarc_report.lua

raw
1 --[[
2 Copyright (c) 2022, Vsevolod Stakhov <[email protected]>
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 ]]--
16
17 local argparse = require "argparse"
18 local lua_util = require "lua_util"
19 local logger = require "rspamd_logger"
20 local lua_redis = require "lua_redis"
21 local lua_maps = require "lua_maps"
22 local dmarc_common = require "plugins/dmarc"
23 local rspamd_mempool = require "rspamd_mempool"
24 local rspamd_url = require "rspamd_url"
25 local rspamd_text = require "rspamd_text"
26 local rspamd_util = require "rspamd_util"
27 local rspamd_dns = require "rspamd_dns"
28 local fun = require "fun"
29
30 local N = 'dmarc_report'
31
32 -- Define command line options
33 local parser = argparse()
34 :name "rspamadm dmarc_report"
35 :description "Dmarc reports sending tool"
36 :help_description_margin(30)
37
38 parser:option "-c --config"
39 :description "Path to config file"
40 :argname("<cfg>")
41 :default(rspamd_paths["CONFDIR"] .. "/" .. "rspamd.conf")
42
43 parser:flag "-v --verbose"
44 :description "Enable dmarc specific logging"
45
46 parser:flag "-n --no-opt"
47 :description "Do not reset reporting data/send reports"
48
49 parser:argument "date"
50 :description "Date to process (yesterday by default)"
51 :argname "<YYYYMMDD>"
52 :args "*"
53 parser:option "-b --batch-size"
54 :description "Send reports in batches up to <batch-size> messages"
55 :argname "<number>"
56 :convert(tonumber)
57 :default "10"
58
59 parser:flag "-r --recheck-rua"
60 :description "Re-check RUA addresses against exclude_rua_addresses map before sending"
61
62 local report_template = [[From: "{= from_name =}" <{= from_addr =}>
63 To: {= rcpt =}
64 {%+ if is_string(bcc) %}Bcc: {= bcc =}{%- endif %}
65 Subject: Report Domain: {= reporting_domain =}
66 Submitter: {= submitter =}
67 Report-ID: {= report_id =}
68 Date: {= report_date =}
69 MIME-Version: 1.0
70 Message-ID: <{= message_id =}>
71 Content-Type: multipart/mixed;
72 boundary="----=_NextPart_{= uuid =}"
73 Auto-Submitted: auto-generated
74 Precedence: bulk
75 X-Auto-Response-Suppress: All
76
77 This is a multipart message in MIME format.
78
79 ------=_NextPart_{= uuid =}
80 Content-Type: text/plain; charset="us-ascii"
81 Content-Transfer-Encoding: 7bit
82
83 This is an aggregate report from {= submitter =}.
84
85 Report domain: {= reporting_domain =}
86 Submitter: {= submitter =}
87 Report ID: {= report_id =}
88
89 ------=_NextPart_{= uuid =}
90 Content-Type: application/gzip
91 Content-Transfer-Encoding: base64
92 Content-Disposition: attachment;
93 filename="{= submitter =}!{= reporting_domain =}!{= report_start =}!{= report_end =}.xml.gz"
94
95 ]]
96 local report_footer = [[
97
98 ------=_NextPart_{= uuid =}--]]
99
100 local dmarc_settings = {}
101 local redis_params
102 local redis_attrs = {
103 config = rspamd_config,
104 ev_base = rspamadm_ev_base,
105 session = rspamadm_session,
106 log_obj = rspamd_config,
107 resolver = rspamadm_dns_resolver,
108 }
109 local redis_attrs_write = lua_util.shallowcopy(redis_attrs)
110 redis_attrs_write['is_write'] = true
111 local pool
112 local exclude_rua_map
113 -- Context for external map queries (used instead of task in rspamadm)
114 local map_context = {
115 config = rspamd_config,
116 ev_base = rspamadm_ev_base,
117 session = rspamadm_session,
118 resolver = rspamadm_dns_resolver,
119 }
120
121 local function load_config(opts)
122 local _r, err = rspamd_config:load_ucl(opts['config'])
123
124 if not _r then
125 logger.errx('cannot parse %s: %s', opts['config'], err)
126 os.exit(1)
127 end
128
129 _r, err = rspamd_config:parse_rcl({ 'logging', 'worker' })
130 if not _r then
131 logger.errx('cannot process %s: %s', opts['config'], err)
132 os.exit(1)
133 end
134 end
135
136 -- Concat elements using redis_keys.join_char
137 local function redis_prefix(...)
138 return table.concat({ ... }, dmarc_settings.reporting.redis_keys.join_char)
139 end
140
141 local function get_rua(rep_key)
142 local parts = lua_util.str_split(rep_key, dmarc_settings.reporting.redis_keys.join_char)
143
144 if #parts >= 3 then
145 return parts[3]
146 end
147
148 return nil
149 end
150
151 local function get_domain(rep_key)
152 local parts = lua_util.str_split(rep_key, dmarc_settings.reporting.redis_keys.join_char)
153
154 if #parts >= 3 then
155 return parts[2]
156 end
157
158 return nil
159 end
160
161 local function gen_uuid()
162 local template = 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'
163 return string.gsub(template, '[xy]', function(c)
164 local v = (c == 'x') and math.random(0, 0xf) or math.random(8, 0xb)
165 return string.format('%x', v)
166 end)
167 end
168
169 local function gen_xml_grammar()
170 local lpeg = require 'lpeg'
171 local lt = lpeg.P('<') / '&lt;'
172 local gt = lpeg.P('>') / '&gt;'
173 local amp = lpeg.P('&') / '&amp;'
174 local quot = lpeg.P('"') / '&quot;'
175 local apos = lpeg.P("'") / '&apos;'
176 local special = lt + gt + amp + quot + apos
177 local grammar = lpeg.Cs((special + 1) ^ 0)
178 return grammar
179 end
180
181 local xml_grammar = gen_xml_grammar()
182
183 local function escape_xml(input)
184 if type(input) == 'string' or type(input) == 'userdata' then
185 return xml_grammar:match(input)
186 else
187 input = tostring(input)
188
189 if input then
190 return xml_grammar:match(input)
191 end
192 end
193
194 return ''
195 end
196
197 -- Creates report XML header
198 local function report_header(reporting_domain, report_start, report_end, domain_policy)
199 local report_id = string.format('%s.%d.%d',
200 reporting_domain, report_start, report_end)
201 local xml_template = [[
202 <?xml version="1.0" encoding="UTF-8" ?>
203 <feedback>
204 <report_metadata>
205 <org_name>{= report_settings.org_name | escape_xml =}</org_name>
206 <email>{= report_settings.email | escape_xml =}</email>
207 <report_id>{= report_id =}</report_id>
208 <date_range>
209 <begin>{= report_start =}</begin>
210 <end>{= report_end =}</end>
211 </date_range>
212 </report_metadata>
213 <policy_published>
214 <domain>{= reporting_domain | escape_xml =}</domain>
215 <adkim>{= domain_policy.adkim | escape_xml =}</adkim>
216 <aspf>{= domain_policy.aspf | escape_xml =}</aspf>
217 <p>{= domain_policy.p | escape_xml =}</p>
218 <sp>{= domain_policy.sp | escape_xml =}</sp>
219 <pct>{= domain_policy.pct | escape_xml =}</pct>
220 </policy_published>
221 ]]
222 return lua_util.jinja_template(xml_template, {
223 report_settings = dmarc_settings.reporting,
224 report_id = report_id,
225 report_start = report_start,
226 report_end = report_end,
227 domain_policy = domain_policy,
228 reporting_domain = reporting_domain,
229 }, true, false,
230 {
231 escape_xml = escape_xml
232 })
233 end
234
235 -- Generate xml entry for a preprocessed redis row
236 local function entry_to_xml(data)
237 local xml_template = [[<record>
238 <row>
239 <source_ip>{= data.ip =}</source_ip>
240 <count>{= data.count =}</count>
241 <policy_evaluated>
242 <disposition>{= data.disposition =}</disposition>
243 <dkim>{= data.dkim_disposition =}</dkim>
244 <spf>{= data.spf_disposition =}</spf>
245 {% if data.override and data.override ~= '' -%}
246 <reason><type>{= data.override =}</type></reason>
247 {%- endif %}
248 </policy_evaluated>
249 </row>
250 <identifiers>
251 <header_from>{= data.header_from =}</header_from>
252 </identifiers>
253 <auth_results>
254 {% if data.dkim_results[1] -%}
255 {% for d in data.dkim_results -%}
256 <dkim>
257 <domain>{= d.domain =}</domain>
258 <result>{= d.result =}</result>
259 </dkim>
260 {%- endfor %}
261 {%- endif %}
262 <spf>
263 <domain>{= data.spf_domain =}</domain>
264 <result>{= data.spf_result =}</result>
265 </spf>
266 </auth_results>
267 </record>
268 ]]
269 return lua_util.jinja_template(xml_template, { data = data }, true,
270 false, {
271 escape_xml = escape_xml
272 })
273 end
274
275 -- Process a report entry stored in Redis splitting it to a lua table
276 local function process_report_entry(data, score)
277 local split = lua_util.str_split(data, ',')
278 local row = {
279 ip = split[1],
280 spf_disposition = split[2],
281 dkim_disposition = split[3],
282 disposition = split[4],
283 override = split[5],
284 header_from = split[6],
285 dkim_results = {},
286 spf_domain = split[11],
287 spf_result = split[12],
288 count = tonumber(score),
289 }
290 -- Process dkim entries
291 local function dkim_entries_process(dkim_data, result)
292 if dkim_data and dkim_data ~= '' then
293 local dkim_elts = lua_util.str_split(dkim_data, '|')
294 for _, d in ipairs(dkim_elts) do
295 table.insert(row.dkim_results, { domain = d, result = result })
296 end
297 end
298 end
299 dkim_entries_process(split[7], 'pass')
300 dkim_entries_process(split[8], 'fail')
301 dkim_entries_process(split[9], 'temperror')
302 dkim_entries_process(split[9], 'permerror')
303
304 return row
305 end
306
307 -- Process a single rua entry, validating in DNS if needed
308 local function process_rua(dmarc_domain, rua)
309 -- Remove size limitation, as we don't care about them
310 local addrs = {}
311 for _, rua_part in fun.map(lua_util.str_trim, lua_util.str_split(rua, ',')) do
312 local u = rspamd_url.create(pool, rua_part:gsub('!%d+[kmg]?$', ''))
313 local u2 = rspamd_url.create(pool, dmarc_domain)
314 if u and (u:get_protocol() or '') == 'mailto' and u:get_user() then
315 -- Check each address for sanity
316 if u:get_tld() == u2:get_tld() then
317 -- Same eSLD - always include
318 table.insert(addrs, u)
319 else
320 -- We need to check authority
321 local resolve_str = string.format('%s._report._dmarc.%s',
322 dmarc_domain, u:get_host())
323 local is_ok, results = rspamd_dns.request({
324 config = rspamd_config,
325 session = rspamadm_session,
326 type = 'txt',
327 name = resolve_str,
328 })
329
330 if not is_ok then
331 logger.errx('cannot resolve %s: %s; exclude %s', resolve_str, results, rua_part)
332 else
333 local found = false
334 for _, t in ipairs(results) do
335 if string.match(t, 'v=DMARC1') then
336 found = true
337 break
338 end
339 end
340
341 if not found then
342 logger.errx('%s is not authorized to process reports on %s', dmarc_domain, u:get_host())
343 else
344 -- All good
345 table.insert(addrs, u)
346 end
347 end
348 end
349 else
350 logger.errx('invalid rua url: "%s""', rua_part)
351 end
352 end
353
354 if #addrs > 0 then
355 return addrs
356 end
357
358 return nil
359 end
360
361 -- Validate reporting domain, extracting rua and checking 3rd party report domains
362 -- This function returns a full dmarc record processed + rua as a list of url objects
363 local function validate_reporting_domain(reporting_domain)
364 local is_ok, results = rspamd_dns.request({
365 config = rspamd_config,
366 session = rspamadm_session,
367 type = 'txt',
368 name = '_dmarc.' .. reporting_domain,
369 })
370
371 if not is_ok or not results then
372 logger.errx('cannot resolve _dmarc.%s: %s', reporting_domain, results)
373 return nil
374 end
375
376 for _, r in ipairs(results) do
377 local processed, rec = dmarc_common.dmarc_check_record(rspamd_config, r, false)
378 if processed and rec.rua then
379 -- We need to check or alter rua if needed
380 local processed_rua = process_rua(reporting_domain, rec.rua)
381 if processed_rua then
382 rec = rec.raw_elts
383 rec.rua = processed_rua
384
385 -- Fill defaults in a record to avoid nils in a report
386 rec['pct'] = rec['pct'] or 100
387 rec['adkim'] = rec['adkim'] or 'r'
388 rec['aspf'] = rec['aspf'] or 'r'
389 rec['p'] = rec['p'] or 'none'
390 rec['sp'] = rec['sp'] or 'none'
391 return rec
392 end
393 return nil
394 end
395 end
396
397 return nil
398 end
399
400 -- Returns a list of recipients from a table as a string processing elements if needed
401 local function rcpt_list(tbl, func)
402 local res = {}
403 for _, r in ipairs(tbl) do
404 if func then
405 table.insert(res, func(r))
406 else
407 table.insert(res, r)
408 end
409 end
410
411 return table.concat(res, ',')
412 end
413
414 -- Synchronous smtp send function
415 local function send_reports_by_smtp(opts, reports, finish_cb)
416 local lua_smtp = require "lua_smtp"
417 local reports_failed = 0
418 local reports_sent = 0
419 local report_settings = dmarc_settings.reporting
420
421 local function gen_sendmail_cb(report, args)
422 return function(ret, err)
423 -- We modify this from all callbacks
424 args.nreports = args.nreports - 1
425 if not ret then
426 logger.errx("Couldn't send mail for %s: %s", report.reporting_domain, err)
427 reports_failed = reports_failed + 1
428 else
429 reports_sent = reports_sent + 1
430 lua_util.debugm(N, 'successfully sent a report for %s: %s bytes sent',
431 report.reporting_domain, #report.message)
432 end
433
434 -- Tail call to the next batch or to the final function
435 if args.nreports == 0 then
436 if args.next_start > #reports then
437 finish_cb(reports_sent, reports_failed)
438 else
439 args.cont_func(args.next_start)
440 end
441 end
442 end
443 end
444
445 local function send_data_in_batches(cur_batch)
446 local nreports = math.min(#reports - cur_batch + 1, opts.batch_size)
447 local next_start = cur_batch + nreports
448 lua_util.debugm(N, 'send data for %s domains (from %s to %s)',
449 nreports, cur_batch, next_start - 1)
450 -- Shared across all closures
451 local gen_args = {
452 cont_func = send_data_in_batches,
453 nreports = nreports,
454 next_start = next_start
455 }
456 for i = cur_batch, next_start - 1 do
457 local report = reports[i]
458 local send_opts = {
459 ev_base = rspamadm_ev_base,
460 session = rspamadm_session,
461 config = rspamd_config,
462 host = report_settings.smtp,
463 port = report_settings.smtp_port or 25,
464 resolver = rspamadm_dns_resolver,
465 from = report_settings.email,
466 recipients = report.rcpts,
467 helo = report_settings.helo or 'rspamd.localhost',
468 }
469
470 lua_smtp.sendmail(send_opts,
471 report.message,
472 gen_sendmail_cb(report, gen_args))
473 end
474 end
475
476 send_data_in_batches(1)
477 end
478
479 local function prepare_report(opts, start_time, end_time, rep_key)
480 local rua = get_rua(rep_key)
481 local reporting_domain = get_domain(rep_key)
482
483 if not rua then
484 logger.errx('report %s has no valid rua, skip it', rep_key)
485 return nil
486 end
487 if not reporting_domain then
488 logger.errx('report %s has no valid reporting_domain, skip it', rep_key)
489 return nil
490 end
491
492 local ret, results = lua_redis.request(redis_params, redis_attrs,
493 { 'EXISTS', rep_key })
494
495 if not ret or not results or results == 0 then
496 return nil
497 end
498
499 -- Rename report key to avoid races
500 if not opts.no_opt then
501 lua_redis.request(redis_params, redis_attrs_write,
502 { 'RENAME', rep_key, rep_key .. '_processing' })
503 rep_key = rep_key .. '_processing'
504 end
505
506 local dmarc_record = validate_reporting_domain(reporting_domain)
507 lua_util.debugm(N, 'process reporting domain %s: %s', reporting_domain, dmarc_record)
508
509 if not dmarc_record then
510 if not opts.no_opt then
511 lua_redis.request(redis_params, redis_attrs_write,
512 { 'DEL', rep_key })
513 end
514 logger.messagex('Cannot process reports for domain %s; invalid dmarc record', reporting_domain)
515 return nil
516 end
517
518 -- Re-check RUA addresses against exclude_rua_addresses map if enabled
519 -- Works with both local maps and external maps (HTTP) using synchronous requests
520 if exclude_rua_map and dmarc_record.rua then
521 local filtered_rua = {}
522 local excluded_count = 0
523 for _, rua_elt in ipairs(dmarc_record.rua) do
524 local rua_email = string.format('%s@%s', rua_elt:get_user(), rua_elt:get_host())
525 -- For external maps, pass map_context to enable synchronous HTTP requests
526 local excluded = exclude_rua_map:get_key(rua_email, nil, map_context)
527 if not excluded then
528 -- Also check just the domain part
529 excluded = exclude_rua_map:get_key(rua_elt:get_host(), nil, map_context)
530 end
531 if excluded then
532 lua_util.debugm(N, 'RUA address %s for domain %s is excluded by map (re-check)',
533 rua_email, reporting_domain)
534 excluded_count = excluded_count + 1
535 else
536 table.insert(filtered_rua, rua_elt)
537 end
538 end
539
540 if #filtered_rua == 0 then
541 if not opts.no_opt then
542 lua_redis.request(redis_params, redis_attrs_write,
543 { 'DEL', rep_key })
544 end
545 logger.messagex('All RUA addresses for domain %s are excluded by map (re-check), skipping report',
546 reporting_domain)
547 return nil
548 elseif excluded_count > 0 then
549 logger.messagex('Filtered %s RUA addresses for domain %s, %s remaining',
550 excluded_count, reporting_domain, #filtered_rua)
551 dmarc_record.rua = filtered_rua
552 end
553 end
554
555 -- Get all reports for a domain
556 ret, results = lua_redis.request(redis_params, redis_attrs,
557 { 'ZRANGE', rep_key, '0', '-1', 'WITHSCORES' })
558 local report_entries = {}
559 table.insert(report_entries,
560 report_header(reporting_domain, start_time, end_time, dmarc_record))
561 for i = 1, #results, 2 do
562 local xml_record = entry_to_xml(process_report_entry(results[i], results[i + 1]))
563 table.insert(report_entries, xml_record)
564 end
565 table.insert(report_entries, '</feedback>')
566 local xml_to_compress = rspamd_text.fromtable(report_entries)
567 lua_util.debugm(N, 'got xml: %s', xml_to_compress)
568
569 -- Prepare SMTP message
570 local report_settings = dmarc_settings.reporting
571 local rcpt_string = rcpt_list(dmarc_record.rua, function(rua_elt)
572 return string.format('%s@%s', rua_elt:get_user(), rua_elt:get_host())
573 end)
574 local bcc_string
575 if report_settings.bcc_addrs then
576 bcc_string = rcpt_list(report_settings.bcc_addrs)
577 end
578 local uuid = gen_uuid()
579 local rhead = lua_util.jinja_template(report_template, {
580 from_name = report_settings.from_name,
581 from_addr = report_settings.email,
582 rcpt = rcpt_string,
583 bcc = bcc_string,
584 uuid = uuid,
585 reporting_domain = reporting_domain,
586 submitter = report_settings.domain,
587 report_id = string.format('%s.%d.%d', reporting_domain, start_time,
588 end_time),
589 report_date = rspamd_util.time_to_string(rspamd_util.get_time()),
590 message_id = rspamd_util.random_hex(16) .. '@' .. report_settings.msgid_from,
591 report_start = start_time,
592 report_end = end_time
593 }, true,
594 false, {
595 escape_xml = escape_xml
596 })
597 local rfooter = lua_util.jinja_template(report_footer, {
598 uuid = uuid,
599 }, true, false, {
600 escape_xml = escape_xml
601 })
602 local message = rspamd_text.fromtable {
603 (rhead:gsub("\n", "\r\n")),
604 rspamd_util.encode_base64(rspamd_util.gzip_compress(xml_to_compress), 73),
605 rfooter:gsub("\n", "\r\n"),
606 }
607
608 lua_util.debugm(N, 'got final message: %s', message)
609
610 if not opts.no_opt then
611 lua_redis.request(redis_params, redis_attrs_write,
612 { 'DEL', rep_key })
613 end
614
615 local report_rcpts = lua_util.str_split(rcpt_string, ',')
616
617 if report_settings.bcc_addrs then
618 for _, b in ipairs(report_settings.bcc_addrs) do
619 table.insert(report_rcpts, b)
620 end
621 end
622
623 return {
624 message = message,
625 rcpts = report_rcpts,
626 reporting_domain = reporting_domain
627 }
628 end
629
630 local function process_report_date(opts, start_time, end_time, date)
631 local idx_key = redis_prefix(dmarc_settings.reporting.redis_keys.index_prefix, date)
632 local ret, results = lua_redis.request(redis_params, redis_attrs,
633 { 'EXISTS', idx_key })
634
635 if not ret or not results or results == 0 then
636 logger.messagex('No reports for %s', date)
637 return {}
638 end
639
640 -- Rename index key to avoid races
641 if not opts.no_opt then
642 lua_redis.request(redis_params, redis_attrs_write,
643 { 'RENAME', idx_key, idx_key .. '_processing' })
644 idx_key = idx_key .. '_processing'
645 end
646 ret, results = lua_redis.request(redis_params, redis_attrs,
647 { 'SMEMBERS', idx_key })
648
649 if not ret or not results then
650 -- Remove bad key
651 if not opts.no_opt then
652 lua_redis.request(redis_params, redis_attrs_write,
653 { 'DEL', idx_key })
654 end
655 logger.messagex('Cannot get reports for %s', date)
656 return {}
657 end
658
659 -- Process reports in batches to limit Redis connections
660 local reports = {}
661
662 for batch_start = 1, #results, opts.batch_size do
663 local batch_end = math.min(batch_start + opts.batch_size - 1, #results)
664 lua_util.debugm(N, 'processing report batch %s to %s (of %s total)',
665 batch_start, batch_end, #results)
666
667 for i = batch_start, batch_end do
668 local rep = results[i]
669 local report = prepare_report(opts, start_time, end_time, rep)
670
671 if report then
672 table.insert(reports, report)
673 end
674 end
675
676 -- Force garbage collection after each batch to release Redis connections
677 collectgarbage("collect")
678 end
679
680 -- Shuffle reports to make sending more fair
681 lua_util.shuffle(reports)
682 -- Remove processed key
683 if not opts.no_opt then
684 lua_redis.request(redis_params, redis_attrs_write,
685 { 'DEL', idx_key })
686 end
687
688 return reports
689 end
690
691
692 -- Returns a day before today at 00:00 as unix seconds
693 local function yesterday_midnight()
694 local piecewise_time = os.date("*t")
695 piecewise_time.day = piecewise_time.day - 1 -- Lua allows negative values here
696 piecewise_time.hour = 0
697 piecewise_time.sec = 0
698 piecewise_time.min = 0
699 return os.time(piecewise_time)
700 end
701
702 -- Returns today time at 00:00 as unix seconds
703 local function today_midnight()
704 local piecewise_time = os.date("*t")
705 piecewise_time.hour = 0
706 piecewise_time.sec = 0
707 piecewise_time.min = 0
708 return os.time(piecewise_time)
709 end
710
711 local function handler(args)
712 local start_time
713 -- Preserve start time as report sending might take some time
714 local start_collection = today_midnight()
715
716 local opts = parser:parse(args)
717
718 -- Normalize batch_size: floor to integer and clamp to >= 1
719 -- Fractional values would break array indexing in batching loops
720 opts.batch_size = math.max(1, math.floor(opts.batch_size or 10))
721
722 pool = rspamd_mempool.create()
723 load_config(opts)
724 rspamd_url.init(rspamd_config:get_tld_path())
725
726 if opts.verbose then
727 lua_util.enable_debug_modules('dmarc', N)
728 end
729
730 dmarc_settings = rspamd_config:get_all_opt('dmarc')
731 if not dmarc_settings or not dmarc_settings.reporting or not dmarc_settings.reporting.enabled then
732 logger.errx('dmarc reporting is not enabled, exiting')
733 os.exit(1)
734 end
735
736 dmarc_settings = lua_util.override_defaults(dmarc_common.default_settings, dmarc_settings)
737 redis_params = lua_redis.parse_redis_server('dmarc', dmarc_settings)
738
739 if not redis_params then
740 logger.errx('Redis is not configured, exiting')
741 os.exit(1)
742 end
743
744 -- Load exclude_rua_addresses map if --recheck-rua flag is set
745 if opts.recheck_rua then
746 if dmarc_settings.reporting.exclude_rua_addresses then
747 exclude_rua_map = lua_maps.map_add_from_ucl(dmarc_settings.reporting.exclude_rua_addresses,
748 'set', 'DMARC RUA exclusion map for report sending')
749 if exclude_rua_map then
750 if exclude_rua_map.__external then
751 logger.messagex('Loaded exclude_rua_addresses external map for RUA re-checking')
752 else
753 logger.messagex('Loaded exclude_rua_addresses map for RUA re-checking')
754 end
755 else
756 logger.warnx('Failed to load exclude_rua_addresses map, RUA re-checking disabled')
757 end
758 else
759 logger.warnx('--recheck-rua specified but no exclude_rua_addresses configured in dmarc settings')
760 end
761 end
762
763 for _, e in ipairs({ 'email', 'domain', 'org_name' }) do
764 if not dmarc_settings.reporting[e] then
765 logger.errx('Missing required setting: dmarc.reporting.%s', e)
766 return
767 end
768 end
769
770 local ret, results = lua_redis.request(redis_params, redis_attrs, {
771 'GET', 'rspamd_dmarc_last_collection'
772 })
773
774 if not ret or not tonumber(results) then
775 start_time = yesterday_midnight()
776 else
777 start_time = tonumber(results)
778 end
779
780 lua_util.debugm(N, 'previous last report date is %s', start_time)
781
782 if not opts.date or #opts.date == 0 then
783 opts.date = {}
784 table.insert(opts.date, os.date('%Y%m%d', yesterday_midnight()))
785 end
786
787 local ndates = 0
788 local nreports = 0
789 local all_reports = {}
790 for _, date in ipairs(opts.date) do
791 lua_util.debugm(N, 'Process date %s', date)
792 local reports_for_date = process_report_date(opts, start_time, start_collection, date)
793 if #reports_for_date > 0 then
794 ndates = ndates + 1
795 nreports = nreports + #reports_for_date
796
797 for _, r in ipairs(reports_for_date) do
798 table.insert(all_reports, r)
799 end
800 end
801 end
802
803 local function finish_cb(nsuccess, nfail)
804 if not opts.no_opt then
805 lua_util.debugm(N, 'set last report date to %s', start_collection)
806 -- Hack to avoid coroutines + async functions mess: we use async redis call here
807 redis_attrs_write.callback = function()
808 logger.messagex('Reporting collection has finished %s dates processed, %s reports: %s completed, %s failed',
809 ndates, nreports, nsuccess, nfail)
810 end
811 lua_redis.request(redis_params, redis_attrs_write,
812 { 'SETEX', 'rspamd_dmarc_last_collection', dmarc_settings.reporting.keys_expire * 2,
813 tostring(start_collection) })
814 else
815 logger.messagex('Reporting collection has finished %s dates processed, %s reports: %s completed, %s failed',
816 ndates, nreports, nsuccess, nfail)
817 end
818
819 pool:destroy()
820 end
821 if not opts.no_opt then
822 send_reports_by_smtp(opts, all_reports, finish_cb)
823 else
824 logger.messagex('Skip sending mails due to -n / --no-opt option')
825 end
826 end
827
828 return {
829 name = 'dmarc_report',
830 aliases = { 'dmarc_reporting' },
831 handler = handler,
832 description = parser._description
833 }