Debian Codegraph @

lualib/rspamadm/corpus_test.lua

raw
1 local rspamd_logger = require "rspamd_logger"
2 local ucl = require "ucl"
3 local lua_util = require "lua_util"
4 local argparse = require "argparse"
5
6 local parser = argparse()
7 :name "rspamadm corpus_test"
8 :description "Create logs files from email corpus"
9 :help_description_margin(32)
10
11 parser:option "-H --ham"
12 :description("Ham directory")
13 :argname("<dir>")
14 parser:option "-S --spam"
15 :description("Spam directory")
16 :argname("<dir>")
17 parser:option "-n --conns"
18 :description("Number of parallel connections")
19 :argname("<N>")
20 :convert(tonumber)
21 :default(10)
22 parser:option "-o --output"
23 :description("Output file")
24 :argname("<file>")
25 :default('results.log')
26 parser:option "-t --timeout"
27 :description("Timeout for client connections")
28 :argname("<sec>")
29 :convert(tonumber)
30 :default(60)
31 parser:option "-c --connect"
32 :description("Connect to specific host")
33 :argname("<host>")
34 :default('localhost:11334')
35 parser:option "-r --rspamc"
36 :description("Use specific rspamc path")
37 :argname("<path>")
38 :default('rspamc')
39
40 local HAM = "HAM"
41 local SPAM = "SPAM"
42 local opts
43
44 local function scan_email(n_parallel, path, timeout)
45
46 local rspamc_command = string.format("%s --connect %s -j --compact -n %s -t %.3f %s",
47 opts.rspamc, opts.connect, n_parallel, timeout, path)
48 local result = assert(io.popen(rspamc_command))
49 result = result:read("*all")
50 return result
51 end
52
53 local function write_results(results, file)
54
55 local f = io.open(file, 'w')
56
57 for _, result in pairs(results) do
58 local log_line = string.format("%s %.2f %s",
59 result.type, result.score, result.action)
60
61 for _, sym in pairs(result.symbols) do
62 log_line = log_line .. " " .. sym
63 end
64
65 log_line = log_line .. " " .. result.scan_time .. " " .. file .. ':' .. result.filename
66
67 log_line = log_line .. "\r\n"
68
69 f:write(log_line)
70 end
71
72 f:close()
73 end
74
75 local function encoded_json_to_log(result)
76 -- Returns table containing score, action, list of symbols
77
78 local filtered_result = {}
79 local ucl_parser = ucl.parser()
80
81 local is_good, err = ucl_parser:parse_string(result)
82
83 if not is_good then
84 rspamd_logger.errx("Parser error: %1", err)
85 return nil
86 end
87
88 result = ucl_parser:get_object()
89
90 filtered_result.score = result.score
91 if not result.action then
92 rspamd_logger.errx("Bad JSON: %1", result)
93 return nil
94 end
95 local action = result.action:gsub("%s+", "_")
96 filtered_result.action = action
97
98 filtered_result.symbols = {}
99
100 for sym, _ in pairs(result.symbols) do
101 table.insert(filtered_result.symbols, sym)
102 end
103
104 filtered_result.filename = result.filename
105 filtered_result.scan_time = result.scan_time
106
107 return filtered_result
108 end
109
110 local function scan_results_to_logs(results, actual_email_type)
111
112 local logs = {}
113
114 results = lua_util.rspamd_str_split(results, "\n")
115
116 if results[#results] == "" then
117 results[#results] = nil
118 end
119
120 for _, result in pairs(results) do
121 result = encoded_json_to_log(result)
122 if result then
123 result['type'] = actual_email_type
124 table.insert(logs, result)
125 end
126 end
127
128 return logs
129 end
130
131 local function handler(args)
132 opts = parser:parse(args)
133 local ham_directory = opts['ham']
134 local spam_directory = opts['spam']
135 local connections = opts["conns"]
136 local output = opts["output"]
137
138 local results = {}
139
140 local start_time = os.time()
141 local no_of_ham = 0
142 local no_of_spam = 0
143
144 if ham_directory then
145 rspamd_logger.messagex("Scanning ham corpus...")
146 local ham_results = scan_email(connections, ham_directory, opts["timeout"])
147 ham_results = scan_results_to_logs(ham_results, HAM)
148
149 no_of_ham = #ham_results
150
151 for _, result in pairs(ham_results) do
152 table.insert(results, result)
153 end
154 end
155
156 if spam_directory then
157 rspamd_logger.messagex("Scanning spam corpus...")
158 local spam_results = scan_email(connections, spam_directory, opts.timeout)
159 spam_results = scan_results_to_logs(spam_results, SPAM)
160
161 no_of_spam = #spam_results
162
163 for _, result in pairs(spam_results) do
164 table.insert(results, result)
165 end
166 end
167
168 rspamd_logger.messagex("Writing results to %s", output)
169 write_results(results, output)
170
171 rspamd_logger.messagex("Stats: ")
172 local elapsed_time = os.time() - start_time
173 local total_msgs = no_of_ham + no_of_spam
174 rspamd_logger.messagex("Elapsed time: %ss", elapsed_time)
175 rspamd_logger.messagex("No of ham: %s", no_of_ham)
176 rspamd_logger.messagex("No of spam: %s", no_of_spam)
177 rspamd_logger.messagex("Messages/sec: %s", (total_msgs / elapsed_time))
178 end
179
180 return {
181 name = 'corpustest',
182 aliases = { 'corpus_test', 'corpus' },
183 handler = handler,
184 description = parser._description
185 }