Debian Codegraph @

lualib/rspamadm/neural_test.lua

raw
1 local rspamd_logger = require "rspamd_logger"
2 local argparse = require "argparse"
3 local lua_util = require "lua_util"
4 local ucl = require "ucl"
5
6 local parser = argparse()
7 :name "rspamadm neural_test"
8 :description "Test the neural network with labelled dataset"
9 :help_description_margin(32)
10
11 parser:option "-c --config"
12 :description "Path to config file"
13 :argname("<cfg>")
14 :default(rspamd_paths["CONFDIR"] .. "/" .. "rspamd.conf")
15 parser:option "-H --hamdir"
16 :description("Ham directory")
17 :argname("<dir>")
18 parser:option "-S --spamdir"
19 :description("Spam directory")
20 :argname("<dir>")
21 parser:option "-t --timeout"
22 :description("Timeout for client connections")
23 :argname("<sec>")
24 :convert(tonumber)
25 :default(60)
26 parser:option "-n --conns"
27 :description("Number of parallel connections")
28 :argname("<N>")
29 :convert(tonumber)
30 :default(10)
31 parser:option "-c --connect"
32 :description("Connect to specific host")
33 :argname("<host>")
34 :default('localhost:11334')
35 parser:option "-r --rspamc"
36 :description("Use specific rspamc path")
37 :argname("<path>")
38 :default('rspamc')
39 parser:option '--rule'
40 :description 'Rule to test'
41 :argname('<rule>')
42
43 local HAM = "HAM"
44 local SPAM = "SPAM"
45
46 local function load_config(opts)
47 local _r, err = rspamd_config:load_ucl(opts['config'])
48
49 if not _r then
50 rspamd_logger.errx('cannot parse %s: %s', opts['config'], err)
51 os.exit(1)
52 end
53
54 _r, err = rspamd_config:parse_rcl({ 'logging', 'worker' })
55 if not _r then
56 rspamd_logger.errx('cannot process %s: %s', opts['config'], err)
57 os.exit(1)
58 end
59 end
60
61 local function scan_email(rspamc_path, host, n_parallel, path, timeout)
62
63 local rspamc_command = string.format("%s --connect %s -j --compact -n %s -t %.3f %s",
64 rspamc_path, host, n_parallel, timeout, path)
65 local result = assert(io.popen(rspamc_command))
66 result = result:read("*all")
67 return result
68 end
69
70 local function encoded_json_to_log(result)
71 -- Returns table containing score, action, list of symbols
72
73 local filtered_result = {}
74 local ucl_parser = ucl.parser()
75
76 local is_good, err = ucl_parser:parse_string(result)
77
78 if not is_good then
79 rspamd_logger.errx("Parser error: %1", err)
80 return nil
81 end
82
83 result = ucl_parser:get_object()
84
85 filtered_result.score = result.score
86 if not result.action then
87 rspamd_logger.errx("Bad JSON: %1", result)
88 return nil
89 end
90 local action = result.action:gsub("%s+", "_")
91 filtered_result.action = action
92
93 filtered_result.symbols = {}
94
95 for sym, _ in pairs(result.symbols) do
96 table.insert(filtered_result.symbols, sym)
97 end
98
99 filtered_result.filename = result.filename
100 filtered_result.scan_time = result.scan_time
101
102 return filtered_result
103 end
104
105 local function filter_scan_results(results, actual_email_type)
106
107 local logs = {}
108
109 results = lua_util.rspamd_str_split(results, "\n")
110
111 if results[#results] == "" then
112 results[#results] = nil
113 end
114
115 for _, result in pairs(results) do
116 result = encoded_json_to_log(result)
117 if result then
118 result['type'] = actual_email_type
119 table.insert(logs, result)
120 end
121 end
122
123 return logs
124 end
125
126 local function get_stats_from_scan_results(results, rules)
127
128 local rule_stats = {}
129 for rule, _ in pairs(rules) do
130 rule_stats[rule] = { tp = 0, tn = 0, fp = 0, fn = 0 }
131 end
132
133 for _, result in ipairs(results) do
134 for _, symbol in ipairs(result["symbols"]) do
135 for name, rule in pairs(rules) do
136 if rule.symbol_spam and rule.symbol_spam == symbol then
137 if result.type == HAM then
138 rule_stats[name].fp = rule_stats[name].fp + 1
139 elseif result.type == SPAM then
140 rule_stats[name].tp = rule_stats[name].tp + 1
141 end
142 elseif rule.symbol_ham and rule.symbol_ham == symbol then
143 if result.type == HAM then
144 rule_stats[name].tn = rule_stats[name].tn + 1
145 elseif result.type == SPAM then
146 rule_stats[name].fn = rule_stats[name].fn + 1
147 end
148 end
149 end
150 end
151 end
152
153 for rule, _ in pairs(rules) do
154 rule_stats[rule].fpr = rule_stats[rule].fp / (rule_stats[rule].fp + rule_stats[rule].tn)
155 rule_stats[rule].fnr = rule_stats[rule].fn / (rule_stats[rule].fn + rule_stats[rule].tp)
156 end
157
158 return rule_stats
159 end
160
161 local function print_neural_stats(neural_stats)
162 for rule, stats in pairs(neural_stats) do
163 rspamd_logger.messagex("\nStats for rule: %s", rule)
164 rspamd_logger.messagex("False positive rate: %s%%", stats.fpr * 100)
165 rspamd_logger.messagex("False negative rate: %s%%", stats.fnr * 100)
166 end
167 end
168
169 local function handler(args)
170 local opts = parser:parse(args)
171
172 local ham_directory = opts['hamdir']
173 local spam_directory = opts['spamdir']
174 local connections = opts["conns"]
175
176 load_config(opts)
177
178 local neural_opts = rspamd_config:get_all_opt('neural')
179
180 if opts["rule"] then
181 local found = false
182 for rule_name, _ in pairs(neural_opts.rules) do
183 if string.lower(rule_name) == string.lower(opts["rule"]) then
184 found = true
185 else
186 neural_opts.rules[rule_name] = nil
187 end
188 end
189
190 if not found then
191 rspamd_logger.errx("Couldn't find the rule %s", opts["rule"])
192 return
193 end
194 end
195
196 local results = {}
197
198 if ham_directory then
199 rspamd_logger.messagex("Scanning ham corpus...")
200 local ham_results = scan_email(opts.rspamc, opts.connect, connections, ham_directory, opts.timeout)
201 ham_results = filter_scan_results(ham_results, HAM)
202
203 for _, result in pairs(ham_results) do
204 table.insert(results, result)
205 end
206 end
207
208 if spam_directory then
209 rspamd_logger.messagex("Scanning spam corpus...")
210 local spam_results = scan_email(opts.rspamc, opts.connect, connections, spam_directory, opts.timeout)
211 spam_results = filter_scan_results(spam_results, SPAM)
212
213 for _, result in pairs(spam_results) do
214 table.insert(results, result)
215 end
216 end
217
218 local neural_stats = get_stats_from_scan_results(results, neural_opts.rules)
219 print_neural_stats(neural_stats)
220
221 end
222
223 return {
224 name = "neuraltest",
225 aliases = { "neural_test" },
226 handler = handler,
227 description = parser._description
228 }