Debian Codegraph @

lualib/rspamadm/classifier_test.lua

raw
1 local rspamd_util = require "rspamd_util"
2 local lua_util = require "lua_util"
3 local argparse = require "argparse"
4 local ucl = require "ucl"
5 local rspamd_logger = require "rspamd_logger"
6
7 local parser = argparse()
8 :name "rspamadm classifier_test"
9 :description "Learn classifier and evaluate its performance"
10 :help_description_margin(32)
11
12 parser:option "-H --ham"
13 :description("Ham directory")
14 :argname("<dir>")
15 parser:option "-S --spam"
16 :description("Spam directory")
17 :argname("<dir>")
18 parser:option "-C --classifier"
19 :description("Classifier type: bayes or neural")
20 :argname("<type>")
21 :default('bayes')
22 parser:flag "-n --no-learning"
23 :description("Do not learn classifier")
24 parser:flag "-T --train-only"
25 :description("Only train, do not evaluate (neural only)")
26 parser:option "--nconns"
27 :description("Number of parallel connections")
28 :argname("<N>")
29 :convert(tonumber)
30 :default(10)
31 parser:option "-t --timeout"
32 :description("Timeout for client connections")
33 :argname("<sec>")
34 :convert(tonumber)
35 :default(60)
36 parser:option "-c --connect"
37 :description("Connect to specific host")
38 :argname("<host>")
39 :default('localhost:11334')
40 parser:option "-r --rspamc"
41 :description("Use specific rspamc path")
42 :argname("<path>")
43 :default('rspamc')
44 parser:option "-f --cv-fraction"
45 :description("Use specific fraction for cross-validation")
46 :argname("<fraction>")
47 :convert(tonumber)
48 :default(0.7)
49 parser:option "--spam-symbol"
50 :description("Use specific spam symbol (auto-detected from classifier type)")
51 :argname("<symbol>")
52 parser:option "--ham-symbol"
53 :description("Use specific ham symbol (auto-detected from classifier type)")
54 :argname("<symbol>")
55 parser:option "--train-wait"
56 :description("Seconds to wait after training for neural network (neural only, should be > watch_interval)")
57 :argname("<sec>")
58 :convert(tonumber)
59 :default(90)
60
61 local opts
62
63 -- Utility function to split a table into two parts randomly
64 local function split_table(t, fraction)
65 local shuffled = {}
66 for _, v in ipairs(t) do
67 local pos = math.random(1, #shuffled + 1)
68 table.insert(shuffled, pos, v)
69 end
70 local split_point = math.floor(#shuffled * tonumber(fraction))
71 local part1 = { lua_util.unpack(shuffled, 1, split_point) }
72 local part2 = { lua_util.unpack(shuffled, split_point + 1) }
73 return part1, part2
74 end
75
76 -- Utility function to get all files in a directory
77 local function get_files(dir)
78 return rspamd_util.glob(dir .. '/*')
79 end
80
81 local function list_to_file(list, fname)
82 local out = assert(io.open(fname, "w"))
83 for _, v in ipairs(list) do
84 out:write(v)
85 out:write("\n")
86 end
87 out:close()
88 end
89
90 -- Function to train the Bayes classifier with given files
91 local function train_bayes(files, command)
92 local fname = os.tmpname()
93 list_to_file(files, fname)
94 local rspamc_command = string.format("%s --connect %s -j --compact -n %s -t %.3f %s --files-list=%s",
95 opts.rspamc, opts.connect, opts.nconns, opts.timeout, command, fname)
96 local handle = assert(io.popen(rspamc_command))
97 handle:read("*all")
98 handle:close()
99 os.remove(fname)
100 end
101
102 -- Function to train with ANN-Train header (for neural classifier)
103 -- Uses settings to enable only NEURAL_LEARN symbol, skipping full scan
104 local function train_neural(files, learn_type)
105 local fname = os.tmpname()
106 list_to_file(files, fname)
107
108 -- Use ANN-Train header with settings to limit scan to NEURAL_LEARN only
109 local rspamc_command = string.format(
110 "%s --connect %s -j --compact -n %s -t %.3f " ..
111 "--settings '{\"symbols_enabled\":[\"NEURAL_LEARN\"]}' " ..
112 "--header 'ANN-Train=%s' --files-list=%s",
113 opts.rspamc, opts.connect, opts.nconns, opts.timeout,
114 learn_type, fname)
115
116 local result = assert(io.popen(rspamc_command))
117 local output = result:read("*all")
118 result:close()
119 os.remove(fname)
120
121 -- Count successful submissions
122 local count = 0
123 for line in output:gmatch("[^\n]+") do
124 local ucl_parser = ucl.parser()
125 local is_good, _ = ucl_parser:parse_string(line)
126 if is_good then
127 count = count + 1
128 end
129 end
130
131 return count
132 end
133
134 -- Function to classify files and return results
135 local function classify_files(files, known_spam_files, known_ham_files)
136 local fname = os.tmpname()
137 list_to_file(files, fname)
138
139 local settings_header = string.format('--header Settings="{symbols_enabled=[%s, %s]}"',
140 opts.spam_symbol, opts.ham_symbol)
141 local rspamc_command = string.format("%s %s --connect %s --compact -n %s -t %.3f --files-list=%s",
142 opts.rspamc,
143 settings_header,
144 opts.connect,
145 opts.nconns,
146 opts.timeout, fname)
147 local result = assert(io.popen(rspamc_command))
148 local results = {}
149 for line in result:lines() do
150 local ucl_parser = ucl.parser()
151 local is_good, err = ucl_parser:parse_string(line)
152 if not is_good then
153 rspamd_logger.errx("Parser error: %1", err)
154 else
155 local obj = ucl_parser:get_object()
156 local file = obj.filename
157 local symbols = obj.symbols or {}
158
159 if symbols[opts.spam_symbol] then
160 local score = symbols[opts.spam_symbol].score
161 table.insert(results, { result = "spam", file = file, score = score })
162 if known_ham_files[file] then
163 rspamd_logger.message("FP: %s is classified as spam but is known ham", file)
164 end
165 elseif symbols[opts.ham_symbol] then
166 local score = symbols[opts.ham_symbol].score
167 table.insert(results, { result = "ham", file = file, score = score })
168 if known_spam_files[file] then
169 rspamd_logger.message("FN: %s is classified as ham but is known spam", file)
170 end
171 else
172 -- No classification result
173 table.insert(results, { result = "unknown", file = file })
174 end
175 end
176 end
177
178 result:close()
179 os.remove(fname)
180
181 return results
182 end
183
184 -- Function to evaluate classifier performance
185 local function evaluate_results(results, spam_label, ham_label,
186 known_spam_files, known_ham_files, total_cv_files, elapsed)
187 local true_positives, false_positives, true_negatives, false_negatives = 0, 0, 0, 0
188 local classified, unclassified = 0, 0
189
190 for _, res in ipairs(results) do
191 if res.result == spam_label then
192 if known_spam_files[res.file] then
193 true_positives = true_positives + 1
194 elseif known_ham_files[res.file] then
195 false_positives = false_positives + 1
196 end
197 classified = classified + 1
198 elseif res.result == ham_label then
199 if known_spam_files[res.file] then
200 false_negatives = false_negatives + 1
201 elseif known_ham_files[res.file] then
202 true_negatives = true_negatives + 1
203 end
204 classified = classified + 1
205 else
206 unclassified = unclassified + 1
207 end
208 end
209
210 print(string.format("\n%-20s %-10s", "Metric", "Value"))
211 print(string.rep("-", 35))
212 print(string.format("%-20s %-10d", "True Positives", true_positives))
213 print(string.format("%-20s %-10d", "False Positives", false_positives))
214 print(string.format("%-20s %-10d", "True Negatives", true_negatives))
215 print(string.format("%-20s %-10d", "False Negatives", false_negatives))
216 print(string.format("%-20s %-10d", "Unclassified", unclassified))
217
218 if classified > 0 then
219 local accuracy = (true_positives + true_negatives) / classified
220 local precision = true_positives > 0 and true_positives / (true_positives + false_positives) or 0
221 local recall = true_positives > 0 and true_positives / (true_positives + false_negatives) or 0
222 local f1_score = (precision + recall) > 0 and 2 * (precision * recall) / (precision + recall) or 0
223
224 print(string.format("%-20s %-10.4f", "Accuracy", accuracy))
225 print(string.format("%-20s %-10.4f", "Precision", precision))
226 print(string.format("%-20s %-10.4f", "Recall", recall))
227 print(string.format("%-20s %-10.4f", "F1 Score", f1_score))
228 end
229
230 print(string.format("%-20s %-10.2f%%", "Classified", classified / total_cv_files * 100))
231 print(string.format("%-20s %-10.2f", "Elapsed (sec)", elapsed))
232 end
233
234 local function handler(args)
235 opts = parser:parse(args)
236
237 local ham_directory = opts['ham']
238 local spam_directory = opts['spam']
239 local classifier_type = opts['classifier']
240
241 if not ham_directory or not spam_directory then
242 print("Error: Both --ham and --spam directories are required")
243 os.exit(1)
244 end
245
246 -- Normalize classifier type (accept legacy 'llm_embeddings' as alias for 'neural')
247 if classifier_type == 'llm_embeddings' then
248 classifier_type = 'neural'
249 end
250
251 -- Set default symbols based on classifier type
252 if not opts.spam_symbol then
253 if classifier_type == 'neural' then
254 opts.spam_symbol = 'NEURAL_SPAM'
255 else
256 opts.spam_symbol = 'BAYES_SPAM'
257 end
258 end
259 if not opts.ham_symbol then
260 if classifier_type == 'neural' then
261 opts.ham_symbol = 'NEURAL_HAM'
262 else
263 opts.ham_symbol = 'BAYES_HAM'
264 end
265 end
266
267 -- Get all files
268 local spam_files = get_files(spam_directory)
269 local known_spam_files = lua_util.list_to_hash(spam_files)
270 local ham_files = get_files(ham_directory)
271 local known_ham_files = lua_util.list_to_hash(ham_files)
272
273 print(string.format("Classifier: %s", classifier_type))
274 print(string.format("Found %d spam files, %d ham files", #spam_files, #ham_files))
275
276 -- Split files into training and cross-validation sets
277 local train_spam, cv_spam = split_table(spam_files, opts.cv_fraction)
278 local train_ham, cv_ham = split_table(ham_files, opts.cv_fraction)
279
280 print(string.format("Split: %d/%d spam (train/test), %d/%d ham (train/test)",
281 #train_spam, #cv_spam, #train_ham, #cv_ham))
282
283 -- Training phase
284 if not opts.no_learning then
285 print("\n=== Training Phase ===")
286
287 local t, train_spam_time, train_ham_time
288
289 if classifier_type == 'neural' then
290 -- Neural training using ANN-Train header
291 -- Interleave spam and ham submissions for balanced training
292 print(string.format("Training %d spam + %d ham messages (interleaved)...", #train_spam, #train_ham))
293 t = rspamd_util.get_time()
294
295 -- Create interleaved list of {file, type} pairs
296 local interleaved = {}
297 local spam_idx, ham_idx = 1, 1
298 while spam_idx <= #train_spam or ham_idx <= #train_ham do
299 if spam_idx <= #train_spam then
300 table.insert(interleaved, { file = train_spam[spam_idx], type = 'spam' })
301 spam_idx = spam_idx + 1
302 end
303 if ham_idx <= #train_ham then
304 table.insert(interleaved, { file = train_ham[ham_idx], type = 'ham' })
305 ham_idx = ham_idx + 1
306 end
307 end
308
309 -- Submit in batches, grouped by type for efficiency
310 local batch_size = math.max(1, math.floor(#interleaved / 10))
311 local spam_batch, ham_batch = {}, {}
312 local spam_trained, ham_trained = 0, 0
313
314 for i, item in ipairs(interleaved) do
315 if item.type == 'spam' then
316 table.insert(spam_batch, item.file)
317 else
318 table.insert(ham_batch, item.file)
319 end
320
321 -- Submit batches periodically
322 if i % batch_size == 0 or i == #interleaved then
323 if #spam_batch > 0 then
324 spam_trained = spam_trained + train_neural(spam_batch, "spam")
325 spam_batch = {}
326 end
327 if #ham_batch > 0 then
328 ham_trained = ham_trained + train_neural(ham_batch, "ham")
329 ham_batch = {}
330 end
331 end
332 end
333
334 train_spam_time = rspamd_util.get_time() - t
335 train_ham_time = 0 -- Combined time
336 print(string.format(" Submitted %d spam + %d ham samples in %.2f seconds",
337 spam_trained, ham_trained, train_spam_time))
338
339 -- Wait for neural network to train using ev_base sleep
340 print(string.format("\nWaiting %d seconds for neural network training...", opts.train_wait))
341 rspamadm_ev_base:sleep(opts.train_wait)
342 print("Training wait complete.")
343 else
344 -- Bayes training using learn_spam/learn_ham
345 print(string.format("Start learn spam, %d messages, %d connections", #train_spam, opts.nconns))
346 t = rspamd_util.get_time()
347 train_bayes(train_spam, "learn_spam")
348 train_spam_time = rspamd_util.get_time() - t
349
350 print(string.format("Start learn ham, %d messages, %d connections", #train_ham, opts.nconns))
351 t = rspamd_util.get_time()
352 train_bayes(train_ham, "learn_ham")
353 train_ham_time = rspamd_util.get_time() - t
354
355 print(string.format("Learning done: %d spam in %.2f sec, %d ham in %.2f sec",
356 #train_spam, train_spam_time, #train_ham, train_ham_time))
357 end
358 else
359 print("\nSkipping training phase (--no-learning)")
360 end
361
362 if opts.train_only then
363 print("\nTraining only mode - skipping evaluation")
364 return
365 end
366
367 -- Cross-validation phase
368 print("\n=== Evaluation Phase ===")
369
370 local cv_files = {}
371 for _, file in ipairs(cv_spam) do
372 table.insert(cv_files, file)
373 end
374 for _, file in ipairs(cv_ham) do
375 table.insert(cv_files, file)
376 end
377
378 -- Shuffle cross-validation files
379 cv_files = split_table(cv_files, 1)
380
381 print(string.format("Classifying %d test messages...", #cv_files))
382
383 -- Get classification results
384 local t = rspamd_util.get_time()
385 local results = classify_files(cv_files, known_spam_files, known_ham_files)
386 local elapsed = rspamd_util.get_time() - t
387
388 -- Evaluate results
389 evaluate_results(results, "spam", "ham",
390 known_spam_files,
391 known_ham_files,
392 #cv_files,
393 elapsed)
394 end
395
396 return {
397 name = 'classifiertest',
398 aliases = { 'classifier_test' },
399 handler = handler,
400 description = parser._description
401 }