| 1 |
local rspamd_util = require "rspamd_util" |
| 2 |
local lua_util = require "lua_util" |
| 3 |
local argparse = require "argparse" |
| 4 |
local ucl = require "ucl" |
| 5 |
local rspamd_logger = require "rspamd_logger" |
| 6 |
|
| 7 |
local parser = argparse() |
| 8 |
:name "rspamadm classifier_test" |
| 9 |
:description "Learn classifier and evaluate its performance" |
| 10 |
:help_description_margin(32) |
| 11 |
|
| 12 |
parser:option "-H --ham" |
| 13 |
:description("Ham directory") |
| 14 |
:argname("<dir>") |
| 15 |
parser:option "-S --spam" |
| 16 |
:description("Spam directory") |
| 17 |
:argname("<dir>") |
| 18 |
parser:option "-C --classifier" |
| 19 |
:description("Classifier type: bayes or neural") |
| 20 |
:argname("<type>") |
| 21 |
:default('bayes') |
| 22 |
parser:flag "-n --no-learning" |
| 23 |
:description("Do not learn classifier") |
| 24 |
parser:flag "-T --train-only" |
| 25 |
:description("Only train, do not evaluate (neural only)") |
| 26 |
parser:option "--nconns" |
| 27 |
:description("Number of parallel connections") |
| 28 |
:argname("<N>") |
| 29 |
:convert(tonumber) |
| 30 |
:default(10) |
| 31 |
parser:option "-t --timeout" |
| 32 |
:description("Timeout for client connections") |
| 33 |
:argname("<sec>") |
| 34 |
:convert(tonumber) |
| 35 |
:default(60) |
| 36 |
parser:option "-c --connect" |
| 37 |
:description("Connect to specific host") |
| 38 |
:argname("<host>") |
| 39 |
:default('localhost:11334') |
| 40 |
parser:option "-r --rspamc" |
| 41 |
:description("Use specific rspamc path") |
| 42 |
:argname("<path>") |
| 43 |
:default('rspamc') |
| 44 |
parser:option "-f --cv-fraction" |
| 45 |
:description("Use specific fraction for cross-validation") |
| 46 |
:argname("<fraction>") |
| 47 |
:convert(tonumber) |
| 48 |
:default(0.7) |
| 49 |
parser:option "--spam-symbol" |
| 50 |
:description("Use specific spam symbol (auto-detected from classifier type)") |
| 51 |
:argname("<symbol>") |
| 52 |
parser:option "--ham-symbol" |
| 53 |
:description("Use specific ham symbol (auto-detected from classifier type)") |
| 54 |
:argname("<symbol>") |
| 55 |
parser:option "--train-wait" |
| 56 |
:description("Seconds to wait after training for neural network (neural only, should be > watch_interval)") |
| 57 |
:argname("<sec>") |
| 58 |
:convert(tonumber) |
| 59 |
:default(90) |
| 60 |
|
| 61 |
local opts |
| 62 |
|
| 63 |
-- Utility function to split a table into two parts randomly |
| 64 |
local function split_table(t, fraction) |
| 65 |
local shuffled = {} |
| 66 |
for _, v in ipairs(t) do |
| 67 |
local pos = math.random(1, #shuffled + 1) |
| 68 |
table.insert(shuffled, pos, v) |
| 69 |
end |
| 70 |
local split_point = math.floor(#shuffled * tonumber(fraction)) |
| 71 |
local part1 = { lua_util.unpack(shuffled, 1, split_point) } |
| 72 |
local part2 = { lua_util.unpack(shuffled, split_point + 1) } |
| 73 |
return part1, part2 |
| 74 |
end |
| 75 |
|
| 76 |
-- Utility function to get all files in a directory |
| 77 |
local function get_files(dir) |
| 78 |
return rspamd_util.glob(dir .. '/*') |
| 79 |
end |
| 80 |
|
| 81 |
local function list_to_file(list, fname) |
| 82 |
local out = assert(io.open(fname, "w")) |
| 83 |
for _, v in ipairs(list) do |
| 84 |
out:write(v) |
| 85 |
out:write("\n") |
| 86 |
end |
| 87 |
out:close() |
| 88 |
end |
| 89 |
|
| 90 |
-- Function to train the Bayes classifier with given files |
| 91 |
local function train_bayes(files, command) |
| 92 |
local fname = os.tmpname() |
| 93 |
list_to_file(files, fname) |
| 94 |
local rspamc_command = string.format("%s --connect %s -j --compact -n %s -t %.3f %s --files-list=%s", |
| 95 |
opts.rspamc, opts.connect, opts.nconns, opts.timeout, command, fname) |
| 96 |
local handle = assert(io.popen(rspamc_command)) |
| 97 |
handle:read("*all") |
| 98 |
handle:close() |
| 99 |
os.remove(fname) |
| 100 |
end |
| 101 |
|
| 102 |
-- Function to train with ANN-Train header (for neural classifier) |
| 103 |
-- Uses settings to enable only NEURAL_LEARN symbol, skipping full scan |
| 104 |
local function train_neural(files, learn_type) |
| 105 |
local fname = os.tmpname() |
| 106 |
list_to_file(files, fname) |
| 107 |
|
| 108 |
-- Use ANN-Train header with settings to limit scan to NEURAL_LEARN only |
| 109 |
local rspamc_command = string.format( |
| 110 |
"%s --connect %s -j --compact -n %s -t %.3f " .. |
| 111 |
"--settings '{\"symbols_enabled\":[\"NEURAL_LEARN\"]}' " .. |
| 112 |
"--header 'ANN-Train=%s' --files-list=%s", |
| 113 |
opts.rspamc, opts.connect, opts.nconns, opts.timeout, |
| 114 |
learn_type, fname) |
| 115 |
|
| 116 |
local result = assert(io.popen(rspamc_command)) |
| 117 |
local output = result:read("*all") |
| 118 |
result:close() |
| 119 |
os.remove(fname) |
| 120 |
|
| 121 |
-- Count successful submissions |
| 122 |
local count = 0 |
| 123 |
for line in output:gmatch("[^\n]+") do |
| 124 |
local ucl_parser = ucl.parser() |
| 125 |
local is_good, _ = ucl_parser:parse_string(line) |
| 126 |
if is_good then |
| 127 |
count = count + 1 |
| 128 |
end |
| 129 |
end |
| 130 |
|
| 131 |
return count |
| 132 |
end |
| 133 |
|
| 134 |
-- Function to classify files and return results |
| 135 |
local function classify_files(files, known_spam_files, known_ham_files) |
| 136 |
local fname = os.tmpname() |
| 137 |
list_to_file(files, fname) |
| 138 |
|
| 139 |
local settings_header = string.format('--header Settings="{symbols_enabled=[%s, %s]}"', |
| 140 |
opts.spam_symbol, opts.ham_symbol) |
| 141 |
local rspamc_command = string.format("%s %s --connect %s --compact -n %s -t %.3f --files-list=%s", |
| 142 |
opts.rspamc, |
| 143 |
settings_header, |
| 144 |
opts.connect, |
| 145 |
opts.nconns, |
| 146 |
opts.timeout, fname) |
| 147 |
local result = assert(io.popen(rspamc_command)) |
| 148 |
local results = {} |
| 149 |
for line in result:lines() do |
| 150 |
local ucl_parser = ucl.parser() |
| 151 |
local is_good, err = ucl_parser:parse_string(line) |
| 152 |
if not is_good then |
| 153 |
rspamd_logger.errx("Parser error: %1", err) |
| 154 |
else |
| 155 |
local obj = ucl_parser:get_object() |
| 156 |
local file = obj.filename |
| 157 |
local symbols = obj.symbols or {} |
| 158 |
|
| 159 |
if symbols[opts.spam_symbol] then |
| 160 |
local score = symbols[opts.spam_symbol].score |
| 161 |
table.insert(results, { result = "spam", file = file, score = score }) |
| 162 |
if known_ham_files[file] then |
| 163 |
rspamd_logger.message("FP: %s is classified as spam but is known ham", file) |
| 164 |
end |
| 165 |
elseif symbols[opts.ham_symbol] then |
| 166 |
local score = symbols[opts.ham_symbol].score |
| 167 |
table.insert(results, { result = "ham", file = file, score = score }) |
| 168 |
if known_spam_files[file] then |
| 169 |
rspamd_logger.message("FN: %s is classified as ham but is known spam", file) |
| 170 |
end |
| 171 |
else |
| 172 |
-- No classification result |
| 173 |
table.insert(results, { result = "unknown", file = file }) |
| 174 |
end |
| 175 |
end |
| 176 |
end |
| 177 |
|
| 178 |
result:close() |
| 179 |
os.remove(fname) |
| 180 |
|
| 181 |
return results |
| 182 |
end |
| 183 |
|
| 184 |
-- Function to evaluate classifier performance |
| 185 |
local function evaluate_results(results, spam_label, ham_label, |
| 186 |
known_spam_files, known_ham_files, total_cv_files, elapsed) |
| 187 |
local true_positives, false_positives, true_negatives, false_negatives = 0, 0, 0, 0 |
| 188 |
local classified, unclassified = 0, 0 |
| 189 |
|
| 190 |
for _, res in ipairs(results) do |
| 191 |
if res.result == spam_label then |
| 192 |
if known_spam_files[res.file] then |
| 193 |
true_positives = true_positives + 1 |
| 194 |
elseif known_ham_files[res.file] then |
| 195 |
false_positives = false_positives + 1 |
| 196 |
end |
| 197 |
classified = classified + 1 |
| 198 |
elseif res.result == ham_label then |
| 199 |
if known_spam_files[res.file] then |
| 200 |
false_negatives = false_negatives + 1 |
| 201 |
elseif known_ham_files[res.file] then |
| 202 |
true_negatives = true_negatives + 1 |
| 203 |
end |
| 204 |
classified = classified + 1 |
| 205 |
else |
| 206 |
unclassified = unclassified + 1 |
| 207 |
end |
| 208 |
end |
| 209 |
|
| 210 |
print(string.format("\n%-20s %-10s", "Metric", "Value")) |
| 211 |
print(string.rep("-", 35)) |
| 212 |
print(string.format("%-20s %-10d", "True Positives", true_positives)) |
| 213 |
print(string.format("%-20s %-10d", "False Positives", false_positives)) |
| 214 |
print(string.format("%-20s %-10d", "True Negatives", true_negatives)) |
| 215 |
print(string.format("%-20s %-10d", "False Negatives", false_negatives)) |
| 216 |
print(string.format("%-20s %-10d", "Unclassified", unclassified)) |
| 217 |
|
| 218 |
if classified > 0 then |
| 219 |
local accuracy = (true_positives + true_negatives) / classified |
| 220 |
local precision = true_positives > 0 and true_positives / (true_positives + false_positives) or 0 |
| 221 |
local recall = true_positives > 0 and true_positives / (true_positives + false_negatives) or 0 |
| 222 |
local f1_score = (precision + recall) > 0 and 2 * (precision * recall) / (precision + recall) or 0 |
| 223 |
|
| 224 |
print(string.format("%-20s %-10.4f", "Accuracy", accuracy)) |
| 225 |
print(string.format("%-20s %-10.4f", "Precision", precision)) |
| 226 |
print(string.format("%-20s %-10.4f", "Recall", recall)) |
| 227 |
print(string.format("%-20s %-10.4f", "F1 Score", f1_score)) |
| 228 |
end |
| 229 |
|
| 230 |
print(string.format("%-20s %-10.2f%%", "Classified", classified / total_cv_files * 100)) |
| 231 |
print(string.format("%-20s %-10.2f", "Elapsed (sec)", elapsed)) |
| 232 |
end |
| 233 |
|
| 234 |
local function handler(args) |
| 235 |
opts = parser:parse(args) |
| 236 |
|
| 237 |
local ham_directory = opts['ham'] |
| 238 |
local spam_directory = opts['spam'] |
| 239 |
local classifier_type = opts['classifier'] |
| 240 |
|
| 241 |
if not ham_directory or not spam_directory then |
| 242 |
print("Error: Both --ham and --spam directories are required") |
| 243 |
os.exit(1) |
| 244 |
end |
| 245 |
|
| 246 |
-- Normalize classifier type (accept legacy 'llm_embeddings' as alias for 'neural') |
| 247 |
if classifier_type == 'llm_embeddings' then |
| 248 |
classifier_type = 'neural' |
| 249 |
end |
| 250 |
|
| 251 |
-- Set default symbols based on classifier type |
| 252 |
if not opts.spam_symbol then |
| 253 |
if classifier_type == 'neural' then |
| 254 |
opts.spam_symbol = 'NEURAL_SPAM' |
| 255 |
else |
| 256 |
opts.spam_symbol = 'BAYES_SPAM' |
| 257 |
end |
| 258 |
end |
| 259 |
if not opts.ham_symbol then |
| 260 |
if classifier_type == 'neural' then |
| 261 |
opts.ham_symbol = 'NEURAL_HAM' |
| 262 |
else |
| 263 |
opts.ham_symbol = 'BAYES_HAM' |
| 264 |
end |
| 265 |
end |
| 266 |
|
| 267 |
-- Get all files |
| 268 |
local spam_files = get_files(spam_directory) |
| 269 |
local known_spam_files = lua_util.list_to_hash(spam_files) |
| 270 |
local ham_files = get_files(ham_directory) |
| 271 |
local known_ham_files = lua_util.list_to_hash(ham_files) |
| 272 |
|
| 273 |
print(string.format("Classifier: %s", classifier_type)) |
| 274 |
print(string.format("Found %d spam files, %d ham files", #spam_files, #ham_files)) |
| 275 |
|
| 276 |
-- Split files into training and cross-validation sets |
| 277 |
local train_spam, cv_spam = split_table(spam_files, opts.cv_fraction) |
| 278 |
local train_ham, cv_ham = split_table(ham_files, opts.cv_fraction) |
| 279 |
|
| 280 |
print(string.format("Split: %d/%d spam (train/test), %d/%d ham (train/test)", |
| 281 |
#train_spam, #cv_spam, #train_ham, #cv_ham)) |
| 282 |
|
| 283 |
-- Training phase |
| 284 |
if not opts.no_learning then |
| 285 |
print("\n=== Training Phase ===") |
| 286 |
|
| 287 |
local t, train_spam_time, train_ham_time |
| 288 |
|
| 289 |
if classifier_type == 'neural' then |
| 290 |
-- Neural training using ANN-Train header |
| 291 |
-- Interleave spam and ham submissions for balanced training |
| 292 |
print(string.format("Training %d spam + %d ham messages (interleaved)...", #train_spam, #train_ham)) |
| 293 |
t = rspamd_util.get_time() |
| 294 |
|
| 295 |
-- Create interleaved list of {file, type} pairs |
| 296 |
local interleaved = {} |
| 297 |
local spam_idx, ham_idx = 1, 1 |
| 298 |
while spam_idx <= #train_spam or ham_idx <= #train_ham do |
| 299 |
if spam_idx <= #train_spam then |
| 300 |
table.insert(interleaved, { file = train_spam[spam_idx], type = 'spam' }) |
| 301 |
spam_idx = spam_idx + 1 |
| 302 |
end |
| 303 |
if ham_idx <= #train_ham then |
| 304 |
table.insert(interleaved, { file = train_ham[ham_idx], type = 'ham' }) |
| 305 |
ham_idx = ham_idx + 1 |
| 306 |
end |
| 307 |
end |
| 308 |
|
| 309 |
-- Submit in batches, grouped by type for efficiency |
| 310 |
local batch_size = math.max(1, math.floor(#interleaved / 10)) |
| 311 |
local spam_batch, ham_batch = {}, {} |
| 312 |
local spam_trained, ham_trained = 0, 0 |
| 313 |
|
| 314 |
for i, item in ipairs(interleaved) do |
| 315 |
if item.type == 'spam' then |
| 316 |
table.insert(spam_batch, item.file) |
| 317 |
else |
| 318 |
table.insert(ham_batch, item.file) |
| 319 |
end |
| 320 |
|
| 321 |
-- Submit batches periodically |
| 322 |
if i % batch_size == 0 or i == #interleaved then |
| 323 |
if #spam_batch > 0 then |
| 324 |
spam_trained = spam_trained + train_neural(spam_batch, "spam") |
| 325 |
spam_batch = {} |
| 326 |
end |
| 327 |
if #ham_batch > 0 then |
| 328 |
ham_trained = ham_trained + train_neural(ham_batch, "ham") |
| 329 |
ham_batch = {} |
| 330 |
end |
| 331 |
end |
| 332 |
end |
| 333 |
|
| 334 |
train_spam_time = rspamd_util.get_time() - t |
| 335 |
train_ham_time = 0 -- Combined time |
| 336 |
print(string.format(" Submitted %d spam + %d ham samples in %.2f seconds", |
| 337 |
spam_trained, ham_trained, train_spam_time)) |
| 338 |
|
| 339 |
-- Wait for neural network to train using ev_base sleep |
| 340 |
print(string.format("\nWaiting %d seconds for neural network training...", opts.train_wait)) |
| 341 |
rspamadm_ev_base:sleep(opts.train_wait) |
| 342 |
print("Training wait complete.") |
| 343 |
else |
| 344 |
-- Bayes training using learn_spam/learn_ham |
| 345 |
print(string.format("Start learn spam, %d messages, %d connections", #train_spam, opts.nconns)) |
| 346 |
t = rspamd_util.get_time() |
| 347 |
train_bayes(train_spam, "learn_spam") |
| 348 |
train_spam_time = rspamd_util.get_time() - t |
| 349 |
|
| 350 |
print(string.format("Start learn ham, %d messages, %d connections", #train_ham, opts.nconns)) |
| 351 |
t = rspamd_util.get_time() |
| 352 |
train_bayes(train_ham, "learn_ham") |
| 353 |
train_ham_time = rspamd_util.get_time() - t |
| 354 |
|
| 355 |
print(string.format("Learning done: %d spam in %.2f sec, %d ham in %.2f sec", |
| 356 |
#train_spam, train_spam_time, #train_ham, train_ham_time)) |
| 357 |
end |
| 358 |
else |
| 359 |
print("\nSkipping training phase (--no-learning)") |
| 360 |
end |
| 361 |
|
| 362 |
if opts.train_only then |
| 363 |
print("\nTraining only mode - skipping evaluation") |
| 364 |
return |
| 365 |
end |
| 366 |
|
| 367 |
-- Cross-validation phase |
| 368 |
print("\n=== Evaluation Phase ===") |
| 369 |
|
| 370 |
local cv_files = {} |
| 371 |
for _, file in ipairs(cv_spam) do |
| 372 |
table.insert(cv_files, file) |
| 373 |
end |
| 374 |
for _, file in ipairs(cv_ham) do |
| 375 |
table.insert(cv_files, file) |
| 376 |
end |
| 377 |
|
| 378 |
-- Shuffle cross-validation files |
| 379 |
cv_files = split_table(cv_files, 1) |
| 380 |
|
| 381 |
print(string.format("Classifying %d test messages...", #cv_files)) |
| 382 |
|
| 383 |
-- Get classification results |
| 384 |
local t = rspamd_util.get_time() |
| 385 |
local results = classify_files(cv_files, known_spam_files, known_ham_files) |
| 386 |
local elapsed = rspamd_util.get_time() - t |
| 387 |
|
| 388 |
-- Evaluate results |
| 389 |
evaluate_results(results, "spam", "ham", |
| 390 |
known_spam_files, |
| 391 |
known_ham_files, |
| 392 |
#cv_files, |
| 393 |
elapsed) |
| 394 |
end |
| 395 |
|
| 396 |
return { |
| 397 |
name = 'classifiertest', |
| 398 |
aliases = { 'classifier_test' }, |
| 399 |
handler = handler, |
| 400 |
description = parser._description |
| 401 |
} |