| 1 |
--[[ |
| 2 |
Copyright (c) 2022, Vsevolod Stakhov <[email protected]> |
| 3 |
|
| 4 |
Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 |
you may not use this file except in compliance with the License. |
| 6 |
You may obtain a copy of the License at |
| 7 |
|
| 8 |
http://www.apache.org/licenses/LICENSE-2.0 |
| 9 |
|
| 10 |
Unless required by applicable law or agreed to in writing, software |
| 11 |
distributed under the License is distributed on an "AS IS" BASIS, |
| 12 |
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 |
See the License for the specific language governing permissions and |
| 14 |
limitations under the License. |
| 15 |
]]-- |
| 16 |
|
| 17 |
local argparse = require "argparse" |
| 18 |
local lua_clickhouse = require "lua_clickhouse" |
| 19 |
local lua_util = require "lua_util" |
| 20 |
local rspamd_http = require "rspamd_http" |
| 21 |
local rspamd_upstream_list = require "rspamd_upstream_list" |
| 22 |
local rspamd_logger = require "rspamd_logger" |
| 23 |
local ucl = require "ucl" |
| 24 |
|
| 25 |
local E = {} |
| 26 |
|
| 27 |
-- Define command line options |
| 28 |
local parser = argparse() |
| 29 |
:name 'rspamadm clickhouse' |
| 30 |
:description 'Retrieve information from Clickhouse' |
| 31 |
:help_description_margin(30) |
| 32 |
:command_target('command') |
| 33 |
:require_command(true) |
| 34 |
|
| 35 |
parser:option '-c --config' |
| 36 |
:description 'Path to config file' |
| 37 |
:argname('config_file') |
| 38 |
:default(rspamd_paths['CONFDIR'] .. '/rspamd.conf') |
| 39 |
parser:option '-d --database' |
| 40 |
:description 'Name of Clickhouse database to use' |
| 41 |
:argname('database') |
| 42 |
:default('default') |
| 43 |
parser:flag '--no-ssl-verify' |
| 44 |
:description 'Disable SSL verification' |
| 45 |
:argname('no_ssl_verify') |
| 46 |
parser:mutex( |
| 47 |
parser:option '-p --password' |
| 48 |
:description 'Password to use for Clickhouse' |
| 49 |
:argname('password'), |
| 50 |
parser:flag '-a --ask-password' |
| 51 |
:description 'Ask password from the terminal' |
| 52 |
:argname('ask_password') |
| 53 |
) |
| 54 |
parser:option '-s --server' |
| 55 |
:description 'Address[:port] to connect to Clickhouse with' |
| 56 |
:argname('server') |
| 57 |
parser:option '-u --user' |
| 58 |
:description 'Username to use for Clickhouse' |
| 59 |
:argname('user') |
| 60 |
parser:option '--use-gzip' |
| 61 |
:description 'Use Gzip with Clickhouse' |
| 62 |
:argname('use_gzip') |
| 63 |
:default(true) |
| 64 |
parser:flag '--use-https' |
| 65 |
:description 'Use HTTPS with Clickhouse' |
| 66 |
:argname('use_https') |
| 67 |
|
| 68 |
local neural_profile = parser:command 'neural_profile' |
| 69 |
:description 'Generate symbols profile using data from Clickhouse' |
| 70 |
neural_profile:option '-w --where' |
| 71 |
:description 'WHERE clause for Clickhouse query' |
| 72 |
:argname('where') |
| 73 |
neural_profile:flag '-j --json' |
| 74 |
:description 'Write output as JSON' |
| 75 |
:argname('json') |
| 76 |
neural_profile:option '--days' |
| 77 |
:description 'Number of days to collect stats for' |
| 78 |
:argname('days') |
| 79 |
:default('7') |
| 80 |
neural_profile:option '--limit -l' |
| 81 |
:description 'Maximum rows to fetch per day' |
| 82 |
:argname('limit') |
| 83 |
neural_profile:option '--settings-id' |
| 84 |
:description 'Settings ID to query' |
| 85 |
:argname('settings_id') |
| 86 |
:default('') |
| 87 |
|
| 88 |
local neural_train = parser:command 'neural_train' |
| 89 |
:description 'Train neural using data from Clickhouse' |
| 90 |
neural_train:option '--days' |
| 91 |
:description 'Number of days to query data for' |
| 92 |
:argname('days') |
| 93 |
:default('7') |
| 94 |
neural_train:option '--column-name-digest' |
| 95 |
:description 'Name of neural profile digest column in Clickhouse' |
| 96 |
:argname('column_name_digest') |
| 97 |
:default('NeuralDigest') |
| 98 |
neural_train:option '--column-name-vector' |
| 99 |
:description 'Name of neural training vector column in Clickhouse' |
| 100 |
:argname('column_name_vector') |
| 101 |
:default('NeuralMpack') |
| 102 |
neural_train:option '--limit -l' |
| 103 |
:description 'Maximum rows to fetch per day' |
| 104 |
:argname('limit') |
| 105 |
neural_train:option '--profile -p' |
| 106 |
:description 'Profile to use for training' |
| 107 |
:argname('profile') |
| 108 |
:default('default') |
| 109 |
neural_train:option '--rule -r' |
| 110 |
:description 'Rule to train' |
| 111 |
:argname('rule') |
| 112 |
:default('default') |
| 113 |
neural_train:option '--spam -s' |
| 114 |
:description 'WHERE clause to use for spam' |
| 115 |
:argname('spam') |
| 116 |
:default("Action == 'reject'") |
| 117 |
neural_train:option '--ham -h' |
| 118 |
:description 'WHERE clause to use for ham' |
| 119 |
:argname('ham') |
| 120 |
:default('Score < 0') |
| 121 |
neural_train:option '--url -u' |
| 122 |
:description 'URL to use for training' |
| 123 |
:argname('url') |
| 124 |
:default('http://127.0.0.1:11334/plugins/neural/learn') |
| 125 |
|
| 126 |
local http_params = { |
| 127 |
config = rspamd_config, |
| 128 |
ev_base = rspamadm_ev_base, |
| 129 |
session = rspamadm_session, |
| 130 |
resolver = rspamadm_dns_resolver, |
| 131 |
} |
| 132 |
|
| 133 |
local function load_config(config_file) |
| 134 |
local _r, err = rspamd_config:load_ucl(config_file) |
| 135 |
|
| 136 |
if not _r then |
| 137 |
rspamd_logger.errx('cannot load %s: %s', config_file, err) |
| 138 |
os.exit(1) |
| 139 |
end |
| 140 |
|
| 141 |
_r, err = rspamd_config:parse_rcl({ 'logging', 'worker' }) |
| 142 |
if not _r then |
| 143 |
rspamd_logger.errx('cannot process %s: %s', config_file, err) |
| 144 |
os.exit(1) |
| 145 |
end |
| 146 |
|
| 147 |
if not rspamd_config:init_modules() then |
| 148 |
rspamd_logger.errx('cannot init modules when parsing %s', config_file) |
| 149 |
os.exit(1) |
| 150 |
end |
| 151 |
|
| 152 |
rspamd_config:init_subsystem('symcache') |
| 153 |
end |
| 154 |
|
| 155 |
local function days_list(days) |
| 156 |
-- Create list of days to query starting with yesterday |
| 157 |
local query_days = {} |
| 158 |
local previous_date = os.time() - 86400 |
| 159 |
local num_days = tonumber(days) |
| 160 |
for _ = 1, num_days do |
| 161 |
table.insert(query_days, os.date('%Y-%m-%d', previous_date)) |
| 162 |
previous_date = previous_date - 86400 |
| 163 |
end |
| 164 |
return query_days |
| 165 |
end |
| 166 |
|
| 167 |
local function get_excluded_symbols(known_symbols, correlations, seen_total) |
| 168 |
-- Walk results once to collect all symbols & count occurrences |
| 169 |
|
| 170 |
local remove = {} |
| 171 |
local known_symbols_list = {} |
| 172 |
local composites = rspamd_config:get_all_opt('composites') |
| 173 |
local all_symbols = rspamd_config:get_symbols() |
| 174 |
local skip_flags = { |
| 175 |
nostat = true, |
| 176 |
skip = true, |
| 177 |
idempotent = true, |
| 178 |
composite = true, |
| 179 |
} |
| 180 |
for k, v in pairs(known_symbols) do |
| 181 |
local lower_count, higher_count |
| 182 |
if v.seen_spam > v.seen_ham then |
| 183 |
lower_count = v.seen_ham |
| 184 |
higher_count = v.seen_spam |
| 185 |
else |
| 186 |
lower_count = v.seen_spam |
| 187 |
higher_count = v.seen_ham |
| 188 |
end |
| 189 |
|
| 190 |
if composites[k] then |
| 191 |
remove[k] = 'composite symbol' |
| 192 |
elseif lower_count / higher_count >= 0.95 then |
| 193 |
remove[k] = 'weak ham/spam correlation' |
| 194 |
elseif v.seen / seen_total >= 0.9 then |
| 195 |
remove[k] = 'omnipresent symbol' |
| 196 |
elseif not all_symbols[k] then |
| 197 |
remove[k] = 'nonexistent symbol' |
| 198 |
else |
| 199 |
for fl, _ in pairs(all_symbols[k].flags or {}) do |
| 200 |
if skip_flags[fl] then |
| 201 |
remove[k] = fl .. ' symbol' |
| 202 |
break |
| 203 |
end |
| 204 |
end |
| 205 |
end |
| 206 |
known_symbols_list[v.id] = { |
| 207 |
seen = v.seen, |
| 208 |
name = k, |
| 209 |
} |
| 210 |
end |
| 211 |
|
| 212 |
-- Walk correlation matrix and check total counts |
| 213 |
for sym_id, row in pairs(correlations) do |
| 214 |
for inner_sym_id, count in pairs(row) do |
| 215 |
local known = known_symbols_list[sym_id] |
| 216 |
local inner = known_symbols_list[inner_sym_id] |
| 217 |
if known and count == known.seen and not remove[inner.name] and not remove[known.name] then |
| 218 |
remove[known.name] = string.format("overlapped by %s", |
| 219 |
known_symbols_list[inner_sym_id].name) |
| 220 |
end |
| 221 |
end |
| 222 |
end |
| 223 |
|
| 224 |
return remove |
| 225 |
end |
| 226 |
|
| 227 |
local function handle_neural_profile(args) |
| 228 |
|
| 229 |
local known_symbols, correlations = {}, {} |
| 230 |
local symbols_count, seen_total = 0, 0 |
| 231 |
|
| 232 |
local function process_row(r) |
| 233 |
local is_spam = true |
| 234 |
if r['Action'] == 'no action' or r['Action'] == 'greylist' then |
| 235 |
is_spam = false |
| 236 |
end |
| 237 |
seen_total = seen_total + 1 |
| 238 |
|
| 239 |
local nsym = #r['Symbols.Names'] |
| 240 |
|
| 241 |
for i = 1, nsym do |
| 242 |
local sym = r['Symbols.Names'][i] |
| 243 |
local t = known_symbols[sym] |
| 244 |
if not t then |
| 245 |
local spam_count, ham_count = 0, 0 |
| 246 |
if is_spam then |
| 247 |
spam_count = spam_count + 1 |
| 248 |
else |
| 249 |
ham_count = ham_count + 1 |
| 250 |
end |
| 251 |
known_symbols[sym] = { |
| 252 |
id = symbols_count, |
| 253 |
seen = 1, |
| 254 |
seen_ham = ham_count, |
| 255 |
seen_spam = spam_count, |
| 256 |
} |
| 257 |
symbols_count = symbols_count + 1 |
| 258 |
else |
| 259 |
known_symbols[sym].seen = known_symbols[sym].seen + 1 |
| 260 |
if is_spam then |
| 261 |
known_symbols[sym].seen_spam = known_symbols[sym].seen_spam + 1 |
| 262 |
else |
| 263 |
known_symbols[sym].seen_ham = known_symbols[sym].seen_ham + 1 |
| 264 |
end |
| 265 |
end |
| 266 |
end |
| 267 |
|
| 268 |
-- Fill correlations |
| 269 |
for i = 1, nsym do |
| 270 |
for j = 1, nsym do |
| 271 |
if i ~= j then |
| 272 |
local sym = r['Symbols.Names'][i] |
| 273 |
local inner_sym_name = r['Symbols.Names'][j] |
| 274 |
local known_sym = known_symbols[sym] |
| 275 |
local inner_sym = known_symbols[inner_sym_name] |
| 276 |
if known_sym and inner_sym then |
| 277 |
if not correlations[known_sym.id] then |
| 278 |
correlations[known_sym.id] = {} |
| 279 |
end |
| 280 |
local n = correlations[known_sym.id][inner_sym.id] or 0 |
| 281 |
n = n + 1 |
| 282 |
correlations[known_sym.id][inner_sym.id] = n |
| 283 |
end |
| 284 |
end |
| 285 |
end |
| 286 |
end |
| 287 |
end |
| 288 |
|
| 289 |
local query_days = days_list(args.days) |
| 290 |
local conditions = {} |
| 291 |
table.insert(conditions, string.format("SettingsId = '%s'", args.settings_id)) |
| 292 |
local limit = '' |
| 293 |
local num_limit = tonumber(args.limit) |
| 294 |
if num_limit then |
| 295 |
limit = string.format(' LIMIT %d', num_limit) -- Contains leading space |
| 296 |
end |
| 297 |
if args.where then |
| 298 |
table.insert(conditions, args.where) |
| 299 |
end |
| 300 |
|
| 301 |
local query_fmt = 'SELECT Action, Symbols.Names FROM rspamd WHERE %s%s' |
| 302 |
for _, query_day in ipairs(query_days) do |
| 303 |
-- Date should be the last condition |
| 304 |
table.insert(conditions, string.format("Date = '%s'", query_day)) |
| 305 |
local query = string.format(query_fmt, table.concat(conditions, ' AND '), limit) |
| 306 |
local upstream = args.upstream:get_upstream_round_robin() |
| 307 |
local err = lua_clickhouse.select_sync(upstream, args, http_params, query, process_row) |
| 308 |
if err ~= nil then |
| 309 |
io.stderr:write(string.format('Error querying Clickhouse: %s\n', err)) |
| 310 |
os.exit(1) |
| 311 |
end |
| 312 |
conditions[#conditions] = nil -- remove Date condition |
| 313 |
end |
| 314 |
|
| 315 |
local remove = get_excluded_symbols(known_symbols, correlations, seen_total) |
| 316 |
if not args.json then |
| 317 |
for k in pairs(known_symbols) do |
| 318 |
if not remove[k] then |
| 319 |
io.stdout:write(string.format('%s\n', k)) |
| 320 |
end |
| 321 |
end |
| 322 |
os.exit(0) |
| 323 |
end |
| 324 |
|
| 325 |
local json_output = { |
| 326 |
all_symbols = {}, |
| 327 |
removed_symbols = {}, |
| 328 |
used_symbols = {}, |
| 329 |
} |
| 330 |
for k in pairs(known_symbols) do |
| 331 |
table.insert(json_output.all_symbols, k) |
| 332 |
local why_removed = remove[k] |
| 333 |
if why_removed then |
| 334 |
json_output.removed_symbols[k] = why_removed |
| 335 |
else |
| 336 |
table.insert(json_output.used_symbols, k) |
| 337 |
end |
| 338 |
end |
| 339 |
io.stdout:write(ucl.to_format(json_output, 'json')) |
| 340 |
end |
| 341 |
|
| 342 |
local function post_neural_training(url, rule, spam_rows, ham_rows) |
| 343 |
-- Prepare JSON payload |
| 344 |
local payload = ucl.to_format( |
| 345 |
{ |
| 346 |
ham_vec = ham_rows, |
| 347 |
rule = rule, |
| 348 |
spam_vec = spam_rows, |
| 349 |
}, 'json') |
| 350 |
|
| 351 |
-- POST the payload |
| 352 |
local err, response = rspamd_http.request({ |
| 353 |
body = payload, |
| 354 |
config = rspamd_config, |
| 355 |
ev_base = rspamadm_ev_base, |
| 356 |
log_obj = rspamd_config, |
| 357 |
resolver = rspamadm_dns_resolver, |
| 358 |
session = rspamadm_session, |
| 359 |
url = url, |
| 360 |
}) |
| 361 |
|
| 362 |
if err then |
| 363 |
io.stderr:write(string.format('HTTP error: %s\n', err)) |
| 364 |
os.exit(1) |
| 365 |
end |
| 366 |
if response.code ~= 200 then |
| 367 |
io.stderr:write(string.format('bad HTTP code: %d\n', response.code)) |
| 368 |
os.exit(1) |
| 369 |
end |
| 370 |
io.stdout:write(string.format('%s\n', response.content)) |
| 371 |
end |
| 372 |
|
| 373 |
local function handle_neural_train(args) |
| 374 |
|
| 375 |
local this_where -- which class of messages are we collecting data for |
| 376 |
local ham_rows, spam_rows = {}, {} |
| 377 |
local want_spam, want_ham = true, true -- keep collecting while true |
| 378 |
|
| 379 |
-- Try find profile in config |
| 380 |
local neural_opts = rspamd_config:get_all_opt('neural') |
| 381 |
local symbols_profile = ((((neural_opts or E).rules or E)[args.rule] or E).profile or E)[args.profile] |
| 382 |
if not symbols_profile then |
| 383 |
io.stderr:write(string.format("Couldn't find profile %s in rule %s\n", args.profile, args.rule)) |
| 384 |
os.exit(1) |
| 385 |
end |
| 386 |
-- Try find max_trains |
| 387 |
local max_trains = (neural_opts.rules[args.rule].train or E).max_trains or 1000 |
| 388 |
|
| 389 |
-- Callback used to process rows from Clickhouse |
| 390 |
local function process_row(r) |
| 391 |
local destination -- which table to collect this information in |
| 392 |
if this_where == args.ham then |
| 393 |
destination = ham_rows |
| 394 |
if #destination >= max_trains then |
| 395 |
want_ham = false |
| 396 |
return |
| 397 |
end |
| 398 |
else |
| 399 |
destination = spam_rows |
| 400 |
if #destination >= max_trains then |
| 401 |
want_spam = false |
| 402 |
return |
| 403 |
end |
| 404 |
end |
| 405 |
local ucl_parser = ucl.parser() |
| 406 |
local ok, err = ucl_parser:parse_string(r[args.column_name_vector], 'msgpack') |
| 407 |
if not ok then |
| 408 |
io.stderr:write(string.format("Couldn't parse [%s]: %s", r[args.column_name_vector], err)) |
| 409 |
os.exit(1) |
| 410 |
end |
| 411 |
table.insert(destination, ucl_parser:get_object()) |
| 412 |
end |
| 413 |
|
| 414 |
-- Generate symbols digest |
| 415 |
table.sort(symbols_profile) |
| 416 |
local symbols_digest = lua_util.table_digest(symbols_profile) |
| 417 |
-- Create list of days to query data for |
| 418 |
local query_days = days_list(args.days) |
| 419 |
-- Set value for limit |
| 420 |
local limit = '' |
| 421 |
local num_limit = tonumber(args.limit) |
| 422 |
if num_limit then |
| 423 |
limit = string.format(' LIMIT %d', num_limit) -- Contains leading space |
| 424 |
end |
| 425 |
-- Prepare query elements |
| 426 |
local conditions = { string.format("%s = '%s'", args.column_name_digest, symbols_digest) } |
| 427 |
local query_fmt = 'SELECT %s FROM rspamd WHERE %s%s' |
| 428 |
|
| 429 |
-- Run queries |
| 430 |
for _, the_where in ipairs({ args.ham, args.spam }) do |
| 431 |
-- Inform callback which group of vectors we're collecting |
| 432 |
this_where = the_where |
| 433 |
table.insert(conditions, the_where) -- should be 2nd from last condition |
| 434 |
-- Loop over days and try collect data |
| 435 |
for _, query_day in ipairs(query_days) do |
| 436 |
-- Break the loop if we have enough data already |
| 437 |
if this_where == args.ham then |
| 438 |
if not want_ham then |
| 439 |
break |
| 440 |
end |
| 441 |
else |
| 442 |
if not want_spam then |
| 443 |
break |
| 444 |
end |
| 445 |
end |
| 446 |
-- Date should be the last condition |
| 447 |
table.insert(conditions, string.format("Date = '%s'", query_day)) |
| 448 |
local query = string.format(query_fmt, args.column_name_vector, table.concat(conditions, ' AND '), limit) |
| 449 |
local upstream = args.upstream:get_upstream_round_robin() |
| 450 |
local err = lua_clickhouse.select_sync(upstream, args, http_params, query, process_row) |
| 451 |
if err ~= nil then |
| 452 |
io.stderr:write(string.format('Error querying Clickhouse: %s\n', err)) |
| 453 |
os.exit(1) |
| 454 |
end |
| 455 |
conditions[#conditions] = nil -- remove Date condition |
| 456 |
end |
| 457 |
conditions[#conditions] = nil -- remove spam/ham condition |
| 458 |
end |
| 459 |
|
| 460 |
-- Make sure we collected enough data for training |
| 461 |
if #ham_rows < max_trains then |
| 462 |
io.stderr:write(string.format('Insufficient ham rows: %d/%d\n', #ham_rows, max_trains)) |
| 463 |
os.exit(1) |
| 464 |
end |
| 465 |
if #spam_rows < max_trains then |
| 466 |
io.stderr:write(string.format('Insufficient spam rows: %d/%d\n', #spam_rows, max_trains)) |
| 467 |
os.exit(1) |
| 468 |
end |
| 469 |
|
| 470 |
return post_neural_training(args.url, args.rule, spam_rows, ham_rows) |
| 471 |
end |
| 472 |
|
| 473 |
local command_handlers = { |
| 474 |
neural_profile = handle_neural_profile, |
| 475 |
neural_train = handle_neural_train, |
| 476 |
} |
| 477 |
|
| 478 |
local function handler(args) |
| 479 |
local cmd_opts = parser:parse(args) |
| 480 |
|
| 481 |
load_config(cmd_opts.config_file) |
| 482 |
local cfg_opts = rspamd_config:get_all_opt('clickhouse') |
| 483 |
|
| 484 |
if cmd_opts.ask_password then |
| 485 |
local rspamd_util = require "rspamd_util" |
| 486 |
|
| 487 |
io.write('Password: ') |
| 488 |
cmd_opts.password = rspamd_util.readpassphrase() |
| 489 |
end |
| 490 |
|
| 491 |
local function override_settings(params) |
| 492 |
for _, which in ipairs(params) do |
| 493 |
if cmd_opts[which] == nil then |
| 494 |
cmd_opts[which] = cfg_opts[which] |
| 495 |
end |
| 496 |
end |
| 497 |
end |
| 498 |
|
| 499 |
override_settings({ |
| 500 |
'database', 'no_ssl_verify', 'password', 'server', |
| 501 |
'use_gzip', 'use_https', 'user', |
| 502 |
}) |
| 503 |
|
| 504 |
local servers = cmd_opts['server'] or cmd_opts['servers'] |
| 505 |
if not servers then |
| 506 |
parser:error("server(s) unspecified & couldn't be fetched from config") |
| 507 |
end |
| 508 |
|
| 509 |
cmd_opts.upstream = rspamd_upstream_list.create(rspamd_config, servers, 8123) |
| 510 |
|
| 511 |
if not cmd_opts.upstream then |
| 512 |
io.stderr:write(string.format("can't parse clickhouse address: %s\n", servers)) |
| 513 |
os.exit(1) |
| 514 |
end |
| 515 |
|
| 516 |
local f = command_handlers[cmd_opts.command] |
| 517 |
if not f then |
| 518 |
parser:error(string.format("command isn't implemented: %s", |
| 519 |
cmd_opts.command)) |
| 520 |
end |
| 521 |
f(cmd_opts) |
| 522 |
end |
| 523 |
|
| 524 |
return { |
| 525 |
handler = handler, |
| 526 |
description = parser._description, |
| 527 |
name = 'clickhouse' |
| 528 |
} |