Debian Codegraph @

lualib/rspamadm/clickhouse.lua

raw
1 --[[
2 Copyright (c) 2022, Vsevolod Stakhov <[email protected]>
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 ]]--
16
17 local argparse = require "argparse"
18 local lua_clickhouse = require "lua_clickhouse"
19 local lua_util = require "lua_util"
20 local rspamd_http = require "rspamd_http"
21 local rspamd_upstream_list = require "rspamd_upstream_list"
22 local rspamd_logger = require "rspamd_logger"
23 local ucl = require "ucl"
24
25 local E = {}
26
27 -- Define command line options
28 local parser = argparse()
29 :name 'rspamadm clickhouse'
30 :description 'Retrieve information from Clickhouse'
31 :help_description_margin(30)
32 :command_target('command')
33 :require_command(true)
34
35 parser:option '-c --config'
36 :description 'Path to config file'
37 :argname('config_file')
38 :default(rspamd_paths['CONFDIR'] .. '/rspamd.conf')
39 parser:option '-d --database'
40 :description 'Name of Clickhouse database to use'
41 :argname('database')
42 :default('default')
43 parser:flag '--no-ssl-verify'
44 :description 'Disable SSL verification'
45 :argname('no_ssl_verify')
46 parser:mutex(
47 parser:option '-p --password'
48 :description 'Password to use for Clickhouse'
49 :argname('password'),
50 parser:flag '-a --ask-password'
51 :description 'Ask password from the terminal'
52 :argname('ask_password')
53 )
54 parser:option '-s --server'
55 :description 'Address[:port] to connect to Clickhouse with'
56 :argname('server')
57 parser:option '-u --user'
58 :description 'Username to use for Clickhouse'
59 :argname('user')
60 parser:option '--use-gzip'
61 :description 'Use Gzip with Clickhouse'
62 :argname('use_gzip')
63 :default(true)
64 parser:flag '--use-https'
65 :description 'Use HTTPS with Clickhouse'
66 :argname('use_https')
67
68 local neural_profile = parser:command 'neural_profile'
69 :description 'Generate symbols profile using data from Clickhouse'
70 neural_profile:option '-w --where'
71 :description 'WHERE clause for Clickhouse query'
72 :argname('where')
73 neural_profile:flag '-j --json'
74 :description 'Write output as JSON'
75 :argname('json')
76 neural_profile:option '--days'
77 :description 'Number of days to collect stats for'
78 :argname('days')
79 :default('7')
80 neural_profile:option '--limit -l'
81 :description 'Maximum rows to fetch per day'
82 :argname('limit')
83 neural_profile:option '--settings-id'
84 :description 'Settings ID to query'
85 :argname('settings_id')
86 :default('')
87
88 local neural_train = parser:command 'neural_train'
89 :description 'Train neural using data from Clickhouse'
90 neural_train:option '--days'
91 :description 'Number of days to query data for'
92 :argname('days')
93 :default('7')
94 neural_train:option '--column-name-digest'
95 :description 'Name of neural profile digest column in Clickhouse'
96 :argname('column_name_digest')
97 :default('NeuralDigest')
98 neural_train:option '--column-name-vector'
99 :description 'Name of neural training vector column in Clickhouse'
100 :argname('column_name_vector')
101 :default('NeuralMpack')
102 neural_train:option '--limit -l'
103 :description 'Maximum rows to fetch per day'
104 :argname('limit')
105 neural_train:option '--profile -p'
106 :description 'Profile to use for training'
107 :argname('profile')
108 :default('default')
109 neural_train:option '--rule -r'
110 :description 'Rule to train'
111 :argname('rule')
112 :default('default')
113 neural_train:option '--spam -s'
114 :description 'WHERE clause to use for spam'
115 :argname('spam')
116 :default("Action == 'reject'")
117 neural_train:option '--ham -h'
118 :description 'WHERE clause to use for ham'
119 :argname('ham')
120 :default('Score < 0')
121 neural_train:option '--url -u'
122 :description 'URL to use for training'
123 :argname('url')
124 :default('http://127.0.0.1:11334/plugins/neural/learn')
125
126 local http_params = {
127 config = rspamd_config,
128 ev_base = rspamadm_ev_base,
129 session = rspamadm_session,
130 resolver = rspamadm_dns_resolver,
131 }
132
133 local function load_config(config_file)
134 local _r, err = rspamd_config:load_ucl(config_file)
135
136 if not _r then
137 rspamd_logger.errx('cannot load %s: %s', config_file, err)
138 os.exit(1)
139 end
140
141 _r, err = rspamd_config:parse_rcl({ 'logging', 'worker' })
142 if not _r then
143 rspamd_logger.errx('cannot process %s: %s', config_file, err)
144 os.exit(1)
145 end
146
147 if not rspamd_config:init_modules() then
148 rspamd_logger.errx('cannot init modules when parsing %s', config_file)
149 os.exit(1)
150 end
151
152 rspamd_config:init_subsystem('symcache')
153 end
154
155 local function days_list(days)
156 -- Create list of days to query starting with yesterday
157 local query_days = {}
158 local previous_date = os.time() - 86400
159 local num_days = tonumber(days)
160 for _ = 1, num_days do
161 table.insert(query_days, os.date('%Y-%m-%d', previous_date))
162 previous_date = previous_date - 86400
163 end
164 return query_days
165 end
166
167 local function get_excluded_symbols(known_symbols, correlations, seen_total)
168 -- Walk results once to collect all symbols & count occurrences
169
170 local remove = {}
171 local known_symbols_list = {}
172 local composites = rspamd_config:get_all_opt('composites')
173 local all_symbols = rspamd_config:get_symbols()
174 local skip_flags = {
175 nostat = true,
176 skip = true,
177 idempotent = true,
178 composite = true,
179 }
180 for k, v in pairs(known_symbols) do
181 local lower_count, higher_count
182 if v.seen_spam > v.seen_ham then
183 lower_count = v.seen_ham
184 higher_count = v.seen_spam
185 else
186 lower_count = v.seen_spam
187 higher_count = v.seen_ham
188 end
189
190 if composites[k] then
191 remove[k] = 'composite symbol'
192 elseif lower_count / higher_count >= 0.95 then
193 remove[k] = 'weak ham/spam correlation'
194 elseif v.seen / seen_total >= 0.9 then
195 remove[k] = 'omnipresent symbol'
196 elseif not all_symbols[k] then
197 remove[k] = 'nonexistent symbol'
198 else
199 for fl, _ in pairs(all_symbols[k].flags or {}) do
200 if skip_flags[fl] then
201 remove[k] = fl .. ' symbol'
202 break
203 end
204 end
205 end
206 known_symbols_list[v.id] = {
207 seen = v.seen,
208 name = k,
209 }
210 end
211
212 -- Walk correlation matrix and check total counts
213 for sym_id, row in pairs(correlations) do
214 for inner_sym_id, count in pairs(row) do
215 local known = known_symbols_list[sym_id]
216 local inner = known_symbols_list[inner_sym_id]
217 if known and count == known.seen and not remove[inner.name] and not remove[known.name] then
218 remove[known.name] = string.format("overlapped by %s",
219 known_symbols_list[inner_sym_id].name)
220 end
221 end
222 end
223
224 return remove
225 end
226
227 local function handle_neural_profile(args)
228
229 local known_symbols, correlations = {}, {}
230 local symbols_count, seen_total = 0, 0
231
232 local function process_row(r)
233 local is_spam = true
234 if r['Action'] == 'no action' or r['Action'] == 'greylist' then
235 is_spam = false
236 end
237 seen_total = seen_total + 1
238
239 local nsym = #r['Symbols.Names']
240
241 for i = 1, nsym do
242 local sym = r['Symbols.Names'][i]
243 local t = known_symbols[sym]
244 if not t then
245 local spam_count, ham_count = 0, 0
246 if is_spam then
247 spam_count = spam_count + 1
248 else
249 ham_count = ham_count + 1
250 end
251 known_symbols[sym] = {
252 id = symbols_count,
253 seen = 1,
254 seen_ham = ham_count,
255 seen_spam = spam_count,
256 }
257 symbols_count = symbols_count + 1
258 else
259 known_symbols[sym].seen = known_symbols[sym].seen + 1
260 if is_spam then
261 known_symbols[sym].seen_spam = known_symbols[sym].seen_spam + 1
262 else
263 known_symbols[sym].seen_ham = known_symbols[sym].seen_ham + 1
264 end
265 end
266 end
267
268 -- Fill correlations
269 for i = 1, nsym do
270 for j = 1, nsym do
271 if i ~= j then
272 local sym = r['Symbols.Names'][i]
273 local inner_sym_name = r['Symbols.Names'][j]
274 local known_sym = known_symbols[sym]
275 local inner_sym = known_symbols[inner_sym_name]
276 if known_sym and inner_sym then
277 if not correlations[known_sym.id] then
278 correlations[known_sym.id] = {}
279 end
280 local n = correlations[known_sym.id][inner_sym.id] or 0
281 n = n + 1
282 correlations[known_sym.id][inner_sym.id] = n
283 end
284 end
285 end
286 end
287 end
288
289 local query_days = days_list(args.days)
290 local conditions = {}
291 table.insert(conditions, string.format("SettingsId = '%s'", args.settings_id))
292 local limit = ''
293 local num_limit = tonumber(args.limit)
294 if num_limit then
295 limit = string.format(' LIMIT %d', num_limit) -- Contains leading space
296 end
297 if args.where then
298 table.insert(conditions, args.where)
299 end
300
301 local query_fmt = 'SELECT Action, Symbols.Names FROM rspamd WHERE %s%s'
302 for _, query_day in ipairs(query_days) do
303 -- Date should be the last condition
304 table.insert(conditions, string.format("Date = '%s'", query_day))
305 local query = string.format(query_fmt, table.concat(conditions, ' AND '), limit)
306 local upstream = args.upstream:get_upstream_round_robin()
307 local err = lua_clickhouse.select_sync(upstream, args, http_params, query, process_row)
308 if err ~= nil then
309 io.stderr:write(string.format('Error querying Clickhouse: %s\n', err))
310 os.exit(1)
311 end
312 conditions[#conditions] = nil -- remove Date condition
313 end
314
315 local remove = get_excluded_symbols(known_symbols, correlations, seen_total)
316 if not args.json then
317 for k in pairs(known_symbols) do
318 if not remove[k] then
319 io.stdout:write(string.format('%s\n', k))
320 end
321 end
322 os.exit(0)
323 end
324
325 local json_output = {
326 all_symbols = {},
327 removed_symbols = {},
328 used_symbols = {},
329 }
330 for k in pairs(known_symbols) do
331 table.insert(json_output.all_symbols, k)
332 local why_removed = remove[k]
333 if why_removed then
334 json_output.removed_symbols[k] = why_removed
335 else
336 table.insert(json_output.used_symbols, k)
337 end
338 end
339 io.stdout:write(ucl.to_format(json_output, 'json'))
340 end
341
342 local function post_neural_training(url, rule, spam_rows, ham_rows)
343 -- Prepare JSON payload
344 local payload = ucl.to_format(
345 {
346 ham_vec = ham_rows,
347 rule = rule,
348 spam_vec = spam_rows,
349 }, 'json')
350
351 -- POST the payload
352 local err, response = rspamd_http.request({
353 body = payload,
354 config = rspamd_config,
355 ev_base = rspamadm_ev_base,
356 log_obj = rspamd_config,
357 resolver = rspamadm_dns_resolver,
358 session = rspamadm_session,
359 url = url,
360 })
361
362 if err then
363 io.stderr:write(string.format('HTTP error: %s\n', err))
364 os.exit(1)
365 end
366 if response.code ~= 200 then
367 io.stderr:write(string.format('bad HTTP code: %d\n', response.code))
368 os.exit(1)
369 end
370 io.stdout:write(string.format('%s\n', response.content))
371 end
372
373 local function handle_neural_train(args)
374
375 local this_where -- which class of messages are we collecting data for
376 local ham_rows, spam_rows = {}, {}
377 local want_spam, want_ham = true, true -- keep collecting while true
378
379 -- Try find profile in config
380 local neural_opts = rspamd_config:get_all_opt('neural')
381 local symbols_profile = ((((neural_opts or E).rules or E)[args.rule] or E).profile or E)[args.profile]
382 if not symbols_profile then
383 io.stderr:write(string.format("Couldn't find profile %s in rule %s\n", args.profile, args.rule))
384 os.exit(1)
385 end
386 -- Try find max_trains
387 local max_trains = (neural_opts.rules[args.rule].train or E).max_trains or 1000
388
389 -- Callback used to process rows from Clickhouse
390 local function process_row(r)
391 local destination -- which table to collect this information in
392 if this_where == args.ham then
393 destination = ham_rows
394 if #destination >= max_trains then
395 want_ham = false
396 return
397 end
398 else
399 destination = spam_rows
400 if #destination >= max_trains then
401 want_spam = false
402 return
403 end
404 end
405 local ucl_parser = ucl.parser()
406 local ok, err = ucl_parser:parse_string(r[args.column_name_vector], 'msgpack')
407 if not ok then
408 io.stderr:write(string.format("Couldn't parse [%s]: %s", r[args.column_name_vector], err))
409 os.exit(1)
410 end
411 table.insert(destination, ucl_parser:get_object())
412 end
413
414 -- Generate symbols digest
415 table.sort(symbols_profile)
416 local symbols_digest = lua_util.table_digest(symbols_profile)
417 -- Create list of days to query data for
418 local query_days = days_list(args.days)
419 -- Set value for limit
420 local limit = ''
421 local num_limit = tonumber(args.limit)
422 if num_limit then
423 limit = string.format(' LIMIT %d', num_limit) -- Contains leading space
424 end
425 -- Prepare query elements
426 local conditions = { string.format("%s = '%s'", args.column_name_digest, symbols_digest) }
427 local query_fmt = 'SELECT %s FROM rspamd WHERE %s%s'
428
429 -- Run queries
430 for _, the_where in ipairs({ args.ham, args.spam }) do
431 -- Inform callback which group of vectors we're collecting
432 this_where = the_where
433 table.insert(conditions, the_where) -- should be 2nd from last condition
434 -- Loop over days and try collect data
435 for _, query_day in ipairs(query_days) do
436 -- Break the loop if we have enough data already
437 if this_where == args.ham then
438 if not want_ham then
439 break
440 end
441 else
442 if not want_spam then
443 break
444 end
445 end
446 -- Date should be the last condition
447 table.insert(conditions, string.format("Date = '%s'", query_day))
448 local query = string.format(query_fmt, args.column_name_vector, table.concat(conditions, ' AND '), limit)
449 local upstream = args.upstream:get_upstream_round_robin()
450 local err = lua_clickhouse.select_sync(upstream, args, http_params, query, process_row)
451 if err ~= nil then
452 io.stderr:write(string.format('Error querying Clickhouse: %s\n', err))
453 os.exit(1)
454 end
455 conditions[#conditions] = nil -- remove Date condition
456 end
457 conditions[#conditions] = nil -- remove spam/ham condition
458 end
459
460 -- Make sure we collected enough data for training
461 if #ham_rows < max_trains then
462 io.stderr:write(string.format('Insufficient ham rows: %d/%d\n', #ham_rows, max_trains))
463 os.exit(1)
464 end
465 if #spam_rows < max_trains then
466 io.stderr:write(string.format('Insufficient spam rows: %d/%d\n', #spam_rows, max_trains))
467 os.exit(1)
468 end
469
470 return post_neural_training(args.url, args.rule, spam_rows, ham_rows)
471 end
472
473 local command_handlers = {
474 neural_profile = handle_neural_profile,
475 neural_train = handle_neural_train,
476 }
477
478 local function handler(args)
479 local cmd_opts = parser:parse(args)
480
481 load_config(cmd_opts.config_file)
482 local cfg_opts = rspamd_config:get_all_opt('clickhouse')
483
484 if cmd_opts.ask_password then
485 local rspamd_util = require "rspamd_util"
486
487 io.write('Password: ')
488 cmd_opts.password = rspamd_util.readpassphrase()
489 end
490
491 local function override_settings(params)
492 for _, which in ipairs(params) do
493 if cmd_opts[which] == nil then
494 cmd_opts[which] = cfg_opts[which]
495 end
496 end
497 end
498
499 override_settings({
500 'database', 'no_ssl_verify', 'password', 'server',
501 'use_gzip', 'use_https', 'user',
502 })
503
504 local servers = cmd_opts['server'] or cmd_opts['servers']
505 if not servers then
506 parser:error("server(s) unspecified & couldn't be fetched from config")
507 end
508
509 cmd_opts.upstream = rspamd_upstream_list.create(rspamd_config, servers, 8123)
510
511 if not cmd_opts.upstream then
512 io.stderr:write(string.format("can't parse clickhouse address: %s\n", servers))
513 os.exit(1)
514 end
515
516 local f = command_handlers[cmd_opts.command]
517 if not f then
518 parser:error(string.format("command isn't implemented: %s",
519 cmd_opts.command))
520 end
521 f(cmd_opts)
522 end
523
524 return {
525 handler = handler,
526 description = parser._description,
527 name = 'clickhouse'
528 }