Debian Codegraph @

lualib/lua_stat.lua

raw
1 --[[
2 Copyright (c) 2022, Vsevolod Stakhov <[email protected]>
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 ]]--
16
17 --[[[
18 -- @module lua_stat
19 -- This module contains helper functions for supporting statistics
20 --]]
21
22 local logger = require "rspamd_logger"
23 local sqlite3 = require "rspamd_sqlite3"
24 local util = require "rspamd_util"
25 local lua_redis = require "lua_redis"
26 local lua_util = require "lua_util"
27 local exports = {}
28
29 local N = "stat_tools" -- luacheck: ignore (maybe unused)
30
31 -- Performs synchronous conversion of redis schema
32 local function convert_bayes_schema(redis_params, symbol_spam, symbol_ham, expire)
33
34 -- Old schema is the following one:
35 -- Keys are named <symbol>[<user>]
36 -- Elements are placed within hash:
37 -- BAYES_SPAM -> {<id1>: <num_hits>, <id2>: <num_hits> ...}
38 -- In new schema it is changed to a more extensible schema:
39 -- Keys are named RS[<user>]_<id> -> {'H': <ham_hits>, 'S': <spam_hits>}
40 -- So we can expire individual records, measure most popular elements by zranges,
41 -- add new fields, such as tokens etc
42
43 local res, conn = lua_redis.redis_connect_sync(redis_params, true)
44
45 if not res then
46 logger.errx("cannot connect to redis server")
47 return false
48 end
49
50 -- KEYS[1]: key to check (e.g. 'BAYES_SPAM')
51 -- KEYS[2]: hash key ('S' or 'H')
52 -- KEYS[3]: expire
53 local lua_script = [[
54 redis.replicate_commands()
55 local keys = redis.call('SMEMBERS', KEYS[1]..'_keys')
56 local nconverted = 0
57 for _,k in ipairs(keys) do
58 local cursor = redis.call('HSCAN', k, 0)
59 local neutral_prefix = string.gsub(k, KEYS[1], 'RS')
60 local elts
61 while cursor[1] ~= "0" do
62 elts = cursor[2]
63 cursor = redis.call('HSCAN', k, cursor[1])
64 local real_key
65 for i,v in ipairs(elts) do
66 if i % 2 ~= 0 then
67 real_key = v
68 else
69 local nkey = string.format('%s_%s', neutral_prefix, real_key)
70 redis.call('HSET', nkey, KEYS[2], v)
71 if KEYS[3] and tonumber(KEYS[3]) > 0 then
72 redis.call('EXPIRE', nkey, KEYS[3])
73 end
74 nconverted = nconverted + 1
75 end
76 end
77 end
78 end
79 return nconverted
80 ]]
81
82 conn:add_cmd('EVAL', { lua_script, '3', symbol_spam, 'S', tostring(expire) })
83 local ret
84 ret, res = conn:exec()
85
86 if not ret then
87 logger.errx('error converting symbol %s: %s', symbol_spam, res)
88 return false
89 else
90 logger.messagex('converted %s elements from symbol %s', res, symbol_spam)
91 end
92
93 conn:add_cmd('EVAL', { lua_script, '3', symbol_ham, 'H', tostring(expire) })
94 ret, res = conn:exec()
95
96 if not ret then
97 logger.errx('error converting symbol %s: %s', symbol_ham, res)
98 return false
99 else
100 logger.messagex('converted %s elements from symbol %s', res, symbol_ham)
101 end
102
103 -- We can now convert metadata: set + learned + version
104 -- KEYS[1]: key to check (e.g. 'BAYES_SPAM')
105 -- KEYS[2]: learn key (e.g. 'learns_spam' or 'learns_ham')
106 lua_script = [[
107 local keys = redis.call('SMEMBERS', KEYS[1]..'_keys')
108
109 for _,k in ipairs(keys) do
110 local learns = redis.call('HGET', k, 'learns') or 0
111 local neutral_prefix = string.gsub(k, KEYS[1], 'RS')
112
113 redis.call('HSET', neutral_prefix, KEYS[2], learns)
114 redis.call('SADD', KEYS[1]..'_keys', neutral_prefix)
115 redis.call('SREM', KEYS[1]..'_keys', k)
116 redis.call('DEL', KEYS[1])
117 redis.call('SET', k ..'_version', '2')
118 end
119 ]]
120
121 conn:add_cmd('EVAL', { lua_script, '2', symbol_spam, 'learns_spam' })
122 ret, res = conn:exec()
123
124 if not ret then
125 logger.errx('error converting metadata for symbol %s: %s', symbol_spam, res)
126 return false
127 end
128
129 conn:add_cmd('EVAL', { lua_script, '2', symbol_ham, 'learns_ham' })
130 ret, res = conn:exec()
131
132 if not ret then
133 logger.errx('error converting metadata for symbol %s', symbol_ham, res)
134 return false
135 end
136
137 return true
138 end
139
140 exports.convert_bayes_schema = convert_bayes_schema
141
142 -- It now accepts both ham and spam databases
143 -- parameters:
144 -- redis_params - how do we connect to a redis server
145 -- sqlite_db_spam - name for sqlite database with spam tokens
146 -- sqlite_db_ham - name for sqlite database with ham tokens
147 -- symbol_ham - name for symbol representing spam, e.g. BAYES_SPAM
148 -- symbol_spam - name for symbol representing ham, e.g. BAYES_HAM
149 -- learn_cache_spam - name for sqlite database with spam learn cache
150 -- learn_cache_ham - name for sqlite database with ham learn cache
151 -- reset_previous - if true, then the old database is flushed (slow)
152 local function convert_sqlite_to_redis(redis_params,
153 sqlite_db_spam, sqlite_db_ham, symbol_spam, symbol_ham,
154 learn_cache_db, expire, reset_previous)
155 local nusers = 0
156 local lim = 1000 -- Update each 1000 tokens
157 local users_map = {}
158 local converted = 0
159
160 local db_spam = sqlite3.open(sqlite_db_spam)
161 if not db_spam then
162 logger.errx('Cannot open source db: %s', sqlite_db_spam)
163 return false
164 end
165 local db_ham = sqlite3.open(sqlite_db_ham)
166 if not db_ham then
167 logger.errx('Cannot open source db: %s', sqlite_db_ham)
168 return false
169 end
170
171 local res, conn = lua_redis.redis_connect_sync(redis_params, true)
172
173 if not res then
174 logger.errx("cannot connect to redis server")
175 return false
176 end
177
178 if reset_previous then
179 -- Do a more complicated cleanup
180 -- execute a lua script that cleans up data
181 local script = [[
182 local members = redis.call('SMEMBERS', KEYS[1]..'_keys')
183
184 for _,prefix in ipairs(members) do
185 local keys = redis.call('KEYS', prefix..'*')
186 redis.call('DEL', keys)
187 end
188 ]]
189 -- Common keys
190 for _, sym in ipairs({ symbol_spam, symbol_ham }) do
191 logger.messagex('Cleaning up old data for %s', sym)
192 conn:add_cmd('EVAL', { script, '1', sym })
193 conn:exec()
194 conn:add_cmd('DEL', { sym .. "_version" })
195 conn:add_cmd('DEL', { sym .. "_keys" })
196 conn:exec()
197 end
198
199 if learn_cache_db then
200 -- Cleanup learned_cache
201 logger.messagex('Cleaning up old data learned cache')
202 conn:add_cmd('DEL', { "learned_ids" })
203 conn:exec()
204 end
205 end
206
207 local function convert_db(db, is_spam)
208 -- Map users and languages
209 local what = 'ham'
210 if is_spam then
211 what = 'spam'
212 end
213
214 local learns = {}
215 db:sql('BEGIN;')
216 -- Fill users mapping
217 for row in db:rows('SELECT * FROM users;') do
218 if row.id == '0' then
219 users_map[row.id] = ''
220 else
221 users_map[row.id] = row.name
222 end
223 learns[row.id] = row.learns
224 nusers = nusers + 1
225 end
226
227 -- Workaround for old databases
228 for row in db:rows('SELECT * FROM languages') do
229 if learns['0'] then
230 learns['0'] = learns['0'] + row.learns
231 else
232 learns['0'] = row.learns
233 end
234 end
235
236 local function send_batch(tokens, prefix)
237 -- We use the new schema: RS[user]_token -> H=ham count
238 -- S=spam count
239 local hash_key = 'H'
240 if is_spam then
241 hash_key = 'S'
242 end
243 for _, tok in ipairs(tokens) do
244 -- tok schema:
245 -- tok[1] = token_id (uint64 represented as a string)
246 -- tok[2] = token value (number)
247 -- tok[3] = user_map[user_id] or ''
248 local rkey = string.format('%s%s_%s', prefix, tok[3], tok[1])
249 conn:add_cmd('HINCRBYFLOAT', { rkey, hash_key, tostring(tok[2]) })
250
251 if expire and expire ~= 0 then
252 conn:add_cmd('EXPIRE', { rkey, tostring(math.floor(expire)) })
253 end
254 end
255
256 return conn:exec()
257 end
258 -- Fill tokens, sending data to redis each `lim` records
259
260 local ntokens = db:query('SELECT count(*) as c FROM tokens')['c']
261 local tokens = {}
262 local num = 0
263 local total = 0
264
265 for row in db:rows('SELECT token,value,user FROM tokens;') do
266 local user = ''
267 if row.user ~= 0 and users_map[row.user] then
268 user = users_map[row.user]
269 end
270
271 table.insert(tokens, { row.token, row.value, user })
272 num = num + 1
273 total = total + 1
274 if num > lim then
275 -- TODO: we use the default 'RS' prefix, it can be false in case of
276 -- classifiers with labels
277 local ret, err_str = send_batch(tokens, 'RS')
278 if not ret then
279 logger.errx('Cannot send tokens to the redis server: ' .. err_str)
280 db:sql('COMMIT;')
281 return false
282 end
283
284 num = 0
285 tokens = {}
286 end
287
288 io.write(string.format('Processed batch %s: %s/%s\r', what, total, ntokens))
289 end
290 -- Last batch
291 if #tokens > 0 then
292 local ret, err_str = send_batch(tokens, 'RS')
293 if not ret then
294 logger.errx('Cannot send tokens to the redis server: ' .. err_str)
295 db:sql('COMMIT;')
296 return false
297 end
298
299 io.write(string.format('Processed batch %s: %s/%s\r', what, total, ntokens))
300 end
301 io.write('\n')
302
303 converted = converted + total
304
305 -- Close DB
306 db:sql('COMMIT;')
307 local symbol = symbol_ham
308 local learns_elt = "learns_ham"
309
310 if is_spam then
311 symbol = symbol_spam
312 learns_elt = "learns_spam"
313 end
314
315 for id, learned in pairs(learns) do
316 local user = users_map[id]
317 if not conn:add_cmd('HSET', { 'RS' .. user, learns_elt, learned }) then
318 logger.errx('Cannot update learns for user: ' .. user)
319 return false
320 end
321 if not conn:add_cmd('SADD', { symbol .. '_keys', 'RS' .. user }) then
322 logger.errx('Cannot update learns for user: ' .. user)
323 return false
324 end
325 end
326 -- Set version
327 conn:add_cmd('SET', { symbol .. '_version', '2' })
328 return conn:exec()
329 end
330
331 logger.messagex('Convert spam tokens')
332 if not convert_db(db_spam, true) then
333 return false
334 end
335
336 logger.messagex('Convert ham tokens')
337 if not convert_db(db_ham, false) then
338 return false
339 end
340
341 if learn_cache_db then
342 logger.messagex('Convert learned ids from %s', learn_cache_db)
343 local db = sqlite3.open(learn_cache_db)
344 local ret = true
345 local total = 0
346
347 if not db then
348 logger.errx('Cannot open cache database: ' .. learn_cache_db)
349 return false
350 end
351
352 db:sql('BEGIN;')
353
354 for row in db:rows('SELECT * FROM learns;') do
355 local is_spam
356 local digest = tostring(util.encode_base32(row.digest))
357
358 if row.flag == '0' then
359 is_spam = '-1'
360 else
361 is_spam = '1'
362 end
363
364 if not conn:add_cmd('HSET', { 'learned_ids', digest, is_spam }) then
365 logger.errx('Cannot add hash: ' .. digest)
366 ret = false
367 else
368 total = total + 1
369 end
370 end
371 db:sql('COMMIT;')
372
373 if ret then
374 conn:exec()
375 end
376
377 if ret then
378 logger.messagex('Converted %s cached items from sqlite3 learned cache to redis',
379 total)
380 else
381 logger.errx('Error occurred during sending data to redis')
382 end
383 end
384
385 logger.messagex('Migrated %s tokens for %s users for symbols (%s, %s)',
386 converted, nusers, symbol_spam, symbol_ham)
387 return true
388 end
389
390 exports.convert_sqlite_to_redis = convert_sqlite_to_redis
391
392 -- Loads sqlite3 based classifiers and output data in form of array of objects:
393 -- [
394 -- {
395 -- symbol_spam = XXX
396 -- symbol_ham = YYY
397 -- db_spam = XXX.sqlite
398 -- db_ham = YYY.sqlite
399 -- learn_cache = ZZZ.sqlite
400 -- per_user = true/false
401 -- label = str
402 -- }
403 -- ]
404 local function load_sqlite_config(cfg)
405 local result = {}
406
407 local function parse_classifier(cls)
408 local tbl = {}
409 if cls.cache then
410 local cache = cls.cache
411 if cache.type == 'sqlite3' and (cache.file or cache.path) then
412 tbl.learn_cache = (cache.file or cache.path)
413 end
414 end
415
416 if cls.per_user then
417 tbl.per_user = cls.per_user
418 end
419
420 if cls.label then
421 tbl.label = cls.label
422 end
423
424 local statfiles = cls.statfile
425 for _, stf in ipairs(statfiles) do
426 local path = (stf.file or stf.path or stf.db or stf.dbname)
427 local symbol = stf.symbol or 'undefined'
428
429 if not path then
430 logger.errx('no path defined for statfile %s', symbol)
431 else
432
433 local spam
434 if stf.spam then
435 spam = stf.spam
436 else
437 if string.match(symbol:upper(), 'SPAM') then
438 spam = true
439 else
440 spam = false
441 end
442 end
443
444 if spam then
445 tbl.symbol_spam = symbol
446 tbl.db_spam = path
447 else
448 tbl.symbol_ham = symbol
449 tbl.db_ham = path
450 end
451 end
452 end
453
454 if tbl.symbol_spam and tbl.symbol_ham and tbl.db_ham and tbl.db_spam then
455 table.insert(result, tbl)
456 end
457 end
458
459 local classifier = cfg.classifier
460
461 if classifier then
462 if classifier[1] then
463 for _, cls in ipairs(classifier) do
464 if cls.bayes then
465 cls = cls.bayes
466 end
467 if cls.backend and cls.backend == 'sqlite3' then
468 parse_classifier(cls)
469 end
470 end
471 else
472 if classifier.bayes then
473 classifier = classifier.bayes
474 if classifier[1] then
475 for _, cls in ipairs(classifier) do
476 if cls.backend and cls.backend == 'sqlite3' then
477 parse_classifier(cls)
478 end
479 end
480 else
481 if classifier.backend and classifier.backend == 'sqlite3' then
482 parse_classifier(classifier)
483 end
484 end
485 end
486 end
487 end
488
489 return result
490 end
491
492 exports.load_sqlite_config = load_sqlite_config
493
494 -- A helper method that suggests a user how to configure Redis based
495 -- classifier based on the existing sqlite classifier
496 local function redis_classifier_from_sqlite(sqlite_classifier, expire)
497 local result = {
498 new_schema = true,
499 backend = 'redis',
500 cache = {
501 backend = 'redis'
502 },
503 statfile = {
504 [sqlite_classifier.symbol_spam] = {
505 spam = true
506 },
507 [sqlite_classifier.symbol_ham] = {
508 spam = false
509 }
510 }
511 }
512
513 if expire then
514 result.expire = expire
515 end
516
517 return { classifier = { bayes = result } }
518 end
519
520 exports.redis_classifier_from_sqlite = redis_classifier_from_sqlite
521
522 -- Reads statistics config and return preprocessed table
523 local function process_stat_config(cfg)
524 local opts_section = cfg:get_all_opt('options') or {}
525
526 -- Check if we have a dedicated section for statistics
527 if opts_section.statistics then
528 opts_section = opts_section.statistics
529 end
530
531 -- Default
532 local res_config = {
533 classify_headers = {
534 "User-Agent",
535 "X-Mailer",
536 "Content-Type",
537 "X-MimeOLE",
538 "Organization",
539 "Organisation"
540 },
541 classify_images = true,
542 classify_mime_info = true,
543 classify_urls = true,
544 classify_meta = true,
545 classify_max_tlds = 10,
546 }
547
548 res_config = lua_util.override_defaults(res_config, opts_section)
549
550 -- Postprocess classify_headers
551 local classify_headers_parsed = {}
552
553 for _, v in ipairs(res_config.classify_headers) do
554 local s1, s2 = v:match("^([A-Z])[^%-]+%-([A-Z]).*$")
555
556 local hname
557 if s1 and s2 then
558 hname = string.format('%s-%s', s1, s2)
559 else
560 s1 = v:match("^X%-([A-Z].*)$")
561
562 if s1 then
563 hname = string.format('x%s', s1:sub(1, 3):lower())
564 else
565 hname = string.format('%s', v:sub(1, 3):lower())
566 end
567 end
568
569 if classify_headers_parsed[hname] then
570 table.insert(classify_headers_parsed[hname], v)
571 else
572 classify_headers_parsed[hname] = { v }
573 end
574 end
575
576 res_config.classify_headers_parsed = classify_headers_parsed
577
578 return res_config
579 end
580
581 -- Split a filename into multiple stat tokens for better generalization.
582 -- Prefixes are passed by the caller to allow differentiation and backward compatibility:
583 -- orig_prefix -- applied to the full filename (case-preserved)
584 -- ext_prefix -- applied to the file extension (lowercased)
585 -- part_prefix -- applied to meaningful parts (lowercased)
586 -- Purely numeric parts and parts shorter than 2 characters are discarded.
587 local function filename_to_tokens(fname, orig_prefix, ext_prefix, part_prefix)
588 local tokens = {}
589
590 -- Original filename (case-preserved) as a whole token
591 table.insert(tokens, orig_prefix .. fname)
592
593 -- Extract extension: everything after the last dot (no spaces in extension),
594 -- requires at least one character before the dot
595 local ext = fname:match('^.+%.([^%.%s]+)$')
596 -- base: filename with the last extension stripped (byte arithmetic is correct
597 -- here because fname:sub uses byte positions and #ext is also in bytes)
598 local base = ext and fname:sub(1, #fname - #ext - 1) or fname
599
600 if ext then
601 local lext = util.lower_utf8(ext)
602 if util.strlen_utf8(lext) >= 2 then
603 table.insert(tokens, ext_prefix .. lext)
604 end
605 -- Double extension (e.g. "tar.gz", "pdf.exe"): strong signal for both
606 -- legitimate compound formats and malware disguise patterns
607 local prev_ext = base:match('^.+%.([^%.%s]+)$')
608 if prev_ext and util.strlen_utf8(prev_ext) >= 2 then
609 table.insert(tokens, ext_prefix .. util.lower_utf8(prev_ext .. '.' .. ext))
610 end
611 end
612
613 -- Split base by common filename delimiters (ASCII)
614 for part in base:gmatch('[^%-%_%s%.%#]+') do
615 -- Split each part further by the following boundaries (byte-level):
616 -- ASCII lowercase (97-122) -> ASCII uppercase (65-90): CamelCase, e.g. "InvoiceReport"
617 -- ASCII letter (65-90, 97-122) or UTF-8 continuation byte (128-191)
618 -- -> ASCII digit (48-57): letter/digit boundary, e.g. "temp36173", "апр26"
619 -- UTF-8 continuation bytes (128-191) are the last bytes of any multi-byte
620 -- Unicode character (Cyrillic, CJK, accented Latin, etc.), so this rule
621 -- correctly detects the boundary between any Unicode letter and an ASCII digit.
622 local sub_start = 1
623 for pos = 1, #part - 1 do
624 local cur_byte = part:byte(pos)
625 local next_byte = part:byte(pos + 1)
626 local is_split =
627 (cur_byte >= 97 and cur_byte <= 122 and next_byte >= 65 and next_byte <= 90) or
628 ((cur_byte >= 65 and cur_byte <= 90 or cur_byte >= 97 and cur_byte <= 122 or
629 cur_byte >= 128 and cur_byte <= 191) and
630 next_byte >= 48 and next_byte <= 57)
631 if is_split then
632 local sub = util.lower_utf8(part:sub(sub_start, pos))
633 if util.strlen_utf8(sub) >= 2 and not sub:match('^%d+$') then
634 table.insert(tokens, part_prefix .. sub)
635 end
636 sub_start = pos + 1
637 end
638 end
639 local sub = util.lower_utf8(part:sub(sub_start))
640 if util.strlen_utf8(sub) >= 2 and not sub:match('^%d+$') then
641 table.insert(tokens, part_prefix .. sub)
642 end
643 end
644
645 return tokens
646 end
647
648 local function get_mime_stat_tokens(task, res, i)
649 local parts = task:get_parts() or {}
650 local seen_multipart = false
651 local seen_plain = false
652 local seen_html = false
653 local empty_plain = false
654 local empty_html = false
655 local online_text = false
656
657 for _, part in ipairs(parts) do
658 local fname = part:get_filename()
659
660 local sz = part:get_length()
661
662 if sz > 0 then
663 rawset(res, i, string.format("#ps:%d",
664 math.floor(math.log(sz))))
665 lua_util.debugm("bayes", task, "part size: %s",
666 res[i])
667 i = i + 1
668 end
669
670 if fname then
671 local fname_tokens = filename_to_tokens(fname, "#f:", "#fe:", "#fp:")
672 for _, ft in ipairs(fname_tokens) do
673 rawset(res, i, ft)
674 i = i + 1
675 end
676 lua_util.debugm("bayes", task, "added attachment filename tokens: %s",
677 table.concat(fname_tokens, ", "))
678 end
679
680 if part:is_text() then
681 local tp = part:get_text()
682
683 if tp:is_html() then
684 seen_html = true
685
686 if tp:get_length() == 0 then
687 empty_html = true
688 end
689 else
690 seen_plain = true
691
692 if tp:get_length() == 0 then
693 empty_plain = true
694 end
695 end
696
697 if tp:get_lines_count() < 2 then
698 online_text = true
699 end
700
701 rawset(res, i, "#lang:" .. (tp:get_language() or 'unk'))
702 lua_util.debugm("bayes", task, "added language: %s",
703 res[i])
704 i = i + 1
705
706 rawset(res, i, "#cs:" .. (tp:get_charset() or 'unk'))
707 lua_util.debugm("bayes", task, "added charset: %s",
708 res[i])
709 i = i + 1
710
711 elseif part:is_multipart() then
712 seen_multipart = true;
713 end
714 end
715
716 -- Create a special token depending on parts structure
717 local st_tok = "#unk"
718 if seen_multipart and seen_html and seen_plain then
719 st_tok = '#mpth'
720 end
721
722 if seen_html and not seen_plain then
723 st_tok = "#ho"
724 end
725
726 if seen_plain and not seen_html then
727 st_tok = "#to"
728 end
729
730 local spec_tok = ""
731 if online_text then
732 spec_tok = "#ot"
733 end
734
735 if empty_plain then
736 spec_tok = spec_tok .. "#ep"
737 end
738
739 if empty_html then
740 spec_tok = spec_tok .. "#eh"
741 end
742
743 rawset(res, i, string.format("#m:%s%s", st_tok, spec_tok))
744 lua_util.debugm("bayes", task, "added mime token: %s",
745 res[i])
746 i = i + 1
747
748 return i
749 end
750
751 local function get_headers_stat_tokens(task, cf, res, i)
752 --[[
753 -- As discussed with Alexander Moisseev, this feature can skew statistics
754 -- especially when learning is separated from scanning, so learning
755 -- has a different set of tokens where this token can have too high weight
756 local hdrs_cksum = task:get_mempool():get_variable("headers_hash")
757
758 if hdrs_cksum then
759 rawset(res, i, string.format("#hh:%s", hdrs_cksum:sub(1, 7)))
760 lua_util.debugm("bayes", task, "added hdrs hash token: %s",
761 res[i])
762 i = i + 1
763 end
764 ]]--
765
766 for k, hdrs in pairs(cf.classify_headers_parsed) do
767 for _, hname in ipairs(hdrs) do
768 local value = task:get_header(hname)
769
770 if value then
771 rawset(res, i, string.format("#h:%s:%s", k, value))
772 lua_util.debugm("bayes", task, "added hdrs token: %s",
773 res[i])
774 i = i + 1
775 end
776 end
777 end
778
779 local from = (task:get_from('mime') or {})[1]
780
781 if from and from.name then
782 rawset(res, i, string.format("#F:%s", from.name))
783 lua_util.debugm("bayes", task, "added from name token: %s",
784 res[i])
785 i = i + 1
786 end
787
788 return i
789 end
790
791 local function get_meta_stat_tokens(task, res, i)
792 local day_and_hour = os.date('%u:%H',
793 task:get_date { format = 'message', gmt = true })
794 rawset(res, i, string.format("#dt:%s", day_and_hour))
795 lua_util.debugm("bayes", task, "added day_of_week token: %s",
796 res[i])
797 i = i + 1
798
799 local pol = {}
800
801 -- Authentication results
802 if task:has_symbol('DKIM_TRACE') then
803 -- Autolearn or scan
804 if task:has_symbol('R_SPF_ALLOW') then
805 table.insert(pol, 's=pass')
806 end
807
808 local trace = task:get_symbol('DKIM_TRACE')
809 local dkim_opts = trace[1]['options']
810 if dkim_opts then
811 for _, o in ipairs(dkim_opts) do
812 local check_res = string.sub(o, -1)
813 local domain = string.sub(o, 1, -3)
814
815 if check_res == '+' then
816 table.insert(pol, string.format('d=%s:%s', "pass", domain))
817 end
818 end
819 end
820 else
821 -- Offline learn
822 local aur = task:get_header('Authentication-Results')
823
824 if aur then
825 local spf = aur:match('spf=([a-z]+)')
826 local dkim, dkim_domain = aur:match('dkim=([a-z]+) header.d=([a-z.%-]+)')
827
828 if spf then
829 table.insert(pol, 's=' .. spf)
830 end
831 if dkim and dkim_domain then
832 table.insert(pol, string.format('d=%s:%s', dkim, dkim_domain))
833 end
834 end
835 end
836
837 if #pol > 0 then
838 rawset(res, i, string.format("#aur:%s", table.concat(pol, ',')))
839 lua_util.debugm("bayes", task, "added policies token: %s",
840 res[i])
841 i = i + 1
842 end
843
844 --[[
845 -- Disabled.
846 -- 1. Depending on the source the message has a different set of Received
847 -- headers as the receiving MTA adds another Received header.
848 -- 2. The usefulness of the Received tokens is questionable.
849 local rh = task:get_received_headers()
850
851 if rh and #rh > 0 then
852 local lim = math.min(5, #rh)
853 for j =1,lim do
854 local rcvd = rh[j]
855 local ip = rcvd.real_ip
856 if ip and ip:is_valid() and ip:get_version() == 4 then
857 local masked = ip:apply_mask(24)
858
859 rawset(res, i, string.format("#rcv:%s:%s", tostring(masked),
860 rcvd.proto))
861 lua_util.debugm("bayes", task, "added received token: %s",
862 res[i])
863 i = i + 1
864 end
865 end
866 end
867 ]]--
868
869 return i
870 end
871
872 local function get_stat_tokens(task, cf)
873 local res = {}
874 local E = {}
875 local i = 1
876
877 if cf.classify_images then
878 local images = task:get_images() or E
879
880 for _, img in ipairs(images) do
881 rawset(res, i, "image")
882 i = i + 1
883 rawset(res, i, tostring(img:get_height()))
884 i = i + 1
885 rawset(res, i, tostring(img:get_width()))
886 i = i + 1
887 rawset(res, i, tostring(img:get_type()))
888 i = i + 1
889
890 local fname = img:get_filename()
891
892 if fname then
893 rawset(res, i, fname)
894 i = i + 1
895 end
896
897 lua_util.debugm("bayes", task, "added image: %s",
898 fname)
899 end
900 end
901
902 if cf.classify_mime_info then
903 i = get_mime_stat_tokens(task, res, i)
904 end
905
906 if cf.classify_headers and #cf.classify_headers > 0 then
907 i = get_headers_stat_tokens(task, cf, res, i)
908 end
909
910 if cf.classify_urls then
911 local urls = lua_util.extract_specific_urls { task = task, limit = 5, esld_limit = 1 }
912
913 if urls then
914 for _, u in ipairs(urls) do
915 rawset(res, i, string.format("#u:%s", u:get_tld()))
916 lua_util.debugm("bayes", task, "added url token: %s",
917 res[i])
918 i = i + 1
919 end
920 end
921 end
922
923 if cf.classify_meta then
924 i = get_meta_stat_tokens(task, res, i)
925 end
926
927 return res
928 end
929
930 exports._filename_to_tokens = filename_to_tokens
931
932 exports.gen_stat_tokens = function(cfg)
933 local stat_config = process_stat_config(cfg)
934
935 return function(task)
936 return get_stat_tokens(task, stat_config)
937 end
938 end
939
940 return exports