Debian Codegraph @

lualib/lua_meta.lua

raw
1 --[[
2 Copyright (c) 2022, Vsevolod Stakhov <[email protected]>
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 ]] --
16
17 local exports = {}
18
19 local N = "metatokens"
20 local T = require "lua_shape.core"
21 local logger = require "rspamd_logger"
22 local lua_mime = require "lua_mime"
23
24 -- Metafunctions
25 local function meta_size_function(task)
26 local sizes = {
27 100,
28 200,
29 500,
30 1000,
31 2000,
32 4000,
33 10000,
34 20000,
35 30000,
36 100000,
37 200000,
38 400000,
39 800000,
40 1000000,
41 2000000,
42 8000000,
43 }
44
45 local size = task:get_size()
46 for i = 1, #sizes do
47 if sizes[i] >= size then
48 return { (1.0 * i) / #sizes }
49 end
50 end
51
52 return { 0 }
53 end
54
55 local function meta_images_function(task)
56 local images = task:get_images()
57 local ntotal = 0
58 local njpg = 0
59 local npng = 0
60 local nlarge = 0
61 local nsmall = 0
62
63 if images then
64 for _, img in ipairs(images) do
65 if img:get_type() == 'png' then
66 npng = npng + 1
67 elseif img:get_type() == 'jpeg' then
68 njpg = njpg + 1
69 end
70
71 local w = img:get_width()
72 local h = img:get_height()
73
74 if w > 0 and h > 0 then
75 if w + h > 256 then
76 nlarge = nlarge + 1
77 else
78 nsmall = nsmall + 1
79 end
80 end
81
82 ntotal = ntotal + 1
83 end
84 end
85 if ntotal > 0 then
86 njpg = 1.0 * njpg / ntotal
87 npng = 1.0 * npng / ntotal
88 nlarge = 1.0 * nlarge / ntotal
89 nsmall = 1.0 * nsmall / ntotal
90 end
91 return { ntotal, npng, njpg, nlarge, nsmall } -- Fixed order to match names
92 end
93
94 local function meta_nparts_function(task)
95 local nattachments = 0
96 local ntextparts = 0
97 local totalparts = 1
98
99 local tp = task:get_text_parts()
100 if tp then
101 ntextparts = #tp
102 end
103
104 local parts = task:get_parts()
105
106 if parts then
107 for _, p in ipairs(parts) do
108 if p:is_attachment() then
109 nattachments = nattachments + 1
110 end
111 totalparts = totalparts + 1
112 end
113 end
114
115 return { (1.0 * ntextparts) / totalparts, (1.0 * nattachments) / totalparts }
116 end
117
118 local function meta_encoding_function(task)
119 local nutf = 0
120 local nother = 0
121
122 local tp = task:get_text_parts()
123 if tp and #tp > 0 then
124 for _, p in ipairs(tp) do
125 if p:is_utf() then
126 nutf = nutf + 1
127 else
128 nother = nother + 1
129 end
130 end
131
132 return { nutf / #tp, nother / #tp }
133 end
134
135 return { 0, 0 }
136 end
137
138 local function meta_recipients_function(task)
139 local nmime = 0
140 local nsmtp = 0
141
142 if task:has_recipients('mime') then
143 nmime = #(task:get_recipients('mime'))
144 end
145 if task:has_recipients('smtp') then
146 nsmtp = #(task:get_recipients('smtp'))
147 end
148
149 if nmime > 0 then
150 nmime = 1.0 / nmime
151 end
152 if nsmtp > 0 then
153 nsmtp = 1.0 / nsmtp
154 end
155
156 return { nmime, nsmtp }
157 end
158
159 local function meta_received_function(task)
160 local count_factor = 0
161 local invalid_factor = 0
162 local rh = task:get_received_headers()
163 local time_factor = 0
164 local secure_factor = 0
165 local fun = require "fun"
166
167 if rh and #rh > 0 then
168 local ntotal = 0.0
169 local init_time = 0
170
171 fun.each(function(rc)
172 ntotal = ntotal + 1.0
173
174 if not rc.by_hostname then
175 invalid_factor = invalid_factor + 1.0
176 end
177 if init_time == 0 and rc.timestamp then
178 init_time = rc.timestamp
179 elseif rc.timestamp then
180 time_factor = time_factor + math.abs(init_time - rc.timestamp)
181 init_time = rc.timestamp
182 end
183 if rc.flags and (rc.flags['ssl'] or rc.flags['authenticated']) then
184 secure_factor = secure_factor + 1.0
185 end
186 end,
187 fun.filter(function(rc)
188 return not rc.flags or not rc.flags['artificial']
189 end, rh))
190
191 if ntotal > 0 then
192 invalid_factor = invalid_factor / ntotal
193 secure_factor = secure_factor / ntotal
194 count_factor = 1.0 / ntotal
195 end
196
197 if time_factor ~= 0 then
198 time_factor = 1.0 / time_factor
199 end
200 end
201
202 return { count_factor, invalid_factor, time_factor, secure_factor }
203 end
204
205 local function meta_urls_function(task)
206 local has_urls, nurls = task:has_urls()
207 if has_urls and nurls > 0 then
208 return { 1.0 / nurls }
209 end
210
211 return { 0 }
212 end
213
214 local function meta_words_function(task)
215 local avg_len = task:get_mempool():get_variable("avg_words_len", "double") or 0.0
216 local short_words = task:get_mempool():get_variable("short_words_cnt", "double") or 0.0
217 local ret_len = 0
218
219 local lens = {
220 2,
221 3,
222 4,
223 5,
224 6,
225 7,
226 8,
227 9,
228 10,
229 15,
230 20,
231 }
232
233 for i = 1, #lens do
234 if lens[i] >= avg_len then
235 ret_len = (1.0 * i) / #lens
236 break
237 end
238 end
239
240 local tp = task:get_text_parts()
241 local wres = {
242 0, -- spaces rate
243 0, -- double spaces rate
244 0, -- non spaces rate
245 0, -- ascii characters rate
246 0, -- non-ascii characters rate
247 0, -- capital characters rate
248 0, -- numeric characters
249 }
250 for _, p in ipairs(tp) do
251 local stats = p:get_stats()
252 local len = p:get_length()
253
254 if len > 0 then
255 wres[1] = wres[1] + stats['spaces'] / len
256 wres[2] = wres[2] + stats['double_spaces'] / len
257 wres[3] = wres[3] + stats['non_spaces'] / len
258 wres[4] = wres[4] + stats['ascii_characters'] / len
259 wres[5] = wres[5] + stats['non_ascii_characters'] / len
260 wres[6] = wres[6] + stats['capital_letters'] / len
261 wres[7] = wres[7] + stats['numeric_characters'] / len
262 end
263 end
264
265 local ret = {
266 ret_len, -- avg_words_len (moved to match the names array)
267 short_words, -- nshort_words
268 }
269
270 local divisor = 1.0
271 if #tp > 0 then
272 divisor = #tp
273 end
274
275 for _, wr in ipairs(wres) do
276 table.insert(ret, wr / divisor)
277 end
278
279 return ret
280 end
281
282 local function meta_html_features_function(task)
283 local lt, http, ql, same, dom_total, max_per_dom = 0, 0, 0, 0, 0, 0
284 local ft, fua, fa = 0, 0, 0
285
286 local sel_part = lua_mime.get_displayed_text_part(task)
287 if sel_part and sel_part:is_html() then
288 local html = sel_part:get_html()
289 if html and html.get_features then
290 local f = html:get_features()
291 if f and f.links then
292 lt = f.links.total_links or 0
293 http = f.links.http_links or 0
294 ql = f.links.query_links or 0
295 same = f.links.same_etld1_links or 0
296 dom_total = f.links.domains_total or 0
297 max_per_dom = f.links.max_links_single_domain or 0
298 end
299 ft = f.forms_count or 0
300 fua = f.forms_post_unaffiliated or 0
301 fa = f.forms_post_affiliated or 0
302 end
303 end
304
305
306 local nhtml_links = 0
307 local http_ratio = 0
308 local query_ratio = 0
309 local same_etld1_ratio = 0
310 local domains_per_link_ratio = 0
311 local max_links_per_domain_ratio = 0
312
313 if lt > 0 then
314 nhtml_links = 1.0 / lt
315 http_ratio = http / lt
316 query_ratio = ql / lt
317 same_etld1_ratio = same / lt
318 domains_per_link_ratio = dom_total / lt
319 max_links_per_domain_ratio = max_per_dom / lt
320 end
321
322 local nhtml_forms = 0
323 local forms_unaff_ratio = 0
324 local forms_aff_ratio = 0
325
326 if ft > 0 then
327 nhtml_forms = 1.0 / ft
328 forms_unaff_ratio = fua / ft
329 forms_aff_ratio = fa / ft
330 end
331
332 return {
333 nhtml_links,
334 http_ratio,
335 query_ratio,
336 same_etld1_ratio,
337 domains_per_link_ratio,
338 max_links_per_domain_ratio,
339 nhtml_forms,
340 forms_unaff_ratio,
341 forms_aff_ratio,
342 }
343 end
344
345 local function meta_cta_function(task)
346 local mp = task:get_mempool()
347 local cta_aff = mp:get_variable("html_cta_affiliated", "double") or 0
348 local cta_w = mp:get_variable("html_cta_weight", "double") or 0
349 local aff_ratio = mp:get_variable("html_affiliated_links_ratio", "double") or 0
350 local tr_ratio = mp:get_variable("html_trackerish_ratio", "double") or 0
351
352 return {
353 cta_aff,
354 cta_w,
355 aff_ratio,
356 tr_ratio,
357 }
358 end
359
360 local function meta_html_visibility_function(task)
361 local sel_part = lua_mime.get_displayed_text_part(task)
362 local hidden_ratio, transparent_ratio = 0, 0
363 local blkh, blkt, off, mref, mrefu = 0, 0, 0, 0, 0
364
365 if sel_part and sel_part:is_html() then
366 local html = sel_part:get_html()
367 if html and html.get_features then
368 local f = html:get_features() or {}
369 local vis = f.text_visible or 0
370 local hid = f.text_hidden or 0
371 local transp = f.text_transparent or 0
372 local total = vis + hid
373 if total > 0 then
374 hidden_ratio = hid / total
375 transparent_ratio = transp / total
376 end
377 blkh = f.blocks_hidden or 0
378 blkt = f.blocks_transparent or 0
379 off = f.offscreen_blocks or 0
380 mref = f.meta_refresh or 0
381 mrefu = f.meta_refresh_urls or 0
382 end
383 end
384
385 -- no mempool fallback; individual mempool exports were removed
386
387 return {
388 hidden_ratio,
389 transparent_ratio,
390 blkh,
391 blkt,
392 off,
393 mref,
394 mrefu,
395 }
396 end
397
398 local metafunctions = {
399 {
400 cb = meta_size_function,
401 ninputs = 1,
402 names = {
403 "size"
404 },
405 description = 'Describes size of the message',
406 },
407 {
408 cb = meta_images_function,
409 ninputs = 5,
410 -- 1 - number of images,
411 -- 2 - number of png images,
412 -- 3 - number of jpeg images
413 -- 4 - number of large images (> 128 x 128)
414 -- 5 - number of small images (< 128 x 128)
415 names = {
416 'nimages',
417 'npng_images',
418 'njpeg_images',
419 'nlarge_images',
420 'nsmall_images'
421 },
422 description = [[Functions for images matching:
423 - number of images,
424 - number of png images,
425 - number of jpeg images
426 - number of large images (> 128 x 128)
427 - number of small images (< 128 x 128)
428 ]]
429 },
430 {
431 cb = meta_nparts_function,
432 ninputs = 2,
433 -- 1 - number of text parts
434 -- 2 - number of attachments
435 names = {
436 'ntext_parts',
437 'nattachments'
438 },
439 description = [[Functions for images matching:
440 - number of text parts
441 - number of attachments
442 ]]
443 },
444 {
445 cb = meta_encoding_function,
446 ninputs = 2,
447 -- 1 - number of utf parts
448 -- 2 - number of non-utf parts
449 names = {
450 'nutf_parts',
451 'nascii_parts'
452 },
453 description = [[Functions for encoding matching:
454 - number of utf parts
455 - number of non-utf parts
456 ]]
457 },
458 {
459 cb = meta_recipients_function,
460 ninputs = 2,
461 -- 1 - number of mime rcpt
462 -- 2 - number of smtp rcpt
463 names = {
464 'nmime_rcpt',
465 'nsmtp_rcpt'
466 },
467 description = [[Functions for recipients data matching:
468 - number of mime rcpt
469 - number of smtp rcpt
470 ]]
471 },
472 {
473 cb = meta_received_function,
474 ninputs = 4,
475 names = {
476 'nreceived',
477 'nreceived_invalid',
478 'nreceived_bad_time',
479 'nreceived_secure'
480 },
481 description = [[Functions for received headers data matching:
482 - number of received headers
483 - number of bad received headers
484 - number of skewed time received headers
485 - number of received via secured relays
486 ]]
487 },
488 {
489 cb = meta_urls_function,
490 ninputs = 1,
491 names = {
492 'nurls'
493 },
494 description = [[Functions for urls data matching:
495 - number of urls
496 ]]
497 },
498 {
499 cb = meta_words_function,
500 ninputs = 9,
501 names = {
502 'avg_words_len',
503 'nshort_words',
504 'spaces_rate',
505 'double_spaces_rate',
506 'non_spaces_rate',
507 'ascii_characters_rate',
508 'non_ascii_characters_rate',
509 'capital_characters_rate',
510 'numeric_characters'
511 },
512 description = [[Functions for words data matching:
513 - average length of the words
514 - number of short words
515 - rate of spaces in the text
516 - rate of multiple spaces
517 - rate of non space characters
518 - rate of ascii characters
519 - rate of non-ascii characters
520 - rate of capital letters
521 - rate of numbers
522 ]]
523 },
524 {
525 cb = meta_html_features_function,
526 ninputs = 9,
527 names = {
528 'nhtml_links',
529 'nhtml_http_links_ratio',
530 'nhtml_query_links_ratio',
531 'nhtml_same_etld1_links_ratio',
532 'nhtml_domains_per_link_ratio',
533 'nhtml_max_links_per_domain_ratio',
534 'nhtml_forms',
535 'nhtml_forms_unaffiliated_ratio',
536 'nhtml_forms_affiliated_ratio',
537 },
538 description = [[HTML link/form aggregated features:
539 - reciprocal of total links
540 - ratio of http(s) links
541 - ratio of links with query
542 - ratio of links with same eTLD+1 as first-party
543 - domains per link ratio
544 - max links per single domain ratio
545 - reciprocal of total forms
546 - ratio of forms posting to unaffiliated domains
547 - ratio of forms posting to affiliated domains
548 ]]
549 },
550 {
551 cb = meta_cta_function,
552 ninputs = 4,
553 names = {
554 'html_cta_affiliated',
555 'html_cta_weight',
556 'html_affiliated_links_ratio',
557 'html_trackerish_ratio',
558 },
559 description = [[CTA and affiliation metrics from lua_cta:
560 - CTA affiliated flag
561 - CTA weight heuristic
562 - affiliated links ratio among candidates
563 - trackerish domains ratio among candidates
564 ]]
565 },
566 {
567 cb = meta_html_visibility_function,
568 ninputs = 7,
569 names = {
570 'html_hidden_text_ratio',
571 'html_transparent_text_ratio',
572 'html_hidden_blocks',
573 'html_transparent_blocks',
574 'html_offscreen_blocks',
575 'html_meta_refresh',
576 'html_meta_refresh_urls',
577 },
578 description = [[HTML hidden/offscreen/obfuscation features and meta refresh counters:
579 - ratio of hidden text to total text
580 - ratio of transparent text to total text
581 - number of hidden text blocks appended
582 - number of transparent blocks
583 - number of offscreen-styled blocks
584 - number of meta refresh tags
585 - number of meta refresh URLs extracted
586 ]]
587 },
588 }
589
590 local meta_schema = T.table({
591 cb = T.callable(),
592 ninputs = T.number(),
593 names = T.array(T.string()),
594 description = T.string():optional()
595 })
596
597 local metatokens_by_name = {}
598
599 local function fill_metatokens_by_name()
600 metatokens_by_name = {}
601
602 for _, mt in ipairs(metafunctions) do
603 for i = 1, mt.ninputs do
604 local name = mt.names[i]
605
606 metatokens_by_name[name] = function(task)
607 local results = mt.cb(task)
608 return results[i]
609 end
610 end
611 end
612 end
613
614 local function calculate_digest()
615 local cr = require "rspamd_cryptobox_hash"
616
617 local h = cr.create()
618 for _, mt in ipairs(metafunctions) do
619 for i = 1, mt.ninputs do
620 local name = mt.names[i]
621 h:update(name)
622 end
623 end
624
625 exports.digest = h:hex()
626 end
627
628 local function rspamd_gen_metatokens(task, names)
629 local lua_util = require "lua_util"
630 local ipairs = ipairs
631 local metatokens = {}
632
633 if not names then
634 local cached = task:cache_get('metatokens')
635
636 if cached then
637 return cached
638 else
639 for _, mt in ipairs(metafunctions) do
640 local ct = mt.cb(task)
641 for i, tok in ipairs(ct) do
642 lua_util.debugm(N, task, "metatoken: %s = %s",
643 mt.names[i], tok)
644 if tok ~= tok or tok == math.huge then
645 logger.errx(task, 'metatoken %s returned %s; replace it with 0 for sanity',
646 mt.names[i], tok)
647 tok = 0.0
648 end
649 table.insert(metatokens, tok)
650 end
651 end
652
653 task:cache_set('metatokens', metatokens)
654 end
655 else
656 for _, n in ipairs(names) do
657 if metatokens_by_name[n] then
658 local tok = metatokens_by_name[n](task)
659 if tok ~= tok or tok == math.huge then
660 logger.errx(task, 'metatoken %s returned %s; replace it with 0 for sanity',
661 n, tok)
662 tok = 0.0
663 end
664 table.insert(metatokens, tok)
665 else
666 logger.errx(task, 'unknown metatoken: %s', n)
667 end
668 end
669 end
670
671 return metatokens
672 end
673
674 exports.rspamd_gen_metatokens = rspamd_gen_metatokens
675 exports.gen_metatokens = rspamd_gen_metatokens
676
677 local function rspamd_gen_metatokens_table(task)
678 local metatokens = {}
679
680 for _, mt in ipairs(metafunctions) do
681 local ct = mt.cb(task)
682 for i, tok in ipairs(ct) do
683 if tok ~= tok or tok == math.huge then
684 logger.errx(task, 'metatoken %s returned %s; replace it with 0 for sanity',
685 mt.names[i], tok)
686 tok = 0.0
687 end
688
689 metatokens[mt.names[i]] = tok
690 end
691 end
692
693 return metatokens
694 end
695
696 exports.rspamd_gen_metatokens_table = rspamd_gen_metatokens_table
697 exports.gen_metatokens_table = rspamd_gen_metatokens_table
698
699 local function rspamd_count_metatokens()
700 local ipairs = ipairs
701 local total = 0
702 for _, mt in ipairs(metafunctions) do
703 total = total + mt.ninputs
704 end
705
706 return total
707 end
708
709 exports.rspamd_count_metatokens = rspamd_count_metatokens
710 exports.count_metatokens = rspamd_count_metatokens
711 exports.version = 3 -- MUST be increased on each change of metatokens
712
713 exports.add_metafunction = function(tbl)
714 local ret, err = meta_schema(tbl)
715
716 if not ret then
717 logger.errx('cannot add metafunction: %s', err)
718 else
719 table.insert(metafunctions, tbl)
720 fill_metatokens_by_name()
721 calculate_digest()
722 end
723 end
724
725 fill_metatokens_by_name()
726 calculate_digest()
727
728 return exports