Debian Codegraph @

lualib/lua_scanners/pyzor.lua

raw
1 --[[
2 Copyright (c) 2021, defkev <[email protected]>
3 Copyright (c) 2018, Carsten Rosenberg <[email protected]>
4 Copyright (c) 2022, Vsevolod Stakhov <[email protected]>
5
6 Licensed under the Apache License, Version 2.0 (the "License");
7 you may not use this file except in compliance with the License.
8 You may obtain a copy of the License at
9
10 http://www.apache.org/licenses/LICENSE-2.0
11
12 Unless required by applicable law or agreed to in writing, software
13 distributed under the License is distributed on an "AS IS" BASIS,
14 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 See the License for the specific language governing permissions and
16 limitations under the License.
17 ]]--
18
19 --[[[
20 -- @module pyzor
21 -- This module contains pyzor access functions
22 --]]
23
24 local lua_util = require "lua_util"
25 local tcp = require "rspamd_tcp"
26 local upstream_list = require "rspamd_upstream_list"
27 local rspamd_logger = require "rspamd_logger"
28 local common = require "lua_scanners/common"
29
30 local N = 'pyzor'
31 local categories = { 'pyzor', 'bulk', 'hash', 'scanner' }
32
33 local function pyzor_config(opts)
34
35 local pyzor_conf = {
36 text_part_min_words = 2,
37 default_port = 5953,
38 timeout = 15.0,
39 log_clean = false,
40 retransmits = 2,
41 detection_category = "hash",
42 cache_expire = 7200, -- expire redis in one hour
43 message = '${SCANNER}: Pyzor bulk message found: "${VIRUS}"',
44 default_score = 1.5,
45 action = false,
46 }
47
48 pyzor_conf = lua_util.override_defaults(pyzor_conf, opts)
49
50 if not pyzor_conf.prefix then
51 pyzor_conf.prefix = 'rext_' .. N .. '_'
52 end
53
54 if not pyzor_conf.log_prefix then
55 pyzor_conf.log_prefix = N .. ' (' .. pyzor_conf.detection_category .. ')'
56 end
57
58 if not pyzor_conf['servers'] then
59 rspamd_logger.errx(rspamd_config, 'no servers defined')
60
61 return nil
62 end
63
64 pyzor_conf['upstreams'] = upstream_list.create(rspamd_config,
65 pyzor_conf['servers'],
66 pyzor_conf.default_port)
67
68 if pyzor_conf['upstreams'] then
69 lua_util.add_debug_alias('external_services', N)
70 return pyzor_conf
71 end
72
73 rspamd_logger.errx(rspamd_config, 'cannot parse servers %s',
74 pyzor_conf['servers'])
75 return nil
76 end
77
78 local function pyzor_check(task, content, digest, rule)
79 local function pyzor_check_uncached ()
80 local upstream = rule.upstreams:get_upstream_round_robin()
81 local addr = upstream:get_addr()
82 local retransmits = rule.retransmits
83
84 local function pyzor_callback(err, data, conn)
85
86 if err then
87
88 -- retry with another upstream until retransmits exceeds
89 if retransmits > 0 then
90
91 retransmits = retransmits - 1
92
93 -- Select a different upstream!
94 upstream = rule.upstreams:get_upstream_round_robin()
95 addr = upstream:get_addr()
96
97 lua_util.debugm(N, task, '%s: retry IP: %s:%s err: %s',
98 rule.log_prefix, addr, addr:get_port(), err)
99
100 tcp.request({
101 task = task,
102 host = addr:to_string(),
103 port = addr:get_port(),
104 upstream = upstream,
105 timeout = rule['timeout'],
106 shutdown = true,
107 data = content,
108 callback = pyzor_callback,
109 })
110 else
111 rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits exceed',
112 rule['symbol'], rule['type'])
113 task:insert_result(rule['symbol_fail'], 0.0,
114 'failed to scan and retransmits exceed')
115 end
116 else
117 -- pyzor output is unicode (\x09 -> tab, \0a -> newline)
118 -- public.pyzor.org:24441 (200, 'OK') 21285091 206759
119 -- server:port Code Diag Count WL-Count
120 local str_data = tostring(data)
121 lua_util.debugm(N, task, '%s: returned data: %s',
122 rule.log_prefix, str_data)
123 -- If pyzor would return JSON this wouldn't be necessary
124 local resp = {}
125 for v in string.gmatch(str_data, '[^\t]+') do
126 table.insert(resp, v)
127 end
128 -- rspamd_logger.infox(task, 'resp: %s', resp)
129 if resp[2] ~= [[(200, 'OK')]] then
130 rspamd_logger.errx(task, "error parsing response: %s", str_data)
131 return
132 end
133
134 local whitelisted = tonumber(resp[4])
135 local reported = tonumber(resp[3])
136
137 --rspamd_logger.infox(task, "%s - count=%s wl=%s", addr:to_string(), reported, whitelisted)
138
139 --[[
140 Weight is Count - WL-Count of rule.default_score in percent, e.g.
141 SPAM:
142 Count: 100 (100%)
143 WL-Count: 1 (1%)
144 rule.default_score: 1
145 Weight: 0.99
146 HAM:
147 Count: 10 (100%)
148 WL-Count: 10 (100%)
149 rule.default_score: 1
150 Weight: 0
151 ]]
152 local weight = tonumber(string.format("%.2f",
153 rule.default_score * (reported - whitelisted) / (reported + whitelisted)))
154 local info = string.format("count=%d wl=%d", reported, whitelisted)
155 local threat_string = string.format("bl_%d_wl_%d",
156 reported, whitelisted)
157
158 if weight > 0 then
159 lua_util.debugm(N, task, '%s: returned result is spam - info: %s',
160 rule.log_prefix, info)
161 common.yield_result(task, rule, threat_string, weight)
162 common.save_cache(task, digest, rule, threat_string, weight)
163 else
164 if rule.log_clean then
165 rspamd_logger.infox(task, '%s: clean, returned result is ham - info: %s',
166 rule.log_prefix, info)
167 else
168 lua_util.debugm(N, task, '%s: returned result is ham - info: %s',
169 rule.log_prefix, info)
170 end
171 common.save_cache(task, digest, rule, 'OK', weight)
172 end
173
174 end
175 end
176
177 if digest == 'da39a3ee5e6b4b0d3255bfef95601890afd80709' then
178 rspamd_logger.infox(task, '%s: not checking default digest', rule.log_prefix)
179 return
180 end
181
182 tcp.request({
183 task = task,
184 host = addr:to_string(),
185 port = addr:get_port(),
186 upstream = upstream,
187 timeout = rule.timeout,
188 shutdown = true,
189 data = content,
190 callback = pyzor_callback,
191 })
192 end
193 if common.condition_check_and_continue(task, content, rule, digest, pyzor_check_uncached) then
194 return
195 else
196 pyzor_check_uncached()
197 end
198 end
199
200 return {
201 type = categories,
202 description = 'pyzor bulk scanner',
203 configure = pyzor_config,
204 check = pyzor_check,
205 name = N
206 }