| 1 |
--[[ |
| 2 |
Copyright (c) 2021, defkev <[email protected]> |
| 3 |
Copyright (c) 2018, Carsten Rosenberg <[email protected]> |
| 4 |
Copyright (c) 2022, Vsevolod Stakhov <[email protected]> |
| 5 |
|
| 6 |
Licensed under the Apache License, Version 2.0 (the "License"); |
| 7 |
you may not use this file except in compliance with the License. |
| 8 |
You may obtain a copy of the License at |
| 9 |
|
| 10 |
http://www.apache.org/licenses/LICENSE-2.0 |
| 11 |
|
| 12 |
Unless required by applicable law or agreed to in writing, software |
| 13 |
distributed under the License is distributed on an "AS IS" BASIS, |
| 14 |
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 15 |
See the License for the specific language governing permissions and |
| 16 |
limitations under the License. |
| 17 |
]]-- |
| 18 |
|
| 19 |
--[[[ |
| 20 |
-- @module pyzor |
| 21 |
-- This module contains pyzor access functions |
| 22 |
--]] |
| 23 |
|
| 24 |
local lua_util = require "lua_util" |
| 25 |
local tcp = require "rspamd_tcp" |
| 26 |
local upstream_list = require "rspamd_upstream_list" |
| 27 |
local rspamd_logger = require "rspamd_logger" |
| 28 |
local common = require "lua_scanners/common" |
| 29 |
|
| 30 |
local N = 'pyzor' |
| 31 |
local categories = { 'pyzor', 'bulk', 'hash', 'scanner' } |
| 32 |
|
| 33 |
local function pyzor_config(opts) |
| 34 |
|
| 35 |
local pyzor_conf = { |
| 36 |
text_part_min_words = 2, |
| 37 |
default_port = 5953, |
| 38 |
timeout = 15.0, |
| 39 |
log_clean = false, |
| 40 |
retransmits = 2, |
| 41 |
detection_category = "hash", |
| 42 |
cache_expire = 7200, -- expire redis in one hour |
| 43 |
message = '${SCANNER}: Pyzor bulk message found: "${VIRUS}"', |
| 44 |
default_score = 1.5, |
| 45 |
action = false, |
| 46 |
} |
| 47 |
|
| 48 |
pyzor_conf = lua_util.override_defaults(pyzor_conf, opts) |
| 49 |
|
| 50 |
if not pyzor_conf.prefix then |
| 51 |
pyzor_conf.prefix = 'rext_' .. N .. '_' |
| 52 |
end |
| 53 |
|
| 54 |
if not pyzor_conf.log_prefix then |
| 55 |
pyzor_conf.log_prefix = N .. ' (' .. pyzor_conf.detection_category .. ')' |
| 56 |
end |
| 57 |
|
| 58 |
if not pyzor_conf['servers'] then |
| 59 |
rspamd_logger.errx(rspamd_config, 'no servers defined') |
| 60 |
|
| 61 |
return nil |
| 62 |
end |
| 63 |
|
| 64 |
pyzor_conf['upstreams'] = upstream_list.create(rspamd_config, |
| 65 |
pyzor_conf['servers'], |
| 66 |
pyzor_conf.default_port) |
| 67 |
|
| 68 |
if pyzor_conf['upstreams'] then |
| 69 |
lua_util.add_debug_alias('external_services', N) |
| 70 |
return pyzor_conf |
| 71 |
end |
| 72 |
|
| 73 |
rspamd_logger.errx(rspamd_config, 'cannot parse servers %s', |
| 74 |
pyzor_conf['servers']) |
| 75 |
return nil |
| 76 |
end |
| 77 |
|
| 78 |
local function pyzor_check(task, content, digest, rule) |
| 79 |
local function pyzor_check_uncached () |
| 80 |
local upstream = rule.upstreams:get_upstream_round_robin() |
| 81 |
local addr = upstream:get_addr() |
| 82 |
local retransmits = rule.retransmits |
| 83 |
|
| 84 |
local function pyzor_callback(err, data, conn) |
| 85 |
|
| 86 |
if err then |
| 87 |
|
| 88 |
-- retry with another upstream until retransmits exceeds |
| 89 |
if retransmits > 0 then |
| 90 |
|
| 91 |
retransmits = retransmits - 1 |
| 92 |
|
| 93 |
-- Select a different upstream! |
| 94 |
upstream = rule.upstreams:get_upstream_round_robin() |
| 95 |
addr = upstream:get_addr() |
| 96 |
|
| 97 |
lua_util.debugm(N, task, '%s: retry IP: %s:%s err: %s', |
| 98 |
rule.log_prefix, addr, addr:get_port(), err) |
| 99 |
|
| 100 |
tcp.request({ |
| 101 |
task = task, |
| 102 |
host = addr:to_string(), |
| 103 |
port = addr:get_port(), |
| 104 |
upstream = upstream, |
| 105 |
timeout = rule['timeout'], |
| 106 |
shutdown = true, |
| 107 |
data = content, |
| 108 |
callback = pyzor_callback, |
| 109 |
}) |
| 110 |
else |
| 111 |
rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits exceed', |
| 112 |
rule['symbol'], rule['type']) |
| 113 |
task:insert_result(rule['symbol_fail'], 0.0, |
| 114 |
'failed to scan and retransmits exceed') |
| 115 |
end |
| 116 |
else |
| 117 |
-- pyzor output is unicode (\x09 -> tab, \0a -> newline) |
| 118 |
-- public.pyzor.org:24441 (200, 'OK') 21285091 206759 |
| 119 |
-- server:port Code Diag Count WL-Count |
| 120 |
local str_data = tostring(data) |
| 121 |
lua_util.debugm(N, task, '%s: returned data: %s', |
| 122 |
rule.log_prefix, str_data) |
| 123 |
-- If pyzor would return JSON this wouldn't be necessary |
| 124 |
local resp = {} |
| 125 |
for v in string.gmatch(str_data, '[^\t]+') do |
| 126 |
table.insert(resp, v) |
| 127 |
end |
| 128 |
-- rspamd_logger.infox(task, 'resp: %s', resp) |
| 129 |
if resp[2] ~= [[(200, 'OK')]] then |
| 130 |
rspamd_logger.errx(task, "error parsing response: %s", str_data) |
| 131 |
return |
| 132 |
end |
| 133 |
|
| 134 |
local whitelisted = tonumber(resp[4]) |
| 135 |
local reported = tonumber(resp[3]) |
| 136 |
|
| 137 |
--rspamd_logger.infox(task, "%s - count=%s wl=%s", addr:to_string(), reported, whitelisted) |
| 138 |
|
| 139 |
--[[ |
| 140 |
Weight is Count - WL-Count of rule.default_score in percent, e.g. |
| 141 |
SPAM: |
| 142 |
Count: 100 (100%) |
| 143 |
WL-Count: 1 (1%) |
| 144 |
rule.default_score: 1 |
| 145 |
Weight: 0.99 |
| 146 |
HAM: |
| 147 |
Count: 10 (100%) |
| 148 |
WL-Count: 10 (100%) |
| 149 |
rule.default_score: 1 |
| 150 |
Weight: 0 |
| 151 |
]] |
| 152 |
local weight = tonumber(string.format("%.2f", |
| 153 |
rule.default_score * (reported - whitelisted) / (reported + whitelisted))) |
| 154 |
local info = string.format("count=%d wl=%d", reported, whitelisted) |
| 155 |
local threat_string = string.format("bl_%d_wl_%d", |
| 156 |
reported, whitelisted) |
| 157 |
|
| 158 |
if weight > 0 then |
| 159 |
lua_util.debugm(N, task, '%s: returned result is spam - info: %s', |
| 160 |
rule.log_prefix, info) |
| 161 |
common.yield_result(task, rule, threat_string, weight) |
| 162 |
common.save_cache(task, digest, rule, threat_string, weight) |
| 163 |
else |
| 164 |
if rule.log_clean then |
| 165 |
rspamd_logger.infox(task, '%s: clean, returned result is ham - info: %s', |
| 166 |
rule.log_prefix, info) |
| 167 |
else |
| 168 |
lua_util.debugm(N, task, '%s: returned result is ham - info: %s', |
| 169 |
rule.log_prefix, info) |
| 170 |
end |
| 171 |
common.save_cache(task, digest, rule, 'OK', weight) |
| 172 |
end |
| 173 |
|
| 174 |
end |
| 175 |
end |
| 176 |
|
| 177 |
if digest == 'da39a3ee5e6b4b0d3255bfef95601890afd80709' then |
| 178 |
rspamd_logger.infox(task, '%s: not checking default digest', rule.log_prefix) |
| 179 |
return |
| 180 |
end |
| 181 |
|
| 182 |
tcp.request({ |
| 183 |
task = task, |
| 184 |
host = addr:to_string(), |
| 185 |
port = addr:get_port(), |
| 186 |
upstream = upstream, |
| 187 |
timeout = rule.timeout, |
| 188 |
shutdown = true, |
| 189 |
data = content, |
| 190 |
callback = pyzor_callback, |
| 191 |
}) |
| 192 |
end |
| 193 |
if common.condition_check_and_continue(task, content, rule, digest, pyzor_check_uncached) then |
| 194 |
return |
| 195 |
else |
| 196 |
pyzor_check_uncached() |
| 197 |
end |
| 198 |
end |
| 199 |
|
| 200 |
return { |
| 201 |
type = categories, |
| 202 |
description = 'pyzor bulk scanner', |
| 203 |
configure = pyzor_config, |
| 204 |
check = pyzor_check, |
| 205 |
name = N |
| 206 |
} |