| 1 |
--[[ |
| 2 |
Copyright (c) 2022, Vsevolod Stakhov <[email protected]> |
| 3 |
|
| 4 |
Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 |
you may not use this file except in compliance with the License. |
| 6 |
You may obtain a copy of the License at |
| 7 |
|
| 8 |
http://www.apache.org/licenses/LICENSE-2.0 |
| 9 |
|
| 10 |
Unless required by applicable law or agreed to in writing, software |
| 11 |
distributed under the License is distributed on an "AS IS" BASIS, |
| 12 |
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 |
See the License for the specific language governing permissions and |
| 14 |
limitations under the License. |
| 15 |
]] -- |
| 16 |
|
| 17 |
--[[[ |
| 18 |
-- @module virustotal |
| 19 |
-- This module contains Virustotal integration support |
| 20 |
-- https://www.virustotal.com/ |
| 21 |
--]] |
| 22 |
|
| 23 |
local lua_util = require "lua_util" |
| 24 |
local http = require "rspamd_http" |
| 25 |
local rspamd_cryptobox_hash = require "rspamd_cryptobox_hash" |
| 26 |
local rspamd_logger = require "rspamd_logger" |
| 27 |
local common = require "lua_scanners/common" |
| 28 |
|
| 29 |
local N = 'virustotal' |
| 30 |
|
| 31 |
local function virustotal_config(opts) |
| 32 |
local default_conf = { |
| 33 |
name = N, |
| 34 |
url = 'https://www.virustotal.com/vtapi/v2/file', |
| 35 |
timeout = 5.0, |
| 36 |
log_clean = false, |
| 37 |
retransmits = 1, |
| 38 |
cache_expire = 7200, -- expire redis in 2h |
| 39 |
message = '${SCANNER}: spam message found: "${VIRUS}"', |
| 40 |
detection_category = "virus", |
| 41 |
default_score = 1, |
| 42 |
action = false, |
| 43 |
scan_mime_parts = true, |
| 44 |
scan_text_mime = false, |
| 45 |
scan_image_mime = false, |
| 46 |
apikey = nil, -- Required to set by user |
| 47 |
-- Specific for virustotal |
| 48 |
minimum_engines = 3, -- Minimum required to get scored |
| 49 |
-- Threshold-based categorization |
| 50 |
low_category = 5, -- Low threat: minimum_engines to low_category-1 |
| 51 |
medium_category = 10, -- Medium threat: low_category to medium_category-1 |
| 52 |
-- High threat: medium_category and above |
| 53 |
-- Symbol categories |
| 54 |
symbols = { |
| 55 |
clean = { |
| 56 |
symbol = 'VIRUSTOTAL_CLEAN', |
| 57 |
score = -0.5, |
| 58 |
description = 'VirusTotal decided attachment to be clean' |
| 59 |
}, |
| 60 |
low = { |
| 61 |
symbol = 'VIRUSTOTAL_LOW', |
| 62 |
score = 2.0, |
| 63 |
description = 'VirusTotal found low number of threats' |
| 64 |
}, |
| 65 |
medium = { |
| 66 |
symbol = 'VIRUSTOTAL_MEDIUM', |
| 67 |
score = 5.0, |
| 68 |
description = 'VirusTotal found medium number of threats' |
| 69 |
}, |
| 70 |
high = { |
| 71 |
symbol = 'VIRUSTOTAL_HIGH', |
| 72 |
score = 8.0, |
| 73 |
description = 'VirusTotal found high number of threats' |
| 74 |
}, |
| 75 |
}, |
| 76 |
} |
| 77 |
|
| 78 |
default_conf = lua_util.override_defaults(default_conf, opts) |
| 79 |
|
| 80 |
if not default_conf.prefix then |
| 81 |
default_conf.prefix = 'rs_' .. default_conf.name .. '_' |
| 82 |
end |
| 83 |
|
| 84 |
if not default_conf.log_prefix then |
| 85 |
if default_conf.name:lower() == default_conf.type:lower() then |
| 86 |
default_conf.log_prefix = default_conf.name |
| 87 |
else |
| 88 |
default_conf.log_prefix = default_conf.name .. ' (' .. default_conf.type .. ')' |
| 89 |
end |
| 90 |
end |
| 91 |
|
| 92 |
if not default_conf.apikey then |
| 93 |
rspamd_logger.errx(rspamd_config, 'no apikey defined for virustotal, disable checks') |
| 94 |
|
| 95 |
return nil |
| 96 |
end |
| 97 |
|
| 98 |
lua_util.add_debug_alias('external_services', default_conf.name) |
| 99 |
return default_conf |
| 100 |
end |
| 101 |
|
| 102 |
local function virustotal_check(task, content, digest, rule, maybe_part) |
| 103 |
local function virustotal_check_uncached() |
| 104 |
local function make_url(hash) |
| 105 |
return string.format('%s/report?apikey=%s&resource=%s', |
| 106 |
rule.url, rule.apikey, hash) |
| 107 |
end |
| 108 |
|
| 109 |
local hash = rspamd_cryptobox_hash.create_specific('md5') |
| 110 |
hash:update(content) |
| 111 |
hash = hash:hex() |
| 112 |
|
| 113 |
local url = make_url(hash) |
| 114 |
lua_util.debugm(N, task, "send request %s", url) |
| 115 |
local request_data = { |
| 116 |
task = task, |
| 117 |
url = url, |
| 118 |
timeout = rule.timeout, |
| 119 |
} |
| 120 |
|
| 121 |
local function vt_http_callback(http_err, code, body, headers) |
| 122 |
if http_err then |
| 123 |
rspamd_logger.errx(task, 'HTTP error: %s, body: %s, headers: %s', http_err, body, headers) |
| 124 |
else |
| 125 |
local cached |
| 126 |
-- Parse the response |
| 127 |
if code ~= 200 then |
| 128 |
if code == 404 then |
| 129 |
cached = 'OK' |
| 130 |
if rule['log_clean'] then |
| 131 |
rspamd_logger.infox(task, '%s: hash %s clean (not found)', |
| 132 |
rule.log_prefix, hash) |
| 133 |
else |
| 134 |
lua_util.debugm(rule.name, task, '%s: hash %s clean (not found)', |
| 135 |
rule.log_prefix, hash) |
| 136 |
end |
| 137 |
elseif code == 204 then |
| 138 |
-- Request rate limit exceeded |
| 139 |
rspamd_logger.infox(task, 'virustotal request rate limit exceeded') |
| 140 |
task:insert_result(rule.symbol_fail, 1.0, 'rate limit exceeded') |
| 141 |
return |
| 142 |
else |
| 143 |
rspamd_logger.errx(task, 'invalid HTTP code: %s, body: %s, headers: %s', code, body, headers) |
| 144 |
task:insert_result(rule.symbol_fail, 1.0, 'Bad HTTP code: ' .. code) |
| 145 |
return |
| 146 |
end |
| 147 |
else |
| 148 |
local ucl = require "ucl" |
| 149 |
local parser = ucl.parser() |
| 150 |
local res, json_err = parser:parse_string(body) |
| 151 |
|
| 152 |
lua_util.debugm(rule.name, task, '%s: got reply data: "%s"', |
| 153 |
rule.log_prefix, body) |
| 154 |
|
| 155 |
if res then |
| 156 |
local obj = parser:get_object() |
| 157 |
if not obj.positives or type(obj.positives) ~= 'number' then |
| 158 |
if obj.response_code then |
| 159 |
if obj.response_code == 0 then |
| 160 |
if rule['log_clean'] then |
| 161 |
rspamd_logger.infox(task, '%s: hash %s clean (not found)', |
| 162 |
rule.log_prefix, hash) |
| 163 |
else |
| 164 |
lua_util.debugm(rule.name, task, '%s: hash %s clean (not found)', |
| 165 |
rule.log_prefix, hash) |
| 166 |
end |
| 167 |
-- Insert CLEAN symbol |
| 168 |
if rule.symbols and rule.symbols.clean then |
| 169 |
local clean_sym = rule.symbols.clean.symbol or 'VIRUSTOTAL_CLEAN' |
| 170 |
local sopt = string.format("%s:0", hash) |
| 171 |
task:insert_result(clean_sym, 1.0, sopt) |
| 172 |
-- Save with symbol name for proper cache retrieval |
| 173 |
cached = string.format("%s\v%s", clean_sym, sopt) |
| 174 |
else |
| 175 |
cached = 'OK' |
| 176 |
end |
| 177 |
else |
| 178 |
rspamd_logger.errx(task, 'invalid JSON reply: %s, body: %s, headers: %s', |
| 179 |
'bad response code: ' .. tostring(obj.response_code), body, headers) |
| 180 |
task:insert_result(rule.symbol_fail, 1.0, 'Bad JSON reply: no `positives` element') |
| 181 |
return |
| 182 |
end |
| 183 |
else |
| 184 |
rspamd_logger.errx(task, 'invalid JSON reply: %s, body: %s, headers: %s', |
| 185 |
'no response_code', body, headers) |
| 186 |
task:insert_result(rule.symbol_fail, 1.0, 'Bad JSON reply: no `positives` element') |
| 187 |
return |
| 188 |
end |
| 189 |
else |
| 190 |
if obj.positives == 0 then |
| 191 |
if rule['log_clean'] then |
| 192 |
rspamd_logger.infox(task, '%s: hash %s clean', |
| 193 |
rule.log_prefix, hash) |
| 194 |
else |
| 195 |
lua_util.debugm(rule.name, task, '%s: hash %s clean', |
| 196 |
rule.log_prefix, hash) |
| 197 |
end |
| 198 |
-- Insert CLEAN symbol |
| 199 |
if rule.symbols and rule.symbols.clean then |
| 200 |
local clean_sym = rule.symbols.clean.symbol or 'VIRUSTOTAL_CLEAN' |
| 201 |
local sopt = string.format("%s:0/%s", hash, obj.total or 0) |
| 202 |
task:insert_result(clean_sym, 1.0, sopt) |
| 203 |
-- Save with symbol name for proper cache retrieval |
| 204 |
cached = string.format("%s\v%s", clean_sym, sopt) |
| 205 |
else |
| 206 |
cached = 'OK' |
| 207 |
end |
| 208 |
elseif obj.positives < rule.minimum_engines then |
| 209 |
lua_util.debugm(rule.name, task, '%s: hash %s has not enough hits: %s where %s is min', |
| 210 |
rule.log_prefix, hash, obj.positives, rule.minimum_engines) |
| 211 |
cached = 'OK' |
| 212 |
else |
| 213 |
-- Determine category based on detection count |
| 214 |
local category |
| 215 |
local category_sym |
| 216 |
local sopt = string.format("%s:%s/%s", hash, obj.positives, obj.total) |
| 217 |
|
| 218 |
if obj.positives >= rule.medium_category then |
| 219 |
category = 'high' |
| 220 |
category_sym = rule.symbols.high.symbol or 'VIRUSTOTAL_HIGH' |
| 221 |
elseif obj.positives >= rule.low_category then |
| 222 |
category = 'medium' |
| 223 |
category_sym = rule.symbols.medium.symbol or 'VIRUSTOTAL_MEDIUM' |
| 224 |
else |
| 225 |
category = 'low' |
| 226 |
category_sym = rule.symbols.low.symbol or 'VIRUSTOTAL_LOW' |
| 227 |
end |
| 228 |
|
| 229 |
rspamd_logger.infox(task, '%s: result - %s: "%s" - category: %s', |
| 230 |
rule.log_prefix, rule.detection_category .. 'found', sopt, category) |
| 231 |
|
| 232 |
task:insert_result(category_sym, 1.0, sopt) |
| 233 |
-- Save with symbol name for proper cache retrieval |
| 234 |
cached = string.format("%s\v%s", category_sym, sopt) |
| 235 |
end |
| 236 |
end |
| 237 |
else |
| 238 |
-- not res |
| 239 |
rspamd_logger.errx(task, 'invalid JSON reply: %s, body: %s, headers: %s', |
| 240 |
json_err, body, headers) |
| 241 |
task:insert_result(rule.symbol_fail, 1.0, 'Bad JSON reply: ' .. json_err) |
| 242 |
return |
| 243 |
end |
| 244 |
end |
| 245 |
|
| 246 |
if cached then |
| 247 |
common.save_cache(task, digest, rule, cached, 1.0, maybe_part) |
| 248 |
end |
| 249 |
end |
| 250 |
end |
| 251 |
|
| 252 |
request_data.callback = vt_http_callback |
| 253 |
http.request(request_data) |
| 254 |
end |
| 255 |
|
| 256 |
if common.condition_check_and_continue(task, content, rule, digest, |
| 257 |
virustotal_check_uncached) then |
| 258 |
return |
| 259 |
else |
| 260 |
virustotal_check_uncached() |
| 261 |
end |
| 262 |
end |
| 263 |
|
| 264 |
return { |
| 265 |
type = 'antivirus', |
| 266 |
description = 'Virustotal integration', |
| 267 |
configure = virustotal_config, |
| 268 |
check = virustotal_check, |
| 269 |
name = N |
| 270 |
} |