Debian Codegraph @

lualib/lua_scanners/cloudmark.lua

raw
1 --[[
2 Copyright (c) 2021, Alexander Moisseev <[email protected]>
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 ]]--
16
17 --[[[
18 -- @module cloudmark
19 -- This module contains Cloudmark v2 interface
20 --]]
21
22 local lua_util = require "lua_util"
23 local http = require "rspamd_http"
24 local upstream_list = require "rspamd_upstream_list"
25 local rspamd_logger = require "rspamd_logger"
26 local ucl = require "ucl"
27 local rspamd_util = require "rspamd_util"
28 local common = require "lua_scanners/common"
29 local fun = require "fun"
30 local lua_mime = require "lua_mime"
31
32 local N = 'cloudmark'
33 -- Boundary for multipart transfers, generated on module init
34 local static_boundary = rspamd_util.random_hex(32)
35
36 local function cloudmark_url(rule, addr, maybe_url)
37 local url
38 local port = addr:get_port()
39
40 maybe_url = maybe_url or rule.url
41 if port == 0 then
42 port = rule.default_port
43 end
44 if rule.use_https then
45 url = string.format('https://%s:%d%s', tostring(addr),
46 port, maybe_url)
47 else
48 url = string.format('http://%s:%d%s', tostring(addr),
49 port, maybe_url)
50 end
51
52 return url
53 end
54
55 -- Detect cloudmark max size
56 local function cloudmark_preload(rule, cfg, ev_base, _)
57 local upstream = rule.upstreams:get_upstream_round_robin()
58 local addr = upstream:get_addr()
59 local function max_message_size_cb(http_err, code, body, _)
60 if http_err then
61 rspamd_logger.errx(ev_base, 'HTTP error when getting max message size: %s',
62 http_err)
63 return
64 end
65 if code ~= 200 then
66 rspamd_logger.errx(ev_base, 'bad HTTP code when getting max message size: %s', code)
67 end
68 local parser = ucl.parser()
69 local ret, err = parser:parse_string(body)
70 if not ret then
71 rspamd_logger.errx(ev_base, 'could not parse response body [%s]: %s', body, err)
72 return
73 end
74 local obj = parser:get_object()
75 local ms = obj.maxMessageSize
76 if not ms then
77 rspamd_logger.errx(ev_base, 'missing maxMessageSize in the response body (JSON): %s', obj)
78 return
79 end
80
81 rule.max_size = ms
82 lua_util.debugm(N, cfg, 'set maximum message size set to %s bytes', ms)
83 end
84 http.request({
85 ev_base = ev_base,
86 config = cfg,
87 url = cloudmark_url(rule, addr, '/score/v2/max-message-size'),
88 callback = max_message_size_cb,
89 })
90 end
91
92 local function numerify(d)
93 local l = {}
94 for k in pairs(d) do
95 table.insert(l, k)
96 end
97 for _, k in ipairs(l) do
98 local new_key = tonumber(k)
99 if new_key then
100 d[new_key] = d[k]
101 d[k] = nil
102 end
103 end
104 end
105
106 local function cloudmark_config(opts)
107
108 local cloudmark_conf = {
109 name = N,
110 default_port = 2713,
111 url = '/score/v2/message',
112 use_https = false,
113 timeout = 5.0,
114 log_clean = false,
115 retransmits = 1,
116 score_threshold = 90, -- minimum score to considerate reply
117 message = '${SCANNER}: spam message found: "${VIRUS}"',
118 max_message = 0,
119 detection_category = "hash",
120 default_score = 1,
121 action = false,
122 log_spamcause = true,
123 symbol_fail = 'CLOUDMARK_FAIL',
124 symbol = 'CLOUDMARK_CHECK',
125 symbol_spam = 'CLOUDMARK_SPAM',
126 add_score_header = false, -- Add X-CMAE-Score header
127 add_headers = false, -- allow addition of the headers from Cloudmark
128 scores_symbols = nil, -- a table with match { [score_threshold] = symbol, ... }
129 }
130
131 cloudmark_conf = lua_util.override_defaults(cloudmark_conf, opts)
132
133 if type(cloudmark_conf.scores_symbols) == 'table' then
134 numerify(cloudmark_conf.scores_symbols)
135 end
136
137 if not cloudmark_conf.prefix then
138 cloudmark_conf.prefix = 'rs_' .. cloudmark_conf.name .. '_'
139 end
140
141 if not cloudmark_conf.log_prefix then
142 if cloudmark_conf.name:lower() == cloudmark_conf.type:lower() then
143 cloudmark_conf.log_prefix = cloudmark_conf.name
144 else
145 cloudmark_conf.log_prefix = cloudmark_conf.name .. ' (' .. cloudmark_conf.type .. ')'
146 end
147 end
148
149 if not cloudmark_conf.servers and cloudmark_conf.socket then
150 cloudmark_conf.servers = cloudmark_conf.socket
151 end
152
153 if not cloudmark_conf.servers then
154 rspamd_logger.errx(rspamd_config, 'no servers defined')
155
156 return nil
157 end
158
159 cloudmark_conf.upstreams = upstream_list.create(rspamd_config,
160 cloudmark_conf.servers,
161 cloudmark_conf.default_port)
162
163 if cloudmark_conf.upstreams then
164
165 cloudmark_conf.symbols = { { symbol = cloudmark_conf.symbol_spam, score = 5.0 } }
166 cloudmark_conf.preloads = { cloudmark_preload }
167 lua_util.add_debug_alias('external_services', cloudmark_conf.name)
168 return cloudmark_conf
169 end
170
171 rspamd_logger.errx(rspamd_config, 'cannot parse servers %s',
172 cloudmark_conf['servers'])
173 return nil
174 end
175
176 local function get_specific_symbol(scores_symbols, score)
177 local selected
178 local sel_thr = -1
179
180 for threshold, sym in pairs(scores_symbols) do
181 if sel_thr < threshold and threshold <= score then
182 selected = sym
183 sel_thr = threshold
184 end
185 end
186
187 return selected
188 end
189
190 assert(get_specific_symbol({ [90] = 'CLOUDMARK_SPAM' }, 100) == 'CLOUDMARK_SPAM')
191 assert(get_specific_symbol({ [90] = 'CLOUDMARK_SPAM' }, 80) == nil)
192 assert(get_specific_symbol({ [90] = 'CLOUDMARK_SPAM', [80] = 'CLOUDMARK_SPAM2' }, 100) == 'CLOUDMARK_SPAM')
193 assert(get_specific_symbol({ [90] = 'CLOUDMARK_SPAM', [80] = 'CLOUDMARK_SPAM2' }, 80) == 'CLOUDMARK_SPAM2')
194 assert(get_specific_symbol({ [90] = 'CLOUDMARK_SPAM', [80] = 'CLOUDMARK_SPAM2' }, 70) == nil)
195 assert(get_specific_symbol({ [90] = 'CLOUDMARK_SPAM', [80] = 'CLOUDMARK_SPAM2' }, 90) == 'CLOUDMARK_SPAM')
196 assert(get_specific_symbol({ }, 80) == nil)
197 assert(get_specific_symbol({ [100] = 'CLOUDMARK_SPAM' }, 100) == 'CLOUDMARK_SPAM')
198 assert(get_specific_symbol({ [0] = 'CLOUDMARK_SPAM' }, 0) == 'CLOUDMARK_SPAM')
199
200 local function parse_cloudmark_reply(task, rule, body)
201 local parser = ucl.parser()
202 local ret, err = parser:parse_string(body)
203 if not ret then
204 rspamd_logger.errx(task, '%s: bad response body (raw): %s', N, body)
205 task:insert_result(rule.symbol_fail, 1.0, 'Parser error: ' .. err)
206 return
207 end
208 local obj = parser:get_object()
209 lua_util.debugm(N, task, 'cloudmark reply is: %s', obj)
210
211 if not obj.score then
212 rspamd_logger.errx(task, '%s: bad response body (raw): %s', N, body)
213 task:insert_result(rule.symbol_fail, 1.0, 'Parser error: no score')
214 return
215 end
216
217 if obj.analysis then
218 -- Report analysis string
219 local qid = task:get_queue_id() or 'unknown'
220 rspamd_logger.infox(task, 'qid: <%s>, cloudmark report string: %s', qid, obj.analysis)
221 end
222
223 local score = tonumber(obj.score) or 0
224 if score >= rule.score_threshold then
225 task:insert_result(rule.symbol_spam, 1.0, tostring(score))
226 end
227
228 if rule.add_headers and type(obj.appendHeaders) == 'table' then
229 local headers_add = fun.tomap(fun.map(function(h)
230 return h.headerField, {
231 order = 1, value = h.body
232 }
233 end, obj.appendHeaders))
234 lua_mime.modify_headers(task, {
235 add = headers_add
236 })
237 end
238
239 if rule.add_score_header then
240 lua_mime.modify_headers(task, {
241 add = {
242 ['X-CMAE-Score'] = {
243 order = 1,
244 value = tostring(score)
245 }
246 }
247 })
248 end
249
250 if type(rule.scores_symbols) == 'table' then
251 local sym = get_specific_symbol(rule.scores_symbols, score)
252 if sym then
253 task:insert_result(sym, 1.0, tostring(score))
254 end
255 end
256
257 end
258
259 local function cloudmark_check(task, content, digest, rule, maybe_part)
260 local function cloudmark_check_uncached()
261 local upstream = rule.upstreams:get_upstream_round_robin()
262 local addr = upstream:get_addr()
263 local retransmits = rule.retransmits
264
265 local url = cloudmark_url(rule, addr)
266 local message_data = task:get_content()
267 if rule.max_message and rule.max_message > 0 and #message_data > rule.max_message then
268 task:insert_result(rule['symbol_fail'], 0.0, 'Message too large: ' .. #message_data)
269 return
270 end
271 local request = {
272 rfc822 = {
273 ['Content-Type'] = 'message/rfc822',
274 data = message_data,
275 }
276 }
277
278 local helo = task:get_helo()
279 if helo then
280 request['heloDomain'] = {
281 data = helo,
282 }
283 end
284 local mail_from = task:get_from('smtp') or {}
285 if mail_from[1] and #mail_from[1].addr > 1 then
286 request['mailFrom'] = {
287 data = mail_from[1].addr
288 }
289 end
290
291 local rcpt_to = task:get_recipients('smtp')
292 if rcpt_to then
293 request['rcptTo'] = {
294 data = table.concat(fun.totable(fun.map(function(r)
295 return r.addr
296 end, rcpt_to)), ',')
297 }
298 end
299
300 local fip = task:get_from_ip()
301 if fip and fip:is_valid() then
302 request['connIp'] = {
303 data = tostring(fip)
304 }
305 end
306
307 local hostname = task:get_hostname()
308 if hostname then
309 request['fromHost'] = hostname
310 end
311
312 local request_data = {
313 task = task,
314 url = url,
315 body = lua_util.table_to_multipart_body(request, static_boundary),
316 headers = {
317 ['Content-Type'] = string.format('multipart/form-data; boundary="%s"', static_boundary)
318 },
319 timeout = rule.timeout,
320 }
321
322 local function cloudmark_callback(http_err, code, body, headers)
323
324 local function cloudmark_requery()
325 -- set current upstream to fail because an error occurred
326 upstream:fail()
327
328 -- retry with another upstream until retransmits exceeds
329 if retransmits > 0 then
330
331 retransmits = retransmits - 1
332
333 lua_util.debugm(rule.name, task,
334 '%s: request Error: %s - retries left: %s',
335 rule.log_prefix, http_err, retransmits)
336
337 -- Select a different upstream!
338 upstream = rule.upstreams:get_upstream_round_robin()
339 addr = upstream:get_addr()
340 url = cloudmark_url(rule, addr)
341
342 lua_util.debugm(rule.name, task, '%s: retry IP: %s:%s',
343 rule.log_prefix, addr, addr:get_port())
344 request_data.url = url
345
346 http.request(request_data)
347 else
348 rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits ' ..
349 'exceed', rule.log_prefix)
350 task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan and ' ..
351 'retransmits exceed')
352 upstream:fail()
353 end
354 end
355
356 if http_err then
357 cloudmark_requery()
358 else
359 -- Parse the response
360 if upstream then
361 upstream:ok()
362 end
363 if code ~= 200 then
364 rspamd_logger.errx(task, 'invalid HTTP code: %s, body: %s, headers: %s', code, body, headers)
365 task:insert_result(rule.symbol_fail, 1.0, 'Bad HTTP code: ' .. code)
366 return
367 end
368 parse_cloudmark_reply(task, rule, body)
369 end
370 end
371
372 request_data.callback = cloudmark_callback
373 http.request(request_data)
374 end
375
376 if common.condition_check_and_continue(task, content, rule, digest,
377 cloudmark_check_uncached, maybe_part) then
378 return
379 else
380 cloudmark_check_uncached()
381 end
382 end
383
384 return {
385 type = { 'cloudmark', 'scanner' },
386 description = 'Cloudmark cartridge interface',
387 configure = cloudmark_config,
388 check = cloudmark_check,
389 name = N,
390 }