Debian Codegraph @

lualib/lua_bayes_redis.lua

raw
1 --[[
2 Copyright (c) 2022, Vsevolod Stakhov <[email protected]>
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 ]]
16
17 -- This file contains functions to support Bayes statistics in Redis
18
19 local exports = {}
20 local lua_redis = require "lua_redis"
21 local logger = require "rspamd_logger"
22 local lua_util = require "lua_util"
23 local ucl = require "ucl"
24
25 local N = "bayes"
26
27 local function gen_classify_functor(redis_params, classify_script_id)
28 return function(task, expanded_key, id, class_labels, stat_tokens, callback)
29 local function classify_redis_cb(err, data)
30 lua_util.debugm(N, task, 'classify redis cb: %s, %s', err, data)
31 if err then
32 callback(task, false, err)
33 else
34 -- Pass the raw data table to the C++ callback for processing
35 -- The C++ callback will handle both binary and multi-class formats
36 callback(task, true, data)
37 end
38 end
39
40 -- Determine class labels to send to Redis script
41 local script_class_labels
42 if type(class_labels) == "table" then
43 -- Use simple comma-separated string instead of messagepack
44 script_class_labels = "TABLE:" .. table.concat(class_labels, ",")
45 else
46 -- Single class label or boolean compatibility
47 if class_labels == true or class_labels == "true" then
48 script_class_labels = "S" -- spam
49 elseif class_labels == false or class_labels == "false" then
50 script_class_labels = "H" -- ham
51 else
52 script_class_labels = class_labels -- string class label
53 end
54 end
55
56 lua_redis.exec_redis_script(classify_script_id,
57 { task = task, is_write = false, key = expanded_key },
58 classify_redis_cb, { expanded_key }, { script_class_labels, stat_tokens })
59 end
60 end
61
62 local function gen_learn_functor(redis_params, learn_script_id)
63 return function(task, expanded_key, id, class_label, symbol, is_unlearn, stat_tokens, callback, maybe_text_tokens)
64 local function learn_redis_cb(err, data)
65 lua_util.debugm(N, task, 'learn redis cb: %s, %s for class %s', err, data, class_label)
66 if err then
67 callback(task, false, err)
68 else
69 callback(task, true)
70 end
71 end
72
73 -- Convert class_label for backward compatibility
74 local script_class_label = class_label
75 if class_label == true or class_label == "true" then
76 script_class_label = "S" -- spam
77 elseif class_label == false or class_label == "false" then
78 script_class_label = "H" -- ham
79 end
80
81 if maybe_text_tokens then
82 lua_redis.exec_redis_script(learn_script_id,
83 { task = task, is_write = true, key = expanded_key },
84 learn_redis_cb,
85 { expanded_key }, { script_class_label, symbol, tostring(is_unlearn), stat_tokens, maybe_text_tokens })
86 else
87 lua_redis.exec_redis_script(learn_script_id,
88 { task = task, is_write = true, key = expanded_key },
89 learn_redis_cb, { expanded_key }, { script_class_label, symbol, tostring(is_unlearn), stat_tokens })
90 end
91 end
92 end
93
94 local function load_redis_params(classifier_ucl, statfile_ucl)
95 local redis_params
96
97 -- Try load from statfile options
98 if statfile_ucl.redis then
99 redis_params = lua_redis.try_load_redis_servers(statfile_ucl.redis, rspamd_config, true)
100 end
101
102 if not redis_params then
103 if statfile_ucl then
104 redis_params = lua_redis.try_load_redis_servers(statfile_ucl, rspamd_config, true)
105 end
106 end
107
108 -- Try load from classifier config
109 if not redis_params and classifier_ucl.backend then
110 redis_params = lua_redis.try_load_redis_servers(classifier_ucl.backend, rspamd_config, true)
111 end
112
113 if not redis_params and classifier_ucl.redis then
114 redis_params = lua_redis.try_load_redis_servers(classifier_ucl.redis, rspamd_config, true)
115 end
116
117 if not redis_params then
118 redis_params = lua_redis.try_load_redis_servers(classifier_ucl, rspamd_config, true)
119 end
120
121 -- Try load global options
122 if not redis_params then
123 redis_params = lua_redis.try_load_redis_servers(rspamd_config:get_all_opt('redis'), rspamd_config, true)
124 end
125
126 if not redis_params then
127 logger.err(rspamd_config, "cannot load Redis parameters for the classifier")
128 return nil
129 end
130
131 return redis_params
132 end
133
134 ---
135 --- Init bayes classifier
136 --- @param classifier_ucl ucl of the classifier config
137 --- @param statfile_ucl ucl of the statfile config
138 --- @return a pair of (classify_functor, learn_functor) or `nil` in case of error
139 exports.lua_bayes_init_statfile = function(classifier_ucl, statfile_ucl, symbol, class_label, ev_base, stat_periodic_cb)
140 local redis_params = load_redis_params(classifier_ucl, statfile_ucl)
141
142 if not redis_params then
143 return nil
144 end
145
146 local classify_script_id, script_err = lua_redis.load_redis_script_from_file("bayes_classify.lua", redis_params)
147 if not classify_script_id then
148 logger.errx(ev_base, script_err)
149 return nil
150 end
151 local learn_script_id
152 learn_script_id, script_err = lua_redis.load_redis_script_from_file("bayes_learn.lua", redis_params)
153 if not learn_script_id then
154 logger.errx(ev_base, script_err)
155 return nil
156 end
157 local stat_script_id
158 stat_script_id, script_err = lua_redis.load_redis_script_from_file("bayes_stat.lua", redis_params)
159 if not stat_script_id then
160 logger.errx(ev_base, script_err)
161 return nil
162 end
163 local max_users = classifier_ucl.max_users or 1000
164
165 local current_data = {
166 users = 0,
167 revision = 0,
168 }
169 local final_data = {
170 users = 0,
171 revision = 0, -- number of learns
172 }
173 local cursor = 0
174
175 if ev_base then
176 rspamd_config:add_periodic(ev_base, 0.0, function(cfg, _)
177 local function stat_redis_cb(err, data)
178 lua_util.debugm(N, cfg, 'stat redis cb: %s, %s', err, data)
179
180 if err then
181 logger.warn(cfg, 'cannot get bayes statistics for %s: %s', symbol, err)
182 else
183 local new_cursor = data[1]
184 current_data.users = current_data.users + data[2]
185 current_data.revision = current_data.revision + data[3]
186 if new_cursor == 0 then
187 -- Done iteration
188 final_data = lua_util.shallowcopy(current_data)
189 current_data = {
190 users = 0,
191 revision = 0,
192 }
193 lua_util.debugm(N, cfg, 'final data: %s', final_data)
194 stat_periodic_cb(cfg, final_data)
195 end
196
197 cursor = new_cursor
198 end
199 end
200
201 -- Convert class_label to learn key
202 local learn_key
203 if class_label == true or class_label == "true" or class_label == "S" then
204 learn_key = "learns_spam"
205 elseif class_label == false or class_label == "false" or class_label == "H" then
206 learn_key = "learns_ham"
207 else
208 -- For other class labels, use learns_<class_label>
209 learn_key = "learns_" .. string.lower(tostring(class_label))
210 end
211
212 lua_redis.exec_redis_script(stat_script_id,
213 { ev_base = ev_base, cfg = cfg, is_write = false },
214 stat_redis_cb, { tostring(cursor),
215 symbol,
216 learn_key,
217 tostring(max_users) })
218 return statfile_ucl.monitor_timeout or classifier_ucl.monitor_timeout or 30.0
219 end)
220 end
221
222 return gen_classify_functor(redis_params, classify_script_id), gen_learn_functor(redis_params, learn_script_id)
223 end
224
225 local function gen_cache_check_functor(redis_params, check_script_id, conf)
226 local packed_conf = ucl.to_format(conf, 'msgpack')
227 return function(task, cache_id, callback)
228 local function classify_redis_cb(err, data)
229 lua_util.debugm(N, task, 'check cache redis cb: %s, %s (%s)', err, data, type(data))
230 if err then
231 callback(task, false, err)
232 else
233 -- The cached value is now a class name string (e.g. "spam", "ham",
234 -- "transactional"). Previously it was a number (numeric class_id hash),
235 -- but that caused precision loss for large uint64 hashes in Lua doubles.
236 if type(data) == 'string' then
237 callback(task, true, data)
238 else
239 callback(task, false, 'not found')
240 end
241 end
242 end
243
244 lua_util.debugm(N, task, 'checking cache: %s', cache_id)
245 lua_redis.exec_redis_script(check_script_id,
246 { task = task, is_write = false, key = cache_id },
247 classify_redis_cb, { cache_id }, { packed_conf })
248 end
249 end
250
251 local function gen_cache_learn_functor(redis_params, learn_script_id, conf)
252 local packed_conf = ucl.to_format(conf, 'msgpack')
253 return function(task, cache_id, class_name)
254 local function learn_redis_cb(err, data)
255 lua_util.debugm(N, task, 'learn_cache redis cb: %s, %s', err, data)
256 end
257
258 lua_util.debugm(N, task, 'try to learn cache: %s as %s', cache_id, class_name)
259 lua_redis.exec_redis_script(learn_script_id,
260 { task = task, is_write = true, key = cache_id },
261 learn_redis_cb,
262 { cache_id }, { class_name, packed_conf })
263 end
264 end
265
266 exports.lua_bayes_init_cache = function(classifier_ucl, statfile_ucl)
267 local redis_params = load_redis_params(classifier_ucl, statfile_ucl)
268
269 if not redis_params then
270 return nil
271 end
272
273 local default_conf = {
274 cache_prefix = "learned_ids",
275 cache_max_elt = 10000, -- Maximum number of elements in the cache key
276 cache_max_keys = 5, -- Maximum number of keys in the cache
277 cache_elt_len = 32, -- Length of the element in the cache (will trim id to that value)
278 }
279
280 local conf = lua_util.override_defaults(default_conf, classifier_ucl)
281 -- Clean all not known configurations
282 for k, _ in pairs(conf) do
283 if default_conf[k] == nil then
284 conf[k] = nil
285 end
286 end
287
288 local check_script_id, err = lua_redis.load_redis_script_from_file("bayes_cache_check.lua", redis_params)
289 if not check_script_id then
290 logger.errx(rspamd_config, err)
291 return nil
292 end
293 local learn_script_id
294 learn_script_id, err = lua_redis.load_redis_script_from_file("bayes_cache_learn.lua", redis_params)
295 if not learn_script_id then
296 logger.errx(rspamd_config, err)
297 return nil
298 end
299
300 return gen_cache_check_functor(redis_params, check_script_id, conf), gen_cache_learn_functor(redis_params,
301 learn_script_id, conf)
302 end
303
304 return exports