Debian Codegraph @

lualib/lua_cache.lua

raw
1 --[[
2 Copyright (c) 2025, Vsevolod Stakhov <[email protected]>
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 ]] --
16
17 --[[[
18 -- @module lua_cache
19 -- This module provides a Redis-based caching API for Rspamd with support for
20 -- concurrent operations across multiple workers. It includes features like
21 -- distributed locking via PENDING markers, automatic key hashing,
22 -- configurable serialization formats, and TTL management.
23 --
24 @example
25 local redis_cache = require "lua_cache"
26 local redis_params = redis_lib.parse_redis_server('reputation')
27
28 -- Create cache context
29 local cache_context = redis_cache.create_cache_context(redis_params, {
30 cache_prefix = "rspamd_reputation",
31 cache_ttl = 86400, -- 1 day
32 cache_format = "json",
33 cache_hash_len = 16,
34 cache_use_hashing = true
35 })
36
37 -- Example usage in a task
38 local function process_url_reputation(task, url)
39 local cache_key = url:get_tld()
40
41 -- Try to get data from cache first
42 redis_cache.cache_get(task, cache_key, cache_context, 5.0,
43 -- This callback is called on cache miss
44 function(task)
45 -- Perform expensive reputation lookup
46 local reputation = calculate_reputation(task, url)
47
48 -- Store result in cache for future use
49 redis_cache.cache_set(task, cache_key, {
50 score = reputation.score,
51 categories = reputation.categories,
52 timestamp = os.time()
53 }, cache_context)
54
55 -- Use the result
56 apply_reputation_rules(task, url, reputation)
57 end,
58 -- This callback is called when cache data is available
59 function(task, err, data)
60 if err then
61 logger.errx(task, "Cache error for %s: %s", cache_key, err)
62 return
63 end
64
65 -- Use the cached data
66 apply_reputation_rules(task, url, data)
67 end
68 )
69 end
70 --]]
71
72 local logger = require "rspamd_logger"
73 local ucl = require "ucl"
74 local lua_util = require "lua_util"
75 local rspamd_util = require "rspamd_util"
76 local lua_redis = require "lua_redis"
77 local hasher = require "rspamd_cryptobox_hash"
78
79 local N = "lua_cache"
80 local exports = {}
81
82 -- Default options
83 local default_opts = {
84 cache_prefix = "rspamd_cache",
85 cache_ttl = 3600, -- 1 hour
86 cache_probes = 5, -- Number of times to check a pending key
87 cache_format = "json", -- Serialization format
88 cache_hash_len = 16, -- Number of hex symbols to use for hashed keys
89 cache_use_hashing = false -- Whether to hash keys by default
90 }
91
92 -- Create a hash of the key using the configured length
93 local function hash_key(key, hash_len)
94 local h = hasher.create(key)
95 local hex = h:hex()
96
97 if hash_len and hash_len > 0 and hash_len < #hex then
98 return string.sub(hex, 1, hash_len)
99 end
100
101 return hex
102 end
103
104 -- Get the appropriate key based on hashing configuration
105 local function get_cache_key(raw_key, cache_context, force_hashing)
106 -- Determine whether to hash based on context settings and force parameter
107 local should_hash = force_hashing
108 if should_hash == nil then
109 should_hash = cache_context.opts.cache_use_hashing
110 end
111
112 if should_hash then
113 local raw_len = (type(raw_key) == 'string') and #raw_key or -1
114 lua_util.debugm(N, rspamd_config, "hashing cache key (len=%s) with hash length %s",
115 raw_len, cache_context.opts.cache_hash_len)
116 return hash_key(raw_key, cache_context.opts.cache_hash_len)
117 else
118 return raw_key
119 end
120 end
121
122 -- Create a caching context with the provided options
123 local function create_cache_context(redis_params, opts, module_name)
124 if not redis_params then
125 return nil, "Redis parameters must be provided"
126 end
127
128 local cache_context = {}
129 cache_context.redis_params = redis_params
130
131 -- Process and merge configuration options
132 cache_context.opts = lua_util.override_defaults(default_opts, opts)
133 cache_context.N = module_name or N
134
135 -- Register Redis prefix
136 lua_redis.register_prefix(cache_context.opts.cache_prefix,
137 "caching",
138 "Cache API prefix")
139
140 lua_util.debugm(N, rspamd_config, "registered redis prefix: %s", cache_context.opts.cache_prefix)
141
142 -- Remove cache related options from opts table
143 if opts then
144 lua_util.debugm(N, rspamd_config, "removing cache options from original opts table")
145 opts.cache_prefix = nil
146 opts.cache_ttl = nil
147 opts.cache_probes = nil
148 opts.cache_format = nil
149 opts.cache_hash_len = nil
150 opts.cache_use_hashing = nil
151 end
152
153 -- Set serialization and deserialization functions
154 if cache_context.opts.cache_format == "messagepack" then
155 lua_util.debugm(cache_context.N, rspamd_config, "using messagepack for serialization")
156
157 cache_context.encode = function(data)
158 return ucl.to_format(data, 'msgpack')
159 end
160
161 cache_context.decode = function(raw_data)
162 local ucl_parser = ucl.parser()
163 local ok, ucl_err = ucl_parser:parse_text(raw_data, 'msgpack')
164 if not ok then
165 lua_util.debugm(cache_context.N, rspamd_config, "failed to parse messagepack data: %s", ucl_err)
166 return nil
167 end
168 return ucl_parser:get_object()
169 end
170 else
171 -- Default to JSON
172 lua_util.debugm(cache_context.N, rspamd_config, "using json for serialization")
173
174 cache_context.encode = function(data)
175 return ucl.to_format(data, 'json')
176 end
177
178 cache_context.decode = function(raw_data)
179 local ucl_parser = ucl.parser()
180 local ok, ucl_err = ucl_parser:parse_text(raw_data)
181 if not ok then
182 lua_util.debugm(cache_context.N, rspamd_config, "failed to parse json data: %s", ucl_err)
183 return nil
184 end
185 return ucl_parser:get_object()
186 end
187 end
188
189 lua_util.debugm(cache_context.N, rspamd_config, "cache context created: %s", cache_context.opts)
190 return cache_context
191 end
192
193 -- Encode data for storage in Redis with proper formatting
194 local function encode_data(data, cache_context)
195 lua_util.debugm(cache_context.N, rspamd_config, "encoding data using %s format", cache_context.opts.cache_format)
196 return cache_context.encode(data)
197 end
198
199 -- Decode data from Redis with proper formatting
200 local function decode_data(data, cache_context)
201 if not data then
202 lua_util.debugm(cache_context.N, rspamd_config, "cannot decode nil data")
203 return nil
204 end
205 lua_util.debugm(cache_context.N, rspamd_config, "decoding data using %s format", cache_context.opts.cache_format)
206 return cache_context.decode(data)
207 end
208
209 -- Check if a value is a PENDING marker and extract its details
210 local function parse_pending_value(value, cache_context)
211 if type(value) ~= 'string' then
212 lua_util.debugm(cache_context.N, rspamd_config, "value is not a string, cannot be a pending marker")
213 return nil
214 end
215
216 -- Check if the value starts with PENDING:
217 if string.sub(value, 1, 8) ~= "PENDING:" then
218 lua_util.debugm(cache_context.N, rspamd_config, "value doesn't start with PENDING: prefix")
219 return nil
220 end
221
222 lua_util.debugm(cache_context.N, rspamd_config, "found PENDING marker, extracting data")
223 local pending_data = string.sub(value, 9)
224 return decode_data(pending_data, cache_context)
225 end
226
227 -- Create a pending marker with hostname and timeout
228 local function create_pending_marker(timeout, cache_context)
229 local hostname = rspamd_util.get_hostname()
230 local pending_data = {
231 hostname = hostname,
232 timeout = timeout,
233 timestamp = os.time()
234 }
235
236 lua_util.debugm(cache_context.N, rspamd_config, "creating PENDING marker for host %s, timeout %s",
237 hostname, timeout)
238
239 return "PENDING:" .. encode_data(pending_data, cache_context)
240 end
241
242 -- Check cache and handle the result appropriately
243 local function cache_get(task, key, cache_context, timeout, callback_uncached, callback_data)
244 if not task or not key or not cache_context or not callback_uncached or not callback_data then
245 logger.errx(task, "missing required parameters for cache_get")
246 return false
247 end
248
249 local full_key = cache_context.opts.cache_prefix .. "_" .. get_cache_key(key, cache_context, nil)
250 lua_util.debugm(cache_context.N, task, "cache lookup for key: %s", full_key)
251
252 -- Function to check a pending key
253 local function check_pending(pending_info)
254 local probe_count = 0
255 local probe_interval = timeout / (cache_context.opts.cache_probes or 5)
256
257 lua_util.debugm(cache_context.N, task, "setting up probes for pending key %s, interval: %s seconds",
258 full_key, probe_interval)
259
260 -- Set up a timer to probe the key
261 local function probe_key()
262 probe_count = probe_count + 1
263 lua_util.debugm(cache_context.N, task, "probe #%s/%s for pending key %s",
264 probe_count, cache_context.opts.cache_probes, full_key)
265
266 if probe_count >= cache_context.opts.cache_probes then
267 logger.infox(task, "maximum probes reached for key %s, considering it failed", full_key)
268 lua_util.debugm(cache_context.N, task, "maximum probes reached for key %s, giving up", full_key)
269 callback_data(task, "timeout waiting for pending key", nil)
270 return
271 end
272
273 lua_util.debugm(cache_context.N, task, "probing redis for key %s", full_key)
274 lua_redis.redis_make_request(task, cache_context.redis_params, key, false,
275 function(err, data)
276 if err then
277 logger.errx(task, "redis error while probing key %s: %s", full_key, err)
278 lua_util.debugm(cache_context.N, task, "redis error during probe: %s, retrying later", err)
279 task:add_timer(probe_interval, probe_key)
280 return
281 end
282
283 if not data or type(data) == 'userdata' then
284 lua_util.debugm(cache_context.N, task, "pending key %s disappeared, calling uncached handler", full_key)
285 callback_uncached(task)
286 return
287 end
288
289 local pending = parse_pending_value(data, cache_context)
290 if pending then
291 lua_util.debugm(cache_context.N, task, "key %s still pending (host: %s), retrying later",
292 full_key, pending.hostname)
293 task:add_timer(probe_interval, probe_key)
294 else
295 lua_util.debugm(cache_context.N, task, "pending key %s resolved to actual data", full_key)
296 callback_data(task, nil, decode_data(data, cache_context))
297 end
298 end,
299 'GET', { full_key }
300 )
301 end
302
303 -- Start the first probe after the initial probe interval
304 lua_util.debugm(cache_context.N, task, "scheduling first probe for %s in %s seconds",
305 full_key, probe_interval)
306 task:add_timer(probe_interval, probe_key)
307 end
308
309 -- Initial cache lookup
310 lua_util.debugm(cache_context.N, task, "making initial redis GET request for key: %s", full_key)
311 lua_redis.redis_make_request(task, cache_context.redis_params, key, false,
312 function(err, data)
313 if err then
314 logger.errx(task, "redis error looking up key %s: %s", full_key, err)
315 lua_util.debugm(cache_context.N, task, "redis error: %s, calling uncached handler", err)
316 callback_uncached(task)
317 return
318 end
319
320 if not data or type(data) == 'userdata' then
321 -- Key not found, set pending and call the uncached callback
322 lua_util.debugm(cache_context.N, task, "key %s not found in cache, creating pending marker", full_key)
323 local pending_marker = create_pending_marker(timeout, cache_context)
324
325 lua_util.debugm(cache_context.N, task, "setting pending marker for key %s with TTL %s",
326 full_key, timeout * 2)
327 lua_redis.redis_make_request(task, cache_context.redis_params, key, true,
328 function(set_err, set_data)
329 if set_err then
330 logger.errx(task, "redis error setting pending marker for %s: %s", full_key, set_err)
331 lua_util.debugm(cache_context.N, task, "failed to set pending marker: %s", set_err)
332 else
333 lua_util.debugm(cache_context.N, task, "successfully set pending marker for %s", full_key)
334 end
335 lua_util.debugm(cache_context.N, task, "calling uncached handler for %s", full_key)
336 callback_uncached(task)
337 end,
338 'SETEX', { full_key, tostring(math.floor(timeout * 2)), pending_marker }
339 )
340 else
341 -- Key found, check if it's a pending marker or actual data
342 local pending = parse_pending_value(data, cache_context)
343
344 if pending then
345 -- Key is being processed by another worker
346 lua_util.debugm(cache_context.N, task, "key %s is pending on host %s, waiting for result",
347 full_key, pending.hostname)
348 check_pending(pending)
349 else
350 -- Extend TTL and return data
351 lua_util.debugm(cache_context.N, task, "found cached data for key %s, extending TTL to %s",
352 full_key, cache_context.opts.cache_ttl)
353 lua_redis.redis_make_request(task, cache_context.redis_params, key, true,
354 function(expire_err, _)
355 if expire_err then
356 logger.errx(task, "redis error extending TTL for %s: %s", full_key, expire_err)
357 lua_util.debugm(cache_context.N, task, "failed to extend TTL: %s", expire_err)
358 else
359 lua_util.debugm(cache_context.N, task, "successfully extended TTL for %s", full_key)
360 end
361 end,
362 'EXPIRE', { full_key, tostring(math.floor(cache_context.opts.cache_ttl)) }
363 )
364
365 lua_util.debugm(cache_context.N, task, "returning cached data for key %s", full_key)
366 callback_data(task, nil, decode_data(data, cache_context))
367 end
368 end
369 end,
370 'GET', { full_key }
371 )
372
373 return true
374 end
375
376 -- Save data to the cache
377 local function cache_set(task, key, data, cache_context)
378 if not task or not key or not data or not cache_context then
379 logger.errx(task, "missing required parameters for cache_set")
380 return false
381 end
382
383 local full_key = cache_context.opts.cache_prefix .. "_" .. get_cache_key(key, cache_context, nil)
384 local ttl = cache_context.opts.cache_ttl
385 local expire_at = os.time() + ttl
386 lua_util.debugm(cache_context.N, task, "caching data for key: %s with TTL: %s (expiring at: %s)",
387 full_key, ttl, os.date('%Y-%m-%d %H:%M:%S', expire_at))
388
389 local encoded_data = encode_data(data, cache_context)
390
391 -- Store the data with expiration
392 lua_util.debugm(cache_context.N, task, "making redis SETEX request for key: %s", full_key)
393 return lua_redis.redis_make_request(task, cache_context.redis_params, key, true,
394 function(err, result)
395 if err then
396 logger.errx(task, "redis error setting cached data for %s: %s", full_key, err)
397 lua_util.debugm(cache_context.N, task, "failed to cache data: %s", err)
398 else
399 lua_util.debugm(cache_context.N, task, "successfully cached data for key %s, expiring at %s",
400 full_key, os.date('%Y-%m-%d %H:%M:%S', expire_at))
401 end
402 end,
403 'SETEX', { full_key, tostring(math.floor(ttl)), encoded_data }
404 )
405 end
406
407 -- Delete a cache entry
408 local function cache_del(task, key, cache_context)
409 if not task or not key or not cache_context then
410 logger.errx(task, "missing required parameters for cache_del")
411 return false
412 end
413
414 local full_key = cache_context.opts.cache_prefix .. "_" .. get_cache_key(key, cache_context, nil)
415 lua_util.debugm(cache_context.N, task, "deleting cache key: %s", full_key)
416
417 return lua_redis.redis_make_request(task, cache_context.redis_params, key, true,
418 function(err, result)
419 if err then
420 logger.errx(task, "redis error deleting cache key %s: %s", full_key, err)
421 lua_util.debugm(cache_context.N, task, "failed to delete cache key: %s", err)
422 else
423 local count = tonumber(result) or 0
424 lua_util.debugm(cache_context.N, task, "successfully deleted cache key %s (%s keys removed)",
425 full_key, count)
426 end
427 end,
428 'DEL', { full_key }
429 )
430 end
431
432 -- Export the API functions
433 ---[[[
434 -- @function lua_cache.create_cache_context(redis_params, opts, module_name)
435 -- Creates a Redis caching context with specified parameters and options
436 -- @param {table} redis_params Redis connection parameters (required)
437 -- @param {table} opts Optional configuration parameters:
438 -- * `cache_prefix`: Key prefix for Redis (default: "rspamd_cache")
439 -- * `cache_ttl`: TTL in seconds for cached entries (default: 3600)
440 -- * `cache_probes`: Number of times to check pending keys (default: 5)
441 -- * `cache_format`: Serialization format - "json" or "messagepack" (default: "json")
442 -- * `cache_hash_len`: Number of hex symbols for hashed keys (default: 16)
443 -- * `cache_use_hashing`: Whether to hash keys by default (default: true)
444 -- @return {table} Cache context or nil + error message on failure
445 --]]
446 exports.create_cache_context = create_cache_context
447 ---[[[
448 -- @function รง.cache_get(task, key, cache_context, timeout, callback_uncached, callback_data)
449 -- Retrieves data from cache, handling pending states and cache misses appropriately
450 -- @param {rspamd_task} task Current task (required)
451 -- @param {string} key Cache key (required)
452 -- @param {table} cache_context Redis cache context from create_cache_context (required)
453 -- @param {number} timeout Timeout for pending operations in seconds (required)
454 -- @param {function} callback_uncached Function to call on cache miss: callback_uncached(task) (required)
455 -- @param {function} callback_data Function to call when data is available: callback_data(task, err, data) (required)
456 -- @return {boolean} true if request was initiated successfully, false otherwise
457 --]]
458 exports.cache_get = cache_get
459 ---[[[
460 -- @function lua_cache.cache_set(task, key, data, cache_context)
461 -- Stores data in the cache with the configured TTL
462 -- @param {rspamd_task} task Current task (required)
463 -- @param {string} key Cache key (required)
464 -- @param {table} data Data to store in the cache (required)
465 -- @param {table} cache_context Redis cache context from create_cache_context (required)
466 -- @return {boolean} true if request was initiated successfully, false otherwise
467 --]]
468 exports.cache_set = cache_set
469 ---[[[
470 -- @function lua_cache.cache_del(task, key, cache_context)
471 -- Deletes data from the cache
472 -- @param {rspamd_task} task Current task (required)
473 -- @param {string} key Cache key (required)
474 -- @param {table} cache_context Redis cache context from create_cache_context (required)
475 -- @return {boolean} true if request was initiated successfully, false otherwise
476 --]]
477 exports.cache_del = cache_del
478
479 return exports