Debian Codegraph @

lualib/lua_hs_cache.lua

raw
1 --[[
2 Copyright (c) 2026, Vsevolod Stakhov <[email protected]>
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 ]]--
16
17 local logger = require "rspamd_logger"
18 local rspamd_util = require "rspamd_util"
19 local lua_redis = require "lua_redis"
20 local lua_util = require "lua_util"
21 local rspamd_http = require "rspamd_http"
22
23 local exports = {}
24 -- Use "hyperscan" module name so debug output is enabled together with C code
25 local N = "hyperscan"
26
27 -- File backend
28 local file_backend = {}
29 file_backend.__index = file_backend
30
31 -- Zstd magic number: 0xFD2FB528 (little-endian bytes: 28 B5 2F FD)
32 local ZSTD_MAGIC = string.char(0x28, 0xB5, 0x2F, 0xFD)
33
34 function file_backend.new(config)
35 local self = setmetatable({}, file_backend)
36 -- Store config for logging context
37 self.config = config.rspamd_config or config
38 -- Remove trailing slashes from cache_dir
39 local cache_dir = config.cache_dir or '/var/lib/rspamd/hs_cache'
40 self.cache_dir = cache_dir:gsub("/+$", "")
41 -- Default to flat directory structure for backward compatibility
42 self.platform_dirs = config.platform_dirs == true
43 -- Enable compression by default (consistent with redis/http backends)
44 local opts = config.file or config
45 self.use_compression = (opts.compression ~= false) and (config.compression ~= false)
46 lua_util.debugm(N, self.config, "file backend config: cache_dir=%s, platform_dirs=%s, compression=%s",
47 self.cache_dir, self.platform_dirs and "yes" or "no",
48 self.use_compression and "enabled" or "disabled")
49 return self
50 end
51
52 -- Get file extension based on compression setting
53 function file_backend:_get_extension()
54 return self.use_compression and '.hs.zst' or '.hs'
55 end
56
57 -- Get the path for a cache file
58 -- @param cache_key string cache key (hash that already includes platform info)
59 -- @param platform_id string platform identifier (unused in flat mode for backward compat)
60 -- @param ext string optional extension override (e.g., '.hs' or '.hs.zst')
61 function file_backend:_get_path(cache_key, platform_id, ext)
62 local extension = ext or self:_get_extension()
63 if self.platform_dirs then
64 -- Optional: use platform subdirectories (not default)
65 return string.format("%s/%s/%s%s", self.cache_dir, platform_id, cache_key, extension)
66 else
67 -- Default: flat structure matching original C code behavior
68 -- Platform info is already embedded in cache_key hash
69 return string.format("%s/%s%s", self.cache_dir, cache_key, extension)
70 end
71 end
72
73 -- Check if data starts with zstd magic bytes
74 function file_backend:_is_zstd(data)
75 if not data or #data < 4 then
76 return false
77 end
78 return data:sub(1, 4) == ZSTD_MAGIC
79 end
80
81 -- Find existing cache file, trying both compressed and uncompressed extensions
82 -- Returns: path, is_compressed (or nil if not found)
83 function file_backend:_find_existing_path(cache_key, platform_id)
84 -- Try compressed first if compression is enabled, otherwise try uncompressed first
85 local primary_ext = self:_get_extension()
86 local secondary_ext = self.use_compression and '.hs' or '.hs.zst'
87
88 local primary_path = self:_get_path(cache_key, platform_id, primary_ext)
89 -- rspamd_util.stat returns (err, stat_table) - check for no error AND valid stat
90 local err, stat = rspamd_util.stat(primary_path)
91 if not err and stat then
92 return primary_path, primary_ext == '.hs.zst'
93 end
94
95 local secondary_path = self:_get_path(cache_key, platform_id, secondary_ext)
96 err, stat = rspamd_util.stat(secondary_path)
97 if not err and stat then
98 return secondary_path, secondary_ext == '.hs.zst'
99 end
100
101 return nil, nil
102 end
103
104 function file_backend:_ensure_dir(path)
105 local dir = path:match("(.*/)")
106 if dir then
107 local ok, err = rspamd_util.mkdir(dir, true)
108 if not ok and err then
109 logger.warnx(N, "failed to create directory %s: %s", dir, err)
110 end
111 end
112 end
113
114 function file_backend:exists(cache_key, platform_id, callback)
115 local path, is_compressed = self:_find_existing_path(cache_key, platform_id)
116
117 if path then
118 local err, stat = rspamd_util.stat(path)
119 if not err and stat then
120 lua_util.debugm(N, self.config, "file exists check: %s found, size: %d, compressed: %s",
121 path, stat.size, is_compressed and "yes" or "no")
122 callback(nil, true, { size = stat.size, mtime = stat.mtime, compressed = is_compressed })
123 else
124 -- Race condition: file disappeared between _find_existing_path and stat
125 lua_util.debugm(N, self.config, "file exists check: %s disappeared (race)", path)
126 callback(nil, false, nil)
127 end
128 else
129 local expected_path = self:_get_path(cache_key, platform_id)
130 lua_util.debugm(N, self.config, "file exists check: %s not found (checked both extensions)", expected_path)
131 callback(nil, false, nil)
132 end
133 end
134
135 function file_backend:load(cache_key, platform_id, callback)
136 local path, expected_compressed = self:_find_existing_path(cache_key, platform_id)
137
138 if not path then
139 local expected_path = self:_get_path(cache_key, platform_id)
140 lua_util.debugm(N, self.config, "file load failed: %s not found (checked both extensions)", expected_path)
141 callback("file not found", nil)
142 return
143 end
144
145 lua_util.debugm(N, self.config, "file load from: %s (expected compressed: %s)", path, expected_compressed and "yes" or "no")
146
147 local f, err = io.open(path, "rb")
148 if not f then
149 lua_util.debugm(N, self.config, "file load failed from %s: %s", path, err or "open error")
150 callback(err or "open error", nil)
151 return
152 end
153 local data = f:read("*a")
154 f:close()
155 if not data then
156 lua_util.debugm(N, self.config, "file read failed from %s", path)
157 callback("read error", nil)
158 return
159 end
160
161 -- Check if data is actually zstd compressed (magic byte verification)
162 local is_zstd = self:_is_zstd(data)
163 lua_util.debugm(N, self.config, "file loaded %d bytes from %s, zstd magic: %s",
164 #data, path, is_zstd and "yes" or "no")
165
166 -- Notify hyperscan cache that this file is known (for cleanup tracking)
167 rspamd_util.hyperscan_notice_known(path)
168
169 if is_zstd then
170 -- Decompress the data
171 local decompress_err, decompressed = rspamd_util.zstd_decompress(data)
172 if not decompress_err and decompressed then
173 lua_util.debugm(N, self.config, "file decompressed %d -> %d bytes from %s (compression ratio: %.1f%%)",
174 #data, #decompressed, path, (1 - #data / #decompressed) * 100)
175 callback(nil, decompressed)
176 else
177 lua_util.debugm(N, self.config, "file decompression failed for %s: %s", path, decompress_err or "unknown error")
178 callback(decompress_err or "decompression failed", nil)
179 end
180 else
181 -- Data is not compressed, return as-is
182 if expected_compressed then
183 lua_util.debugm(N, self.config, "file %s has .zst extension but no zstd magic - treating as uncompressed", path)
184 end
185 callback(nil, data)
186 end
187 end
188
189 function file_backend:store(cache_key, platform_id, data, _ttl, callback)
190 local path = self:_get_path(cache_key, platform_id)
191
192 lua_util.debugm(N, self.config, "file store to: %s, original size: %d bytes, compression: %s",
193 path, #data, self.use_compression and "enabled" or "disabled")
194
195 self:_ensure_dir(path)
196
197 local store_data = data
198 -- Compress if enabled
199 if self.use_compression then
200 local compressed, compress_err = rspamd_util.zstd_compress(data)
201 if compressed then
202 lua_util.debugm(N, self.config, "file compressed %d -> %d bytes (%.1f%% size reduction) for %s",
203 #data, #compressed, (1 - #compressed / #data) * 100, path)
204 store_data = compressed
205 else
206 logger.warnx(N, "compression failed: %s, storing uncompressed to %s", compress_err, path)
207 end
208 end
209
210 -- Write to temp file first, then rename atomically
211 local tmp_path = path .. ".tmp." .. rspamd_util.random_hex(8)
212 -- store_data can be string or rspamd_text userdata
213 local ok, write_err
214 if type(store_data) == "userdata" and store_data.save_in_file then
215 ok, write_err = store_data:save_in_file(tmp_path)
216 else
217 local f, err = io.open(tmp_path, "wb")
218 if not f then
219 callback(err or "open failed")
220 return
221 end
222 ok, write_err = f:write(store_data)
223 f:close()
224 end
225 if not ok then
226 os.remove(tmp_path)
227 callback(write_err or "write failed")
228 return
229 end
230
231 do
232 local renamed, rename_err = os.rename(tmp_path, path)
233 if renamed then
234 lua_util.debugm(N, self.config, "stored %d bytes to %s", #store_data, path)
235 -- Notify hyperscan cache that this file is known (for cleanup tracking)
236 rspamd_util.hyperscan_notice_known(path)
237 -- Remove old file with opposite extension if it exists (migration cleanup)
238 local old_ext = self.use_compression and '.hs' or '.hs.zst'
239 local old_path = self:_get_path(cache_key, platform_id, old_ext)
240 local old_err, old_stat = rspamd_util.stat(old_path)
241 if not old_err and old_stat then
242 local removed = os.remove(old_path)
243 if removed then
244 lua_util.debugm(N, self.config, "removed old cache file %s (migrated to %s)", old_path, path)
245 end
246 end
247 callback(nil)
248 else
249 os.remove(tmp_path)
250 callback(rename_err or "rename failed")
251 end
252 end
253 end
254
255 function file_backend:delete(cache_key, platform_id, callback)
256 -- Try to delete both compressed and uncompressed versions
257 local deleted_any = false
258 local last_err = nil
259
260 for _, ext in ipairs({'.hs', '.hs.zst'}) do
261 local path = self:_get_path(cache_key, platform_id, ext)
262 local stat_err, stat = rspamd_util.stat(path)
263 if not stat_err and stat then
264 local ok, err = os.remove(path)
265 if ok then
266 lua_util.debugm(N, self.config, "deleted %s", path)
267 deleted_any = true
268 else
269 last_err = err
270 end
271 end
272 end
273
274 if deleted_any then
275 callback(nil)
276 else
277 callback(last_err or "file not found")
278 end
279 end
280
281 function file_backend:exists_sync(cache_key, platform_id)
282 local path, is_compressed = self:_find_existing_path(cache_key, platform_id)
283 if path then
284 lua_util.debugm(N, self.config, "file sync exists check: %s found (compressed: %s)",
285 path, is_compressed and "yes" or "no")
286 return true, nil
287 else
288 local expected_path = self:_get_path(cache_key, platform_id)
289 lua_util.debugm(N, self.config, "file sync exists check: %s not found (checked both extensions)", expected_path)
290 return false, nil
291 end
292 end
293
294 function file_backend:save_async(cache_key, platform_id, data, callback)
295 self:store(cache_key, platform_id, data, nil, callback)
296 end
297
298 function file_backend:load_async(cache_key, platform_id, callback)
299 self:load(cache_key, platform_id, callback)
300 end
301
302 function file_backend:exists_async(cache_key, platform_id, callback)
303 local exists, err = self:exists_sync(cache_key, platform_id)
304 callback(err, exists)
305 end
306
307 function file_backend:load_sync(cache_key, platform_id)
308 local path, expected_compressed = self:_find_existing_path(cache_key, platform_id)
309
310 if not path then
311 local expected_path = self:_get_path(cache_key, platform_id)
312 lua_util.debugm(N, self.config, "file sync load failed: %s not found (checked both extensions)", expected_path)
313 return nil, "file not found"
314 end
315
316 lua_util.debugm(N, self.config, "file sync load from: %s (expected compressed: %s)",
317 path, expected_compressed and "yes" or "no")
318
319 local f, err = io.open(path, "rb")
320 if not f then
321 lua_util.debugm(N, self.config, "file sync load failed from %s: %s", path, err or "open error")
322 return nil, err or "open error"
323 end
324 local data = f:read("*a")
325 f:close()
326 if not data then
327 lua_util.debugm(N, self.config, "file sync read failed from %s", path)
328 return nil, "read error"
329 end
330
331 -- Check if data is actually zstd compressed (magic byte verification)
332 local is_zstd = self:_is_zstd(data)
333 lua_util.debugm(N, self.config, "file sync loaded %d bytes from %s, zstd magic: %s",
334 #data, path, is_zstd and "yes" or "no")
335
336 -- Notify hyperscan cache that this file is known (for cleanup tracking)
337 rspamd_util.hyperscan_notice_known(path)
338
339 if is_zstd then
340 -- Decompress the data
341 local decompress_err, decompressed = rspamd_util.zstd_decompress(data)
342 if not decompress_err and decompressed then
343 lua_util.debugm(N, self.config, "file sync decompressed %d -> %d bytes from %s (compression ratio: %.1f%%)",
344 #data, #decompressed, path, (1 - #data / #decompressed) * 100)
345 return decompressed, nil
346 else
347 lua_util.debugm(N, self.config, "file sync decompression failed for %s: %s", path, decompress_err or "unknown error")
348 return nil, decompress_err or "decompression failed"
349 end
350 else
351 -- Data is not compressed, return as-is
352 if expected_compressed then
353 lua_util.debugm(N, self.config, "file %s has .zst extension but no zstd magic - treating as uncompressed", path)
354 end
355 return data, nil
356 end
357 end
358
359 function file_backend:save_sync(cache_key, platform_id, data)
360 local path = self:_get_path(cache_key, platform_id)
361 lua_util.debugm(N, self.config, "file sync save to: %s, original size: %d bytes, compression: %s",
362 path, #data, self.use_compression and "enabled" or "disabled")
363 self:_ensure_dir(path)
364
365 local store_data = data
366 -- Compress if enabled
367 if self.use_compression then
368 local compressed, compress_err = rspamd_util.zstd_compress(data)
369 if compressed then
370 lua_util.debugm(N, self.config, "file sync compressed %d -> %d bytes (%.1f%% size reduction) for %s",
371 #data, #compressed, (1 - #compressed / #data) * 100, path)
372 store_data = compressed
373 else
374 logger.warnx(N, "compression failed: %s, storing uncompressed to %s", compress_err, path)
375 end
376 end
377
378 local tmp_path = path .. ".tmp." .. rspamd_util.random_hex(8)
379 -- store_data can be string or rspamd_text userdata
380 local ok, write_err
381 if type(store_data) == "userdata" and store_data.save_in_file then
382 ok, write_err = store_data:save_in_file(tmp_path)
383 else
384 local f, err = io.open(tmp_path, "wb")
385 if not f then
386 lua_util.debugm(N, self.config, "file sync open failed for %s: %s", tmp_path, err)
387 return false, err
388 end
389 ok, write_err = f:write(store_data)
390 f:close()
391 end
392 if not ok then
393 lua_util.debugm(N, self.config, "file sync write failed to %s: %s", tmp_path, write_err)
394 os.remove(tmp_path)
395 return false, write_err
396 end
397
398 local renamed, rename_err = os.rename(tmp_path, path)
399 if not renamed then
400 lua_util.debugm(N, self.config, "file sync rename failed %s -> %s: %s", tmp_path, path, rename_err)
401 os.remove(tmp_path)
402 return false, rename_err
403 end
404
405 lua_util.debugm(N, self.config, "file sync stored %d bytes to %s", #store_data, path)
406
407 -- Notify hyperscan cache that this file is known (for cleanup tracking)
408 rspamd_util.hyperscan_notice_known(path)
409
410 -- Remove old file with opposite extension if it exists (migration cleanup)
411 local old_ext = self.use_compression and '.hs' or '.hs.zst'
412 local old_path = self:_get_path(cache_key, platform_id, old_ext)
413 local old_err, old_stat = rspamd_util.stat(old_path)
414 if not old_err and old_stat then
415 local removed = os.remove(old_path)
416 if removed then
417 lua_util.debugm(N, self.config, "removed old cache file %s (migrated to %s)", old_path, path)
418 end
419 end
420
421 return true, nil
422 end
423
424 -- Redis backend
425 local redis_backend = {}
426 redis_backend.__index = redis_backend
427
428 function redis_backend.new(config)
429 local self = setmetatable({}, redis_backend)
430
431 -- Redis config can be:
432 -- 1. In a 'redis' sub-section of hs_helper worker options
433 -- 2. Directly in the hs_helper worker options (servers, write_servers, etc.)
434 -- 3. Fallback to global 'redis' configuration section
435 local redis_opts = config.redis or config
436 self.redis_params = lua_redis.parse_redis_server(nil, redis_opts, true)
437 if not self.redis_params then
438 -- Fallback to global redis config
439 self.redis_params = lua_redis.parse_redis_server('redis')
440 end
441
442 if not self.redis_params then
443 logger.warnx(N, "redis backend: no redis configuration found in hs_helper worker or global redis section")
444 end
445
446 if config.ev_base and self.redis_params then
447 self.redis_params.ev_base = config.ev_base
448 end
449
450 if config.rspamd_config then
451 self.config = config.rspamd_config
452 else
453 self.config = config
454 end
455
456 -- Config options can be in redis sub-section or at top level
457 local opts = config.redis or config
458 self.default_ttl = opts.ttl or config.ttl or (86400 * 30) -- 30 days default
459 self.refresh_ttl = (opts.refresh_ttl ~= false) and (config.refresh_ttl ~= false)
460 self.use_compression = (opts.compression ~= false) and (config.compression ~= false)
461 -- Use different default prefix for compressed (rspamd_zhs) vs uncompressed (rspamd_hs)
462 local default_prefix = self.use_compression and 'rspamd_zhs' or 'rspamd_hs'
463 self.prefix = opts.prefix or config.prefix or default_prefix
464
465 lua_util.debugm(N, self.config, "redis backend config: prefix=%s, ttl=%s, refresh_ttl=%s, compression=%s",
466 self.prefix, self.default_ttl, self.refresh_ttl, self.use_compression)
467
468 return self
469 end
470
471 function redis_backend:_get_key(cache_key, platform_id)
472 return string.format("%s:%s:%s", self.prefix, platform_id, cache_key)
473 end
474
475 function redis_backend:exists(cache_key, platform_id, callback)
476 local key = self:_get_key(cache_key, platform_id)
477
478 if not self.redis_params then
479 callback("redis not configured", false, nil)
480 return
481 end
482
483 lua_util.debugm(N, self.config, "redis EXISTS check for key: %s", key)
484
485 local attrs = {
486 ev_base = self.redis_params.ev_base,
487 config = self.config,
488 callback = function(err, data)
489 if err then
490 lua_util.debugm(N, self.config, "redis EXISTS failed for key %s: %s", key, err)
491 callback(err, false, nil)
492 else
493 lua_util.debugm(N, self.config, "redis EXISTS result for key %s: %s", key, data == 1 and "found" or "not found")
494 callback(nil, data == 1, nil)
495 end
496 end
497 }
498
499 local req = {'EXISTS', key}
500 lua_redis.request(self.redis_params, attrs, req)
501 end
502
503 function redis_backend:load(cache_key, platform_id, callback)
504 local key = self:_get_key(cache_key, platform_id)
505
506 if not self.redis_params then
507 callback("redis not configured", nil)
508 return
509 end
510
511 -- Use GETEX to refresh TTL on read if enabled
512 local req
513 if self.refresh_ttl then
514 lua_util.debugm(N, self.config, "redis GETEX (with TTL refresh %d) for key: %s", self.default_ttl, key)
515 req = {'GETEX', key, 'EX', tostring(self.default_ttl)}
516 else
517 lua_util.debugm(N, self.config, "redis GET for key: %s", key)
518 req = {'GET', key}
519 end
520
521 local attrs = {
522 ev_base = self.redis_params.ev_base,
523 config = self.config,
524 callback = function(err, data)
525 if err then
526 lua_util.debugm(N, self.config, "redis GET failed for key %s: %s", key, err)
527 callback(err, nil)
528 elseif not data then
529 lua_util.debugm(N, self.config, "redis cache miss for key %s", key)
530 callback("not found", nil)
531 else
532 -- Decompress if needed
533 if self.use_compression then
534 -- Defensive check: ensure data is a string or text before decompression
535 local data_type = type(data)
536 if data_type ~= 'string' and data_type ~= 'userdata' then
537 logger.errx(self.config, "redis GET returned unexpected type %s for key %s, data=%s",
538 data_type, key, tostring(data))
539 callback("unexpected data type: " .. data_type, nil)
540 return
541 end
542 local ok, decompress_err, decompressed = pcall(rspamd_util.zstd_decompress, data)
543 if not ok then
544 -- pcall failed - decompress_err contains the error message
545 logger.errx(self.config, "zstd_decompress pcall failed for key %s: %s, data type=%s, len=%s",
546 key, tostring(decompress_err), data_type, data_type == 'string' and #data or 'N/A')
547 callback("decompression error: " .. tostring(decompress_err), nil)
548 return
549 end
550 if not decompress_err and decompressed then
551 lua_util.debugm(N, self.config, "redis loaded and decompressed %d -> %d bytes from key %s (compression ratio: %.1f%%)",
552 #data, #decompressed, key, (1 - #data / #decompressed) * 100)
553 callback(nil, decompressed)
554 else
555 lua_util.debugm(N, self.config, "redis decompression failed for key %s: %s", key, decompress_err)
556 callback(decompress_err or "decompression failed", nil)
557 end
558 else
559 lua_util.debugm(N, self.config, "redis loaded %d bytes (uncompressed) from key %s", #data, key)
560 callback(nil, data)
561 end
562 end
563 end
564 }
565
566 lua_redis.request(self.redis_params, attrs, req)
567 end
568
569 function redis_backend:store(cache_key, platform_id, data, ttl, callback)
570 local key = self:_get_key(cache_key, platform_id)
571 local actual_ttl = ttl or self.default_ttl
572
573 if not self.redis_params then
574 callback("redis not configured")
575 return
576 end
577
578 lua_util.debugm(N, self.config, "redis SETEX for key: %s, original size: %d bytes, TTL: %d, compression: %s",
579 key, #data, actual_ttl, self.use_compression and "enabled" or "disabled")
580
581 local store_data = data
582 -- Compress if enabled
583 if self.use_compression then
584 local compressed, compress_err = rspamd_util.zstd_compress(data)
585 if compressed then
586 lua_util.debugm(N, self.config, "redis compressed %d -> %d bytes (%.1f%% size reduction) for key %s",
587 #data, #compressed, (1 - #compressed / #data) * 100, key)
588 store_data = compressed
589 else
590 logger.warnx(N, "compression failed: %s, storing uncompressed", compress_err)
591 end
592 end
593
594 local attrs = {
595 ev_base = self.redis_params.ev_base,
596 config = self.config,
597 is_write = true,
598 callback = function(err)
599 if err then
600 lua_util.debugm(N, self.config, "redis SETEX failed for key %s: %s", key, err)
601 callback(err)
602 else
603 lua_util.debugm(N, self.config, "redis stored %d bytes to key %s with TTL %d",
604 #store_data, key, actual_ttl)
605 callback(nil)
606 end
607 end
608 }
609
610 local req = {'SETEX', key, tostring(actual_ttl), store_data}
611 lua_redis.request(self.redis_params, attrs, req)
612 end
613
614 function redis_backend:delete(cache_key, platform_id, callback)
615 local key = self:_get_key(cache_key, platform_id)
616
617 if not self.redis_params then
618 callback("redis not configured")
619 return
620 end
621
622 lua_util.debugm(N, self.config, "redis DEL for key: %s", key)
623
624 local attrs = {
625 ev_base = self.redis_params.ev_base,
626 config = self.config,
627 is_write = true,
628 callback = function(err)
629 if err then
630 lua_util.debugm(N, self.config, "redis DEL failed for key %s: %s", key, err)
631 callback(err)
632 else
633 lua_util.debugm(N, self.config, "redis deleted key %s", key)
634 callback(nil)
635 end
636 end
637 }
638
639 local req = {'DEL', key}
640 lua_redis.request(self.redis_params, attrs, req)
641 end
642
643 function redis_backend:save_async(cache_key, platform_id, data, callback)
644 self:store(cache_key, platform_id, data, nil, callback)
645 end
646
647 function redis_backend:load_async(cache_key, platform_id, callback)
648 self:load(cache_key, platform_id, callback)
649 end
650
651 function redis_backend:exists_async(cache_key, platform_id, callback)
652 self:exists(cache_key, platform_id, callback)
653 end
654
655 -- HTTP backend
656 local http_backend = {}
657 http_backend.__index = http_backend
658
659 function http_backend.new(config)
660 local self = setmetatable({}, http_backend)
661 -- Store config for logging context
662 self.config = config.rspamd_config or config
663
664 -- HTTP config can be in 'http' sub-section or at top level
665 local opts = config.http or config
666 self.base_url = opts.base_url or opts.url
667 self.timeout = opts.timeout or config.timeout or 30
668 self.auth_header = opts.auth_header or config.auth_header
669 self.auth_value = opts.auth_value or config.auth_value
670 self.use_compression = (opts.compression ~= false) and (config.compression ~= false)
671 return self
672 end
673
674 function http_backend:_get_url(cache_key, platform_id)
675 return string.format("%s/%s/%s", self.base_url, platform_id, cache_key)
676 end
677
678 function http_backend:_get_headers()
679 local headers = {}
680 if self.auth_header and self.auth_value then
681 headers[self.auth_header] = self.auth_value
682 end
683 return headers
684 end
685
686 function http_backend:exists(cache_key, platform_id, callback)
687 local url = self:_get_url(cache_key, platform_id)
688
689 lua_util.debugm(N, self.config, "http HEAD check for url: %s", url)
690
691 rspamd_http.request({
692 url = url,
693 method = 'HEAD',
694 headers = self:_get_headers(),
695 timeout = self.timeout,
696 callback = function(err, code, _, headers)
697 if err then
698 lua_util.debugm(N, self.config, "http HEAD failed for %s: %s", url, err)
699 callback(err, false, nil)
700 elseif code == 200 then
701 local size = headers and headers['content-length']
702 lua_util.debugm(N, self.config, "http HEAD found %s, size: %s", url, size or "unknown")
703 callback(nil, true, { size = tonumber(size) })
704 else
705 lua_util.debugm(N, self.config, "http HEAD not found %s (code: %d)", url, code)
706 callback(nil, false, nil)
707 end
708 end
709 })
710 end
711
712 function http_backend:load(cache_key, platform_id, callback)
713 local url = self:_get_url(cache_key, platform_id)
714
715 lua_util.debugm(N, self.config, "http GET for url: %s", url)
716
717 rspamd_http.request({
718 url = url,
719 method = 'GET',
720 headers = self:_get_headers(),
721 timeout = self.timeout,
722 callback = function(err, code, body, headers)
723 if err then
724 lua_util.debugm(N, self.config, "http GET failed for %s: %s", url, err)
725 callback(err, nil)
726 elseif code == 200 and body then
727 -- Check if content is compressed
728 local content_encoding = headers and headers['content-encoding']
729 if content_encoding == 'zstd' or self.use_compression then
730 local decompress_err, decompressed = rspamd_util.zstd_decompress(body)
731 if not decompress_err and decompressed then
732 lua_util.debugm(N, self.config, "http loaded and decompressed %d -> %d bytes from %s",
733 #body, #decompressed, url)
734 callback(nil, decompressed)
735 else
736 lua_util.debugm(N, self.config, "http loaded %d bytes (no decompression) from %s", #body, url)
737 callback(nil, body)
738 end
739 else
740 lua_util.debugm(N, self.config, "http loaded %d bytes from %s", #body, url)
741 callback(nil, body)
742 end
743 elseif code == 404 then
744 lua_util.debugm(N, self.config, "http cache miss (404) for %s", url)
745 callback("not found", nil)
746 else
747 lua_util.debugm(N, self.config, "http GET failed for %s: HTTP %d", url, code)
748 callback(string.format("HTTP %d", code), nil)
749 end
750 end
751 })
752 end
753
754 function http_backend:store(cache_key, platform_id, data, ttl, callback)
755 local url = self:_get_url(cache_key, platform_id)
756 local headers = self:_get_headers()
757
758 lua_util.debugm(N, self.config, "http PUT for url: %s, original size: %d bytes, compression: %s",
759 url, #data, self.use_compression and "enabled" or "disabled")
760
761 local store_data = data
762 if self.use_compression then
763 local compressed = rspamd_util.zstd_compress(data)
764 if compressed then
765 lua_util.debugm(N, self.config, "http compressed %d -> %d bytes (%.1f%% size reduction) for %s",
766 #data, #compressed, (1 - #compressed / #data) * 100, url)
767 store_data = compressed
768 headers['Content-Encoding'] = 'zstd'
769 end
770 end
771
772 if ttl then
773 headers['X-TTL'] = tostring(ttl)
774 end
775
776 rspamd_http.request({
777 url = url,
778 method = 'PUT',
779 headers = headers,
780 body = store_data,
781 timeout = self.timeout,
782 callback = function(err, code)
783 if err then
784 lua_util.debugm(N, self.config, "http PUT failed for %s: %s", url, err)
785 callback(err)
786 elseif code >= 200 and code < 300 then
787 lua_util.debugm(N, self.config, "http stored %d bytes to %s", #store_data, url)
788 callback(nil)
789 else
790 lua_util.debugm(N, self.config, "http PUT failed for %s: HTTP %d", url, code)
791 callback(string.format("HTTP %d", code))
792 end
793 end
794 })
795 end
796
797 function http_backend:delete(cache_key, platform_id, callback)
798 local url = self:_get_url(cache_key, platform_id)
799
800 lua_util.debugm(N, self.config, "http DELETE for url: %s", url)
801
802 rspamd_http.request({
803 url = url,
804 method = 'DELETE',
805 headers = self:_get_headers(),
806 timeout = self.timeout,
807 callback = function(err, code)
808 if err then
809 lua_util.debugm(N, self.config, "http DELETE failed for %s: %s", url, err)
810 callback(err)
811 elseif code >= 200 and code < 300 or code == 404 then
812 lua_util.debugm(N, self.config, "http deleted %s", url)
813 callback(nil)
814 else
815 lua_util.debugm(N, self.config, "http DELETE failed for %s: HTTP %d", url, code)
816 callback(string.format("HTTP %d", code))
817 end
818 end
819 })
820 end
821
822 -- Backend factory
823
824 -- Create a backend instance based on configuration
825 -- @param config table with:
826 -- - backend: "file"|"redis"|"http" (default: "file")
827 -- - cache_dir: directory for file backend
828 -- - redis: redis configuration table
829 -- - http: http configuration table
830 -- @return backend instance
831 function exports.create_backend(config)
832 local backend_type = config.backend or config.cache_backend or 'file'
833
834 local cfg = config.rspamd_config or config
835 lua_util.debugm(N, cfg, "creating hyperscan cache backend: %s", backend_type)
836
837 -- Always pass full config - backends will extract what they need
838 -- (config contains ev_base, rspamd_config at top level, plus optional
839 -- redis/http sub-sections for backend-specific settings)
840 if backend_type == 'file' then
841 local be = file_backend.new(config)
842 lua_util.debugm(N, be.config, "file backend created, cache_dir: %s, compression: %s",
843 be.cache_dir or "not set", be.use_compression and "enabled" or "disabled")
844 return be
845 elseif backend_type == 'redis' then
846 local be = redis_backend.new(config)
847 if be.redis_params then
848 lua_util.debugm(N, be.config, "redis backend created, prefix: %s, compression: %s",
849 be.prefix, be.use_compression and "enabled" or "disabled")
850 else
851 logger.errx(N, "redis backend created but no redis params - operations will fail!")
852 end
853 return be
854 elseif backend_type == 'http' then
855 local be = http_backend.new(config)
856 lua_util.debugm(N, be.config, "http backend created, base_url: %s", be.base_url or "not set")
857 return be
858 else
859 logger.errx(N, "unknown hyperscan cache backend: %s, falling back to file", backend_type)
860 return file_backend.new(config)
861 end
862 end
863
864 -- Export individual backend constructors for direct use
865 exports.file_backend = file_backend
866 exports.redis_backend = redis_backend
867 exports.http_backend = http_backend
868
869 return exports