Debian Codegraph @

lualib/llm_context.lua

raw
1 --[[
2 Context management for LLM-based spam detection
3
4 Provides:
5 - fetch(task, redis_params, opts, callback, debug_module): load context JSON from Redis and format prompt snippet
6 - update_after_classification(task, redis_params, opts, result, sel_part, debug_module): update context after LLM result
7
8 Opts (all optional, safe defaults applied):
9 enabled: boolean
10 level: 'user' | 'domain' | 'esld' (scope for context key)
11 key_prefix: string (prefix before scope)
12 key_suffix: string (suffix after identity)
13 max_messages: number (sliding window size)
14 message_ttl: seconds
15 ttl: seconds (Redis key TTL)
16 top_senders: number (how many to keep in top_senders)
17 summary_max_chars: number (truncate stored text)
18 flagged_phrases: array of strings (case-insensitive match)
19 last_labels_count: number
20
21 debug_module: optional string, module name for debug logging (default: 'llm_context')
22 ]]
23
24 local M = {}
25
26 local lua_redis = require "lua_redis"
27 local lua_util = require "lua_util"
28 local rspamd_logger = require "rspamd_logger"
29 local ucl = require "ucl"
30 local rspamd_util = require "rspamd_util"
31 local llm_common = require "llm_common"
32
33 local EMPTY = {}
34
35 local DEFAULTS = {
36 enabled = false,
37 level = 'user',
38 key_prefix = 'user',
39 key_suffix = 'mail_context',
40 max_messages = 40,
41 min_messages = 5, -- minimum messages in context before injecting into prompt
42 message_ttl = 14 * 24 * 3600,
43 ttl = 30 * 24 * 3600,
44 top_senders = 5,
45 summary_max_chars = 512,
46 flagged_phrases = {
47 'reset your password',
48 'click here to verify',
49 'confirm your account',
50 'urgent invoice',
51 'wire transfer',
52 },
53 last_labels_count = 10,
54 }
55
56 local function to_seconds(v)
57 if type(v) == 'number' then return v end
58 return tonumber(v) or 0
59 end
60
61 local function get_domain_from_addr(addr)
62 if not addr then return nil end
63 return string.match(addr, '.*@(.+)')
64 end
65
66 -- Determine our user/domain - same identity for both incoming and outgoing mail
67 local function get_our_identity(task, scope)
68 -- For outgoing mail: authenticated user or sender from local network
69 -- For incoming mail: principal recipient
70 local user = task:get_user()
71 local ip = task:get_ip()
72 local is_outgoing = user or (ip and ip:is_local())
73
74 local identity
75 if scope == 'user' then
76 if is_outgoing then
77 -- Outgoing: use sender (authenticated user or from address)
78 identity = user or task:get_reply_sender()
79 if not identity then
80 local from = ((task:get_from('smtp') or EMPTY)[1] or EMPTY)['addr']
81 identity = from
82 end
83 else
84 -- Incoming: use recipient
85 identity = task:get_principal_recipient()
86 end
87 elseif scope == 'domain' then
88 if is_outgoing then
89 -- Outgoing: domain of sender
90 if user then
91 identity = get_domain_from_addr(user)
92 end
93 if not identity then
94 identity = ((task:get_from('smtp') or EMPTY)[1] or EMPTY)['domain']
95 end
96 else
97 -- Incoming: domain of recipient
98 local rcpt = task:get_principal_recipient()
99 identity = get_domain_from_addr(rcpt)
100 end
101 elseif scope == 'esld' then
102 if is_outgoing then
103 -- Outgoing: eSLD of sender domain
104 local d
105 if user then
106 d = get_domain_from_addr(user)
107 end
108 if not d then
109 d = ((task:get_from('smtp') or EMPTY)[1] or EMPTY)['domain']
110 end
111 if d then identity = rspamd_util.get_tld(d) end
112 else
113 -- Incoming: eSLD of recipient domain
114 local rcpt = task:get_principal_recipient()
115 local d = get_domain_from_addr(rcpt)
116 if d then
117 identity = rspamd_util.get_tld(d)
118 end
119 end
120 end
121
122 return identity
123 end
124
125 local function compute_identity(task, opts, debug_module)
126 local N = debug_module or 'llm_context'
127 local scope = opts.level or DEFAULTS.level
128 local identity = get_our_identity(task, scope)
129
130 if not identity or identity == '' then
131 return nil
132 end
133
134 -- Log direction for debugging
135 local user = task:get_user()
136 local ip = task:get_ip()
137 local is_outgoing = user or (ip and ip:is_local())
138 lua_util.debugm(N, task, 'computed identity for %s (%s): %s',
139 scope, is_outgoing and 'outgoing' or 'incoming', tostring(identity))
140
141 local key_prefix = opts.key_prefix or DEFAULTS.key_prefix
142 local key_suffix = opts.key_suffix or DEFAULTS.key_suffix
143 local key = string.format('%s:%s:%s', key_prefix, identity, key_suffix)
144
145 return {
146 scope = scope,
147 identity = identity,
148 key = key,
149 }
150 end
151
152 local function parse_json(data)
153 if not data then return nil end
154 -- Redis can return userdata nil or empty string
155 if type(data) == 'userdata' then
156 data = tostring(data)
157 end
158 if type(data) ~= 'string' or data == '' then
159 return nil
160 end
161 local parser = ucl.parser()
162 local ok, err = parser:parse_text(data)
163 if not ok then return nil, err end
164 return parser:get_object()
165 end
166
167 local function encode_json(obj)
168 return ucl.to_format(obj, 'json-compact', true)
169 end
170
171 local function now()
172 return os.time()
173 end
174
175 local function truncate_text(txt, limit)
176 if not txt then return '' end
177 if #txt <= limit then return txt end
178 return txt:sub(1, limit)
179 end
180
181 local function has_flag(flags, flag_name)
182 if type(flags) ~= 'table' then return false end
183 for _, f in ipairs(flags) do
184 if f == flag_name then return true end
185 end
186 return false
187 end
188
189 local function extract_keywords(text_part, limit)
190 if not text_part then return {} end
191 local words = text_part:get_words('full')
192 if not words or #words == 0 then return {} end
193
194 local counts = {}
195 for _, w in ipairs(words) do
196 local norm_word = w[2] or '' -- normalized
197 local flags = w[4] or {}
198 -- Skip stop words, too short, or non-text
199 if not has_flag(flags, 'stop_word') and #norm_word > 2 and has_flag(flags, 'text') then
200 counts[norm_word] = (counts[norm_word] or 0) + 1
201 end
202 end
203
204 local arr = {}
205 for word, cnt in pairs(counts) do
206 table.insert(arr, { w = word, c = cnt })
207 end
208 table.sort(arr, function(a, b)
209 if a.c == b.c then return a.w < b.w end
210 return a.c > b.c
211 end)
212
213 local res = {}
214 for i = 1, math.min(limit or 12, #arr) do
215 table.insert(res, arr[i].w)
216 end
217 return res
218 end
219
220 local function safe_array(arr)
221 if type(arr) ~= 'table' then return {} end
222 return arr
223 end
224
225 local function build_message_summary(task, sel_part, opts)
226 local model_cfg = { max_tokens = 256 }
227 local content_tbl
228 if sel_part then
229 local itbl = llm_common.build_llm_input(task, {
230 max_tokens = model_cfg.max_tokens,
231 reply_trim_mode = opts.reply_trim_mode,
232 })
233 content_tbl = itbl
234 else
235 content_tbl = llm_common.build_llm_input(task, {
236 max_tokens = model_cfg.max_tokens,
237 reply_trim_mode = opts.reply_trim_mode,
238 })
239 end
240 if type(content_tbl) ~= 'table' then
241 return nil
242 end
243 local txt = content_tbl.text or ''
244 local summary_max = opts.summary_max_chars or DEFAULTS.summary_max_chars
245 local msg = {
246 from = content_tbl.from or ((task:get_from('smtp') or EMPTY)[1] or EMPTY)['addr'],
247 subject = content_tbl.subject or '',
248 ts = now(),
249 keywords = extract_keywords(sel_part, 12),
250 }
251 if txt and #txt > 0 then
252 msg.text = truncate_text(txt, summary_max)
253 end
254 return msg
255 end
256
257 local function trim_messages(recent_messages, max_messages, min_ts)
258 local res = {}
259 for _, m in ipairs(recent_messages) do
260 if not min_ts or (m.ts and m.ts >= min_ts) then
261 table.insert(res, m)
262 end
263 end
264 table.sort(res, function(a, b)
265 local ta = a.ts or 0
266 local tb = b.ts or 0
267 return ta > tb
268 end)
269 while #res > max_messages do
270 table.remove(res)
271 end
272 return res
273 end
274
275 local function recompute_top_senders(sender_counts, limit_n)
276 local arr = {}
277 for s, c in pairs(sender_counts or {}) do
278 table.insert(arr, { s = s, c = c })
279 end
280 table.sort(arr, function(a, b)
281 if a.c == b.c then return a.s < b.s end
282 return a.c > b.c
283 end)
284 local res = {}
285 for i = 1, math.min(limit_n, #arr) do
286 table.insert(res, arr[i].s)
287 end
288 return res
289 end
290
291 local function ensure_defaults(ctx)
292 if type(ctx) ~= 'table' then ctx = {} end
293 ctx.recent_messages = safe_array(ctx.recent_messages)
294 ctx.top_senders = safe_array(ctx.top_senders)
295 ctx.flagged_phrases = safe_array(ctx.flagged_phrases)
296 ctx.last_spam_labels = safe_array(ctx.last_spam_labels)
297 ctx.sender_counts = ctx.sender_counts or {}
298 return ctx
299 end
300
301 local function contains_ci(haystack, needle)
302 if not haystack or not needle then return false end
303 return string.find(string.lower(haystack), string.lower(needle), 1, true) ~= nil
304 end
305
306 local function update_flagged_phrases(ctx, text_part, opts)
307 local phrases = opts.flagged_phrases or DEFAULTS.flagged_phrases
308 if not text_part then return end
309 local words = text_part:get_words('norm')
310 if not words or #words == 0 then return end
311 local text_lower = table.concat(words, ' ')
312 for _, p in ipairs(phrases) do
313 if contains_ci(text_lower, p) then
314 local present = false
315 for _, e in ipairs(ctx.flagged_phrases) do
316 if string.lower(e) == string.lower(p) then
317 present = true
318 break
319 end
320 end
321 if not present then
322 table.insert(ctx.flagged_phrases, p)
323 end
324 end
325 end
326 end
327
328 local function to_bullets_recent(recent_messages, limit_n)
329 local lines = {}
330 local n = math.min(limit_n, #recent_messages)
331 for i = 1, n do
332 local m = recent_messages[i]
333 local from = m.from or m.sender or ''
334 local subj = m.subject or ''
335 table.insert(lines, string.format('- %s: %s', from, subj))
336 end
337 return table.concat(lines, '\n')
338 end
339
340 local function join_list(arr)
341 if not arr or #arr == 0 then return '' end
342 return table.concat(arr, ', ')
343 end
344
345 local function format_context_prompt(ctx, task)
346 local bullets = to_bullets_recent(ctx.recent_messages or {}, 5)
347 local top_senders = join_list(ctx.top_senders or {})
348 local flagged = join_list(ctx.flagged_phrases or {})
349 local spam_types = join_list(ctx.last_spam_labels or {})
350
351 -- Check if current sender is known
352 local sender_frequency = 'new'
353 if task then
354 local from = ((task:get_from('smtp') or EMPTY)[1] or EMPTY)['addr']
355 if from and ctx.sender_counts and ctx.sender_counts[from] then
356 local count = ctx.sender_counts[from]
357 if count >= 10 then
358 sender_frequency = 'frequent'
359 elseif count >= 3 then
360 sender_frequency = 'known'
361 else
362 sender_frequency = 'occasional'
363 end
364 end
365 end
366
367 local parts = {}
368 table.insert(parts, 'User recent correspondence summary:')
369 if bullets ~= '' then
370 table.insert(parts, bullets)
371 else
372 table.insert(parts, '- (no recent messages)')
373 end
374 table.insert(parts, string.format('Top senders in mailbox: %s', top_senders))
375 if flagged ~= '' then
376 table.insert(parts, string.format('Recently flagged suspicious phrases: %s', flagged))
377 end
378 if spam_types ~= '' then
379 table.insert(parts, string.format('Last detected spam types: %s', spam_types))
380 end
381 table.insert(parts, string.format('Current sender: %s', sender_frequency))
382
383 return table.concat(parts, '\n')
384 end
385
386 function M.fetch(task, redis_params, opts, callback, debug_module)
387 local N = debug_module or 'llm_context'
388 opts = lua_util.override_defaults(DEFAULTS, opts or {})
389 if not opts.enabled then
390 callback(nil, nil, nil)
391 return
392 end
393 if not redis_params then
394 callback('no redis', nil, nil)
395 return
396 end
397
398 local ident = compute_identity(task, opts, N)
399 if not ident then
400 lua_util.debugm(N, task, 'no identity computed, skipping context')
401 callback('no identity', nil, nil)
402 return
403 end
404
405 lua_util.debugm(N, task, 'fetching context for %s: %s',
406 tostring(ident.scope), tostring(ident.identity))
407
408 local function on_get(err, data)
409 if err then
410 rspamd_logger.errx(task, 'llm_context: get failed: %s', tostring(err))
411 callback(err, nil, nil)
412 return
413 end
414 local ctx
415 if data then
416 lua_util.debugm(N, task, 'got context data from redis, parsing')
417 ctx = ensure_defaults(select(1, parse_json(data)) or {})
418 else
419 lua_util.debugm(N, task, 'no context data in redis, using empty')
420 ctx = ensure_defaults({})
421 end
422
423 -- Check if context has enough messages for warm-up
424 local min_msgs = opts.min_messages or DEFAULTS.min_messages
425 local msg_count = #(ctx.recent_messages or {})
426 if msg_count < min_msgs then
427 lua_util.debugm(N, task, 'context has only %s messages (min: %s), not injecting into prompt',
428 tostring(msg_count), tostring(min_msgs))
429 callback(nil, ctx, nil) -- return ctx but no prompt snippet
430 return
431 end
432
433 lua_util.debugm(N, task, 'context warm-up OK: %s messages, generating snippet',
434 tostring(msg_count))
435 local prompt_snippet = format_context_prompt(ctx, task)
436 callback(nil, ctx, prompt_snippet)
437 end
438
439 local ok = lua_redis.redis_make_request(task, redis_params, ident.key, false, on_get, 'GET', { ident.key })
440 if not ok then
441 callback('request not scheduled', nil, nil)
442 end
443 end
444
445 function M.update_after_classification(task, redis_params, opts, result, sel_part, debug_module)
446 local N = debug_module or 'llm_context'
447 opts = lua_util.override_defaults(DEFAULTS, opts or {})
448 if not opts.enabled then return end
449 if not redis_params then return end
450
451 local ident = compute_identity(task, opts, N)
452 if not ident then return end
453
454 local function on_get(err, data)
455 if err then
456 rspamd_logger.errx(task, 'llm_context: get for update failed: %s', tostring(err))
457 return
458 end
459 lua_util.debugm(N, task, 'updating context for %s: %s',
460 tostring(ident.scope), tostring(ident.identity))
461 local ctx = ensure_defaults(select(1, parse_json(data)) or {})
462
463 local msg = build_message_summary(task, sel_part, opts)
464 if msg then
465 table.insert(ctx.recent_messages, 1, msg)
466 local sender = msg.from or ''
467 if sender ~= '' then
468 ctx.sender_counts[sender] = (ctx.sender_counts[sender] or 0) + 1
469 end
470 update_flagged_phrases(ctx, sel_part, opts)
471 end
472
473 local min_ts = now() - to_seconds(opts.message_ttl)
474 ctx.recent_messages = trim_messages(ctx.recent_messages, opts.max_messages, min_ts)
475 ctx.top_senders = recompute_top_senders(ctx.sender_counts, opts.top_senders)
476
477 local labels = {}
478 if result then
479 if result.categories and type(result.categories) == 'table' then
480 for _, c in ipairs(result.categories) do table.insert(labels, tostring(c)) end
481 end
482 if result.probability then
483 if result.probability > 0.5 then
484 table.insert(labels, 'spam')
485 else
486 table.insert(labels, 'ham')
487 end
488 end
489 end
490 for _, l in ipairs(labels) do table.insert(ctx.last_spam_labels, 1, l) end
491 while #ctx.last_spam_labels > opts.last_labels_count do table.remove(ctx.last_spam_labels) end
492
493 ctx.updated_at = now()
494
495 local payload = encode_json(ctx)
496 local ttl = to_seconds(opts.ttl)
497 local expire_at = now() + ttl
498
499 -- Log what we're storing in context
500 lua_util.debugm(N, task,
501 'storing context for %s: %s messages, labels=%s, top_senders=%s, flagged=%s, payload_size=%s bytes, expiring at %s',
502 tostring(ident.identity or '(none)'),
503 tostring(#ctx.recent_messages),
504 table.concat(ctx.last_spam_labels or {}, ','),
505 table.concat(ctx.top_senders or {}, ','),
506 table.concat(ctx.flagged_phrases or {}, ','),
507 tostring(#payload),
508 os.date('%Y-%m-%d %H:%M:%S', expire_at))
509
510 if msg then
511 lua_util.debugm(N, task,
512 'added message: from=%s, subject=%s, keywords=%s',
513 tostring(msg.from or '(none)'),
514 tostring(msg.subject or '(none)'),
515 table.concat(msg.keywords or {}, ','))
516 end
517
518 local function on_set(set_err)
519 if set_err then
520 rspamd_logger.errx(task, 'llm_context: set failed: %s', tostring(set_err))
521 else
522 lua_util.debugm(N, task, 'context saved to redis: key=%s, ttl=%s, expiring at %s',
523 tostring(ident.key), tostring(ttl), os.date('%Y-%m-%d %H:%M:%S', expire_at))
524 end
525 end
526 local ok = lua_redis.redis_make_request(task, redis_params, ident.key, true, on_set, 'SETEX',
527 { ident.key, tostring(math.floor(ttl)), payload })
528 if not ok then
529 rspamd_logger.errx(task, 'llm_context: set request was not scheduled')
530 end
531 end
532
533 local ok = lua_redis.redis_make_request(task, redis_params, ident.key, false, on_get, 'GET', { ident.key })
534 if not ok then
535 rspamd_logger.errx(task, 'llm_context: initial get request was not scheduled')
536 end
537 end
538
539 return M