Debian Codegraph @

lualib/llm_common.lua

raw
1 --[[
2 Common helpers for building LLM input content from a task
3 ]] --
4
5 local lua_util = require "lua_util"
6 local lua_mime = require "lua_mime"
7 local fun = require "fun"
8
9 local M = {}
10 local N = 'llm_common'
11
12 local function get_meta_llm_content(task)
13 local url_content = "Url domains: no urls found"
14 if task:has_urls() then
15 local urls = lua_util.extract_specific_urls { task = task, limit = 5, esld_limit = 1 }
16 url_content = "Url domains: " .. table.concat(fun.totable(fun.map(function(u)
17 return u:get_tld() or ''
18 end, urls or {})), ', ')
19 end
20
21 local from_or_empty = ((task:get_from('mime') or {})[1] or {})
22 local from_name = from_or_empty.name or ''
23 local from_addr = from_or_empty.addr or ''
24 local from_content = string.format('From: %s <%s>', from_name, from_addr)
25
26 return url_content, from_content
27 end
28
29 -- Build structured payload suitable for LLM embeddings and chat
30 -- Returns: table { subject = <string>, from = <string>, url_domains = <string>, text = <rspamd_text|string> }, part
31 function M.build_llm_input(task, opts)
32 opts = opts or {}
33 local subject = task:get_subject() or ''
34 local url_content, from_content = get_meta_llm_content(task)
35
36 -- Use extract_text_limited for content
37 local max_tokens = tonumber(opts.max_tokens) or 1024
38 -- Rough estimation: 1 token approx 4 bytes (english), but let's be generous
39 -- However, we can use max_words as a proxy for tokens?
40 -- opts.max_tokens is typically tokens.
41 -- Rspamd uses bytes for limit.
42 -- Let's stick with what we had but using extract_text_limited
43
44 local reply_trim_mode = opts.reply_trim_mode or 'replies'
45 local trim_replies = false
46 if reply_trim_mode == 'always' then
47 trim_replies = true
48 elseif reply_trim_mode == 'none' then
49 trim_replies = false
50 else
51 trim_replies = task:has_header('In-Reply-To') or task:has_header('References')
52 end
53
54 local extraction_opts = {
55 max_bytes = max_tokens * 6, -- Rough estimate
56 max_words = max_tokens, -- Better estimate if available
57 min_words = opts.min_words,
58 strip_quotes = trim_replies,
59 strip_reply_headers = trim_replies,
60 smart_trim = trim_replies,
61 }
62
63 local res = lua_mime.extract_text_limited(task, extraction_opts)
64
65 if not res or res.text == "" then
66 lua_util.debugm(N, task, 'no text extracted')
67 return nil, nil
68 end
69
70 return {
71 subject = subject,
72 from = from_content,
73 url_domains = url_content,
74 text = res.text,
75 }, nil -- part is not available as before since we extract from task directly
76 end
77
78 -- Backwards-compat alias
79 M.build_embedding_input = M.build_llm_input
80
81 M.get_meta_llm_content = get_meta_llm_content
82
83 return M