| 1 |
--[[ |
| 2 |
Common helpers for building LLM input content from a task |
| 3 |
]] -- |
| 4 |
|
| 5 |
local lua_util = require "lua_util" |
| 6 |
local lua_mime = require "lua_mime" |
| 7 |
local fun = require "fun" |
| 8 |
|
| 9 |
local M = {} |
| 10 |
local N = 'llm_common' |
| 11 |
|
| 12 |
local function get_meta_llm_content(task) |
| 13 |
local url_content = "Url domains: no urls found" |
| 14 |
if task:has_urls() then |
| 15 |
local urls = lua_util.extract_specific_urls { task = task, limit = 5, esld_limit = 1 } |
| 16 |
url_content = "Url domains: " .. table.concat(fun.totable(fun.map(function(u) |
| 17 |
return u:get_tld() or '' |
| 18 |
end, urls or {})), ', ') |
| 19 |
end |
| 20 |
|
| 21 |
local from_or_empty = ((task:get_from('mime') or {})[1] or {}) |
| 22 |
local from_name = from_or_empty.name or '' |
| 23 |
local from_addr = from_or_empty.addr or '' |
| 24 |
local from_content = string.format('From: %s <%s>', from_name, from_addr) |
| 25 |
|
| 26 |
return url_content, from_content |
| 27 |
end |
| 28 |
|
| 29 |
-- Build structured payload suitable for LLM embeddings and chat |
| 30 |
-- Returns: table { subject = <string>, from = <string>, url_domains = <string>, text = <rspamd_text|string> }, part |
| 31 |
function M.build_llm_input(task, opts) |
| 32 |
opts = opts or {} |
| 33 |
local subject = task:get_subject() or '' |
| 34 |
local url_content, from_content = get_meta_llm_content(task) |
| 35 |
|
| 36 |
-- Use extract_text_limited for content |
| 37 |
local max_tokens = tonumber(opts.max_tokens) or 1024 |
| 38 |
-- Rough estimation: 1 token approx 4 bytes (english), but let's be generous |
| 39 |
-- However, we can use max_words as a proxy for tokens? |
| 40 |
-- opts.max_tokens is typically tokens. |
| 41 |
-- Rspamd uses bytes for limit. |
| 42 |
-- Let's stick with what we had but using extract_text_limited |
| 43 |
|
| 44 |
local reply_trim_mode = opts.reply_trim_mode or 'replies' |
| 45 |
local trim_replies = false |
| 46 |
if reply_trim_mode == 'always' then |
| 47 |
trim_replies = true |
| 48 |
elseif reply_trim_mode == 'none' then |
| 49 |
trim_replies = false |
| 50 |
else |
| 51 |
trim_replies = task:has_header('In-Reply-To') or task:has_header('References') |
| 52 |
end |
| 53 |
|
| 54 |
local extraction_opts = { |
| 55 |
max_bytes = max_tokens * 6, -- Rough estimate |
| 56 |
max_words = max_tokens, -- Better estimate if available |
| 57 |
min_words = opts.min_words, |
| 58 |
strip_quotes = trim_replies, |
| 59 |
strip_reply_headers = trim_replies, |
| 60 |
smart_trim = trim_replies, |
| 61 |
} |
| 62 |
|
| 63 |
local res = lua_mime.extract_text_limited(task, extraction_opts) |
| 64 |
|
| 65 |
if not res or res.text == "" then |
| 66 |
lua_util.debugm(N, task, 'no text extracted') |
| 67 |
return nil, nil |
| 68 |
end |
| 69 |
|
| 70 |
return { |
| 71 |
subject = subject, |
| 72 |
from = from_content, |
| 73 |
url_domains = url_content, |
| 74 |
text = res.text, |
| 75 |
}, nil -- part is not available as before since we extract from task directly |
| 76 |
end |
| 77 |
|
| 78 |
-- Backwards-compat alias |
| 79 |
M.build_embedding_input = M.build_llm_input |
| 80 |
|
| 81 |
M.get_meta_llm_content = get_meta_llm_content |
| 82 |
|
| 83 |
return M |