Debian Codegraph @

lualib/plugins/neural/providers/llm.lua

raw
1 --[[
2 LLM provider for neural feature fusion
3 Collects text from the most relevant part and requests embeddings from an LLM API.
4 Supports minimal OpenAI- and Ollama-compatible embedding endpoints.
5 ]] --
6
7 local rspamd_http = require "rspamd_http"
8 local rspamd_logger = require "rspamd_logger"
9 local ucl = require "ucl"
10 local neural_common = require "plugins/neural"
11 local lua_cache = require "lua_cache"
12 local llm_common = require "llm_common"
13 local lua_mime = require "lua_mime"
14
15 local N = "neural.llm"
16
17 local function select_text(task, opts)
18 return llm_common.build_llm_input(task, opts)
19 end
20
21 -- Detect primary language from the displayed text part
22 local function detect_language(task)
23 local part = lua_mime.get_displayed_text_part(task)
24 if part then
25 local lang = part:get_language()
26 if lang and lang ~= '' then
27 return lang
28 end
29 end
30 return nil
31 end
32
33 local function compose_llm_settings(pcfg, language)
34 local gpt_settings = rspamd_config:get_all_opt('gpt') or {}
35 -- Provider identity is pcfg.type=='llm'; backend type is specified via one of these keys
36 local llm_type = pcfg.llm_type or pcfg.api or pcfg.backend or gpt_settings.type or 'openai'
37 local model = pcfg.model or gpt_settings.model
38 local model_params = gpt_settings.model_parameters or {}
39 local model_cfg = model and model_params[model] or {}
40 local max_tokens = pcfg.max_tokens
41 if not max_tokens then
42 max_tokens = model_cfg.max_completion_tokens or model_cfg.max_tokens or gpt_settings.max_tokens
43 end
44 local timeout = pcfg.timeout or gpt_settings.timeout or 2.0
45 local url = pcfg.url
46 local api_key = pcfg.api_key or gpt_settings.api_key
47
48 -- Language-specific model/URL selection
49 -- Config format: language_models = { en = { model = "...", url = "..." }, ru = { model = "..." }, ... }
50 -- Or shorthand: language_models = { en = "model-name", ru = "model-name", ... }
51 local language_models = pcfg.language_models
52 if language and language_models then
53 local lang_cfg = language_models[language]
54 if lang_cfg then
55 if type(lang_cfg) == 'string' then
56 -- Shorthand: just model name
57 model = lang_cfg
58 elseif type(lang_cfg) == 'table' then
59 -- Full config: { model = "...", url = "...", api_key = "..." }
60 if lang_cfg.model then
61 model = lang_cfg.model
62 end
63 if lang_cfg.url then
64 url = lang_cfg.url
65 end
66 if lang_cfg.api_key then
67 api_key = lang_cfg.api_key
68 end
69 end
70 end
71 end
72
73 if not url then
74 if llm_type == 'openai' then
75 url = 'https://api.openai.com/v1/embeddings'
76 elseif llm_type == 'ollama' then
77 url = 'http://127.0.0.1:11434/api/embeddings'
78 end
79 end
80
81 return {
82 type = llm_type,
83 model = model,
84 max_tokens = max_tokens,
85 timeout = timeout,
86 url = url,
87 api_key = api_key,
88 cache_ttl = pcfg.cache_ttl or 86400,
89 cache_prefix = pcfg.cache_prefix or 'neural_llm',
90 cache_hash_len = pcfg.cache_hash_len or 32,
91 cache_use_hashing = (pcfg.cache_use_hashing ~= false),
92 -- Optional staged timeouts (inherit from global gpt if present)
93 connect_timeout = pcfg.connect_timeout or gpt_settings.connect_timeout,
94 ssl_timeout = pcfg.ssl_timeout or gpt_settings.ssl_timeout,
95 write_timeout = pcfg.write_timeout or gpt_settings.write_timeout,
96 read_timeout = pcfg.read_timeout or gpt_settings.read_timeout,
97 reply_trim_mode = pcfg.reply_trim_mode or gpt_settings.reply_trim_mode,
98 }
99 end
100
101 local function normalize_cache_key_input(input_string)
102 if type(input_string) == 'userdata' then
103 return input_string:str()
104 end
105 return tostring(input_string)
106 end
107
108 local function extract_embedding(llm_type, parsed)
109 if llm_type == 'openai' then
110 -- { data = [ { embedding = [...] } ] }
111 if parsed and parsed.data and parsed.data[1] and parsed.data[1].embedding then
112 return parsed.data[1].embedding
113 end
114 elseif llm_type == 'ollama' then
115 -- { embedding = [...] }
116 if parsed and parsed.embedding then
117 return parsed.embedding
118 end
119 end
120 return nil
121 end
122
123 neural_common.register_provider('llm', {
124 collect_async = function(task, ctx, cont)
125 local pcfg = ctx.config or {}
126
127 -- Detect language from displayed text part for model/URL selection
128 local language = detect_language(task)
129 local llm = compose_llm_settings(pcfg, language)
130
131 if not llm.model then
132 rspamd_logger.debugm(N, task, 'llm provider missing model; skip')
133 cont(nil)
134 return
135 end
136
137 -- Do not run embeddings on infer if ANN is not loaded for this set/profile
138 if ctx.phase == 'infer' then
139 local set_or_profile = ctx.profile or ctx.set
140 if not set_or_profile or not set_or_profile.ann then
141 rspamd_logger.debugm(N, task, 'skip llm on infer: ANN not loaded for current settings')
142 cont(nil)
143 return
144 end
145 end
146
147 local input_tbl = select_text(task, { reply_trim_mode = llm.reply_trim_mode })
148 if not input_tbl then
149 rspamd_logger.debugm(N, task, 'llm provider has no content to embed; skip')
150 cont(nil)
151 return
152 end
153
154 -- Build request input string: subject first (more valuable for spam detection),
155 -- then text content. Subject-first ensures it's always included even if text is truncated.
156 local input_string
157 if input_tbl.subject and input_tbl.subject ~= '' then
158 input_string = "Subject: " .. input_tbl.subject .. "\n" .. (input_tbl.text or '')
159 else
160 input_string = input_tbl.text or ''
161 end
162
163 local input_key = normalize_cache_key_input(input_string)
164 rspamd_logger.debugm(N, task, 'llm embedding request: model=%s url=%s lang=%s len=%s',
165 tostring(llm.model), tostring(llm.url), tostring(language or 'unknown'), tostring(#input_key))
166
167 local body
168 if llm.type == 'openai' then
169 body = { model = llm.model, input = input_string }
170 elseif llm.type == 'ollama' then
171 body = { model = llm.model, prompt = input_string }
172 else
173 rspamd_logger.debugm(N, task, 'unsupported llm type: %s', llm.type)
174 cont(nil)
175 return
176 end
177
178 -- Redis cache: hash the final input string only
179 local cache_ctx = lua_cache.create_cache_context(neural_common.redis_params, {
180 cache_prefix = llm.cache_prefix,
181 cache_ttl = llm.cache_ttl,
182 cache_format = 'messagepack',
183 cache_hash_len = llm.cache_hash_len,
184 cache_use_hashing = llm.cache_use_hashing,
185 }, N)
186
187 -- Use raw key and allow cache module to hash/shorten it per context
188 -- Include language in cache key for proper separation
189 local key = string.format('%s:%s:%s:%s', llm.type, llm.model or 'model', language or 'unk', input_key)
190
191 local function finish_with_vec(vec)
192 if type(vec) == 'table' and #vec > 0 then
193 local meta = {
194 name = pcfg.name or 'llm',
195 type = 'llm',
196 dim = #vec,
197 weight = ctx.weight or 1.0,
198 model = llm.model,
199 provider = llm.type,
200 language = language,
201 }
202 rspamd_logger.debugm(N, task, 'llm embedding result: dim=%s lang=%s', #vec, language or 'unknown')
203 cont(vec, meta)
204 else
205 rspamd_logger.debugm(N, task, 'llm embedding result: empty')
206 cont(nil)
207 end
208 end
209
210 local function http_cb(err, code, resp, _)
211 if err then
212 rspamd_logger.debugm(N, task, 'llm http error: %s', err)
213 cont(nil)
214 return
215 end
216 if code ~= 200 or not resp then
217 rspamd_logger.debugm(N, task, 'llm bad http code: %s', code)
218 cont(nil)
219 return
220 end
221
222 local parser = ucl.parser()
223 local ok, perr = parser:parse_string(resp)
224 if not ok then
225 rspamd_logger.debugm(N, task, 'llm cannot parse reply: %s', perr)
226 cont(nil)
227 return
228 end
229 local parsed = parser:get_object()
230 local emb = extract_embedding(llm.type, parsed)
231 if type(emb) == 'table' then
232 lua_cache.cache_set(task, key, emb, cache_ctx)
233 finish_with_vec(emb)
234 else
235 rspamd_logger.debugm(N, task, 'llm embedding parse: no embedding field')
236 cont(nil)
237 end
238 end
239
240 local function do_request_and_cache()
241 local headers = { ['Content-Type'] = 'application/json' }
242 if llm.type == 'openai' and llm.api_key then
243 headers['Authorization'] = 'Bearer ' .. llm.api_key
244 end
245
246 local http_params = {
247 url = llm.url,
248 mime_type = 'application/json',
249 timeout = llm.timeout,
250 log_obj = task,
251 headers = headers,
252 body = ucl.to_format(body, 'json-compact', true),
253 task = task,
254 method = 'POST',
255 use_gzip = true,
256 keepalive = true,
257 callback = http_cb,
258 -- staged timeouts
259 connect_timeout = llm.connect_timeout,
260 ssl_timeout = llm.ssl_timeout,
261 write_timeout = llm.write_timeout,
262 read_timeout = llm.read_timeout,
263 }
264
265 rspamd_http.request(http_params)
266 end
267
268 -- Use async cache API
269 lua_cache.cache_get(task, key, cache_ctx, llm.timeout or 2.0,
270 function()
271 -- Uncached path
272 do_request_and_cache()
273 end,
274 function(_, err, data)
275 if data and type(data) == 'table' then
276 finish_with_vec(data)
277 else
278 do_request_and_cache()
279 end
280 end)
281 end,
282 })