Debian Codegraph @

lualib/llm_search_context.lua

raw
1 --[[
2 Copyright (c) 2024, Vsevolod Stakhov <[email protected]>
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 ]] --
16
17 --[[
18 Web search context module for LLM-based spam detection
19
20 This module extracts domains from email URLs, queries a search API to fetch
21 relevant information about those domains, and formats the results as context
22 for LLM-based classification.
23
24 Main function:
25 - fetch_and_format(task, redis_params, opts, callback, debug_module): Fetch search context and format for LLM
26
27 Options (all optional with safe defaults):
28 enabled: boolean (default: false)
29 search_url: string (default: "https://leta.mullvad.net/api/search")
30 max_domains: number (default: 3) - max domains to search
31 max_results_per_query: number (default: 3) - max results per domain
32 timeout: number (default: 5) - HTTP request timeout in seconds
33 cache_ttl: number (default: 3600) - cache TTL in seconds
34 cache_key_prefix: string (default: "gpt_search")
35 as_system: boolean (default: true) - inject as system message vs user message
36 enable_expression: table - optional gating expression
37 disable_expression: table - optional negative gating expression
38 ]]
39
40 local N = 'llm_search_context'
41
42 local M = {}
43
44 local rspamd_http = require "rspamd_http"
45 local rspamd_logger = require "rspamd_logger"
46 local lua_util = require "lua_util"
47 local lua_cache = require "lua_cache"
48 local lua_mime = require "lua_mime"
49 local ucl = require "ucl"
50
51 local DEFAULTS = {
52 enabled = false,
53 search_url = "https://leta.mullvad.net/search/__data.json",
54 search_engine = "brave", -- Search engine to use (brave, google, etc.)
55 max_domains = 3,
56 max_results_per_query = 3,
57 timeout = 5,
58 cache_ttl = 3600, -- 1 hour
59 cache_key_prefix = "llm_search",
60 as_system = true,
61 enable_expression = nil,
62 disable_expression = nil,
63 }
64
65 -- Extract unique domains from task URLs, prioritizing CTA (call-to-action) links
66 local function extract_domains(task, max_domains, debug_module)
67 local Np = debug_module or N
68 local domains = {}
69 local seen = {}
70
71 -- Skip common domains that won't provide useful context
72 local skip_domains = {
73 ['localhost'] = true,
74 ['127.0.0.1'] = true,
75 ['example.com'] = true,
76 ['example.org'] = true,
77 }
78
79 -- First, try to get CTA URLs from HTML (most relevant for spam detection)
80 -- Uses button weight and HTML structure analysis from C code
81 local cta_urls = {}
82 local sel_part = lua_mime.get_displayed_text_part(task)
83 if sel_part then
84 cta_urls = sel_part:get_cta_urls()
85 end
86 lua_util.debugm(Np, task,
87 "CTA analysis found %d URLs across", #cta_urls)
88
89 for _, url in ipairs(cta_urls) do
90 if #domains >= max_domains then
91 break
92 end
93
94 local host = url:get_host()
95 if host and not skip_domains[host:lower()] and not seen[host] then
96 seen[host] = true
97 table.insert(domains, host)
98 lua_util.debugm(Np, task, "added CTA domain: %s", host)
99 end
100 end
101
102 -- If we don't have enough domains from CTA, get more from content URLs
103 if #domains < max_domains then
104 lua_util.debugm(Np, task, "need more domains (%d/%d), extracting from content URLs",
105 #domains, max_domains)
106
107 local urls = lua_util.extract_specific_urls({
108 task = task,
109 limit = max_domains * 3,
110 esld_limit = max_domains,
111 need_content = true, -- Content URLs (buttons, links in text)
112 need_images = false,
113 }) or {}
114
115 lua_util.debugm(Np, task, "extracted %d content URLs", #urls)
116
117 for _, url in ipairs(urls) do
118 if #domains >= max_domains then
119 break
120 end
121
122 local host = url:get_host()
123 if host and not seen[host] and not skip_domains[host:lower()] then
124 seen[host] = true
125 table.insert(domains, host)
126 lua_util.debugm(Np, task, "added content domain: %s", host)
127 end
128 end
129 end
130
131 -- Still need more? Get from any URLs
132 if #domains < max_domains then
133 lua_util.debugm(Np, task, "still need more domains (%d/%d), extracting from all URLs",
134 #domains, max_domains)
135
136 local urls = lua_util.extract_specific_urls({
137 task = task,
138 limit = max_domains * 3,
139 esld_limit = max_domains,
140 }) or {}
141
142 lua_util.debugm(Np, task, "extracted %d all URLs", #urls)
143
144 for _, url in ipairs(urls) do
145 if #domains >= max_domains then
146 break
147 end
148
149 local host = url:get_host()
150 if host and not seen[host] and not skip_domains[host:lower()] then
151 seen[host] = true
152 table.insert(domains, host)
153 lua_util.debugm(Np, task, "added general domain: %s", host)
154 end
155 end
156 end
157
158 return domains
159 end
160
161 -- Query search API for a single domain
162 local function query_search_api(task, domain, opts, callback, debug_module)
163 local Np = debug_module or N
164
165 -- Prepare search query for Leta Mullvad API
166 local query_params = {
167 q = domain,
168 engine = opts.search_engine,
169 }
170
171 -- Build query string
172 local query_string = ""
173 for k, v in pairs(query_params) do
174 if query_string ~= "" then
175 query_string = query_string .. "&"
176 end
177 query_string = query_string .. k .. "=" .. lua_util.url_encode_string(v)
178 end
179
180 local full_url = opts.search_url .. "?" .. query_string
181
182 local function http_callback(err, code, body, _)
183 if err then
184 lua_util.debugm(Np, task, "search API error for domain '%s': %s", domain, err)
185 callback(nil, domain, err)
186 return
187 end
188
189 if code ~= 200 then
190 rspamd_logger.infox(task, "search API returned code %s for domain '%s', url: %s, body: %s",
191 code, domain, full_url, body and body:sub(1, 200) or 'nil')
192 callback(nil, domain, string.format("HTTP %s", code))
193 return
194 end
195
196 lua_util.debugm(Np, task, "search API success for domain '%s', url: %s", domain, full_url)
197
198 -- Parse Leta Mullvad JSON response
199 local parser = ucl.parser()
200 local ok, parse_err = parser:parse_string(body)
201 if not ok then
202 rspamd_logger.errx(task, "%s: failed to parse search API response for %s: %s",
203 Np, domain, parse_err)
204 callback(nil, domain, parse_err)
205 return
206 end
207
208 local data = parser:get_object()
209
210 -- Extract search results from Leta Mullvad's nested structure
211 -- Structure: data.nodes[3].data is a flat array with indices as pointers
212 -- data[1] = metadata with pointers, data[5] = items array (Lua 1-indexed)
213 local search_results = { results = {} }
214
215 if data and data.nodes and type(data.nodes) == 'table' and #data.nodes >= 3 then
216 local search_node = data.nodes[3] -- Third node contains search data (Lua 1-indexed)
217
218 if search_node and search_node.data and type(search_node.data) == 'table' then
219 local flat_data = search_node.data
220 local metadata = flat_data[1]
221
222 lua_util.debugm(Np, task, "parsing domain '%s': flat_data has %d elements, metadata type: %s",
223 domain, #flat_data, type(metadata))
224
225 if metadata and metadata.items and type(metadata.items) == 'number' then
226 -- metadata.items is a 0-indexed pointer, add 1 for Lua
227 local items_idx = metadata.items + 1
228 local items = flat_data[items_idx]
229
230 if items and type(items) == 'table' then
231 lua_util.debugm(Np, task, "found %d item indices for domain '%s', items_idx=%d",
232 #items, domain, items_idx)
233
234 local count = 0
235
236 for _, result_idx in ipairs(items) do
237 if count >= opts.max_results_per_query then
238 break
239 end
240
241 -- result_idx is 0-indexed, add 1 for Lua
242 local result_template_idx = result_idx + 1
243 local result_template = flat_data[result_template_idx]
244
245 if result_template and type(result_template) == 'table' then
246 -- Extract values using the template's pointers (also 0-indexed)
247 local link = result_template.link and flat_data[result_template.link + 1]
248 local snippet = result_template.snippet and flat_data[result_template.snippet + 1]
249 local title = result_template.title and flat_data[result_template.title + 1]
250
251 lua_util.debugm(Np, task, "result %d template: link_idx=%s, snippet_idx=%s, title_idx=%s",
252 count + 1, tostring(result_template.link), tostring(result_template.snippet),
253 tostring(result_template.title))
254
255 if link or title or snippet then
256 table.insert(search_results.results, {
257 title = title or "",
258 snippet = snippet or "",
259 url = link or ""
260 })
261 count = count + 1
262 lua_util.debugm(Np, task, "extracted result %d: title='%s', snippet_len=%d",
263 count, title or "nil", snippet and #snippet or 0)
264 end
265 else
266 lua_util.debugm(Np, task, "result_template at idx %d is not a table: %s",
267 result_template_idx, type(result_template))
268 end
269 end
270 else
271 lua_util.debugm(Np, task, "items is not a table for domain '%s', type: %s",
272 domain, type(items))
273 end
274 else
275 lua_util.debugm(Np, task, "no valid metadata.items for domain '%s'", domain)
276 end
277 end
278 end
279
280 lua_util.debugm(Np, task, "extracted %d search results for domain '%s'",
281 #search_results.results, domain)
282 callback(search_results, domain, nil)
283 end
284
285 rspamd_http.request({
286 url = full_url,
287 timeout = opts.timeout,
288 callback = http_callback,
289 task = task,
290 log_obj = task,
291 })
292 end
293
294 -- Format search results as context
295 local function format_search_results(all_results, opts)
296 if not all_results or #all_results == 0 then
297 return nil
298 end
299
300 local context_lines = {
301 "Web search context for domains in email:"
302 }
303
304 for _, domain_result in ipairs(all_results) do
305 local domain = domain_result.domain
306 local results = domain_result.results
307
308 if results and results.results and #results.results > 0 then
309 table.insert(context_lines, string.format("\nDomain: %s", domain))
310
311 for i, result in ipairs(results.results) do
312 if i > opts.max_results_per_query then
313 break
314 end
315
316 local title = result.title or "No title"
317 local snippet = result.snippet or result.description or "No description"
318
319 -- Truncate snippet if too long
320 if #snippet > 200 then
321 snippet = snippet:sub(1, 197) .. "..."
322 end
323
324 table.insert(context_lines, string.format(" - %s: %s", title, snippet))
325 end
326 else
327 table.insert(context_lines, string.format("\nDomain: %s - No search results found", domain))
328 end
329 end
330
331 return table.concat(context_lines, "\n")
332 end
333
334 -- Main function to fetch and format search context
335 function M.fetch_and_format(task, redis_params, opts, callback, debug_module)
336 local Np = debug_module or N
337
338 -- Apply defaults
339 opts = lua_util.override_defaults(DEFAULTS, opts or {})
340
341 if not opts.enabled then
342 lua_util.debugm(Np, task, "search context disabled")
343 callback(task, false, nil)
344 return
345 end
346
347 -- Extract domains from task
348 local domains = extract_domains(task, opts.max_domains, Np)
349
350 if #domains == 0 then
351 lua_util.debugm(Np, task, "no domains to search")
352 callback(task, false, nil)
353 return
354 end
355
356 lua_util.debugm(Np, task, "final domain list (%d domains) for search: %s",
357 #domains, table.concat(domains, ", "))
358
359 -- Create cache context
360 local cache_ctx = nil
361 if redis_params then
362 cache_ctx = lua_cache.create_cache_context(redis_params, {
363 cache_prefix = opts.cache_key_prefix,
364 cache_ttl = opts.cache_ttl,
365 cache_format = 'messagepack',
366 cache_hash_len = 16,
367 cache_use_hashing = true,
368 }, Np)
369 end
370
371 local pending_queries = #domains
372 local all_results = {}
373
374 -- Callback for each domain query complete
375 local function domain_complete(domain, results)
376 pending_queries = pending_queries - 1
377
378 if results then
379 table.insert(all_results, {
380 domain = domain,
381 results = results
382 })
383 end
384
385 if pending_queries == 0 then
386 -- All queries complete
387 if #all_results == 0 then
388 lua_util.debugm(Np, task, "no search results obtained")
389 callback(task, false, nil)
390 else
391 local context_snippet = format_search_results(all_results, opts)
392 lua_util.debugm(Np, task, "search context formatted (%s bytes)",
393 context_snippet and #context_snippet or 0)
394 callback(task, true, context_snippet)
395 end
396 end
397 end
398
399 -- Process each domain
400 for _, domain in ipairs(domains) do
401 local cache_key = string.format("search:%s:%s", opts.search_engine, domain)
402
403 if cache_ctx then
404 -- Use lua_cache for caching
405 lua_cache.cache_get(task, cache_key, cache_ctx, opts.timeout,
406 function()
407 -- Cache miss - query API
408 query_search_api(task, domain, opts, function(api_results, d, api_err)
409 if api_results then
410 lua_cache.cache_set(task, cache_key, api_results, cache_ctx)
411 domain_complete(d, api_results)
412 else
413 lua_util.debugm(Np, task, "search failed for domain %s: %s", d, api_err)
414 domain_complete(d, nil)
415 end
416 end, Np)
417 end,
418 function(_, err, data)
419 -- Cache hit or after miss callback
420 if data and type(data) == 'table' then
421 lua_util.debugm(Np, task, "cache hit for domain %s", domain)
422 domain_complete(domain, data)
423 -- If no data and no error, the miss callback was already invoked
424 elseif err then
425 lua_util.debugm(Np, task, "cache error for domain %s: %s", domain, err)
426 domain_complete(domain, nil)
427 end
428 end)
429 else
430 -- No Redis, query directly
431 query_search_api(task, domain, opts, function(api_results, d, api_err)
432 if not api_results then
433 lua_util.debugm(Np, task, "search failed for domain %s: %s", d, api_err)
434 end
435 domain_complete(d, api_results)
436 end, Np)
437 end
438 end
439 end
440
441 return M