Debian Codegraph @

lualib/plugins/neural/providers/fasttext_embed.lua

raw
1 --[[
2 FastText embedding provider for neural feature fusion.
3 Loads a FastText model (supervised or unsupervised) and computes sentence
4 embeddings from message text. Supports per-language models for multilingual
5 deployments.
6
7 By default, all configured language_models are used for every message
8 (multi_model = true), producing richer cross-lingual representations.
9 Set multi_model = false to select a single model based on detected language.
10
11 Pooling modes (pooling = "mean_max" by default):
12 "mean" - average of word vectors (classic fasttext sentence vector)
13 "mean_max" - concatenation of mean and element-wise max pooling
14
15 Conv1d output mode (output_mode = "conv1d"):
16 Multi-scale max-over-time pooling over sliding word windows.
17 For each kernel size, averages word vectors in a window, then
18 max-pools across all window positions per channel.
19 Produces compact features: n_scales * pools_per_scale * channels.
20 Options:
21 kernel_sizes = [1, 3, 5] - window sizes (default)
22 conv_pooling = "max" - per-scale pooling: "max", "mean", "mean_max"
23 max_words = 32 - max word positions (default)
24
25 SIF (Smooth Inverse Frequency) weighting is enabled by default (sif_weight = true).
26 Common words (the, is, a) get near-zero weight, distinctive words get high weight.
27 Tune with sif_a parameter (default 1e-3). Set sif_weight = false to disable.
28
29 Configuration example in neural.conf:
30 providers = [
31 {
32 type = "fasttext_embed";
33 model = "/path/to/default_model.bin";
34 # Optional per-language models
35 language_models = {
36 en = "/path/to/en_model.bin";
37 ru = "/path/to/ru_model.bin";
38 };
39 weight = 1.0;
40 multi_model = true; # use all language models (default)
41 pooling = "mean_max"; # mean + max pooling (default)
42 sif_weight = true; # SIF word weighting (default)
43 sif_a = 1e-3; # SIF smoothing parameter (default)
44 }
45 ];
46 ]] --
47
48 local rspamd_logger = require "rspamd_logger"
49 local lua_mime = require "lua_mime"
50 local neural_common = require "plugins/neural"
51
52 local N = "neural.fasttext_embed"
53
54 -- Cache of loaded FastText models: path -> model userdata
55 local loaded_models = {}
56
57 -- Try to load rspamd_fasttext; may be nil if not compiled with FastText
58 local ok, rspamd_fasttext = pcall(require, "rspamd_fasttext")
59 if not ok then
60 rspamd_fasttext = nil
61 end
62
63 local function load_model(path)
64 if not rspamd_fasttext then
65 rspamd_logger.errx(rspamd_config, '%s: rspamd_fasttext module not available (compiled without FastText?)', N)
66 return nil
67 end
68
69 if loaded_models[path] then
70 local cached = loaded_models[path]
71 if cached:is_loaded() then
72 return cached
73 end
74 -- Cached but map not ready yet
75 return nil
76 end
77
78 -- Use load_map for map-backed loading (supports HTTP URLs + file reload)
79 rspamd_logger.infox(rspamd_config, '%s: loading FastText model from %s', N, path)
80 local model = rspamd_fasttext.load_map(rspamd_config, path)
81
82 if model then
83 loaded_models[path] = model
84 if model:is_loaded() then
85 rspamd_logger.infox(rspamd_config, '%s: loaded FastText model %s, dimension=%s',
86 N, path, model:get_dimension())
87 return model
88 end
89 end
90
91 return nil
92 end
93
94 -- Collect all available models (for multi_model mode)
95 local function collect_all_models(pcfg)
96 local models = {}
97
98 if pcfg.language_models then
99 -- Sort by language key for deterministic order
100 local langs = {}
101 for lang, _ in pairs(pcfg.language_models) do
102 langs[#langs + 1] = lang
103 end
104 table.sort(langs)
105
106 for _, lang in ipairs(langs) do
107 local path = pcfg.language_models[lang]
108 local model = load_model(path)
109 if model then
110 models[#models + 1] = { model = model, path = path, lang = lang }
111 end
112 end
113 end
114
115 -- Add default model if configured and not already loaded
116 if pcfg.model then
117 local already_loaded = false
118 for _, m in ipairs(models) do
119 if m.path == pcfg.model then
120 already_loaded = true
121 break
122 end
123 end
124 if not already_loaded then
125 local model = load_model(pcfg.model)
126 if model then
127 models[#models + 1] = { model = model, path = pcfg.model, lang = 'default' }
128 end
129 end
130 end
131
132 return models
133 end
134
135 -- Select a single model based on language (for single-model mode)
136 local function select_model(pcfg, language)
137 -- Check per-language models first
138 if language and pcfg.language_models then
139 local lang_path = pcfg.language_models[language]
140 if lang_path then
141 local model = load_model(lang_path)
142 if model then
143 return { { model = model, path = lang_path, lang = language } }
144 end
145 end
146 end
147
148 -- Fallback to default model
149 if pcfg.model then
150 local model = load_model(pcfg.model)
151 if model then
152 return { { model = model, path = pcfg.model, lang = 'default' } }
153 end
154 end
155
156 return {}
157 end
158
159 -- Extract words from text parts
160 local function extract_words(task, opts)
161 local words = {}
162 local how = opts.word_form or 'norm'
163
164 -- Get text parts
165 local parts
166 if opts.all_parts then
167 parts = task:get_text_parts()
168 else
169 local sel = lua_mime.get_displayed_text_part(task)
170 if sel then
171 parts = { sel }
172 else
173 parts = task:get_text_parts()
174 end
175 end
176
177 if not parts then
178 return words
179 end
180
181 for _, part in ipairs(parts) do
182 local pw = part:get_words(how)
183 if pw then
184 for _, w in ipairs(pw) do
185 if type(w) == 'string' and #w > 0 then
186 words[#words + 1] = w
187 end
188 end
189 end
190 end
191
192 return words
193 end
194
195 -- Compute mean and optionally max pooling from word vectors
196 -- When sif_a > 0, uses SIF (Smooth Inverse Frequency) weighting:
197 -- w(word) = a / (a + p(word))
198 -- where p(word) is the word probability from the model's vocabulary.
199 -- Common words get near-zero weight, distinctive words get high weight.
200 local function compute_pooled_vectors(model, words, pooling, sif_a)
201 local dim = model:get_dimension()
202 local mean_vec = {}
203 local max_vec = {}
204 local need_max = (pooling == 'mean_max')
205 local use_sif = sif_a and sif_a > 0
206
207 for d = 1, dim do
208 mean_vec[d] = 0.0
209 if need_max then
210 max_vec[d] = -math.huge
211 end
212 end
213
214 local total_weight = 0.0
215 for _, w in ipairs(words) do
216 local wv = model:get_word_vector(w)
217 if wv and #wv >= dim then
218 -- SIF weight: a / (a + p(word)); unknown words get weight 1.0
219 local weight = 1.0
220 if use_sif then
221 local freq = model:get_word_frequency(w)
222 if freq > 0 then
223 weight = sif_a / (sif_a + freq)
224 end
225 end
226
227 total_weight = total_weight + weight
228 for d = 1, dim do
229 mean_vec[d] = mean_vec[d] + wv[d] * weight
230 if need_max and wv[d] > max_vec[d] then
231 max_vec[d] = wv[d]
232 end
233 end
234 end
235 end
236
237 if total_weight == 0 then
238 return nil
239 end
240
241 -- Normalize weighted mean
242 for d = 1, dim do
243 mean_vec[d] = mean_vec[d] / total_weight
244 end
245
246 -- L2-normalize mean vector (match fasttext behavior)
247 local norm = 0.0
248 for d = 1, dim do
249 norm = norm + mean_vec[d] * mean_vec[d]
250 end
251 norm = math.sqrt(norm)
252 if norm > 0 then
253 for d = 1, dim do
254 mean_vec[d] = mean_vec[d] / norm
255 end
256 end
257
258 if need_max then
259 -- L2-normalize max vector
260 norm = 0.0
261 for d = 1, dim do
262 norm = norm + max_vec[d] * max_vec[d]
263 end
264 norm = math.sqrt(norm)
265 if norm > 0 then
266 for d = 1, dim do
267 max_vec[d] = max_vec[d] / norm
268 end
269 end
270
271 -- Concatenate mean + max
272 for d = 1, dim do
273 mean_vec[dim + d] = max_vec[d]
274 end
275 end
276
277 return mean_vec
278 end
279
280 -- Multi-scale pooling for conv1d feature extraction.
281 -- Instead of storing raw NCW matrices and relying on KANN conv1d,
282 -- we apply fixed convolution-like operations in Lua and store compact features.
283 --
284 -- For each window size k in kernel_sizes (default {1, 3, 5}):
285 -- 1. Slide a window of k words over the sequence
286 -- 2. Average word vectors within each window (like a fixed conv filter)
287 -- 3. Max-pool (and optionally mean-pool) over all window positions
288 -- Each scale's features are L2-normalized independently for balanced contribution.
289 -- Output: flat table of n_scales * n_pool * C floats (e.g., 3 * 2 * 100 = 600 for mean_max)
290 local function compute_conv1d_features(models, words, max_words, sif_a, opts)
291 opts = opts or {}
292 local use_sif = sif_a and sif_a > 0
293 local nwords = math.min(#words, max_words)
294 local kernel_sizes = opts.kernel_sizes or { 1, 3, 5 }
295 local conv_pooling = opts.conv_pooling or 'mean_max'
296 local need_mean = (conv_pooling == 'mean_max' or conv_pooling == 'mean')
297 local need_max = (conv_pooling == 'mean_max' or conv_pooling == 'max')
298
299 if nwords == 0 then
300 return nil, 0, 0
301 end
302
303 -- Compute total channels (sum of all model dimensions)
304 local total_channels = 0
305 for _, m in ipairs(models) do
306 total_channels = total_channels + m.model:get_dimension()
307 end
308
309 -- Collect word vectors: word_vecs[w][c] for word position w, channel c
310 local word_vecs = {}
311 for w = 1, nwords do
312 local wv_all = {}
313 for _, m in ipairs(models) do
314 local wv = m.model:get_word_vector(words[w])
315 local dim = m.model:get_dimension()
316 if wv and #wv >= dim then
317 local weight = 1.0
318 if use_sif then
319 local freq = m.model:get_word_frequency(words[w])
320 if freq > 0 then
321 weight = sif_a / (sif_a + freq)
322 end
323 end
324 for d = 1, dim do
325 wv_all[#wv_all + 1] = wv[d] * weight
326 end
327 else
328 for _ = 1, dim do
329 wv_all[#wv_all + 1] = 0.0
330 end
331 end
332 end
333 word_vecs[w] = wv_all
334 end
335
336 -- Multi-scale pooling with per-scale L2 normalization
337 local output = {}
338
339 for _, k in ipairs(kernel_sizes) do
340 local scale_mean = need_mean and {} or nil
341 local scale_max = need_max and {} or nil
342
343 -- Initialize per-scale accumulators
344 for c = 1, total_channels do
345 if scale_mean then
346 scale_mean[c] = 0.0
347 end
348 if scale_max then
349 scale_max[c] = -math.huge
350 end
351 end
352
353 -- Slide window of size k over word positions
354 local n_windows = nwords - k + 1
355 if n_windows < 1 then
356 -- Sequence too short for this kernel; treat each word as a window
357 n_windows = nwords
358 for c = 1, total_channels do
359 for w = 1, nwords do
360 local val = word_vecs[w][c] or 0.0
361 if scale_mean then
362 scale_mean[c] = scale_mean[c] + val
363 end
364 if scale_max and val > scale_max[c] then
365 scale_max[c] = val
366 end
367 end
368 if scale_mean then
369 scale_mean[c] = scale_mean[c] / nwords
370 end
371 end
372 else
373 for c = 1, total_channels do
374 for start = 1, n_windows do
375 -- Average word vectors within this window
376 local sum = 0.0
377 for w = start, start + k - 1 do
378 sum = sum + (word_vecs[w][c] or 0.0)
379 end
380 local avg = sum / k
381 if scale_mean then
382 scale_mean[c] = scale_mean[c] + avg
383 end
384 if scale_max and avg > scale_max[c] then
385 scale_max[c] = avg
386 end
387 end
388 if scale_mean then
389 scale_mean[c] = scale_mean[c] / n_windows
390 end
391 end
392 end
393
394 -- L2-normalize and append mean features for this scale
395 if scale_mean then
396 local norm = 0.0
397 for c = 1, total_channels do
398 norm = norm + scale_mean[c] * scale_mean[c]
399 end
400 norm = math.sqrt(norm)
401 for c = 1, total_channels do
402 output[#output + 1] = norm > 0 and (scale_mean[c] / norm) or 0.0
403 end
404 end
405
406 -- L2-normalize and append max features for this scale
407 if scale_max then
408 local norm = 0.0
409 for c = 1, total_channels do
410 norm = norm + scale_max[c] * scale_max[c]
411 end
412 norm = math.sqrt(norm)
413 for c = 1, total_channels do
414 output[#output + 1] = norm > 0 and (scale_max[c] / norm) or 0.0
415 end
416 end
417 end
418
419 local pools_per_scale = (need_mean and 1 or 0) + (need_max and 1 or 0)
420 return output, total_channels, pools_per_scale
421 end
422
423 neural_common.register_provider('fasttext_embed', {
424 init = function(pcfg)
425 -- Pre-register map-backed models at config time
426 if pcfg.model then
427 load_model(pcfg.model)
428 end
429 if pcfg.language_models then
430 for _, path in pairs(pcfg.language_models) do
431 load_model(path)
432 end
433 end
434 end,
435 collect_async = function(task, ctx, cont)
436 local pcfg = ctx.config or {}
437
438 if not rspamd_fasttext then
439 rspamd_logger.debugm(N, task, 'fasttext_embed: rspamd_fasttext not available; skip')
440 cont(nil)
441 return
442 end
443
444 -- Select models: all models or single based on language
445 local multi_model = pcfg.multi_model ~= false -- default true
446 local models
447 if multi_model and pcfg.language_models then
448 models = collect_all_models(pcfg)
449 else
450 local language = task and (function()
451 local part = lua_mime.get_displayed_text_part(task)
452 if part then
453 local lang = part:get_language()
454 if lang and lang ~= '' then
455 return lang
456 end
457 end
458 return nil
459 end)()
460 models = select_model(pcfg, language)
461 end
462
463 if #models == 0 then
464 rspamd_logger.debugm(N, task, 'fasttext_embed: no models available; skip')
465 cont(nil)
466 return
467 end
468
469 -- Extract words; use 'norm' by default for FastText (it expects lowercased tokens)
470 local words = extract_words(task, {
471 word_form = pcfg.word_form or 'norm',
472 all_parts = pcfg.all_parts,
473 })
474
475 -- Optionally prepend subject words
476 if pcfg.include_subject ~= false then
477 local subj = task:get_subject()
478 if subj and #subj > 0 then
479 -- Simple whitespace tokenization for subject
480 for w in subj:gmatch('%S+') do
481 table.insert(words, 1, w:lower())
482 end
483 end
484 end
485
486 if #words == 0 then
487 rspamd_logger.debugm(N, task, 'fasttext_embed: no words found; skip')
488 cont(nil)
489 return
490 end
491
492 -- Conv1d output mode: multi-scale max-over-time pooling.
493 -- For each kernel size, averages word vectors in sliding windows, then
494 -- max-pools across all positions per channel. Produces compact features:
495 -- n_scales * pools_per_scale * total_channels.
496 if pcfg.output_mode == 'conv1d' then
497 local max_words = pcfg.max_words or 32
498 local sif_a = pcfg.sif_a
499 if sif_a == nil then
500 sif_a = (pcfg.sif_weight ~= false) and 1e-3 or 0
501 end
502
503 local kernel_sizes = pcfg.kernel_sizes or { 1, 3, 5 }
504 local conv_pooling = pcfg.conv_pooling or 'max'
505 local model_names = {}
506 for _, m in ipairs(models) do
507 model_names[#model_names + 1] = m.lang
508 end
509
510 local combined_vec, total_channels, pools_per_scale = compute_conv1d_features(
511 models, words, max_words, sif_a,
512 { kernel_sizes = kernel_sizes, conv_pooling = conv_pooling })
513
514 if not combined_vec or #combined_vec == 0 then
515 rspamd_logger.debugm(N, task, 'fasttext_embed: conv1d produced empty features; skip')
516 cont(nil)
517 return
518 end
519
520 local meta = {
521 name = pcfg.name or 'fasttext_embed',
522 type = 'fasttext_embed',
523 output_mode = 'conv1d',
524 channels = total_channels,
525 n_scales = #kernel_sizes,
526 pools_per_scale = pools_per_scale,
527 dim = #combined_vec,
528 weight = ctx.weight or 1.0,
529 models = table.concat(model_names, '+'),
530 }
531
532 rspamd_logger.debugm(N, task,
533 'fasttext_embed: conv1d k=%s pool=%s dim=%s (%s models, %s words)',
534 table.concat(kernel_sizes, ','), conv_pooling,
535 #combined_vec, #models, math.min(#words, max_words))
536 cont(combined_vec, meta)
537 return
538 end
539
540 local pooling = pcfg.pooling or 'mean_max'
541 -- SIF weighting: enabled by default with a=1e-3
542 local sif_a = pcfg.sif_a
543 if sif_a == nil then
544 sif_a = (pcfg.sif_weight ~= false) and 1e-3 or 0
545 end
546 local combined_vec = {}
547 local model_names = {}
548 local total_dim = 0
549
550 for _, m in ipairs(models) do
551 local vec = compute_pooled_vectors(m.model, words, pooling, sif_a)
552 if vec then
553 for _, v in ipairs(vec) do
554 combined_vec[#combined_vec + 1] = v
555 end
556 total_dim = total_dim + #vec
557 model_names[#model_names + 1] = m.lang
558 end
559 end
560
561 if #combined_vec == 0 then
562 rspamd_logger.debugm(N, task, 'fasttext_embed: empty vectors from all models; skip')
563 cont(nil)
564 return
565 end
566
567 local meta = {
568 name = pcfg.name or 'fasttext_embed',
569 type = 'fasttext_embed',
570 dim = total_dim,
571 weight = ctx.weight or 1.0,
572 models = table.concat(model_names, '+'),
573 pooling = pooling,
574 }
575
576 rspamd_logger.debugm(N, task, 'fasttext_embed: produced %s-dim vector (%s models, %s pooling, %s words)',
577 total_dim, #models, pooling, #words)
578 cont(combined_vec, meta)
579 end,
580 })