Debian Codegraph @

lualib/plugins/neural/providers/text_hash.lua

raw
1 --[[
2 Text hash provider for neural feature fusion.
3 Extracts stemmed tokens from text parts, computes unigram and bigram hashes,
4 and accumulates them into a fixed-size feature vector using the hashing trick.
5
6 This is a cheap, zero-latency alternative to LLM embeddings that captures
7 bag-of-words and bigram structure from message content.
8 ]] --
9
10 local rspamd_logger = require "rspamd_logger"
11 local rspamd_util = require "rspamd_util"
12 local lua_mime = require "lua_mime"
13 local neural_common = require "plugins/neural"
14
15 local N = "neural.text_hash"
16
17 -- Default configuration
18 local default_hash_size = 16384 -- 2^14; must be power of 2
19 local default_seed = 0xBEEF1234
20 local default_bigram_seed = 0xDEAD5678
21
22 -- Fast modular reduction for power-of-2 sizes
23 local function hash_to_bucket(hash_val, size)
24 -- Lua 5.1/LuaJIT: use math.fmod since we can't do bitwise AND
25 -- hash_val from caseless_hash_fast is a double, take absolute value
26 if hash_val < 0 then
27 hash_val = -hash_val
28 end
29 return (math.floor(hash_val) % size) + 1
30 end
31
32 -- Extract words from all relevant text parts
33 local function extract_words(task, opts)
34 local words = {}
35 local how = opts.word_form or 'stem'
36
37 -- Get text parts; prefer displayed text part for focused signal
38 local parts
39 if opts.all_parts then
40 parts = task:get_text_parts()
41 else
42 local sel = lua_mime.get_displayed_text_part(task)
43 if sel then
44 parts = { sel }
45 else
46 parts = task:get_text_parts()
47 end
48 end
49
50 if not parts then
51 return words
52 end
53
54 for _, part in ipairs(parts) do
55 local pw = part:get_words(how)
56 if pw then
57 for _, w in ipairs(pw) do
58 if type(w) == 'string' and #w > 0 then
59 words[#words + 1] = w
60 end
61 end
62 end
63 end
64
65 return words
66 end
67
68 -- Build the hashed feature vector from words
69 local function build_hash_vector(words, hash_size, seed, bigram_seed)
70 local vec = {}
71 for i = 1, hash_size do
72 vec[i] = 0.0
73 end
74
75 local n_words = #words
76
77 -- Unigrams
78 for i = 1, n_words do
79 local h = rspamd_util.caseless_hash_fast(words[i], seed)
80 local bucket = hash_to_bucket(h, hash_size)
81 -- Use sign trick to reduce collisions: hash determines +1 or -1
82 local sign_h = rspamd_util.caseless_hash_fast(words[i], seed + 1)
83 local sign = (math.floor(sign_h) % 2 == 0) and 1.0 or -1.0
84 vec[bucket] = vec[bucket] + sign
85 end
86
87 -- Bigrams
88 for i = 2, n_words do
89 local bigram = words[i - 1] .. '\0' .. words[i]
90 local h = rspamd_util.caseless_hash_fast(bigram, bigram_seed)
91 local bucket = hash_to_bucket(h, hash_size)
92 local sign_h = rspamd_util.caseless_hash_fast(bigram, bigram_seed + 1)
93 local sign = (math.floor(sign_h) % 2 == 0) and 1.0 or -1.0
94 vec[bucket] = vec[bucket] + sign
95 end
96
97 -- L2 normalize
98 local sumsq = 0.0
99 for i = 1, hash_size do
100 sumsq = sumsq + vec[i] * vec[i]
101 end
102 if sumsq > 0 then
103 local inv = 1.0 / math.sqrt(sumsq)
104 for i = 1, hash_size do
105 vec[i] = vec[i] * inv
106 end
107 end
108
109 return vec
110 end
111
112 -- Also hash subject line and headers as additional signal
113 local function add_header_features(task, vec, hash_size, seed)
114 -- Subject
115 local subj = task:get_subject()
116 if subj and #subj > 0 then
117 local h = rspamd_util.caseless_hash_fast(subj, seed + 100)
118 local bucket = hash_to_bucket(h, hash_size)
119 vec[bucket] = vec[bucket] + 0.5
120 end
121
122 -- From domain
123 local from = task:get_from('mime')
124 if from and from[1] and from[1].domain then
125 local h = rspamd_util.caseless_hash_fast('FROM:' .. from[1].domain, seed + 200)
126 local bucket = hash_to_bucket(h, hash_size)
127 vec[bucket] = vec[bucket] + 0.5
128 end
129 end
130
131 neural_common.register_provider('text_hash', {
132 collect_async = function(task, ctx, cont)
133 local pcfg = ctx.config or {}
134 local hash_size = pcfg.hash_size or default_hash_size
135 local seed = pcfg.seed or default_seed
136 local bigram_seed = pcfg.bigram_seed or default_bigram_seed
137
138 local words = extract_words(task, {
139 word_form = pcfg.word_form or 'stem',
140 all_parts = pcfg.all_parts,
141 })
142
143 if #words == 0 then
144 rspamd_logger.debugm(N, task, 'text_hash: no words found; skip')
145 cont(nil)
146 return
147 end
148
149 rspamd_logger.debugm(N, task, 'text_hash: hashing %s words into %s buckets',
150 #words, hash_size)
151
152 local vec = build_hash_vector(words, hash_size, seed, bigram_seed)
153
154 -- Optionally add header features
155 if pcfg.include_headers ~= false then
156 add_header_features(task, vec, hash_size, seed)
157 end
158
159 local meta = {
160 name = pcfg.name or 'text_hash',
161 type = 'text_hash',
162 dim = hash_size,
163 weight = ctx.weight or 1.0,
164 }
165
166 rspamd_logger.debugm(N, task, 'text_hash: produced %s-dim vector from %s words',
167 hash_size, #words)
168 cont(vec, meta)
169 end,
170 })