| 1 |
--[[ |
| 2 |
Text hash provider for neural feature fusion. |
| 3 |
Extracts stemmed tokens from text parts, computes unigram and bigram hashes, |
| 4 |
and accumulates them into a fixed-size feature vector using the hashing trick. |
| 5 |
|
| 6 |
This is a cheap, zero-latency alternative to LLM embeddings that captures |
| 7 |
bag-of-words and bigram structure from message content. |
| 8 |
]] -- |
| 9 |
|
| 10 |
local rspamd_logger = require "rspamd_logger" |
| 11 |
local rspamd_util = require "rspamd_util" |
| 12 |
local lua_mime = require "lua_mime" |
| 13 |
local neural_common = require "plugins/neural" |
| 14 |
|
| 15 |
local N = "neural.text_hash" |
| 16 |
|
| 17 |
-- Default configuration |
| 18 |
local default_hash_size = 16384 -- 2^14; must be power of 2 |
| 19 |
local default_seed = 0xBEEF1234 |
| 20 |
local default_bigram_seed = 0xDEAD5678 |
| 21 |
|
| 22 |
-- Fast modular reduction for power-of-2 sizes |
| 23 |
local function hash_to_bucket(hash_val, size) |
| 24 |
-- Lua 5.1/LuaJIT: use math.fmod since we can't do bitwise AND |
| 25 |
-- hash_val from caseless_hash_fast is a double, take absolute value |
| 26 |
if hash_val < 0 then |
| 27 |
hash_val = -hash_val |
| 28 |
end |
| 29 |
return (math.floor(hash_val) % size) + 1 |
| 30 |
end |
| 31 |
|
| 32 |
-- Extract words from all relevant text parts |
| 33 |
local function extract_words(task, opts) |
| 34 |
local words = {} |
| 35 |
local how = opts.word_form or 'stem' |
| 36 |
|
| 37 |
-- Get text parts; prefer displayed text part for focused signal |
| 38 |
local parts |
| 39 |
if opts.all_parts then |
| 40 |
parts = task:get_text_parts() |
| 41 |
else |
| 42 |
local sel = lua_mime.get_displayed_text_part(task) |
| 43 |
if sel then |
| 44 |
parts = { sel } |
| 45 |
else |
| 46 |
parts = task:get_text_parts() |
| 47 |
end |
| 48 |
end |
| 49 |
|
| 50 |
if not parts then |
| 51 |
return words |
| 52 |
end |
| 53 |
|
| 54 |
for _, part in ipairs(parts) do |
| 55 |
local pw = part:get_words(how) |
| 56 |
if pw then |
| 57 |
for _, w in ipairs(pw) do |
| 58 |
if type(w) == 'string' and #w > 0 then |
| 59 |
words[#words + 1] = w |
| 60 |
end |
| 61 |
end |
| 62 |
end |
| 63 |
end |
| 64 |
|
| 65 |
return words |
| 66 |
end |
| 67 |
|
| 68 |
-- Build the hashed feature vector from words |
| 69 |
local function build_hash_vector(words, hash_size, seed, bigram_seed) |
| 70 |
local vec = {} |
| 71 |
for i = 1, hash_size do |
| 72 |
vec[i] = 0.0 |
| 73 |
end |
| 74 |
|
| 75 |
local n_words = #words |
| 76 |
|
| 77 |
-- Unigrams |
| 78 |
for i = 1, n_words do |
| 79 |
local h = rspamd_util.caseless_hash_fast(words[i], seed) |
| 80 |
local bucket = hash_to_bucket(h, hash_size) |
| 81 |
-- Use sign trick to reduce collisions: hash determines +1 or -1 |
| 82 |
local sign_h = rspamd_util.caseless_hash_fast(words[i], seed + 1) |
| 83 |
local sign = (math.floor(sign_h) % 2 == 0) and 1.0 or -1.0 |
| 84 |
vec[bucket] = vec[bucket] + sign |
| 85 |
end |
| 86 |
|
| 87 |
-- Bigrams |
| 88 |
for i = 2, n_words do |
| 89 |
local bigram = words[i - 1] .. '\0' .. words[i] |
| 90 |
local h = rspamd_util.caseless_hash_fast(bigram, bigram_seed) |
| 91 |
local bucket = hash_to_bucket(h, hash_size) |
| 92 |
local sign_h = rspamd_util.caseless_hash_fast(bigram, bigram_seed + 1) |
| 93 |
local sign = (math.floor(sign_h) % 2 == 0) and 1.0 or -1.0 |
| 94 |
vec[bucket] = vec[bucket] + sign |
| 95 |
end |
| 96 |
|
| 97 |
-- L2 normalize |
| 98 |
local sumsq = 0.0 |
| 99 |
for i = 1, hash_size do |
| 100 |
sumsq = sumsq + vec[i] * vec[i] |
| 101 |
end |
| 102 |
if sumsq > 0 then |
| 103 |
local inv = 1.0 / math.sqrt(sumsq) |
| 104 |
for i = 1, hash_size do |
| 105 |
vec[i] = vec[i] * inv |
| 106 |
end |
| 107 |
end |
| 108 |
|
| 109 |
return vec |
| 110 |
end |
| 111 |
|
| 112 |
-- Also hash subject line and headers as additional signal |
| 113 |
local function add_header_features(task, vec, hash_size, seed) |
| 114 |
-- Subject |
| 115 |
local subj = task:get_subject() |
| 116 |
if subj and #subj > 0 then |
| 117 |
local h = rspamd_util.caseless_hash_fast(subj, seed + 100) |
| 118 |
local bucket = hash_to_bucket(h, hash_size) |
| 119 |
vec[bucket] = vec[bucket] + 0.5 |
| 120 |
end |
| 121 |
|
| 122 |
-- From domain |
| 123 |
local from = task:get_from('mime') |
| 124 |
if from and from[1] and from[1].domain then |
| 125 |
local h = rspamd_util.caseless_hash_fast('FROM:' .. from[1].domain, seed + 200) |
| 126 |
local bucket = hash_to_bucket(h, hash_size) |
| 127 |
vec[bucket] = vec[bucket] + 0.5 |
| 128 |
end |
| 129 |
end |
| 130 |
|
| 131 |
neural_common.register_provider('text_hash', { |
| 132 |
collect_async = function(task, ctx, cont) |
| 133 |
local pcfg = ctx.config or {} |
| 134 |
local hash_size = pcfg.hash_size or default_hash_size |
| 135 |
local seed = pcfg.seed or default_seed |
| 136 |
local bigram_seed = pcfg.bigram_seed or default_bigram_seed |
| 137 |
|
| 138 |
local words = extract_words(task, { |
| 139 |
word_form = pcfg.word_form or 'stem', |
| 140 |
all_parts = pcfg.all_parts, |
| 141 |
}) |
| 142 |
|
| 143 |
if #words == 0 then |
| 144 |
rspamd_logger.debugm(N, task, 'text_hash: no words found; skip') |
| 145 |
cont(nil) |
| 146 |
return |
| 147 |
end |
| 148 |
|
| 149 |
rspamd_logger.debugm(N, task, 'text_hash: hashing %s words into %s buckets', |
| 150 |
#words, hash_size) |
| 151 |
|
| 152 |
local vec = build_hash_vector(words, hash_size, seed, bigram_seed) |
| 153 |
|
| 154 |
-- Optionally add header features |
| 155 |
if pcfg.include_headers ~= false then |
| 156 |
add_header_features(task, vec, hash_size, seed) |
| 157 |
end |
| 158 |
|
| 159 |
local meta = { |
| 160 |
name = pcfg.name or 'text_hash', |
| 161 |
type = 'text_hash', |
| 162 |
dim = hash_size, |
| 163 |
weight = ctx.weight or 1.0, |
| 164 |
} |
| 165 |
|
| 166 |
rspamd_logger.debugm(N, task, 'text_hash: produced %s-dim vector from %s words', |
| 167 |
hash_size, #words) |
| 168 |
cont(vec, meta) |
| 169 |
end, |
| 170 |
}) |