Debian Codegraph @

lualib/lua_cta.lua

raw
1 --[[
2 CTA and link affiliation analysis
3
4 Purpose:
5 - Given a capped list of candidate links extracted in C during HTML parsing,
6 compute simple affiliation scores between those links and the sender’s
7 first-party domain, and pick a likely CTA (call-to-action) link.
8
9 How it is called:
10 - C code (message processing after HTML parsing) loads this function via
11 `rspamd_lua_require_function(L, "lua_cta", "process_html_links")` and calls
12 `process_html_links(task, part, ctx)` once per HTML text part.
13
14 Inputs (ctx table):
15 - links_total: total number of links in the part (summary; may be omitted)
16 - domains_total: number of distinct link domains (summary)
17 - max_links_single_domain: maximum links seen for a single domain (summary)
18 - candidates: array (capped in C, default 24) of small objects with fields:
19 - host: link host (string)
20 - idn, numeric, has_port, has_query, display_mismatch: booleans
21 - order, part_order: integers (ordering hints)
22 - etld1: optional eTLD+1 (if not set, this module approximates from host)
23
24 Outputs (returned table):
25 - cta_affiliated: boolean – whether the selected CTA appears affiliated
26 - cta_weight: number – simple weight hint (e.g. 1.0 if display mismatch)
27 - affiliated_ratio: number – fraction of candidates considered affiliated
28 - trackerish_ratio: number – fraction of candidates that look trackerish
29
30 Configuration (rspamd.conf):
31 - Use the `link_affiliation { ... }` section.
32 - Options:
33 - stopwords: map (set/regexp/glob) used to strip common tracking tokens from
34 domains when computing token overlap
35 - whitelist / blacklist: optional maps (set) to tweak affiliation
36 - min_similarity: number (default 0.5) – Jaccard threshold for affiliation
37 - max_candidates: number (default 24) – extra Lua-side cap (C caps as well)
38
39 This module keeps all heavy config logic in Lua using lua_maps and only relies
40 on C to provide a bounded set of safe, pre-filtered candidates.
41 ]]
42 local M = {}
43
44 local lua_util = require "lua_util"
45 local lua_maps = require "lua_maps"
46 local rspamd_util = require "rspamd_util"
47
48 -- Reasonable defaults (can be overridden in rspamd.conf: link_affiliation { ... })
49 local DEFAULT_STOPWORDS = {
50 -- Common TLD tokens and ccTLDs
51 "com", "net", "org", "info", "biz", "co", "io", "me", "us", "uk", "ru", "de", "fr", "au", "ca", "cn", "jp", "kr", "in",
52 "eu",
53 "es", "it", "nl", "pl", "se", "no", "fi", "dk", "cz", "sk", "pt", "tr", "gr", "hu", "ro", "bg", "ua", "by", "lt", "lv",
54 "ee",
55 "br", "mx", "ch", "be", "at", "dk", "cz", "sk", "pt", "ar", "cl", "pe", "tw", "th", "ph", "vn", "id", "hk", "sg", "nz",
56 "za",
57 "il", "ie", "is", "lu", "si", "hr", "rs", "gl", "ly",
58 -- Generic / infrastructural
59 "www", "web", "site", "app", "apps", "cloud", "cdn", "edge", "fastly", "akamai", "akamaihd", "edgesuite", "cloudfront",
60 -- Tracking/redirect/marketing tokens
61 "mail", "email", "news", "newsletter", "click", "link", "links", "go", "redir", "redirect", "rdir", "safe", "safelinks",
62 "trk", "track", "tracking", "ref", "mkt", "mktg", "campaign", "promo", "offer", "offers",
63 -- ESPs and bulk mailers (tokens found in their eTLD+1)
64 "mailchimp", "mandrill", "sendgrid", "sparkpost", "sparkpostmail", "amazonses", "ses", "postmark", "postmarkapp",
65 "mailgun",
66 "sendinblue", "constantcontact", "list", "manage", "rs6", "aweber", "hubspot", "campaignmonitor", "cmail", "klaviyo",
67 "sailthru",
68 "drip", "convertkit", "getresponse", "mautic", "braze", "acoustic", "responsys", "eloqua", "iterable", "sendy",
69 "emarsys", "mailjet",
70 "mailerlite", "mailerq", "mailrelay", "mailup", "omnisend", "clickdimensions", "dotdigital", "pepipost"
71 }
72
73 local DEFAULT_WHITELIST = {
74 -- Intentionally empty by default. Users can add trusted eTLD+1 domains here
75 }
76
77 local DEFAULT_BLACKLIST = {
78 -- Popular shorteners / redirection eTLD+1
79 "t.co", "bit.ly", "goo.gl", "tinyurl.com", "lnkd.in", "buff.ly", "ow.ly", "rebrand.ly", "bitly.com", "is.gd", "v.gd",
80 "t.ly",
81 "cutt.ly", "shorturl.at", "reurl.cc", "rb.gy", "s.id", "trib.al",
82 -- Common ESP/tracker link domains (treat as non-affiliated by default)
83 "list-manage.com", "mandrillapp.com", "sendgrid.net", "sparkpostmail.com", "amazonses.com", "postmarkapp.com",
84 "mailgun.org",
85 "sendinblue.com", "constantcontact.com", "campaignmonitor.com", "cmail1.com", "cmail2.com", "aweber.com", "hubspot.com",
86 "exacttarget.com", "clickdimensions.com", "eloqua.com", "responsys.net", "emarsys.net", "mailjet.com", "klaviyo.com",
87 "dripemail2.com",
88 "getresponse.com", "benchmarkemail.com", "omnisend.com", "mailerlite.com", "dotdigital.com"
89 }
90
91 local settings = {
92 min_similarity = 0.5,
93 max_candidates = 24,
94 stopwords = nil,
95 whitelist = nil,
96 blacklist = nil,
97 }
98
99 local function load_settings()
100 local cfg = rawget(_G, 'rspamd_config')
101 local opts = (cfg and cfg:get_all_opt('link_affiliation')) or {}
102 settings = lua_util.override_defaults(settings, opts)
103 -- Convert map definitions to maps if needed
104 if settings.stopwords then
105 if type(settings.stopwords) ~= 'table' or not settings.stopwords.get_key then
106 settings.stopwords = lua_maps.map_add_from_ucl(settings.stopwords, 'set', 'link affiliation stopwords')
107 end
108 else
109 settings.stopwords = lua_maps.map_add_from_ucl(DEFAULT_STOPWORDS, 'set', 'link affiliation stopwords (default)')
110 end
111 if settings.whitelist then
112 if type(settings.whitelist) ~= 'table' or not settings.whitelist.get_key then
113 settings.whitelist = lua_maps.map_add_from_ucl(settings.whitelist, 'set', 'link affiliation whitelist')
114 end
115 else
116 settings.whitelist = lua_maps.map_add_from_ucl(DEFAULT_WHITELIST, 'set', 'link affiliation whitelist (default)')
117 end
118 if settings.blacklist then
119 if type(settings.blacklist) ~= 'table' or not settings.blacklist.get_key then
120 settings.blacklist = lua_maps.map_add_from_ucl(settings.blacklist, 'set', 'link affiliation blacklist')
121 end
122 else
123 settings.blacklist = lua_maps.map_add_from_ucl(DEFAULT_BLACKLIST, 'set', 'link affiliation blacklist (default)')
124 end
125 end
126
127 load_settings()
128
129 local function etld1_tokens(dom)
130 local t = {}
131 for token in string.gmatch(string.lower(dom or ''), "[a-z0-9]+") do
132 if not (settings.stopwords and settings.stopwords:get_key(token)) then
133 t[token] = true
134 end
135 end
136 return t
137 end
138
139 local function jaccard(a, b)
140 local inter, uni = 0, 0
141 for k in pairs(a) do
142 if b[k] then inter = inter + 1 end
143 uni = uni + 1
144 end
145 for k in pairs(b) do
146 if not a[k] then uni = uni + 1 end
147 end
148 if uni == 0 then return 0 end
149 return inter / uni
150 end
151
152 M.process_html_links = function(task, part, ctx)
153 local first_party = nil
154 -- Derive first-party from From: if not provided
155 do
156 local from = task:get_from('mime') or {}
157 if from[1] and from[1].domain then
158 first_party = from[1].domain
159 end
160 end
161
162 local cands = ctx.candidates or {}
163 if #cands > settings.max_candidates then
164 local tmp = {}
165 for i = 1, settings.max_candidates do tmp[i] = cands[i] end
166 cands = tmp
167 end
168 local affiliated = 0
169 local trackerish = 0
170
171 local fp_tokens = etld1_tokens(first_party)
172
173 for _, c in ipairs(cands) do
174 local etld1 = c.etld1 or rspamd_util.get_tld(c.host or '') or (c.host or '')
175
176 local toks = etld1_tokens(etld1)
177 local sim = jaccard(fp_tokens, toks)
178
179 if sim >= settings.min_similarity then
180 affiliated = affiliated + 1
181 end
182
183 -- very naive trackerish: all tokens are stopwords or too few tokens
184 local n_tokens, n_nonstop = 0, 0
185 for _ in pairs(toks) do
186 n_tokens = n_tokens + 1; n_nonstop = n_nonstop + 1
187 end
188 if n_nonstop == 0 then trackerish = trackerish + 1 end
189 end
190
191 local res = {
192 affiliated_ratio = (#cands > 0) and (affiliated / #cands) or 0,
193 trackerish_ratio = (#cands > 0) and (trackerish / #cands) or 0,
194 }
195
196 -- Simple CTA guess: prefer higher C-side weight, then display_mismatch, then earliest order
197 if #cands > 0 then
198 table.sort(cands, function(a, b)
199 local aw, bw = tonumber(a.weight) or 0, tonumber(b.weight) or 0
200 if aw ~= bw then return aw > bw end
201 if a.display_mismatch ~= b.display_mismatch then return a.display_mismatch end
202 if a.order ~= b.order then return a.order < b.order end
203 return a.part_order < b.part_order
204 end)
205 local cta = cands[1]
206 local etld1 = cta.etld1 or rspamd_util.get_tld(cta.host or '') or (cta.host or '')
207 local toks = etld1_tokens(etld1)
208 local sim = jaccard(fp_tokens, toks)
209 res.cta_affiliated = (sim >= settings.min_similarity)
210 res.cta_weight = (cta.display_mismatch and 1.0 or 0.5)
211 end
212
213 return res
214 end
215
216 return M