Debian Codegraph @

lualib/lua_urls_compose.lua

raw
1 --[[
2 Copyright (c) 2022, Vsevolod Stakhov <[email protected]>
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 ]]--
16
17 --[[[
18 -- @module lua_urls_compose
19 -- This module contains functions to compose urls queries from hostname
20 -- to TLD part
21 --]]
22
23 local N = "lua_urls_compose"
24 local lua_util = require "lua_util"
25 local rspamd_util = require "rspamd_util"
26 local bit = require "bit"
27 local rspamd_trie = require "rspamd_trie"
28 local fun = require "fun"
29 local rspamd_regexp = require "rspamd_regexp"
30
31 local maps_cache = {}
32
33 local exports = {}
34
35 local function process_url(self, log_obj, url_tld, url_host)
36 local tld_elt = self.tlds[url_tld]
37
38 if tld_elt then
39 lua_util.debugm(N, log_obj, 'found compose tld for %s (host = %s)',
40 url_tld, url_host)
41
42 for _, excl in ipairs(tld_elt.except_rules) do
43 local matched, ret = excl[2](url_tld, url_host)
44 if matched then
45 lua_util.debugm(N, log_obj, 'found compose exclusion for %s (%s) -> %s',
46 url_host, excl[1], ret)
47
48 return ret
49 end
50 end
51
52 if tld_elt.multipattern_compose_rules then
53 local matches = tld_elt.multipattern_compose_rules:match(url_host)
54
55 if matches then
56 local lua_pat_idx = math.huge
57
58 for m, _ in pairs(matches) do
59 if m < lua_pat_idx then
60 lua_pat_idx = m
61 end
62 end
63
64 if #tld_elt.compose_rules >= lua_pat_idx then
65 local lua_pat = tld_elt.compose_rules[lua_pat_idx]
66 local matched, ret = lua_pat[2](url_tld, url_host)
67
68 if not matched then
69 lua_util.debugm(N, log_obj, 'NOT found compose inclusion for %s (%s) -> %s',
70 url_host, lua_pat[1], url_tld)
71
72 return url_tld
73 else
74 lua_util.debugm(N, log_obj, 'found compose inclusion for %s (%s) -> %s',
75 url_host, lua_pat[1], ret)
76
77 return ret
78 end
79 else
80 lua_util.debugm(N, log_obj, 'NOT found compose inclusion for %s (%s) -> %s',
81 url_host, lua_pat_idx, url_tld)
82
83 return url_tld
84 end
85 end
86 else
87 -- Match one by one
88 for _, lua_pat in ipairs(tld_elt.compose_rules) do
89 local matched, ret = lua_pat[2](url_tld, url_host)
90 if matched then
91 lua_util.debugm(N, log_obj, 'found compose inclusion for %s (%s) -> %s',
92 url_host, lua_pat[1], ret)
93
94 return ret
95 end
96 end
97 end
98
99 lua_util.debugm(N, log_obj, 'not found compose inclusion for %s in %s -> %s',
100 url_host, url_tld, url_tld)
101 else
102 lua_util.debugm(N, log_obj, 'not found compose tld for %s in %s -> %s',
103 url_host, url_tld, url_tld)
104 end
105
106 return url_tld
107 end
108
109 local function tld_pattern_transform(tld_pat)
110 -- Convert tld like pattern to a lua match pattern
111 -- blah -> %.blah
112 -- *.blah -> .*%.blah
113 local ret
114 if tld_pat:sub(1, 2) == '*.' then
115 ret = string.format('^((?:[^.]+\\.)*%s)$', tld_pat:sub(3))
116 else
117 ret = string.format('(?:^|\\.)((?:[^.]+\\.)?%s)$', tld_pat)
118 end
119
120 lua_util.debugm(N, nil, 'added pattern %s -> %s',
121 tld_pat, ret)
122
123 return ret
124 end
125
126 local function include_elt_gen(pat)
127 pat = rspamd_regexp.create(tld_pattern_transform(pat), 'i')
128 return function(_, host)
129 local matches = pat:search(host, false, true)
130 if matches then
131 return true, matches[1][2]
132 end
133
134 return false
135 end
136 end
137
138 local function exclude_elt_gen(pat)
139 pat = rspamd_regexp.create(tld_pattern_transform(pat))
140 return function(tld, host)
141 if pat:search(host) then
142 return true, tld
143 end
144
145 return false
146 end
147 end
148
149 local function compose_map_cb(self, map_text)
150 local lpeg = require "lpeg"
151
152 local singleline_comment = lpeg.P '#' * (1 - lpeg.S '\r\n\f') ^ 0
153 local comments_strip_grammar = lpeg.C((1 - lpeg.P '#') ^ 1) * lpeg.S(' \t') ^ 0 * singleline_comment ^ 0
154
155 local function process_tld_rule(tld_elt, l)
156 if l:sub(1, 1) == '!' then
157 -- Exclusion elt
158 table.insert(tld_elt.except_rules, { l, exclude_elt_gen(l:sub(2)) })
159 else
160 table.insert(tld_elt.compose_rules, { l, include_elt_gen(l) })
161 end
162 end
163
164 local function process_map_line(l)
165 -- Skip empty lines and comments
166 if #l == 0 then
167 return
168 end
169 l = comments_strip_grammar:match(l)
170 if not l or #l == 0 then
171 return
172 end
173
174 -- Get TLD
175 local tld = rspamd_util.get_tld(l)
176
177 if tld then
178 local tld_elt = self.tlds[tld]
179
180 if not tld_elt then
181 tld_elt = {
182 compose_rules = {},
183 except_rules = {},
184 multipattern_compose_rules = nil
185 }
186
187 lua_util.debugm(N, rspamd_config, 'processed new tld rule for %s', tld)
188 self.tlds[tld] = tld_elt
189 end
190
191 process_tld_rule(tld_elt, l)
192 else
193 lua_util.debugm(N, rspamd_config, 'cannot read tld from compose map line: %s', l)
194 end
195 end
196
197 for line in map_text:lines() do
198 process_map_line(line)
199 end
200
201 local multipattern_threshold = 1
202 for tld, tld_elt in pairs(self.tlds) do
203 -- Sort patterns to have longest labels before shortest ones,
204 -- so we can ensure that they match before
205 table.sort(tld_elt.compose_rules, function(e1, e2)
206 local _, ndots1 = string.gsub(e1[1], '(%.)', '')
207 local _, ndots2 = string.gsub(e2[1], '(%.)', '')
208
209 return ndots1 > ndots2
210 end)
211 if rspamd_trie.has_hyperscan() and #tld_elt.compose_rules >= multipattern_threshold then
212 lua_util.debugm(N, rspamd_config, 'tld %s has %s rules, apply multipattern',
213 tld, #tld_elt.compose_rules)
214 local flags = bit.bor(rspamd_trie.flags.re,
215 rspamd_trie.flags.dot_all,
216 rspamd_trie.flags.no_start,
217 rspamd_trie.flags.icase)
218
219
220 -- We now convert our internal patterns to multipattern patterns
221 local mp_table = fun.totable(fun.map(function(pat_elt)
222 return tld_pattern_transform(pat_elt[1])
223 end, tld_elt.compose_rules))
224 tld_elt.multipattern_compose_rules = rspamd_trie.create(mp_table, flags)
225 end
226 end
227 end
228
229 exports.add_composition_map = function(cfg, map_obj)
230 local hash_key = map_obj
231 if type(map_obj) == 'table' then
232 hash_key = lua_util.unordered_table_digest(map_obj)
233 end
234
235 local map = maps_cache[hash_key]
236
237 if not map then
238 local ret = {
239 process_url = process_url,
240 hash = hash_key,
241 tlds = {},
242 }
243
244 map = cfg:add_map {
245 type = 'callback',
246 description = 'URL compose map',
247 url = map_obj,
248 callback = function(input)
249 compose_map_cb(ret, input)
250 end,
251 opaque_data = true,
252 }
253
254 ret.map = map
255 maps_cache[hash_key] = ret
256 map = ret
257 end
258
259 return map
260 end
261
262 exports.inject_composition_rules = function(cfg, rules)
263 local hash_key = rules
264 local rspamd_text = require "rspamd_text"
265 if type(rules) == 'table' then
266 hash_key = lua_util.unordered_table_digest(rules)
267 end
268
269 local map = maps_cache[hash_key]
270
271 if not map then
272 local ret = {
273 process_url = process_url,
274 hash = hash_key,
275 tlds = {},
276 }
277
278 compose_map_cb(ret, rspamd_text.fromtable(rules, '\n'))
279 maps_cache[hash_key] = ret
280 map = ret
281 end
282
283 return map
284 end
285
286 return exports