| 1 |
--[[ |
| 2 |
Copyright (c) 2022, Vsevolod Stakhov <[email protected]> |
| 3 |
|
| 4 |
Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 |
you may not use this file except in compliance with the License. |
| 6 |
You may obtain a copy of the License at |
| 7 |
|
| 8 |
http://www.apache.org/licenses/LICENSE-2.0 |
| 9 |
|
| 10 |
Unless required by applicable law or agreed to in writing, software |
| 11 |
distributed under the License is distributed on an "AS IS" BASIS, |
| 12 |
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 |
See the License for the specific language governing permissions and |
| 14 |
limitations under the License. |
| 15 |
]]-- |
| 16 |
|
| 17 |
--[[[ |
| 18 |
-- @module lua_urls_compose |
| 19 |
-- This module contains functions to compose urls queries from hostname |
| 20 |
-- to TLD part |
| 21 |
--]] |
| 22 |
|
| 23 |
local N = "lua_urls_compose" |
| 24 |
local lua_util = require "lua_util" |
| 25 |
local rspamd_util = require "rspamd_util" |
| 26 |
local bit = require "bit" |
| 27 |
local rspamd_trie = require "rspamd_trie" |
| 28 |
local fun = require "fun" |
| 29 |
local rspamd_regexp = require "rspamd_regexp" |
| 30 |
|
| 31 |
local maps_cache = {} |
| 32 |
|
| 33 |
local exports = {} |
| 34 |
|
| 35 |
local function process_url(self, log_obj, url_tld, url_host) |
| 36 |
local tld_elt = self.tlds[url_tld] |
| 37 |
|
| 38 |
if tld_elt then |
| 39 |
lua_util.debugm(N, log_obj, 'found compose tld for %s (host = %s)', |
| 40 |
url_tld, url_host) |
| 41 |
|
| 42 |
for _, excl in ipairs(tld_elt.except_rules) do |
| 43 |
local matched, ret = excl[2](url_tld, url_host) |
| 44 |
if matched then |
| 45 |
lua_util.debugm(N, log_obj, 'found compose exclusion for %s (%s) -> %s', |
| 46 |
url_host, excl[1], ret) |
| 47 |
|
| 48 |
return ret |
| 49 |
end |
| 50 |
end |
| 51 |
|
| 52 |
if tld_elt.multipattern_compose_rules then |
| 53 |
local matches = tld_elt.multipattern_compose_rules:match(url_host) |
| 54 |
|
| 55 |
if matches then |
| 56 |
local lua_pat_idx = math.huge |
| 57 |
|
| 58 |
for m, _ in pairs(matches) do |
| 59 |
if m < lua_pat_idx then |
| 60 |
lua_pat_idx = m |
| 61 |
end |
| 62 |
end |
| 63 |
|
| 64 |
if #tld_elt.compose_rules >= lua_pat_idx then |
| 65 |
local lua_pat = tld_elt.compose_rules[lua_pat_idx] |
| 66 |
local matched, ret = lua_pat[2](url_tld, url_host) |
| 67 |
|
| 68 |
if not matched then |
| 69 |
lua_util.debugm(N, log_obj, 'NOT found compose inclusion for %s (%s) -> %s', |
| 70 |
url_host, lua_pat[1], url_tld) |
| 71 |
|
| 72 |
return url_tld |
| 73 |
else |
| 74 |
lua_util.debugm(N, log_obj, 'found compose inclusion for %s (%s) -> %s', |
| 75 |
url_host, lua_pat[1], ret) |
| 76 |
|
| 77 |
return ret |
| 78 |
end |
| 79 |
else |
| 80 |
lua_util.debugm(N, log_obj, 'NOT found compose inclusion for %s (%s) -> %s', |
| 81 |
url_host, lua_pat_idx, url_tld) |
| 82 |
|
| 83 |
return url_tld |
| 84 |
end |
| 85 |
end |
| 86 |
else |
| 87 |
-- Match one by one |
| 88 |
for _, lua_pat in ipairs(tld_elt.compose_rules) do |
| 89 |
local matched, ret = lua_pat[2](url_tld, url_host) |
| 90 |
if matched then |
| 91 |
lua_util.debugm(N, log_obj, 'found compose inclusion for %s (%s) -> %s', |
| 92 |
url_host, lua_pat[1], ret) |
| 93 |
|
| 94 |
return ret |
| 95 |
end |
| 96 |
end |
| 97 |
end |
| 98 |
|
| 99 |
lua_util.debugm(N, log_obj, 'not found compose inclusion for %s in %s -> %s', |
| 100 |
url_host, url_tld, url_tld) |
| 101 |
else |
| 102 |
lua_util.debugm(N, log_obj, 'not found compose tld for %s in %s -> %s', |
| 103 |
url_host, url_tld, url_tld) |
| 104 |
end |
| 105 |
|
| 106 |
return url_tld |
| 107 |
end |
| 108 |
|
| 109 |
local function tld_pattern_transform(tld_pat) |
| 110 |
-- Convert tld like pattern to a lua match pattern |
| 111 |
-- blah -> %.blah |
| 112 |
-- *.blah -> .*%.blah |
| 113 |
local ret |
| 114 |
if tld_pat:sub(1, 2) == '*.' then |
| 115 |
ret = string.format('^((?:[^.]+\\.)*%s)$', tld_pat:sub(3)) |
| 116 |
else |
| 117 |
ret = string.format('(?:^|\\.)((?:[^.]+\\.)?%s)$', tld_pat) |
| 118 |
end |
| 119 |
|
| 120 |
lua_util.debugm(N, nil, 'added pattern %s -> %s', |
| 121 |
tld_pat, ret) |
| 122 |
|
| 123 |
return ret |
| 124 |
end |
| 125 |
|
| 126 |
local function include_elt_gen(pat) |
| 127 |
pat = rspamd_regexp.create(tld_pattern_transform(pat), 'i') |
| 128 |
return function(_, host) |
| 129 |
local matches = pat:search(host, false, true) |
| 130 |
if matches then |
| 131 |
return true, matches[1][2] |
| 132 |
end |
| 133 |
|
| 134 |
return false |
| 135 |
end |
| 136 |
end |
| 137 |
|
| 138 |
local function exclude_elt_gen(pat) |
| 139 |
pat = rspamd_regexp.create(tld_pattern_transform(pat)) |
| 140 |
return function(tld, host) |
| 141 |
if pat:search(host) then |
| 142 |
return true, tld |
| 143 |
end |
| 144 |
|
| 145 |
return false |
| 146 |
end |
| 147 |
end |
| 148 |
|
| 149 |
local function compose_map_cb(self, map_text) |
| 150 |
local lpeg = require "lpeg" |
| 151 |
|
| 152 |
local singleline_comment = lpeg.P '#' * (1 - lpeg.S '\r\n\f') ^ 0 |
| 153 |
local comments_strip_grammar = lpeg.C((1 - lpeg.P '#') ^ 1) * lpeg.S(' \t') ^ 0 * singleline_comment ^ 0 |
| 154 |
|
| 155 |
local function process_tld_rule(tld_elt, l) |
| 156 |
if l:sub(1, 1) == '!' then |
| 157 |
-- Exclusion elt |
| 158 |
table.insert(tld_elt.except_rules, { l, exclude_elt_gen(l:sub(2)) }) |
| 159 |
else |
| 160 |
table.insert(tld_elt.compose_rules, { l, include_elt_gen(l) }) |
| 161 |
end |
| 162 |
end |
| 163 |
|
| 164 |
local function process_map_line(l) |
| 165 |
-- Skip empty lines and comments |
| 166 |
if #l == 0 then |
| 167 |
return |
| 168 |
end |
| 169 |
l = comments_strip_grammar:match(l) |
| 170 |
if not l or #l == 0 then |
| 171 |
return |
| 172 |
end |
| 173 |
|
| 174 |
-- Get TLD |
| 175 |
local tld = rspamd_util.get_tld(l) |
| 176 |
|
| 177 |
if tld then |
| 178 |
local tld_elt = self.tlds[tld] |
| 179 |
|
| 180 |
if not tld_elt then |
| 181 |
tld_elt = { |
| 182 |
compose_rules = {}, |
| 183 |
except_rules = {}, |
| 184 |
multipattern_compose_rules = nil |
| 185 |
} |
| 186 |
|
| 187 |
lua_util.debugm(N, rspamd_config, 'processed new tld rule for %s', tld) |
| 188 |
self.tlds[tld] = tld_elt |
| 189 |
end |
| 190 |
|
| 191 |
process_tld_rule(tld_elt, l) |
| 192 |
else |
| 193 |
lua_util.debugm(N, rspamd_config, 'cannot read tld from compose map line: %s', l) |
| 194 |
end |
| 195 |
end |
| 196 |
|
| 197 |
for line in map_text:lines() do |
| 198 |
process_map_line(line) |
| 199 |
end |
| 200 |
|
| 201 |
local multipattern_threshold = 1 |
| 202 |
for tld, tld_elt in pairs(self.tlds) do |
| 203 |
-- Sort patterns to have longest labels before shortest ones, |
| 204 |
-- so we can ensure that they match before |
| 205 |
table.sort(tld_elt.compose_rules, function(e1, e2) |
| 206 |
local _, ndots1 = string.gsub(e1[1], '(%.)', '') |
| 207 |
local _, ndots2 = string.gsub(e2[1], '(%.)', '') |
| 208 |
|
| 209 |
return ndots1 > ndots2 |
| 210 |
end) |
| 211 |
if rspamd_trie.has_hyperscan() and #tld_elt.compose_rules >= multipattern_threshold then |
| 212 |
lua_util.debugm(N, rspamd_config, 'tld %s has %s rules, apply multipattern', |
| 213 |
tld, #tld_elt.compose_rules) |
| 214 |
local flags = bit.bor(rspamd_trie.flags.re, |
| 215 |
rspamd_trie.flags.dot_all, |
| 216 |
rspamd_trie.flags.no_start, |
| 217 |
rspamd_trie.flags.icase) |
| 218 |
|
| 219 |
|
| 220 |
-- We now convert our internal patterns to multipattern patterns |
| 221 |
local mp_table = fun.totable(fun.map(function(pat_elt) |
| 222 |
return tld_pattern_transform(pat_elt[1]) |
| 223 |
end, tld_elt.compose_rules)) |
| 224 |
tld_elt.multipattern_compose_rules = rspamd_trie.create(mp_table, flags) |
| 225 |
end |
| 226 |
end |
| 227 |
end |
| 228 |
|
| 229 |
exports.add_composition_map = function(cfg, map_obj) |
| 230 |
local hash_key = map_obj |
| 231 |
if type(map_obj) == 'table' then |
| 232 |
hash_key = lua_util.unordered_table_digest(map_obj) |
| 233 |
end |
| 234 |
|
| 235 |
local map = maps_cache[hash_key] |
| 236 |
|
| 237 |
if not map then |
| 238 |
local ret = { |
| 239 |
process_url = process_url, |
| 240 |
hash = hash_key, |
| 241 |
tlds = {}, |
| 242 |
} |
| 243 |
|
| 244 |
map = cfg:add_map { |
| 245 |
type = 'callback', |
| 246 |
description = 'URL compose map', |
| 247 |
url = map_obj, |
| 248 |
callback = function(input) |
| 249 |
compose_map_cb(ret, input) |
| 250 |
end, |
| 251 |
opaque_data = true, |
| 252 |
} |
| 253 |
|
| 254 |
ret.map = map |
| 255 |
maps_cache[hash_key] = ret |
| 256 |
map = ret |
| 257 |
end |
| 258 |
|
| 259 |
return map |
| 260 |
end |
| 261 |
|
| 262 |
exports.inject_composition_rules = function(cfg, rules) |
| 263 |
local hash_key = rules |
| 264 |
local rspamd_text = require "rspamd_text" |
| 265 |
if type(rules) == 'table' then |
| 266 |
hash_key = lua_util.unordered_table_digest(rules) |
| 267 |
end |
| 268 |
|
| 269 |
local map = maps_cache[hash_key] |
| 270 |
|
| 271 |
if not map then |
| 272 |
local ret = { |
| 273 |
process_url = process_url, |
| 274 |
hash = hash_key, |
| 275 |
tlds = {}, |
| 276 |
} |
| 277 |
|
| 278 |
compose_map_cb(ret, rspamd_text.fromtable(rules, '\n')) |
| 279 |
maps_cache[hash_key] = ret |
| 280 |
map = ret |
| 281 |
end |
| 282 |
|
| 283 |
return map |
| 284 |
end |
| 285 |
|
| 286 |
return exports |