Debian Codegraph @

lualib/lua_magic/init.lua

raw
1 --[[
2 Copyright (c) 2022, Vsevolod Stakhov <[email protected]>
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 ]] --
16
17 --[[[
18 -- @module lua_magic
19 -- This module contains file types detection logic
20 --]]
21
22 local patterns = require "lua_magic/patterns"
23 local types = require "lua_magic/types"
24 local heuristics = require "lua_magic/heuristics"
25 local fun = require "fun"
26 local lua_util = require "lua_util"
27
28 local rspamd_text = require "rspamd_text"
29 local rspamd_trie = require "rspamd_trie"
30
31 local N = "lua_magic"
32 local exports = {}
33 -- trie objects
34 local compiled_patterns
35 local compiled_short_patterns
36 local compiled_tail_patterns
37 -- {<str>, <match_object>, <pattern_object>} indexed by pattern number
38 local processed_patterns = {}
39 local short_patterns = {}
40 local tail_patterns = {}
41
42 local short_match_limit = 128
43 local max_short_offset = -1
44 local min_tail_offset = math.huge
45
46 local function process_patterns(log_obj)
47 -- Add pattern to either short patterns or to normal patterns
48 local function add_processed(str, match, pattern)
49 if match.position and type(match.position) == 'number' then
50 if match.tail then
51 -- Tail pattern
52 tail_patterns[#tail_patterns + 1] = {
53 str, match, pattern
54 }
55 if min_tail_offset > match.tail then
56 min_tail_offset = match.tail
57 end
58
59 lua_util.debugm(N, log_obj, 'add tail pattern %s for ext %s',
60 str, pattern.ext)
61 elseif match.position < short_match_limit then
62 short_patterns[#short_patterns + 1] = {
63 str, match, pattern
64 }
65 if str:sub(1, 1) == '^' then
66 lua_util.debugm(N, log_obj, 'add head pattern %s for ext %s',
67 str, pattern.ext)
68 else
69 lua_util.debugm(N, log_obj, 'add short pattern %s for ext %s',
70 str, pattern.ext)
71 end
72
73 if max_short_offset < match.position then
74 max_short_offset = match.position
75 end
76 else
77 processed_patterns[#processed_patterns + 1] = {
78 str, match, pattern
79 }
80
81 lua_util.debugm(N, log_obj, 'add long pattern %s for ext %s',
82 str, pattern.ext)
83 end
84 else
85 processed_patterns[#processed_patterns + 1] = {
86 str, match, pattern
87 }
88
89 lua_util.debugm(N, log_obj, 'add long pattern %s for ext %s',
90 str, pattern.ext)
91 end
92 end
93
94 if not compiled_patterns then
95 for ext, pattern in pairs(patterns) do
96 assert(types[ext], 'not found type: ' .. ext)
97 pattern.ext = ext
98 for _, match in ipairs(pattern.matches) do
99 if match.string then
100 if match.relative_position and not match.position then
101 match.position = match.relative_position + #match.string
102
103 if match.relative_position == 0 then
104 if match.string:sub(1, 1) ~= '^' then
105 match.string = '^' .. match.string
106 end
107 end
108 end
109 add_processed(match.string, match, pattern)
110 elseif match.hex then
111 local hex_table = {}
112
113 for i = 1, #match.hex, 2 do
114 local subc = match.hex:sub(i, i + 1)
115 hex_table[#hex_table + 1] = string.format('\\x{%s}', subc)
116 end
117
118 if match.relative_position and not match.position then
119 match.position = match.relative_position + #match.hex / 2
120 end
121 if match.relative_position == 0 then
122 table.insert(hex_table, 1, '^')
123 end
124 add_processed(table.concat(hex_table), match, pattern)
125 end
126 end
127 end
128 local bit = require "bit"
129 local compile_flags = bit.bor(rspamd_trie.flags.re, rspamd_trie.flags.dot_all)
130 compile_flags = bit.bor(compile_flags, rspamd_trie.flags.single_match)
131 compile_flags = bit.bor(compile_flags, rspamd_trie.flags.no_start)
132 compiled_patterns = rspamd_trie.create(fun.totable(
133 fun.map(function(t)
134 return t[1]
135 end, processed_patterns)),
136 compile_flags
137 )
138 compiled_short_patterns = rspamd_trie.create(fun.totable(
139 fun.map(function(t)
140 return t[1]
141 end, short_patterns)),
142 compile_flags
143 )
144 compiled_tail_patterns = rspamd_trie.create(fun.totable(
145 fun.map(function(t)
146 return t[1]
147 end, tail_patterns)),
148 compile_flags
149 )
150
151 lua_util.debugm(N, log_obj,
152 'compiled %s (%s short; %s long; %s tail) patterns',
153 #processed_patterns + #short_patterns + #tail_patterns,
154 #short_patterns, #processed_patterns, #tail_patterns)
155 end
156 end
157
158 process_patterns(rspamd_config)
159
160 local function match_chunk(chunk, input, tlen, offset, trie, processed_tbl, log_obj, res, part)
161 local matches = trie:match(chunk)
162
163 local last = tlen
164
165 local function add_result(weight, ext)
166 if not res[ext] then
167 res[ext] = 0
168 end
169 if weight then
170 res[ext] = res[ext] + weight
171 else
172 res[ext] = res[ext] + 1
173 end
174
175 lua_util.debugm(N, log_obj, 'add pattern for %s, weight %s, total weight %s',
176 ext, weight, res[ext])
177 end
178
179 local function match_position(pos, expected)
180 local cmp = function(a, b)
181 return a == b
182 end
183 if type(expected) == 'table' then
184 -- Something like {'>', 0}
185 if expected[1] == '>' then
186 cmp = function(a, b)
187 return a > b
188 end
189 elseif expected[1] == '>=' then
190 cmp = function(a, b)
191 return a >= b
192 end
193 elseif expected[1] == '<' then
194 cmp = function(a, b)
195 return a < b
196 end
197 elseif expected[1] == '<=' then
198 cmp = function(a, b)
199 return a <= b
200 end
201 elseif expected[1] == '!=' then
202 cmp = function(a, b)
203 return a ~= b
204 end
205 end
206 expected = expected[2]
207 end
208
209 -- Tail match
210 if expected < 0 then
211 expected = last + expected + 1
212 end
213 return cmp(pos, expected)
214 end
215
216 for npat, matched_positions in pairs(matches) do
217 local pat_data = processed_tbl[npat]
218 local pattern = pat_data[3]
219 local match = pat_data[2]
220
221 -- Single position
222 if match.position then
223 local position = match.position
224
225 for _, pos in ipairs(matched_positions) do
226 lua_util.debugm(N, log_obj, 'found match %s at offset %s(from %s)',
227 pattern.ext, pos, offset)
228 if match_position(pos + offset, position) then
229 if match.heuristic then
230 local ext, weight = match.heuristic(input, log_obj, pos + offset, part)
231
232 if ext then
233 add_result(weight, ext)
234 break
235 end
236 else
237 add_result(match.weight, pattern.ext)
238 break
239 end
240 end
241 end
242 elseif match.positions then
243 -- Match all positions
244 local all_right = true
245 local matched_pos = 0
246 for _, position in ipairs(match.positions) do
247 local matched = false
248 for _, pos in ipairs(matched_positions) do
249 lua_util.debugm(N, log_obj, 'found match %s at offset %s(from %s)',
250 pattern.ext, pos, offset)
251 if not match_position(pos + offset, position) then
252 matched = true
253 matched_pos = pos
254 break
255 end
256 end
257 if not matched then
258 all_right = false
259 break
260 end
261 end
262
263 if all_right then
264 if match.heuristic then
265 local ext, weight = match.heuristic(input, log_obj, matched_pos + offset, part)
266
267 if ext then
268 add_result(weight, ext)
269 break
270 end
271 else
272 add_result(match.weight, pattern.ext)
273 break
274 end
275 end
276 end
277 end
278 end
279
280 local function process_detected(res)
281 local extensions = lua_util.keys(res)
282
283 if #extensions > 0 then
284 table.sort(extensions, function(ex1, ex2)
285 return res[ex1] > res[ex2]
286 end)
287
288 return extensions, res[extensions[1]]
289 end
290
291 return nil
292 end
293
294 exports.detect = function(part, log_obj)
295 if not log_obj then
296 log_obj = rspamd_config
297 end
298 local input = part:get_content()
299
300 local res = {}
301
302 if type(input) == 'string' then
303 -- Convert to rspamd_text
304 input = rspamd_text.fromstring(input)
305 end
306
307 if type(input) == 'userdata' then
308 local inplen = #input
309 if inplen == 0 then
310 -- Empty input: nothing to match
311 return nil
312 end
313
314 -- Check tail matches
315 if inplen > min_tail_offset then
316 local tail = input:span(inplen - min_tail_offset, min_tail_offset)
317 match_chunk(tail, input, inplen, inplen - min_tail_offset,
318 compiled_tail_patterns, tail_patterns, log_obj, res, part)
319 end
320
321 -- Try short match
322 if max_short_offset > 0 then
323 local head = input:span(1, math.min(max_short_offset, inplen))
324 match_chunk(head, input, inplen, 0,
325 compiled_short_patterns, short_patterns, log_obj, res, part)
326 end
327
328 -- Check if we have enough data or go to long patterns
329 local extensions, confidence = process_detected(res)
330
331 if extensions and #extensions > 0 and confidence > 30 then
332 -- We are done on short patterns
333 return extensions[1], types[extensions[1]]
334 end
335
336 -- No way, let's check data in chunks or just the whole input if it is small enough
337 if #input > exports.chunk_size * 3 then
338 -- Chunked version as input is too long
339 local chunk1, chunk2 = input:span(1, exports.chunk_size * 2),
340 input:span(inplen - exports.chunk_size, exports.chunk_size)
341 local offset1, offset2 = 0, inplen - exports.chunk_size
342
343 match_chunk(chunk1, input, inplen,
344 offset1, compiled_patterns, processed_patterns, log_obj, res, part)
345 match_chunk(chunk2, input, inplen,
346 offset2, compiled_patterns, processed_patterns, log_obj, res, part)
347 else
348 -- Input is short enough to match it at all
349 match_chunk(input, input, inplen, 0,
350 compiled_patterns, processed_patterns, log_obj, res, part)
351 end
352 else
353 -- Table input is NYI
354 assert(0, 'table input for match')
355 end
356
357 local extensions = process_detected(res)
358
359 if extensions and #extensions > 0 then
360 return extensions[1], types[extensions[1]]
361 end
362
363 -- Nothing found
364 return nil
365 end
366
367 exports.detect_mime_part = function(part, log_obj)
368 local ext, weight = heuristics.mime_part_heuristic(part, log_obj)
369
370 if ext and weight and weight > 20 then
371 return ext, types[ext]
372 end
373
374 ext = exports.detect(part, log_obj)
375
376 if ext then
377 return ext, types[ext]
378 end
379
380 -- Fallback by filename extension (e.g. .eml attachments with generic content-type)
381 local fname
382 if part.get_filename then
383 fname = part:get_filename()
384 end
385 if type(fname) == 'string' then
386 local lfn = fname:lower()
387 if #lfn > 4 and lfn:sub(-4) == '.eml' then
388 return 'eml', types['eml']
389 end
390 end
391
392 -- Text/html and other parts
393 ext, weight = heuristics.text_part_heuristic(part, log_obj)
394 if ext and weight and weight > 20 then
395 return ext, types[ext]
396 end
397 end
398
399 -- This parameter specifies how many bytes are checked in the input
400 -- Rspamd checks 2 chunks at start and 1 chunk at the end
401 exports.chunk_size = 32768
402
403 exports.types = types
404
405 return exports