Debian Codegraph @

lualib/rspamadm/fuzzy_convert.lua

raw
1 local sqlite3 = require "rspamd_sqlite3"
2 local redis = require "rspamd_redis"
3 local util = require "rspamd_util"
4
5 local function connect_redis(server, username, password, db)
6 local ret
7 local conn, err = redis.connect_sync({
8 host = server,
9 })
10
11 if not conn then
12 return nil, 'Cannot connect: ' .. err
13 end
14
15 if username then
16 if password then
17 ret = conn:add_cmd('AUTH', { username, password })
18 if not ret then
19 return nil, 'Cannot queue command'
20 end
21 else
22 return nil, 'Redis requires a password when username is supplied'
23 end
24 elseif password then
25 ret = conn:add_cmd('AUTH', { password })
26 if not ret then
27 return nil, 'Cannot queue command'
28 end
29 end
30 if db then
31 ret = conn:add_cmd('SELECT', { db })
32 if not ret then
33 return nil, 'Cannot queue command'
34 end
35 end
36
37 return conn, nil
38 end
39
40 local function send_digests(digests, redis_host, redis_username, redis_password, redis_db)
41 local conn, err = connect_redis(redis_host, redis_username, redis_password, redis_db)
42 if err then
43 print(err)
44 return false
45 end
46 local ret
47 for _, v in ipairs(digests) do
48 ret = conn:add_cmd('HMSET', {
49 'fuzzy' .. v[1],
50 'F', v[2],
51 'V', v[3],
52 })
53 if not ret then
54 print('Cannot batch command')
55 return false
56 end
57 ret = conn:add_cmd('EXPIRE', {
58 'fuzzy' .. v[1],
59 tostring(v[4]),
60 })
61 if not ret then
62 print('Cannot batch command')
63 return false
64 end
65 end
66 ret, err = conn:exec()
67 if not ret then
68 print('Cannot execute batched commands: ' .. err)
69 return false
70 end
71 return true
72 end
73
74 local function send_shingles(shingles, redis_host, redis_username, redis_password, redis_db)
75 local conn, err = connect_redis(redis_host, redis_username, redis_password, redis_db)
76 if err then
77 print("Redis error: " .. err)
78 return false
79 end
80 local ret
81 for _, v in ipairs(shingles) do
82 ret = conn:add_cmd('SET', {
83 'fuzzy_' .. v[2] .. '_' .. v[1],
84 v[4],
85 })
86 if not ret then
87 print('Cannot batch SET command: ' .. err)
88 return false
89 end
90 ret = conn:add_cmd('EXPIRE', {
91 'fuzzy_' .. v[2] .. '_' .. v[1],
92 tostring(v[3]),
93 })
94 if not ret then
95 print('Cannot batch command')
96 return false
97 end
98 end
99 ret, err = conn:exec()
100 if not ret then
101 print('Cannot execute batched commands: ' .. err)
102 return false
103 end
104 return true
105 end
106
107 local function update_counters(total, redis_host, redis_username, redis_password, redis_db)
108 local conn, err = connect_redis(redis_host, redis_username, redis_password, redis_db)
109 if err then
110 print(err)
111 return false
112 end
113 local ret
114 ret = conn:add_cmd('SET', {
115 'fuzzylocal',
116 total,
117 })
118 if not ret then
119 print('Cannot batch command')
120 return false
121 end
122 ret = conn:add_cmd('SET', {
123 'fuzzy_count',
124 total,
125 })
126 if not ret then
127 print('Cannot batch command')
128 return false
129 end
130 ret, err = conn:exec()
131 if not ret then
132 print('Cannot execute batched commands: ' .. err)
133 return false
134 end
135 return true
136 end
137
138 return function(_, res)
139 local db = sqlite3.open(res['source_db'])
140 local shingles = {}
141 local digests = {}
142 local num_batch_digests = 0
143 local num_batch_shingles = 0
144 local total_digests = 0
145 local total_shingles = 0
146 local lim_batch = 1000 -- Update each 1000 entries
147 local redis_username = res['redis_username']
148 local redis_password = res['redis_password']
149 local redis_db = nil
150
151 if res['redis_db'] then
152 redis_db = tostring(res['redis_db'])
153 end
154
155 if not db then
156 print('Cannot open source db: ' .. res['source_db'])
157 return
158 end
159
160 local now = util.get_time()
161 for row in db:rows('SELECT id, flag, digest, value, time FROM digests') do
162
163 local expire_in = math.floor(now - row.time + res['expiry'])
164 if expire_in >= 1 then
165 table.insert(digests, { row.digest, row.flag, row.value, expire_in })
166 num_batch_digests = num_batch_digests + 1
167 total_digests = total_digests + 1
168 for srow in db:rows('SELECT value, number FROM shingles WHERE digest_id = ' .. row.id) do
169 table.insert(shingles, { srow.value, srow.number, expire_in, row.digest })
170 total_shingles = total_shingles + 1
171 num_batch_shingles = num_batch_shingles + 1
172 end
173 end
174 if num_batch_digests >= lim_batch then
175 if not send_digests(digests, res['redis_host'], redis_username, redis_password, redis_db) then
176 return
177 end
178 num_batch_digests = 0
179 digests = {}
180 end
181 if num_batch_shingles >= lim_batch then
182 if not send_shingles(shingles, res['redis_host'], redis_username, redis_password, redis_db) then
183 return
184 end
185 num_batch_shingles = 0
186 shingles = {}
187 end
188 end
189 if digests[1] then
190 if not send_digests(digests, res['redis_host'], redis_username, redis_password, redis_db) then
191 return
192 end
193 end
194 if shingles[1] then
195 if not send_shingles(shingles, res['redis_host'], redis_username, redis_password, redis_db) then
196 return
197 end
198 end
199
200 local message = string.format(
201 'Migrated %d digests and %d shingles',
202 total_digests, total_shingles
203 )
204 if not update_counters(total_digests, res['redis_host'], redis_username, redis_password, redis_db) then
205 message = message .. ' but failed to update counters'
206 end
207 print(message)
208 end