| 1 |
local sqlite3 = require "rspamd_sqlite3" |
| 2 |
local redis = require "rspamd_redis" |
| 3 |
local util = require "rspamd_util" |
| 4 |
|
| 5 |
local function connect_redis(server, username, password, db) |
| 6 |
local ret |
| 7 |
local conn, err = redis.connect_sync({ |
| 8 |
host = server, |
| 9 |
}) |
| 10 |
|
| 11 |
if not conn then |
| 12 |
return nil, 'Cannot connect: ' .. err |
| 13 |
end |
| 14 |
|
| 15 |
if username then |
| 16 |
if password then |
| 17 |
ret = conn:add_cmd('AUTH', { username, password }) |
| 18 |
if not ret then |
| 19 |
return nil, 'Cannot queue command' |
| 20 |
end |
| 21 |
else |
| 22 |
return nil, 'Redis requires a password when username is supplied' |
| 23 |
end |
| 24 |
elseif password then |
| 25 |
ret = conn:add_cmd('AUTH', { password }) |
| 26 |
if not ret then |
| 27 |
return nil, 'Cannot queue command' |
| 28 |
end |
| 29 |
end |
| 30 |
if db then |
| 31 |
ret = conn:add_cmd('SELECT', { db }) |
| 32 |
if not ret then |
| 33 |
return nil, 'Cannot queue command' |
| 34 |
end |
| 35 |
end |
| 36 |
|
| 37 |
return conn, nil |
| 38 |
end |
| 39 |
|
| 40 |
local function send_digests(digests, redis_host, redis_username, redis_password, redis_db) |
| 41 |
local conn, err = connect_redis(redis_host, redis_username, redis_password, redis_db) |
| 42 |
if err then |
| 43 |
print(err) |
| 44 |
return false |
| 45 |
end |
| 46 |
local ret |
| 47 |
for _, v in ipairs(digests) do |
| 48 |
ret = conn:add_cmd('HMSET', { |
| 49 |
'fuzzy' .. v[1], |
| 50 |
'F', v[2], |
| 51 |
'V', v[3], |
| 52 |
}) |
| 53 |
if not ret then |
| 54 |
print('Cannot batch command') |
| 55 |
return false |
| 56 |
end |
| 57 |
ret = conn:add_cmd('EXPIRE', { |
| 58 |
'fuzzy' .. v[1], |
| 59 |
tostring(v[4]), |
| 60 |
}) |
| 61 |
if not ret then |
| 62 |
print('Cannot batch command') |
| 63 |
return false |
| 64 |
end |
| 65 |
end |
| 66 |
ret, err = conn:exec() |
| 67 |
if not ret then |
| 68 |
print('Cannot execute batched commands: ' .. err) |
| 69 |
return false |
| 70 |
end |
| 71 |
return true |
| 72 |
end |
| 73 |
|
| 74 |
local function send_shingles(shingles, redis_host, redis_username, redis_password, redis_db) |
| 75 |
local conn, err = connect_redis(redis_host, redis_username, redis_password, redis_db) |
| 76 |
if err then |
| 77 |
print("Redis error: " .. err) |
| 78 |
return false |
| 79 |
end |
| 80 |
local ret |
| 81 |
for _, v in ipairs(shingles) do |
| 82 |
ret = conn:add_cmd('SET', { |
| 83 |
'fuzzy_' .. v[2] .. '_' .. v[1], |
| 84 |
v[4], |
| 85 |
}) |
| 86 |
if not ret then |
| 87 |
print('Cannot batch SET command: ' .. err) |
| 88 |
return false |
| 89 |
end |
| 90 |
ret = conn:add_cmd('EXPIRE', { |
| 91 |
'fuzzy_' .. v[2] .. '_' .. v[1], |
| 92 |
tostring(v[3]), |
| 93 |
}) |
| 94 |
if not ret then |
| 95 |
print('Cannot batch command') |
| 96 |
return false |
| 97 |
end |
| 98 |
end |
| 99 |
ret, err = conn:exec() |
| 100 |
if not ret then |
| 101 |
print('Cannot execute batched commands: ' .. err) |
| 102 |
return false |
| 103 |
end |
| 104 |
return true |
| 105 |
end |
| 106 |
|
| 107 |
local function update_counters(total, redis_host, redis_username, redis_password, redis_db) |
| 108 |
local conn, err = connect_redis(redis_host, redis_username, redis_password, redis_db) |
| 109 |
if err then |
| 110 |
print(err) |
| 111 |
return false |
| 112 |
end |
| 113 |
local ret |
| 114 |
ret = conn:add_cmd('SET', { |
| 115 |
'fuzzylocal', |
| 116 |
total, |
| 117 |
}) |
| 118 |
if not ret then |
| 119 |
print('Cannot batch command') |
| 120 |
return false |
| 121 |
end |
| 122 |
ret = conn:add_cmd('SET', { |
| 123 |
'fuzzy_count', |
| 124 |
total, |
| 125 |
}) |
| 126 |
if not ret then |
| 127 |
print('Cannot batch command') |
| 128 |
return false |
| 129 |
end |
| 130 |
ret, err = conn:exec() |
| 131 |
if not ret then |
| 132 |
print('Cannot execute batched commands: ' .. err) |
| 133 |
return false |
| 134 |
end |
| 135 |
return true |
| 136 |
end |
| 137 |
|
| 138 |
return function(_, res) |
| 139 |
local db = sqlite3.open(res['source_db']) |
| 140 |
local shingles = {} |
| 141 |
local digests = {} |
| 142 |
local num_batch_digests = 0 |
| 143 |
local num_batch_shingles = 0 |
| 144 |
local total_digests = 0 |
| 145 |
local total_shingles = 0 |
| 146 |
local lim_batch = 1000 -- Update each 1000 entries |
| 147 |
local redis_username = res['redis_username'] |
| 148 |
local redis_password = res['redis_password'] |
| 149 |
local redis_db = nil |
| 150 |
|
| 151 |
if res['redis_db'] then |
| 152 |
redis_db = tostring(res['redis_db']) |
| 153 |
end |
| 154 |
|
| 155 |
if not db then |
| 156 |
print('Cannot open source db: ' .. res['source_db']) |
| 157 |
return |
| 158 |
end |
| 159 |
|
| 160 |
local now = util.get_time() |
| 161 |
for row in db:rows('SELECT id, flag, digest, value, time FROM digests') do |
| 162 |
|
| 163 |
local expire_in = math.floor(now - row.time + res['expiry']) |
| 164 |
if expire_in >= 1 then |
| 165 |
table.insert(digests, { row.digest, row.flag, row.value, expire_in }) |
| 166 |
num_batch_digests = num_batch_digests + 1 |
| 167 |
total_digests = total_digests + 1 |
| 168 |
for srow in db:rows('SELECT value, number FROM shingles WHERE digest_id = ' .. row.id) do |
| 169 |
table.insert(shingles, { srow.value, srow.number, expire_in, row.digest }) |
| 170 |
total_shingles = total_shingles + 1 |
| 171 |
num_batch_shingles = num_batch_shingles + 1 |
| 172 |
end |
| 173 |
end |
| 174 |
if num_batch_digests >= lim_batch then |
| 175 |
if not send_digests(digests, res['redis_host'], redis_username, redis_password, redis_db) then |
| 176 |
return |
| 177 |
end |
| 178 |
num_batch_digests = 0 |
| 179 |
digests = {} |
| 180 |
end |
| 181 |
if num_batch_shingles >= lim_batch then |
| 182 |
if not send_shingles(shingles, res['redis_host'], redis_username, redis_password, redis_db) then |
| 183 |
return |
| 184 |
end |
| 185 |
num_batch_shingles = 0 |
| 186 |
shingles = {} |
| 187 |
end |
| 188 |
end |
| 189 |
if digests[1] then |
| 190 |
if not send_digests(digests, res['redis_host'], redis_username, redis_password, redis_db) then |
| 191 |
return |
| 192 |
end |
| 193 |
end |
| 194 |
if shingles[1] then |
| 195 |
if not send_shingles(shingles, res['redis_host'], redis_username, redis_password, redis_db) then |
| 196 |
return |
| 197 |
end |
| 198 |
end |
| 199 |
|
| 200 |
local message = string.format( |
| 201 |
'Migrated %d digests and %d shingles', |
| 202 |
total_digests, total_shingles |
| 203 |
) |
| 204 |
if not update_counters(total_digests, res['redis_host'], redis_username, redis_password, redis_db) then |
| 205 |
message = message .. ' but failed to update counters' |
| 206 |
end |
| 207 |
print(message) |
| 208 |
end |