Debian Codegraph @

lualib/lua_clickhouse.lua

raw
1 --[[
2 Copyright (c) 2022, Vsevolod Stakhov <[email protected]>
3 Copyright (c) 2018, Mikhail Galanin <[email protected]>
4
5 Licensed under the Apache License, Version 2.0 (the "License");
6 you may not use this file except in compliance with the License.
7 You may obtain a copy of the License at
8
9 http://www.apache.org/licenses/LICENSE-2.0
10
11 Unless required by applicable law or agreed to in writing, software
12 distributed under the License is distributed on an "AS IS" BASIS,
13 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 See the License for the specific language governing permissions and
15 limitations under the License.
16 ]] --
17
18 --[[[
19 -- @module lua_clickhouse
20 -- This module contains Clickhouse access functions
21 --]]
22
23 local rspamd_logger = require "rspamd_logger"
24 local rspamd_http = require "rspamd_http"
25 local lua_util = require "lua_util"
26 local rspamd_text = require "rspamd_text"
27
28 local exports = {}
29 local N = 'clickhouse'
30
31 local default_timeout = 10.0
32
33 local function escape_spaces(query)
34 return query:gsub('%s', '%%20')
35 end
36
37 local function ch_number(a)
38 if (a + 2 ^ 52) - 2 ^ 52 == a then
39 -- Integer
40 return tostring(math.floor(a))
41 end
42
43 return tostring(a)
44 end
45
46 local function clickhouse_quote(str)
47 if str then
48 return str:gsub('[\'\\\n\t\r]', {
49 ['\''] = [[\']],
50 ['\\'] = [[\\]],
51 ['\n'] = [[\n]],
52 ['\t'] = [[\t]],
53 ['\r'] = [[\r]],
54 })
55 end
56
57 return ''
58 end
59
60 -- Converts an array to a string suitable for clickhouse
61 local function array_to_string(ar)
62 for i, elt in ipairs(ar) do
63 local t = type(elt)
64 if t == 'string' then
65 ar[i] = string.format('\'%s\'', clickhouse_quote(elt))
66 elseif t == 'userdata' then
67 ar[i] = string.format('\'%s\'', clickhouse_quote(tostring(elt)))
68 elseif t == 'number' then
69 ar[i] = ch_number(elt)
70 end
71 end
72
73 return table.concat(ar, ',')
74 end
75
76 -- Converts a row into TSV, taking extra care about arrays
77 local function row_to_tsv(row)
78 for i, elt in ipairs(row) do
79 local t = type(elt)
80 if t == 'table' then
81 row[i] = '[' .. array_to_string(elt) .. ']'
82 elseif t == 'number' then
83 row[i] = ch_number(elt)
84 elseif t == 'userdata' then
85 row[i] = clickhouse_quote(tostring(elt))
86 else
87 row[i] = clickhouse_quote(elt)
88 end
89 end
90
91 return rspamd_text.fromtable(row, '\t')
92 end
93
94 exports.row_to_tsv = row_to_tsv
95
96 -- Parses JSONEachRow reply from CH
97 local function parse_clickhouse_response_json_eachrow(params, data, row_cb)
98 local ucl = require "ucl"
99
100 if data == nil then
101 -- clickhouse returned no data (i.e. empty result set): exiting
102 return {}
103 end
104
105 local function parse_string(s)
106 local parser = ucl.parser()
107 local res, err
108 if type(s) == 'string' then
109 res, err = parser:parse_string(s)
110 else
111 res, err = parser:parse_text(s)
112 end
113
114 if not res then
115 rspamd_logger.errx(params.log_obj, 'Parser error: %s', err)
116 return nil
117 end
118 return parser:get_object()
119 end
120
121 -- iterate over rows and parse
122 local parsed_rows = {}
123 for plain_row in data:lines() do
124 if plain_row and #plain_row > 1 then
125 local parsed_row = parse_string(plain_row)
126 if parsed_row then
127 if row_cb then
128 row_cb(parsed_row)
129 else
130 table.insert(parsed_rows, parsed_row)
131 end
132 end
133 end
134 end
135
136 return parsed_rows
137 end
138
139 -- Parses JSON reply from CH
140 local function parse_clickhouse_response_json(params, data)
141 local ucl = require "ucl"
142
143 if data == nil then
144 -- clickhouse returned no data (i.e. empty result set) considered valid!
145 return nil, {}
146 end
147
148 local function parse_string(s)
149 local parser = ucl.parser()
150 local res, err
151
152 if type(s) == 'string' then
153 res, err = parser:parse_string(s)
154 else
155 res, err = parser:parse_text(s)
156 end
157
158 if not res then
159 rspamd_logger.errx(params.log_obj, 'Parser error: %s', err)
160 return nil
161 end
162 return parser:get_object()
163 end
164
165 local json = parse_string(data)
166
167 if not json then
168 return 'bad json', {}
169 end
170
171 return nil, json
172 end
173
174 -- Helper to generate HTTP closure
175 local function mk_http_select_cb(upstream, params, ok_cb, fail_cb, row_cb)
176 local function http_cb(err_message, code, data, _)
177 if code ~= 200 or err_message then
178 if not err_message then
179 err_message = data
180 end
181 local ip_addr = upstream:get_addr():to_string(true)
182
183 if fail_cb then
184 fail_cb(params, err_message, data)
185 else
186 rspamd_logger.errx(params.log_obj,
187 "request failed on clickhouse server %s: %s",
188 ip_addr, err_message)
189 end
190 upstream:fail()
191 else
192 upstream:ok()
193 local rows = parse_clickhouse_response_json_eachrow(params, data, row_cb)
194
195 if rows then
196 if ok_cb then
197 ok_cb(params, rows)
198 else
199 lua_util.debugm(N, params.log_obj,
200 "http_select_cb ok: %s, %s, %s, %s", err_message, code,
201 data:gsub('[\n%s]+', ' '), _)
202 end
203 else
204 if fail_cb then
205 fail_cb(params, 'failed to parse reply', data)
206 else
207 local ip_addr = upstream:get_addr():to_string(true)
208 rspamd_logger.errx(params.log_obj,
209 "request failed on clickhouse server %s: %s",
210 ip_addr, 'failed to parse reply')
211 end
212 end
213 end
214 end
215
216 return http_cb
217 end
218
219 -- Helper to generate HTTP closure
220 local function mk_http_insert_cb(upstream, params, ok_cb, fail_cb)
221 local function http_cb(err_message, code, data, _)
222 if code ~= 200 or err_message then
223 if not err_message then
224 err_message = data
225 end
226 local ip_addr = upstream:get_addr():to_string(true)
227
228 if fail_cb then
229 fail_cb(params, err_message, data)
230 else
231 rspamd_logger.errx(params.log_obj,
232 "request failed on clickhouse server %s: %s",
233 ip_addr, err_message)
234 end
235 upstream:fail()
236 else
237 upstream:ok()
238
239 if ok_cb then
240 local err, parsed = parse_clickhouse_response_json(data)
241
242 if err then
243 fail_cb(params, err, data)
244 else
245 ok_cb(params, parsed)
246 end
247 else
248 lua_util.debugm(N, params.log_obj,
249 "http_insert_cb ok: %s, %s, %s, %s", err_message, code,
250 data:gsub('[\n%s]+', ' '), _)
251 end
252 end
253 end
254
255 return http_cb
256 end
257
258 --[[[
259 -- @function lua_clickhouse.select(upstream, settings, params, query,
260 ok_cb, fail_cb)
261 -- Make select request to clickhouse
262 -- @param {upstream} upstream clickhouse server upstream
263 -- @param {table} settings global settings table:
264 -- * use_gsip: use gzip compression
265 -- * timeout: request timeout
266 -- * no_ssl_verify: skip SSL verification
267 -- * user: HTTP user
268 -- * password: HTTP password
269 -- @param {params} HTTP request params
270 -- @param {string} query select query (passed in HTTP body)
271 -- @param {function} ok_cb callback to be called in case of success
272 -- @param {function} fail_cb callback to be called in case of some error
273 -- @param {function} row_cb optional callback to be called on each parsed data row (instead of table insertion)
274 -- @return {boolean} whether a connection was successful
275 -- @example
276 --
277 --]]
278 exports.select = function(upstream, settings, params, query, ok_cb, fail_cb, row_cb)
279 local http_params = {}
280
281 for k, v in pairs(params) do
282 http_params[k] = v
283 end
284
285 http_params.callback = mk_http_select_cb(upstream, http_params, ok_cb, fail_cb, row_cb)
286 http_params.gzip = settings.use_gzip
287 http_params.mime_type = 'text/plain'
288 http_params.timeout = settings.timeout or default_timeout
289 http_params.no_ssl_verify = settings.no_ssl_verify
290 http_params.user = settings.user
291 http_params.password = settings.password
292 http_params.body = query
293 http_params.log_obj = params.task or params.config
294 http_params.opaque_body = true
295 -- staged timeouts
296 http_params.connect_timeout = settings.connect_timeout
297 http_params.ssl_timeout = settings.ssl_timeout
298 http_params.write_timeout = settings.write_timeout
299 http_params.read_timeout = settings.read_timeout
300
301 lua_util.debugm(N, http_params.log_obj, "clickhouse select request: %s", http_params.body)
302
303 if not http_params.url then
304 local connect_prefix = "http://"
305 if settings.use_https then
306 connect_prefix = 'https://'
307 end
308 local ip_addr = upstream:get_addr():to_string(true)
309 local database = settings.database or 'default'
310 http_params.url = string.format('%s%s/?database=%s&default_format=JSONEachRow',
311 connect_prefix, ip_addr, escape_spaces(database))
312 end
313
314 return rspamd_http.request(http_params)
315 end
316
317 --[[[
318 -- @function lua_clickhouse.select_sync(upstream, settings, params, query,
319 ok_cb, fail_cb, row_cb)
320 -- Make select request to clickhouse
321 -- @param {upstream} upstream clickhouse server upstream
322 -- @param {table} settings global settings table:
323 -- * use_gsip: use gzip compression
324 -- * timeout: request timeout
325 -- * no_ssl_verify: skip SSL verification
326 -- * user: HTTP user
327 -- * password: HTTP password
328 -- @param {params} HTTP request params
329 -- @param {string} query select query (passed in HTTP body)
330 -- @param {function} ok_cb callback to be called in case of success
331 -- @param {function} fail_cb callback to be called in case of some error
332 -- @param {function} row_cb optional callback to be called on each parsed data row (instead of table insertion)
333 -- @return
334 -- {string} error message if exists
335 -- nil | {rows} | {http_response}
336 -- @example
337 --
338 --]]
339 exports.select_sync = function(upstream, settings, params, query, row_cb)
340 local http_params = {}
341
342 for k, v in pairs(params) do
343 http_params[k] = v
344 end
345
346 http_params.gzip = settings.use_gzip
347 http_params.mime_type = 'text/plain'
348 http_params.timeout = settings.timeout or default_timeout
349 http_params.no_ssl_verify = settings.no_ssl_verify
350 http_params.user = settings.user
351 http_params.password = settings.password
352 http_params.body = query
353 http_params.log_obj = params.task or params.config
354 http_params.opaque_body = true
355 -- staged timeouts
356 http_params.connect_timeout = settings.connect_timeout
357 http_params.ssl_timeout = settings.ssl_timeout
358 http_params.write_timeout = settings.write_timeout
359 http_params.read_timeout = settings.read_timeout
360
361 lua_util.debugm(N, http_params.log_obj, "clickhouse select request: %s", http_params.body)
362
363 if not http_params.url then
364 local connect_prefix = "http://"
365 if settings.use_https then
366 connect_prefix = 'https://'
367 end
368 local ip_addr = upstream:get_addr():to_string(true)
369 local database = settings.database or 'default'
370 http_params.url = string.format('%s%s/?database=%s&default_format=JSONEachRow',
371 connect_prefix, ip_addr, escape_spaces(database))
372 end
373
374 local err, response = rspamd_http.request(http_params)
375
376 if err then
377 return err, nil
378 elseif response.code ~= 200 then
379 return response.content, response
380 else
381 lua_util.debugm(N, http_params.log_obj, "clickhouse select response: %1", response)
382 local rows = parse_clickhouse_response_json_eachrow(params, response.content, row_cb)
383 return nil, rows
384 end
385 end
386
387 --[[[
388 -- @function lua_clickhouse.insert(upstream, settings, params, query, rows,
389 ok_cb, fail_cb)
390 -- Insert data rows to clickhouse
391 -- @param {upstream} upstream clickhouse server upstream
392 -- @param {table} settings global settings table:
393 -- * use_gsip: use gzip compression
394 -- * timeout: request timeout
395 -- * no_ssl_verify: skip SSL verification
396 -- * user: HTTP user
397 -- * password: HTTP password
398 -- @param {params} HTTP request params
399 -- @param {string} query select query (passed in `query` request element with spaces escaped)
400 -- @param {table|mixed} rows mix of strings, numbers or tables (for arrays)
401 -- @param {function} ok_cb callback to be called in case of success
402 -- @param {function} fail_cb callback to be called in case of some error
403 -- @return {boolean} whether a connection was successful
404 -- @example
405 --
406 --]]
407 exports.insert = function(upstream, settings, params, query, rows,
408 ok_cb, fail_cb)
409 local http_params = {}
410
411 for k, v in pairs(params) do
412 http_params[k] = v
413 end
414
415 http_params.callback = mk_http_insert_cb(upstream, http_params, ok_cb, fail_cb)
416 http_params.gzip = settings.use_gzip
417 http_params.mime_type = 'text/plain'
418 http_params.timeout = settings.timeout or default_timeout
419 http_params.no_ssl_verify = settings.no_ssl_verify
420 http_params.user = settings.user
421 http_params.password = settings.password
422 http_params.method = 'POST'
423 http_params.body = { rspamd_text.fromtable(rows, '\n'), '\n' }
424 http_params.log_obj = params.task or params.config
425 -- staged timeouts
426 http_params.connect_timeout = settings.connect_timeout
427 http_params.ssl_timeout = settings.ssl_timeout
428 http_params.write_timeout = settings.write_timeout
429 http_params.read_timeout = settings.read_timeout
430
431 if not http_params.url then
432 local connect_prefix = "http://"
433 if settings.use_https then
434 connect_prefix = 'https://'
435 end
436 local ip_addr = upstream:get_addr():to_string(true)
437 local database = settings.database or 'default'
438 http_params.url = string.format('%s%s/?database=%s&query=%s%%20FORMAT%%20TabSeparated',
439 connect_prefix,
440 ip_addr,
441 escape_spaces(database),
442 escape_spaces(query))
443 end
444
445 return rspamd_http.request(http_params)
446 end
447
448 --[[[
449 -- @function lua_clickhouse.generic(upstream, settings, params, query,
450 ok_cb, fail_cb)
451 -- Make a generic request to Clickhouse (e.g. alter)
452 -- @param {upstream} upstream clickhouse server upstream
453 -- @param {table} settings global settings table:
454 -- * use_gsip: use gzip compression
455 -- * timeout: request timeout
456 -- * no_ssl_verify: skip SSL verification
457 -- * user: HTTP user
458 -- * password: HTTP password
459 -- @param {params} HTTP request params
460 -- @param {string} query Clickhouse query (passed in `query` request element with spaces escaped)
461 -- @param {function} ok_cb callback to be called in case of success
462 -- @param {function} fail_cb callback to be called in case of some error
463 -- @return {boolean} whether a connection was successful
464 -- @example
465 --
466 --]]
467 exports.generic = function(upstream, settings, params, query,
468 ok_cb, fail_cb)
469 local http_params = {}
470
471 for k, v in pairs(params) do
472 http_params[k] = v
473 end
474
475 http_params.callback = mk_http_insert_cb(upstream, http_params, ok_cb, fail_cb)
476 http_params.gzip = settings.use_gzip
477 http_params.mime_type = 'text/plain'
478 http_params.timeout = settings.timeout or default_timeout
479 http_params.no_ssl_verify = settings.no_ssl_verify
480 http_params.user = settings.user
481 http_params.password = settings.password
482 http_params.log_obj = params.task or params.config
483 http_params.body = query
484 -- staged timeouts
485 http_params.connect_timeout = settings.connect_timeout
486 http_params.ssl_timeout = settings.ssl_timeout
487 http_params.write_timeout = settings.write_timeout
488 http_params.read_timeout = settings.read_timeout
489
490 if not http_params.url then
491 local connect_prefix = "http://"
492 if settings.use_https then
493 connect_prefix = 'https://'
494 end
495 local ip_addr = upstream:get_addr():to_string(true)
496 local database = settings.database or 'default'
497 http_params.url = string.format('%s%s/?database=%s&default_format=JSONEachRow',
498 connect_prefix, ip_addr, escape_spaces(database))
499 end
500
501 return rspamd_http.request(http_params)
502 end
503
504 --[[[
505 -- @function lua_clickhouse.generic_sync(upstream, settings, params, query,
506 ok_cb, fail_cb)
507 -- Make a generic request to Clickhouse (e.g. alter)
508 -- @param {upstream} upstream clickhouse server upstream
509 -- @param {table} settings global settings table:
510 -- * use_gsip: use gzip compression
511 -- * timeout: request timeout
512 -- * no_ssl_verify: skip SSL verification
513 -- * user: HTTP user
514 -- * password: HTTP password
515 -- @param {params} HTTP request params
516 -- @param {string} query Clickhouse query (passed in `query` request element with spaces escaped)
517 -- @return {boolean} whether a connection was successful
518 -- @example
519 --
520 --]]
521 exports.generic_sync = function(upstream, settings, params, query)
522 local http_params = {}
523
524 for k, v in pairs(params) do
525 http_params[k] = v
526 end
527
528 http_params.gzip = settings.use_gzip
529 http_params.mime_type = 'text/plain'
530 http_params.timeout = settings.timeout or default_timeout
531 http_params.no_ssl_verify = settings.no_ssl_verify
532 http_params.user = settings.user
533 http_params.password = settings.password
534 http_params.log_obj = params.task or params.config
535 http_params.body = query
536 -- staged timeouts
537 http_params.connect_timeout = settings.connect_timeout
538 http_params.ssl_timeout = settings.ssl_timeout
539 http_params.write_timeout = settings.write_timeout
540 http_params.read_timeout = settings.read_timeout
541
542 if not http_params.url then
543 local connect_prefix = "http://"
544 if settings.use_https then
545 connect_prefix = 'https://'
546 end
547 local ip_addr = upstream:get_addr():to_string(true)
548 local database = settings.database or 'default'
549 http_params.url = string.format('%s%s/?database=%s&default_format=JSON',
550 connect_prefix, ip_addr, escape_spaces(database))
551 end
552
553 local err, response = rspamd_http.request(http_params)
554
555 if err then
556 return err, nil
557 elseif response.code ~= 200 then
558 return response.content, response
559 else
560 lua_util.debugm(N, http_params.log_obj, "clickhouse generic response: %1", response)
561 local e, obj = parse_clickhouse_response_json(params, response.content)
562
563 if e then
564 return e, nil
565 end
566 return nil, obj
567 end
568 end
569
570 return exports