Debian Codegraph @

src/calibre/db/sqlite_extension.cpp

raw
1 /*
2 * sqlite_extension.cpp
3 * Copyright (C) 2021 Kovid Goyal <kovid at kovidgoyal.net>
4 *
5 * Distributed under terms of the GPL3 license.
6 */
7
8 #define PY_SSIZE_T_CLEAN
9 #define UNICODE
10 #include <Python.h>
11 #include <stdlib.h>
12 #include <string>
13 #include <vector>
14 #include <unordered_map>
15 #include <mutex>
16 #include <cstring>
17 #include <sqlite3ext.h>
18 #include <unicode/unistr.h>
19 #include <unicode/uchar.h>
20 #include <unicode/translit.h>
21 #include <unicode/errorcode.h>
22 #include <unicode/brkiter.h>
23 #include <unicode/uscript.h>
24 #if __has_include(<libstemmer.h>)
25 #include <libstemmer.h>
26 #else
27 #include <libstemmer/libstemmer.h>
28 #endif
29 #include "../utils/cpp_binding.h"
30 SQLITE_EXTENSION_INIT1
31
32 typedef int (*token_callback_func)(void *, int, const char *, int, int, int);
33
34
35 // Converting SQLITE text to ICU strings {{{
36 // UTF-8 decode taken from: https://bjoern.hoehrmann.de/utf-8/decoder/dfa/
37
38 static const uint8_t utf8_data[] = {
39 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 00..1f
40 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 20..3f
41 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 40..5f
42 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 60..7f
43 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, // 80..9f
44 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, // a0..bf
45 8, 8, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // c0..df
46 0xa, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x4, 0x3, 0x3, // e0..ef
47 0xb, 0x6, 0x6, 0x6, 0x5, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, // f0..ff
48 0x0, 0x1, 0x2, 0x3, 0x5, 0x8, 0x7, 0x1, 0x1, 0x1, 0x4, 0x6, 0x1, 0x1, 0x1, 0x1, // s0..s0
49 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, // s1..s2
50 1, 2, 1, 1, 1, 1, 1, 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, // s3..s4
51 1, 2, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 1, 3, 1, 1, 1, 1, 1, 1, // s5..s6
52 1, 3, 1, 1, 1, 1, 1, 3, 1, 3, 1, 1, 1, 1, 1, 1, 1, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // s7..s8
53 };
54
55
56 typedef enum UTF8State { UTF8_ACCEPT = 0, UTF8_REJECT = 1 } UTF8State;
57
58 uint32_t
59 decode_utf8(UTF8State *state, uint32_t *codep, uint8_t byte) {
60 uint32_t type = utf8_data[byte];
61
62 *codep = (*state != UTF8_ACCEPT) ? (byte & 0x3fu) | (*codep << 6) : (0xff >> type) & (byte);
63
64 *state = (UTF8State)utf8_data[256 + *state * 16 + type];
65 return *state;
66 }
67
68
69 static void
70 populate_icu_string(const char *text, int text_sz, icu::UnicodeString &str, std::vector<int> &byte_offsets) {
71 UTF8State state = UTF8_ACCEPT, prev = UTF8_ACCEPT;
72 uint32_t codep = 0;
73 for (int i = 0, pos = 0; i < text_sz; i++) {
74 switch (decode_utf8(&state, &codep, text[i])) {
75 case UTF8_ACCEPT: {
76 size_t sz = str.length();
77 str.append((UChar32)codep);
78 sz = str.length() - sz;
79 for (size_t x = 0; x < sz; x++) byte_offsets.push_back(pos);
80 pos = i + 1;
81 } break;
82 case UTF8_REJECT:
83 state = UTF8_ACCEPT;
84 if (prev != UTF8_ACCEPT && i > 0) i--;
85 break;
86 }
87 prev = state;
88 }
89 byte_offsets.push_back(text_sz);
90 }
91 // }}}
92
93 static char ui_language[16] = {'e', 'n', 0};
94 static std::mutex global_mutex;
95
96 class IteratorDescription {
97 public:
98 const char *language;
99 UScriptCode script;
100 };
101
102 class Stemmer {
103 private:
104 struct sb_stemmer *handle;
105 char lang_name[32];
106
107 public:
108 Stemmer() : handle(NULL), lang_name() {}
109 Stemmer(const char *lang) {
110 size_t len = strlen(lang);
111 for (size_t i = 0; i < sizeof(lang_name) - 1 && i < len; i++) {
112 lang_name[i] = lang[i];
113 if ('A' <= lang_name[i] && lang_name[i] <= 'Z') lang_name[i] += 'a' - 'A';
114 }
115 lang_name[std::min(len, sizeof(lang_name) - 1)] = 0;
116 handle = sb_stemmer_new(lang_name, NULL);
117 }
118 const char *
119 language_name() const {
120 return lang_name;
121 }
122
123 ~Stemmer() {
124 if (handle) {
125 sb_stemmer_delete(handle);
126 handle = NULL;
127 }
128 }
129
130 const char *
131 stem(const char *token, size_t token_sz, int *sz) {
132 const char *ans = NULL;
133 *sz = 0;
134 if (handle) {
135 ans = reinterpret_cast<const char *>(sb_stemmer_stem(handle, reinterpret_cast<const sb_symbol *>(token), (int)token_sz));
136 if (ans) *sz = sb_stemmer_length(handle);
137 }
138 return ans;
139 }
140
141 explicit
142 operator bool() const noexcept {
143 return handle != NULL;
144 }
145 };
146
147 typedef std::unique_ptr<icu::BreakIterator> BreakIterator;
148 typedef std::unique_ptr<Stemmer> StemmerPtr;
149 static const std::string empty_string("");
150
151 class Tokenizer {
152 private:
153 bool remove_diacritics, stem_words;
154 std::unique_ptr<icu::Transliterator> diacritics_remover;
155 std::vector<int> byte_offsets;
156 std::string token_buf, current_ui_language;
157 token_callback_func current_callback;
158 void *current_callback_ctx;
159 std::unordered_map<std::string, BreakIterator> iterators;
160 std::unordered_map<std::string, StemmerPtr> stemmers;
161
162 bool
163 is_token_char(UChar32 ch) const {
164 switch (u_charType(ch)) {
165 case U_UPPERCASE_LETTER:
166 case U_LOWERCASE_LETTER:
167 case U_TITLECASE_LETTER:
168 case U_MODIFIER_LETTER:
169 case U_OTHER_LETTER:
170 case U_DECIMAL_DIGIT_NUMBER:
171 case U_LETTER_NUMBER:
172 case U_OTHER_NUMBER:
173 case U_CURRENCY_SYMBOL:
174 case U_OTHER_SYMBOL:
175 case U_PRIVATE_USE_CHAR: return true;
176 default: break; ;
177 }
178 return false;
179 }
180
181 int
182 send_token(const icu::UnicodeString &token, int32_t start_offset, int32_t end_offset, StemmerPtr &stemmer, int flags = 0) {
183 token_buf.clear();
184 token_buf.reserve(4 * token.length());
185 token.toUTF8String(token_buf);
186 const char *root = token_buf.c_str();
187 int sz = (int)token_buf.size();
188 if (stem_words && stemmer->operator bool()) {
189 root = stemmer->stem(root, sz, &sz);
190 if (!root) {
191 root = token_buf.c_str();
192 sz = (int)token_buf.size();
193 }
194 }
195 return current_callback(current_callback_ctx, flags, root, (int)sz, byte_offsets.at(start_offset), byte_offsets.at(end_offset));
196 }
197
198 const char *
199 iterator_language_for_script(UScriptCode script) const {
200 switch (script) {
201 default: return "";
202 case USCRIPT_THAI:
203 case USCRIPT_LAO: return "th_TH";
204 case USCRIPT_KHMER: return "km_KH";
205 case USCRIPT_MYANMAR: return "my_MM";
206 case USCRIPT_HIRAGANA:
207 case USCRIPT_KATAKANA: return "ja_JP";
208 case USCRIPT_HANGUL: return "ko_KR";
209 case USCRIPT_HAN:
210 case USCRIPT_SIMPLIFIED_HAN:
211 case USCRIPT_TRADITIONAL_HAN:
212 case USCRIPT_HAN_WITH_BOPOMOFO: return "zh";
213 }
214 }
215
216 bool
217 at_script_boundary(IteratorDescription &current, UChar32 next_codepoint) const {
218 icu::ErrorCode err;
219 UScriptCode script = uscript_getScript(next_codepoint, err);
220 if (script == USCRIPT_COMMON || script == USCRIPT_INVALID_CODE || script == USCRIPT_INHERITED || current.script == script) return false;
221 const char *lang = iterator_language_for_script(script);
222 if (strcmp(current.language, lang) == 0) return false;
223 current.script = script;
224 current.language = lang;
225 return true;
226 }
227
228 void
229 ensure_basic_iterator(void) {
230 std::lock_guard<std::mutex> lock(global_mutex);
231 if (current_ui_language != ui_language || iterators.find(empty_string) == iterators.end()) {
232 current_ui_language.clear();
233 current_ui_language = ui_language;
234 icu::ErrorCode status;
235 if (current_ui_language.empty()) {
236 iterators[empty_string] = BreakIterator(icu::BreakIterator::createWordInstance(icu::Locale::getDefault(), status));
237 } else {
238 ensure_lang_iterator(ui_language);
239 }
240 }
241 }
242
243 BreakIterator &
244 ensure_lang_iterator(const char *lang = "") {
245 auto ans = iterators.find(lang);
246 if (ans == iterators.end()) {
247 icu::ErrorCode status;
248 iterators[lang] = BreakIterator(icu::BreakIterator::createWordInstance(icu::Locale::createCanonical(lang), status));
249 if (status.isFailure()) { iterators[lang] = BreakIterator(icu::BreakIterator::createWordInstance(icu::Locale::getDefault(), status)); }
250 ans = iterators.find(lang);
251 }
252 return ans->second;
253 }
254
255 StemmerPtr &
256 ensure_stemmer(const char *lang = "") {
257 if (!lang[0]) lang = current_ui_language.c_str();
258 auto ans = stemmers.find(lang);
259 if (ans == stemmers.end()) {
260 stemmers[lang] = stem_words ? std::make_unique<Stemmer>(lang) : std::make_unique<Stemmer>();
261 ans = stemmers.find(lang);
262 }
263 return ans->second;
264 }
265
266 int
267 tokenize_script_block(
268 const icu::UnicodeString &str,
269 int32_t block_start,
270 int32_t block_limit,
271 bool for_query_or_aux,
272 token_callback_func callback,
273 void *callback_ctx,
274 BreakIterator &word_iterator,
275 StemmerPtr &stemmer) {
276 word_iterator->setText(str.tempSubStringBetween(block_start, block_limit));
277 int32_t token_start_pos = word_iterator->first() + block_start, token_end_pos;
278 int rc = SQLITE_OK;
279 do {
280 token_end_pos = word_iterator->next();
281 if (token_end_pos == icu::BreakIterator::DONE) token_end_pos = block_limit;
282 else token_end_pos += block_start;
283 if (token_end_pos > token_start_pos) {
284 bool is_token = false;
285 for (int32_t pos = token_start_pos; !is_token && pos < token_end_pos; pos = str.moveIndex32(pos, 1)) {
286 if (is_token_char(str.char32At(pos))) is_token = true;
287 }
288 if (is_token) {
289 icu::UnicodeString token(str, token_start_pos, token_end_pos - token_start_pos);
290 token.foldCase();
291 if ((rc = send_token(token, token_start_pos, token_end_pos, stemmer)) != SQLITE_OK) return rc;
292 if (!for_query_or_aux && remove_diacritics) {
293 icu::UnicodeString tt(str, token_start_pos, token_end_pos - token_start_pos);
294 diacritics_remover->transliterate(tt);
295 tt.foldCase();
296 if (tt != token) {
297 if ((rc = send_token(tt, token_start_pos, token_end_pos, stemmer, FTS5_TOKEN_COLOCATED)) != SQLITE_OK) return rc;
298 }
299 }
300 }
301 }
302 token_start_pos = token_end_pos;
303 } while (token_end_pos < block_limit);
304 return rc;
305 }
306
307 public:
308 int constructor_error;
309
310 Tokenizer(const char **args, int nargs, bool stem_words = false)
311 : remove_diacritics(true), stem_words(stem_words), diacritics_remover(), byte_offsets(), token_buf(), current_ui_language(""), current_callback(NULL),
312 current_callback_ctx(NULL), iterators(), stemmers(),
313
314 constructor_error(SQLITE_OK) {
315 for (int i = 0; i < nargs; i++) {
316 if (strcmp(args[i], "remove_diacritics") == 0) {
317 i++;
318 if (i < nargs && strcmp(args[i], "0") == 0) remove_diacritics = false;
319 } else if (strcmp(args[i], "stem_words") == 0) {
320 i++;
321 if (i < nargs && strcmp(args[i], "0") == 0) stem_words = false;
322 else stem_words = true;
323 }
324 }
325 if (remove_diacritics) {
326 icu::ErrorCode status;
327 diacritics_remover.reset(icu::Transliterator::createInstance("NFD; [:M:] Remove; NFC", UTRANS_FORWARD, status));
328 if (status.isFailure()) {
329 fprintf(stderr, "Failed to create ICU transliterator to remove diacritics with error: %s\n", status.errorName());
330 constructor_error = SQLITE_INTERNAL;
331 diacritics_remover.reset(NULL);
332 remove_diacritics = false;
333 }
334 }
335 std::lock_guard<std::mutex> lock(global_mutex);
336 current_ui_language = ui_language;
337 }
338
339 int
340 tokenize(void *callback_ctx, int flags, const char *text, int text_sz, token_callback_func callback) {
341 ensure_basic_iterator();
342 current_callback = callback;
343 current_callback_ctx = callback_ctx;
344 icu::UnicodeString str(text_sz, 0, 0);
345 byte_offsets.clear();
346 byte_offsets.reserve(text_sz + 8);
347 populate_icu_string(text, text_sz, str, byte_offsets);
348 int32_t offset = str.getChar32Start(0);
349 int rc = SQLITE_OK;
350 const bool for_query_or_aux = (flags & (FTS5_TOKENIZE_QUERY | FTS5_TOKENIZE_AUX)) != 0;
351 IteratorDescription state;
352 state.language = "";
353 state.script = USCRIPT_COMMON;
354 int32_t start_script_block_at = offset;
355 auto word_iterator = std::ref(ensure_lang_iterator(state.language));
356 auto stemmer = std::ref(ensure_stemmer(state.language));
357 while (offset < str.length()) {
358 UChar32 ch = str.char32At(offset);
359 if (at_script_boundary(state, ch)) {
360 if (offset > start_script_block_at) {
361 if ((rc = tokenize_script_block(str, start_script_block_at, offset, for_query_or_aux, callback, callback_ctx, word_iterator, stemmer)) !=
362 SQLITE_OK)
363 return rc;
364 }
365 start_script_block_at = offset;
366 word_iterator = ensure_lang_iterator(state.language);
367 stemmer = ensure_stemmer(state.language);
368 }
369 offset = str.moveIndex32(offset, 1);
370 }
371 if (offset > start_script_block_at) {
372 rc = tokenize_script_block(str, start_script_block_at, offset, for_query_or_aux, callback, callback_ctx, word_iterator, stemmer);
373 }
374 return rc;
375 }
376 };
377
378 // boilerplate {{{
379 static int
380 fts5_api_from_db(sqlite3 *db, fts5_api **ppApi) {
381 sqlite3_stmt *pStmt = 0;
382 *ppApi = 0;
383 int rc = sqlite3_prepare(db, "SELECT fts5(?1)", -1, &pStmt, 0);
384 if (rc == SQLITE_OK) {
385 sqlite3_bind_pointer(pStmt, 1, reinterpret_cast<void *>(ppApi), "fts5_api_ptr", 0);
386 (void)sqlite3_step(pStmt);
387 rc = sqlite3_finalize(pStmt);
388 }
389 return rc;
390 }
391
392 static int
393 _tok_create(void *sqlite3, const char **azArg, int nArg, Fts5Tokenizer **ppOut, bool stem_words = false) {
394 int rc = SQLITE_OK;
395 try {
396 Tokenizer *p = new Tokenizer(azArg, nArg, stem_words);
397 if (p->constructor_error != SQLITE_OK) {
398 rc = p->constructor_error;
399 delete p;
400 } else {
401 *ppOut = reinterpret_cast<Fts5Tokenizer *>(p);
402 }
403 } catch (std::bad_alloc const &) { return SQLITE_NOMEM; } catch (...) {
404 return SQLITE_ERROR;
405 }
406 return rc;
407 }
408
409 static int
410 tok_create(void *sqlite3, const char **azArg, int nArg, Fts5Tokenizer **ppOut) {
411 return _tok_create(sqlite3, azArg, nArg, ppOut);
412 }
413
414 static int
415 tok_create_with_stemming(void *sqlite3, const char **azArg, int nArg, Fts5Tokenizer **ppOut) {
416 return _tok_create(sqlite3, azArg, nArg, ppOut, true);
417 }
418
419 static int
420 tok_tokenize(Fts5Tokenizer *tokenizer_ptr, void *callback_ctx, int flags, const char *text, int text_sz, token_callback_func callback) {
421 Tokenizer *p = reinterpret_cast<Tokenizer *>(tokenizer_ptr);
422 try {
423 return p->tokenize(callback_ctx, flags, text, text_sz, callback);
424 } catch (std::bad_alloc const &) { return SQLITE_NOMEM; } catch (...) {
425 return SQLITE_ERROR;
426 }
427 }
428
429 static void
430 tok_delete(Fts5Tokenizer *p) {
431 Tokenizer *t = reinterpret_cast<Tokenizer *>(p);
432 delete t;
433 }
434
435 extern "C" {
436 #ifdef _MSC_VER
437 #define MYEXPORT __declspec(dllexport)
438 #else
439 #define MYEXPORT __attribute__((visibility("default")))
440 #endif
441
442 MYEXPORT int
443 calibre_sqlite_extension_init(sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi) {
444 SQLITE_EXTENSION_INIT2(pApi);
445 fts5_api *fts5api = NULL;
446 int rc = fts5_api_from_db(db, &fts5api);
447 if (rc != SQLITE_OK) {
448 *pzErrMsg = (char *)"Failed to get FTS 5 API with error code";
449 return rc;
450 }
451 if (!fts5api || fts5api->iVersion < 2) {
452 *pzErrMsg = (char *)"FTS 5 iVersion too old or NULL pointer";
453 return SQLITE_ERROR;
454 }
455 fts5_tokenizer tok = {tok_create, tok_delete, tok_tokenize};
456 fts5api->xCreateTokenizer(fts5api, "unicode61", reinterpret_cast<void *>(fts5api), &tok, NULL);
457 fts5api->xCreateTokenizer(fts5api, "calibre", reinterpret_cast<void *>(fts5api), &tok, NULL);
458 fts5_tokenizer tok2 = {tok_create_with_stemming, tok_delete, tok_tokenize};
459 fts5api->xCreateTokenizer(fts5api, "porter", reinterpret_cast<void *>(fts5api), &tok2, NULL);
460 return SQLITE_OK;
461 }
462
463 MYEXPORT int
464 sqlite3_sqliteextension_init(sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi) {
465 return calibre_sqlite_extension_init(db, pzErrMsg, pApi);
466 }
467 }
468
469 static PyObject *
470 get_locales_for_break_iteration(PyObject *self, PyObject *args) {
471 std::unique_ptr<icu::StringEnumeration> locs(icu::BreakIterator::getAvailableLocales());
472 icu::ErrorCode status;
473 pyobject_raii ans(PyList_New(0));
474 if (ans) {
475 const icu::UnicodeString *item;
476 while ((item = locs->snext(status))) {
477 std::string name;
478 item->toUTF8String(name);
479 pyobject_raii pn(PyUnicode_FromString(name.c_str()));
480 if (pn) PyList_Append(ans.ptr(), pn.ptr());
481 }
482 if (status.isFailure()) {
483 PyErr_Format(PyExc_RuntimeError, "Failed to iterate over locales with error: %s", status.errorName());
484 return NULL;
485 }
486 }
487 return ans.detach();
488 }
489
490 static PyObject *
491 set_ui_language(PyObject *self, PyObject *args) {
492 std::lock_guard<std::mutex> lock(global_mutex);
493 const char *val;
494 if (!PyArg_ParseTuple(args, "s", &val)) return NULL;
495 strncpy(ui_language, val, sizeof(ui_language) - 1);
496 Py_RETURN_NONE;
497 }
498
499 static int
500 py_callback(void *ctx, int flags, const char *text, int text_length, int start_offset, int end_offset) {
501 PyObject *ans = reinterpret_cast<PyObject *>(ctx);
502 Py_ssize_t psz = text_length;
503 pyobject_raii item(Py_BuildValue("{ss# si si si}", "text", text, psz, "start", start_offset, "end", end_offset, "flags", flags));
504 if (item) PyList_Append(ans, item.ptr());
505 return SQLITE_OK;
506 }
507
508 static PyObject *
509 tokenize(PyObject *self, PyObject *args) {
510 const char *text;
511 Py_ssize_t text_length, remove_diacritics = 1, flags = FTS5_TOKENIZE_DOCUMENT;
512 if (!PyArg_ParseTuple(args, "s#|pi", &text, &text_length, &remove_diacritics, &flags)) return NULL;
513 const char *targs[2] = {"remove_diacritics", "2"};
514 if (!remove_diacritics) targs[1] = "0";
515 Tokenizer t(targs, sizeof(targs) / sizeof(targs[0]));
516 pyobject_raii ans(PyList_New(0));
517 if (!ans) return NULL;
518 t.tokenize(ans.ptr(), flags, text, text_length, py_callback);
519 return ans.detach();
520 }
521
522 static PyObject *
523 stem(PyObject *self, PyObject *args) {
524 const char *text, *lang = "en";
525 Py_ssize_t text_length;
526 if (!PyArg_ParseTuple(args, "s#|s", &text, &text_length, &lang)) return NULL;
527 Stemmer s(lang);
528 if (!s) {
529 PyErr_SetString(PyExc_ValueError, "No stemmer for the specified language");
530 return NULL;
531 }
532 int sz;
533 const char *result = s.stem(text, text_length, &sz);
534 Py_ssize_t a = sz;
535 if (!result) return PyErr_NoMemory();
536 return Py_BuildValue("s#", result, a);
537 }
538
539 static PyMethodDef methods[] = {
540 {"get_locales_for_break_iteration", get_locales_for_break_iteration, METH_NOARGS, "Get list of available locales for break iteration"},
541 {"set_ui_language", set_ui_language, METH_VARARGS, "Set the current UI language"},
542 {"tokenize", tokenize, METH_VARARGS, "Tokenize a string, useful for testing"},
543 {"stem", stem, METH_VARARGS, "Stem a word in the specified language, defaulting to English"},
544 {NULL, NULL, 0, NULL}};
545
546 static int
547 exec_module(PyObject *mod) {
548 if (PyModule_AddIntMacro(mod, FTS5_TOKENIZE_QUERY) != 0) return 1;
549 if (PyModule_AddIntMacro(mod, FTS5_TOKENIZE_DOCUMENT) != 0) return 1;
550 if (PyModule_AddIntMacro(mod, FTS5_TOKENIZE_PREFIX) != 0) return 1;
551 if (PyModule_AddIntMacro(mod, FTS5_TOKENIZE_AUX) != 0) return 1;
552 if (PyModule_AddIntMacro(mod, FTS5_TOKEN_COLOCATED) != 0) return 1;
553 return 0;
554 }
555
556 static PyModuleDef_Slot slots[] = {{Py_mod_exec, (void *)exec_module}, {0, NULL}};
557
558 static struct PyModuleDef module_def = {PyModuleDef_HEAD_INIT};
559
560 extern "C" {
561 CALIBRE_MODINIT_FUNC
562 PyInit_sqlite_extension(void) {
563 module_def.m_name = "sqlite_extension";
564 module_def.m_doc = "Implement ICU based tokenizer for FTS5";
565 module_def.m_methods = methods;
566 module_def.m_slots = slots;
567 return PyModuleDef_Init(&module_def);
568 }
569 } // }}}