| 1 |
import valkey |
| 2 |
|
| 3 |
from ...asyncio.client import Pipeline as AsyncioPipeline |
| 4 |
from .commands import ( |
| 5 |
AGGREGATE_CMD, |
| 6 |
CONFIG_CMD, |
| 7 |
INFO_CMD, |
| 8 |
PROFILE_CMD, |
| 9 |
SEARCH_CMD, |
| 10 |
SPELLCHECK_CMD, |
| 11 |
SYNDUMP_CMD, |
| 12 |
AsyncSearchCommands, |
| 13 |
SearchCommands, |
| 14 |
) |
| 15 |
|
| 16 |
|
| 17 |
class Search(SearchCommands): |
| 18 |
""" |
| 19 |
Create a client for talking to search. |
| 20 |
It abstracts the API of the module and lets you just use the engine. |
| 21 |
""" |
| 22 |
|
| 23 |
class BatchIndexer: |
| 24 |
""" |
| 25 |
A batch indexer allows you to automatically batch |
| 26 |
document indexing in pipelines, flushing it every N documents. |
| 27 |
""" |
| 28 |
|
| 29 |
def __init__(self, client, chunk_size=1000): |
| 30 |
self.client = client |
| 31 |
self.execute_command = client.execute_command |
| 32 |
self._pipeline = client.pipeline(transaction=False, shard_hint=None) |
| 33 |
self.total = 0 |
| 34 |
self.chunk_size = chunk_size |
| 35 |
self.current_chunk = 0 |
| 36 |
|
| 37 |
def __del__(self): |
| 38 |
if self.current_chunk: |
| 39 |
self.commit() |
| 40 |
|
| 41 |
def add_document( |
| 42 |
self, |
| 43 |
doc_id, |
| 44 |
nosave=False, |
| 45 |
score=1.0, |
| 46 |
payload=None, |
| 47 |
replace=False, |
| 48 |
partial=False, |
| 49 |
no_create=False, |
| 50 |
**fields, |
| 51 |
): |
| 52 |
""" |
| 53 |
Add a document to the batch query |
| 54 |
""" |
| 55 |
self.client._add_document( |
| 56 |
doc_id, |
| 57 |
conn=self._pipeline, |
| 58 |
nosave=nosave, |
| 59 |
score=score, |
| 60 |
payload=payload, |
| 61 |
replace=replace, |
| 62 |
partial=partial, |
| 63 |
no_create=no_create, |
| 64 |
**fields, |
| 65 |
) |
| 66 |
self.current_chunk += 1 |
| 67 |
self.total += 1 |
| 68 |
if self.current_chunk >= self.chunk_size: |
| 69 |
self.commit() |
| 70 |
|
| 71 |
def add_document_hash(self, doc_id, score=1.0, replace=False): |
| 72 |
""" |
| 73 |
Add a hash to the batch query |
| 74 |
""" |
| 75 |
self.client._add_document_hash( |
| 76 |
doc_id, conn=self._pipeline, score=score, replace=replace |
| 77 |
) |
| 78 |
self.current_chunk += 1 |
| 79 |
self.total += 1 |
| 80 |
if self.current_chunk >= self.chunk_size: |
| 81 |
self.commit() |
| 82 |
|
| 83 |
def commit(self): |
| 84 |
""" |
| 85 |
Manually commit and flush the batch indexing query |
| 86 |
""" |
| 87 |
self._pipeline.execute() |
| 88 |
self.current_chunk = 0 |
| 89 |
|
| 90 |
def __init__(self, client, index_name="idx"): |
| 91 |
""" |
| 92 |
Create a new Client for the given index_name. |
| 93 |
The default name is `idx` |
| 94 |
|
| 95 |
If conn is not None, we employ an already existing valkey connection |
| 96 |
""" |
| 97 |
self._MODULE_CALLBACKS = {} |
| 98 |
self.client = client |
| 99 |
self.index_name = index_name |
| 100 |
self.execute_command = client.execute_command |
| 101 |
self._pipeline = client.pipeline |
| 102 |
self._RESP2_MODULE_CALLBACKS = { |
| 103 |
INFO_CMD: self._parse_info, |
| 104 |
SEARCH_CMD: self._parse_search, |
| 105 |
AGGREGATE_CMD: self._parse_aggregate, |
| 106 |
PROFILE_CMD: self._parse_profile, |
| 107 |
SPELLCHECK_CMD: self._parse_spellcheck, |
| 108 |
CONFIG_CMD: self._parse_config_get, |
| 109 |
SYNDUMP_CMD: self._parse_syndump, |
| 110 |
} |
| 111 |
|
| 112 |
def pipeline(self, transaction=True, shard_hint=None): |
| 113 |
"""Creates a pipeline for the SEARCH module, that can be used for executing |
| 114 |
SEARCH commands, as well as classic core commands. |
| 115 |
""" |
| 116 |
p = Pipeline( |
| 117 |
connection_pool=self.client.connection_pool, |
| 118 |
response_callbacks=self._MODULE_CALLBACKS, |
| 119 |
transaction=transaction, |
| 120 |
shard_hint=shard_hint, |
| 121 |
) |
| 122 |
p.index_name = self.index_name |
| 123 |
return p |
| 124 |
|
| 125 |
|
| 126 |
class AsyncSearch(Search, AsyncSearchCommands): |
| 127 |
class BatchIndexer(Search.BatchIndexer): |
| 128 |
""" |
| 129 |
A batch indexer allows you to automatically batch |
| 130 |
document indexing in pipelines, flushing it every N documents. |
| 131 |
""" |
| 132 |
|
| 133 |
async def add_document( |
| 134 |
self, |
| 135 |
doc_id, |
| 136 |
nosave=False, |
| 137 |
score=1.0, |
| 138 |
payload=None, |
| 139 |
replace=False, |
| 140 |
partial=False, |
| 141 |
no_create=False, |
| 142 |
**fields, |
| 143 |
): |
| 144 |
""" |
| 145 |
Add a document to the batch query |
| 146 |
""" |
| 147 |
self.client._add_document( |
| 148 |
doc_id, |
| 149 |
conn=self._pipeline, |
| 150 |
nosave=nosave, |
| 151 |
score=score, |
| 152 |
payload=payload, |
| 153 |
replace=replace, |
| 154 |
partial=partial, |
| 155 |
no_create=no_create, |
| 156 |
**fields, |
| 157 |
) |
| 158 |
self.current_chunk += 1 |
| 159 |
self.total += 1 |
| 160 |
if self.current_chunk >= self.chunk_size: |
| 161 |
await self.commit() |
| 162 |
|
| 163 |
async def commit(self): |
| 164 |
""" |
| 165 |
Manually commit and flush the batch indexing query |
| 166 |
""" |
| 167 |
await self._pipeline.execute() |
| 168 |
self.current_chunk = 0 |
| 169 |
|
| 170 |
def pipeline(self, transaction=True, shard_hint=None): |
| 171 |
"""Creates a pipeline for the SEARCH module, that can be used for executing |
| 172 |
SEARCH commands, as well as classic core commands. |
| 173 |
""" |
| 174 |
p = AsyncPipeline( |
| 175 |
connection_pool=self.client.connection_pool, |
| 176 |
response_callbacks=self._MODULE_CALLBACKS, |
| 177 |
transaction=transaction, |
| 178 |
shard_hint=shard_hint, |
| 179 |
) |
| 180 |
p.index_name = self.index_name |
| 181 |
return p |
| 182 |
|
| 183 |
|
| 184 |
class Pipeline(SearchCommands, valkey.client.Pipeline): |
| 185 |
"""Pipeline for the module.""" |
| 186 |
|
| 187 |
|
| 188 |
class AsyncPipeline(AsyncSearchCommands, AsyncioPipeline, Pipeline): |
| 189 |
"""AsyncPipeline for the module.""" |