| 1 |
from typing import List, Optional, Union |
| 2 |
|
| 3 |
|
| 4 |
class Query: |
| 5 |
""" |
| 6 |
Query is used to build complex queries that have more parameters than just |
| 7 |
the query string. The query string is set in the constructor, and other |
| 8 |
options have setter functions. |
| 9 |
|
| 10 |
The setter functions return the query object, so they can be chained, |
| 11 |
i.e. `Query("foo").verbatim().filter(...)` etc. |
| 12 |
""" |
| 13 |
|
| 14 |
def __init__(self, query_string: str) -> None: |
| 15 |
""" |
| 16 |
Create a new query object. |
| 17 |
The query string is set in the constructor, and other options have |
| 18 |
setter functions. |
| 19 |
""" |
| 20 |
|
| 21 |
self._query_string: str = query_string |
| 22 |
self._offset: int = 0 |
| 23 |
self._num: int = 10 |
| 24 |
self._no_content: bool = False |
| 25 |
self._no_stopwords: bool = False |
| 26 |
self._fields: Optional[List[str]] = None |
| 27 |
self._verbatim: bool = False |
| 28 |
self._with_payloads: bool = False |
| 29 |
self._with_scores: bool = False |
| 30 |
self._scorer: Optional[str] = None |
| 31 |
self._filters: List = list() |
| 32 |
self._ids: Optional[List[str]] = None |
| 33 |
self._slop: int = -1 |
| 34 |
self._timeout: Optional[float] = None |
| 35 |
self._in_order: bool = False |
| 36 |
self._sortby: Optional[SortbyField] = None |
| 37 |
self._return_fields: List = [] |
| 38 |
self._summarize_fields: List = [] |
| 39 |
self._highlight_fields: List = [] |
| 40 |
self._language: Optional[str] = None |
| 41 |
self._expander: Optional[str] = None |
| 42 |
self._dialect: Optional[int] = None |
| 43 |
|
| 44 |
def query_string(self) -> str: |
| 45 |
"""Return the query string of this query only.""" |
| 46 |
return self._query_string |
| 47 |
|
| 48 |
def limit_ids(self, *ids) -> "Query": |
| 49 |
"""Limit the results to a specific set of pre-known document |
| 50 |
ids of any length.""" |
| 51 |
self._ids = ids |
| 52 |
return self |
| 53 |
|
| 54 |
def return_fields(self, *fields) -> "Query": |
| 55 |
"""Add fields to return fields.""" |
| 56 |
self._return_fields += fields |
| 57 |
return self |
| 58 |
|
| 59 |
def return_field(self, field: str, as_field: Optional[str] = None) -> "Query": |
| 60 |
"""Add field to return fields (Optional: add 'AS' name |
| 61 |
to the field).""" |
| 62 |
self._return_fields.append(field) |
| 63 |
if as_field is not None: |
| 64 |
self._return_fields += ("AS", as_field) |
| 65 |
return self |
| 66 |
|
| 67 |
def _mk_field_list(self, fields: List[str]) -> List: |
| 68 |
if not fields: |
| 69 |
return [] |
| 70 |
return [fields] if isinstance(fields, str) else list(fields) |
| 71 |
|
| 72 |
def summarize( |
| 73 |
self, |
| 74 |
fields: Optional[List] = None, |
| 75 |
context_len: Optional[int] = None, |
| 76 |
num_frags: Optional[int] = None, |
| 77 |
sep: Optional[str] = None, |
| 78 |
) -> "Query": |
| 79 |
""" |
| 80 |
Return an abridged format of the field, containing only the segments of |
| 81 |
the field which contain the matching term(s). |
| 82 |
|
| 83 |
If `fields` is specified, then only the mentioned fields are |
| 84 |
summarized; otherwise all results are summarized. |
| 85 |
|
| 86 |
Server side defaults are used for each option (except `fields`) |
| 87 |
if not specified |
| 88 |
|
| 89 |
- **fields** List of fields to summarize. All fields are summarized |
| 90 |
if not specified |
| 91 |
- **context_len** Amount of context to include with each fragment |
| 92 |
- **num_frags** Number of fragments per document |
| 93 |
- **sep** Separator string to separate fragments |
| 94 |
""" |
| 95 |
args = ["SUMMARIZE"] |
| 96 |
fields = self._mk_field_list(fields) |
| 97 |
if fields: |
| 98 |
args += ["FIELDS", str(len(fields))] + fields |
| 99 |
|
| 100 |
if context_len is not None: |
| 101 |
args += ["LEN", str(context_len)] |
| 102 |
if num_frags is not None: |
| 103 |
args += ["FRAGS", str(num_frags)] |
| 104 |
if sep is not None: |
| 105 |
args += ["SEPARATOR", sep] |
| 106 |
|
| 107 |
self._summarize_fields = args |
| 108 |
return self |
| 109 |
|
| 110 |
def highlight( |
| 111 |
self, fields: Optional[List[str]] = None, tags: Optional[List[str]] = None |
| 112 |
) -> None: |
| 113 |
""" |
| 114 |
Apply specified markup to matched term(s) within the returned field(s). |
| 115 |
|
| 116 |
- **fields** If specified then only those mentioned fields are |
| 117 |
highlighted, otherwise all fields are highlighted |
| 118 |
- **tags** A list of two strings to surround the match. |
| 119 |
""" |
| 120 |
args = ["HIGHLIGHT"] |
| 121 |
fields = self._mk_field_list(fields) |
| 122 |
if fields: |
| 123 |
args += ["FIELDS", str(len(fields))] + fields |
| 124 |
if tags: |
| 125 |
args += ["TAGS"] + list(tags) |
| 126 |
|
| 127 |
self._highlight_fields = args |
| 128 |
return self |
| 129 |
|
| 130 |
def language(self, language: str) -> "Query": |
| 131 |
""" |
| 132 |
Analyze the query as being in the specified language. |
| 133 |
|
| 134 |
:param language: The language (e.g. `chinese` or `english`) |
| 135 |
""" |
| 136 |
self._language = language |
| 137 |
return self |
| 138 |
|
| 139 |
def slop(self, slop: int) -> "Query": |
| 140 |
"""Allow a maximum of N intervening non matched terms between |
| 141 |
phrase terms (0 means exact phrase). |
| 142 |
""" |
| 143 |
self._slop = slop |
| 144 |
return self |
| 145 |
|
| 146 |
def timeout(self, timeout: float) -> "Query": |
| 147 |
"""overrides the timeout parameter of the module""" |
| 148 |
self._timeout = timeout |
| 149 |
return self |
| 150 |
|
| 151 |
def in_order(self) -> "Query": |
| 152 |
""" |
| 153 |
Match only documents where the query terms appear in |
| 154 |
the same order in the document. |
| 155 |
i.e. for the query "hello world", we do not match "world hello" |
| 156 |
""" |
| 157 |
self._in_order = True |
| 158 |
return self |
| 159 |
|
| 160 |
def scorer(self, scorer: str) -> "Query": |
| 161 |
""" |
| 162 |
Use a different scoring function to evaluate document relevance. |
| 163 |
Default is `TFIDF`. |
| 164 |
|
| 165 |
:param scorer: The scoring function to use |
| 166 |
(e.g. `TFIDF.DOCNORM` or `BM25`) |
| 167 |
""" |
| 168 |
self._scorer = scorer |
| 169 |
return self |
| 170 |
|
| 171 |
def get_args(self) -> List[str]: |
| 172 |
"""Format the valkey arguments for this query and return them.""" |
| 173 |
args = [self._query_string] |
| 174 |
args += self._get_args_tags() |
| 175 |
args += self._summarize_fields + self._highlight_fields |
| 176 |
args += ["LIMIT", self._offset, self._num] |
| 177 |
return args |
| 178 |
|
| 179 |
def _get_args_tags(self) -> List[str]: |
| 180 |
args = [] |
| 181 |
if self._no_content: |
| 182 |
args.append("NOCONTENT") |
| 183 |
if self._fields: |
| 184 |
args.append("INFIELDS") |
| 185 |
args.append(len(self._fields)) |
| 186 |
args += self._fields |
| 187 |
if self._verbatim: |
| 188 |
args.append("VERBATIM") |
| 189 |
if self._no_stopwords: |
| 190 |
args.append("NOSTOPWORDS") |
| 191 |
if self._filters: |
| 192 |
for flt in self._filters: |
| 193 |
if not isinstance(flt, Filter): |
| 194 |
raise AttributeError("Did not receive a Filter object.") |
| 195 |
args += flt.args |
| 196 |
if self._with_payloads: |
| 197 |
args.append("WITHPAYLOADS") |
| 198 |
if self._scorer: |
| 199 |
args += ["SCORER", self._scorer] |
| 200 |
if self._with_scores: |
| 201 |
args.append("WITHSCORES") |
| 202 |
if self._ids: |
| 203 |
args.append("INKEYS") |
| 204 |
args.append(len(self._ids)) |
| 205 |
args += self._ids |
| 206 |
if self._slop >= 0: |
| 207 |
args += ["SLOP", self._slop] |
| 208 |
if self._timeout is not None: |
| 209 |
args += ["TIMEOUT", self._timeout] |
| 210 |
if self._in_order: |
| 211 |
args.append("INORDER") |
| 212 |
if self._return_fields: |
| 213 |
args.append("RETURN") |
| 214 |
args.append(len(self._return_fields)) |
| 215 |
args += self._return_fields |
| 216 |
if self._sortby: |
| 217 |
if not isinstance(self._sortby, SortbyField): |
| 218 |
raise AttributeError("Did not receive a SortByField.") |
| 219 |
args.append("SORTBY") |
| 220 |
args += self._sortby.args |
| 221 |
if self._language: |
| 222 |
args += ["LANGUAGE", self._language] |
| 223 |
if self._expander: |
| 224 |
args += ["EXPANDER", self._expander] |
| 225 |
if self._dialect: |
| 226 |
args += ["DIALECT", self._dialect] |
| 227 |
|
| 228 |
return args |
| 229 |
|
| 230 |
def paging(self, offset: int, num: int) -> "Query": |
| 231 |
""" |
| 232 |
Set the paging for the query (defaults to 0..10). |
| 233 |
|
| 234 |
- **offset**: Paging offset for the results. Defaults to 0 |
| 235 |
- **num**: How many results do we want |
| 236 |
""" |
| 237 |
self._offset = offset |
| 238 |
self._num = num |
| 239 |
return self |
| 240 |
|
| 241 |
def verbatim(self) -> "Query": |
| 242 |
"""Set the query to be verbatim, i.e. use no query expansion |
| 243 |
or stemming. |
| 244 |
""" |
| 245 |
self._verbatim = True |
| 246 |
return self |
| 247 |
|
| 248 |
def no_content(self) -> "Query": |
| 249 |
"""Set the query to only return ids and not the document content.""" |
| 250 |
self._no_content = True |
| 251 |
return self |
| 252 |
|
| 253 |
def no_stopwords(self) -> "Query": |
| 254 |
""" |
| 255 |
Prevent the query from being filtered for stopwords. |
| 256 |
Only useful in very big queries that you are certain contain |
| 257 |
no stopwords. |
| 258 |
""" |
| 259 |
self._no_stopwords = True |
| 260 |
return self |
| 261 |
|
| 262 |
def with_payloads(self) -> "Query": |
| 263 |
"""Ask the engine to return document payloads.""" |
| 264 |
self._with_payloads = True |
| 265 |
return self |
| 266 |
|
| 267 |
def with_scores(self) -> "Query": |
| 268 |
"""Ask the engine to return document search scores.""" |
| 269 |
self._with_scores = True |
| 270 |
return self |
| 271 |
|
| 272 |
def limit_fields(self, *fields: List[str]) -> "Query": |
| 273 |
""" |
| 274 |
Limit the search to specific TEXT fields only. |
| 275 |
|
| 276 |
- **fields**: A list of strings, case-sensitive field names |
| 277 |
from the defined schema. |
| 278 |
""" |
| 279 |
self._fields = fields |
| 280 |
return self |
| 281 |
|
| 282 |
def add_filter(self, flt: "Filter") -> "Query": |
| 283 |
""" |
| 284 |
Add a numeric or geo filter to the query. |
| 285 |
**Currently only one of each filter is supported by the engine** |
| 286 |
|
| 287 |
- **flt**: A NumericFilter or GeoFilter object, used on a |
| 288 |
corresponding field |
| 289 |
""" |
| 290 |
|
| 291 |
self._filters.append(flt) |
| 292 |
return self |
| 293 |
|
| 294 |
def sort_by(self, field: str, asc: bool = True) -> "Query": |
| 295 |
""" |
| 296 |
Add a sortby field to the query. |
| 297 |
|
| 298 |
- **field** - the name of the field to sort by |
| 299 |
- **asc** - when `True`, sorting will be done in ascending order |
| 300 |
""" |
| 301 |
self._sortby = SortbyField(field, asc) |
| 302 |
return self |
| 303 |
|
| 304 |
def expander(self, expander: str) -> "Query": |
| 305 |
""" |
| 306 |
Add a expander field to the query. |
| 307 |
|
| 308 |
- **expander** - the name of the expander |
| 309 |
""" |
| 310 |
self._expander = expander |
| 311 |
return self |
| 312 |
|
| 313 |
def dialect(self, dialect: int) -> "Query": |
| 314 |
""" |
| 315 |
Add a dialect field to the query. |
| 316 |
|
| 317 |
- **dialect** - dialect version to execute the query under |
| 318 |
""" |
| 319 |
self._dialect = dialect |
| 320 |
return self |
| 321 |
|
| 322 |
|
| 323 |
class Filter: |
| 324 |
def __init__(self, keyword: str, field: str, *args: List[str]) -> None: |
| 325 |
self.args = [keyword, field] + list(args) |
| 326 |
|
| 327 |
|
| 328 |
class NumericFilter(Filter): |
| 329 |
INF = "+inf" |
| 330 |
NEG_INF = "-inf" |
| 331 |
|
| 332 |
def __init__( |
| 333 |
self, |
| 334 |
field: str, |
| 335 |
minval: Union[int, str], |
| 336 |
maxval: Union[int, str], |
| 337 |
minExclusive: bool = False, |
| 338 |
maxExclusive: bool = False, |
| 339 |
) -> None: |
| 340 |
args = [ |
| 341 |
minval if not minExclusive else f"({minval}", |
| 342 |
maxval if not maxExclusive else f"({maxval}", |
| 343 |
] |
| 344 |
|
| 345 |
Filter.__init__(self, "FILTER", field, *args) |
| 346 |
|
| 347 |
|
| 348 |
class GeoFilter(Filter): |
| 349 |
METERS = "m" |
| 350 |
KILOMETERS = "km" |
| 351 |
FEET = "ft" |
| 352 |
MILES = "mi" |
| 353 |
|
| 354 |
def __init__( |
| 355 |
self, field: str, lon: float, lat: float, radius: float, unit: str = KILOMETERS |
| 356 |
) -> None: |
| 357 |
Filter.__init__(self, "GEOFILTER", field, lon, lat, radius, unit) |
| 358 |
|
| 359 |
|
| 360 |
class SortbyField: |
| 361 |
def __init__(self, field: str, asc=True) -> None: |
| 362 |
self.args = [field, "ASC" if asc else "DESC"] |