| 1 |
"Base Cache class." |
| 2 |
|
| 3 |
import time |
| 4 |
import warnings |
| 5 |
|
| 6 |
from asgiref.sync import sync_to_async |
| 7 |
|
| 8 |
from django.core.exceptions import ImproperlyConfigured |
| 9 |
from django.utils.module_loading import import_string |
| 10 |
from django.utils.regex_helper import _lazy_re_compile |
| 11 |
|
| 12 |
|
| 13 |
class InvalidCacheBackendError(ImproperlyConfigured): |
| 14 |
pass |
| 15 |
|
| 16 |
|
| 17 |
class CacheKeyWarning(RuntimeWarning): |
| 18 |
pass |
| 19 |
|
| 20 |
|
| 21 |
class InvalidCacheKey(ValueError): |
| 22 |
pass |
| 23 |
|
| 24 |
|
| 25 |
# Stub class to ensure not passing in a `timeout` argument results in |
| 26 |
# the default timeout |
| 27 |
DEFAULT_TIMEOUT = object() |
| 28 |
|
| 29 |
# Memcached does not accept keys longer than this. |
| 30 |
MEMCACHE_MAX_KEY_LENGTH = 250 |
| 31 |
|
| 32 |
|
| 33 |
def default_key_func(key, key_prefix, version): |
| 34 |
""" |
| 35 |
Default function to generate keys. |
| 36 |
|
| 37 |
Construct the key used by all other methods. By default, prepend |
| 38 |
the `key_prefix`. KEY_FUNCTION can be used to specify an alternate |
| 39 |
function with custom key making behavior. |
| 40 |
""" |
| 41 |
return "%s:%s:%s" % (key_prefix, version, key) |
| 42 |
|
| 43 |
|
| 44 |
def get_key_func(key_func): |
| 45 |
""" |
| 46 |
Function to decide which key function to use. |
| 47 |
|
| 48 |
Default to ``default_key_func``. |
| 49 |
""" |
| 50 |
if key_func is not None: |
| 51 |
if callable(key_func): |
| 52 |
return key_func |
| 53 |
else: |
| 54 |
return import_string(key_func) |
| 55 |
return default_key_func |
| 56 |
|
| 57 |
|
| 58 |
class BaseCache: |
| 59 |
_missing_key = object() |
| 60 |
|
| 61 |
def __init__(self, params): |
| 62 |
timeout = params.get("timeout", params.get("TIMEOUT", 300)) |
| 63 |
if timeout is not None: |
| 64 |
try: |
| 65 |
timeout = int(timeout) |
| 66 |
except (ValueError, TypeError): |
| 67 |
timeout = 300 |
| 68 |
self.default_timeout = timeout |
| 69 |
|
| 70 |
options = params.get("OPTIONS", {}) |
| 71 |
max_entries = params.get("max_entries", options.get("MAX_ENTRIES", 300)) |
| 72 |
try: |
| 73 |
self._max_entries = int(max_entries) |
| 74 |
except (ValueError, TypeError): |
| 75 |
self._max_entries = 300 |
| 76 |
|
| 77 |
cull_frequency = params.get("cull_frequency", options.get("CULL_FREQUENCY", 3)) |
| 78 |
try: |
| 79 |
self._cull_frequency = int(cull_frequency) |
| 80 |
except (ValueError, TypeError): |
| 81 |
self._cull_frequency = 3 |
| 82 |
|
| 83 |
self.key_prefix = params.get("KEY_PREFIX", "") |
| 84 |
self.version = params.get("VERSION", 1) |
| 85 |
self.key_func = get_key_func(params.get("KEY_FUNCTION")) |
| 86 |
|
| 87 |
def get_backend_timeout(self, timeout=DEFAULT_TIMEOUT): |
| 88 |
""" |
| 89 |
Return the timeout value usable by this backend based upon the provided |
| 90 |
timeout. |
| 91 |
""" |
| 92 |
if timeout == DEFAULT_TIMEOUT: |
| 93 |
timeout = self.default_timeout |
| 94 |
elif timeout == 0: |
| 95 |
# ticket 21147 - avoid time.time() related precision issues |
| 96 |
timeout = -1 |
| 97 |
return None if timeout is None else time.time() + timeout |
| 98 |
|
| 99 |
def make_key(self, key, version=None): |
| 100 |
""" |
| 101 |
Construct the key used by all other methods. By default, use the |
| 102 |
key_func to generate a key (which, by default, prepends the |
| 103 |
`key_prefix' and 'version'). A different key function can be provided |
| 104 |
at the time of cache construction; alternatively, you can subclass the |
| 105 |
cache backend to provide custom key making behavior. |
| 106 |
""" |
| 107 |
if version is None: |
| 108 |
version = self.version |
| 109 |
|
| 110 |
return self.key_func(key, self.key_prefix, version) |
| 111 |
|
| 112 |
def validate_key(self, key): |
| 113 |
""" |
| 114 |
Warn about keys that would not be portable to the memcached |
| 115 |
backend. This encourages (but does not force) writing backend-portable |
| 116 |
cache code. |
| 117 |
""" |
| 118 |
for warning in memcache_key_warnings(key): |
| 119 |
warnings.warn(warning, CacheKeyWarning) |
| 120 |
|
| 121 |
def make_and_validate_key(self, key, version=None): |
| 122 |
"""Helper to make and validate keys.""" |
| 123 |
key = self.make_key(key, version=version) |
| 124 |
self.validate_key(key) |
| 125 |
return key |
| 126 |
|
| 127 |
def add(self, key, value, timeout=DEFAULT_TIMEOUT, version=None): |
| 128 |
""" |
| 129 |
Set a value in the cache if the key does not already exist. If |
| 130 |
timeout is given, use that timeout for the key; otherwise use the |
| 131 |
default cache timeout. |
| 132 |
|
| 133 |
Return True if the value was stored, False otherwise. |
| 134 |
""" |
| 135 |
raise NotImplementedError( |
| 136 |
"subclasses of BaseCache must provide an add() method" |
| 137 |
) |
| 138 |
|
| 139 |
async def aadd(self, key, value, timeout=DEFAULT_TIMEOUT, version=None): |
| 140 |
return await sync_to_async(self.add, thread_sensitive=True)( |
| 141 |
key, value, timeout, version |
| 142 |
) |
| 143 |
|
| 144 |
def get(self, key, default=None, version=None): |
| 145 |
""" |
| 146 |
Fetch a given key from the cache. If the key does not exist, return |
| 147 |
default, which itself defaults to None. |
| 148 |
""" |
| 149 |
raise NotImplementedError("subclasses of BaseCache must provide a get() method") |
| 150 |
|
| 151 |
async def aget(self, key, default=None, version=None): |
| 152 |
return await sync_to_async(self.get, thread_sensitive=True)( |
| 153 |
key, default, version |
| 154 |
) |
| 155 |
|
| 156 |
def set(self, key, value, timeout=DEFAULT_TIMEOUT, version=None): |
| 157 |
""" |
| 158 |
Set a value in the cache. If timeout is given, use that timeout for the |
| 159 |
key; otherwise use the default cache timeout. |
| 160 |
""" |
| 161 |
raise NotImplementedError("subclasses of BaseCache must provide a set() method") |
| 162 |
|
| 163 |
async def aset(self, key, value, timeout=DEFAULT_TIMEOUT, version=None): |
| 164 |
return await sync_to_async(self.set, thread_sensitive=True)( |
| 165 |
key, value, timeout, version |
| 166 |
) |
| 167 |
|
| 168 |
def touch(self, key, timeout=DEFAULT_TIMEOUT, version=None): |
| 169 |
""" |
| 170 |
Update the key's expiry time using timeout. Return True if successful |
| 171 |
or False if the key does not exist. |
| 172 |
""" |
| 173 |
raise NotImplementedError( |
| 174 |
"subclasses of BaseCache must provide a touch() method" |
| 175 |
) |
| 176 |
|
| 177 |
async def atouch(self, key, timeout=DEFAULT_TIMEOUT, version=None): |
| 178 |
return await sync_to_async(self.touch, thread_sensitive=True)( |
| 179 |
key, timeout, version |
| 180 |
) |
| 181 |
|
| 182 |
def delete(self, key, version=None): |
| 183 |
""" |
| 184 |
Delete a key from the cache and return whether it succeeded, failing |
| 185 |
silently. |
| 186 |
""" |
| 187 |
raise NotImplementedError( |
| 188 |
"subclasses of BaseCache must provide a delete() method" |
| 189 |
) |
| 190 |
|
| 191 |
async def adelete(self, key, version=None): |
| 192 |
return await sync_to_async(self.delete, thread_sensitive=True)(key, version) |
| 193 |
|
| 194 |
def get_many(self, keys, version=None): |
| 195 |
""" |
| 196 |
Fetch a bunch of keys from the cache. For certain backends (memcached, |
| 197 |
pgsql) this can be *much* faster when fetching multiple values. |
| 198 |
|
| 199 |
Return a dict mapping each key in keys to its value. If the given |
| 200 |
key is missing, it will be missing from the response dict. |
| 201 |
""" |
| 202 |
d = {} |
| 203 |
for k in keys: |
| 204 |
val = self.get(k, self._missing_key, version=version) |
| 205 |
if val is not self._missing_key: |
| 206 |
d[k] = val |
| 207 |
return d |
| 208 |
|
| 209 |
async def aget_many(self, keys, version=None): |
| 210 |
"""See get_many().""" |
| 211 |
d = {} |
| 212 |
for k in keys: |
| 213 |
val = await self.aget(k, self._missing_key, version=version) |
| 214 |
if val is not self._missing_key: |
| 215 |
d[k] = val |
| 216 |
return d |
| 217 |
|
| 218 |
def get_or_set(self, key, default, timeout=DEFAULT_TIMEOUT, version=None): |
| 219 |
""" |
| 220 |
Fetch a given key from the cache. If the key does not exist, |
| 221 |
add the key and set it to the default value. The default value can |
| 222 |
also be any callable. If timeout is given, use that timeout for the |
| 223 |
key; otherwise use the default cache timeout. |
| 224 |
|
| 225 |
Return the value of the key stored or retrieved. |
| 226 |
""" |
| 227 |
val = self.get(key, self._missing_key, version=version) |
| 228 |
if val is self._missing_key: |
| 229 |
if callable(default): |
| 230 |
default = default() |
| 231 |
self.add(key, default, timeout=timeout, version=version) |
| 232 |
# Fetch the value again to avoid a race condition if another caller |
| 233 |
# added a value between the first get() and the add() above. |
| 234 |
return self.get(key, default, version=version) |
| 235 |
return val |
| 236 |
|
| 237 |
async def aget_or_set(self, key, default, timeout=DEFAULT_TIMEOUT, version=None): |
| 238 |
"""See get_or_set().""" |
| 239 |
val = await self.aget(key, self._missing_key, version=version) |
| 240 |
if val is self._missing_key: |
| 241 |
if callable(default): |
| 242 |
default = default() |
| 243 |
await self.aadd(key, default, timeout=timeout, version=version) |
| 244 |
# Fetch the value again to avoid a race condition if another caller |
| 245 |
# added a value between the first aget() and the aadd() above. |
| 246 |
return await self.aget(key, default, version=version) |
| 247 |
return val |
| 248 |
|
| 249 |
def has_key(self, key, version=None): |
| 250 |
""" |
| 251 |
Return True if the key is in the cache and has not expired. |
| 252 |
""" |
| 253 |
return ( |
| 254 |
self.get(key, self._missing_key, version=version) is not self._missing_key |
| 255 |
) |
| 256 |
|
| 257 |
async def ahas_key(self, key, version=None): |
| 258 |
return ( |
| 259 |
await self.aget(key, self._missing_key, version=version) |
| 260 |
is not self._missing_key |
| 261 |
) |
| 262 |
|
| 263 |
def incr(self, key, delta=1, version=None): |
| 264 |
""" |
| 265 |
Add delta to value in the cache. If the key does not exist, raise a |
| 266 |
ValueError exception. |
| 267 |
""" |
| 268 |
value = self.get(key, self._missing_key, version=version) |
| 269 |
if value is self._missing_key: |
| 270 |
raise ValueError("Key '%s' not found" % key) |
| 271 |
new_value = value + delta |
| 272 |
self.set(key, new_value, version=version) |
| 273 |
return new_value |
| 274 |
|
| 275 |
async def aincr(self, key, delta=1, version=None): |
| 276 |
"""See incr().""" |
| 277 |
value = await self.aget(key, self._missing_key, version=version) |
| 278 |
if value is self._missing_key: |
| 279 |
raise ValueError("Key '%s' not found" % key) |
| 280 |
new_value = value + delta |
| 281 |
await self.aset(key, new_value, version=version) |
| 282 |
return new_value |
| 283 |
|
| 284 |
def decr(self, key, delta=1, version=None): |
| 285 |
""" |
| 286 |
Subtract delta from value in the cache. If the key does not exist, raise |
| 287 |
a ValueError exception. |
| 288 |
""" |
| 289 |
return self.incr(key, -delta, version=version) |
| 290 |
|
| 291 |
async def adecr(self, key, delta=1, version=None): |
| 292 |
return await self.aincr(key, -delta, version=version) |
| 293 |
|
| 294 |
def __contains__(self, key): |
| 295 |
""" |
| 296 |
Return True if the key is in the cache and has not expired. |
| 297 |
""" |
| 298 |
# This is a separate method, rather than just a copy of has_key(), |
| 299 |
# so that it always has the same functionality as has_key(), even |
| 300 |
# if a subclass overrides it. |
| 301 |
return self.has_key(key) |
| 302 |
|
| 303 |
def set_many(self, data, timeout=DEFAULT_TIMEOUT, version=None): |
| 304 |
""" |
| 305 |
Set a bunch of values in the cache at once from a dict of key/value |
| 306 |
pairs. For certain backends (memcached), this is much more efficient |
| 307 |
than calling set() multiple times. |
| 308 |
|
| 309 |
If timeout is given, use that timeout for the key; otherwise use the |
| 310 |
default cache timeout. |
| 311 |
|
| 312 |
On backends that support it, return a list of keys that failed |
| 313 |
insertion, or an empty list if all keys were inserted successfully. |
| 314 |
""" |
| 315 |
for key, value in data.items(): |
| 316 |
self.set(key, value, timeout=timeout, version=version) |
| 317 |
return [] |
| 318 |
|
| 319 |
async def aset_many(self, data, timeout=DEFAULT_TIMEOUT, version=None): |
| 320 |
for key, value in data.items(): |
| 321 |
await self.aset(key, value, timeout=timeout, version=version) |
| 322 |
return [] |
| 323 |
|
| 324 |
def delete_many(self, keys, version=None): |
| 325 |
""" |
| 326 |
Delete a bunch of values in the cache at once. For certain backends |
| 327 |
(memcached), this is much more efficient than calling delete() multiple |
| 328 |
times. |
| 329 |
""" |
| 330 |
for key in keys: |
| 331 |
self.delete(key, version=version) |
| 332 |
|
| 333 |
async def adelete_many(self, keys, version=None): |
| 334 |
for key in keys: |
| 335 |
await self.adelete(key, version=version) |
| 336 |
|
| 337 |
def clear(self): |
| 338 |
"""Remove *all* values from the cache at once.""" |
| 339 |
raise NotImplementedError( |
| 340 |
"subclasses of BaseCache must provide a clear() method" |
| 341 |
) |
| 342 |
|
| 343 |
async def aclear(self): |
| 344 |
return await sync_to_async(self.clear, thread_sensitive=True)() |
| 345 |
|
| 346 |
def incr_version(self, key, delta=1, version=None): |
| 347 |
""" |
| 348 |
Add delta to the cache version for the supplied key. Return the new |
| 349 |
version. |
| 350 |
""" |
| 351 |
if version is None: |
| 352 |
version = self.version |
| 353 |
|
| 354 |
value = self.get(key, self._missing_key, version=version) |
| 355 |
if value is self._missing_key: |
| 356 |
raise ValueError("Key '%s' not found" % key) |
| 357 |
|
| 358 |
self.set(key, value, version=version + delta) |
| 359 |
self.delete(key, version=version) |
| 360 |
return version + delta |
| 361 |
|
| 362 |
async def aincr_version(self, key, delta=1, version=None): |
| 363 |
"""See incr_version().""" |
| 364 |
if version is None: |
| 365 |
version = self.version |
| 366 |
|
| 367 |
value = await self.aget(key, self._missing_key, version=version) |
| 368 |
if value is self._missing_key: |
| 369 |
raise ValueError("Key '%s' not found" % key) |
| 370 |
|
| 371 |
await self.aset(key, value, version=version + delta) |
| 372 |
await self.adelete(key, version=version) |
| 373 |
return version + delta |
| 374 |
|
| 375 |
def decr_version(self, key, delta=1, version=None): |
| 376 |
""" |
| 377 |
Subtract delta from the cache version for the supplied key. Return the |
| 378 |
new version. |
| 379 |
""" |
| 380 |
return self.incr_version(key, -delta, version) |
| 381 |
|
| 382 |
async def adecr_version(self, key, delta=1, version=None): |
| 383 |
return await self.aincr_version(key, -delta, version) |
| 384 |
|
| 385 |
def close(self, **kwargs): |
| 386 |
"""Close the cache connection""" |
| 387 |
pass |
| 388 |
|
| 389 |
async def aclose(self, **kwargs): |
| 390 |
pass |
| 391 |
|
| 392 |
|
| 393 |
memcached_error_chars_re = _lazy_re_compile(r"[\x00-\x20\x7f]") |
| 394 |
|
| 395 |
|
| 396 |
def memcache_key_warnings(key): |
| 397 |
if len(key) > MEMCACHE_MAX_KEY_LENGTH: |
| 398 |
yield ( |
| 399 |
"Cache key will cause errors if used with memcached: %r " |
| 400 |
"(longer than %s)" % (key, MEMCACHE_MAX_KEY_LENGTH) |
| 401 |
) |
| 402 |
if memcached_error_chars_re.search(key): |
| 403 |
yield ( |
| 404 |
"Cache key contains characters that will cause errors if used with " |
| 405 |
f"memcached: {key!r}" |
| 406 |
) |