|
| 1 | +import pickle |
| 2 | +import time |
| 3 | +import logging |
| 4 | +from os import makedirs |
| 5 | +from os.path import dirname, exists, isabs, join |
| 6 | +from typing import Any, Optional |
| 7 | + |
| 8 | +from privex.helpers.cache.CacheAdapter import CacheAdapter |
| 9 | +from privex.helpers.exceptions import CacheNotFound |
| 10 | +from privex.helpers.common import empty, empty_if, is_true |
| 11 | +from privex.helpers import settings |
| 12 | + |
| 13 | + |
| 14 | +log = logging.getLogger(__name__) |
| 15 | + |
| 16 | + |
| 17 | +def _cache_result_expired(res, _auto_purge=True) -> bool: |
| 18 | + if empty(res): return False |
| 19 | + if empty(res.expires_at, zero=True) or not _auto_purge: return False |
| 20 | + return float(res.expires_at) <= time.time() |
| 21 | + |
| 22 | + |
| 23 | +class SqliteCache(CacheAdapter): |
| 24 | + """ |
| 25 | + An SQLite3 backed implementation of :class:`.CacheAdapter`. Creates and uses a semi-global Sqlite instance via |
| 26 | + :py:mod:`privex.helpers.plugin` by default. |
| 27 | +
|
| 28 | + To allow for a wide variety of Python objects to be safely stored and retrieved from Sqlite, this class |
| 29 | + uses the :py:mod:`pickle` module for serialising + un-serialising values to/from Sqlite. |
| 30 | +
|
| 31 | + **Basic Usage**:: |
| 32 | +
|
| 33 | + >>> from privex.helpers.cache import SqliteCache |
| 34 | + >>> rc = SqliteCache() |
| 35 | + >>> rc.set('hello', 'world') |
| 36 | + >>> rc['hello'] |
| 37 | + 'world' |
| 38 | +
|
| 39 | +
|
| 40 | + **Disabling Pickling** |
| 41 | +
|
| 42 | + In some cases, you may need interoperable caching with other languages. The :py:mod:`pickle` serialisation |
| 43 | + technique is extremely specific to Python and is largely unsupported outside of Python. Thus if you need |
| 44 | + to share Sqlite cache data with applications in other languages, then you must disable pickling. |
| 45 | +
|
| 46 | + **WARNING:** If you disable pickling, then you must perform your own serialisation + de-serialization on |
| 47 | + complex objects such as ``dict``, ``list``, ``Decimal``, or arbitrary classes/functions after getting |
| 48 | + or setting cache keys. |
| 49 | +
|
| 50 | + **Disabling Pickle per instance** |
| 51 | +
|
| 52 | + Pass ``use_pickle=False`` to the constructor, or access the attribute directly to disable pickling for a |
| 53 | + single instance of SqliteCache (not globally):: |
| 54 | +
|
| 55 | + >>> rc = SqliteCache(use_pickle=False) # Opt 1. Disable pickle in constructor |
| 56 | + >>> rc.use_pickle = False # Opt 2. Disable pickle on an existing instance |
| 57 | +
|
| 58 | +
|
| 59 | + **Disabling Pickle by default on any new instances** |
| 60 | +
|
| 61 | + Change the static attribute :py:attr:`.pickle_default` to ``False`` to disable the use of pickle by default |
| 62 | + across any new instances of SqliteCache:: |
| 63 | +
|
| 64 | + >>> SqliteCache.pickle_default = False |
| 65 | +
|
| 66 | +
|
| 67 | +
|
| 68 | + """ |
| 69 | + |
| 70 | + pickle_default: bool = True |
| 71 | + """ |
| 72 | + Change this to ``False`` to disable the use of :py:mod:`pickle` by default for any new instances |
| 73 | + of this class. |
| 74 | + """ |
| 75 | + |
| 76 | + use_pickle: bool |
| 77 | + """If ``True``, will use :py:mod:`pickle` for serializing objects before inserting into Sqlite, and |
| 78 | + un-serialising objects retrieved from Sqlite3. This attribute is set in :py:meth:`.__init__`. |
| 79 | +
|
| 80 | + Change this to ``False`` to disable the use of :py:mod:`pickle` - instead values will be passed to / returned |
| 81 | + from Sqlite3 as-is, with no serialisation (this may require you to manually serialize complex types such |
| 82 | + as ``dict`` and ``Decimal`` before insertion, and un-serialise after retrieval). |
| 83 | + """ |
| 84 | + |
| 85 | + last_purged_expired: Optional[int] = None |
| 86 | + |
| 87 | + def __init__(self, db_file: str = None, memory_persist=False, use_pickle: bool = None, connection_kwargs: dict = None, *args, **kwargs): |
| 88 | + """ |
| 89 | + :class:`.SqliteCache` uses an auto-generated database filename / path by default, based on the name of the currently running |
| 90 | + script ( retrieved from ``sys.argv[0]`` ), allowing for persistent caching - without any manual configuration of the adapter, |
| 91 | + nor the requirement for any running background services such as ``redis`` / ``memcached``. |
| 92 | + |
| 93 | + |
| 94 | +
|
| 95 | + :param str db_file: (Optional) Name of / path to Sqlite3 database file to create/use for the cache. |
| 96 | + |
| 97 | + :param bool memory_persist: Use a shared in-memory database, which can be accessed by other instances of this class (in this |
| 98 | + process) - which is cleared after all memory connections are closed. |
| 99 | + Shortcut for ``db_file='file::memory:?cache=shared'`` |
| 100 | +
|
| 101 | + :param bool use_pickle: (Default: ``True``) Use the built-in ``pickle`` to serialise values before |
| 102 | + storing in Sqlite3, and un-serialise when loading from Sqlite3 |
| 103 | + |
| 104 | + :param dict connection_kwargs: (Optional) Additional / overriding kwargs to pass to :meth:`sqlite3.connect` when |
| 105 | + :class:`.SqliteCacheManager` initialises it's sqlite3 connection. |
| 106 | + |
| 107 | + :keyword int purge_every: (Default: 300) Expired + abandoned cache records are purged using the DB manager method |
| 108 | + :meth:`.SqliteCacheManager.purge_expired` during :meth:`.get` / :meth:`.set` calls. To avoid |
| 109 | + performance issues, the actual :meth:`.SqliteCacheManager.purge_expired` method is only called |
| 110 | + if at least ``purge_every`` seconds have passed since the last purge was |
| 111 | + triggered ( :attr:`.last_purged_expired` ) |
| 112 | +
|
| 113 | + """ |
| 114 | + from privex.helpers.cache.post_deps import SqliteCacheManager |
| 115 | + super().__init__(*args, **kwargs) |
| 116 | + self.db_file: str = empty_if(db_file, SqliteCacheManager.DEFAULT_DB) |
| 117 | + self.db_folder = None |
| 118 | + if ':memory:' not in self.db_file: |
| 119 | + if not isabs(self.db_file): self.db_file = join(SqliteCacheManager.DEFAULT_DB_FOLDER, self.db_file) |
| 120 | + self.db_folder = dirname(self.db_file) |
| 121 | + if not exists(self.db_folder): |
| 122 | + log.debug("Folder for database doesn't exist. Creating: %s", self.db_folder) |
| 123 | + makedirs(self.db_folder) |
| 124 | + self.connection_kwargs = empty_if(connection_kwargs, {}, itr=True) |
| 125 | + self.memory_persist = is_true(memory_persist) |
| 126 | + self._wrapper = None |
| 127 | + self.purge_every = kwargs.get('purge_every', 300) |
| 128 | + self.use_pickle = self.pickle_default if use_pickle is None else use_pickle |
| 129 | + |
| 130 | + @property |
| 131 | + def purge_due(self) -> bool: |
| 132 | + lpe = SqliteCache.last_purged_expired |
| 133 | + return empty(lpe, zero=True) or (time.time() - lpe) >= self.purge_every |
| 134 | + |
| 135 | + @property |
| 136 | + def wrapper(self): |
| 137 | + if not self._wrapper: |
| 138 | + from privex.helpers.cache.post_deps import SqliteCacheManager |
| 139 | + self._wrapper = SqliteCacheManager( |
| 140 | + db=self.db_file, connection_kwargs=self.connection_kwargs, memory_persist=self.memory_persist |
| 141 | + ) |
| 142 | + return self._wrapper |
| 143 | + |
| 144 | + def purge_expired(self, force=False) -> Optional[int]: |
| 145 | + if self.purge_due or force: |
| 146 | + log.debug("%s - Expired items purge is due (or force is True). Purging expired cache items...", self.__class__.__name__) |
| 147 | + res = self.wrapper.purge_expired() |
| 148 | + log.debug("Finished purging expired items. Total expired cache items deleted: %s", res) |
| 149 | + SqliteCache.last_purged_expired = time.time() |
| 150 | + return res |
| 151 | + return None |
| 152 | + |
| 153 | + def get(self, key: str, default: Any = None, fail: bool = False, _auto_purge=True) -> Any: |
| 154 | + key = str(key) |
| 155 | + _not_found_msg = f'Cache key "{key}" was not found.' |
| 156 | + if _auto_purge: self.purge_expired() |
| 157 | + res = self.wrapper.find_cache_key(key) |
| 158 | + if _cache_result_expired(res, _auto_purge=_auto_purge): |
| 159 | + log.debug("Caller attempted to retrieve expired key '%s', but _auto_purge is True - auto-removing expired key %s", key, key) |
| 160 | + self.remove(key) |
| 161 | + res = None |
| 162 | + _not_found_msg += ' (key was expired - auto-removed)' |
| 163 | + if empty(res): |
| 164 | + if fail: raise CacheNotFound(_not_found_msg) |
| 165 | + return default |
| 166 | + return pickle.loads(res.value) if self.use_pickle else res.value |
| 167 | + |
| 168 | + def set(self, key: str, value: Any, timeout: Optional[int] = settings.DEFAULT_CACHE_TIMEOUT, _auto_purge=True): |
| 169 | + if _auto_purge: self.purge_expired() |
| 170 | + v = pickle.dumps(value) if self.use_pickle else value |
| 171 | + return self.wrapper.set_cache_key(str(key), v, expires_secs=timeout) |
| 172 | + |
| 173 | + def remove(self, *key: str) -> bool: |
| 174 | + removed = 0 |
| 175 | + for k in key: |
| 176 | + k = str(k) |
| 177 | + try: |
| 178 | + self.get(k, fail=True, _auto_purge=False) |
| 179 | + self.wrapper.delete_cache_key(k) |
| 180 | + removed += 1 |
| 181 | + except CacheNotFound: |
| 182 | + pass |
| 183 | + |
| 184 | + return removed == len(key) |
| 185 | + |
| 186 | + def update_timeout(self, key: str, timeout: int = settings.DEFAULT_CACHE_TIMEOUT) -> Any: |
| 187 | + key, timeout = str(key), int(timeout) |
| 188 | + v = self.get(key=key, fail=True, _auto_purge=False) |
| 189 | + self.set(key=key, value=v, timeout=timeout, _auto_purge=False) |
| 190 | + return v |
| 191 | + |
| 192 | + def connect(self, db=None, *args, connection_kwargs=None, memory_persist=None, **kwargs): |
| 193 | + c_kwargs = dict( |
| 194 | + connection_kwargs=empty_if(connection_kwargs, self.connection_kwargs), |
| 195 | + memory_persist=empty_if(memory_persist, self.memory_persist) |
| 196 | + ) |
| 197 | + c_kwargs = {**c_kwargs, **kwargs} |
| 198 | + from privex.helpers.cache.post_deps import SqliteCacheManager |
| 199 | + self._wrapper = SqliteCacheManager(empty_if(db, self.db_file), *args, **c_kwargs) |
| 200 | + return self._wrapper |
| 201 | + |
| 202 | + # noinspection PyProtectedMember |
| 203 | + def close(self): |
| 204 | + cls_name = self.__class__.__name__ |
| 205 | + if self._wrapper is not None: |
| 206 | + log.debug("Closing Synchronous Sqlite3 instance %s._wrapper", cls_name) |
| 207 | + if self._wrapper._conn: |
| 208 | + try: |
| 209 | + self._wrapper._conn.close() |
| 210 | + except Exception: |
| 211 | + log.exception("Unexpected error while closing %s._wrapper._conn", cls_name) |
| 212 | + self._wrapper._conn = None |
| 213 | + self._wrapper = None |
0 commit comments