bccache.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350
  1. # -*- coding: utf-8 -*-
  2. """The optional bytecode cache system. This is useful if you have very
  3. complex template situations and the compilation of all those templates
  4. slows down your application too much.
  5. Situations where this is useful are often forking web applications that
  6. are initialized on the first request.
  7. """
  8. import errno
  9. import fnmatch
  10. import os
  11. import stat
  12. import sys
  13. import tempfile
  14. from hashlib import sha1
  15. from os import listdir
  16. from os import path
  17. from ._compat import BytesIO
  18. from ._compat import marshal_dump
  19. from ._compat import marshal_load
  20. from ._compat import pickle
  21. from ._compat import text_type
  22. from .utils import open_if_exists
  23. bc_version = 4
  24. # Magic bytes to identify Jinja bytecode cache files. Contains the
  25. # Python major and minor version to avoid loading incompatible bytecode
  26. # if a project upgrades its Python version.
  27. bc_magic = (
  28. b"j2"
  29. + pickle.dumps(bc_version, 2)
  30. + pickle.dumps((sys.version_info[0] << 24) | sys.version_info[1], 2)
  31. )
  32. class Bucket(object):
  33. """Buckets are used to store the bytecode for one template. It's created
  34. and initialized by the bytecode cache and passed to the loading functions.
  35. The buckets get an internal checksum from the cache assigned and use this
  36. to automatically reject outdated cache material. Individual bytecode
  37. cache subclasses don't have to care about cache invalidation.
  38. """
  39. def __init__(self, environment, key, checksum):
  40. self.environment = environment
  41. self.key = key
  42. self.checksum = checksum
  43. self.reset()
  44. def reset(self):
  45. """Resets the bucket (unloads the bytecode)."""
  46. self.code = None
  47. def load_bytecode(self, f):
  48. """Loads bytecode from a file or file like object."""
  49. # make sure the magic header is correct
  50. magic = f.read(len(bc_magic))
  51. if magic != bc_magic:
  52. self.reset()
  53. return
  54. # the source code of the file changed, we need to reload
  55. checksum = pickle.load(f)
  56. if self.checksum != checksum:
  57. self.reset()
  58. return
  59. # if marshal_load fails then we need to reload
  60. try:
  61. self.code = marshal_load(f)
  62. except (EOFError, ValueError, TypeError):
  63. self.reset()
  64. return
  65. def write_bytecode(self, f):
  66. """Dump the bytecode into the file or file like object passed."""
  67. if self.code is None:
  68. raise TypeError("can't write empty bucket")
  69. f.write(bc_magic)
  70. pickle.dump(self.checksum, f, 2)
  71. marshal_dump(self.code, f)
  72. def bytecode_from_string(self, string):
  73. """Load bytecode from a string."""
  74. self.load_bytecode(BytesIO(string))
  75. def bytecode_to_string(self):
  76. """Return the bytecode as string."""
  77. out = BytesIO()
  78. self.write_bytecode(out)
  79. return out.getvalue()
  80. class BytecodeCache(object):
  81. """To implement your own bytecode cache you have to subclass this class
  82. and override :meth:`load_bytecode` and :meth:`dump_bytecode`. Both of
  83. these methods are passed a :class:`~jinja2.bccache.Bucket`.
  84. A very basic bytecode cache that saves the bytecode on the file system::
  85. from os import path
  86. class MyCache(BytecodeCache):
  87. def __init__(self, directory):
  88. self.directory = directory
  89. def load_bytecode(self, bucket):
  90. filename = path.join(self.directory, bucket.key)
  91. if path.exists(filename):
  92. with open(filename, 'rb') as f:
  93. bucket.load_bytecode(f)
  94. def dump_bytecode(self, bucket):
  95. filename = path.join(self.directory, bucket.key)
  96. with open(filename, 'wb') as f:
  97. bucket.write_bytecode(f)
  98. A more advanced version of a filesystem based bytecode cache is part of
  99. Jinja.
  100. """
  101. def load_bytecode(self, bucket):
  102. """Subclasses have to override this method to load bytecode into a
  103. bucket. If they are not able to find code in the cache for the
  104. bucket, it must not do anything.
  105. """
  106. raise NotImplementedError()
  107. def dump_bytecode(self, bucket):
  108. """Subclasses have to override this method to write the bytecode
  109. from a bucket back to the cache. If it unable to do so it must not
  110. fail silently but raise an exception.
  111. """
  112. raise NotImplementedError()
  113. def clear(self):
  114. """Clears the cache. This method is not used by Jinja but should be
  115. implemented to allow applications to clear the bytecode cache used
  116. by a particular environment.
  117. """
  118. def get_cache_key(self, name, filename=None):
  119. """Returns the unique hash key for this template name."""
  120. hash = sha1(name.encode("utf-8"))
  121. if filename is not None:
  122. filename = "|" + filename
  123. if isinstance(filename, text_type):
  124. filename = filename.encode("utf-8")
  125. hash.update(filename)
  126. return hash.hexdigest()
  127. def get_source_checksum(self, source):
  128. """Returns a checksum for the source."""
  129. return sha1(source.encode("utf-8")).hexdigest()
  130. def get_bucket(self, environment, name, filename, source):
  131. """Return a cache bucket for the given template. All arguments are
  132. mandatory but filename may be `None`.
  133. """
  134. key = self.get_cache_key(name, filename)
  135. checksum = self.get_source_checksum(source)
  136. bucket = Bucket(environment, key, checksum)
  137. self.load_bytecode(bucket)
  138. return bucket
  139. def set_bucket(self, bucket):
  140. """Put the bucket into the cache."""
  141. self.dump_bytecode(bucket)
  142. class FileSystemBytecodeCache(BytecodeCache):
  143. """A bytecode cache that stores bytecode on the filesystem. It accepts
  144. two arguments: The directory where the cache items are stored and a
  145. pattern string that is used to build the filename.
  146. If no directory is specified a default cache directory is selected. On
  147. Windows the user's temp directory is used, on UNIX systems a directory
  148. is created for the user in the system temp directory.
  149. The pattern can be used to have multiple separate caches operate on the
  150. same directory. The default pattern is ``'__jinja2_%s.cache'``. ``%s``
  151. is replaced with the cache key.
  152. >>> bcc = FileSystemBytecodeCache('/tmp/jinja_cache', '%s.cache')
  153. This bytecode cache supports clearing of the cache using the clear method.
  154. """
  155. def __init__(self, directory=None, pattern="__jinja2_%s.cache"):
  156. if directory is None:
  157. directory = self._get_default_cache_dir()
  158. self.directory = directory
  159. self.pattern = pattern
  160. def _get_default_cache_dir(self):
  161. def _unsafe_dir():
  162. raise RuntimeError(
  163. "Cannot determine safe temp directory. You "
  164. "need to explicitly provide one."
  165. )
  166. tmpdir = tempfile.gettempdir()
  167. # On windows the temporary directory is used specific unless
  168. # explicitly forced otherwise. We can just use that.
  169. if os.name == "nt":
  170. return tmpdir
  171. if not hasattr(os, "getuid"):
  172. _unsafe_dir()
  173. dirname = "_jinja2-cache-%d" % os.getuid()
  174. actual_dir = os.path.join(tmpdir, dirname)
  175. try:
  176. os.mkdir(actual_dir, stat.S_IRWXU)
  177. except OSError as e:
  178. if e.errno != errno.EEXIST:
  179. raise
  180. try:
  181. os.chmod(actual_dir, stat.S_IRWXU)
  182. actual_dir_stat = os.lstat(actual_dir)
  183. if (
  184. actual_dir_stat.st_uid != os.getuid()
  185. or not stat.S_ISDIR(actual_dir_stat.st_mode)
  186. or stat.S_IMODE(actual_dir_stat.st_mode) != stat.S_IRWXU
  187. ):
  188. _unsafe_dir()
  189. except OSError as e:
  190. if e.errno != errno.EEXIST:
  191. raise
  192. actual_dir_stat = os.lstat(actual_dir)
  193. if (
  194. actual_dir_stat.st_uid != os.getuid()
  195. or not stat.S_ISDIR(actual_dir_stat.st_mode)
  196. or stat.S_IMODE(actual_dir_stat.st_mode) != stat.S_IRWXU
  197. ):
  198. _unsafe_dir()
  199. return actual_dir
  200. def _get_cache_filename(self, bucket):
  201. return path.join(self.directory, self.pattern % bucket.key)
  202. def load_bytecode(self, bucket):
  203. f = open_if_exists(self._get_cache_filename(bucket), "rb")
  204. if f is not None:
  205. try:
  206. bucket.load_bytecode(f)
  207. finally:
  208. f.close()
  209. def dump_bytecode(self, bucket):
  210. f = open(self._get_cache_filename(bucket), "wb")
  211. try:
  212. bucket.write_bytecode(f)
  213. finally:
  214. f.close()
  215. def clear(self):
  216. # imported lazily here because google app-engine doesn't support
  217. # write access on the file system and the function does not exist
  218. # normally.
  219. from os import remove
  220. files = fnmatch.filter(listdir(self.directory), self.pattern % "*")
  221. for filename in files:
  222. try:
  223. remove(path.join(self.directory, filename))
  224. except OSError:
  225. pass
  226. class MemcachedBytecodeCache(BytecodeCache):
  227. """This class implements a bytecode cache that uses a memcache cache for
  228. storing the information. It does not enforce a specific memcache library
  229. (tummy's memcache or cmemcache) but will accept any class that provides
  230. the minimal interface required.
  231. Libraries compatible with this class:
  232. - `cachelib <https://github.com/pallets/cachelib>`_
  233. - `python-memcached <https://pypi.org/project/python-memcached/>`_
  234. (Unfortunately the django cache interface is not compatible because it
  235. does not support storing binary data, only unicode. You can however pass
  236. the underlying cache client to the bytecode cache which is available
  237. as `django.core.cache.cache._client`.)
  238. The minimal interface for the client passed to the constructor is this:
  239. .. class:: MinimalClientInterface
  240. .. method:: set(key, value[, timeout])
  241. Stores the bytecode in the cache. `value` is a string and
  242. `timeout` the timeout of the key. If timeout is not provided
  243. a default timeout or no timeout should be assumed, if it's
  244. provided it's an integer with the number of seconds the cache
  245. item should exist.
  246. .. method:: get(key)
  247. Returns the value for the cache key. If the item does not
  248. exist in the cache the return value must be `None`.
  249. The other arguments to the constructor are the prefix for all keys that
  250. is added before the actual cache key and the timeout for the bytecode in
  251. the cache system. We recommend a high (or no) timeout.
  252. This bytecode cache does not support clearing of used items in the cache.
  253. The clear method is a no-operation function.
  254. .. versionadded:: 2.7
  255. Added support for ignoring memcache errors through the
  256. `ignore_memcache_errors` parameter.
  257. """
  258. def __init__(
  259. self,
  260. client,
  261. prefix="jinja2/bytecode/",
  262. timeout=None,
  263. ignore_memcache_errors=True,
  264. ):
  265. self.client = client
  266. self.prefix = prefix
  267. self.timeout = timeout
  268. self.ignore_memcache_errors = ignore_memcache_errors
  269. def load_bytecode(self, bucket):
  270. try:
  271. code = self.client.get(self.prefix + bucket.key)
  272. except Exception:
  273. if not self.ignore_memcache_errors:
  274. raise
  275. code = None
  276. if code is not None:
  277. bucket.bytecode_from_string(code)
  278. def dump_bytecode(self, bucket):
  279. args = (self.prefix + bucket.key, bucket.bytecode_to_string())
  280. if self.timeout is not None:
  281. args += (self.timeout,)
  282. try:
  283. self.client.set(*args)
  284. except Exception:
  285. if not self.ignore_memcache_errors:
  286. raise