_reloader.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341
  1. import os
  2. import subprocess
  3. import sys
  4. import threading
  5. import time
  6. from itertools import chain
  7. from ._compat import iteritems
  8. from ._compat import PY2
  9. from ._compat import text_type
  10. from ._internal import _log
  11. def _iter_module_files():
  12. """This iterates over all relevant Python files. It goes through all
  13. loaded files from modules, all files in folders of already loaded modules
  14. as well as all files reachable through a package.
  15. """
  16. # The list call is necessary on Python 3 in case the module
  17. # dictionary modifies during iteration.
  18. for module in list(sys.modules.values()):
  19. if module is None:
  20. continue
  21. filename = getattr(module, "__file__", None)
  22. if filename:
  23. if os.path.isdir(filename) and os.path.exists(
  24. os.path.join(filename, "__init__.py")
  25. ):
  26. filename = os.path.join(filename, "__init__.py")
  27. old = None
  28. while not os.path.isfile(filename):
  29. old = filename
  30. filename = os.path.dirname(filename)
  31. if filename == old:
  32. break
  33. else:
  34. if filename[-4:] in (".pyc", ".pyo"):
  35. filename = filename[:-1]
  36. yield filename
  37. def _find_observable_paths(extra_files=None):
  38. """Finds all paths that should be observed."""
  39. rv = set(
  40. os.path.dirname(os.path.abspath(x)) if os.path.isfile(x) else os.path.abspath(x)
  41. for x in sys.path
  42. )
  43. for filename in extra_files or ():
  44. rv.add(os.path.dirname(os.path.abspath(filename)))
  45. for module in list(sys.modules.values()):
  46. fn = getattr(module, "__file__", None)
  47. if fn is None:
  48. continue
  49. fn = os.path.abspath(fn)
  50. rv.add(os.path.dirname(fn))
  51. return _find_common_roots(rv)
  52. def _get_args_for_reloading():
  53. """Determine how the script was executed, and return the args needed
  54. to execute it again in a new process.
  55. """
  56. rv = [sys.executable]
  57. py_script = sys.argv[0]
  58. args = sys.argv[1:]
  59. # Need to look at main module to determine how it was executed.
  60. __main__ = sys.modules["__main__"]
  61. # The value of __package__ indicates how Python was called. It may
  62. # not exist if a setuptools script is installed as an egg. It may be
  63. # set incorrectly for entry points created with pip on Windows.
  64. if getattr(__main__, "__package__", None) is None or (
  65. os.name == "nt"
  66. and __main__.__package__ == ""
  67. and not os.path.exists(py_script)
  68. and os.path.exists(py_script + ".exe")
  69. ):
  70. # Executed a file, like "python app.py".
  71. py_script = os.path.abspath(py_script)
  72. if os.name == "nt":
  73. # Windows entry points have ".exe" extension and should be
  74. # called directly.
  75. if not os.path.exists(py_script) and os.path.exists(py_script + ".exe"):
  76. py_script += ".exe"
  77. if (
  78. os.path.splitext(sys.executable)[1] == ".exe"
  79. and os.path.splitext(py_script)[1] == ".exe"
  80. ):
  81. rv.pop(0)
  82. rv.append(py_script)
  83. else:
  84. # Executed a module, like "python -m werkzeug.serving".
  85. if sys.argv[0] == "-m":
  86. # Flask works around previous behavior by putting
  87. # "-m flask" in sys.argv.
  88. # TODO remove this once Flask no longer misbehaves
  89. args = sys.argv
  90. else:
  91. if os.path.isfile(py_script):
  92. # Rewritten by Python from "-m script" to "/path/to/script.py".
  93. py_module = __main__.__package__
  94. name = os.path.splitext(os.path.basename(py_script))[0]
  95. if name != "__main__":
  96. py_module += "." + name
  97. else:
  98. # Incorrectly rewritten by pydevd debugger from "-m script" to "script".
  99. py_module = py_script
  100. rv.extend(("-m", py_module.lstrip(".")))
  101. rv.extend(args)
  102. return rv
  103. def _find_common_roots(paths):
  104. """Out of some paths it finds the common roots that need monitoring."""
  105. paths = [x.split(os.path.sep) for x in paths]
  106. root = {}
  107. for chunks in sorted(paths, key=len, reverse=True):
  108. node = root
  109. for chunk in chunks:
  110. node = node.setdefault(chunk, {})
  111. node.clear()
  112. rv = set()
  113. def _walk(node, path):
  114. for prefix, child in iteritems(node):
  115. _walk(child, path + (prefix,))
  116. if not node:
  117. rv.add("/".join(path))
  118. _walk(root, ())
  119. return rv
  120. class ReloaderLoop(object):
  121. name = None
  122. # monkeypatched by testsuite. wrapping with `staticmethod` is required in
  123. # case time.sleep has been replaced by a non-c function (e.g. by
  124. # `eventlet.monkey_patch`) before we get here
  125. _sleep = staticmethod(time.sleep)
  126. def __init__(self, extra_files=None, interval=1):
  127. self.extra_files = set(os.path.abspath(x) for x in extra_files or ())
  128. self.interval = interval
  129. def run(self):
  130. pass
  131. def restart_with_reloader(self):
  132. """Spawn a new Python interpreter with the same arguments as this one,
  133. but running the reloader thread.
  134. """
  135. while 1:
  136. _log("info", " * Restarting with %s" % self.name)
  137. args = _get_args_for_reloading()
  138. # a weird bug on windows. sometimes unicode strings end up in the
  139. # environment and subprocess.call does not like this, encode them
  140. # to latin1 and continue.
  141. if os.name == "nt" and PY2:
  142. new_environ = {}
  143. for key, value in iteritems(os.environ):
  144. if isinstance(key, text_type):
  145. key = key.encode("iso-8859-1")
  146. if isinstance(value, text_type):
  147. value = value.encode("iso-8859-1")
  148. new_environ[key] = value
  149. else:
  150. new_environ = os.environ.copy()
  151. new_environ["WERKZEUG_RUN_MAIN"] = "true"
  152. exit_code = subprocess.call(args, env=new_environ, close_fds=False)
  153. if exit_code != 3:
  154. return exit_code
  155. def trigger_reload(self, filename):
  156. self.log_reload(filename)
  157. sys.exit(3)
  158. def log_reload(self, filename):
  159. filename = os.path.abspath(filename)
  160. _log("info", " * Detected change in %r, reloading" % filename)
  161. class StatReloaderLoop(ReloaderLoop):
  162. name = "stat"
  163. def run(self):
  164. mtimes = {}
  165. while 1:
  166. for filename in chain(_iter_module_files(), self.extra_files):
  167. try:
  168. mtime = os.stat(filename).st_mtime
  169. except OSError:
  170. continue
  171. old_time = mtimes.get(filename)
  172. if old_time is None:
  173. mtimes[filename] = mtime
  174. continue
  175. elif mtime > old_time:
  176. self.trigger_reload(filename)
  177. self._sleep(self.interval)
  178. class WatchdogReloaderLoop(ReloaderLoop):
  179. def __init__(self, *args, **kwargs):
  180. ReloaderLoop.__init__(self, *args, **kwargs)
  181. from watchdog.observers import Observer
  182. from watchdog.events import FileSystemEventHandler
  183. self.observable_paths = set()
  184. def _check_modification(filename):
  185. if filename in self.extra_files:
  186. self.trigger_reload(filename)
  187. dirname = os.path.dirname(filename)
  188. if dirname.startswith(tuple(self.observable_paths)):
  189. if filename.endswith((".pyc", ".pyo", ".py")):
  190. self.trigger_reload(filename)
  191. class _CustomHandler(FileSystemEventHandler):
  192. def on_created(self, event):
  193. _check_modification(event.src_path)
  194. def on_modified(self, event):
  195. _check_modification(event.src_path)
  196. def on_moved(self, event):
  197. _check_modification(event.src_path)
  198. _check_modification(event.dest_path)
  199. def on_deleted(self, event):
  200. _check_modification(event.src_path)
  201. reloader_name = Observer.__name__.lower()
  202. if reloader_name.endswith("observer"):
  203. reloader_name = reloader_name[:-8]
  204. reloader_name += " reloader"
  205. self.name = reloader_name
  206. self.observer_class = Observer
  207. self.event_handler = _CustomHandler()
  208. self.should_reload = False
  209. def trigger_reload(self, filename):
  210. # This is called inside an event handler, which means throwing
  211. # SystemExit has no effect.
  212. # https://github.com/gorakhargosh/watchdog/issues/294
  213. self.should_reload = True
  214. self.log_reload(filename)
  215. def run(self):
  216. watches = {}
  217. observer = self.observer_class()
  218. observer.start()
  219. try:
  220. while not self.should_reload:
  221. to_delete = set(watches)
  222. paths = _find_observable_paths(self.extra_files)
  223. for path in paths:
  224. if path not in watches:
  225. try:
  226. watches[path] = observer.schedule(
  227. self.event_handler, path, recursive=True
  228. )
  229. except OSError:
  230. # Clear this path from list of watches We don't want
  231. # the same error message showing again in the next
  232. # iteration.
  233. watches[path] = None
  234. to_delete.discard(path)
  235. for path in to_delete:
  236. watch = watches.pop(path, None)
  237. if watch is not None:
  238. observer.unschedule(watch)
  239. self.observable_paths = paths
  240. self._sleep(self.interval)
  241. finally:
  242. observer.stop()
  243. observer.join()
  244. sys.exit(3)
  245. reloader_loops = {"stat": StatReloaderLoop, "watchdog": WatchdogReloaderLoop}
  246. try:
  247. __import__("watchdog.observers")
  248. except ImportError:
  249. reloader_loops["auto"] = reloader_loops["stat"]
  250. else:
  251. reloader_loops["auto"] = reloader_loops["watchdog"]
  252. def ensure_echo_on():
  253. """Ensure that echo mode is enabled. Some tools such as PDB disable
  254. it which causes usability issues after reload."""
  255. # tcgetattr will fail if stdin isn't a tty
  256. if not sys.stdin.isatty():
  257. return
  258. try:
  259. import termios
  260. except ImportError:
  261. return
  262. attributes = termios.tcgetattr(sys.stdin)
  263. if not attributes[3] & termios.ECHO:
  264. attributes[3] |= termios.ECHO
  265. termios.tcsetattr(sys.stdin, termios.TCSANOW, attributes)
  266. def run_with_reloader(main_func, extra_files=None, interval=1, reloader_type="auto"):
  267. """Run the given function in an independent python interpreter."""
  268. import signal
  269. reloader = reloader_loops[reloader_type](extra_files, interval)
  270. signal.signal(signal.SIGTERM, lambda *args: sys.exit(0))
  271. try:
  272. if os.environ.get("WERKZEUG_RUN_MAIN") == "true":
  273. ensure_echo_on()
  274. t = threading.Thread(target=main_func, args=())
  275. t.setDaemon(True)
  276. t.start()
  277. reloader.run()
  278. else:
  279. sys.exit(reloader.restart_with_reloader())
  280. except KeyboardInterrupt:
  281. pass