Debian Codegraph @

django/utils/autoreload.py

raw
1 import itertools
2 import logging
3 import os
4 import signal
5 import subprocess
6 import sys
7 import threading
8 import time
9 import traceback
10 import weakref
11 from collections import defaultdict
12 from functools import lru_cache, wraps
13 from pathlib import Path
14 from types import ModuleType
15 from zipimport import zipimporter
16
17 import django
18 from django.apps import apps
19 from django.core.signals import request_finished
20 from django.dispatch import Signal
21 from django.utils.functional import cached_property
22 from django.utils.version import get_version_tuple
23
24 autoreload_started = Signal()
25 file_changed = Signal()
26
27 DJANGO_AUTORELOAD_ENV = "RUN_MAIN"
28
29 logger = logging.getLogger("django.utils.autoreload")
30
31 # If an error is raised while importing a file, it's not placed in sys.modules.
32 # This means that any future modifications aren't caught. Keep a list of these
33 # file paths to allow watching them in the future.
34 _error_files = []
35 _exception = None
36
37 try:
38 import termios
39 except ImportError:
40 termios = None
41
42
43 try:
44 import pywatchman
45 except ImportError:
46 pywatchman = None
47
48
49 def is_django_module(module):
50 """Return True if the given module is nested under Django."""
51 return module.__name__.startswith("django.")
52
53
54 def is_django_path(path):
55 """Return True if the given file path is nested under Django."""
56 return Path(django.__file__).parent in Path(path).parents
57
58
59 def check_errors(fn):
60 @wraps(fn)
61 def wrapper(*args, **kwargs):
62 global _exception
63 try:
64 fn(*args, **kwargs)
65 except Exception:
66 _exception = sys.exc_info()
67
68 et, ev, tb = _exception
69
70 if getattr(ev, "filename", None) is None:
71 # get the filename from the last item in the stack
72 filename = traceback.extract_tb(tb)[-1][0]
73 else:
74 filename = ev.filename
75
76 if filename not in _error_files:
77 _error_files.append(filename)
78
79 raise
80
81 return wrapper
82
83
84 def raise_last_exception():
85 if _exception is not None:
86 raise _exception[1]
87
88
89 def ensure_echo_on():
90 """
91 Ensure that echo mode is enabled. Some tools such as PDB disable
92 it which causes usability issues after reload.
93 """
94 if not termios or not sys.stdin.isatty():
95 return
96 attr_list = termios.tcgetattr(sys.stdin)
97 if not attr_list[3] & termios.ECHO:
98 attr_list[3] |= termios.ECHO
99 if hasattr(signal, "SIGTTOU"):
100 old_handler = signal.signal(signal.SIGTTOU, signal.SIG_IGN)
101 else:
102 old_handler = None
103 termios.tcsetattr(sys.stdin, termios.TCSANOW, attr_list)
104 if old_handler is not None:
105 signal.signal(signal.SIGTTOU, old_handler)
106
107
108 def iter_all_python_module_files():
109 # This is a hot path during reloading. Create a stable sorted list of
110 # modules based on the module name and pass it to iter_modules_and_files().
111 # This ensures cached results are returned in the usual case that modules
112 # aren't loaded on the fly.
113 keys = sorted(sys.modules)
114 modules = tuple(
115 m
116 for m in map(sys.modules.__getitem__, keys)
117 if not isinstance(m, weakref.ProxyTypes)
118 )
119 return iter_modules_and_files(modules, frozenset(_error_files))
120
121
122 @lru_cache(maxsize=1)
123 def iter_modules_and_files(modules, extra_files):
124 """Iterate through all modules needed to be watched."""
125 sys_file_paths = []
126 for module in modules:
127 # During debugging (with PyDev) the 'typing.io' and 'typing.re' objects
128 # are added to sys.modules, however they are types not modules and so
129 # cause issues here.
130 if not isinstance(module, ModuleType):
131 continue
132 if module.__name__ in ("__main__", "__mp_main__"):
133 # __main__ (usually manage.py) doesn't always have a __spec__ set.
134 # Handle this by falling back to using __file__, resolved below.
135 # See https://docs.python.org/reference/import.html#main-spec
136 # __file__ may not exists, e.g. when running ipdb debugger.
137 if hasattr(module, "__file__"):
138 sys_file_paths.append(module.__file__)
139 continue
140 if getattr(module, "__spec__", None) is None:
141 continue
142 spec = module.__spec__
143 # Modules could be loaded from places without a concrete location. If
144 # this is the case, skip them.
145 if spec.has_location:
146 origin = (
147 spec.loader.archive
148 if isinstance(spec.loader, zipimporter)
149 else spec.origin
150 )
151 sys_file_paths.append(origin)
152
153 results = set()
154 for filename in itertools.chain(sys_file_paths, extra_files):
155 if not filename:
156 continue
157 path = Path(filename)
158 try:
159 if not path.exists():
160 # The module could have been removed, don't fail loudly if this
161 # is the case.
162 continue
163 except ValueError as e:
164 # Network filesystems may return null bytes in file paths.
165 logger.debug('"%s" raised when resolving path: "%s"', e, path)
166 continue
167 resolved_path = path.resolve().absolute()
168 results.add(resolved_path)
169 return frozenset(results)
170
171
172 @lru_cache(maxsize=1)
173 def common_roots(paths):
174 """
175 Return a tuple of common roots that are shared between the given paths.
176 File system watchers operate on directories and aren't cheap to create.
177 Try to find the minimum set of directories to watch that encompass all of
178 the files that need to be watched.
179 """
180 # Inspired from Werkzeug:
181 # https://github.com/pallets/werkzeug/blob/7477be2853df70a022d9613e765581b9411c3c39/werkzeug/_reloader.py
182 # Create a sorted list of the path components, longest first.
183 path_parts = sorted([x.parts for x in paths], key=len, reverse=True)
184 tree = {}
185 for chunks in path_parts:
186 node = tree
187 # Add each part of the path to the tree.
188 for chunk in chunks:
189 node = node.setdefault(chunk, {})
190 # Clear the last leaf in the tree.
191 node.clear()
192
193 # Turn the tree into a list of Path instances.
194 def _walk(node, path):
195 for prefix, child in node.items():
196 yield from _walk(child, path + (prefix,))
197 if not node:
198 yield Path(*path)
199
200 return tuple(_walk(tree, ()))
201
202
203 def sys_path_directories():
204 """
205 Yield absolute directories from sys.path, ignoring entries that don't
206 exist.
207 """
208 for path in sys.path:
209 path = Path(path)
210 if not path.exists():
211 continue
212 resolved_path = path.resolve().absolute()
213 # If the path is a file (like a zip file), watch the parent directory.
214 if resolved_path.is_file():
215 yield resolved_path.parent
216 else:
217 yield resolved_path
218
219
220 def get_child_arguments():
221 """
222 Return the executable. This contains a workaround for Windows if the
223 executable is reported to not have the .exe extension which can cause bugs
224 on reloading.
225 """
226 import __main__
227
228 py_script = Path(sys.argv[0])
229 exe_entrypoint = py_script.with_suffix(".exe")
230
231 args = [sys.executable] + ["-W%s" % o for o in sys.warnoptions]
232 if sys.implementation.name in ("cpython", "pypy"):
233 args.extend(
234 f"-X{key}" if value is True else f"-X{key}={value}"
235 for key, value in sys._xoptions.items()
236 )
237 # __spec__ is set when the server was started with the `-m` option,
238 # see https://docs.python.org/3/reference/import.html#main-spec
239 # __spec__ may not exist, e.g. when running in a Conda env.
240 if getattr(__main__, "__spec__", None) is not None and not exe_entrypoint.exists():
241 spec = __main__.__spec__
242 if (spec.name == "__main__" or spec.name.endswith(".__main__")) and spec.parent:
243 name = spec.parent
244 else:
245 name = spec.name
246 args += ["-m", name]
247 args += sys.argv[1:]
248 elif not py_script.exists():
249 # sys.argv[0] may not exist for several reasons on Windows.
250 # It may exist with a .exe extension or have a -script.py suffix.
251 if exe_entrypoint.exists():
252 # Should be executed directly, ignoring sys.executable.
253 return [exe_entrypoint, *sys.argv[1:]]
254 script_entrypoint = py_script.with_name("%s-script.py" % py_script.name)
255 if script_entrypoint.exists():
256 # Should be executed as usual.
257 return [*args, script_entrypoint, *sys.argv[1:]]
258 raise RuntimeError("Script %s does not exist." % py_script)
259 else:
260 args += sys.argv
261 return args
262
263
264 def trigger_reload(filename):
265 logger.info("%s changed, reloading.", filename)
266 sys.exit(3)
267
268
269 def restart_with_reloader():
270 new_environ = {**os.environ, DJANGO_AUTORELOAD_ENV: "true"}
271 args = get_child_arguments()
272 while True:
273 p = subprocess.run(args, env=new_environ, close_fds=False)
274 if p.returncode != 3:
275 return p.returncode
276
277
278 class BaseReloader:
279 def __init__(self):
280 self.extra_files = set()
281 self.directory_globs = defaultdict(set)
282 self._stop_condition = threading.Event()
283
284 def watch_dir(self, path, glob):
285 path = Path(path)
286 try:
287 path = path.absolute()
288 except FileNotFoundError:
289 logger.debug(
290 "Unable to watch directory %s as it cannot be resolved.",
291 path,
292 exc_info=True,
293 )
294 return
295 logger.debug("Watching dir %s with glob %s.", path, glob)
296 self.directory_globs[path].add(glob)
297
298 def watched_files(self, include_globs=True):
299 """
300 Yield all files that need to be watched, including module files and
301 files within globs.
302 """
303 yield from iter_all_python_module_files()
304 yield from self.extra_files
305 if include_globs:
306 for directory, patterns in self.directory_globs.items():
307 for pattern in patterns:
308 yield from directory.glob(pattern)
309
310 def wait_for_apps_ready(self, app_reg, django_main_thread):
311 """
312 Wait until Django reports that the apps have been loaded. If the given
313 thread has terminated before the apps are ready, then a SyntaxError or
314 other non-recoverable error has been raised. In that case, stop waiting
315 for the apps_ready event and continue processing.
316
317 Return True if the thread is alive and the ready event has been
318 triggered, or False if the thread is terminated while waiting for the
319 event.
320 """
321 while django_main_thread.is_alive():
322 if app_reg.ready_event.wait(timeout=0.1):
323 return True
324 else:
325 logger.debug("Main Django thread has terminated before apps are ready.")
326 return False
327
328 def run(self, django_main_thread):
329 logger.debug("Waiting for apps ready_event.")
330 self.wait_for_apps_ready(apps, django_main_thread)
331 from django.urls import get_resolver
332
333 # Prevent a race condition where URL modules aren't loaded when the
334 # reloader starts by accessing the urlconf_module property.
335 try:
336 get_resolver().urlconf_module
337 except Exception:
338 # Loading the urlconf can result in errors during development.
339 # If this occurs then swallow the error and continue.
340 pass
341 logger.debug("Apps ready_event triggered. Sending autoreload_started signal.")
342 autoreload_started.send(sender=self)
343 self.run_loop()
344
345 def run_loop(self):
346 ticker = self.tick()
347 while not self.should_stop:
348 try:
349 next(ticker)
350 except StopIteration:
351 break
352 self.stop()
353
354 def tick(self):
355 """
356 This generator is called in a loop from run_loop. It's important that
357 the method takes care of pausing or otherwise waiting for a period of
358 time. This split between run_loop() and tick() is to improve the
359 testability of the reloader implementations by decoupling the work they
360 do from the loop.
361 """
362 raise NotImplementedError("subclasses must implement tick().")
363
364 @classmethod
365 def check_availability(cls):
366 raise NotImplementedError("subclasses must implement check_availability().")
367
368 def notify_file_changed(self, path):
369 results = file_changed.send(sender=self, file_path=path)
370 logger.debug("%s notified as changed. Signal results: %s.", path, results)
371 if not any(res[1] for res in results):
372 trigger_reload(path)
373
374 # These are primarily used for testing.
375 @property
376 def should_stop(self):
377 return self._stop_condition.is_set()
378
379 def stop(self):
380 self._stop_condition.set()
381
382
383 class StatReloader(BaseReloader):
384 SLEEP_TIME = 1 # Check for changes once per second.
385
386 def tick(self):
387 mtimes = {}
388 while True:
389 for filepath, mtime in self.snapshot_files():
390 old_time = mtimes.get(filepath)
391 mtimes[filepath] = mtime
392 if old_time is None:
393 logger.debug("File %s first seen with mtime %s", filepath, mtime)
394 continue
395 elif mtime > old_time:
396 logger.debug(
397 "File %s previous mtime: %s, current mtime: %s",
398 filepath,
399 old_time,
400 mtime,
401 )
402 self.notify_file_changed(filepath)
403
404 time.sleep(self.SLEEP_TIME)
405 yield
406
407 def snapshot_files(self):
408 # watched_files may produce duplicate paths if globs overlap.
409 seen_files = set()
410 for file in self.watched_files():
411 if file in seen_files:
412 continue
413 try:
414 mtime = file.stat().st_mtime
415 except OSError:
416 # This is thrown when the file does not exist.
417 continue
418 seen_files.add(file)
419 yield file, mtime
420
421 @classmethod
422 def check_availability(cls):
423 return True
424
425
426 class WatchmanUnavailable(RuntimeError):
427 pass
428
429
430 class WatchmanReloader(BaseReloader):
431 def __init__(self):
432 self.roots = defaultdict(set)
433 self.processed_request = threading.Event()
434 self.client_timeout = int(os.environ.get("DJANGO_WATCHMAN_TIMEOUT", 5))
435 super().__init__()
436
437 @cached_property
438 def client(self):
439 return pywatchman.client(timeout=self.client_timeout)
440
441 def _watch_root(self, root):
442 # In practice this shouldn't occur, however, it's possible that a
443 # directory that doesn't exist yet is being watched. If it's outside of
444 # sys.path then this will end up a new root. How to handle this isn't
445 # clear: Not adding the root will likely break when subscribing to the
446 # changes, however, as this is currently an internal API, no files
447 # will be being watched outside of sys.path. Fixing this by checking
448 # inside watch_glob() and watch_dir() is expensive, instead this could
449 # could fall back to the StatReloader if this case is detected? For
450 # now, watching its parent, if possible, is sufficient.
451 if not root.exists():
452 if not root.parent.exists():
453 logger.warning(
454 "Unable to watch root dir %s as neither it or its parent exist.",
455 root,
456 )
457 return
458 root = root.parent
459 result = self.client.query("watch-project", str(root.absolute()))
460 if "warning" in result:
461 logger.warning("Watchman warning: %s", result["warning"])
462 logger.debug("Watchman watch-project result: %s", result)
463 return result["watch"], result.get("relative_path")
464
465 @lru_cache
466 def _get_clock(self, root):
467 return self.client.query("clock", root)["clock"]
468
469 def _subscribe(self, directory, name, expression):
470 root, rel_path = self._watch_root(directory)
471 # Only receive notifications of files changing, filtering out other types
472 # like special files: https://facebook.github.io/watchman/docs/type
473 only_files_expression = [
474 "allof",
475 ["anyof", ["type", "f"], ["type", "l"]],
476 expression,
477 ]
478 query = {
479 "expression": only_files_expression,
480 "fields": ["name"],
481 "since": self._get_clock(root),
482 "dedup_results": True,
483 }
484 if rel_path:
485 query["relative_root"] = rel_path
486 logger.debug(
487 "Issuing watchman subscription %s, for root %s. Query: %s",
488 name,
489 root,
490 query,
491 )
492 self.client.query("subscribe", root, name, query)
493
494 def _subscribe_dir(self, directory, filenames):
495 if not directory.exists():
496 if not directory.parent.exists():
497 logger.warning(
498 "Unable to watch directory %s as neither it or its parent exist.",
499 directory,
500 )
501 return
502 prefix = "files-parent-%s" % directory.name
503 filenames = ["%s/%s" % (directory.name, filename) for filename in filenames]
504 directory = directory.parent
505 expression = ["name", filenames, "wholename"]
506 else:
507 prefix = "files"
508 expression = ["name", filenames]
509 self._subscribe(directory, "%s:%s" % (prefix, directory), expression)
510
511 def _watch_glob(self, directory, patterns):
512 """
513 Watch a directory with a specific glob. If the directory doesn't yet
514 exist, attempt to watch the parent directory and amend the patterns to
515 include this. It's important this method isn't called more than one per
516 directory when updating all subscriptions. Subsequent calls will
517 overwrite the named subscription, so it must include all possible glob
518 expressions.
519 """
520 prefix = "glob"
521 if not directory.exists():
522 if not directory.parent.exists():
523 logger.warning(
524 "Unable to watch directory %s as neither it or its parent exist.",
525 directory,
526 )
527 return
528 prefix = "glob-parent-%s" % directory.name
529 patterns = ["%s/%s" % (directory.name, pattern) for pattern in patterns]
530 directory = directory.parent
531
532 expression = ["anyof"]
533 for pattern in patterns:
534 expression.append(["match", pattern, "wholename"])
535 self._subscribe(directory, "%s:%s" % (prefix, directory), expression)
536
537 def watched_roots(self, watched_files):
538 extra_directories = self.directory_globs.keys()
539 watched_file_dirs = [f.parent for f in watched_files]
540 sys_paths = list(sys_path_directories())
541 return frozenset((*extra_directories, *watched_file_dirs, *sys_paths))
542
543 def _update_watches(self):
544 watched_files = list(self.watched_files(include_globs=False))
545 found_roots = common_roots(self.watched_roots(watched_files))
546 logger.debug("Watching %s files", len(watched_files))
547 logger.debug("Found common roots: %s", found_roots)
548 # Setup initial roots for performance, shortest roots first.
549 for root in sorted(found_roots):
550 self._watch_root(root)
551 for directory, patterns in self.directory_globs.items():
552 self._watch_glob(directory, patterns)
553 # Group sorted watched_files by their parent directory.
554 sorted_files = sorted(watched_files, key=lambda p: p.parent)
555 for directory, group in itertools.groupby(sorted_files, key=lambda p: p.parent):
556 # These paths need to be relative to the parent directory.
557 self._subscribe_dir(
558 directory, [str(p.relative_to(directory)) for p in group]
559 )
560
561 def update_watches(self):
562 try:
563 self._update_watches()
564 except Exception as ex:
565 # If the service is still available, raise the original exception.
566 if self.check_server_status(ex):
567 raise
568
569 def _check_subscription(self, sub):
570 subscription = self.client.getSubscription(sub)
571 if not subscription:
572 return
573 logger.debug("Watchman subscription %s has results.", sub)
574 for result in subscription:
575 # When using watch-project, it's not simple to get the relative
576 # directory without storing some specific state. Store the full
577 # path to the directory in the subscription name, prefixed by its
578 # type (glob, files).
579 root_directory = Path(result["subscription"].split(":", 1)[1])
580 logger.debug("Found root directory %s", root_directory)
581 for file in result.get("files", []):
582 self.notify_file_changed(root_directory / file)
583
584 def request_processed(self, **kwargs):
585 logger.debug("Request processed. Setting update_watches event.")
586 self.processed_request.set()
587
588 def tick(self):
589 request_finished.connect(self.request_processed)
590 self.update_watches()
591 while True:
592 if self.processed_request.is_set():
593 self.update_watches()
594 self.processed_request.clear()
595 try:
596 self.client.receive()
597 except pywatchman.SocketTimeout:
598 pass
599 except pywatchman.WatchmanError as ex:
600 logger.debug("Watchman error: %s, checking server status.", ex)
601 self.check_server_status(ex)
602 else:
603 for sub in list(self.client.subs.keys()):
604 self._check_subscription(sub)
605 yield
606 # Protect against busy loops.
607 time.sleep(0.1)
608
609 def stop(self):
610 self.client.close()
611 super().stop()
612
613 def check_server_status(self, inner_ex=None):
614 """Return True if the server is available."""
615 try:
616 self.client.query("version")
617 except Exception:
618 raise WatchmanUnavailable(str(inner_ex)) from inner_ex
619 return True
620
621 @classmethod
622 def check_availability(cls):
623 if not pywatchman:
624 raise WatchmanUnavailable("pywatchman not installed.")
625 client = pywatchman.client(timeout=0.1)
626 try:
627 result = client.capabilityCheck()
628 except Exception:
629 # The service is down?
630 raise WatchmanUnavailable("Cannot connect to the watchman service.")
631 version = get_version_tuple(result["version"])
632 # Watchman 4.9 includes multiple improvements to watching project
633 # directories as well as case insensitive filesystems.
634 logger.debug("Watchman version %s", version)
635 if version < (4, 9):
636 raise WatchmanUnavailable("Watchman 4.9 or later is required.")
637
638
639 def get_reloader():
640 """Return the most suitable reloader for this environment."""
641 try:
642 WatchmanReloader.check_availability()
643 except WatchmanUnavailable:
644 return StatReloader()
645 return WatchmanReloader()
646
647
648 def start_django(reloader, main_func, *args, **kwargs):
649 ensure_echo_on()
650
651 main_func = check_errors(main_func)
652 django_main_thread = threading.Thread(
653 target=main_func, args=args, kwargs=kwargs, name="django-main-thread"
654 )
655 django_main_thread.daemon = True
656 django_main_thread.start()
657
658 while not reloader.should_stop:
659 reloader.run(django_main_thread)
660
661
662 def run_with_reloader(main_func, *args, **kwargs):
663 signal.signal(signal.SIGTERM, lambda *args: sys.exit(0))
664 try:
665 if os.environ.get(DJANGO_AUTORELOAD_ENV) == "true":
666 reloader = get_reloader()
667 logger.info(
668 "Watching for file changes with %s", reloader.__class__.__name__
669 )
670 start_django(reloader, main_func, *args, **kwargs)
671 else:
672 exit_code = restart_with_reloader()
673 sys.exit(exit_code)
674 except KeyboardInterrupt:
675 pass