restarter.py 4.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120
  1. """A basic kernel monitor with autorestarting.
  2. This watches a kernel's state using KernelManager.is_alive and auto
  3. restarts the kernel if it dies.
  4. It is an incomplete base class, and must be subclassed.
  5. """
  6. # Copyright (c) Jupyter Development Team.
  7. # Distributed under the terms of the Modified BSD License.
  8. from traitlets.config.configurable import LoggingConfigurable
  9. from traitlets import (
  10. Instance, Float, Dict, Bool, Integer,
  11. )
  12. class KernelRestarter(LoggingConfigurable):
  13. """Monitor and autorestart a kernel."""
  14. kernel_manager = Instance('jupyter_client.KernelManager')
  15. debug = Bool(False, config=True,
  16. help="""Whether to include every poll event in debugging output.
  17. Has to be set explicitly, because there will be *a lot* of output.
  18. """
  19. )
  20. time_to_dead = Float(3.0, config=True,
  21. help="""Kernel heartbeat interval in seconds."""
  22. )
  23. restart_limit = Integer(5, config=True,
  24. help="""The number of consecutive autorestarts before the kernel is presumed dead."""
  25. )
  26. random_ports_until_alive = Bool(True, config=True,
  27. help="""Whether to choose new random ports when restarting before the kernel is alive."""
  28. )
  29. _restarting = Bool(False)
  30. _restart_count = Integer(0)
  31. _initial_startup = Bool(True)
  32. callbacks = Dict()
  33. def _callbacks_default(self):
  34. return dict(restart=[], dead=[])
  35. def start(self):
  36. """Start the polling of the kernel."""
  37. raise NotImplementedError("Must be implemented in a subclass")
  38. def stop(self):
  39. """Stop the kernel polling."""
  40. raise NotImplementedError("Must be implemented in a subclass")
  41. def add_callback(self, f, event='restart'):
  42. """register a callback to fire on a particular event
  43. Possible values for event:
  44. 'restart' (default): kernel has died, and will be restarted.
  45. 'dead': restart has failed, kernel will be left dead.
  46. """
  47. self.callbacks[event].append(f)
  48. def remove_callback(self, f, event='restart'):
  49. """unregister a callback to fire on a particular event
  50. Possible values for event:
  51. 'restart' (default): kernel has died, and will be restarted.
  52. 'dead': restart has failed, kernel will be left dead.
  53. """
  54. try:
  55. self.callbacks[event].remove(f)
  56. except ValueError:
  57. pass
  58. def _fire_callbacks(self, event):
  59. """fire our callbacks for a particular event"""
  60. for callback in self.callbacks[event]:
  61. try:
  62. callback()
  63. except Exception as e:
  64. self.log.error("KernelRestarter: %s callback %r failed", event, callback, exc_info=True)
  65. def poll(self):
  66. if self.debug:
  67. self.log.debug('Polling kernel...')
  68. if not self.kernel_manager.is_alive():
  69. if self._restarting:
  70. self._restart_count += 1
  71. else:
  72. self._restart_count = 1
  73. if self._restart_count >= self.restart_limit:
  74. self.log.warning("KernelRestarter: restart failed")
  75. self._fire_callbacks('dead')
  76. self._restarting = False
  77. self._restart_count = 0
  78. self.stop()
  79. else:
  80. newports = self.random_ports_until_alive and self._initial_startup
  81. self.log.info('KernelRestarter: restarting kernel (%i/%i), %s random ports',
  82. self._restart_count,
  83. self.restart_limit,
  84. 'new' if newports else 'keep'
  85. )
  86. self._fire_callbacks('restart')
  87. self.kernel_manager.restart_kernel(now=True, newports=newports)
  88. self._restarting = True
  89. else:
  90. if self._initial_startup:
  91. self._initial_startup = False
  92. if self._restarting:
  93. self.log.debug("KernelRestarter: restart apparently succeeded")
  94. self._restarting = False