| 12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879 |
- """Helper functions for working with signals"""
- import logging
- from twisted.internet.defer import maybeDeferred, DeferredList, Deferred
- from twisted.python.failure import Failure
- from pydispatch.dispatcher import Any, Anonymous, liveReceivers, \
- getAllReceivers, disconnect
- from pydispatch.robustapply import robustApply
- from scrapy.utils.log import failure_to_exc_info
- logger = logging.getLogger(__name__)
- class _IgnoredException(Exception):
- pass
- def send_catch_log(signal=Any, sender=Anonymous, *arguments, **named):
- """Like pydispatcher.robust.sendRobust but it also logs errors and returns
- Failures instead of exceptions.
- """
- dont_log = named.pop('dont_log', _IgnoredException)
- spider = named.get('spider', None)
- responses = []
- for receiver in liveReceivers(getAllReceivers(sender, signal)):
- try:
- response = robustApply(receiver, signal=signal, sender=sender,
- *arguments, **named)
- if isinstance(response, Deferred):
- logger.error("Cannot return deferreds from signal handler: %(receiver)s",
- {'receiver': receiver}, extra={'spider': spider})
- except dont_log:
- result = Failure()
- except Exception:
- result = Failure()
- logger.error("Error caught on signal handler: %(receiver)s",
- {'receiver': receiver},
- exc_info=True, extra={'spider': spider})
- else:
- result = response
- responses.append((receiver, result))
- return responses
- def send_catch_log_deferred(signal=Any, sender=Anonymous, *arguments, **named):
- """Like send_catch_log but supports returning deferreds on signal handlers.
- Returns a deferred that gets fired once all signal handlers deferreds were
- fired.
- """
- def logerror(failure, recv):
- if dont_log is None or not isinstance(failure.value, dont_log):
- logger.error("Error caught on signal handler: %(receiver)s",
- {'receiver': recv},
- exc_info=failure_to_exc_info(failure),
- extra={'spider': spider})
- return failure
- dont_log = named.pop('dont_log', None)
- spider = named.get('spider', None)
- dfds = []
- for receiver in liveReceivers(getAllReceivers(sender, signal)):
- d = maybeDeferred(robustApply, receiver, signal=signal, sender=sender,
- *arguments, **named)
- d.addErrback(logerror, receiver)
- d.addBoth(lambda result: (receiver, result))
- dfds.append(d)
- d = DeferredList(dfds)
- d.addCallback(lambda out: [x[1] for x in out])
- return d
- def disconnect_all(signal=Any, sender=Any):
- """Disconnect all signal handlers. Useful for cleaning up after running
- tests
- """
- for receiver in liveReceivers(getAllReceivers(sender, signal)):
- disconnect(receiver, signal=signal, sender=sender)
|
