| 12345678910111213141516171819202122232425262728293031323334 |
- from scrapy.exceptions import NotConfigured
- from scrapy.utils.request import request_httprepr
- from scrapy.utils.response import response_httprepr
- from scrapy.utils.python import global_object_name
- class DownloaderStats(object):
- def __init__(self, stats):
- self.stats = stats
- @classmethod
- def from_crawler(cls, crawler):
- if not crawler.settings.getbool('DOWNLOADER_STATS'):
- raise NotConfigured
- return cls(crawler.stats)
- def process_request(self, request, spider):
- self.stats.inc_value('downloader/request_count', spider=spider)
- self.stats.inc_value('downloader/request_method_count/%s' % request.method, spider=spider)
- reqlen = len(request_httprepr(request))
- self.stats.inc_value('downloader/request_bytes', reqlen, spider=spider)
- def process_response(self, request, response, spider):
- self.stats.inc_value('downloader/response_count', spider=spider)
- self.stats.inc_value('downloader/response_status_count/%s' % response.status, spider=spider)
- reslen = len(response_httprepr(response))
- self.stats.inc_value('downloader/response_bytes', reslen, spider=spider)
- return response
- def process_exception(self, request, exception, spider):
- ex_class = global_object_name(exception.__class__)
- self.stats.inc_value('downloader/exception_count', spider=spider)
- self.stats.inc_value('downloader/exception_type_count/%s' % ex_class, spider=spider)
|
