| 1234567891011121314151617181920212223 |
- """Set User-Agent header per spider or use a default value from settings"""
- from scrapy import signals
- class UserAgentMiddleware(object):
- """This middleware allows spiders to override the user_agent"""
- def __init__(self, user_agent='Scrapy'):
- self.user_agent = user_agent
- @classmethod
- def from_crawler(cls, crawler):
- o = cls(crawler.settings['USER_AGENT'])
- crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
- return o
- def spider_opened(self, spider):
- self.user_agent = getattr(spider, 'user_agent', self.user_agent)
- def process_request(self, request, spider):
- if self.user_agent:
- request.headers.setdefault(b'User-Agent', self.user_agent)
|
