Scrapy学习笔记(8)-使用signals来监控spider
场景介绍
有时候我们需要在spider启动或者结束的时候执行一些特定的操作，比如说记录日志之类的，在scrapy中我们可以使用signals来实现。
主要实现代码如下：
# -*- coding: utf-8 -*-from scrapy import signalsfrom scrapy.xlib.pydispatch import dispatcherfrom scrapy.linkextractors import LinkExtractorfrom scrapy.spiders import CrawlSpider, Rulefrom items import IpProxyPoolItemfrom model.spider_running_log import SpiderCrawlLogfrom model import loadSessionfrom datetime import datetimeclass ProxySpiderSpider(CrawlSpider): name = 'MagicSpider' def __init__(self,rule): #spider启动信号和spider_opened函数绑定 dispatcher.connect(self.spider_opened, signals.spider_opened) #spider关闭信号和spider_spider_closed函数绑定 dispatcher.connect(self.spider_closed, signals.spider_closed) self.rule = rule
self.name = rule.name
self.allowed_domains = rule.allowed_domains.split(',') self.start_urls = rule.start_urls.split(',') rule_list = [] # 添加`下一页`的规则 if len(rule.next_page): rule_list.append(Rule(LinkExtractor(restrict_xpaths=rule.next_page), follow=True)) rule_list.append(Rule(LinkExtractor( allow=rule.allow_url.split(','), unique=True), follow=True, callback='parse_item')) self.rules = tuple(rule_list) super(ProxySpiderSpider, self).__init__() #spider关闭时的逻辑 def spider_closed(self, spider): print "spider is closed!" session = loadSession() log = session.query(SpiderCrawlLog).filter(SpiderCrawlLog.spiderID == self.rule.id, SpiderCrawlLog.endTime == None ).first() log.endTime = datetime.now().strftime("%Y-%m-%d %H:%M:%S") log.status = "closed" session.commit() #spider启动时的逻辑 def spider_opened(self, spider): print "spider is running!" item = SpiderCrawlLog( spiderID=self.rule.id, spiderName=self.rule.name, status="Running...", startTime=datetime.now().strftime("%Y-%m-%d %H:%M:%S"), endTime=None, pages=0, items=0 ) session = loadSession() log = session.query(SpiderCrawlLog).filter( SpiderCrawlLog.spiderID == self.rule.id, SpiderCrawlLog.endTime == None) # 查询当前spider是否有未结束的日志 if log.count() == 0 : session.add(item) session.commit() else: pass def parse_item(self, response): # print 'Hi, this is an item page! %s' % response.url #print response.body session = loadSession() log = session.query(SpiderCrawlLog).filter(SpiderCrawlLog.spiderID == self.rule.id, SpiderCrawlLog.endTime == None).first() log.pages = int(log.pages) + 1 session.commit() item=IpProxyPoolItem() if len(self.rule.loop_xpath): # print 'Find %d items!'% len(response.xpath(self.rule.loop_xpath)) for proxy in response.xpath(self.rule.loop_xpath): if len(self.rule.ip_xpath): tmp_ip = proxy.xpath(self.rule.ip_xpath).extract_first() ip = tmp_ip.strip() if tmp_ip is not None else "" else: ip = "" if len(self.rule.port_xpath): tmp_port = proxy.xpath(self.rule.port_xpath).extract_first() port = tmp_port.strip() if tmp_port is not None else "" else: port = "" if len(self.rule.location1_xpath): tmp_location1 = proxy.xpath(self.rule.location1_xpath).extract_first() location1 = tmp_location1.strip() if tmp_location1 is not None else "" else: location1 = "" if len(self.rule.location2_xpath): tmp_location2 = proxy.xpath(self.rule.location2_xpath).extract_first() location2 = tmp_location2.strip() if tmp_location2 is not None else "" else: location2 = "" if len(self.rule.lifetime_xpath): tmp_lifetime = proxy.xpath(self.rule.lifetime_xpath).extract_first() lifetime = tmp_lifetime.strip() if tmp_lifetime is not None else "" else: lifetime = "" if len(self.rule.lastcheck_xpath): tmp_lastcheck = proxy.xpath(self.rule.lastcheck_xpath).extract_first() lastcheck = tmp_lastcheck.strip() if tmp_lastcheck is not None else "" else: lastcheck = "" if len(self.rule.level_xpath): tmp_level = proxy.xpath(self.rule.level_xpath).extract_first() level = tmp_level.strip() if tmp_level is not None else "" else: level = "" if len(self.rule.type_xpath): tmp_type = proxy.xpath(self.rule.type_xpath).extract_first() type = tmp_type.strip() if tmp_type is not None else "" else: type = "" if len(self.rule.speed_xpath): tmp_speed = proxy.xpath(self.rule.speed_xpath).extract_first() speed = tmp_speed.strip() if tmp_speed is not None else "" else: speed = "" item['ip_port']=(":".join([ip,port])) if len(port) else ip
item['type']=type
item['level']=level
item['location']=(" ".join([location1,location2])) if location2 is not None and len(location2) else location1
item['speed']=speed
item['lifetime']=lifetime
item['lastcheck']=lastcheck
item['rule_id']=self.rule.id
item['source']=response.url
yield item
应用截图：
使用signals机制来插入日志并实时更新spider的运行和数据爬取状态
