Scrapy 自定义扩展

自定义扩展时，利用信号在指定位置注册指定操作。

新建custom_extensions.py

from scrapy import signals
class MyExtend:

    def __init__(self,crawler):
        self.crawler = crawler
        # 在指定信号上注册操作
        crawler.signals.connect(self.start, signals.engine_started)
        crawler.signals.connect(self.close, signals.spider_closed)

    @classmethod
    def from_crawler(cls, crawler):
        return cls(crawler)

    def start(self):
        print('signals.engine_started.start')

    def close(self):
        print('signals.spider_closed.close')

最后需要在settings.py里的修改EXTENSIONS：

EXTENSIONS = {
   'scrapy_learn.custom_extensions.MyExtend': 300,
}

可以用的信号

engine_started = object()  # 引擎启动时
engine_stopped = object()  # 引擎停止时
spider_opened = object()   # 爬虫启动时
spider_idle = object()     # 爬虫闲置时
spider_closed = object()   # 爬虫停止时
spider_error = object()    # 爬虫错误时
request_scheduled = object() # 调度器调度时
request_dropped = object() # 调取器丢弃时
response_received = object() # 得到response时
response_downloaded = object() # response下载时
item_scraped = object()    # yield item 时
item_dropped = object()    # drop item 时

有了这些信号，就可以在指定时刻自定义某些操作。

配置文件（settings.py）详解

# 1. 爬虫名称，不是spider,name里的名称，而是整个爬虫项目的名称，
# 很多网站都会有自己的爬虫(百度，谷歌等都有)。
BOT_NAME = 'scrapy_learn'

# 2. 爬虫应用路径
SPIDER_MODULES = ['scrapy_learn.spiders']
NEWSPIDER_MODULE = 'scrapy_learn.spiders'

# 3. 客户端 user-agent请求头，常伪造成浏览器
USER_AGENT = 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.167 Safari/537.36'

# 4. 是否遵循爬虫规则，正经的要遵循，但我们搞爬虫都不正经
ROBOTSTXT_OBEY = False

# 5. 并发请求数，默认16
CONCURRENT_REQUESTS = 32

# 6. 延迟下载秒数，默认0
DOWNLOAD_DELAY = 3

# 7. 单域名访问并发数，并且延迟下次秒数也应用在每个域名，比CONCURRENT_REQUESTS更加细致的并发
CONCURRENT_REQUESTS_PER_DOMAIN = 16

# 单IP访问并发数，如果有值则忽略：CONCURRENT_REQUESTS_PER_DOMAIN，
# 并且延迟下次秒数也应用在每个IP
CONCURRENT_REQUESTS_PER_IP = 16

# 8. 是否支持cookie，cookiejar进行操作cookie，默认支持
COOKIES_ENABLED = True
# 是否是调试模式，调试模式下每次得到cookie都会打印
COOKIES_DEBUG = True

# 9. Telnet用于查看当前爬虫的信息(爬了多少，还剩多少等)，操作爬虫(暂停等)等...，
# cmd中:telnet 127.0.0.1 6023(6023是专门给爬虫用的端口)
# telnet 命令
# est() 检查引擎状态
# engine.pass 暂停引擎， 还有很多命令，在网上可搜
TELNETCONSOLE_ENABLED = True

# 10. 默认请求头
#DEFAULT_REQUEST_HEADERS = {
#   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
#   'Accept-Language': 'en',
#}

# 中间件，需要详细讲，另写
# Enable or disable spider middlewares
# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
#    'scrapy_learn.middlewares.ScrapyLearnSpiderMiddleware': 543,
#}
# Enable or disable downloader middlewares
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
#DOWNLOADER_MIDDLEWARES = {
#    'scrapy_learn.middlewares.ScrapyLearnDownloaderMiddleware': 543,
#}

# 11. 定义pipeline处理请求
# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
   'scrapy_learn.pipelines.ScrapyLearnPipeline': 300,
}

# 12. 自定义扩展，基于信号进行调用
# See https://doc.scrapy.org/en/latest/topics/extensions.html
EXTENSIONS = {
   # 'scrapy.extensions.telnet.TelnetConsole': None,
}

# 自动限速算法(智能请求)
# See https://doc.scrapy.org/en/latest/topics/autothrottle.html
AUTOTHROTTLE_ENABLED = True
# 第一次下载延迟几秒
AUTOTHROTTLE_START_DELAY = 5
# 最大延迟
AUTOTHROTTLE_MAX_DELAY = 60
# 波动范围，不用管
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
#AUTOTHROTTLE_DEBUG = False

# 做缓存的，以后说
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'

# 13. 爬虫允许的最大深度，可以通过meta查看当前深度；0表示无深度
DEPTH_LIMIT = 4

# DEPTH_PRIORITY只能设置为0或1，
# 0深度优先，一下找到底，然后再找其他的
# 1广度优先，一层一层找
# 他们内部的原理就是根据response.meta里的depth(层数)来找。
# DEPTH_PRIORITY = 0