三十九. Scrapy实战 - 简书推荐信息

爬取网址：https://www.jianshu.com/recommendations/users
爬取内容：作者URL、最近更新文章；作者ID、“关注、粉丝、文章、字数、收获喜欢”
爬取方式：Scrapy框架
储存方式：mongodb
主要目的：实现跨页面爬虫

image.png image.png

页面规律比较简单，通过F12查看XHR，即可获取翻页信息。

1.items.py文件

import scrapy

class AuthorItem(scrapy.Item):
    # define the fields for your item here:
    author_url = scrapy.Field()
    new_article = scrapy.Field()
    author_name = scrapy.Field()
    focus = scrapy.Field()
    fans = scrapy.Field()
    article_num = scrapy.Field()
    write_num = scrapy.Field()
    like_num = scrapy.Field()

2.authorspider.py文件

from scrapy.spiders import CrawlSpider
from scrapy.selector import Selector
from scrapy.http import Request
from author.items import AuthorItem

class author(CrawlSpider):
    name = "author"
    start_urls = ["https://www.jianshu.com/recommendations/users?page=1"]

    def parse(self, response):
        base_url = "https://www.jianshu.com/u/"
        selector = Selector(response)
        infos = selector.xpath('//div[@class="col-xs-8"]')
        for info in infos:
            author_url = base_url + info.xpath('div/a/@href').extract()[0].strip('/users/')
            new_article = info.xpath('div/div[@class="recent-update"]')[0].xpath('string(.)').extract()[0].strip().replace(' ','').replace('\n', '')
            yield Request(author_url,meta={'author_url':author_url,'new_article':new_article},callback=self.parse_item)
            #通过meta进行爬虫信息参数的传递，并通过Request请求author_url,回调parse_item()函数。

        urls = ['https://www.jianshu.com/recommendations/users?page={}'.format(i) for i in range(2,30)]
        for url in urls:
            yield Request(url,callback=self.parse)  #回调parse()函数

    def parse_item(self,response): #定义parse_item()爬取详细页面的信息
        item = AuthorItem()
        item['author_url'] = response.meta['author_url']   #取出传递的参数meta
        item['new_article'] = response.meta['new_article']

        try:
            selector = Selector(response)
            focus = selector.xpath('//div[@class="info"]/ul/li[1]/div/a/p/text()').extract()[0]
            fans = selector.xpath('//div[@class="info"]/ul/li[2]/div/a/p/text()').extract()[0]
            article_num = selector.xpath('//div[@class="info"]/ul/li[3]/div/a/p/text()').extract()[0]
            write_num = selector.xpath('//div[@class="info"]/ul/li[4]/div/p/text()').extract()[0]
            like_num = selector.xpath('//div[@class="info"]/ul/li[5]/div/p/text()').extract()[0]

            item['focus'] = focus
            item['fans'] = fans
            item['article_num'] = article_num
            item['write_num'] = write_num
            item['like_num'] = like_num
            yield item

        except IndexError:
            pass

3.pipelines.py文件

import pymongo

class AuthorPipeline(object):
    
    def __init__(self):
        client = pymongo.MongoClient('localhost', 27017)
        mydb = client['mydb']
        author = mydb['author']
        self.post = author  ##连接数据库
    
    
    def process_item(self, item, spider):
        info = dict(item)
        self.post.insert(info)  ##插入数据库
        return item

4.settings.py文件

USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3294.6 Safari/537.36'     #请求头
DOWNLOAD_DELAY = 1                 #睡眠时间1秒
ITEM_PIPELINES = {'author.pipelines.AuthorPipeline': 300}

5.在author文件夹下运行CMD命令：scrapy crawl author，可以看到MongoDB的结果。

image.png