Python爬虫学习16-Scrapy模拟登陆知乎

Scrapy登录知乎要解决两个问题
1、session的传递，保证处理登录是同一个状态。
2、首个登录页面的改变，由直接爬取的页面变为登录页面，再去爬取页面。

上代码

# -*- coding: utf-8 -*-
import scrapy
import re
import json


class ZhihuSpider(scrapy.Spider):
    name = "zhihu"
    allowed_domains = ["www.zhihu.com"]
    start_urls = ['http://www.zhihu.com/']
    headers = {
        "Accept": 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
        "Host": 'www.zhihu.com',
        "User-Agent": 'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36',
    }

    def parse(self, response):
        text = response.text
        with open("index_page.html", 'wb') as f:
            f.write(response.text.encode('utf-8'))
            f.close()

    def start_requests(self):
        return [scrapy.Request('https://www.zhihu.com/#signin', headers=self.headers, callback=self.login)]

    def login(self, response):
        response_text = response.text
        match_obj = re.match('.*name="_xsrf" value="(.*)"/>', response_text, re.S)
        xsrf = ''
        if match_obj:
            xsrf = match_obj.group(1)
        if xsrf:
            post_data = {
                "_xsrf": xsrf,
                "email": '',
                "password": ''
            }
            import time
            t = str(int(time.time()*1000))
            captcha_url = "https://www.zhihu.com/captcha.gif?r={}&type=login".format(t)
            yield scrapy.Request(captcha_url, headers=self.headers, meta={'post_data': post_data}, callback=self.login_after_capthcha)

    def login_after_capthcha(self, response):

        with open('captpcha.jpg', 'wb') as f:
            f.write(response.body)
            f.close()
        from PIL import Image
        try:
            im = Image.open('captpcha.jpg')
            im.show()
            im.close()
        except:
            pass
        captcha = input("输入验证码")
        post_data = response.meta.get("post_data", {})
        post_url = "https://www.zhihu.com/login/email"
        post_data['captcha'] = captcha
        return [scrapy.FormRequest(
            url=post_url,
            formdata=post_data,
            headers=self.headers,
            callback=self.check_login
        )]

    def check_login(self, response):
        # 验证服务器返回数据判断是否成功
        text_json = json.loads(response.text)
        if 'msg' in text_json and text_json["msg"] == "登录成功":
            for url in self.start_urls:  # 从继承的Spider类中拿的内容，恢复到正确执行。
                yield scrapy.Request(url, dont_filter=True, headers=self.headers)

首先对scrapy.Spider类中的start_requests(self)进行重载，改变首先要处理的页面为登录页面。
得到登录页面后，获得xsrf，并下载验证码，通过scrapy.FormRequest构造登录数据，通过check_login回调函数判断登录是否成功。在代码的最后一行转回正常的登录流程。