gevent爬取豆瓣电影top250

采用协程来实现快速抓取页面信息
出处：https://github.com/jingsupo/python-spider/blob/master/day07/douban/04douban_gevent.py

# -*- coding:utf-8 -*-

import requests, time, gevent
from lxml import etree
from gevent import monkey
monkey.patch_all()

class DoubanMovie(object):
    def __init__(self):
        self.base_url = "https://movie.douban.com/top250"
        self.headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko"}
        # 电影总数量
        self.count = 0

    def send_request(self, url):
        time.sleep(2)
        try:
            response = requests.get(url, headers=self.headers)
            data = response.content
            self.parse_data(data)
        except Exception as e:
            print e

    def parse_data(self, data):
        html_data = etree.HTML(data)

        # 电影名字
        name_list = html_data.xpath('//*[@id="content"]/div/div[1]/ol/li/div/div[2]/div[1]/a/span[1]/text()')

        for name in name_list:
            print name
            # 每获取一部电影名字电影数量增加1
            self.count += 1

    def run(self):
        start_time = time.time()

        gevent_list = []

        for page in range(0, 225 + 1, 25):
            url = self.base_url + '?start=' + str(page)

            # 创建协程
            gevents = gevent.spawn(self.send_request, url)
            gevent_list.append(gevents)

        # 主线程等待
        gevent.joinall(gevent_list)

        end_time = time.time()

        total_time = end_time - start_time

        print '全部电影%d部' % self.count
        print '总时间%ss' % total_time


if __name__ == '__main__':
    spider = DoubanMovie()
    spider.run()