Downland Middleware中写cookie池

class CookieMiddleware(object):

    def __init__(self, cookies_pool_url):
        self.logger = logging.getLogger(__name__)
        self.cookies_pool_url = cookies_pool_url

def _get_random_cookies(self):
        try:
            response = requests.get(self.cookies_pool_url)
            if response.status_code == 200:
                return json.loads(response.text)
         except ConnectionError:
            return None

    @classmethod
    def from_crawler(cls, crawler):

        # 实例化的时候把settings带过去给init函数
        return cls(
             cookies_pool_url = crawler.settings.get('COOKIES_POOL_URL')
        )

    def process_request(self, request, spider):
        cookies = self._get_random_cookies()
        if  cookies:
            request.cookies = cookies
            self.logger.debug('Using Cookies' + json.dumps(cookies))
        else:
            self.logger.debug('No Valid Cookies')