Python_scrapy—突破反爬虫的限制

一：自由切换user-agent

• 在Scrapy框架下的middleware.py文件中添加如下代码:

class RandomUserAgentMiddlware(object):
    #随机更换user-agent
    def __init__(self,crawler):
        super(RandomUserAgentMiddlware,self).__init__()
        self.ua=UserAgent()
        self.ua_type=crawler.settings.get("RANDOM_UA_TYPE","random")
        #调用你所定义的UA类型，并随机给出索要类型的user-agent

    def from_crawler(cls,crawler):
        return cls(crawler)

    def process_request(self,request,spider):
        def get_ua():#动态语言中函数里可以定义函数，例如js
            return getattr(self.ua,self.ua_type)

        random_agent=get_ua()#断点调试获取UA结果
        request.header.setdefault('User-Agent',get_ua())

• 并在表头导入UserAgent，并创建UA实例，以便调用。

from fake_useragent import UserAgent
ua = UserAgent()#生成实例，直接调用其中的user-agent

• 在settings.py文件中定义你所需要的UA类型，当前为随机：

RANDOM_UA_TYPE="random"#定义你想要的UA类型，

• 将middlewares中的RandomUserAgentMiddlware类配置到settings当中：

DOWNLOADER_MIDDLEWARES = {
    'ArticleSpider.middlewares.RandomUserAgentMiddlware': 543,
    'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware':None
}

• 运行后报错

NameError: Module 'ArticleSpider.middlewares' doesn't define any object named 'MyCustomDownloaderMiddleware'

原因：没有将setting文件中将DOWNLOADER_MIDDLEWARES中的默认middleware类名改为你所定义的类名。更改后如下：

DOWNLOADER_MIDDLEWARES = {
    'ArticleSpider.middlewares.RandomUserAgentMiddlware': 543,
    #'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware':None
}

二：动态设置ip代理

import requests
from scrapy.selector import Selector
import MySQLdb
#链接数据库
conn=MySQLdb.connect(host="localhost",user="root",password="123456",db="article_spider",charset="utf8",)
cursor=conn.cursor()

def crawl_ips():
    #爬取西刺的免费IP代理
    headers={"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:56.0) Gecko/20100101 Firefox/56.0"}
    for i in range(1568):
        re=requests.get("http://www.xicidaili.com/nn/{0}".format(i),headers=headers)

        selector=Selector(text=re.text)
        all_trs= selector.css("#ip_list tr")

        ip_list=[]
        for tr in all_trs[1:]:
            speed_str=tr.css(".bar::attr(title)").extract()[0]
            if speed_str:
                speed=float(speed_str.split("秒")[0])
            all_texts=tr.css("td::text").extract()

            ip=all_texts[0]
            port=all_texts[1]
            proxy_type=all_texts[5]

            ip_list.append((ip,port,speed,proxy_type))

        for ip_info in ip_list:
            cursor.execute(
                "insert proxy_ip(ip,port,speed,proxy_type) VALUES('{0}','{1}',{2},'{3}')".format(#当值为字符串时，传值要为单引号。
                    ip_info[0],ip_info[1],ip_info[2],ip_info[3]
                )
            )
            conn.commit()
class GetIP(object):
    def delete_ip(self, ip):
        #从数据库中删除无效的ip
        delete_sql = """
            delete from proxy_ip where ip='{0}'
        """.format(ip)
        cursor.execute(delete_sql)
        conn.commit()
        return True

    def judge_ip(self,ip,port):
       #判断ip是否可用
       http_url="https://www.baidu.com"
       proxy_url="https://{0}:{1}".format(ip,port)
       try:
           proxy_dict={
              "http":proxy_url
            }
           response = requests.get(http_url, proxies=proxy_dict)
       except Exception as e:
           print ("invalid ip and port")
           self.delete_ip(ip)
           return False
       else:
           code = response.status_code
           if code >= 200 and code < 300:
               print ("effective ip")
               return True
           else:
               print  ("invalid ip and port")
               self.delete_ip(ip)
               return False

    def get_random_ip(self):
        #从数据库中随机获取一个可用的ip
        random_sql="""
        SELECT ip,port FROM proxy_ip
        ORDER BY RAND()
        LIMIT 1
        """
        result=cursor.execute(random_sql)
        for ip_info in cursor.fetchall():
            ip=ip_info[0]
            port=ip_info[1]

            judge_re = self.judge_ip(ip, port)
        if judge_re:
            return "http://{0}:{1}".format(ip, port)
        else:
            return self.get_random_ip()


#print(crawl_ips())
if __name__ == "__main__":
    get_ip = GetIP()
    get_ip.get_random_ip()