之所以叫“笔记”，并非原创，只是总结。原创作者是慕课老师--七月，很喜欢他的课买了他很多课（打个广告）。仅供学习交流。

爬虫分析

熊猫tv.png

----需求：对一个直播网站的某个分类模块人气进行排名。

----分析：对图中红色区域“video-nickname”、“video-number”进行抓取和排序。

----代码实现：这里是对正则表达式的及python语法的一些考核。

---

具体代码实现

第一阶段：发起网络请求请求到所需分析的html页面

# 引入网络框架
from urllib import request


class Spider:
    url = 'https://www.panda.tv/cate/lol?pdt=1.24.s1.3.4jhlkpnlco3'
#第一阶段：发起网络请求请求到所需分析的html页面
    def __fetch_content(self):

        r = request.urlopen(Spider.url)
        html = r.read()
        html = str(html, encoding='utf-8')
        print(html)

    def run(self):
        self.__fetch_content()#第一阶段


spider = Spider()
spider.run()

第二阶段：分析这个html页面，提取到我们所要关注的标签

# 引入网络框架\正则框架
from urllib import request
import re


class Spider:
    url = 'https://www.panda.tv/cate/lol?pdt=1.24.s1.3.4jhlkpnlco3'
    root_pattern = '<div class="video-info">([\s\S]*?)</div>'
    #第二阶段
    # \s 匹配空白字符，\S 匹配非空白字符 []字符集
    # *匹配*号前面的字符0次或跟多次 ？非贪婪 匹配到<d就结束了

    def __fetch_content(self):

        r = request.urlopen(Spider.url)
        html = r.read()
        html = str(html, encoding='utf-8')
        return html
#第二阶段：分析这个html页面，提取到我们所要关注的标签
    def __analysis(self, html):
        root_html = re.findall(Spider.root_pattern, html)
        print(root_html[0])

    def run(self):
        html = self.__fetch_content()
        self.__analysis(html)


spider = Spider()
spider.run()

获得到结果：

获得所需.png

第三阶段：获取“video-nickname”、“video-number”

# 引入网络框架\正则框架
from urllib import request
import re


class Spider:
    url = 'https://www.panda.tv/cate/lol?pdt=1.24.s1.3.4jhlkpnlco3'
    root_pattern = '<div class="video-info">([\s\S]*?)</div>'
    name_pattern = '</i>([\s\S]*?)</span>'
    # number_pattern = '<span class="video-number">15.1万</span>'
    number_pattern = '<span class="video-number">([\s\S]*?)</span>'
    # \s 匹配空白字符，\S 匹配非空白字符 []字符集
    # *匹配*号前面的字符0次或跟多次 ？非贪婪 匹配到<d就结束了

    def __fetch_content(self):

        r = request.urlopen(Spider.url)
        html = r.read()
        html = str(html, encoding='utf-8')
        return html

    def __analysis(self, html):
        root_html = re.findall(Spider.root_pattern, html)
        anchors = []
        for html in root_html:
            name = re.findall(Spider.name_pattern, html)
            number = re.findall(Spider.number_pattern, html)
            anchor = {"name" :name, "number":number }
            anchors.append(anchor)
        print(anchors[0])

    def run(self):
        html = self.__fetch_content()
        self.__analysis(html)


spider = Spider()
spider.run()

结果
{'name': ['\n 守卫者 ', '\n '], 'number': ['1.9万']}

测试结果.png

继续优化：将空格删除

    # 精炼我们的函数
    def __refine(self, anchors):

        express = lambda anchor: {
            'name': anchor['name'][0].strip(),
            'number': anchor['number'][0]
        }
        return map(express, anchors)

    def run(self):
        html = self.__fetch_content()
        anchors = self.__analysis(html)
        anchors = list(self.__refine(anchors))
        print(anchors)

结果

测试结果.png

第四阶段 对抓取的数据进行排序

# 引入网络框架\正则框架
from urllib import request
import re


class Spider:
    url = 'https://www.panda.tv/cate/lol?pdt=1.24.s1.3.4jhlkpnlco3'
    root_pattern = '<div class="video-info">([\s\S]*?)</div>'
    name_pattern = '</i>([\s\S]*?)</span>'
    # number_pattern = '<span class="video-number">15.1万</span>'
    number_pattern = '<span class="video-number">([\s\S]*?)</span>'
    # \s 匹配空白字符，\S 匹配非空白字符 []字符集
    # *匹配*号前面的字符0次或跟多次 ？非贪婪 匹配到<d就结束了

    def __fetch_content(self):

        r = request.urlopen(Spider.url)
        html = r.read()
        html = str(html, encoding='utf-8')
        return html

    def __analysis(self, html):
        root_html = re.findall(Spider.root_pattern, html)
        anchors = []
        for html in root_html:
            name = re.findall(Spider.name_pattern, html)
            number = re.findall(Spider.number_pattern, html)
            anchor = {"name": name, "number": number}
            anchors.append(anchor)
        return anchors

    # 精炼我们的函数
    def __refine(self, anchors):

        express = lambda anchor: {
            'name': anchor['name'][0].strip(),
            'number': anchor['number'][0]
        }
        return map(express, anchors)

    # 根据number字段值进行排序
    def __sort_anchors(self, anchors):
        anchors = sorted(anchors, key=self.__sort_seed, reverse=True)
        return anchors

    def __sort_seed(self, anchor):
        r = re.findall("\d*", anchor['number'])
        number = float(r[0])
        if '万'in anchor['number']:
            number *= 10000

        return number

    def __show(self, anchors):
        for anchor in anchors:
            print(anchor['name']+'-----'+anchor['number'])

    def run(self):
        html = self.__fetch_content()
        anchors = self.__analysis(html)
        anchors = list(self.__refine(anchors))
        anchors = self.__sort_anchors(anchors)
        self.__show(anchors)


spider = Spider()
spider.run()

测试结果

测试结果.png

最后显示小小的优化：

 def __show(self, anchors):

        # for anchor in anchors:
          # print(anchor['name']+'-----'+anchor['number'])

        for rank in range(0, len(anchors)):
            print('rank'+str(rank + 1) + "----" +
                  anchors[rank]['name'] + "----" +
                  anchors[rank]['number'])

测试结果：

测试结果.png

撒花。。。。花