爬虫常用代码

xpath抓取指定贴吧所有的图片

# 思路
# 1.获取贴吧主页URL，下一页，找URL规律
# 2.获取 1 页中每个帖子的URL
# 3.对每个帖子 URL 发请求，获取帖子中 图片URL
# 4.以此对图片URL发请求，以 wb 方式保存到本地

# 帖子链接链表 = parseHtml.xpath('..')
# for 1个帖子链接 in 帖子链接列表:
#   html = 对每个帖子发请求得到响应
#   for 1个图片链接 in 图片链接列表
#       with open('ll.jpg', 'wb') as f:
#           f.write()

# //div[@class="t_con cleafix"]/div/div/div/a/@href
# //div[@class="d_post_content j_d_post_content  clearfix"]/img[@class="BDE_Image"]/@src

# kw=%E6%A0%A1%E8%8A%B1&pn=100 

from lxml import etree
import requests
import urllib.parse

class BaiduImgSpider:
    def __init__(self):
        self.baseurl = 'http://tieba.baidu.com'
        self.headers = {"User-Agent":"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0"}
        self.mainurl = 'http://tieba.baidu.com/f?'
    
    # 获取所有帖子的 URL 列表
    def getPageUrl(self, params):
        # 发请求
        res = requests.get(self.mainurl,params=params,headers=self.headers)
        res.encoding = 'utf-8'
        html = res.text
        
        # 提取页面中的href
        parseHtml = etree.HTML(html)
        tList = parseHtml.xpath('//div[@class="t_con cleafix"]/div/div/div/a/@href')
        
        for t in tList:
            tLink = self.baseurl + t
            self.getImgUrl(tLink)        
    
    # 获取 一个 帖子中所有图片的URL列表
    def getImgUrl(self, tLink):
        # 获取一个贴子的响应内容
        res = requests.get(tLink,headers=self.headers)
        res.encoding = 'utf-8'
        html = res.text
        
        # 提取图片的 src
        parseHtml = etree.HTML(html)
        imgList = parseHtml.xpath('//div[@class="video_src_wrapper"]/embed/@data-video | //div[@class="d_post_content j_d_post_content  clearfix"]/img[@class="BDE_Image"]/@src')
        
        # 依次遍历图片链接调用写入函数
        for img in imgList:
            self.writeImage(img)
    
    # 把图片保存到本地
    def writeImage(self, img):
        # 对图片链接发起请求，获取res.content
        res = requests.get(img, headers=self.headers)
        res.encoding = 'utf-8'
        # 二进制文件
        html = res.content
        # 写入本地文件
        filename = img[-12:]
        with open(filename, 'wb') as f:
            f.write(html)
            print("%s下载成功" % filename)
    
    # 主函数
    def workOn(self):
        name = input('输入要爬取的贴吧名称：')
        begin = int(input("起始页："))
        end = int(input("终止页："))
        for n in range(begin, end+1):
            pn = (n-1) * 50
            params = {
                    "kw": name,
                    "pn": pn
                }
            self.getPageUrl(params)
#            params = urllib.parse.urlencode(params)
#            # 拼接URL
#            url = self.baseurl + "/f?" + params
#            self.getPageUrl(url)
    
if __name__ == "__main__":
    spider = BaiduImgSpider()
    spider.workOn()

xpath爬取糗事百科的文字

import requests
from lxml import etree
import pymongo

class QiushSpider:
    def __init__(self):
        self.url = "https://www.qiushibaike.com/text/"
        self.headers = {"User-Agent":"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0"}
        # 连接对象
        self.conn = pymongo.MongoClient("10.8.20.56",27017)
        # 库对象
        self.db = self.conn["Qiushidb"]
        # 集合对象
        self.myset = self.db["zhuanye"]
    
    # 获取页面
    def getPage(self):
        res = requests.get(self.url, headers=self.headers)
        res.encoding = 'utf-8'
        html = res.text
        
        self.parsePage(html)
    
    # 解析并写入数据库
    def parsePage(self, html):
        # 创建解析对象，也是 节点对象
        parseHtml = etree.HTML(html)
        # 利用解析对象调用xpath,每个段子的对象
        baseList = parseHtml.xpath('//div[contains(@id,"qiushi_tag_")]')
        # 遍历每个段子对象，1个1个提取
        for base in baseList:
            # base: <element at ...> 节点对象
            # 用户昵称
            username = base.xpath('./div/a/h2')
            if username:
                username = username[0].text
            else:
                username = '匿名用户'
                
            # 段子内容
            content = base.xpath('./a/div[@class="content"]/span/text()')
            content = "".join(content).strip()
            # 好笑的数量
            laugh = base.xpath('.//i[@class="number"]')[0].text
            # 评论的数量
            comments = base.xpath('.//i[@class="number"]')[1].text
            
            # 存入 mongo 数据库，先定义成字典
            d = {
                "username": username.strip(),    
                "content": content.strip(),
                "laugh": laugh.strip(),
                "comments": comments.strip()
                }
            self.myset.insert_one(d)        
    
    # 主函数
    def workOn(self):
        print('正在爬取中...')
        self.getPage()
        print('爬取结束，存入Qiushidb库')
    
if __name__ == '__main__':
    spider = QiushSpider()
    spider.workOn()
    

requests模块案例 : 豆瓣电影排行榜数据抓取

抓取目标 : 豆瓣电影 - 排行榜 - 剧情电影名称 评分

import requests
import json
import pymysql

class DoubanSpider:
    def __init__(self):
        self.url = "https://movie.douban.com/j/chart/top_list?"
        self.headers = {"User-Agent":"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0"}
        
        self.db = pymysql.connect("10.8.20.56","kk","123456","spiderdb",charset="utf8")
        self.cursor = self.db.cursor()
    
    # 获取页面
    def getPage(self, params):
        res = requests.get(self.url,params=params,headers=self.headers)
        res.encoding = "utf-8"
        html = res.text
#        print(html)
        # html 为[{1个电影信息},{},{}]
        self.parsePage(html)
    
    # 解析页面
    def parsePage(self, html):
        ins = "insert into douban(name,score) values(%s,%s)"        
        rList = json.loads(html)
        for rDict in rList:
            name = rDict["title"]
            score = rDict["score"]
            L = [name.strip(),float(score.strip())]
            print(L)
#            print(name, score)
            self.cursor.execute(ins,L)
            self.db.commit()
            print("插入成功")
        self.cursor.close()
        self.db.close()
                
    # 主函数
    def workOn(self):
        number = input("请输入数量:")
        params = {
                "type": 11,
                "interval_id": "100:90",
                "action": "",
                "start": "0",
                "limit": number
                }
        self.getPage(params)
    
if __name__ == '__main__':
    spider = DoubanSpider()
    spider.workOn()

selenium+chromedriver京东商品信息爬取

from selenium import webdriver
import time 

# 创建浏览器对象
driver = webdriver.Chrome()
# 访问京东首页
driver.get('https://www.jd.com/')
# 找到搜索框按钮,接收终端输入,发送到搜索框
text = driver.find_element_by_class_name('text')
key = input("请输入要搜索的内容:")
text.send_keys(key)
# 点击 搜索按钮
button = driver.find_element_by_class_name('button')
button.click()
time.sleep(2)

while True:
    # 执行脚本,进度条拉到最底部
    driver.execute_script(
     'window.scrollTo(0,document.body.\
    scrollHeight)')
    time.sleep(3)
    # 提取数据,分析数据
    rList = driver.find_elements_by_xpath(
               '//div[@id="J_goodsList"]//li')
    # rList : ['商品1节点对象','商品2节点对象']
    for r in rList:
        contentList = r.text.split('\n')
        price = contentList[0]
        name = contentList[1]
        commit = contentList[2]
        market = contentList[3]
        
        d = {
                "价格":price,
                "名称":name,
                "评论":commit,
                "商家":market,
                }
        with open("jd.json","a",encoding="utf-8") as f:
            f.write(str(d) + '\n')
        
    # 点击下一页,-1表示没找到
    if driver.page_source.find(
            'pn-next disabled') == -1:
        driver.find_element_by_class_name\
                   ('pn-next').click()
        time.sleep(3)
    else:
        print("爬取结束")
        break
          
#下一页能点 ：  pn-next
#下一页不能点： pn-next disabled
# 关闭浏览器
driver.quit()