B站爬虫代码

import requests
from bs4 import BeautifulSoup
from selenium import webdriver
import re
import pymongo
import time
import datetime as dt
import random
import pandas as pd
import warnings
warnings.filterwarnings('ignore')
print('导入成功!','\n','-'*20)

def get_urls(n):
    #【分页网页url采集】函数
    #n：页数参数
    lst = []
    for i in range(n):
        lst.append('https://space.bilibili.com/82366241/video?tid=0&page=%i&keyword=&order=pubdate' %(i+1))
    return lst

def get_data(ui,d_h,d_c1,d_c2,table):
    #【视频网页url采集】函数
    #u：起始网址
    #d_h：user-agent信息
    #d_c1,d_c2：cookies信息(两个网址)
    r1 = requests.get(url = ui,headers = d_h,cookies = d_c1)
    soup1 = BeautifulSoup(r1.text,'lxml')
    title = soup1.h1['title']
    time = re.search(r'(\d*-\d*-\d* \d*:\d*:\d*)',soup1.find('div',class_="video-data").text).group(1)
    aid = re.search(r'av(\d*).*',ui).group(1)
    up = soup1.find('div',class_="name").a.text
    r2 = requests.get(url = 'https://api.bilibili.com/x/web-interface/archive/stat?aid=%s' %aid,
                      headers = d_h,cookies = d_c2)
    soup2 = BeautifulSoup(r2.text,'lxml')
    name = soup1.find('div',class_="info open").text
    info = re.search(r'"view.*(\d*),"now_rank"',soup2.text).group().split(',')[:-2]
    date = str(dt.date.today())
    dic = {}
    dic['标题'] = title
    dic['介绍'] = name
    dic['aid'] = aid
    dic['上线时间'] = time
    dic['up主'] = up
    dic['采集时间'] = date
    for i in info:
        dic[i.split(':')[0].replace('"','')] = i.split(':')[1]
    table.insert_one(dic)
    return len(dic)

if __name__ == "__main__": 
    urllst = get_urls(14)
    lilst = []       
    brower = webdriver.Chrome()
    for u in urllst:  
        brower.get(u)
        time.sleep(1)
        ul = brower.find_element_by_class_name('list-list')
        lis = ul.find_elements_by_tag_name('a')
        for li in lis:
            lilst.append(li.get_attribute('href'))
    lilst = list(set(lilst))
            
    dic_h = {'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/70.0.3521.2 Safari/537.36'}
    dic_c1 = {}
    cookies1 = '''xxx'''#替换cookies
    for i in cookies1.split('; '):
        dic_c1[i.split('=')[0]] = i.split('=')[1]    
    dic_c2 = {}
    cookies2 = '''xxx'''#替换cookies
    for i in cookies2.split('; '):
        dic_c2[i.split('=')[0]] = i.split('=')[1]
        
    myclient = pymongo.MongoClient("mongodb://localhost:27017/")
    db = myclient['看电影了没']
    datatable = db['视频信息'+ str(dt.date.today())]
    starttime = time.time()
    errorlst = []
    datalst = []
    for ui in lilst:
        try:
            datalst.append(get_data(ui,dic_h,dic_c1,dic_c2,datatable))
            print('数据采集成功，总共采集%i条数据' % len(datalst))
        except:
            errorlst.append(ui)
            print('数据采集失败，数据网址为：',ui)
        time.sleep(random.randint(1,3))