BeautifulSoup4

BeautifulSoup是一个HTML/XML的解析器，主要用于解析和提取HTML/XML数据。BeautifulSoup用来解析HTML比较简单，API非常人性化，支持CSS选择器，Python标准库中的HTML解析器，也支持lxml的XML解析器。

安装

pip install beautifulsoup4

使用

from bs4 import BeautifulSoup
html = '''
'''
soup = BeautifulSoup(html, 'lxml')
print(soup.title)

案例

from bs4 import BeautifulSoup
import random
import requests
import os
import time

__author__ = 'wangff'

class StewPic:
    def __init__(self, start_url, page):
        self.start_url = start_url
        self.page = page
        self.i = 1
        self.user_agent_list = [
            'Mozilla/5.0 (Windows NT 6.1; rv,2.0.1) Gecko/20100101 Firefox/4.0.1',
            'Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11',
            'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',
            'Mozilla/5.0 (Windows NT 6.1; rv,2.0.1) Gecko/20100101 Firefox/4.0.1',
            'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.106 Safari/537.36',
            'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Maxthon/4.9.2.1000 Chrome/39.0.2146.0 Safari/537.36',
            'Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11',
            'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3',
            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3',
            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/532.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/532.3',
            'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5',
            'Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11',
        ]
        self.headers = {
            'User-Agent': random.choice(self.user_agent_list),
        }

    def get_url_list(self):
        url_list = [self.start_url.format(x) for x in range(1, self.page)]
        return url_list

    def get_item(self):
        url_list = self.get_url_list()

        for url in url_list:
            print(url)
            response = requests.get(url=url, headers=self.headers)
            # print(response.text)
            soup = BeautifulSoup(response.text, 'lxml')
            people_list = soup.select('div.ptw li div.c')
            for people in people_list:
                print(people.a['href'])
                self.into_personal(people.a['href'])
                # break
                time.sleep(random.randint(0, 3))
            # break

    def into_personal(self, url):
        _response = requests.get(url=url, headers=self.headers)
        if _response.text:
            soup = BeautifulSoup(_response.text, 'lxml')
            pic_url_list = soup.select('div.bm_c ul.ptw li a img')
            os.chdir(r'F:\airline stewardess')
            for pic_url in pic_url_list:
                print(pic_url['src'])
                self.download_pic(pic_url['src'])
                time.sleep(random.randint(0, 2))
        else:
            print('照片有误！')

    def download_pic(self, url):
        response = requests.get(url=url, headers=self.headers)
        p = response.content

        try:
            with open(str(self.i) + '.jpg', 'ab') as f:
                f.write(p)
                f.close()
        except Exception as e:
            print(e)

        self.i += 1


if __name__ == '__main__':
    stewardss_pic = StewPic('http://www.kongjie.com/home.php?mod=space&do=album&view=all&page={}', 100)
    stewardss_pic.get_item()