python 网页爬虫

centos 7下python2.7实现网页爬虫
使用requests、BeautifulSoup

1. 安装

方法一：

下载：http://www.crummy.com/software/BeautifulSoup/bs4/download/4.2/

解压：tar -xzvf beautifulsoup4-4.2.0.tar.gz

安装：进入解压后的目录

python setup.py build
sudo python setup.py install
方法二（快速安装）

(Ubuntu) sudo apt-get install python-bs4
或者
install beautifulsoup4
或着
easy_install beautifulsoup4

2.使用
import requests
from bs4 import BeautifulSoup

3.实例
import requests
from bs4 import BeautifulSoup

class Grab(object):
def tzuqiu_parse_url_to_html(self,url,page):
response=requests.get(url+'?page='+page)
soup=BeautifulSoup(response.content, "html.parser")
body=soup.find_all(class_='new-item')
for div in body:
img=div.find_all(class_='new-img')[0].contents[1].get('src')
title=div.find_all(class_='new-title')[0]
content=div.find_all(class_='limittext-multirow')[0]
tags=div.find_all(class_='new-tag limittext')[0]
print('img:'+str(img))
print('title:'+(title.string).encode('utf-8').strip())
print('content:'+(content.string).encode('utf-8').strip())
print('time:'+(tags.find_all('span')[0].string).encode('utf-8').strip())
print('tag:'+(tags.find_all('span')[1].string).encode('utf-8').strip())

def tzuqiu_next_page(self,url,page):
self.tzuqiu_parse_url_to_html(url,page)

if name=='main':
grab=Grab()
for i in range(2):
grab.tzuqiu_next_page('http://www.tzuqiu.cc/news/list.do',str(i))

4.执行
python grab.py