爬取一页商品数据
2016-06-18 本文已影响126人
CxyFreedom
目标
- 爬取一页商品数据
- 不要推广商品和转转商品
- 在详情页获取信息(类目 | 标题 | 发帖时间 | 价格 | 成色 | 区域 | 浏览量)
环境
Python 3.5.1
-
Windows7
或者Ubuntu 14.04
结果示例
{
'areas': ['丰台', '-', '西客站'],
'datetime': '2016-03-06',
'price': '500',
'quality': '95成新',
'title': '北京实体店出售苹果IPAD迷你AIR三星平板电脑',
'type': '北京二手平板电脑',
'views': '6103'
}
代码
# -*- coding: utf-8 -*-
from bs4 import BeautifulSoup
import requests, time
from pprint import pprint
base_url = 'http://bj.58.com/pbdn/0/'
def get_all_links(url):
"""获取一页商品的链接,除了推广和转转"""
urls = []
data = requests.get(url)
if data.status_code == 200:
soup = BeautifulSoup(data.text, 'lxml')
for tb in soup.select('div#infolist > table.tbimg'):
for item in tb.find_all("tr", class_=""):
urls.append(item.find("a", class_="t").get('href'))
return urls
def get_views(url):
"""获取单个商品的浏览量"""
info_id = url.split('?')[0].split('/')[-1].strip('x.shtml')
interface = 'http://jst1.58.com/counter?infoid={}'.format(info_id)
headers = {
'Host': 'jst1.58.com',
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) \
Chrome/50.0.2661.102 Safari/537.36',
'Accept': '*/*',
'Accept-Language': 'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3',
'Accept-Encoding': 'gzip, deflate',
'Referer': url,
'Cookie': r'id58=c5/ns1djWc+KP2U1DXI4Ag==; city=bj; 58home=bj; ipcity=sh%7C%u4E0A%u6D77%7C0; \
58tj_uuid=93b726b3-9742-4270-9d11-dcd665bea67b; new_uv=3; myfeet_tooltip=end; \
als=0; bj58_new_uv=3; bj58_id58s="SUg0UjFZMFI0K3RSMDIyMw=="; \
sessionid=37cd7e97-40fe-40ac-b903-d6e85ffd8690; __utma=253535702.1612735289.1466129063.1466129063.1466129063.1; \
__utmc=253535702; __utmz=253535702.1466129063.1.1.utmcsr=baidu|utmccn=(organic)|utmcmd=organic; \
final_history=26364215035854%2C25245379987755%2C26062681492781; bangbigtip2=1; new_session=0; utm_source=; spm=; \
init_refer=; bj58_new_session=0; bj58_init_refer="{}"'.format(url),
'Connection': 'keep-alive',
'Cache-Control': 'max-age=0'
}
resp = requests.get(interface, headers=headers)
if resp.status_code == 200:
return resp.text.split('=')[-1]
else:
return 0
def get_info(url):
"""获取一页中所有的商品信息"""
url_list = get_all_links(url)
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) \
Chrome/50.0.2661.102 Safari/537.36'
}
for link in url_list:
data = requests.get(link, headers=headers)
if data.status_code == 200:
soup = BeautifulSoup(data.text, 'lxml')
areas = soup.select('.c_25d')
info = {
'type': soup.select('div.breadCrumb span:nth-of-type(3) a')[0].get_text(),
'title': soup.select('.mainTitle > h1:nth-of-type(1)')[0].get_text(),
'datetime': soup.select('.time')[0].get_text(),
'price': soup.select('.price')[0].get_text(),
'quality': soup.select('.suUl > li:nth-of-type(2) > div:nth-of-type(2) > span:nth-of-type(1)')[0].get_text(strip=True),
'areas': list(areas[0].stripped_strings) if len(areas) > 0 else [],
'views': get_views(data.url) # 使用重定向后的链接
}
pprint(info)
time.sleep(1)
if __name__ == '__main__':
get_info(base_url)
总结
- 由于链接中存在重定向的链接,所以需要注意要使用重定向后的链接
- 获取浏览量需要通过其他API接口来进行获取
- 注意网站的反爬虫机制,可以加上请求头或者通过代理IP