Python实战 - 第3节:真实世界中的网页解析

2016-10-30  本文已影响0人  辉叔不太萌

笔记

作业

代码:

from bs4 import BeautifulSoup
import requests
import time

urls = ['http://bj.xiaozhu.com/search-duanzufang-p{}-0/'.format(str(i)) for i in range(1, 14)]

def parse_gender(gender_class):

    if gender_class == 'member_ico1':
        return '女'
    elif gender_class == 'member_ico':
        return '男'
    else:
        return '未知'

def parse_datail_page(url):

    web_data = requests.get(url)
    soap = BeautifulSoup(web_data.text, 'lxml')

    titles = soap.select('body > div.wrap.clearfix.con_bg > div.con_l > div.pho_info > h4 > em')
    addrs = soap.select('body > div.wrap.clearfix.con_bg > div.con_l > div.pho_info > p > span')
    rants = soap.select('#pricePart > div.day_l > span')
    pics = soap.select('#curBigImage')
    owner_pics = soap.select('#floatRightBox > div.js_box.clearfix > div.member_pic > a > img')
    gender_class = soap.select('#floatRightBox > div.js_box.clearfix > div.member_pic > div')[0]['class'][0]
    owner_names = soap.select('#floatRightBox > div.js_box.clearfix > div.w_240 > h6 > a')

    data = {
        'title': titles[0].get_text(),
        'addr': addrs[0].get_text().strip(),
        'rant': rants[0].get_text(),
        'pic': pics[0].get('src'),
        'owner_pic': owner_pics[0].get('src'),
        'owner_name': owner_names[0].get_text(),
        'gender': parse_gender(gender_class),
    }

    print(data)
    time.sleep(2)


def parse_list_page(url):

    web_data = requests.get(url)
    soap = BeautifulSoup(web_data.text, 'lxml')

    detail_urls = soap.select('#page_list > ul > li > a')
    for detail_url in detail_urls:
        parse_datail_page(detail_url.get('href'))


for url in urls:
    parse_list_page(url)

运行结果(部分):

{'addr': '北京市昌平区天通苑地铁5号线附近', 'rant': '385', 'gender': '女', 'owner_pic': 'http://image.xiaozhustatic1.com/21/7,0,79,9253,260,260,8afea7d2.jpg', 'title': '地铁5号13号线直达天安门、雍和宫、鸟巢', 'pic': 'http://image.xiaozhustatic1.com/00,800,533/7,0,53,12783,1800,1200,5b45ee3f.jpg', 'owner_name': '故事还长呢'}
{'addr': '北京市东城区东二环广渠门北里55号', 'rant': '500', 'gender': '女', 'owner_pic': 'http://image.xiaozhustatic1.com/21/4,0,30,8173,260,260,21e70d52.jpg', 'title': '两居72平市中心天安门北京站天坛国贸临铁7号', 'pic': 'http://image.xiaozhustatic1.com/00,800,533/7,0,47,8376,1800,1200,5f37670c.jpg', 'owner_name': '美丽的北京你的家'}
{'addr': '北京市昌平区新龙城小区(昌平路380号院)', 'rant': '450', 'gender': '女', 'owner_pic': 'http://image.xiaozhustatic1.com/21/1,0,71,6035,350,350,a514acd6.jpg', 'title': '【艾窝窝整租】北京大三居•家的味道 13号线', 'pic': 'http://image.xiaozhustatic1.com/00,800,533/1,0,4,6127,825,550,9db282eb.jpg', 'owner_name': '艾窝窝'}
{'addr': '北京市朝阳区6号地铁常营天街豪华公寓', 'rant': '197', 'gender': '未知', 'owner_pic': 'http://image.xiaozhustatic1.com/21/3,0,95,5809,359,359,9c0c2802.jpg', 'title': '地铁6号线常营长楹天街豪华公寓', 'pic': 'http://image.xiaozhustatic1.com/00,800,533/4,0,26,6729,1800,1200,768006fe.jpg', 'owner_name': '爱猪我家萍姐'}
{'addr': '北京市海淀区五道口 清华东路27号院', 'rant': '338', 'gender': '男', 'owner_pic': 'http://image.xiaozhustatic1.com/21/8,0,12,23,414,414,7ba0eebb.jpg', 'title': '五道口六道口地铁鸟巢清华农大林大北语高校附近', 'pic': 'http://image.xiaozhustatic1.com/00,800,533/8,0,35,877,1800,1200,9c0fa827.jpg', 'owner_name': 'xo0v0ox'}
{'addr': '北京市丰台区万科红公寓', 'rant': '358', 'gender': '男', 'owner_pic': 'http://image.xiaozhustatic1.com/21/6,0,40,6670,260,260,377c1f3c.jpg', 'title': '国贸CBD商圈天坛北京南站5,10号线地铁\n', 'pic': 'http://image.xiaozhustatic1.com/00,800,533/3,0,85,5997,1800,1200,56cd3eea.jpg', 'owner_name': 'simoe陈'}
{'addr': '北京市朝阳区柏林爱乐3期5号楼', 'rant': '318', 'gender': '未知', 'owner_pic': 'http://image.xiaozhustatic1.com/21/7,0,91,11996,329,329,89bf5aaf.jpg', 'title': '直达国贸CBD六号线黄渠站整租大两居', 'pic': 'http://image.xiaozhustatic1.com/00,800,533/7,0,64,13088,1800,1200,d40b0666.jpg', 'owner_name': '冯赢Emma'}
{'addr': '北京市海淀区学院路逸成东苑19号楼', 'rant': '328', 'gender': '未知', 'owner_pic': 'http://image.xiaozhustatic1.com/21/3,0,13,2417,329,329,957e12fb.jpg', 'title': '六道口、林业大学温馨舒适一居室超值整租', 'pic': 'http://image.xiaozhustatic1.com/00,800,533/7,0,14,13043,1800,1200,064c14b3.jpg', 'owner_name': '四海为一家'}
{'addr': '北京市朝阳区后现代城', 'rant': '1598', 'gender': '女', 'owner_pic': 'http://image.xiaozhustatic1.com/21/6,0,76,2881,363,363,ca2c4b1a.jpg', 'title': '国贸双井西大望路7号地铁超豪五星级艺术复式', 'pic': 'http://image.xiaozhustatic1.com/00,800,533/8,0,36,760,1800,1200,8d8606eb.jpg', 'owner_name': 'celinna'}
{'addr': '北京市海淀区积水潭小西天志强南园9号楼', 'rant': '328', 'gender': '女', 'owner_pic': 'http://image.xiaozhustatic1.com/21/8,0,61,1483,375,376,8c9f4abd.jpg', 'title': '二环积水潭地铁德胜门后海北师大西直门一居整租', 'pic': 'http://image.xiaozhustatic1.com/00,800,533/6,0,64,3967,1800,1200,fb028f16.jpg', 'owner_name': '天天Tinny'}
{'addr': '北京市海淀区西四环中路41号院', 'rant': '398', 'gender': '女', 'owner_pic': 'http://image.xiaozhustatic1.com/21/8,0,91,453,378,378,34b2042e.jpg', 'title': '一号线五棵松地铁北欧风两居室', 'pic': 'http://image.xiaozhustatic1.com/00,800,533/8,0,3,1208,1800,1200,d3f6df0a.jpg', 'owner_name': 'Cathy77dudu'}
{'addr': '北京市西城区德胜门大街265号', 'rant': '259', 'gender': '未知', 'owner_pic': 'http://image.xiaozhustatic1.com/21/5,0,21,2825,363,364,dffb9e72.jpg', 'title': '特价德内复式,步行后海南锣,护国寺小吃街东头', 'pic': 'http://image.xiaozhustatic1.com/00,800,533/8,0,76,1386,1800,1200,09d607c8.jpg', 'owner_name': '隋青洋'}
{'addr': '北京市西城区南新华街18号安平里', 'rant': '388', 'gender': '未知', 'owner_pic': 'http://image.xiaozhustatic1.com/21/7,0,10,14183,281,281,2c5b73f8.jpg', 'title': '天安门、故宫,和平门地铁首次出租南北通透两居', 'pic': 'http://image.xiaozhustatic1.com/00,800,533/8,0,11,921,1800,1200,b63ea5d3.jpg', 'owner_name': '大美的家'}
{'addr': '北京市东城区贡院西街9号贡院9号', 'rant': '1080', 'gender': '男', 'owner_pic': 'http://image.xiaozhustatic1.com/21/7,0,76,9589,281,281,c6cee3c1.jpg', 'title': '特惠 商旅首选 精英型高端公寓 性价比超赞', 'pic': 'http://image.xiaozhustatic1.com/00,800,533/8,0,2,1371,1800,1200,7bb52d6b.jpg', 'owner_name': 'AZheng_123'}
{'addr': '北京市大兴区清源西路', 'rant': '138', 'gender': '男', 'owner_pic': 'http://image.xiaozhustatic1.com/21/3,0,15,339,375,375,742e0af4.jpg', 'title': '地铁四号线清源路站 (艺术气息温馨大主卧)', 'pic': 'http://image.xiaozhustatic1.com/00,800,533/6,0,98,1041,2666,2000,fda9c1a7.jpg', 'owner_name': '给您一个温暖的家'}
{'addr': '北京市海淀区学院路逸城东苑15号楼(林业大学北 ...', 'rant': '398', 'gender': '未知', 'owner_pic': 'http://image.xiaozhustatic1.com/21/3,0,13,2417,329,329,957e12fb.jpg', 'title': '【两居特价】新东方 五道口 林大农大两居整租', 'pic': 'http://image.xiaozhustatic1.com/00,800,533/6,0,0,4582,1800,1200,f4f3efb7.jpg', 'owner_name': '四海为一家'}
{'addr': '北京市丰台区蒲黄榆GOGO新世代小区', 'rant': '298', 'gender': '男', 'owner_pic': 'http://image.xiaozhustatic1.com/21/2,0,12,3663,376,376,949053ce.jpg', 'title': '蒲黄榆地铁上整租简约时尚洁净温馨^-^', 'pic': 'http://image.xiaozhustatic1.com/00,800,533/4,0,86,8672,1800,1200,b0e2d100.jpg', 'owner_name': '少冰'}
{'addr': '北京市丰台区宋家庄23号院', 'rant': '358', 'gender': '女', 'owner_pic': 'http://image.xiaozhustatic1.com/21/7,0,58,11817,260,260,aa24d08d.jpg', 'title': '北京站南站天安门南锣鼓巷后海5号10号线地铁', 'pic': 'http://image.xiaozhustatic1.com/00,800,533/6,0,63,5608,1800,1200,d358591f.jpg', 'owner_name': '暖心大姐'}
{'addr': '北京市朝阳区胜古中路甲一号院', 'rant': '188', 'gender': '未知', 'owner_pic': 'http://image.xiaozhustatic1.com/21/6,0,63,671,289,289,e1fa1351.jpg', 'title': '安贞近地铁奢华次卧鸟巢安贞医院北服', 'pic': 'http://image.xiaozhustatic1.com/00,800,533/7,0,53,11959,1800,1200,3af333ce.jpg', 'owner_name': '王博洋1989'}
{'addr': '北京市朝阳区太阳宫半岛国际公寓', 'rant': '718', 'gender': '女', 'owner_pic': 'http://image.xiaozhustatic1.com/21/7,0,84,10831,329,329,1867ecd6.jpg', 'title': '10号线地铁太阳宫高端公寓进口床垫轻奢大三房', 'pic': 'http://image.xiaozhustatic1.com/00,800,533/7,0,23,12784,1800,1200,1ba52195.jpg', 'owner_name': '快快乐乐的苹果'}
{'addr': '北京市丰台区宋家庄23号院万科红狮小区', 'rant': '400', 'gender': '女', 'owner_pic': 'http://image.xiaozhustatic1.com/21/7,0,58,11817,260,260,aa24d08d.jpg', 'title': '前门5、10号线宋家庄天坛大两居阳光暖心房', 'pic': 'http://image.xiaozhustatic1.com/00,800,533/5,0,3,1182,1800,1200,b7078939.jpg', 'owner_name': '暖心大姐'}
{'addr': '北京市丰台区万科红11号楼(宋家庄地铁c口10米即...', 'rant': '358', 'gender': '女', 'owner_pic': 'http://image.xiaozhustatic1.com/21/7,0,58,11817,260,260,aa24d08d.jpg', 'title': '天坛前门地铁5,10号线大型交通枢纽暖心公寓', 'pic': 'http://image.xiaozhustatic1.com/00,800,533/3,0,44,3940,1800,1200,2ff6a063.jpg', 'owner_name': '暖心大姐'}
{'addr': '北京市西城区高粱桥斜街长河湾', 'rant': '398', 'gender': '女', 'owner_pic': 'http://image.xiaozhustatic1.com/21/1,0,8,5386,333,333,764cfdb4.jpg', 'title': '西直门 长河湾 韩式田园温馨2居', 'pic': 'http://image.xiaozhustatic1.com/00,800,533/1,0,3,5482,825,550,f874984d.jpg', 'owner_name': '豹子楠'}
{'addr': '北京市东城区广渠门北里55号', 'rant': '500', 'gender': '女', 'owner_pic': 'http://image.xiaozhustatic1.com/21/4,0,30,8173,260,260,21e70d52.jpg', 'title': '市中心两居天安门故宫北京站天坛国贸临铁7号', 'pic': 'http://image.xiaozhustatic1.com/00,800,533/7,0,28,8368,1800,1200,916ca9e8.jpg', 'owner_name': '美丽的北京你的家'}
{'addr': '北京市海淀区地铁8号线西小口D口文龙家园二里(地...', 'rant': '168', 'gender': '男', 'owner_pic': 'http://image.xiaozhustatic1.com/21/5,0,76,1757,341,341,4f1bb296.jpg', 'title': '【主卧特价】8号线地铁口,直达鸟巢南锣鼓巷', 'pic': 'http://image.xiaozhustatic1.com/00,800,533/5,0,24,799,1800,1200,295e36e4.jpg', 'owner_name': '花花he浩子的家'}
{'addr': '北京市朝阳区百子湾路16号后现代城', 'rant': '456', 'gender': '女', 'owner_pic': 'http://image.xiaozhustatic1.com/21/8,0,48,47,334,334,d6d989f6.jpg', 'title': 'CBD国贸 近地铁温馨两居整租', 'pic': 'http://image.xiaozhustatic1.com/00,800,533/8,0,32,1048,1800,1200,10b2517f.jpg', 'owner_name': '知足小家'}
{'addr': '北京市东城区鼓楼大街安德路55号院', 'rant': '398', 'gender': '女', 'owner_pic': 'http://image.xiaozhustatic1.com/21/8,0,61,1483,375,376,8c9f4abd.jpg', 'title': '【两居特价】二环鼓楼大街后海鸟巢直达全新两居', 'pic': 'http://image.xiaozhustatic1.com/00,800,533/8,0,79,1519,1800,1200,0af68cad.jpg', 'owner_name': '天天Tinny'}
{'addr': '北京市通州区后南仓38号楼富力金禧花园', 'rant': '399', 'gender': '女', 'owner_pic': 'http://image.xiaozhustatic1.com/21/3,0,87,1257,260,260,148b29fa.jpg', 'title': '通州万达2居新婚房配纯乳胶床消毒柜安睡又卫生', 'pic': 'http://image.xiaozhustatic1.com/00,800,533/8,0,0,758,1800,1200,62a7cda1.jpg', 'owner_name': '寒海洋'}
{'addr': '北京市朝阳区锦芳路旭辉奥都7号楼', 'rant': '288', 'gender': '女', 'owner_pic': 'http://image.xiaozhustatic1.com/21/1,0,92,4005,334,334,fb27fbdf.jpg', 'title': '9折特价!地铁立水桥南 落地窗精致一居', 'pic': 'http://image.xiaozhustatic1.com/00,800,533/4,0,21,6808,1800,1200,65a24abd.jpg', 'owner_name': '白羊女'}
{'addr': '北京市西城区前马厂胡同', 'rant': '1280', 'gender': '男', 'owner_pic': 'http://image.xiaozhustatic1.com/21/7,0,75,13275,260,260,d9d46a16.jpg', 'title': '峰会-鼓楼风韵(后海、恭王府、南锣鼓巷)', 'pic': 'http://image.xiaozhustatic1.com/00,800,533/8,0,78,1155,1800,1200,0bf99345.jpg', 'owner_name': '峰会g'}
{'addr': '北京市海淀区学院路逸成东苑22号楼', 'rant': '398', 'gender': '未知', 'owner_pic': 'http://image.xiaozhustatic1.com/21/3,0,13,2417,329,329,957e12fb.jpg', 'title': '[特价]学院路 五道口 清华可住5人', 'pic': 'http://image.xiaozhustatic1.com/00,800,533/7,0,69,11545,1800,1200,96423ff4.jpg', 'owner_name': '四海为一家'}
{'addr': '北京市朝阳区百子湾', 'rant': '198', 'gender': '男', 'owner_pic': 'http://image.xiaozhustatic1.com/21/7,0,23,14129,260,260,51c4e4b8.jpg', 'title': '北欧风格的文艺宅', 'pic': 'http://image.xiaozhustatic1.com/00,800,533/8,0,68,732,1800,1200,d404f617.jpg', 'owner_name': '阳光的诱惑'}
{'addr': '北京市朝阳区劲松地铁附近', 'rant': '368', 'gender': '未知', 'owner_pic': 'http://image.xiaozhustatic1.com/21/7,0,85,14196,369,369,4951119d.jpg', 'title': '【特价】近国贸~劲松地铁步行五分钟温馨两居', 'pic': 'http://image.xiaozhustatic1.com/00,800,533/6,0,21,131,1800,1200,408fab6d.jpg', 'owner_name': '一只呆呆i'}
{'addr': '北京市朝阳区小关北里', 'rant': '218', 'gender': '未知', 'owner_pic': 'http://image.xiaozhustatic1.com/21/8,0,31,839,260,260,b0f1d4f3.jpg', 'title': '惠新西街南口阳光大主卧', 'pic': 'http://image.xiaozhustatic1.com/00,800,533/5,0,47,2122,1800,1200,8830e613.jpg', 'owner_name': '最美好的时光遇到你'}
{'addr': '北京市丰台区科怡路', 'rant': '769', 'gender': '女', 'owner_pic': 'http://image.xiaozhustatic1.com/21/2,0,50,1103,260,260,af7b8829.jpg', 'title': '简约大气,浪漫舒适,紧邻9号线科怡路', 'pic': 'http://image.xiaozhustatic1.com/00,800,533/2,0,17,728,824,550,f70b7f18.jpg', 'owner_name': '三毛的家'}
{'addr': '北京市东城区演乐胡同', 'rant': '1499', 'gender': '男', 'owner_pic': 'http://image.xiaozhustatic1.com/21/3,0,12,4011,278,278,178728c4.jpg', 'title': '东单金宝街美术馆超赞四合院', 'pic': 'http://image.xiaozhustatic1.com/00,800,533/8,0,40,1689,3000,2000,1991eaa5.jpg', 'owner_name': '王小山和他的朋友们'}
{'addr': '北京市西城区广安门外大街305号', 'rant': '388', 'gender': '未知', 'owner_pic': 'http://image.xiaozhustatic1.com/21/3,0,71,1932,375,375,cacc9a48.jpg', 'title': '毗邻西站长安街音乐学院金融街儿童医院LOFT', 'pic': 'http://image.xiaozhustatic1.com/00,800,533/3,0,2,2838,1800,1200,ddea66ba.jpg', 'owner_name': '海洋阿姨'}
{'addr': '北京市朝阳区垡头北里15号楼', 'rant': '280', 'gender': '女', 'owner_pic': 'http://image.xiaozhustatic1.com/21/5,0,63,906,260,260,1e5d7816.jpg', 'title': '欢乐谷附近温馨舒适小两居整租', 'pic': 'http://image.xiaozhustatic1.com/00,800,533/5,0,4,1933,1800,1200,3805f74c.jpg', 'owner_name': '安心1983'}
{'addr': '北京市西城区椿树园奥运社区2号楼', 'rant': '438', 'gender': '女', 'owner_pic': 'http://image.xiaozhustatic1.com/21/5,0,49,536,160,160,fd1d9c6c.jpg', 'title': '惠!高星酒店品质,天安门西单,2,4,7号线', 'pic': 'http://image.xiaozhustatic1.com/00,800,533/5,0,28,497,1800,1200,6295fb46.jpg', 'owner_name': '哈利_小狗'}
{'addr': '北京市海淀区知春路52号', 'rant': '228', 'gender': '未知', 'owner_pic': 'http://image.xiaozhustatic1.com/21/1,0,30,4607,467,467,1e0b1a0b.jpg', 'title': '知春路地铁B口希格玛对面独享阳台超值温馨大房', 'pic': 'http://image.xiaozhustatic1.com/00,800,533/4,0,40,10434,1800,1200,ad3e5572.jpg', 'owner_name': '石大双'}
{'addr': '北京市丰台区宋家庄家园3区(宋家庄地铁A口200米...', 'rant': '299', 'gender': '男', 'owner_pic': 'http://image.xiaozhustatic1.com/21/8,0,46,1243,260,260,1b10bd08.jpg', 'title': '北京丰台区宋家庄地铁旁温馨精品公寓大一居', 'pic': 'http://image.xiaozhustatic1.com/00,800,533/7,0,1,8814,1800,1200,fa1f807a.jpg', 'owner_name': 'woaider000'}
{'addr': '北京市东城区鼓楼', 'rant': '196', 'gender': '女', 'owner_pic': 'http://image.xiaozhustatic1.com/21/7,0,97,11132,329,330,90a55b36.jpg', 'title': '【清新】鼓楼后海+胡同独立小院老北京民宿', 'pic': 'http://image.xiaozhustatic1.com/00,800,533/7,0,47,11491,1800,1200,855161e4.jpg', 'owner_name': '心旅民宿'}
{'addr': '北京市昌平区太平庄中二街', 'rant': '178', 'gender': '男', 'owner_pic': 'http://image.xiaozhustatic1.com/21/1,0,71,394,212,212,7f7358df.jpg', 'title': '天通苑复式顶层整租带独立卫生间书房露台', 'pic': 'http://image.xiaozhustatic1.com/00,800,533/1,0,11,3261,825,550,551187ee.jpg', 'owner_name': 'Kelvin陈驰'}
{'addr': '北京市东城区东直门外大街春秀路15号楼(地铁东直...', 'rant': '199', 'gender': '女', 'owner_pic': 'http://image.xiaozhustatic1.com/21/2,0,56,418,375,375,d734cf69.jpg', 'title': '毗邻地铁13/2号东直门簋街工体三里屯大主卧', 'pic': 'http://image.xiaozhustatic1.com/00,800,533/3,0,51,418,825,550,0f6f805c.jpg', 'owner_name': '丁小烦'}
{'addr': '北京市丰台区蒲黄榆GOGO新世代小区', 'rant': '298', 'gender': '男', 'owner_pic': 'http://image.xiaozhustatic1.com/21/2,0,12,3663,376,376,949053ce.jpg', 'title': '音乐学院附中蒲黄榆方庄天坛长安街5-14号线', 'pic': 'http://image.xiaozhustatic1.com/00,800,533/6,0,30,516,1800,1200,3a63b1de.jpg', 'owner_name': '少冰'}
{'addr': '北京市海淀区双榆树南里二区', 'rant': '168', 'gender': '男', 'owner_pic': 'http://image.xiaozhustatic1.com/21/1,0,45,1836,260,260,9573f11f.jpg', 'title': '人大 中关村 4号线 温暖单间', 'pic': 'http://image.xiaozhustatic1.com/00,800,533/1,0,68,2058,825,550,ad67bf83.jpg', 'owner_name': '北京好房东'}
{'addr': '北京市东城区金鱼池东区小区', 'rant': '288', 'gender': '男', 'owner_pic': 'http://image.xiaozhustatic1.com/21/6,0,40,2087,378,378,a7c575ce.jpg', 'title': '天坛前门地处市中心安享一居', 'pic': 'http://image.xiaozhustatic1.com/00,800,533/2,0,5,1092,1800,1200,18140599.jpg', 'owner_name': '太子牛'}
{'addr': '北京市昌平区立汤路', 'rant': '298', 'gender': '女', 'owner_pic': 'http://image.xiaozhustatic1.com/21/8,0,47,1283,414,415,0f56e909.jpg', 'title': '距离地铁一分钟,直达鸟巢水立方浪漫一居', 'pic': 'http://image.xiaozhustatic1.com/00,800,533/2,0,24,3835,1800,1200,b7d49f09.jpg', 'owner_name': '梦想佳de家'}
上一篇下一篇

猜你喜欢

热点阅读