Python爬虫作业 | 爬取拉勾职位信息

2017-04-26 本文已影响215人 JaeGwen

主要爬了以下几个字段：

岗位名称
公司名称
要求的工作经验
薪资
工作地点。

感谢罗攀攀和向右奔跑以及朋友wiliiwin的指导

#coding:utf-8

import json
import requests


class LagoupositionSpider():
    name = "LagouPosition"
    totalPageCount = 1
    curpage = 1
    city = '杭州'
    myurl = 'https://www.lagou.com/jobs/positionAjax.json?px=default&city={}&needAddtionalResult=false'.format(city)

    header = {
         'Cookie': 'JSESSIONID=2FC28971BBE032152E26B3EDC53E5856; user_trace_token=20170426193638-4a8fe6d996f3492492303a4d78b079b4; PRE_UTM=; PRE_HOST=; PRE_SITE=; PRE_LAND=https%3A%2F%2Fwww.lagou.com%2F; LGUID=20170426193639-9caaa97f-2a74-11e7-8138-525400f775ce; index_location_city=%E6%9D%AD%E5%B7%9E; TG-TRACK-CODE=index_search; _gat=1; Hm_lvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1493206604; Hm_lpvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1493206703; _ga=GA1.2.736962990.1493206603; LGSID=20170426193639-9caaa76a-2a74-11e7-8138-525400f775ce; LGRID=20170426193818-d7ae7795-2a74-11e7-b3b0-5254005c3644; SEARCH_ID=88bbbc48ca4448218bea2cd41926b5c6'，'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.221 Safari/537.36 SE 2.X MetaSr 1.0'
    }

    kds = ['Python工程师', '大数据', '云计算', 'docker', '中间件', 'Node.js', '数据挖掘', \
           '自然语言处理', '搜索算法', '精准推荐', '全栈工程师', '图像处理','机器学习', '语音识别']
    kd = kds[0]

    def start_requests(self,cur):
        post_data = {'first': 'false', 'pn': str(cur), 'kd': self.kd}
        html = requests.post(myurl, data=post_data, headers=self.header)
        html_text = html.text
        return html_text

    def get_result(self):
        result = self.start_requests(self.curpage)
        jdict = json.loads(result)
        jcontent = jdict['content']
        jposresult = jcontent['positionResult']
        jresult = jposresult['result']
        self.totalPageCount = jposresult['totalCount'] / 15 + 1
        for each in jresult:
            positionName = each['positionName'].encode('gbk')
            companyFullName = each['companyFullName'].encode('gbk')
            workYear = each['workYear'].encode('gbk')
            salary = each['salary'].encode('gbk')
            district = self.city.decode('utf-8').encode('gbk')
            with open('F:\python.csv', 'ab+') as f:
                f.write('{},{},{},{},{},{}'.format(positionName, district, companyFullName, workYear, salary, '\n'))
        if self.curpage <=  self.totalPageCount:
            self.curpage += 1
            self.start_requests(self.curpage)
            self.get_result()

if __name__ == '__main__':
    lagouspider = LagoupositionSpider()
    lagouspider.get_result()