今日头条爬虫分析-搜索

今天开始爬取今日头条的第二个入口搜索，搜索有四个tab分别为综合，视频，图集，用户。先来分析一下综合的搜索接口
https://www.toutiao.com/search_content/?offset=0&format=json&keyword=%E7%A4%BE%E5%8C%BA%E6%96%B0%E9%9B%B6%E5%94%AE&autoload=true&count=20&cur_tab=1&from=search_tab
只需要传递keyword 搜索关键字即可，无需进行其他处理，直接解析接口返回数据，该接口返回的数据有点杂数据结构不统一，有文章，用户，微头条，问答，搜索推荐，其他关键字搜索推荐，在做数据解析的时候要针对这集中数据结构分别做处理。老样子直接上代码：

    def get_search_article(self, keyword, offset=0):
        keyword = urllib.request.quote(keyword)
        req_url = "https://www.toutiao.com/search_content/?offset={}&format=json&keyword={}&autoload=true&count=20&cur_tab=1&from=search_tab".format(offset,keyword)
        headers = {
            'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36',
            'Connection': 'keep-alive',
            'authority': 'www.toutiao.com',
            'referer': "https://www.toutiao.com/search/?keyword={}".format(keyword),
            'method': 'GET',
            'path': "/search_content/?offset={}&format=json&keyword={}&autoload=true&count=20&cur_tab=1&from=search_tab".format(offset,keyword),
            'scheme': 'https'
        }
        self.s.headers.update(headers)
        req = self.s.get(req_url, proxies=get_proxy_ip())
        time.sleep(random.random() * 2 + 3)
        data = json.loads(req.text)
        items = data['data']
        if data['has_more'] == 1:
            self.page = self.page + 1
            offset = 20 * self.page
            self.parse_data(items)
            time.sleep(2)
            self.get_search_article(keyword, offset)
        else:
            self.parse_data(items)
            toutiaodb.save(self.search_item_list)

    def parse_data(self, items):
        for item in items:
            try:
                type = item['cell_type']
            except:
                type = 0

            if type == 37:    #微头条
                pass
            elif type == 50:
                pass
            elif type == 66:
                pass
            elif type == 26:   #内容推荐
                pass
            elif type == 20:   #搜索推荐
                pass
            elif type == 38:  #用户
                pass
            else:
                titem = toutiaoitem()
                titem.user_id = item['user_id']
                try:
                    titem.source = item['source']
                except:
                    titem.source = item['name']
                titem.title = item['title']
                titem.source_url = item['article_url']
                titem.media_url = item['media_url']
                titem.item_id = item['item_id']
                titem.abstract = item['abstract']
                titem.comments_count = item['comments_count']
                titem.behot_time = item['behot_time']
                titem.image_url = item['image_url']
                titem.image_list = item['image_list']
                titem.tag = item['tag']
                if 'play_effective_count' in item:
                    titem.article_genre = 'vedio'
                    titem.read_count = item['play_effective_count']
                else:
                    titem.article_genre = 'article'
                self.search_item_list.append(titem)

源码在：https://github.com/fywb251/toutiao-crawler