python requests pymongo

2017-10-13  本文已影响0人  站长_郭生
try:
    import json as simplejson
except:
    import simplejson

from pymongo import MongoClient
import requests


def GetMongoDb():
    mongo = MongoClient('127.0.0.1', 27017)
    db = mongo.filminfo
    return db


url = "http://www.baidu.com"
# res = http.HttpReq("GET", url, {}, {}, 35)
headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.104 Safari/537.36 Core/1.53.3427.400 QQBrowser/9.6.12201.400"}
res = requests.get(url, headers=headers)
# print(res)
if res.status_code == 200:
    db = GetMongoDb()
    resp = simplejson.loads(res.content, strict=False)
    for val in resp["urlset"]:
          db.test.update({"id": val["data"]["id"]}, {"$set": val["data"]}, upsert = True)
上一篇下一篇

猜你喜欢

热点阅读