python requests pymongo
2017-10-13 本文已影响0人
站长_郭生
try:
import json as simplejson
except:
import simplejson
from pymongo import MongoClient
import requests
def GetMongoDb():
mongo = MongoClient('127.0.0.1', 27017)
db = mongo.filminfo
return db
url = "http://www.baidu.com"
# res = http.HttpReq("GET", url, {}, {}, 35)
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.104 Safari/537.36 Core/1.53.3427.400 QQBrowser/9.6.12201.400"}
res = requests.get(url, headers=headers)
# print(res)
if res.status_code == 200:
db = GetMongoDb()
resp = simplejson.loads(res.content, strict=False)
for val in resp["urlset"]:
db.test.update({"id": val["data"]["id"]}, {"$set": val["data"]}, upsert = True)