python的requests库
2018-07-12 本文已影响41人
_Haimei
requests各种请求方式
#版本号
In [46]: requests.__version__
Out[46]: '2.19.1'
In [6]: import requests
In [7]: r = requests.get('https://api.github.com/events')
In [8]: r = requests.post('http://httpbin.org/post', data = {'key':'value'})
In [9]: r = requests.put('http://httpbin.org/put', data = {'key':'value'})
In [10]: r = requests.delete('http://httpbin.org/delete')
In [11]: r = requests.head('http://httpbin.org/get')
In [12]: r = requests.options('http://httpbin.org/get')
url拼接
In [13]: payload = {'key1':'value1','v2':'value2'}
In [14]: r = requests.get("http://httpbin.org/get", params=payload)
In [15]: r.url
Out[15]: 'http://httpbin.org/get?key1=value1&v2=value2'
requests响应内容
In [1]: import requests
In [2]: r = requests.get("http://bj.meituan.com/meishi/")
In [3]: r.status_code
Out[3]: 200
In [4]: r.encoding
Out[4]: 'utf-8'
In [5]: r.headers
Out[5]: {'Server': 'Tengine', 'Date': 'Thu, 12 Jul 2018 09:50:36 GMT', 'Content-Type': 'text/html; charset=utf-8', 'Transfer-Encoding': 'chunked', 'Connection': 'keep-alive', 'X-Powered-By': 'dx-web-meishife-pcweb04.dx.sankuai.com_production', 'Set-Cookie': 'client-id=2247b634-52ff-4a27-a0a9-01b1810e6d9f; path=/; expires=Fri, 13 Jul 2018 09:50:36 GMT; httponly, uuid=9b6cad4c-90fa-4f51-a59c-5f4f37cd4464; path=/; domain=meituan.com; httponly', 'Cache-Control': 'no-cache', 'Vary': 'Accept-Encoding', 'Content-Encoding': 'gzip'}
#请求头大小写不敏感可以不区分大小写调用,例:r.headers.get('server')
In [6]: r.text
#内容太多
In [47]: r = requests.get('https://api.github.com/events')
In [48]: r.json()
#示例返回2条内容
[{'id': '7959832918',
'type': 'PushEvent',
'actor': {'id': 7868421,
'login': 'whitestone8214',
'display_login': 'whitestone8214',
'gravatar_id': '',
'url': 'https://api.github.com/users/whitestone8214',
'avatar_url': 'https://avatars.githubusercontent.com/u/7868421?'},
'repo': {'id': 139670300,
'name': 'whitestone8214/Crawler',
'url': 'https://api.github.com/repos/whitestone8214/Crawler'},
'payload': {'push_id': 2716420119,
'size': 1,
'distinct_size': 1,
'ref': 'refs/heads/master',
'head': 'd178c8282b826cf360e3e4656d0258aedaaace36',
'before': '6c08e2c6f66243c1003df07a80af10e337def37b',
'commits': [{'sha': 'd178c8282b826cf360e3e4656d0258aedaaace36',
'author': {'email': 'whitestone8214@openmailbox.org', 'name': 'Minho Jo'},
'message': "#2: whiteline.js 0.2.1 (과 그에 따른 마이그레이션); 입력란 크기 조절; 다시 그린 툴바 아이콘(일단은 툴바 쪽만); 컨텍스트 메뉴 '텍스트 에디터로 열기'; 기타 등등",
'distinct': True,
'url': 'https://api.github.com/repos/whitestone8214/Crawler/commits/d178c8282b826cf360e3e4656d0258aedaaace36'}]},
'public': True,
'created_at': '2018-07-13T03:27:17Z'},
{'id': '7959832917',
'type': 'WatchEvent',
'actor': {'id': 5679265,
'login': 'LFH104',
'display_login': 'LFH104',
'gravatar_id': '',
'url': 'https://api.github.com/users/LFH104',
'avatar_url': 'https://avatars.githubusercontent.com/u/5679265?'},
'repo': {'id': 129860458,
'name': 'phobal/ivideo',
'url': 'https://api.github.com/repos/phobal/ivideo'},
'payload': {'action': 'started'},
'public': True,
'created_at': '2018-07-13T03:27:17Z'}]
In [48]: r = requests.get('https://api.github.com/events', stream=True)
ln [50]: r.raw
Out[50]: <urllib3.response.HTTPResponse at 0x7fe0a51a4278>
In [51]: r.raw.read(10)
Out[51]: b'\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\x03'
In [74]: r = requests.get('http://github.com')
In [75]: r.url
Out[75]: 'https://github.com/'
In [76]: r.history
Out[76]: [<Response [301]>]
post请求form表单
#字典形式
In [13]: payload = {'key1':'value1','v2':'value2'}
In [16]: r = requests.post("http://httpbin.org/post", data=payload)
In [17]: r.text
Out[17]: '{"args":{},"data":"","files":{},"form":{"key1":"value1","v2":"value2"},"headers":{"Accept":"*/*","Accept-Encoding":"gzip, deflate","Connection":"close","Content-Length":"21","Content-Type":"application/x-www-form-urlencoded","Host":"httpbin.org","User-Agent":"python-requests/2.18.4"},"json":null,"origin":"47.90.49.89","url":"http://httpbin.org/post"}\n'
#元组形式
In [18]: payload = (('key1', 'value1'), ('key1', 'value2'))
In [19]: r = requests.post("http://httpbin.org/post", data=payload)
In [20]: r.text
Out[20]: '{"args":{},"data":"","files":{},"form":{"key1":["value1","value2"]},"headers":{"Accept":"*/*","Accept-Encoding":"gzip, deflate","Connection":"close","Content-Length":"23","Content-Type":"application/x-www-form-urlencoded","Host":"httpbin.org","User-Agent":"python-requests/2.18.4"},"json":null,"origin":"47.90.49.89","url":"http://httpbin.org/post"}\n'
自定义请求头
In [52]: url = 'https://api.github.com/some/endpoint'
In [53]: headers = {'user-agent': 'my-app/0.0.1'}
In [54]: r = requests.get(url, headers=headers)
栗子:
import requests
import json
from datetime import date,datetime
import time
# 发送群消息
url = "https://api.telegram.org/bot"
headers = {'Content-Type': 'application/json'}
data = {
"chat_id": -12345,
"text": "**hi**",
"reply_markup": {
"inline_keyboard": [[{
"text":"google",
"url":"www.google.com"
}]]
},
"parse_mode":"Markdown"
}
content = requests.post(url,headers=headers,data=json.dumps(data))
data =json.loads(content.text)
print(data)
自定义cookie
In [61]: url = 'http://httpbin.org/cookies'
In [62]: cookies = dict(cookies_are='working')
In [63]: r = requests.get(url, cookies=cookies)
In [64]: r.text
Out[64]: '{"cookies":{"cookies_are":"working"}}\n'
In [65]: jar = requests.cookies.RequestsCookieJar()
In [66]: jar.set('tasty_cookie', 'yum', domain='httpbin.org', path='/cookies')
Out[66]: Cookie(version=0, name='tasty_cookie', value='yum', port=None, port_specified=False, domain='httpbin.org', domain_specified=True, domain_initial_dot=False, path='/cookies', path_specified=True, secure=False, expires=None, discard=True, comment=None, comment_url=None, rest={'HttpOnly': None}, rfc2109=False)
In [67]: jar.set('gross_cookie', 'blech', domain='httpbin.org', path='/elsewhere ')
Out[67]: Cookie(version=0, name='gross_cookie', value='blech', port=None, port_specified=False, domain='httpbin.org', domain_specified=True, domain_initial_dot=False, path='/elsewhere', path_specified=True, secure=False, expires=None, discard=True, comment=None, comment_url=None, rest={'HttpOnly': None}, rfc2109=False)
In [68]: url = 'http://httpbin.org/cookies'
In [69]: r = requests.get(url, cookies=jar)
In [70]: r.text
Out[70]: '{"cookies":{"tasty_cookie":"yum"}}\n'
In [71]: url = 'http://httpbin.org/elsewhere'
In [72]: r = requests.get(url, cookies=jar)
In [73]: r.text
Out[73]: '<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">\n<title>404 Not Found</title>\n<h1>Not Found</h1>\n<p>The requested URL was not found on the server. If you entered the URL manually please check your spelling and try again.</p>\n'
requests爬虫小栗子
import requests
import json
import redis
from datetime import date,datetime
import time
r = redis.StrictRedis('localhost',6379,decode_responses=True,db = 3)
class JsonExtendEncoder(json.JSONEncoder):
"""
This class provide an extension to json serialization for datetime/date.
"""
def default(self, obj):
"""
provide a interface for datetime/date
"""
if isinstance(obj, datetime):
return obj.strftime('%Y-%m-%d %H:%M:%S')
elif isinstance(obj, date):
return obj.strftime('%Y-%m-%d')
else:
return json.JSONEncoder.default(self, obj)
i = 0
while True:
p = r.pipeline()
url = "https://api.schail.com/v3/ticker/summary?type=0&sort=1&offset=0&limit=100&top=0"
time.sleep(2)
content = requests.get(url)
data =json.loads(content.text)
data_dict = {}
data = data.get('data')
data_dict['data'] = data
current_time = datetime.now()
data_dict['current_time'] = current_time
# print(data_dict)
exchange_list = []
for tid_dict in data.get('summaryList'):
pair_url = "https://api.schail.com/v1/ticker/exchange?tickerId=%s&offset=0&limit=50"%tid_dict['tickerId']
pair_content = requests.get(pair_url)
pair_data = json.loads(pair_content.text)
exchange_dict = {}
exchange_dict['data'] = pair_data['data']
current_time = datetime.now()
exchange_dict['current_time'] = current_time
exchange_list.append(exchange_dict)
data_dict['exchange_list'] = exchange_list
print(data_dict)
i += 1
p.execute()
r.zadd("tokenclubdata", i,json.dumps(data_dict,ensure_ascii=False,cls=JsonExtendEncoder))
# https://api.schail.com/v1/ticker/exchange?tickerId=bitcoin&offset=0&limit=50
# https://api.schail.com/v1/ticker/exchange?tickerId=ethereum&offset=0&limit=50
