python获取播测数据通过grafana展示（一）

前期准备，有一台已经安装好influxdb数据库和grafana的主机
influxdb的安装也很简单，这里就不重点强调了，需要了解的同学可以在这个网站上去了解https://www.linuxdaxue.com/how-to-install-influxdb.html，讲的很不错
我的grafana和influxdb选择的都是最新稳定版的。
influxdb数据库是时序性数据库，用法类似与mysql这样的关系型数据库，但是没有数据表列的alter，update这样的动作。并且输入在数据表中的第一条数据就定义了表中这一列数据的数据类型
从一台主机向N台主机发起请求，获取建立时间，下载速度，丢包，网络延时等数据存入influxdb数据库并且通过grafana展示
N台主机上都 有同一个文件，通过http://ip/file_path方式可以访问
通过python的pycurl模块和发起ICMP的ping包获取想要获取的数据，直接上代码:

#!/usr/bin/env python2
#-*- coding: UTF-8 -*-

import pycurl
import sys
import os
import re
import subprocess
from influxdb import InfluxDBClient

http_code = ''
http_conn_time = ''
http_start_tran = ''
http_total_time = ''
http_speed_download = ''
http_content_type = ''
reip = re.compile(r'(\b(?:[0-9]{1,3}\.){3}[0-9]{1,3}\b)')

class Test():
    def __init__(self):
        self.contents = ''
    def body_callback(self,buf):
        self.contents = self.contents + buf
 
 
def get_http_infor(input_url):
    global http_ip
    global http_code
    global http_resolu_time
    global http_re_size
    global http_size
    global http_conn_time
    global http_pre_tran
    global http_start_tran
    global http_total_time
    global http_speed_download
    #global http_content_type
    t = Test()
    #gzip_test = file("gzip_test.txt", 'w')
    c = pycurl.Curl()
    c.setopt(pycurl.WRITEFUNCTION,t.body_callback)
    c.setopt(pycurl.FOLLOWLOCATION, 2)
    c.setopt(pycurl.ENCODING, 'gzip')
    c.setopt(pycurl.TIMEOUT, 5) 
    c.setopt(pycurl.CONNECTTIMEOUT, 5) 
    c.setopt(pycurl.HTTPHEADER,['Host:appdl.hicloud.com','Range:bytes=0-100000'])
    c.setopt(pycurl.URL,input_url)
    try:
        c.perform()
    except Exception as e:
        print "connecion error:"+str(e)
        #c.close()
        #sys.exit()
    
    http_ip = str(re.findall(reip,input_url.rstrip())[0]) #提取url中ip
    http_code = c.getinfo(pycurl.HTTP_CODE)  #响应状态码
    http_re_size = c.getinfo(pycurl.REQUEST_SIZE)  #请求大小
    http_size = c.getinfo(pycurl.SIZE_DOWNLOAD)  #下载的数据大小
    http_conn_time =  c.getinfo(pycurl.CONNECT_TIME)  #连接服务器时间
    http_pre_tran =  c.getinfo(pycurl.PRETRANSFER_TIME)  #连接上服务器后开始传输时的时间
    http_start_tran =  c.getinfo(pycurl.STARTTRANSFER_TIME)  #接收到第一个字节的时间
    http_total_time = c.getinfo(pycurl.TOTAL_TIME)  #请求花费总时间
    http_speed_download = c.getinfo(pycurl.SPEED_DOWNLOAD)  #下载速度
    #http_content_type = c.getinfo(pycurl.CONTENT_TYPE)  #请求内容类型

def get_ping_data():
    global pack_lost1
    global avg_sec1
    p = subprocess.Popen("ping -c 2 -i 0.5 %s" %http_ip, stdin = subprocess.PIPE, stdout = subprocess.PIPE, stderr = subprocess.PIPE, shell = True)
    out = p.stdout.read()
    regLost = r'(\d+)%'
    regSec = r'min/avg/max/mdev = ([0-9.]+)/([0-9.]+)/([0-9.]+)/([0-9.]+)\sms'
    lost = re.search(regLost,out)
    pack_lost1 = float(lost.group().split("%")[0]) 
    try:
        sec = re.search(regSec,out)
        avg_sec1 = float(sec.groups()[1])
    except AttributeError:
        avg_sec = 1000.00
    #print 'http_ip http_code  http_conn_time http_start_tran http_total_time http_speed_download pack_lost avg_sec'
    #print "{0},{1},{2},{3},{4},{5},{6},{7}".format(http_ip,http_code,http_conn_time,http_start_tran,http_total_time,http_speed_download,pack_lost,avg_sec)

def deposit_database():
    conn_db = InfluxDBClient('数据库ip','8086','数据库用户','数据库用户密码','存储数据数据库')  #连接上grafana数据库
    json_body = [{"measurement": "http_info","tags":{"http_ip":http_ip},"fields":{"http_code":http_code,"http_conn_time":http_conn_time,"http_start_tran":http_start_tran,"http_total_time":http_total_time,"http_speed_download":http_speed_download,"pack_lost1":pack_lost1,"avg_sec1":avg_sec1}}]
    try:
        conn_db.write_points(json_body)
    except Exception as e:
        print e

if __name__ == '__main__':
    with open('url.list','r+') as url_file:
        urls = url_file.readlines()
    for input_url in urls:
        if input_url:
            input_url = input_url.strip('\n')
            get_http_infor(input_url)
            get_ping_data()
            deposit_database()

url.list这个目录里面放置的是需要请求的主机文件
数据库需要定义两张表，因为最终要在grafana中实现下拉列表选择节点来展示不同节点的相应信息。"http_info"是其中的一张数据表。另外一张表我起名为"http_ip_address".
"http_info"中将"http_ip"定义为tags，获取的其他数据定义为fileds。
下节 重点说下关于"http_ip_address"数据表中tags，fileds的定义。这点很关键
将这个脚本放入定时任务，5分钟或者10 分钟执行一次脚本，将脚本存储数据库