grafana prometheus 监控

1.grafana安装

sudo yum install https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana-5.3.4-1.x86_64.rpm 

修改配置

vim /etc/grafana/grafana.ini

  [paths]

# Path to where grafana can store temp files, sessions, and the sqlite3 db (if that is used)

data = /var/lib/grafana

# Directory where grafana can store logs

logs = /var/log/grafana

# Directory where grafana will automatically scan and look for plugins

plugins = /var/lib/grafana/plugins

# The ip address to bind to, empty will bind to all interfaces

http_addr =0.0.0.0

# The http port  to use

http_port = 9300

# If you use reverse proxy and sub path specify full url (with sub path)

root_url = http://127.0.0.1:9300

安装插件

grafana-cli plugins install grafana-piechart-panel

grafana-cli plugins install grafana-worldmap-panel

重启配置 

service grafana-server restart

2.prometheus安装

wget  https://github.com/prometheus/prometheus/releases/download/v2.5.0/prometheus-2.5.0.linux-amd64.tar.gz

tar -zxvf prometheus-2.5.0.linux-amd64.tar.gz

配置prometheus.yml

global:

  scrape_interval:    1s

  evaluation_interval: 1s

scrape_configs:

  - job_name: prometheus

    static_configs:

      - targets: ['127.0.0.1:9301']

        labels:

          instance: prometheus

  - job_name: linux

    static_configs:

      - targets: ['127.0.0.1:9100']

        labels:

          instance:  Nelab-3-server

      - targets: ['127.0.0.1:9100']

        labels:

          instance: crontab-server

      - targets: ['127.0.0.1:9100']

        labels:

          instance: socket-server

  - job_name: 'mysql'

    static_configs:

      - targets: ['127.0.0.1:9104']

        labels:

          instance: Nelab-3-db

  - job_name: redis

    static_configs:

      - targets: ['127.0.0.1:9121']

        labels:

          instance: db1

启动

  ./prometheus --config.file=prometheus.yml --web.listen-address=:9301

3.安装 prometheus_exporters

wget https://github.com/prometheus/mysqld_exporter/releases/download/v0.11.0/mysqld_exporter-0.11.0.linux-amd64.tar.gz

wget https://github.com/prometheus/node_exporter/releases/download/v0.17.0-rc.0/node_exporter-0.17.0-rc.0.linux-amd64.tar.gz

wget https://github.com//oliver006/redis_exporter/releases/download/v0.22.0/redis_exporter-v0.22.0.linux-amd64.tar.gz

【mysql】

  vim .my.cnf

  [client]

  host=172.31.26.69

  port=3306

  user=game

  password=CvSMOyYW

  nohup ./mysqld_exporter --config.my-cnf=".my.cnf" &

  【redis】

  nohup ./redis_exporter --redis.addr  redis://127.0.0.1:6379 --redis.password irbouh92 &

  【node_exporter】

  nohup ./node_exporter &

  4.grafana模板

  redis模板

  https://grafana.com/api/dashboards/763/revisions/1/download

  nginx 模板

  https://grafana.com//api/dashboards/2292/revisions/5/download

  mysql MySQL Overview

  https://github.com/percona/grafana-dashboards

  Elasticsearch Nginx Logs

  https://grafana.com//api/dashboards/2292/revisions/5/download

  Node Exporter Server Metrics

  https://grafana.com/api/dashboards/405/revisions/8/download

  5.nginx日志收集

  配置NGINX

          log_format main  '{"@timestamp":"$time_iso8601",'

                        '"@source":"$server_addr",'

                        '"hostname":"$hostname",'

                        '"ip":"$http_x_forwarded_for",'

                        '"client":"$remote_addr",'

                        '"request_method":"$request_method",'

                        '"scheme":"$scheme",'

                        '"domain":"$server_name",'

                        '"referer":"$http_referer",'

                        '"request":"$request_uri",'

                        '"args":"$args",'

                        '"size":$body_bytes_sent,'

                        '"status": $status,'

                        '"responsetime":$request_time,'

                        '"upstreamtime":"$upstream_response_time",'

                        '"upstreamaddr":"$upstream_addr",'

                        '"http_user_agent":"$http_user_agent",'

                        '"https":"$https"'

                        '}';

  安装elasticsearch

  wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.6.11.tar.gz

  配置elasticsearch.yml

http.port: 9200

network.host: 0.0.0.0

启动

 nohup bin/elasticsearch &

  安装logstash

  wget https://artifacts.elastic.co/downloads/logstash/logstash-6.5.0.tar.gz

  添加配置 nginx_log.conf

input {

    file {

        #这里根据自己日志命名使用正则匹配所有域名访问日志

        path => [ "/var/log/nginx/*.log" ]

        ignore_older => 0

    codec => json

    }

}

filter {

    mutate {

      convert => [ "status","integer" ]

      convert => [ "size","integer" ]

      convert => [ "upstreatime","float" ]

      remove_field => "message"

    }

    geoip {

        source => "ip"

    }

}

output {

    elasticsearch {

        hosts => "127.0.0.1:9200"

        index => "logstash-nginx-access-%{+YYYY.MM.dd}"

    }

#    stdout {codec => rubydebug}

}

启动 

./bin/logstash -f config/nginx_log.conf  --path.data=/root/