011.ELK使用Kafka做缓存收集Nginx日志
2020-04-27 本文已影响0人
CoderJed
1. 流程说明
2. 配置过程
2.1 nginx配置
log_format json '{"time_local": "$time_local", '
'"remote_addr": "$remote_addr", '
'"referer": "$http_referer", '
'"request": "$request", '
'"status": $status, '
'"bytes": $body_bytes_sent, '
'"agent": "$http_user_agent", '
'"x_forwarded": "$http_x_forwarded_for", '
'"up_addr": "$upstream_addr", '
'"up_host": "$upstream_http_host", '
'"upstream_time": "$upstream_response_time", '
'"request_time": "$request_time"}';
# 使用json日志格式
access_log /var/log/nginx/access.log main;
2.2 filebeat配置
filebeat.inputs:
- type: log
enabled: true
paths:
- /var/log/nginx/access.log
json.keys_under_root: true
json.overwrite_keys: true
tags: ["access"]
- type: log
enabled: true
paths:
- /var/log/nginx/error.log
tags: ["error"]
output.kafka:
hosts: ["10.0.0.110:9092","10.0.0.111:9092","10.0.0.112:9092"]
topic: nginx_log
2.3 logstash配置
input {
kafka {
bootstrap_servers => "10.0.0.110:9092,10.0.0.111:9092,10.0.0.112:9092"
topics => ["nginx_log"]
group_id => "logstash"
codec => "json"
}
}
filter {
mutate {
convert => ["upstream_time", "float"]
convert => ["request_time", "float"]
}
}
output {
if "access" in [tags] {
elasticsearch {
hosts => "http://10.0.0.101:9200"
manage_template => false
index => "nginx_access-%{+yyyy.MM}"
}
}
if "error" in [tags] {
elasticsearch {
hosts => "http://10.0.0.101:9200"
manage_template => false
index => "nginx_error-%{+yyyy.MM}"
}
}
}
3. 测试
-
创建kafka topic
[root@kafka01 ~]# /opt/kafka/bin/kafka-topics.sh --create --bootstrap-server 10.0.0.110:9092,10.0.0.111:9092,10.0.0.111:9092 --replication-factor 3 --partitions 3 --topic nginx_log -
监听kafka topic
[root@kafka03 ~]# /opt/kafka/bin/kafka-console-consumer.sh --bootstrap-server 10.0.0.110:9092,10.0.0.111:9092,10.0.0.112:9092 --topic nginx_log --from-beginning -
启动相关服务
[root@nginx01 ~]# systemctl start nginx [root@es01 ~]# systemctl start elasticsearch [root@es01 ~]# systemctl start kibana [root@nginx01 ~]# systemctl start filebeat [root@es01 ~]# /usr/share/logstash/bin/logstash -f /root/logstash.yml -
发送测试请求
[root@nginx01 opt]# ab -c 10 -n 1000 http://10.0.0.109:80/ [root@nginx01 opt]# ab -c 10 -n 1000 http://10.0.0.109:80/baidu -
kafka-console-consumer消费到数据
[root@kafka03 ~]# /opt/kafka/bin/kafka-console-consumer.sh --bootstrap-server 10.0.0.110:9092,10.0.0.111:9092,10.0.0.112:9092 --topic nginx_log --from-beginning {"@timestamp":"2020-04-27T09:09:34.585Z","@metadata":{"beat":"filebeat","type":"doc","version":"6.6.0","topic":"nginx_log"},"beat":{"name":"nginx01","hostname":"nginx01","version":"6.6.0"},"x_forwarded":"-","source":"/var/log/nginx/access.log","bytes":153,"request":"GET /baidu HTTP/1.0","status":404,"offset":552760,"up_host":"-","input":{"type":"log"},"time_local":"27/Apr/2020:17:09:34 +0800","tags":["access"],"host":{"name":"nginx01"},"log":{"file":{"path":"/var/log/nginx/access.log"}},"up_addr":"-","remote_addr":"10.0.0.109","request_time":"0.000","upstream_time":"-","referer":"-","agent":"ApacheBench/2.3","prospector":{"type":"log"}} {"@timestamp":"2020-04-27T09:09:34.585Z","@metadata":{"beat":"filebeat","type":"doc","version":"6.6.0","topic":"nginx_log"},"agent":"ApacheBench/2.3","request_time":"0.000","prospector":{"type":"log"},"beat":{"version":"6.6.0","name":"nginx01","hostname":"nginx01"},"host":{"name":"nginx01"},"status":404,"up_host":"-","remote_addr":"10.0.0.109","tags":["access"],"source":"/var/log/nginx/access.log","log":{"file":{"path":"/var/log/nginx/access.log"}},"up_addr":"-","referer":"-","request":"GET /baidu HTTP/1.0","x_forwarded":"-","bytes":153,"time_local":"27/Apr/2020:17:09:34 +0800","input":{"type":"log"},"offset":553880,"upstream_time":"-"} ...... -
查看kibana
GET _cat/indices yellow open nginx_access-2020.04 apimPU-QTAmP7GeE7l8evQ 5 1 2000 0 689kb 689kb yellow open nginx_error-2020.04 WH3Lme3gQuqBSK8MFYfsSw 5 1 1000 0 754.2kb 754.2kb GET nginx_access-2020.04/_search { "took" : 4, "timed_out" : false, "_shards" : { "total" : 5, "successful" : 5, "skipped" : 0, "failed" : 0 }, "hits" : { "total" : 1999, "max_score" : 1.0, "hits" : [ { "_index" : "nginx_access-2020.04", "_type" : "doc", "_id" : "avBpunEBINm9vG5xGD9v", "_score" : 1.0, "_source" : { "tags" : [ "access" ], "request" : "GET / HTTP/1.0", "offset" : 246975, "time_local" : "27/Apr/2020:14:49:37 +0800", "referer" : "-", "beat" : { "hostname" : "nginx01", "version" : "6.6.0", "name" : "nginx01" }, "input" : { "type" : "log" }, "host" : { "name" : "nginx01" }, "status" : 200, "up_addr" : "-", "up_host" : "-", "prospector" : { "type" : "log" }, "bytes" : 612, "@version" : "1", "agent" : "ApacheBench/2.3", "upstream_time" : 0.0, "request_time" : 0.0, "@timestamp" : "2020-04-27T06:49:45.660Z", "source" : "/var/log/nginx/access.log", "log" : { "file" : { "path" : "/var/log/nginx/access.log" } }, "x_forwarded" : "-", "remote_addr" : "10.0.0.109" } } ] } }
