009.ELK使用Redis做缓存收集Nginx日志
2020-04-27 本文已影响0人
CoderJed
1. 流程说明
2. 配置过程
2.1 nginx配置
log_format json '{"time_local": "$time_local", '
'"remote_addr": "$remote_addr", '
'"referer": "$http_referer", '
'"request": "$request", '
'"status": $status, '
'"bytes": $body_bytes_sent, '
'"agent": "$http_user_agent", '
'"x_forwarded": "$http_x_forwarded_for", '
'"up_addr": "$upstream_addr", '
'"up_host": "$upstream_http_host", '
'"upstream_time": "$upstream_response_time", '
'"request_time": "$request_time"}';
# 使用json日志格式
access_log /var/log/nginx/access.log main;
2.2 filebeat配置
filebeat.inputs:
- type: log
enabled: true
paths:
- /var/log/nginx/access.log
json.keys_under_root: true
json.overwrite_keys: true
tags: ["access"]
- type: log
enabled: true
paths:
- /var/log/nginx/error.log
tags: ["error"]
output.redis:
hosts: ["10.0.0.104:6379"]
keys:
- key: "nginx_access"
when.contains:
tags: "access"
- key: "nginx_error"
when.contains:
tags: "error"
2.3 logstash配置
input {
redis {
host => "10.0.0.104"
port => "6379"
db => "0"
key => "nginx_access"
data_type => "list"
}
redis {
host => "10.0.0.104"
port => "6379"
db => "0"
key => "nginx_error"
data_type => "list"
}
}
filter {
mutate {
# 这两个字段转为float类型
convert => ["upstream_time", "float"]
convert => ["request_time", "float"]
}
}
output {
stdout {}
if "access" in [tags] {
elasticsearch {
hosts => ["10.0.0.101:9200"]
manage_template => false
index => "nginx_access-%{+yyyy.MM}"
}
}
if "error" in [tags] {
elasticsearch {
hosts => ["10.0.0.101:9200"]
manage_template => false
index => "nginx_error-%{+yyyy.MM}"
}
}
}
3. 测试
-
启动以上服务
[root@nginx01 ~]# systemctl start nginx [root@redis01 ~]# /opt/redis/bin/redis-server /opt/redis/conf/redis_6379.conf [root@es01 ~]# systemctl start elasticsearch [root@es01 ~]# systemctl start kibana [root@nginx01 ~]# systemctl start filebeat [root@es01 ~]# /usr/share/logstash/bin/logstash -f /root/logstash.yml -
发送测试请求
[root@nginx01 opt]# ab -c 10 -n 1000 http://10.0.0.109:80/ [root@nginx01 opt]# ab -c 10 -n 1000 http://10.0.0.109:80/baidu -
查看redis
10.0.0.104:6379> keys * 1) "nginx_error" 2) "nginx_access" 10.0.0.104:6379> lpop nginx_access "{\"@timestamp\":\"2020-04-27T06:49:45.566Z\",\"@metadata\":{\"beat\":\"filebeat\",\"type\":\"doc\",\"version\":\"6.6.0\"},\"source\":\"/var/log/nginx/access.log\",\"time_local\":\"27/Apr/2020:14:49:37 +0800\",\"x_forwarded\":\"-\",\"log\":{\"file\":{\"path\":\"/var/log/nginx/access.log\"}},\"up_addr\":\"-\",\"offset\":2775,\"request\":\"GET / HTTP/1.0\",\"request_time\":\"0.000\",\"up_host\":\"-\",\"remote_addr\":\"10.0.0.109\",\"host\":{\"name\":\"nginx01\"},\"status\":200,\"referer\":\"-\",\"tags\":[\"access\"],\"prospector\":{\"type\":\"log\"},\"input\":{\"type\":\"log\"},\"beat\":{\"name\":\"nginx01\",\"hostname\":\"nginx01\",\"version\":\"6.6.0\"},\"agent\":\"ApacheBench/2.3\",\"upstream_time\":\"-\",\"bytes\":612}" 10.0.0.104:6379> lpop nginx_error "{\"@timestamp\":\"2020-04-27T06:49:55.558Z\",\"@metadata\":{\"beat\":\"filebeat\",\"type\":\"doc\",\"version\":\"6.6.0\"},\"log\":{\"file\":{\"path\":\"/var/log/nginx/error.log\"}},\"source\":\"/var/log/nginx/error.log\",\"tags\":[\"error\"],\"prospector\":{\"type\":\"log\"},\"input\":{\"type\":\"log\"},\"host\":{\"name\":\"nginx01\"},\"message\":\"2020/04/27 14:49:47 [error] 3031#3031: *1009 open() \\\"/usr/share/nginx/html/baidu\\\" failed (2: No such file or directory), client: 10.0.0.109, server: localhost, request: \\\"GET /baidu HTTP/1.0\\\", host: \\\"10.0.0.109\\\"\",\"offset\":1040,\"beat\":{\"version\":\"6.6.0\",\"name\":\"nginx01\",\"hostname\":\"nginx01\"}}" -
logstash控制台输出
{ "tags" => [ [0] "access" ], "request" => "GET /baidu HTTP/1.0", "offset" => 554720, "beat" => { "hostname" => "nginx01", "version" => "6.6.0", "name" => "nginx01" }, "referer" => "-", "time_local" => "27/Apr/2020:14:49:47 +0800", "input" => { "type" => "log" }, "host" => { "name" => "nginx01" }, "status" => 404, "up_addr" => "-", "up_host" => "-", "prospector" => { "type" => "log" }, "bytes" => 153, "@version" => "1", "agent" => "ApacheBench/2.3", "request_time" => 0.0, "upstream_time" => 0.0, "@timestamp" => 2020-04-27T06:49:48.731Z, "source" => "/var/log/nginx/access.log", "log" => { "file" => { "path" => "/var/log/nginx/access.log" } }, "x_forwarded" => "-", "remote_addr" => "10.0.0.109" } -
查看kibana
GET _cat/indices yellow open nginx_access-2020.04 hikPROoJR0OK3YiX1a-ztA 5 1 2000 0 643.2kb 643.2kb yellow open nginx_error-2020.04 QUpHiZuuQSetl0m04xrMRQ 5 1 1000 0 995.9kb 995.9kb GET nginx_access-2020.04/_search { "took" : 4, "timed_out" : false, "_shards" : { "total" : 5, "successful" : 5, "skipped" : 0, "failed" : 0 }, "hits" : { "total" : 1999, "max_score" : 1.0, "hits" : [ { "_index" : "nginx_access-2020.04", "_type" : "doc", "_id" : "avBpunEBINm9vG5xGD9v", "_score" : 1.0, "_source" : { "tags" : [ "access" ], "request" : "GET / HTTP/1.0", "offset" : 246975, "time_local" : "27/Apr/2020:14:49:37 +0800", "referer" : "-", "beat" : { "hostname" : "nginx01", "version" : "6.6.0", "name" : "nginx01" }, "input" : { "type" : "log" }, "host" : { "name" : "nginx01" }, "status" : 200, "up_addr" : "-", "up_host" : "-", "prospector" : { "type" : "log" }, "bytes" : 612, "@version" : "1", "agent" : "ApacheBench/2.3", "upstream_time" : 0.0, "request_time" : 0.0, "@timestamp" : "2020-04-27T06:49:45.660Z", "source" : "/var/log/nginx/access.log", "log" : { "file" : { "path" : "/var/log/nginx/access.log" } }, "x_forwarded" : "-", "remote_addr" : "10.0.0.109" } } ] } }
