logstash 解析 nginx 日志

配置

input {
    stdin {
    }
}

filter {
    grok {
        match => {
            "message" => "%{COMBINEDAPACHELOG}"
        }
    }
}

output {
    stdout {
        codec => rubydebug
    }
}

nginx 日志样例

192.168.0.104 - - [11/Nov/2018:21:53:26 +0800] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36 Edge/17.17134" "-"                               

解析结果

192.168.0.104 - - [11/Nov/2018:21:53:26 +0800] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36 Edge/17.17134" "-"                               
{
"@timestamp" => 2018-11-25T14:51:11.821Z,                                                                                     
"host" => "TingShuo",                                                                                                
"request" => "/",                                                                                                         
"ident" => "-",                                                                                                      
"response" => "304",                                                                                                       
"agent" => "\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36 Edge/17.17134\"",
"httpversion" => "1.1",                                                                                                       
"bytes" => "0",                                                                                                      
"referrer" => "\"-\"",                                                                                                  
"@version" => "1",                                                                                                       
"message" => "192.168.0.104 - - [11/Nov/2018:21:53:26 +0800] \"GET / HTTP/1.1\" 304 0 \"-\" \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36 Edge/17.17134\" \"-\"\r",                                                                                                                            
"clientip" => "192.168.0.104",                                                                                         
"timestamp" => "11/Nov/2018:21:53:26 +0800",                                                                                 
"verb" => "GET",                                                                                                        
"auth" => "-"
}                                                                                                                           

---

配置样例

样例一

[root@log-monitor ~]# cat /etc/logstash/conf.d/nginx_access.conf
input {
    file {
        path => [ "/data/nginx-logs/access.log" ]
        start_position => "beginning"
        ignore_older => 0
    }
}

filter {
    grok {
        match => { "message" => "%{NGINXACCESS}" }

    }
    geoip {
      source => "clientip"
      target => "geoip"
      database => "/etc/logstash/GeoLiteCity.dat"
      add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
      add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ]
    }

    mutate {
      convert => [ "[geoip][coordinates]", "float" ]
      convert => [ "response","integer" ]
      convert => [ "bytes","integer" ]
      replace => { "type" => "nginx_access" }
      remove_field => "message"
    }

    date {
      match => [ "timestamp","dd/MMM/yyyy:HH:mm:ss Z"]

    }
    mutate {
      remove_field => "timestamp"

    }


}
output {
    elasticsearch {
        hosts => ["127.0.0.1:9200"]
        index => "logstash-nginx-access-%{+YYYY.MM.dd}"
    }
    stdout {codec => rubydebug}
}