filebeat

架构图

1636241770081.png

配置文件

filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/test/product.log
  tags: ["nginx"]
  fields_under_root: true
  fields:
    project: microservice
    app: product
    
- type: log
  enabled: true
  paths:
    - /var/log/test/gateway.log
  tags: ["nginx"]
  fields_under_root: true
  fields:
    project: microservice
    app: gateway
    
output.logstash:
  hosts: ["192.168.153.26:5044"]
    

配置服务

[root@bogon filebeat]# vi /usr/lib/systemd/system/filebeat.service  
[Unit]
Description=filebeat
[Service]
ExecStart=/opt/elk/filebeat/filebeat -c /opt/elk/filebeat/filebeat.yml
ExecReload=/bin/kill -HUP $MAINPID
KillMode=process
Restart=on-failure
[Install]
WantedBy=multi-user.target

[root@bogon filebeat]# systemctl daemon-reload                           
[root@bogon filebeat]# systemctl restart filebeat

logstash配置

input {
  beats {
    host => "0.0.0.0"
    port => 5044
  }
}

filter {
  if [app] == "product" {
    mutate {
      add_field => {
        "[@metadata][target_index]" => "microservice-product-%{+YYYY.MM}"
      }
    }
  } else if [app] == "gateway" {
    mutate {
      add_field => {
        "[@metadata][target_index]" => "microservice-gateway-%{+YYYY.MM.dd}"
      }
    }
  } else {
    mutate {
      add_field => {
        "[@metadata][target_index]" => "unknown-%{+YYYY}"
      }
    }
  }



}

output {
  elasticsearch {
    hosts => "192.168.153.25:9200"
    index => "%{[@metadata][target_index]}"
  }
}

增加缓冲队列

1636242034923.png

安装redis

[root@es3 ~]# docker run -itd --name redis -p 6379:6379 redis --requirepass 123456

filebeat配置输出到redis

output.redis:
  hosts: ["192.168.153.27:6379"]
  password: "123456"
  key: "filebeat"
  db: 0
  datatype: "list" 

logstash配置从redis里读

input {
  redis {
    host => "192.168.153.27"
    port => 6379
    password => "123456"
    key => "filebeat"
    db => 0
    data_type => "list"
  }
}