ELK环境搭建
2020-04-10 本文已影响0人
苏水的北
1、基础环境搭建：
1.1 准备一台服务器：
系统：CentOS7.6版本Linux系统
ip:10.0.0.5
备注：ELK服务均在一台服务器上部署。
1.2修改repo源，yum安装基础插件：
[root@elk ~]# curl -o /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo
[root@elk ~]#curl -o /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo
[root@elk ~]#yum repolist
[root@elk ~]#yum install tree vim wget bash-completion bash-completion-extras lrzsz net-tools sysstat iotop iftop htop unzip nc nmap telnet bc psmisc -y
1.3永久关闭并暂停selinux和filewalld：
[root@elk ~]#vim /etc/selinux/config
# This file controls the state of SELinux on the system.
# SELINUX= can take one of these three values:
# enforcing - SELinux security policy is enforced.
# permissive - SELinux prints warnings instead of enforcing.
# disabled - No SELinux policy is loaded.
SELINUX=disabled
# SELINUXTYPE= can take one of three values:
# targeted - Targeted processes are protected,
# minimum - Modification of targeted policy. Only selected processes are protected.
# mls - Multi Level Security protection.
SELINUXTYPE=targeted
[root@elk ~]#systemctl disable firewalld
[root@elk ~]#systemctl stop firewalld
[root@elk ~]#systemctl status firewalld
[root@elk ~]#systemctl disable NetworkManager
[root@elk ~]#systemctl stop NetworkManager
[root@elk ~]#systemctl is-active firewalld NetworkManager
[root@elk ~]#systemctl is-enabled firewalld NetworkManager
2、软件环境安装：
2.1 把提前准备好的rpm包放到root用户下：
[root@elk ~]#unzip ELK软件包最新版7.6.1版本.zip
[root@elk ~]#mv ELK软件包最新版7.6.1版本/* .
[root@elk ~]#ll
total 1457296
-rw-------. 1 root root 1610 Mar 11 11:13 anaconda-ks.cfg
-rw-r--r--. 1 root root 296519136 Mar 9 2020 elasticsearch-7.6.1-x86_64.rpm
-rw-r--r--. 1 root root 452827 Aug 29 2019 elasticsearch-head-0.1.2_0.crx.rar
-rw-r--r--. 1 root root 740255976 Apr 9 2020 ELK软件包最新版7.6.1版本.zip
-rw-r--r--. 1 root root 24694569 Mar 9 2020 filebeat-7.6.1-x86_64.rpm
-rw-r--r--. 1 root root 257513099 Mar 9 2020 kibana-7.6.1-x86_64.rpm
-rw-r--r--. 1 root root 172821011 Mar 9 2020 logstash-7.6.1.rpm
2.2 java环境安装：
[root@elk ~]#yum install -y java
[root@elk ~]#java -version
openjdk version "1.8.0_242"
OpenJDK Runtime Environment (build 1.8.0_242-b08)
OpenJDK 64-Bit Server VM (build 25.242-b08, mixed mode)
2.3 安装elasticsearch并验证是否启动以及优化：
[root@elk ~]#rpm -ivh elasticsearch-7.6.1-x86_64.rpm
warning: elasticsearch-7.6.1-x86_64.rpm: Header V4 RSA/SHA512 Signature, key ID d88e42
Preparing... ################################# [100%]
Creating elasticsearch group... OK
Creating elasticsearch user... OK
Updating / installing...
1:elasticsearch-0:7.6.1-1 ################################# [100%]
### NOT starting on installation, please execute the following statements to configuresing systemd
sudo systemctl daemon-reload
sudo systemctl enable elasticsearch.service
### You can start elasticsearch service by executing
sudo systemctl start elasticsearch.service
Created elasticsearch keystore in /etc/elasticsearch
[root@elk ~]#systemctl daemon-reload
[root@elk ~]#systemctl start elasticsearch.service
[root@elk ~]#
[root@elk ~]#netstat -lntup
Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name
tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN 15308/sshd
tcp 0 0 127.0.0.1:25 0.0.0.0:* LISTEN 5945/master
tcp6 0 0 127.0.0.1:9200 :::* LISTEN 15915/java
tcp6 0 0 ::1:9200 :::* LISTEN 15915/java
tcp6 0 0 127.0.0.1:9300 :::* LISTEN 15915/java
tcp6 0 0 ::1:9300 :::* LISTEN 15915/java
tcp6 0 0 :::22 :::* LISTEN 15308/sshd
tcp6 0 0 ::1:25 :::* LISTEN 5945/master
[root@elk ~]#netstat -lntup
Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address Foreign Address State PID/Pr
tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN 15308/
tcp 0 0 127.0.0.1:25 0.0.0.0:* LISTEN 5945/m
tcp6 0 0 127.0.0.1:9200 :::* LISTEN 15915/
tcp6 0 0 ::1:9200 :::* LISTEN 15915/
tcp6 0 0 127.0.0.1:9300 :::* LISTEN 15915/
tcp6 0 0 ::1:9300 :::* LISTEN 15915/
tcp6 0 0 :::22 :::* LISTEN 15308/
tcp6 0 0 ::1:25 :::* LISTEN 5945/m
[root@elk ~]# curl 127.0.0.1:9200 /出现以下信息以及出现9200端口就表明elasticsearch安装成功。
{
"name" : "elk",
"cluster_name" : "elasticsearch",
"cluster_uuid" : "On6iNJhtQLO2g43B9VN29A",
"version" : {
"number" : "7.6.1",
"build_flavor" : "default",
"build_type" : "rpm",
"build_hash" : "aa751e09be0a5072e8570670309b1f12348f023b",
"build_date" : "2020-02-29T00:15:25.529771Z",
"build_snapshot" : false,
"lucene_version" : "8.4.0",
"minimum_wire_compatibility_version" : "6.8.0",
"minimum_index_compatibility_version" : "6.0.0-beta1"
},
"tagline" : "You Know, for Search"
}
[root@elk ~]# rpm -qc elasticsearch /查看关键配置文件
/etc/elasticsearch/elasticsearch.yml #主配置文件
/etc/elasticsearch/jvm.options #java虚拟机配置文件
/etc/init.d/elasticsearch #init.d的启动文件
/etc/sysconfig/elasticsearch #与环境变量相关的设置,不需要动
/usr/lib/sysctl.d/elasticsearch.conf #最大连接数,不需要动
/usr/lib/systemd/system/elasticsearch.service #systemd启动文件
[root@lb01 ~]# grep "\-Xm" /etc/elasticsearch/jvm.options /修改内存大小
-Xms512m
-Xmx512m
[root@elk ~]#grep -v '^#' /etc/elasticsearch/elasticsearch.yml /主配置文件
23 node.name: node-1 #节点名称,每个节点不一样
33 path.data: /var/lib/elasticsearch #数据目录
37 path.logs: /var/log/elasticsearch #日志目录
43 bootstrap.memory_lock: true #锁定内存设置
55 network.host: 192.168.98.49,127.0.0.1 #监听网卡地址（主机地址）
59 http.port: 9200
68 discovery.seed_hosts: ["192.168.98.49"]
9.解决内存锁定
官方解决方案:
[root@elk ~]#systemctl edit elasticsearch
[Service]
LimitMEMLOCK=infinity
重启服务
[root@elk ~]#systemctl daemon-reload
[root@elk ~]#systemctl restart elasticsearch
[root@elk ~]#curl 192.168.98.49:9200
{
"name" : "node-1",
"cluster_name" : "elasticsearch",
"cluster_uuid" : "On6iNJhtQLO2g43B9VN29A",
"version" : {
"number" : "7.6.1",
"build_flavor" : "default",
"build_type" : "rpm",
"build_hash" : "aa751e09be0a5072e8570670309b1f12348f023b",
"build_date" : "2020-02-29T00:15:25.529771Z",
"build_snapshot" : false,
"lucene_version" : "8.4.0",
"minimum_wire_compatibility_version" : "6.8.0",
"minimum_index_compatibility_version" : "6.0.0-beta1"
},
"tagline" : "You Know, for Search"
}
3、在谷歌浏览器中添加es-head-0.1.4_0.crx并在浏览器中验证连接情况：
3.1 谷歌浏览器中添加es-head-0.1.4_0.crx：
image.png
image.png
3.2验证是否连接：
image.png
4、安装配置kibana：
4.1 安装kibana：
[root@elk ~]#rpm -ivh kibana-7.6.1-x86_64.rpm
warning: kibana-7.6.1-x86_64.rpm: Header V4 RSA/SHA512 Signature, key ID d88e42b4: NOKEY
Preparing... ################################# [100%]
Updating / installing...
1:kibana-7.6.1-1 ################################# [100%]
4.2 kibana配置：
[root@elk ~]#grep -n '^[a-z]' /etc/kibana/kibana.yml
2:server.port: 5601
7:server.host: "192.168.98.49"
28:elasticsearch.hosts: ["http://192.168.98.49:9200"]
37:kibana.index: ".kibana"
115:i18n.locale: "zh-CN" //汉化设置
4.3 启动kibana，并设置开机自启：
[root@elk ~]#systemctl start kibana.service
[root@elk ~]#systemctl enable kibana.service
Created symlink from /etc/systemd/system/multi-user.target.wants/kibana.service to /etc/systemd/system/kibana.service.
备注：启动kibana的前提是es必须先启动，因为kibana是依赖es启动的。
4.4 在web检查kibana是否可以打开：
image.png
5、安装nginx，并访问nginx，使其产生日志（下面的filebeat需要使用nginx日志）：
[root@elk ~]#yum install -y nginx
[root@elk ~]#cat /etc/nginx/nginx.conf
user nginx;
worker_processes 1;
error_log /var/log/nginx/error.log;
pid /run/nginx.pid;
events {
worker_connections 1024;
}
http {
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
access_log /var/log/nginx/access.log main;
sendfile on;
keepalive_timeout 65;
#include /etc/nginx/conf.d/*.conf;
server {
listen 80;
server_name www.ht.com;
location / {
root /usr/share/nginx/html;
index index.html index.htm;
}
}
}
[root@elk ~]#nginx -t
nginx: the configuration file /etc/nginx/nginx.conf syntax is ok
nginx: configuration file /etc/nginx/nginx.conf test is successful
[root@elk ~]#systemctl start nginx
访问nginx页面，使其产生nginx日志：
image.png
6、安装filebeat并启动：
6.1 安装filebeat：
[root@elk ~]#rpm -ivh filebeat-7.6.1-x86_64.rpm
warning: filebeat-7.6.1-x86_64.rpm: Header V4 RSA/SHA512 Signature, key ID d88e42b4: NOKEY
Preparing... ################################# [100%]
Updating / installing...
1:filebeat-7.6.1-1 ################################# [100%]
6.2 配置filebeat配置文件：
[root@elk ~]#vim /etc/filebeat/filebeat.yml
filebeat.inputs:
- type: log
enabled: true
paths:
- /var/log/nginx/access.log //nginx日志路径（这里可以是任何日志文件路径）
output.elasticsearch:
hosts: ["10.0.0.101:9200"] //elasticsearch端的ip
6.3 启动filebeat:
[root@elk ~]#systemctl start filebeat.service
[root@elk ~]#systemctl enable filebeat.service
6.4 kibana的web界面，展示filebeat拉取日志数据操作步骤：
image.png
image.png
image.png
image.png
