从0开始配置基于drbd的高可用nfs双机集群全过程（下）
2021-11-05 本文已影响0人
阿乐_822e
3 配置资源
两台机分别操作
$ dd if=/dev/zero of=/dev/sdb1 bs=1M count=100 ##用指定大小的块拷贝一个文件
$ drbdadm -c /etc/drbd.conf create-md all ##创建设备元数据
initializing activity log
......
success
$ drbdadm create-md r1 ##创建r1设备元
You want me to create a v09 style flexible-size internal meta data block.
......
New drbd meta data block successfully created.
$ drbdadm up r1 ##启用该资源
$ drbdadm role r1
# 此时两台机都是Secodnary
4. 将DC-NFS00置主并格式化新分区并挂载
# 在DC-NFS00上操作
$ drbdadm primary r1 --force
$ drbdadm role r1 # 强行置主
$ /root/drbd-utils-9.12.2/scripts/drbd-overview.pl ##查看状态
..... 1:r1/1 Connected(2*) Primar/Second UpToDa/Incons /nfs ext4 394G 73M 374G
1%
$ mkfs.ext4 /dev/drbd1 ##格式化/dev/drbd1
$ mount /dev/drbd1 /nfs ##挂载
$ drbdadm status r1
r1 role:Primary
volume:1 disk:UpToDate
DC-NFS01 role:Secondary
volume:1 replication:SyncSource peer-disk:Inconsistent done:12.65
# 在DC-NFS01上
$ drbdadm status r1
r1 role:Secondary
volume:1 disk:Inconsistent
DC-NFS00 role:Primary
volume:1 replication:SyncTarget peer-disk:UpToDate done:12.80 # 正在同步中
# 以下两台机上都做
$ systemctl start drbd && systemctl enable drbd
5. 安装并配置Keepalived
两台机都做
$ yum -y install keepalived.x86_64
已加载插件：fastestmirror
......
完毕！
$ rm -f /etc/keepalived/keepalived.conf
$ cat << EOF | tee /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
router_id NFS-Master
}
vrrp_script chk_nfs
{
script "/etc/keepalived/nfs_check.sh" #监控脚本
interval 2 # 2秒一次
weight -20 # keepalived部署了两台所以设为20，如果三台就设为30
}
vrrp_instance VI_1 {
state BACKUP # 两台主机都设为backup非抢占模式
interface ens32 # 网卡文件名称，根据实际修改
virtual_router_id 127 # 在同一子网中的id要唯一
priority 100
advert_int 1
nopreempt # 设置为非抢占模式必须要该参数
authentication {
auth_type PASS
auth_pass 123456
}
track_script {
chk_nfs
}
notify_stop /etc/keepalived/notify_stop.sh # keepalived停服时调用
virtual_ipaddress {
192.168.106.127/24 # 虚拟出的公共IP
}
}
EOF
# 配置监控脚本
$ cat << EOF | tee /etc/keepalived/nfs_check.sh
#!/bin/bash
# 日志文件大于5M截取最后50行
[ `du -m /tmp/nfs-chk.log | awk '{print $1}'` -gt 5 ] && tail -50 /tmp/nfs-chk.log>/tmp/nfs-tmp && mv /tmp/nfs-tmp /tmp/nfs-chk.log -f
vip=`ip a |grep 106.127|wc -l`
if [ $vip -eq 1 ];then # 主keepalived机器检查
service nfs status &>/dev/null # 检查nfs可用性
if [ $? -ne 0 ];then # 如果服务状态不正常，先尝试重启服务
time=`date "+%F %H:%M:%S"`
echo -e "$time ------主机NFS服务故障，重启之！------\n" >>/tmp/nfs-chk.log
service nfs start &>/dev/null
fi
nfsStatus=`ps -C nfsd --no-header | wc -l`
if [ $nfsStatus -eq 0 ];then # 若重启nfs服务后，仍不正常
time=`date "+%F %H:%M:%S"`
echo -e "$time ------nfs服务故障且重启失败，切换到备用服务器------\n" >>/tmp/nfs-chk.log
service nfs stop &>>/tmp/nfs-chk.log # 停止nfs服务
umount /dev/drbd1 &>>/tmp/nfs-chk.log # 卸载drbd设备
drbdadm secondary r1 &>>/tmp/nfs-chk.log # 将drbd主降级为备
service keepalived stop &>>/tmp/nfs-chk.log # 关闭keepalived
time=`date "+%F %H:%M:%S"`
echo -e "$time ------切换结束！------\n" >>/tmp/nfs-chk.log
sleep 2
service keepalived start &>>/tmp/nfs-chk.log # 再开启keepalived
else
# drbd置主没有，挂载没有
drbdadm role r1|grep Primary &>/dev/null
if [ $? -ne 0 ];then # drbd未置Primary
time=`date "+%F %H:%M:%S"`
echo -e "$time ------将本机置为DRBD主机并挂载/nfs目录------\n" >>/tmp/nfs-chk.log
drbdadm primary --force r1 &>>/tmp/nfs-chk.log # 将drbd置为主
mount /dev/drbd1 /nfs &>>/tmp/nfs-chk.log # 挂载drbd设备
fi
fi
else # keepalived备机检查
service nfs status &>/dev/null
if [ $? -eq 0 ];then # NFS服务必须处于关闭状态
time=`date "+%F %H:%M:%S"`
echo -e "$time ------关闭备机NFS服务------\n" >>/tmp/nfs-chk.log
service nfs stop &>>/tmp/nfs-chk.log
fi
drbdadm role r1|grep Primary &>/dev/null
if [ $? -eq 0 ];then # drbd必须置备并卸载drbd设备
time=`date "+%F %H:%M:%S"`
echo -e "$time ------备机置secondary并卸载备机drbd设备------\n" >>/tmp/nfs-chk.log
drbdadm secondary r1 &>>/tmp/nfs-chk.log
umount /dev/drbd1 &>>/tmp/nfs-chk.log &>>/tmp/nfs-chk.log
fi
fi
EOF
# 配置keepalived停服脚本
$ cat << EOF | tee /etc/keepalived/notify_stop.sh
#!/bin/bash
time=`date "+%F %H:%M:%S"`
echo -e "$time ------开始切换到备用服务器------\n" >>/tmp/nfs-chk.log
service nfs stop &>>/tmp/nfs-chk.log # 停止nfs服务
umount /dev/drbd1 &>>/tmp/nfs-chk.log # 卸载drbd设备
drbdadm secondary r1 &>>/tmp/nfs-chk.log # 将drbd主降级为备
time=`date "+%F %H:%M:%S"`
echo -e "$time ------切换结束！------\n" >>/tmp/nfs-chk.log
sleep 2
service keepalived start &>>/tmp/nfs-chk.log # 再开启keepalived
EOF
$ chmod +x /etc/keepalived/*.sh
$ systemctl start keepalived.service && systemctl enable keepalived.service
# 以下在备机上，修改/etc/keepalived/keepalived.conf
router_id NFS-Slave
priority 80 #从节点的权重要比主节点低
# 再重启keepalived服务
$ service keepalived restart
说明：在实际工作中，一定要连接到VIP进行操作，如果直连到slave机上对同步目录进行操作，是不会同步到master上的。
至此，NFS-SERVER的所有配置工作完毕！下面介绍NFS客户端的配置。
6. 配置NFS客户端
$ yum -y install nfs-utils
$ systemctl start nfs && systemctl enable nfs && systemctl status nfs
$ showmount -e 192.168.106.127 # 查看可挂载NFS磁盘
Export list for 192.168.106.127:
/nfs 192.168.106.127/24
$ mkdir /nfs
$ mount -t nfs 192.168.106.127:/nfs /nfs # 挂载
$ df /nfs # 查看一下挂载情况
文件系统 1K-块 已用 可用 已用% 挂载点
192.168.106.127:/nfs 412702720 73728 391642112 1% /nfs
# 再配置开机自动挂载
$ echo '192.168.106.127:/nfs /nfs nfs rw,tcp,intr 0 0' >> /etc/fstab
7. NFS简单性能测试
[root@k8s-node1 ~]# mkdir /nfs
[root@k8s-node1 ~]# mount -t nfs 192.168.106..127:/nfs /nfs
# 写测试
[root@k8s-node1 ~]# time dd if=/dev/zero of=/nfs/testfile bs=32k count=16384
......536870912字节(537 MB)已复制，1.9337 秒，278 MB/秒......
# 读测试
[root@k8s-node1 ~]# time dd if=/nfs/testfile of=/dev/null bs=16k
......536870912字节(537 MB)已复制，0.123949 秒，4.3 GB/秒......
可以多重复几次，计算一下平均速率
--------------------------OVER------------------------------------------------------
参考文档：
1、NFS传输速度优化
2、DRBD编译安装配置
3、Centos下部署DRBD+NFS+Keepalived高可用环境记录
