GCDW云数仓ELK安装部署步骤（案例）

1）请提前借出该集群所有机器的root账号密码、提前准备软件安装包、gcdw_workspace运维脚本、ES认证证书和证书密码
2）替换本安装步骤中的GGG为OS用户gbase的密码
3）替换本安装步骤中的XXX为gbase数据库用户的密码

集群搭建版本：ELK-9.8.0.8.3_patch.8

##集群基本信息
##IP列表（需提前核对 主机名 与 业务IP 是否一致）
GCDWELK01    172.16.5.28
GCDWELK02    172.16.5.29

#### 一、ELK集群安装准备步骤
## 1.1 上传运维脚本
# 上传gcdw_workspace.zip至/opt目录下
[root@GCDWELK01 ~]# cd /opt
[root@GCDWELK01 opt]# unzip gcdw_workspace.zip
[root@GCDWELK01 opt]# cd /opt/gcdw_workspace; ll
[root@GCDWELK01 gcdw_workspace]# mkdir backup
[root@GCDWELK01 gcdw_workspace]# mv ../gcdw_workspace.zip backup

## 1.2 上传ELK软件包
#上传ELK相应版本安装包、ES证书至至/opt/gcdw_workspace/setup目录下
[root@GCDWELK01 ~]# cd /opt/gcdw_workspace/setup; ll
[root@GCDWELK01 setup]# md5sum ELK-9.8.0.8.3_patch.8-*.tar.gz
[root@GCDWELK01 setup]# tar -xvf ELK-9.8.0.8.3_patch.8-centos8.2-x86_64.tar.gz

## 1.3 更换ES基线版本软件包
# 上传7.17.23版本的elasticsearch和kibana软件包至/opt/gcdw_workspace/setup/ELKDeploy/package目录下
[root@GCDWELK01 ~]# cd /opt/gcdw_workspace/setup/ELKDeploy; ll
[root@GCDWELK01 ELKDeploy]# cat var/variables.yml | grep -Ei "elasticsearch_tar|elastic_name|kibana_tar|kibana_name"
[root@GCDWELK01 ELKDeploy]# sed -i 's/7.17.9/7.17.23/g' var/variables.yml
[root@GCDWELK01 ELKDeploy]# cat var/variables.yml | grep -Ei "elasticsearch_tar|elastic_name|kibana_tar|kibana_name"

## 1.4 root用户互信配置
[root@GCDWELK01 ~]# yum install -y expect
[root@GCDWELK01 ~]# cd /opt/gcdw_workspace/scripts/autossh
[root@GCDWELK01 autossh]# echo -e "172.16.5.28\n172.16.5.29" > ip.list
[root@GCDWELK01 autossh]# cat ip.list
[root@GCDWELK01 autossh]# ssh-keygen
[root@GCDWELK01 autossh]# read -s -p "rootPwd: " rootPwd; echo -e "\r"; expect autossh_par.sh ip.list root $rootPwd

## 1.5 C3工具配置 (root用户执行)
[root@GCDWELK01 ~]# rpm -ivh --force /opt/gcdw_workspace/tools/c3-5.1.2-1.noarch.rpm
[root@GBASEMA01 ~]# ips=''; for ip in $(cat /opt/gcdw_workspace/scripts/autossh/ip.list); do ips=$ips$ip"\n\t"; done;
[root@GBASEMA01 ~]# clusterIP=$(cat /opt/gcdw_workspace/scripts/autossh/ip.list|head -n1)
[root@GBASEMA01 ~]# echo -e "cluster elk {\n\t${clusterIP}:${clusterIP}\n\tdead remove-index-00\n\t$ips}" >> /etc/c3.conf
[root@GCDWELK01 ~]# chmod 666 /etc/c3.conf; cat /etc/c3.conf
[root@GCDWELK01 ~]# cexec -p elk: hostname

## 1.6 环境检查及配置 (root用户执行)
# 1) 检查硬件信息
[root@GCDWELK01 ~]# cexec elk: 'dmidecode |grep -A4 "System Information"'
[root@GCDWELK01 ~]# cexec elk: 'cat /proc/cpuinfo |grep "model name"|uniq'
[root@GCDWELK01 ~]# cexec elk: 'nkvers | grep -A4 "Release"'
# 2) 操作系统版本一致
[root@GCDWELK01 ~]# cexec elk: 'cat /proc/version'
[root@GCDWELK01 ~]# cexec elk: 'cat /etc/system-release'
# 3) 检查磁盘容量一致
[root@GCDWELK01 ~]# cexec elk: 'df -h|grep opt'
# 4) 检查CPU信息一致
[root@GCDWELK01 ~]# cexec -p elk: 'cat /proc/cpuinfo |grep "model name" | head -n1'
[root@GCDWELK01 ~]# cexec -p elk: 'grep "model name" /proc/cpuinfo|wc -l'
[root@GCDWELK01 ~]# cexec -p elk: 'grep "physical id" /proc/cpuinfo|sort|uniq|wc -l'
[root@GCDWELK01 ~]# cexec -p elk: 'grep "cpu cores" /proc/cpuinfo|sort|uniq'
# 5) 检查内存容量一致
[root@GCDWELK01 ~]# cexec -p elk: 'free -g'
# 6) 检查各节点主机名
[root@GCDWELK01 ~]# cexec -p elk: 'hostname'
# 6) python版本检查（3.6及以上版本）
[root@GCDWELK01 ~]# cexec -p elk: 'python3 --version'
# 7) 检查IP和万兆网
[root@GCDWELK01 ~]# cexec elk: 'ip a|grep inet'
[root@GCDWELK01 ~]# echo "nets=\$(ip a |grep BROADCAST|awk -F: '{print \$2}'); for n in \${nets[@]}; do echo \"\$n : \$(ethtool \$n | grep Speed)\"; done" > /tmp/net_check.sh; cpush elk: /tmp/net_check.sh /tmp; cexec elk: 'sh /tmp/net_check.sh'
# 8) 检查防火墙（关闭防火墙）
[root@GCDWELK01 ~]# cexec elk: 'systemctl status firewalld.service |grep Active'
#设置: cexec elk: 'systemctl stop firewalld.service; systemctl disable firewalld.service'
# 9) 检查SELINUX关闭（disabled）
[root@GCDWELK01 ~]# cexec elk: 'cat /etc/selinux/config | grep "SELINUX=" | grep -Ev "grep|#"'
#设置: cexec elk: 'sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config'
# 10) 检查透明大页关闭（never）及IO调度策略（kyber）
# sdb为数据盘，据实调整
[root@GCDWELK01 ~]# cexec elk: 'cat /sys/kernel/mm/transparent_hugepage/enabled; cat /sys/kernel/mm/transparent_hugepage/defrag; cat /sys/block/sdb/queue/scheduler'
#设置: 临时生效
cexec elk: 'echo never > /sys/kernel/mm/transparent_hugepage/enabled; echo never > /sys/kernel/mm/transparent_hugepage/defrag; echo kyber > /sys/block/sdb/queue/scheduler'
#设置: 永久生效
echo -e "sed -i '/^GRUB_CMDLINE_LINUX/s/\\\"$/ transparent_hugepage=never elevator=kyber\\\"/' /etc/default/grub;\ngrub2-mkconfig -o /boot/efi/EFI/kylin/grub.cfg" > /tmp/huge_set.sh; cpush elk: /tmp/huge_set.sh /tmp; cexec elk: 'sh /tmp/huge_set.sh'
# 11) 检查swappiness（vm.swappiness = 1）、max_map_count（vm.max_map_count = 262144）
[root@GCDWELK01 ~]# cexec elk: 'sysctl -a | grep -E "vm.swappiness|vm.max_map_count"'
#设置: cexec elk: 'cp /etc/sysctl.conf /etc/sysctl.conf.bak; sed -i "/^vm.swappiness/d;/^vm.max_map_count/d" /etc/sysctl.conf; sed -i "\$a\\vm.swappiness = 1" /etc/sysctl.conf; sed -i "\$a\\vm.max_map_count = 262144" /etc/sysctl.conf; sysctl -p /etc/sysctl.conf'
# 12) 检查NUMA是否开启（大于1个node）
[root@GCDWELK01 ~]# cexec elk: 'lscpu|grep -i "numa"'
# 13) 检查最大打开文件数（655360）、进程数（4096）、进程锁定内存数（unlimited）
[root@GCDWELK01 ~]# cexec elk: 'cat /etc/security/limits.conf | grep -Ei "nofile|nproc|memlock" | grep -Ev "grep|#"'
#设置: cexec elk: 'cp /etc/security/limits.conf /etc/security/limits.conf.bak; echo -e "*   soft    nofile  655360\n*   hard    nofile  655360\n*   soft    nproc   unlimited\n*   hard    nproc   unlimited\n* - memlock unlimited" >> /etc/security/limits.conf'
# 14) 检查文件创建、虚拟内存最大值（unlimited）
[root@GCDWELK01 ~]# cexec elk: 'cat /etc/profile |grep ulimit'
#设置: cexec elk: 'cp /etc/profile /etc/profile.bak; echo -e "ulimit -v unlimited\nulimit -f unlimited" >> /etc/profile'
[root@GCDWELK01 ~]#

#### 二、ELK集群安装实施步骤
## 2.1 ELK软件安装
[root@GCDWELK01 ~]# cd /opt/gcdw_workspace/setup/ELKDeploy/
[root@GCDWELK01 ELKDeploy]# sh auto_elk.sh install
[root@GCDWELK01 ELKDeploy]# read -s -p "rootPwd: " rootPwd; echo -e "\r";
[root@GCDWELK01 ELKDeploy]# esIPs=''; for ip in $(cat /opt/gcdw_workspace/scripts/autossh/ip.list); do esIPs=${esIPs}"${ip} root ${rootPwd} "; done;
[root@GCDWELK01 ELKDeploy]# sh auto_elk.sh -E "${esIPs}" -k "${esIPs}" -e "elk_user=esuser elk_pass=X@gbase2024 ansible_python_interpreter=/usr/bin/python3 elastic_dest_dir=/elk/elasticsearch/ elastic_jvm_max=512m elastic_jvm_start=512m kibana_dest_dir=/elk/kibana/"
#ELK卸载: sh auto_elk.sh elk_uninstall -E "${esIPs}" -k "${esIPs}" -e "elk_user=esuser elk_pass=X@gbase2024 elastic_dest_dir=/elk/elasticsearch/ kibana_dest_dir=/elk/kibana/"

## 2.2 停止ES和kibana服务
# 停止ES服务
[root@GCDWELK01 setup]# cexec elk: 'systemctl status elasticsearch | grep Active'
[root@GCDWELK01 setup]# cexec elk: 'systemctl stop elasticsearch'
[root@GCDWELK01 setup]# cexec elk: 'systemctl status elasticsearch | grep Active'
## 停止kibana服务
[root@GCDWELK01 setup]# cexec elk: 'systemctl status kibana | grep Active'
[root@GCDWELK01 setup]# cexec elk: 'systemctl stop kibana'
[root@GCDWELK01 setup]# cexec elk: 'systemctl status kibana | grep Active'

## 2.3 ES安全认证配置
[root@GCDWELK01 ~]# cd /opt/gcdw_workspace/setup
# 证书文件以实际为准
[root@GCDWELK01 setup]# cpush elk: elastic-certificates.p12 /elk/elasticsearch/elasticsearch-7.17.23/config/
[root@GCDWELK01 setup]# cexec elk: 'cd /elk/elasticsearch/elasticsearch-7.17.23/config; chmod 755 *.p12; chown esuser:esuser *.p12; ls -l *.p12'
[root@GCDWELK01 setup]# cexec elk: 'echo -e "xpack.security.enabled: true\nxpack.security.transport.ssl.enabled: true\nxpack.security.transport.ssl.verification_mode: certificate\nxpack.security.transport.ssl.keystore.path: elastic-certificates.p12\nxpack.security.transport.ssl.truststore.path: elastic-certificates.p12\nxpack.security.transport.ssl.client_authentication: optional" >> /elk/elasticsearch/elasticsearch-7.17.23/config/elasticsearch.yml'
[root@GCDWELK01 setup]# cexec elk: 'cat /elk/elasticsearch/elasticsearch-7.17.23/config/elasticsearch.yml | grep xpack'
# 证书密码以实际为准
[root@GCDWELK01 setup]# read -s -p "esCaPwd: " esCaPwd; echo -e "\r"; 
[root@GCDWELK01 setup]# cexec elk: "cd /elk/elasticsearch/elasticsearch-7.17.23/bin; echo '${esCaPwd}' |./elasticsearch-keystore add xpack.security.transport.ssl.keystore.secure_password"
[root@GCDWELK01 setup]# cexec elk: "cd /elk/elasticsearch/elasticsearch-7.17.23/bin; echo '${esCaPwd}' |./elasticsearch-keystore add xpack.security.transport.ssl.truststore.secure_password"
[root@GCDWELK01 setup]# cexec elk: 'cd /elk/elasticsearch/elasticsearch-7.17.23/config; chmod 755 elasticsearch.keystore; chown esuser:esuser elasticsearch.keystore; ls -l elasticsearch.keystore'

## 2.4 启动ES服务
[root@GCDWELK01 setup]# cexec elk: 'systemctl status elasticsearch | grep Active'
[root@GCDWELK01 setup]# cexec elk: 'systemctl start elasticsearch'
[root@GCDWELK01 setup]# cexec elk: 'systemctl status elasticsearch | grep Active'
# ES安全认证验证，IP以实际为准
[root@GCDWELK01 setup]# clusterIP=$(cat /opt/gcdw_workspace/scripts/autossh/ip.list|head -n1)
[root@GCDWELK01 setup]# curl -X GET "http://${clusterIP}:9200/_cat/nodes"

## 2.5 配置对外访问的用户名和密码
[root@GCDWELK01 ~]# cd /elk/elasticsearch/elasticsearch-7.17.23
# 在配置文件中使用的用户有 kibana_system 和 给 filbeat创建的filebeat_internal 这两个用户
# 如果修改kibana_system用户的密码，则kibana需要修改配置文件kibana.yml并重启服务
# 如果修改filebeat_internal用户的密码，则云仓修改values.yaml并重新安装集群，或修改filebeat配置
[root@GCDWELK01 elasticsearch-7.17.23]# ./bin/elasticsearch-setup-passwords interactive
# elastic密码以实际为准
[root@GCDWELK01 setup]# read -s -p "esPwd: " esPwd; echo -e "\r"; 
# 能查看到集群所有节点信息，则证明配置成功
[root@GCDWELK01 elasticsearch-7.17.23]# curl -X GET -u elastic:${esPwd} "http://${clusterIP}:9200/_cat/nodes"

## 2.6 创建filebeat角色和用户，修改ES参数配置
[root@GCDWELK01 ~]# cd /opt/gcdw_workspace/scripts/elk
[root@GCDWELK01 elk]# clusterIP=$(cat /opt/gcdw_workspace/scripts/autossh/ip.list|head -n1); sed -i "s/^.*ClusterIP.*$/ClusterIP=${clusterIP}/g" config
# elastic密码配置到config文件
[root@GCDWELK01 elk]# sh set_esuser_pwd.sh
# 创建filebeat_writer角色
[root@GCDWELK01 elk]# sh es_addrole_filebeat.sh filebeat_writer
# 创建filebeat_internal用户
[root@GCDWELK01 elk]# read -s -p "fbPwd: " fbPwd; echo -e "\r"; sh es_adduser_filebeat.sh filebeat_internal ${fbPwd}
# 调整ES参数配置：ES主备为2主2备, 分片上限为10000, 生命周期为40G
[root@GCDWELK01 elk]# sh set_escluster_para.sh

## 2.7 配置Kibana访问用户
[root@GCDWELK01 ~]# cd /opt/gcdw_workspace/setup
[root@GCDWELK01 setup]# cexec elk: 'cat /elk/kibana/kibana-7.17.23-linux-x86_64/config/kibana.yml | grep -Ei "elasticsearch.username|elasticsearch.password" | grep -Ev "grep|#"'
[root@GCDWELK01 setup]# read -s -p "kibPwd: " kibPwd; echo -e "\r";
[root@GCDWELK01 setup]# cexec elk: 'echo -e "elasticsearch.username: \"kibana_system\"\nelasticsearch.password: \"'${kibPwd}'\"" >> /elk/kibana/kibana-7.17.23-linux-x86_64/config/kibana.yml'
[root@GCDWELK01 setup]# cexec elk: 'cat /elk/kibana/kibana-7.17.23-linux-x86_64/config/kibana.yml | grep -Ei "elasticsearch.username|elasticsearch.password" | grep -Ev "grep|#"'

## 2.8 启动Kibana服务
[root@GCDWELK01 setup]# cexec elk: 'systemctl status kibana | grep Active'
[root@GCDWELK01 setup]# cexec elk: 'systemctl start kibana'
[root@GCDWELK01 setup]# cexec elk: 'systemctl status kibana | grep Active'
# ES安全认证验证，IP以实际为准
[root@GCDWELK01 setup]# curl -X GET -u kibana_system:${kibPwd} "http://172.16.5.28:9200/_cat/nodes"
# 浏览器输入elastic:${esPwd}登录kibana开发工具页面，IP以实际为准
http://172.16.5.28:5601/app/dev_tools#/console