GBase 8a 定时任务自动执行巡检脚本及步骤

一、部署步骤
1.1 上传GBASE巡检脚本，并授权
#上传inspection_gbase.sh脚本至/opt/gbase_workspace/scripts/inspection/目录下

[root@GBASEMA01 ~]# chown -R gbase:gbase /opt/gbase_workspace/scripts
[root@GBASEMA01 ~]# chmod 755 /opt/gbase_workspace/scripts/inspection/inspection_gbase.sh

1.2 配置巡检定时任务

[root@GBASEMA01 ~]# su - gbase
[gbase@GBASEMA01 ~]$ crontab -e
#每周三16时执行
0 16 * * 3 sh /opt/gbase_workspace/scripts/inspection/inspection_gbase.sh

1.3 检查config文件
#确认GBaseName、GBaseMode、InstanceType、UserName、Password参数与该集群匹配

[gbase@GBASEMA01 ~]$ cat /opt/gbase_workspace/scripts/config/config

1.4 验证脚本执行结果

[gbase@GBASEMA01 ~]$ sh /opt/gbase_workspace/scripts/inspection/inspection_gbase.sh
[gbase@GBASEMA01 ~]$ cd /opt/gbase_workspace/scripts/inspection/logs/inspection_gbase; ll
[gbase@GBASEMA01 ~]$ cat *.log

config内容样例如下：

GBaseName=TEST
##环境名称，每套环境一个名称。如果该环境部署多vc，且vc没有任何关联关系，设置成NULL
GBaseMode=1
##集群模式，v953多vc模式设置成1，v86版本及v95版本兼容设置成2
InstanceType=1
##集群是否多实例，是：1，否：0
UserName=check_user
##登录用户名，建议为gbase
Password=check_passwd
##登录密码

inspection_gbase.sh内容如下：

#!/bin/bash
###############################################################################
##creator : Junbing Tu
##create time: 2024-04-24
##Decription: 系统巡检（每月）
##Method: sh ​inspection_gbase.sh
##Version: 1.0 
###############################################################################

#配置文件路径
configDir=/opt/gbase_workspace/scripts/config
#脚本路径
ScriptsDir=/opt/gbase_workspace/scripts/inspection
#日志路径
logDir=/opt/gbase_workspace/scripts/inspection/logs

current_time=$(date +%F" "%T)
current_date=$(date +%F)

#获取配置信息
config()
{
    gbaseName=$(cat ${configDir}/config |grep ^GBaseName |awk -F= '{print $2}' | awk '{print $1}')
    userName=$(cat ${configDir}/config |grep ^UserName |awk -F= '{print $2}' | awk '{print $1}')
    userPasswd=$(cat ${configDir}/config |grep ^Password |awk -F= '{print $2}' | awk '{print $1}')
	#客户端命令
	gccli_command="gccli -u${userName} -p${userPasswd}"
}

#读取所有IP集合
get_iplist()
{
	source /home/gbase/.bash_profile
	if ! isCmdExist gcadmin; then echo "  Failed: execute command (gcadmin) denied to the user"; exit 1; fi
	
	gbaseMode=$(gcadmin showcluster d|grep "virtual cluster" |wc -l)
	vcList=$(gcadmin showcluster d|grep "virtual cluster" | awk -F: '{print $2}' |sed 's/,//g')
	
	gcadmin |grep coordinator[1-9] |awk '{print $4}' > ${configDir}/coor.list
	if [ ${gbaseMode} -gt 0 ];then
		> ${configDir}/data.list
		for vcname in ${vcList}
		do
    		gcadmin showcluster vc ${vcname}|grep node[1-9]|awk '{print $4}' >> ${configDir}/data.list
		done
		#gcadmin showcluster |grep FreeNode[1-9] | awk '{print $4}' >> ${configDir}/data.list
	else
		gcadmin |grep node[1-9]|awk '{print $4}' > ${configDir}/data.list
	fi
}

#集群检查
inspction_cluster()
{
	if ! isCmdExist gcadmin; then
		echo "Failed: execute command (gcadmin) denied to the user."
		exit 1;
	fi
	if ! isCmdExist $gccli_command; then
		echo "Failed: execute command (gccli) denied to the user. "
		exit 1;
	fi
	
	resultcode=0
	resultInfo="Unknown"
	inspctionMsg=""
	
	coorIp=$(gcadmin |grep "coordinator1 " |awk '{print $4}')
	gbaseMode=$(gcadmin showcluster d|grep "virtual cluster" |wc -l)
	vcList=$(gcadmin showcluster d|grep "virtual cluster" | awk -F: '{print $2}' |sed 's/,//g')
	
	if [ ! -d ${logDir}/inspection_gbase ]; then
		mkdir -p ${logDir}/inspection_gbase
	fi
	
	clusterlogName="${gbaseName}_${coorIp}_${current_date}.log"
	clusterLog="${logDir}/inspection_gbase/${clusterlogName}"
	
	echo -e "==== ${gbaseName} MPP Cluster Inscpection Begin now: $(date +%F" "%T) ====\r\n"			>> ${clusterLog}
	source /home/gbase/.bash_profile
	
	echo "*** 一、检查集群总统情况 ***  "																>> ${clusterLog}
	echo -e "1.1 集群模式："																			>> ${clusterLog}
	check_cluster_mode
	echo -e "1.2 集群状态："																			>> ${clusterLog}
	check_cluster_state
	echo -e "1.3 节点状态："																			>> ${clusterLog}
	check_node_state
	echo -e "1.4 组件状态："																			>> ${clusterLog}
	check_process_state
	echo -e "1.5 磁盘容量："																			>> ${clusterLog}
	check_cluster_disk
	
	echo "*** 二、检查集群连接情况 ***  "																>> ${clusterLog}
	echo -e "2.1 管理节点连接："																		>> ${clusterLog}
	check_cluster_connect
	echo -e "2.2 数据节点连接："																		>> ${clusterLog}
	check_node_connect
	echo -e "2.3 集群并发数："																			>> ${clusterLog}
	check_parallel
	echo -e "2.4 集群连接数："																			>> ${clusterLog}
	check_connect_count
	echo -e "2.5 超时SQL检查："																			>> ${clusterLog}
	check_slowQuery
	
	echo "*** 三、检查表数据情况 ***  "																	>> ${clusterLog}
	echo -e "3.1 表空洞率检查："																		>> ${clusterLog}
	check_hole
	echo -e "3.2 表倾斜率检查："																		>> ${clusterLog}
	check_lean

	echo "*** 四、检查结果汇总 ***  "																	>> ${clusterLog}
	if [ ${resultcode} -eq 0 ]; then
		resultInfo="Normal"
		inspctionMsg=""
	elif [ ${resultcode} -eq 1 ]; then
		resultInfo="Abnormal"
	fi
	
	echo -e "集群名称：${gbaseName} 检查时间：${current_time} 检查结果：${resultInfo} ${inspctionMsg}\r\n" >> ${clusterLog}
	
	
	echo "==== ${gbaseName} MPP Cluster Inscpection End now: $(date +%F" "%T) ===="						>> ${clusterLog}
	
}

##Cluster Mode
check_cluster_mode()
{
	#echo "  *** Check the Cluster Mode ***  "															>> ${clusterLog}
	msg=""
	
	if [ ${gbaseMode} -gt 0 ]; then
		for vcname in ${vcList}
		do
    		cluster_mode=$(gcadmin showcluster vc ${vcname}|grep "CLUSTER MODE"|awk '{print $4}')
			if [ "${cluster_mode}" != "NORMAL" ]; then 
				resultcode=1;
				inspctionMsg=${inspctionMsg}"集群${vcname}模式状态为:${cluster_mode}. ";
			fi
			msg=${msg}"${vcname} Cluster Mode: ${cluster_mode}. ""\n"
		done
	else
		dbversion=`${gccli_command} -Ns -e "select @@version"`
		if [ `echo ${dbversion} | grep '^8.6' | wc -l ` -gt 0 ]; then
			cluster_mode=$(gcadmin |grep "CLUSTER MODE"|awk '{print $3}')
		else
			cluster_mode=$(gcadmin |grep "CLUSTER MODE"|awk '{print $4}')
		fi
		
		if [ "${cluster_mode}" != "NORMAL" ]; then 
			resultcode=1;
			inspctionMsg=${inspctionMsg}"集群模式状态为:${cluster_mode}. ";
		fi
		msg=${msg}"Cluster Mode: ${cluster_mode}. ""\n"
	fi
	
	echo -e "${msg} \n"																					>> ${clusterLog}
	
	#echo $resultcode $msg
}


##Cluster State
check_cluster_state()
{
	#echo "  *** Check the Cluster State ***  "															>> ${clusterLog}
	msg=""
	
	if [ ${gbaseMode} -gt 0 ]; then
		for vcname in ${vcList}
		do
    		cluster_State=$(gcadmin showcluster vc ${vcname}|grep "CLUSTER STATE"|awk '{print $3}')
			if [ "${cluster_State}" != "ACTIVE" ]; then 
				resultcode=1;
				inspctionMsg=${inspctionMsg}"集群${vcname}运行状态为:${cluster_State}. ";
			fi
			msg=${msg}"${vcname} Cluster State: ${cluster_State}. ""\n"
		done
	else
		cluster_State=$(gcadmin |grep "CLUSTER STATE"|awk '{print $3}')
		if [ "${cluster_State}" != "ACTIVE" ]; then 
			resultcode=1;
			inspctionMsg=${inspctionMsg}"集群运行状态为:${cluster_State}. ";
		fi
		msg=${msg}"Cluster State: ${cluster_State}. ""\n"
	fi
	
	echo -e "${msg} \n"																					>> ${clusterLog}
	
	#echo $resultcode $msg
}



##Node Statuts
check_node_state()
{
	#echo "  *** Check the Node State ***  "															>> ${clusterLog}
	msg=""
	
	gcware_open=$(gcadmin |grep gcware[1-9] | grep "OPEN")
	gcware_abnl=$(gcadmin |grep gcware[1-9] | grep -v "OPEN")
	if [ "${gcware_abnl}" != "" ]; then 
		resultcode=1;
		nodeMsg=""
		for line in `echo ${gcware_abnl// /}`
		do
			nodeIp=$(echo ${line} | awk -F"|" '{print $3}')
			nodeState=$(echo ${line} | awk -F"|" '{print $4}')
			nodeMsg=${nodeMsg}" "${nodeIp}":"${nodeState}
		done
		inspctionMsg=${inspctionMsg}"集群GcWare节点状态为${nodeMsg}. ";
		gcware_abnl=${gcware_abnl}"\n";
	fi
	msg=${msg}${gcware_abnl// /}${gcware_open// /}"\n";
	
	coor_open=$(gcadmin |grep coordinator[1-9] | grep "OPEN")
	coor_abnl=$(gcadmin |grep coordinator[1-9] | grep -v "OPEN")
	if [ "${coor_abnl}" != "" ]; then 
		resultcode=1;
		nodeMsg=""
		for line in `echo ${coor_abnl// /}`
		do
			nodeIp=$(echo ${line} | awk -F"|" '{print $3}')
			nodeState=$(echo ${line} | awk -F"|" '{print $4}')
			nodeMsg=${nodeMsg}" "${nodeIp}":"${nodeState}
		done
		inspctionMsg=${inspctionMsg}"集群Coor节点状态为${nodeMsg}. ";
		coor_abnl=${coor_abnl}"\n";
	fi
	msg=${msg}${coor_abnl// /}${coor_open// /}"\n";
	
	if [ ${gbaseMode} -gt 0 ]; then
		for vcname in ${vcList}
		do
    		gnode_open=$(gcadmin showcluster vc ${vcname}|grep node[1-9] | grep "OPEN")
			gnode_abnl=$(gcadmin showcluster vc ${vcname}|grep node[1-9] | grep -v "OPEN")
			if [ "${gnode_abnl}" != "" ]; then 
				resultcode=1;
				nodeMsg=""
				for line in `echo ${gnode_abnl// /}`
				do
					nodeIp=$(echo ${line} | awk -F"|" '{print $3}')
					nodeState=$(echo ${line} | awk -F"|" '{print $5}')
					nodeMsg=${nodeMsg}" "${nodeIp}":"${nodeState}
				done
				inspctionMsg=${inspctionMsg}"${vcname} Gnode节点状态为${nodeMsg}. ";
				gnode_abnl=${gnode_abnl}"\n";
			fi
			msg=${msg}${vcname}:"\n"${gnode_abnl// /}${gnode_open// /}"\n";
		done
	else
		gnode_open=$(gcadmin showcluster|grep node[1-9] | grep "OPEN")
		gnode_abnl=$(gcadmin showcluster|grep node[1-9] | grep -v "OPEN")
		if [ "${gnode_abnl}" != "" ]; then 
			resultcode=1;
			nodeMsg=""
			for line in `echo ${gnode_abnl// /}`
			do
				nodeIp=$(echo ${line} | awk -F"|" '{print $3}')
				dbversion=`${gccli_command} -Ns -e "select @@version"`
				if [ `echo ${dbversion} | grep '^8.6' | wc -l ` -gt 0 ]; then
					nodeState=$(echo ${line} | awk -F"|" '{print $4}')
				else
					nodeState=$(echo ${line} | awk -F"|" '{print $5}')
				fi
				
				nodeMsg=${nodeMsg}" "${nodeIp}":"${nodeState}
			done
			inspctionMsg=${inspctionMsg}"集群Gnode节点状态为${nodeMsg}. ";
			gnode_abnl=${gnode_abnl}"\n";
		fi
		msg=${msg}${gnode_abnl// /}${gnode_open// /}"\n";
	fi
	
	echo -e "${msg} \n"																					>> ${clusterLog}
	
	#echo $resultcode $msg
}

##Gcluster Process Connect
check_cluster_connect()
{
	#echo "  *** Check the GCluster Connect ***  "														>> ${clusterLog}
	msg=""
	
	for cip in `cat ${configDir}/coor.list`
	do
		gcRes=$(${gccli_command} -h${cip} -P5258 -Ns --connect_timeout=100 -e "select 1")
		if [ -z ${gcRes} ]; then 
			resultcode=1;
			inspctionMsg=${inspctionMsg}"超时连接管理节点${cip}. ";
			msg=${msg}"${cip} gclusterd Connect Error.""\n";
		else
			msg=${msg}"${cip} gclusterd Connect Normal.""\n";
		fi
	done
	
	echo -e "${msg} \n"																					>> ${clusterLog}
	
	#echo $resultcode $msg
}

##Gnode Process Connect
check_node_connect()
{
	#echo "  *** Check the Gnode Connect ***  "															>> ${clusterLog}
	msg=""
	
	for dip in `cat ${configDir}/data.list`
	do
		gnRes=$(${gccli_command} -h${dip} -P5050 -Ns --connect_timeout=100 -e "select 1")
		if [ -z ${gnRes} ]; then 
			resultcode=1; 
			inspctionMsg=${inspctionMsg}"超时连接数据节点${dip}. ";
			msg=${msg}"${dip} gbased Connect Error or maybe hang.""\n";
		else
			msg=${msg}"${dip} gbased Connect Normal.""\n";
		fi
	done
	
	echo -e "${msg} \n"																					>> ${clusterLog}
	
	#echo $resultcode $msg
}

##Slow Query
check_slowQuery()
{
	#echo "  *** Check the Slow Query ***  "															>> ${clusterLog}
	msg=""
	
	for cip in `cat ${configDir}/coor.list`
	do
		gcRes=$(${gccli_command} -h${cip} -P5258  -Ns --connect_timeout=100 -e"select ID,USER,Time,INFO from information_schema.processlist where Time >=36000 and command not in ('Sleep','CTQ','GNS') and user not in ('root', 'event_scheduler')" | wc -l)
		if [ ${gcRes} -ne 0 ]; then
			resultcode=1;
			inspctionMsg=${inspctionMsg}"集群${cip}节点超时慢SQL数量为:${gcRes}. ";
			msg=${msg}"SlowQuery IP:${cip} Number:${gcRes}.""\n";
		else
			msg=${msg}"${cip} gclusterd Query Normal.""\n";
		fi
	done
	
	echo -e "${msg} \n"																					>> ${clusterLog}
	
	#echo $resultcode $msg
}

##ParallelNumber
check_parallel()
{
	#echo "  *** Check the parallelNumber ***  "														>> ${clusterLog}
	msg=""
	
	session_count_sum=0
	for cip in `cat ${configDir}/coor.list`
	do
		session_count=$(${gccli_command} -h${cip} -Ns --connect_timeout=100 -e"show processlist"|grep -ivE "sleep|command|daemon|processlist"|wc -l)
		if [ ${session_count} -gt 0 ]; then 
			let session_count_sum=${session_count}+${session_count_sum}
		fi
	done
	
	if [ ${session_count_sum} -gt 500 ]; then 
		resultcode=1;
		inspctionMsg=${inspctionMsg}"集群SQL并发数量为:${session_count_sum}. ";
	fi
	msg=${msg}"parallelNumber:${session_count_sum}.""\n";
	
	echo -e "${msg} \n"																					>> ${clusterLog}
	
	#echo $resultcode $msg
}

#SessionNumber
check_connect_count()
{
	#echo "  *** Check the Session Number ***  "														>> ${clusterLog}
	msg=""
	
	for cip in `cat ${configDir}/coor.list`
	do
		session_number=$(${gccli_command} -h${cip} -Ns --connect_timeout=100 -e"show processlist"|grep -ivE "command|daemon|processlist"|wc -l)
		if [ ${session_number} -gt 7000 ]; then 
			resultcode=1;
			inspctionMsg=${inspctionMsg}"集群${cip}节点SQL连接数量为:${session_number}. ";
		fi
		msg=${msg}"Session IP:${cip} Number:${session_number}.""\n";
	done
	
	echo -e "${msg} \n"																					>> ${clusterLog}
	
	#echo $resultcode $msg
}

#Process Status
check_process_state()
{
	#echo "  *** Check the Process State ***  "															>> ${clusterLog}
	msg=""
	
	gclusterMsg=""
	gclusterInfos=""
	for cip in `cat ${configDir}/coor.list`
	do
		gclusterInfo=$(ssh -o ConnectTimeout=15 ${cip} "gcluster_services gcluster info" 2>&1)
		gclusterInfos=${gclusterInfos}" ${cip} ${gclusterInfo}""\n";
		
		if [[ "${gclusterInfo}" != *"running"* ]]; then
			resultcode=1;
			gclusterMsg=${gclusterMsg}" "${cip}
		fi
		
	done
	msg=${msg}"gcluster: \n${gclusterInfos}";
	
	if [  -n "${gclusterMsg}" ]; then
		inspctionMsg=${inspctionMsg}"集群未启动gcluster服务的节点为:${gclusterMsg}. ";
	fi
	
	gbaseMsg=""
	gbaseInfos=""
	for dip in `cat ${configDir}/data.list`
	do
		gbaseInfo=$(ssh -o ConnectTimeout=15 ${dip} "gcluster_services gbase info" 2>&1)
		gbaseInfos=${gbaseInfos}" ${dip} ${gbaseInfo}""\n";
		
		if [[ "${gbaseInfo}" != *"running"* ]]; then
			resultcode=1;
			gbaseMsg=${gbaseMsg}" "${dip}
		fi
	done
	msg=${msg}"gbased: \n${gbaseInfos}";
	
	if [  -n "${gbaseMsg}" ]; then
		inspctionMsg=${inspctionMsg}"集群未启动gbase服务的节点为:${gbaseMsg}. ";
	fi
	
	
	echo -e "${msg} \n"																					>> ${clusterLog}
	
	#echo $resultcode $msg
}


##Cluster Disk
check_cluster_disk()
{
	#echo "  *** Check the Cluster Disk ***  "															>> ${clusterLog}
	msg=""
	
	coorDiskList=$(cexec -p coor: "df|grep /opt|grep -v grep |grep /opt"|awk '{print $2 $7}')
	cnodeMsg=""
	for cdisk in `echo ${coorDiskList}`
    do
		coorDiskNum=`echo $cdisk|awk -F[:] '{print $2}'|awk -F[%] '{print $1}'`
		if [[ -z ${coorDiskNum} ]] || [[ ${coorDiskNum} == *[!0-9]* ]]; then coorDiskNum=0; fi
		if [ ${coorDiskNum} -gt 80 ];then   
			resultcode=1;
			cnodeMsg=${cnodeMsg}${cdisk}" "
		fi
    done
	
	if [  -n "${cnodeMsg}" ]; then
		inspctionMsg=${inspctionMsg}"集群管理节点OPT磁盘使用情况:${cnodeMsg}. ";
	fi
	
	gnodeMsg=""
	nodeAllDiskList=""
	if [ ${gbaseMode} -gt 0 ]; then
		for vcname in ${vcList}
		do
    		nodeDiskList=$(cexec -p ${vcname}: "df|grep /opt|grep -v grep |grep /opt"|awk '{print $2 $7}')
			for gdisk in `echo ${nodeDiskList}`
			do
				nodeDiskNum=`echo $gdisk|awk -F[:] '{print $2}'|awk -F[%] '{print $1}'`
				if [[ -z ${nodeDiskNum} ]] || [[ ${nodeDiskNum} == *[!0-9]* ]]; then nodeDiskNum=0; fi
				if [ ${nodeDiskNum} -gt 80 ];then         
					resultcode=1;
					gnodeMsg+="${vcname}:${gdisk} "
				fi
				nodeAllDiskList+="${vcname}:${gdisk}\n"
			done
		done
	else
		nodeDiskList=$(cexec -p data: "df|grep /opt|grep -v grep |grep /opt"|awk '{print $2 $7}')
		for gdisk in `echo ${nodeDiskList}`
		do
			nodeDiskNum=`echo $gdisk|awk -F[:] '{print $2}'|awk -F[%] '{print $1}'`
			if [[ -z ${nodeDiskNum} ]] || [[ ${nodeDiskNum} == *[!0-9]* ]]; then nodeDiskNum=0; fi
			if [ ${nodeDiskNum} -gt 80 ];then         
				resultcode=1;
				gnodeMsg=${gnodeMsg}${gdisk}" "
			fi
		done
		nodeAllDiskList=${nodeDiskList}
	fi
	
	if [  -n "${gnodeMsg}" ]; then
		inspctionMsg=${inspctionMsg}"集群数据节点OPT磁盘使用情况:${gnodeMsg}. ";
	fi
	msg=${msg}"coorDiskList:\n${coorDiskList}""\n""nodeDiskList:\n${nodeAllDiskList}""\n";
	
	echo -e "${msg} \n"																					>> ${clusterLog}
	
	#echo $resultcode $msg
}


##Cluster Memory
check_cluster_memory()
{
	#echo "  *** Check the Cluster Memory ***  "														>> ${clusterLog}
	msg=""
	
	coorSwapList=$(cexec -p coor: "free -g|grep Swap"|awk '{print $2 $5}')
	cnodeMsg=""
	for cswap in `echo ${coorSwapList}`
    do
		coorSwapNum=`echo $cswap|awk -F[:] '{print $2}'`
		if [[ -z ${coorSwapNum} ]] || [[ ${coorSwapNum} == *[!0-9]* ]]; then coorSwapNum=0; fi
		if [ ${coorSwapNum} -gt 10 ];then   
			resultcode=1;
			cnodeMsg=${cnodeMsg}${cswap}" "
		fi
    done
	
	if [  -n "${cnodeMsg}" ]; then
		inspctionMsg=${inspctionMsg}"集群管理节点使用SWAP情况:${cnodeMsg}. ";
	fi
	
	gnodeMsg=""
	nodeAllSwapList=""
	if [ ${gbaseMode} -gt 0 ]; then
		for vcname in ${vcList}
		do
    		nodeSwapList=$(cexec -p ${vcname}: "free -g|grep Swap"|awk '{print $2 $5}')
			for gswap in `echo ${nodeSwapList}`
			do
				nodeSwapNum=`echo $gswap|awk -F[:] '{print $2}'`
				if [[ -z ${nodeSwapNum} ]] || [[ ${nodeSwapNum} == *[!0-9]* ]]; then nodeSwapNum=0; fi
				if [ ${nodeSwapNum} -gt 10 ];then         
					resultcode=1;
					gnodeMsg+="${vcname}:${gswap} "
				fi
				nodeAllSwapList+="${vcname}:${gswap}\n"
			done
		done
	else
		nodeSwapList=$(cexec -p data: "free -g|grep Swap"|awk '{print $2 $5}')
		for gswap in `echo ${nodeSwapList}`
		do
			nodeSwapNum=`echo $gswap|awk -F[:] '{print $2}'`
			if [[ -z ${nodeSwapNum} ]] || [[ ${nodeSwapNum} == *[!0-9]* ]]; then nodeSwapNum=0; fi
			if [ ${nodeSwapNum} -gt 10 ];then         
				resultcode=1;
				gnodeMsg=${gnodeMsg}${gswap}" "
			fi
		done
		nodeAllSwapList=${nodeSwapList}
	fi
	
	if [  -n "${gnodeMsg}" ]; then
		inspctionMsg=${inspctionMsg}"集群数据节点使用SWAP情况:${gnodeMsg}. ";
	fi
	msg=${msg}"coorSwapList:\n${coorSwapList}""\n""nodeSwapList:\n${nodeAllSwapList}""\n";
	
	echo -e "${msg} \n"																					>> ${clusterLog}
	
	#echo $resultcode $msg
}

##Maintenance Program
check_maintain_script()
{
	#echo "  *** Check the Maintenance Program ***  "													>> ${clusterLog}
	msg=""
	
	always=$(ps -ef|grep always | grep -v grep)
	if [ "" == "${always}" ]; then 
		resultcode=1;
		inspctionMsg=${inspctionMsg}"集群未运行“统计SQL记录always.sh”脚本. ";
		msg=${msg}"The program of always.sh is not running""\n";
	else
		msg=${msg}"The program of always.sh is running""\n";
	fi
	
	monitor=$(crontab -l | grep monitor.sh | grep -v "#")
	if [ "" == "${monitor}" ]; then 
		resultcode=1;
		inspctionMsg=${inspctionMsg}"集群未配置“监控告警monitor.sh”脚本. ";
		msg=${msg}"The program of monitor.sh is not configured""\n";
	else
		msg=${msg}"The program of monitor.sh has been configured""\n";
	fi
	
	rclocal=$(cexec -p all: 'cat /etc/rc.d/rc.local | grep gbase | grep -v "#"' | grep -v gbase | awk '{print $2}')
	if [ "" != "${rclocal}" ] && [ "opening" != "${rclocal}" ] ; then 
		resultcode=1;
		nodeMsg=""
		for line in `echo ${rclocal//:/}`
		do
			nodeMsg=${nodeMsg}" "${line}
		done
		inspctionMsg=${inspctionMsg}"集群未配置开机自启动:${nodeMsg}. ";
		msg=${msg}"The /etc/rc.d/rc.local is not configured in the ${nodeMsg}""\n";
	else
		msg=${msg}"The /etc/rc.d/rc.local has been configured""\n";
	fi
	
	echo -e "${msg} \n"																					>> ${clusterLog}
	
	#echo $resultcode $msg
}

	
##Table Hole
check_hole()
{	
	#echo "  *** Check the Table Hole ***  "															>> ${clusterLog}
	msg=""
	
	dbversion=`${gccli_command} -Ns -e "select @@version"`
	if [ `echo ${dbversion} | grep '^9.5' | wc -l ` -gt 0 ]; then
		tabl_holes=$($gccli_command -Ns -e "select vc_id, dbname, tbname, check_hole_time, rows, all_rows, del_rows, hole_rate from vc00001.testperfdb.datahole where check_hole_time >= now()-interval 15 day and del_rows >= 100000 and hole_rate >= 50 and old_flag = 0 order by del_rows desc, hole_rate desc limit 10" 2>&1)
	else
		tabl_holes=$($gccli_command -Ns -e "select dbname, tbname, check_hole_time, rows, all_rows, del_rows, hole_rate from testperfdb.datahole where check_hole_time >= now()-interval 15 day and del_rows >= 100000 and hole_rate >= 50 and old_flag = 0 order by del_rows desc, hole_rate desc limit 10" 2>&1) 
	fi
	
	msg=${msg}"${tabl_holes}""\n";
	
	echo -e "${msg} \n"																					>> ${clusterLog}
	
	#echo $resultcode $msg
}


##Table Lean
check_lean()
{	
	#echo "  *** Check the Table Lean ***  "															>> ${clusterLog}
	msg=""
	
	dbversion=`${gccli_command} -Ns -e "select @@version"`
	if [ `echo ${dbversion} | grep '^9.5' | wc -l ` -gt 0 ]; then
		tabl_leans=$($gccli_command -Ns -e "select vc_id, dbname, tbname, hash_column, check_lean_time, maxslicerows, minslicerows, avgrows, lean from vc00001.testperfdb.datahole where check_lean_time >= now()-interval 15 day and maxslicerows >= 100000 and lean >= 4 and old_flag = 0 order by maxslicerows desc, lean desc limit 10" 2>&1)
	else
		tabl_leans=$($gccli_command -Ns -e "select dbname, tbname, hash_column, check_lean_time, maxslicerows, minslicerows, avgrows, lean from testperfdb.datahole where check_lean_time >= now()-interval 15 day and maxslicerows >= 100000 and lean >= 4 and old_flag = 0 order by maxslicerows desc, lean desc limit 10" 2>&1)
	fi
	
	msg=${msg}"${tabl_leans}""\n";
	
	echo -e "${msg} \n"																					>> ${clusterLog}
	
	#echo $resultcode $msg
}


#校验命令是否有效
isCmdExist() {
	local cmd="$1"
  	if [ -z "$cmd" ]; then
		echo "Usage isCmdExist yourCmd"
		return 1
	fi

	which "$cmd" >/dev/null 2>&1
	if [ $? -eq 0 ]; then
		return 0
	fi

	return 2
}

#主函数
main()
{
	config
	get_iplist
	inspction_cluster
}

main $@