ZH-升级步骤-GCDW云数仓K8S版

必要前提条件：
1）提前将镜像文件上传至镜像仓库
2）停止集群监控：登录到GCDW监控页面，【连接信息配置】-【监控配置】-【监控开关】选择为【关闭】点击【完成】关闭集群监控

变更内容：9.8.0.8.3_patch.18 升级 至 9.8.0.8.3_patch.23

#### 一、准备工作
## 1.1 上传软件包及归档旧版本
# 上传GCDW-k8s-helm-9.8.0.8.3_patch.23.tar.bz2、cloud_monitor_db_update_9.8.0.8.3_patch.23.tar.gz软件包至/opt/gcdw_workspace/setup目录下
[root@GBASEMA01 ~]# cd /opt/gcdw_workspace/setup; ll
## 归档旧版本安装包
[root@GBASEMA01 setup]# cp helm_gcdw/chart/values.yaml /opt/gcdw_workspace/setup/; cp helm_gcdw/gcdw_confg /opt/gcdw_workspace/setup/;
[root@GBASEMA01 setup]# mv helm_gcdw helm_gcdw_98083p18
## 解压新软件包
[root@GBASEMA01 setup]# tar -xvf GCDW-k8s-helm-9.8.0.8.3_patch.23.tar.bz2
[root@GBASEMA01 setup]# mv helm/helm_gcdw /opt/gcdw_workspace/setup; mv helm /tmp; ll

## 1.2 集群状态和版本核查
[root@gcdw1 ~]# cd /opt/gcdw_workspace/setup/helm_gcdw
[root@node8 helm_gcdw]# baseNm="gcdwbase"; dbaUser="gbase"; dbaPwd="X@gbase2025"
[root@node8 helm_gcdw]# kubectl -it exec pod/gcdw-saas-backend-0 -c gcdw-saas-backend -n "${baseNm}" -- bash -c "cd /backend; ./account --show| grep accountname" |awk -F':' '{print $2}' > atmp; cat atmp|tr -d '\r' > accounts; cat accounts
# 集群运行状态检查正常
[root@node8 helm_gcdw]# kubectl get all -n $baseNm
[root@node8 helm_gcdw]# for nm in $(cat accounts); do echo -e " \033[0;33m *** Check status for ${nm} *** \033[0m"; kubectl get all -n ${nm}; done
# 核查各租户的版本信息
[root@node8 helm_gcdw]# for nm in $(cat accounts); do echo -e " \033[0;33m *** Check version for ${nm} *** \033[0m"; kubectl exec -it coordinator-1-0 -c coordinator -n ${nm} -- bash -c "source ~/.bash_profile; gclusterd -V; gccli -V; gccli -u$dbaUser -p$dbaPwd -e \"show variables like 'version'\""; done

## 1.3 备份基础服务组件配置
[root@node8 helm_gcdw]# mkdir -p backup/${baseNm}_old; for nm in $(cat accounts); do mkdir -p backup/${nm}_old; done
# CM配置备份
[root@node8 helm_gcdw]# cmlist=$(kubectl get cm -n ${baseNm}|grep -Ev "NAME|grep"|awk '{print $1}'); for cm in ${cmlist[@]}; do echo -e " \033[0;33m *** Backuping CM ${cm} for ${baseNm} *** \033[0m"; kubectl get cm "${cm}" -n "${baseNm}" -o yaml > backup/${baseNm}_old/${cm}.yaml; done; ll backup/*_old/*.yaml
# 租户的账户信息备份
[root@node8 helm_gcdw]# kubectl -it exec pod/gcdw-saas-backend-0 -c gcdw-saas-backend -n ${baseNm} -- bash -c "cd /backend; ./account --show" > backup/${baseNm}_old/account_old;
[root@node8 helm_gcdw]# cat backup/*_old/account_old

## 1.4 备份租户配置信息
## 租户的配置文件备份
[root@node8 helm_gcdw]# for nm in $(cat accounts); do cmlist=$(kubectl get cm -n $nm|grep -Ev "NAME|grep"|awk '{print $1}'); for cm in ${cmlist[@]}; do echo -e " \033[0;33m *** Backuping CM ${cm} for ${nm} *** \033[0m"; kubectl get cm "${cm}" -n "${nm}" -o yaml > backup/${nm}_old/${cm}.yaml; done; done;
[root@node8 helm_gcdw]# ll backup/*_old/*.yaml
## 租户的参数信息备份
[root@node8 helm_gcdw]# for nm in $(cat accounts); do kubectl exec -it coordinator-1-0 -c coordinator -n ${nm} -- bash -c "source ~/.bash_profile; gccli -u$dbaUser -p$dbaPwd -Ns -e \"show variables\"" > backup/${nm}_old/coordinator_variables_old; done
[root@node8 helm_gcdw]# cat backup/*_old/coordinator_variables_old
# 租户的用户角色备份
[root@node8 helm_gcdw]# for nm in $(cat accounts); do kubectl exec -it coordinator-1-0 -c coordinator -n ${nm} -- bash -c "source ~/.bash_profile; gccli -u$dbaUser -p$dbaPwd -s -e \"select * from cloud.user; select * from cloud.role\"" > backup/${nm}_old/user_role_old; done
[root@node8 helm_gcdw]# cat backup/*_old/user_role_old
# 租户的库表信息备份
[root@node8 helm_gcdw]# for nm in $(cat accounts); do kubectl cp tools/upgrade/get_gcdw_tables_information.sh ${nm}/coordinator-1-0:home/gbase/get_gcdw_tables_information.sh; kubectl exec -it coordinator-1-0 -c coordinator -n ${nm} -- bash -c "cd /home/gbase; chmod +x get_gcdw_tables_information.sh; sh get_gcdw_tables_information.sh $dbaUser $dbaPwd 2&>1"; done
[root@node8 helm_gcdw]# for nm in $(cat accounts); do kubectl cp ${nm}/coordinator-1-0:home/gbase/get_mpp_tables_information_$(date +%Y%m%d).log backup/${nm}_old/get_mpp_tables_information_$(date +%Y%m%d).log; done
[root@node8 helm_gcdw]# ll backup/*_old/get_mpp_tables_information*.log

## 1.5 调整values.ymal配置文件
[root@gcdw1 helm_gcdw]# cp chart/values.yaml chart/values_bak.yaml
[root@gcdw1 helm_gcdw]# diff chart/values.yaml /opt/gcdw_workspace/setup/values.yaml
[root@gcdw1 helm_gcdw]# mv /opt/gcdw_workspace/setup/gcdw_confg /opt/gcdw_workspace/setup/helm_gcdw; cat gcdw_confg
# 镜像仓库地址配置
[root@gcdw1 helm_gcdw]# registryUrl=$(cat gcdw_confg |grep ^registryUrl |awk -F= '{print $2}' | awk '{print $1}'); registrySecret=$(cat gcdw_confg |grep ^registrySecret |awk -F= '{print $2}' | awk '{print $1}'); 
[root@gcdw1 helm_gcdw]# cat -n chart/values.yaml | grep -A2 registry
[root@gcdw1 helm_gcdw]# sed -i "s|^.*registry.*$|    registry: \"${registryUrl}\"|g;s/^.*imagePullSecrets.*$/    imagePullSecrets: \"${registrySecret}\"/g" chart/values.yaml
[root@gcdw1 helm_gcdw]# cat -n chart/values.yaml | grep -A2 registry
# 镜像仓库账号配置
[root@gcdw1 helm_gcdw]# registryAPI=$(cat gcdw_confg |grep ^registryAPI |awk -F= '{print $2}' | awk '{print $1}'); registryUser=$(cat gcdw_confg |grep ^registryUser |awk -F= '{print $2}' | awk '{print $1}'); registryPwd=$(cat gcdw_confg |grep ^registryPwd |awk -F= '{print $2}' | awk '{print $1}')
[root@gcdw1 helm_gcdw]# regUserBase64=$(echo $registryUser|base64 -w 0); regPwdBase64=$(echo $registryPwd|base64 -w 0)
[root@gcdw1 helm_gcdw]# cat -n chart/values.yaml | grep -A6 multiTenantMode
[root@gcdw1 helm_gcdw]# sed -i "s|      server: \".*$|      server: \"${registryAPI}\"|g;s/      username: \".*$/      username: \"${regUserBase64}\"/g;s/      password: \".*$/      password: \"${regPwdBase64}\"/g" chart/values.yaml
[root@gcdw1 helm_gcdw]# cat -n chart/values.yaml | grep -A6 multiTenantMode
# FDB账号配置
[root@gcdw1 helm_gcdw]# fdbCluster=$(cat gcdw_confg |grep ^fdbCluster |awk -F= '{print $2}' | awk '{print $1}'); fdbBase64=$(echo $fdbCluster|base64 -w 0);
[root@gcdw1 helm_gcdw]# cat -n chart/values.yaml | grep fdb_cluster
[root@gcdw1 helm_gcdw]# sed -i "s/^.*fdb_cluster.*$/    fdb_cluster: \"${fdbBase64}\"/g" chart/values.yaml
[root@gcdw1 helm_gcdw]# cat -n chart/values.yaml | grep fdb_cluster
# 租户DBA账户密码设置
[root@gcdw1 helm_gcdw]# accountInitPwd=$(cat gcdw_confg |grep ^accountInitPwd |awk -F= '{print $2}' | awk '{print $1}'); accountPwdBase64=$(echo $accountInitPwd|base64 -w 0);
[root@gcdw1 helm_gcdw]# cat -n chart/values.yaml | grep account_init_password
[root@gcdw1 helm_gcdw]# sed -i "s/  account_init_password: \".*$/  account_init_password: \"${accountPwdBase64}\"/g" chart/values.yaml
[root@gcdw1 helm_gcdw]# cat -n chart/values.yaml | grep account_init_password
# ELK集群host配置
[root@gcdw1 helm_gcdw]# elkHosts=$(cat gcdw_confg|grep ^elkHosts|awk -F= '{print $2}'|awk '{print $1}'); fbUser=$(cat gcdw_confg|grep ^fbUser|awk -F= '{print $2}' |awk '{print $1}'); fbPwd=$(cat gcdw_confg|grep ^fbPwd|awk -F= '{print $2}'|awk '{print $1}');
[root@gcdw1 helm_gcdw]# cat chart/values.yaml | grep filebeatImage; cat -n chart/values.yaml | grep -A2 hosts
[root@gcdw1 helm_gcdw]# sed -i "s|filebeat:7.17.9|filebeat:7.17.23|g" chart/values.yaml
[root@gcdw1 helm_gcdw]# sed -i "s|^.*hosts.*$|        hosts: ${elkHosts}\n        username: \"${fbUser}\"\n        password: \"${fbPwd}\"|g" chart/values.yaml
[root@gcdw1 helm_gcdw]# cat chart/values.yaml | grep filebeatImage; cat -n chart/values.yaml | grep -A2 hosts
# 字符集参数配置
[root@gcdw1 helm_gcdw]# cat -n chart/values.yaml | grep characterSet
[root@gcdw1 helm_gcdw]# sed -i "s/^.*characterSet.*$/  characterSet: \"utf8mb4\"/g" chart/values.yaml
[root@gcdw1 helm_gcdw]# cat -n chart/values.yaml | grep characterSet
# PV存储卷存储配置（测试环境取消）
[root@gcdw1 helm_gcdw]# cat -n chart/values.yaml | grep enabled
[root@gcdw1 helm_gcdw]# sed -i "s/^.*characterSet.*$/  characterSet: \"utf8mb4\"/g" chart/values.yaml
[root@gcdw1 helm_gcdw]# cat -n chart/values.yaml | grep characterSet
... ...

#### 二、升级工作
## 2.1 检查租户是否已经停业
[root@gcdw1 helm_gcdw]# sh /opt/gcdw_workspace/scripts/k8s/showprocesslist.sh c all
[root@gcdw1 helm_gcdw]# sh /opt/gcdw_workspace/scripts/k8s/showprocesslist.sh n all

## 2.2 关闭租户对外服务
# 检查warehouse资源是否已全部停止（删除/挂起）
[root@gcdw1 helm_gcdw]# for nm in $(cat accounts); do echo -e " \033[0;33m *** Check warehouse for ${nm}: *** \033[0m";  kubectl get warehouse -n ${nm}; done
# 手动删除warehouse资源
[root@gcdw1 helm_gcdw]# for nm in $(cat accounts); do whs=$(kubectl get warehouse -n ${nm} 2>&1 |grep -Ev "NAME|No resources found" |awk '{print $1}'); for wh in ${whs[@]}; do kubectl delete warehouse ${wh} -n ${nm}; done; done
[root@gcdw1 helm_gcdw]# for nm in $(cat accounts); do echo -e " \033[0;33m *** Check warehouse for ${nm}: *** \033[0m";  kubectl get warehouse -n ${nm}; done
# 检查coodinator资源是否已停止（删除）
[root@gcdw1 helm_gcdw]# for nm in $(cat accounts); do echo -e " \033[0;33m *** Check coordinator for ${nm}: *** \033[0m";  kubectl get coordinator -n ${nm}; done
# 手动删除coodinator资源
[root@gcdw1 helm_gcdw]# for nm in $(cat accounts); do cos=$(kubectl get coordinator -n ${nm} 2>&1 |grep -Ev "NAME|No resources found" |awk '{print $1}'); for co in ${cos[@]}; do kubectl delete coordinator ${co} -n ${nm}; done; done
[root@gcdw1 helm_gcdw]# for nm in $(cat accounts); do echo -e " \033[0;33m *** Check coordinator for ${nm}: *** \033[0m";  kubectl get coordinator -n ${nm}; done

## 2.3 升级GCDW监控库(8a单机库)
[root@gcdw1 ~]# cd /opt/gcdw_workspace/setup; ll
[root@gcdw1 setup]# tar -xvf cloud_monitor_db_update_9.8.0.8.3_patch.23.tar.gz -C /home/gbase/
[root@gcdw1 setup]# chown -R gbase:gbase /home/gbase/cloud_monitor_db_update_9.8.0.8.3_patch.23
[root@gcdw1 setup]# su - gbase
[gbase@gcdw1 ~]# cd /home/gbase/cloud_monitor_db_update_9.8.0.8.3_patch.23/; ll
# 参数说明：当前版本编号 更新版本编号 DBA用户 DBA密码 名称 IP 端口
[gbase@gcdw1 ~]# source ~/.bashrc; sh upgrade.sh 98080318 98080323 gbase X@gbase2025 monitor 127.0.0.1 5259
# 回退命令 sh rollback.sh 98080323 98080318 gbase X@gbase2025 monitor 127.0.0.1 5259

## 2.4 执行升级操作
[root@gcdw1 helm_gcdw]# cd /opt/gcdw_workspace/setup/helm_gcdw/tools/upgrade/; ll
[root@node8 upgrade]# sh upgrade_gcdw_k8s2.sh ${baseNm}

## 2.5 集群状态和版本确认
# 集群运行状态检查正常
[root@node8 ~]# cd /opt/gcdw_workspace/setup/helm_gcdw
[root@node8 helm_gcdw]# kubectl get all -n $baseNm
[root@node8 helm_gcdw]# for nm in $(cat accounts); do echo -e " \033[0;33m *** Check status for ${nm} *** \033[0m"; kubectl get all -n ${nm}; done
# 核查各租户的版本信息
[root@node8 helm_gcdw]# for nm in $(cat accounts); do echo -e " \033[0;33m *** Check version for ${nm} *** \033[0m"; kubectl exec -it coordinator-1-0 -c coordinator -n ${nm} -- bash -c "source ~/.bash_profile; gclusterd -V; gccli -V; gccli -u$dbaUser -p$dbaPwd -e \"show variables like 'version'\""; done

## 2.6 核对基础组件配置文件
[root@node8 ~]# cd /opt/gcdw_workspace/setup/helm_gcdw
[root@node8 helm_gcdw]# mkdir -p backup/${baseNm}_new;
# 记录升级后的基础组件配置文件
[root@node8 helm_gcdw]# cmlist=$(kubectl get cm -n ${baseNm}|grep -Ev "NAME|grep"|awk '{print $1}'); for cm in ${cmlist[@]}; do echo -e " \033[0;33m *** Backuping CM ${cm} for ${baseNm} *** \033[0m"; kubectl get cm "${cm}" -n "${baseNm}" -o yaml > backup/${baseNm}_new/${cm}.yaml; done; ll backup/*_new/*.yaml
# gcdw-coordinator-default-config
[root@node8 helm_gcdw]# diff backup/${baseNm}_old/gcdw-coordinator-default-config.yaml backup/${baseNm}_new/gcdw-coordinator-default-config.yaml
[root@node8 helm_gcdw]# kubectl delete cm gcdw-coordinator-default-config -n ${baseNm}; kubectl apply -f backup/${baseNm}_old/gcdw-coordinator-default-config.yaml -n ${baseNm}; kubectl get cm -n ${baseNm}
# gcdw-warehouse-default-config
[root@node8 helm_gcdw]# diff backup/${baseNm}_old/gcdw-warehouse-default-config.yaml backup/${baseNm}_new/gcdw-warehouse-default-config.yaml
[root@node8 helm_gcdw]# kubectl delete cm gcdw-warehouse-default-config -n ${baseNm}; kubectl apply -f backup/${baseNm}_old/gcdw-warehouse-default-config.yaml -n ${baseNm}; kubectl get cm -n ${baseNm}
# 重启POD生效参数
[root@node8 helm_gcdw]# kubectl get pods -n ${baseNm}
[root@node8 helm_gcdw]# podlist=$(kubectl get pods -n ${baseNm}|grep -Ev "NAME|grep"|awk '{print $1}'); for pod in ${podlist[@]}; do echo -e " \033[0;33m *** Restarting POD ${pod} for ${nm} *** \033[0m"; kubectl delete pod "${pod}" -n ${nm}; done;
[root@node8 helm_gcdw]# kubectl get pods -n ${baseNm}
[root@node8 helm_gcdw]# kubectl get all -n ${baseNm}

## 2.7 核对租户配置文件
[root@node8 ~]# cd /opt/gcdw_workspace/setup/helm_gcdw
[root@node8 helm_gcdw]# for nm in $(cat accounts); do mkdir -p backup/${nm}_new; done
# 记录升级后的租户配置文件
[root@node8 helm_gcdw]# for nm in $(cat accounts); do cmlist=$(kubectl get cm -n $nm|grep -Ev "NAME|grep"|awk '{print $1}'); for cm in ${cmlist[@]}; do echo -e " \033[0;33m *** Backuping CM ${cm} for ${nm} *** \033[0m"; kubectl get cm "${cm}" -n "${nm}" -o yaml > backup/${nm}_new/${cm}.yaml; done; done; ll backup/*_new/*.yaml
# gcdw-coordinator-default-config
[root@node8 helm_gcdw]# for nm in $(cat accounts); do echo -e " \033[0;33m *** Diff CM gcdw-coordinator-default-config for ${nm} *** \033[0m"; diff backup/${nm}_old/gcdw-coordinator-default-config.yaml backup/${nm}_new/gcdw-coordinator-default-config.yaml; done
[root@node8 helm_gcdw]# vim backup/${nm}_new/gcdw-coordinator-default-config.yaml
[root@node8 helm_gcdw]# kubectl delete cm gcdw-coordinator-default-config -n ${baseNm}; kubectl apply -f backup/${baseNm}_new/gcdw-coordinator-default-config.yaml -n ${baseNm}; kubectl get cm -n ${baseNm}
# gcdw-warehouse-default-config
[root@node8 helm_gcdw]# for nm in $(cat accounts); do echo -e " \033[0;33m *** Diff CM gcdw-warehouse-default-config for ${nm} *** \033[0m"; diff backup/${nm}_old/gcdw-warehouse-default-config.yaml backup/${nm}_new/gcdw-coordinator-default-config.yaml; done
[root@node8 helm_gcdw]# vim backup/${nm}_new/gcdw-warehouse-default-config.yaml
[root@node8 helm_gcdw]# kubectl delete cm gcdw-warehouse-default-config -n ${baseNm}; kubectl apply -f backup/${baseNm}_new/gcdw-warehouse-default-config.yaml -n ${baseNm}; kubectl get cm -n ${baseNm}
# 重启POD生效参数
[root@node8 helm_gcdw]# kubectl get pods -n ${baseNm}
[root@node8 helm_gcdw]# podlist=$(kubectl get pods -n ${baseNm}|grep -Ev "NAME|grep"|awk '{print $1}'); for pod in ${podlist[@]}; do echo -e " \033[0;33m *** Restarting POD ${pod} for ${nm} *** \033[0m"; kubectl delete pod "${pod}" -n ${nm}; done;
[root@node8 helm_gcdw]# kubectl get pods -n ${baseNm}
[root@node8 helm_gcdw]# kubectl get all -n ${baseNm}

## 2.8 启动所有租户的coor和warehouse资源(upgrade_gcdw_k8s2.sh脚本已支持自动拉起，无需重复拉起)
[root@node8 helm_gcdw]# for nm in $(cat accounts); do echo -e " \033[0;33m *** Starting resource for all tenant in ${nm} *** \033[0m"; kubectl -it exec pod/gcdw-saas-backend-0 -c gcdw-saas-backend -n ${baseNm} -- bash -c "cd /backend; ./account --create_resource -A ${nm}"; done
[root@node8 helm_gcdw]# for nm in $(cat accounts); do echo -e " \033[0;33m *** Check the status for ${nm} *** \033[0m"; kubectl get all -n ${nm}; done

## 2.9 核对租户参数、角色信息
## 租户的参数信息核对
[root@node8 helm_gcdw]# for nm in $(cat accounts); do kubectl exec -it coordinator-1-0 -c coordinator -n ${nm} -- bash -c "source ~/.bash_profile; gccli -u$dbaUser -p$dbaPwd -Ns -e \"show variables\"" > backup/${nm}_new/coordinator_variables_new; done
[root@node8 helm_gcdw]# for nm in $(cat accounts); do echo -e " \033[0;33m *** Diff coordinator variables for ${nm} *** \033[0m"; diff backup/${nm}_old/coordinator_variables_old backup/${nm}_new/coordinator_variables_new; done
# 租户的用户角色核对
[root@node8 helm_gcdw]# for nm in $(cat accounts); do kubectl exec -it coordinator-1-0 -c coordinator -n ${nm} -- bash -c "source ~/.bash_profile; gccli -u$dbaUser -p$dbaPwd -s -e \"select * from cloud.user; select * from cloud.role\"" > backup/${nm}_new/user_role_new; done
[root@node8 helm_gcdw]# for nm in $(cat accounts); do echo -e " \033[0;33m *** Diff user_role for ${nm} *** \033[0m"; diff backup/${nm}_old/user_role_old backup/${nm}_new/user_role_new; done

## 2.10 归档升级文件
[root@node8 helm_gcdw]# mv backup upgrade_backup$(date +%Y%m%d); mv upgrade_backup$(date +%Y%m%d) /opt/gcdw_workspace/backup

## 2.11 开启集群监控
# 登录到GCDW监控页面（切记勿勾选initDB页面中“是否初始话数据库”框），【连接信息配置】-【监控配置】-【监控开关】选择为【开启】点击【完成】开启集群监控