GCDW K8S版安装实施步骤（案例）

#### 一、准备工作
## 1.1 上传镜像至镜像仓库
[root@gcdw1 ~]# cd /opt/GCDW-k8s-9.8.0.8.3_patch.18/gcdw/images; ll
[root@gcdw1 images]# docker images
# 镜像文件加载至本地
[root@gcdw1 images]# for imageFile in gcdw*.tar saas*.tar fdb*.tar filebeat*.tar logrotate*.tar; do echo -e " \033[0;33m *** Loading ${imageFile} *** \033[0m"; docker load -i "${imageFile}"; done
[root@gcdw1 images]# docker images
# 定义镜像文件名及版本
[root@gcdw1 images]# gcdwimages=("gcdw-operator:9.8.0.8.3_patch.18" "gcdw-server:9.8.0.8.3_patch.18" "gcdw-logrotate:9.8.0.8.3_patch.18" "cloud-database-agent:9.8.0.8.3_patch.18" "clound-database-backend:9.8.0.8.3_patch.18" "clound-database-monitor:9.8.0.8.3_patch.18" "clound-database-frontend:9.8.0.8.3_patch.18" "fdb-update:1.0.1" "filebeat:7.17.23" "redis:6.2.3")
[root@gcdw1 images]# for imageInfo in ${gcdwimages[@]}; do docker images|grep $(echo ${imageInfo}|awk -F':' '{print $1}')|grep $(echo ${imageInfo}|awk -F':' '{print $2}')|grep -v grep; done
# 镜像文件标签指向仓库
[root@gcdw1 images]# for imageInfo in ${gcdwimages[@]}; do imageId=$(docker images|grep $(echo ${imageInfo}|cut -d ':' -f 1)|grep $(echo ${imageInfo}|cut -d ':' -f 2)|grep -v grep|awk '{print $3}'|uniq); targetImage="harbor.gbase.cn/gcdw/"; echo -e " \033[0;33m *** Taging ${imageInfo} ${imageId} ${targetImage}${imageInfo} *** \033[0m"; docker tag "${imageId}" "${targetImage}${imageInfo}"; done
[root@gcdw1 images]# for imageInfo in ${gcdwimages[@]}; do docker images|grep $(echo ${imageInfo}|cut -d ':' -f 1)|grep $(echo ${imageInfo}|cut -d ':' -f 2)|grep -v grep; done
# 上传镜像至镜像仓库
[root@gcdw1 images]# for imageInfo in ${gcdwimages[@]}; do targetImage="harbor.gbase.cn/gcdw/"; echo -e " \033[0;33m *** Pushing ${targetImage}${imageInfo} *** \033[0m"; docker push "${targetImage}${imageInfo}"; done

## 1.2 创建基础服务和租户命名空间（CCE操作）
[root@gcdw1 images]# kubectl get ns
# 创建基础服务组件命名空间
[root@gcdw1 images]# kubectl create ns gcdwbase
[root@gcdw1 images]# kubectl get ns
# 创建镜像仓库访问凭证（指向访问镜像仓库的SERVER、用户名、密码）
[root@gcdw1 images]# kubectl create secret --namespace gcdwbase docker-registry gcdwbase-regcred --docker-server=172.16.5.29:443 --docker-username=admin --docker-password=Harbor12345
[root@gcdw1 images]# kubectl get secret -n gcdwbase
# 创建某个租户的命名空间（若CCE云仓库不允许自建命名空间，则先在k8s创建/提供租户的命名空间后，再注册租户；租户的namespace不能包含特殊字符，租户名与此一致；若CCE云数仓允许自建租户的命名空间则设置value.yaml中autoCreateNamespace: true）
[root@gcdw1 images]# kubectl create ns tujbgcdw
[root@gcdw1 images]# kubectl get ns

## 1.3 准备软件运维脚本
## 上传gcdw_workspace.tar.gz至/opt目录下
[root@gcdw1 ~]# cd /opt
[root@gcdw1 opt]# unzip gcdw_workspace.zip
[root@gcdw1 opt]# cd /opt/gbase_workspace; ll
[root@gcdw1 gbase_workspace]# mkdir backup
[root@gcdw1 gbase_workspace]# mv ../gbase_workspace.zip backup

## 1.3 环境所需基础信息登记
# 镜像仓库地址、密钥、账户、密码
[root@gcdw1 helm_gcdw]# echo -e "registryUrl=harbor.gbase.cn/gcdw" > gcdw_confg; echo -e "registryAPI=harbor.gbase.cn/gcdw" >> gcdw_confg; echo -e "registrySecret=gcdwbase-regcred" >> gcdw_confg; echo -e "registryUser=admin" >> gcdw_confg; echo -e "registryPwd=Harbor12345" >> gcdw_confg;
# OBS对象存储EndPoint、Region、Bucket、AK、SK
[root@gcdw1 helm_gcdw]# echo -e "obsEndPoint=172.16.5.28:8000" >> gcdw_confg; echo -e "obsRegion=test" >> gcdw_confg; echo -e "obsBucket=tujbbucket" >> gcdw_confg; echo -e "obsAK=tujunbing" >> gcdw_confg; echo -e "obsSK=tujunbing" >> gcdw_confg;
# FDB元数据库IP+端口、连接串
[root@gcdw1 helm_gcdw]# echo -e "fdbUrl=172.16.5.170:4500" >> gcdw_confg; echo -e "fdbCluster=AZDPsCO3:HtfBNbvuNPujYpI3Mvz0L5DbyLK5NaRh@172.16.5.170:4500,172.16.5.171:4500" >> gcdw_confg;
# GCDW租户DBA初始密码
[root@gcdw1 helm_gcdw]# echo -e "accountInitPwd=X@gbase2024" >> gcdw_confg;
# ELK集群host及账户信息
[root@gcdw1 helm_gcdw]# echo -e "elkHosts=[\"http://172.16.5.29:9200\",\"http://172.16.5.28:9200\"]" >> gcdw_confg; echo -e "fbUser=filebeat_internal" >> gcdw_confg; echo -e "fbPwd=X@gbase2025" >> gcdw_confg;
[root@gcdw1 helm_gcdw]# cat gcdw_confg

## 1.4 外部服务连通性验证
# 镜像仓库账号验证
[root@gcdw1 helm_gcdw]# registryUrl=$(cat gcdw_confg |grep ^registryUrl |awk -F= '{print $2}' | awk '{print $1}'); registryUser=$(cat gcdw_confg |grep ^registryUser |awk -F= '{print $2}' | awk '{print $1}'); registryPwd=$(cat gcdw_confg |grep ^registryPwd |awk -F= '{print $2}' | awk '{print $1}')
[root@gcdw1 helm_gcdw]# docker login $registryUrl -u $registryUser -p $registryPwd
# 对象存储连通验证
[root@gcdw1 helm_gcdw]# obsEndPoint=$(cat gcdw_confg |grep ^obsEndPoint |awk -F= '{print $2}' | awk '{print $1}'); obsIP=$(echo $obsEndPoint|cut -d ':' -f 1); obsPort=$(echo $obsEndPoint|cut -d ':' -f 2)
[root@gcdw1 helm_gcdw]# telnet $obsIP $obsPort
# FDB元数据库验证
[root@gcdw1 helm_gcdw]# fdbUrl=$(cat gcdw_confg |grep ^fdbUrl |awk -F= '{print $2}' | awk '{print $1}'); fdbIP=$(echo $fdbUrl|cut -d ':' -f 1); fdbPort=$(echo $fdbUrl|cut -d ':' -f 2)
[root@gcdw1 helm_gcdw]# telnet $fdbIP $fdbPort

## 1.5 修改配置文件
[root@gcdw1 ~]# cd /opt/GCDW-k8s-9.8.0.8.3_patch.18/gcdw/helm/helm_gcdw
[root@gcdw1 helm_gcdw]# cp chart/values.yaml chart/values_bak.yaml
# 镜像仓库地址配置
[root@gcdw1 helm_gcdw]# registryUrl=$(cat gcdw_confg |grep ^registryUrl |awk -F= '{print $2}' | awk '{print $1}'); registrySecret=$(cat gcdw_confg |grep ^registrySecret |awk -F= '{print $2}' | awk '{print $1}'); 
[root@gcdw1 helm_gcdw]# cat -n chart/values.yaml | grep -A2 registry
[root@gcdw1 helm_gcdw]# sed -i "s|^.*registry.*$|    registry: \"${registryUrl}\"|g;s/^.*imagePullSecrets.*$/    imagePullSecrets: \"${registrySecret}\"/g" chart/values.yaml
[root@gcdw1 helm_gcdw]# cat -n chart/values.yaml | grep -A2 registry
# 镜像仓库账号配置
[root@gcdw1 helm_gcdw]# registryAPI=$(cat gcdw_confg |grep ^registryAPI |awk -F= '{print $2}' | awk '{print $1}'); registryUser=$(cat gcdw_confg |grep ^registryUser |awk -F= '{print $2}' | awk '{print $1}'); registryPwd=$(cat gcdw_confg |grep ^registryPwd |awk -F= '{print $2}' | awk '{print $1}')
[root@gcdw1 helm_gcdw]# regUserBase64=$(echo $registryUser|base64 -w 0); regPwdBase64=$(echo $registryPwd|base64 -w 0)
[root@gcdw1 helm_gcdw]# cat -n chart/values.yaml | grep -A6 multiTenantMode
[root@gcdw1 helm_gcdw]# sed -i "s|      server: \".*$|      server: \"${registryAPI}\"|g;s/      username: \".*$/      username: \"${regUserBase64}\"/g;s/      password: \".*$/      password: \"${regPwdBase64}\"/g" chart/values.yaml
[root@gcdw1 helm_gcdw]# cat -n chart/values.yaml | grep -A6 multiTenantMode
# FDB账号配置
[root@gcdw1 helm_gcdw]# fdbCluster=$(cat gcdw_confg |grep ^fdbCluster |awk -F= '{print $2}' | awk '{print $1}'); fdbBase64=$(echo $fdbCluster|base64 -w 0);
[root@gcdw1 helm_gcdw]# cat -n chart/values.yaml | grep fdb_cluster
[root@gcdw1 helm_gcdw]# sed -i "s/^.*fdb_cluster.*$/    fdb_cluster: \"${fdbBase64}\"/g" chart/values.yaml
[root@gcdw1 helm_gcdw]# cat -n chart/values.yaml | grep fdb_cluster
# 租户DBA账户密码设置
[root@gcdw1 helm_gcdw]# accountInitPwd=$(cat gcdw_confg |grep ^accountInitPwd |awk -F= '{print $2}' | awk '{print $1}'); accountPwdBase64=$(echo $accountInitPwd|base64 -w 0);
[root@gcdw1 helm_gcdw]# cat -n chart/values.yaml | grep account_init_password
[root@gcdw1 helm_gcdw]# sed -i "s/  account_init_password: \".*$/  account_init_password: \"${accountPwdBase64}\"/g" chart/values.yaml
[root@gcdw1 helm_gcdw]# cat -n chart/values.yaml | grep account_init_password
# ELK集群host及账号配置
[root@gcdw1 helm_gcdw]# elkHosts=$(cat gcdw_confg|grep ^elkHosts|awk -F= '{print $2}'|awk '{print $1}'); fbUser=$(cat gcdw_confg|grep ^fbUser|awk -F= '{print $2}' |awk '{print $1}'); fbPwd=$(cat gcdw_confg|grep ^fbPwd|awk -F= '{print $2}'|awk '{print $1}'); 
[root@gcdw1 helm_gcdw]# cat chart/values.yaml | grep filebeatImage; cat -n chart/values.yaml | grep -A2 hosts
[root@gcdw1 helm_gcdw]# sed -i "s|filebeat:7.17.9|filebeat:7.17.23|g" chart/values.yaml
[root@gcdw1 helm_gcdw]# sed -i "s|^.*hosts.*$|        hosts: ${elkHosts}\n        username: \"${fbUser}\"\n        password: \"${fbPwd}\"|g" chart/values.yaml
[root@gcdw1 helm_gcdw]# cat chart/values.yaml | grep filebeatImage; cat -n chart/values.yaml | grep -A2 hosts
# 字符集参数配置
[root@gcdw1 helm_gcdw]# cat -n chart/values.yaml | grep characterSet
[root@gcdw1 helm_gcdw]# sed -i "s/^.*characterSet.*$/  characterSet: \"utf8mb4\"/g" chart/values.yaml
[root@gcdw1 helm_gcdw]# cat -n chart/values.yaml | grep characterSet
# PV存储卷存储配置（测试环境取消storage.enabled: false; 所有服务调小requests.cpu2 memory4Gi；warehouse设置为requests.cpu4 memory16Gi）
[root@gcdw1 helm_gcdw]# 
[root@gcdw1 helm_gcdw]# 
[root@gcdw1 helm_gcdw]#

#### 二、安装步骤
## 2.1 安装GCDW基础服务组件
# 创建自定义资源
[root@gcdw1 ~]# cd /opt/GCDW-k8s-9.8.0.8.3_patch.18/gcdw/helm/helm_gcdw
[root@gcdw1 helm_gcdw]# kubectl apply -f crds
# 安装基础服务组件
[root@gcdw1 helm_gcdw]# ./bin/helm install gcdw chart -n gcdwbase
# 查看基础服务组件部署情况
[root@gcdw1 helm_gcdw]# ./bin/helm ls -n gcdwbase
# 查看GCDW组件情况（所有组件均为 running）
[root@gcdw1 helm_gcdw]# kubectl get all -n gcdwbase
[root@gcdw1 helm_gcdw]# kubectl get all -n gcdwbase -o wide
[root@gcdw1 helm_gcdw]# kubectl get pods -n gcdwbase
[root@gcdw1 helm_gcdw]# kubectl describe pods gcdw-saas-backend-0 -n gcdwbase
# 卸载命令 ./bin/helm uninstall gcdw -n gcdwbase

## 2.2 注册租户
[root@gcdw1 ~]# kubectl -it exec pod/gcdw-saas-backend-0 -n gcdwbase -- bash
[root@gcdw-saas-backend-0 backend]# ll
# 初始化 account 工具密码（使用新的fdb集群，第一次使用account的时候执行，后续就不用执行，否则会报错open tenant error, ins_name: gcdw_account, msg: no next child）
[root@gcdw-saas-backend-0 backend]# ./account --init --admin_password=X@gbase2025
[root@gcdw-saas-backend-0 backend]# ./account --show
# 注册新租户
# 若云仓库不允许自建命名空间，则先在k8s创建/提供租户的命名空间后，再注册租户；租户的namespace不能包含特殊字符，租户名与此一致；若云数仓允许自建租户的命名空间则value.yaml中autoCreateNamespace: false
[root@gcdw-saas-backend-0 backend]# ./account --create -A tujbgcdw -U gbase -P X@gbase2025 --storage_style=s3 --S3_accessKey=tujunbing --S3_secretKey=tujunbing --S3_endPoint=172.16.5.28:8000 --S3_region=test --S3_bucket=tujbbucket --admin_password=X@gbase2024
# 查看租户
[root@gcdw-saas-backend-0 backend]# ./account --show
[root@gcdw-saas-backend-0 backend]# exit
# 删除租户命令 ./account --drop --admin_password X@gbase2024 -A tujbgcdw

## 2.2 租户的gcadmin和gccli操作
[root@gcdw1 helm_gcdw]# kubectl get all -n tujbgcdw
# 进gcware pod执行gcadmin
[root@gcdw1 helm_gcdw]# kubectl -it exec pod/gcware-0 -n tujbgcdw -- bash
[gbase@gcware-0 /]$ gcadmin
[gbase@gcware-0 /]$ exit
# 进coordinator pod执行gccli
[root@gcdw1 helm_gcdw]# kubectl -it exec coordinator-1-0 -n tujbgcdw -- bash
[gbase@coordinator-1-0 /]$ gccli -ugbase -pX@gbase2025
gbase> show warehouses;
gbase> show vcs;
gbase> exit
[gbase@coordinator-1-0 /]$ exit
[root@gcdw1 helm_gcdw]#

## 2.3 k8s主机安装gccli工具
[root@gcdw1 helm_gcdw]# mkdir -p gcluster/server/bin; kubectl cp coordinator-1-0:opt/gcluster/server/bin/gccli gcluster/server/bin/gccli -n tujbgcdw
[root@gcdw1 helm_gcdw]# mkdir -p gcluster/config; kubectl cp coordinator-1-0:opt/gcluster/config/gbase_8a_gcluster.cnf gcluster/config/gbase_8a_gcluster.cnf -n tujbgcdw
[root@gcdw1 helm_gcdw]# useradd gcuser; cp -r gcluster /home/gcuser; chown -R gcuser:gcuser /home/gcuser/gcluster; chmod +x gcluster/server/bin/gccli
[root@gcdw1 helm_gcdw]# su - gcuser -c 'chmod +x /home/gcuser/gcluster/server/bin/gccli; echo -e "export GCLUSTER_BASE=\$HOME/gcluster\nexport GCLUSTER_HOME=\$GCLUSTER_BASE/server\nexport PATH=\$PATH:\$GCLUSTER_HOME/bin\nexport GCLUSTER_SID=gcluster" >> /home/gcuser/.bashrc'
[root@gcdw1 helm_gcdw]# kubectl get service -n tujbgcdw
[root@gcdw1 helm_gcdw]# cat -n tujbgcdw-nodeport.yaml
    1  kind: Service
    2  apiVersion: v1
    3  metadata:
    4    name: tujbgcdwnp
    5    namespace: tujbgcdw
    6  spec:
    7    ports:
    8    - name: coordinator
    9      protocol: TCP
   10      port: 5258
   11      targetPort: 5258
   12      nodePort: 25258
   13    selector:
   14      app: coordiantor
   15      coordinator_cr: tujbgcdw
   16    type: NodePort
   17    sessionAffinity: None
   18    externalTrafficPolicy: Cluster
[root@node8 helm_gcdw]# kubectl apply -f tujbgcdw-nodeport.yaml
[root@gcdw1 helm_gcdw]# su - gcuser -c 'source /home/gcuser/.bashrc; gccli -ugbase -=X@gbase2025 -h 172.16.5.29:23128,172.16.5.29:39903,172.16.5.29:34944 -e "select now()"'

## 2.4 外部主机安装gccli工具
[root@etl1 ~]# useradd gcuser; 
# gccli工具上传至/home/user目录下
[root@etl1 ~]# chown -R gcuser:gcuser /home/gcuser/gccli_MPP_Cluster-NoLicense-9.8.0.8.3_patch.18-redhat7-x86_64.tar.bz2
[root@etl1 ~]# su - gcuser
[gcuser@etl1 ~]$ tar xvf gccli_MPP_Cluster-NoLicense-9.8.0.8.3_patch.18-redhat7-x86_64.tar.bz2
[gcuser@etl1 ~]$ cd gccli_install; sh gccli_install.sh gccli_standalone.tar.bz2
[gcuser@etl1 ~]$ echo -e "export GCLUSTER_BASE=\$HOME/gccli_install/gcluster\nexport GCLUSTER_HOME=\$HOME/gccli_install/gcluster/server\nexport GCLUSTER_SID=gcluster\nexport LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$GCLUSTER_HOME/lib/gbase\nexport PATH=$PATH:$GCLUSTER_HOME/bin" >> /home/gcuser/.bashrc
[gcuser@etl1 ~]$ soucre /home/gcuser/.bashrc; ./gccli_install/gcluster/server/bin/gccli -ugbase -pX@gbase2025 -h 172.16.5.29:23128,172.16.5.29:39903,172.16.5.29:34944 -e "select now()"

## 2.5 创建warehouse（后台操作+唤醒）
[root@gcdw1 helm_gcdw]# kubectl -it exec coordinator-1-0 -n tujbgcdw -- bash
[gbase@coordinator-1-0 /]$ gccli -ugbase -pX@gbase2025
gbase> CREATE WAREHOUSE wh1 WITH WAREHOUSE_SIZE = 'XSMALL' MIN_CLUSTER_COUNT= 1 MAX_CLUSTER_COUNT = 3 SCALING_POLICY = 'standard' AUTO_SUSPEND =600 AUTO_RESUME = 'true' COMMENT= 'gbase cloud data warehouse';
gbase> show warehouses;
gbase> alter warehouse wh1 resume if suspended;
gbase> show warehouses;
gbase> exit
[gbase@coordinator-1-0 /]$ exit
[root@gcdw1 helm_gcdw]#
# 查看wh pod
[root@gcdw1 helm_gcdw]# kubectl get all -n tujbgcdw