GBase 8a
适配迁移
文章
基于Rtsync工具从Goldendb迁移数据到Gbase8a
发表于2024-12-31 21:48:43104次浏览0个评论
软件列表
jdk-8u102-linux-x64.tar.gz
kafka_2.11-0.10.2.1.tgz
zookeeper-3.4.9.tar.gz
GBase_RTSync_8.6.26.2_8a.tar.gz
jdk、kafka、zookeeper的安装包可以自行安装，也可以使用RTSync包中自带的安装包
RTSync各个组件安装
cd /usr/local/hadoop/kafka/bin
# 启动zk
./zookeeper-server-start.sh -daemon ../conf/zookeeper.properties
# 创建topic
./kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1
--partitions 1 --topic topic01
# 查看topic列表
./kafka-topics.sh --list --zookeeper localhost:2181
# 详细信息
./kafka-topics.sh --describe --zookeeper localhost:2181RTSync各个组件启动
# 启动zk
cd /usr/local/hadoop/kafka && ./bin/zookeeper-server-start.sh -daemon
./config/zookeeper.properties
# 启动kafka
cd /usr/local/hadoop/kafka && ./bin/kafka-server-start.sh -daemon
./config/server.properties
RTSync配置文件（单分片集群为例）
1.config_kafka_mysqlto8a.properties
#modify kafka topic inFri Oct 29 11:33:00 CST 2021
#Fri Oct 29 11:33:00 CST 2021
kafka.producer.paramers=request.timeout.ms=600000;metadata.fetch.timeout.ms=60000;linger.ms=10;
kafka.send.issuccess=true
kafka.consumer.paramers=session.timeout.ms=10000;request.timeout.ms=60000;enable.auto.commit=false;max.poll.records=100;max.partition.fetch.bytes=909200000
zookeeper.session.timeout.ms=8000
kafka.resend.max.retries=3
topic.replication.num=1
send.data.max.size=30485760
kafka.batch.commit.time=300
fetch.message.max.bytes=104857600
group.id=demo211209
zookeeper.sync.time.ms=4000
bootstrap.servers=192.168.199.131:9092 -- 修改kafka ip和端口信息
auto.commit.offset.enable=false
kafka.acks=all
kafka.batch.commit.count=1000
topic.enable.auto.create=true
topic.name=mysqlto8a -- 修改topic的名字
auto.commit.interval.ms=5000
zookeeper.connect=192.168.199.131:2181 -- 修改zookeeper ip和端口信息2.config_task.xml
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<servers>
<!-- 同步服务节点配置信息 -->
<server syncMode="increment"
dataFormatType="PUREDATA" -- 指定同步类型和数据格式，Json类型有些不支持
id="mysqlto8a"
mqType="kafka"
queueName="mysqlto8a"
isHighAvailable="false"
dataRecoveryMode="file">
<!-- manager 同步工具管理组件的配置信息 -->
<manager ip="192.168.199.131" -- RTSync服务所在ip
port="9432"
heartbeatPort="9000"
heartbeatTimeOut="500"
httpPort="8087"
isValidateMetadata="false"
useErrorDataRecovery="false"
isTableHotPatch="true"/>
<!-- source 同步工具读模块配置信息 -->
<source ip="192.168.199.131"
user="root" -- RTSync服务器的用户密码
password="111111"
dataFormatParallel="1"
dbObjToUpperCase="true"
isConvertSingleQuote="false"
monitorInterval="100"
openMonitor="true"
path="/opt/RTSync_8.6.26.0_8a" -- RTSync包路径
queuePollTimeOut="20"
queueSize="100"
readParseAdapter="adapter"
rpcPort="9191"/>
<!-- mappings 读写端数据库表映射关系 -->
<mappings>
<!-- source-target 映射实例 -->
<source-target groupName="gn1" id="mysqlto8a1">
<!-- db 读写端数据库相关信息 -->
<db>
<!-- sourcedb 源端(读端)数据库配置信息 -->
<sourcedb driver="com.mysql.jdbc.Driver"
url="jdbc:mysql://192.168.199.129:6676/sk_test?user=rtsync&password=rtsync&useUnicode=true&characterEncoding=utf8&characterSetResults=utf8&useSSL=false" -- Goldendb连接信息
type="MYSQL"
user="rtsync"
password="rtsync"
port="3306"
host="192.168.199.129"
subType="mysql"
sshUser="gbase"
sshPassword="gbase"
sshPort="22"
charset="UTF8"
columnTypeFormat="time=hh:mm:ss.SSSSSSS;date=yyyy-MM-dd;datetime=yyyy-MM-dd HH:mm:ss;smalldatetime=yyyy-MM-dd HH:mm:ss.SSSSSS;datetime2=yyyy-MM-dd HH:mm:ss.SSSSSS"
allowPrimaryKeyNull="true"
catalog="sk_test"
delayTimeThreshold="0"
dyntal="true"
fetchSize="100"
isGetDBspace="true"
isMineWholeTransaction="fasle"
isOverDelayTimeThresholdExit="false"
isParallelForMysql="true"
isUseMinerCache="true"
longTxCheckIntervalTime="10"
longTxMaxWaitTime="60"
maxRecordsPerRead="200"
maxSizeOfPerRecord="1024"
mysqlSlaveId="00"
operationType="dml"
packetMaxRecord="2"
parallel="4"
startLSN="0"
timeOut="2"
timestampWithFraction="true"
transMaxCount="10000"
useAddFile="false"/>
<targetdb charset="UTF8"
type="GCLUSTER"
commitSize="100"
queueSize="20000"
timeOut="30"
user="gbase"
password="gbase20110531"
driver="com.gbase.jdbc.Driver"
catalog="test"
url="jdbc:gbase://192.168.199.130:5258/test?useOldAliasMetadataBehavior=true&rewriteBatchedStatements=true&connectTimeout=0&socketTimeout=0&vcName=vcname000001"/> --GBase 8a连接信息
<!-- tables 读写端表映射关系 -->
<tables isInclude="TRUE">
<table deleteMode="NORMAL"
dyncol="true"
sourcePkColName="id" -- 按表同步就写，按库同步就为空
sourceTableName="demo"
targetPkColName=""
targetTableName="demo">
</table>
</tables>
</db>
<full tableParallel="3" isSupportLO="true" memorySize="500" parameter="fileName=config_full_increment_mysql.properties">
<!--dataTransfer 数据传输方式，目前支持:local，即使用jdbc进行导出导入 ;remote:仅支持oracleto8a-->
<dataTransfer>
<exportTransferor type="remote" />
<importTransferor type="remote" />
</dataTransfer>
<!-- dbExport相关参数,path oracle导出用，设置数据文件路径 -->
<dbExport parameter="" threadNum="10" path="/var/www/html/"/>
<dbImport parameter="maxErrorCount=100;batchSize=500"
threadNum="10" path="http://192.168.199.131"/> -- RTSync服务所在ip
</full>
</source-target>
</mappings>
</server>
</servers>
启动RTSync
sh /opt/RTSync/RTSyncManagerServer.sh start该报错可忽略
修改gbase8a gcluser和gnode参数
gcluster
gcluster_kafka_consumer_enable=1 # 必须参数，不加报错leader thread not started
gcluster_lock_level=10
_gbase_transaction_disable=1 #必须参数
_gcluster_insert_cache_buffer_flag=1
gcluster_assign_kafka_topic_period=20
#gcluster_kafka_max_message_size=10000 #此参数服务器内存较小时，报错can not connect to kafka
gcluster_kafka_batch_commit_dml_count=1000
gcluster_kafka_local_queue_size=2100
gcluster_kafka_user_allowed_max_latency=20000
gcluster_kafka_consume_batch=100
gcluster_kafka_primarykey_can_be_null=1
gcluster_kafka_data_buf_size=20
gcluster_kafka_message_format_type=PUREDATA #必须参数
gnode
_gbase_transaction_disable=1 #必须参数为1
_gbase_kafka_transaction_mode=1 #必须参数为1
gbase_buffer_insert=256M
gbase_tx_log_flush_time=5创建kafka consumer，启动kafka consumer即可正常消费
注意事项
（1）不支持连接goldendb的转发ip，需要连接数据节点的主节点
（2）多分片集群 复制表只需连接其中任意一个主数据节点，hash表需要连接所有分片的主数据节点，配置文件如下
config_task.xml
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<servers>
<!-- 同步服务节点配置信息 -->
<server syncMode="increment"
dataFormatType="PUREDATA"
id="mysqlto8a"
mqType="kafka"
queueName="mysqlto8a"
isHighAvailable="false"
dataRecoveryMode="file">
<!-- manager 同步工具管理组件的配置信息 -->
<manager ip="192.168.217.140"
port="9432"
heartbeatPort="9000"
heartbeatTimeOut="500"
httpPort="8087"
isValidateMetadata="false"
useErrorDataRecovery="false"
isTableHotPatch="true"/>
<!-- source 同步工具读模块配置信息 -->
<source ip="192.168.217.140"
user="gbase"
password="gbase123"
dataFormatParallel="1"
dbObjToUpperCase="true"
isConvertSingleQuote="false"
monitorInterval="100"
openMonitor="true"
path="/opt/GBase_RTSync_8.6.26.2_8a/RTSync_8.6.26.2_8a"
queuePollTimeOut="20"
queueSize="100"
readParseAdapter="adapter"
rpcPort="9191"/>
<!-- mappings 读写端数据库表映射关系 -->
<mappings>
<!-- source-target 映射实例 -->
<source-target groupName="gn1" id="mysqlto8a1">
<!-- db 读写端数据库相关信息 -->
<db>
<!-- sourcedb 源端(读端)数据库配置信息 -->
<sourcedb driver="com.mysql.jdbc.Driver"
url="jdbc:mysql://192.168.217.141:3306/testdb?user=root&password=Jrf200888@&useUnicode=true&characterEncoding=utf8&characterSetResults=utf8&useSSL=false" -- 分片1节点
type="MYSQL"
user="root"
password="Jrf200888@"
port="3306"
host="192.168.217.141"
subType="mysql"
sshUser="root"
sshPassword="123456"
sshPort="22"
charset="UTF8"
columnTypeFormat="time=hh:mm:ss.SSSSSSS;date=yyyy-MM-dd;datetime=yyyy-MM-dd HH:mm:ss;smalldatetime=yyyy-MM-dd HH:mm:ss.SSSSSS;datetime2=yyyy-MM-dd HH:mm:ss.SSSSSS"
allowPrimaryKeyNull="true"
catalog="testdb"
delayTimeThreshold="0"
dyntal="true"
fetchSize="100"
isGetDBspace="true"
isMineWholeTransaction="fasle"
isOverDelayTimeThresholdExit="false"
isParallelForMysql="true"
isUseMinerCache="true"
longTxCheckIntervalTime="10"
longTxMaxWaitTime="60"
maxRecordsPerRead="200"
maxSizeOfPerRecord="1024"
mysqlSlaveId="00"
operationType="dml"
packetMaxRecord="2"
parallel="4"
startLSN="0"
timeOut="2"
timestampWithFraction="true"
transMaxCount="10000"
useAddFile="false"/>
<targetdb charset="UTF8"
type="GCLUSTER"
commitSize="100"
queueSize="20000"
timeOut="30"
user="gbase"
password="gbase20110531"
driver="com.gbase.jdbc.Driver"
catalog="testdb"
url="jdbc:gbase://192.168.217.129:5258/testdb?useOldAliasMetadataBehavior=true&rewriteBatchedStatements=true&connectTimeout=0&socketTimeout=0"/>
<!-- tables 读写端表映射关系 -->
<tables isInclude="TRUE">
<table deleteMode="NORMAL"
dyncol="true"
sourcePkColName="id"
sourceTableName="test"
targetPkColName=""
targetTableName="test">
</table>
</tables>
</db>
<full tableParallel="3" isSupportLO="true" memorySize="500" parameter="fileName=config_full_increment_mysql.properties">
<!--dataTransfer 数据传输方式，目前支持:local，即使用jdbc进行导出导入 ;remote:仅支持oracleto8a-->
<dataTransfer>
<exportTransferor type="remote" />
<importTransferor type="remote" />
</dataTransfer>
<!-- dbExport相关参数,path oracle导出用，设置数据文件路径 -->
<dbExport parameter="" threadNum="10" path="/var/www/html/"/>
<dbImport parameter="maxErrorCount=100;batchSize=500"
threadNum="10" path="http://192.168.217.140"/>
</full>
</source-target>
<!-- source-target 映射实例 -->
<source-target groupName="gn2" id="mysqlto8a2">
<!-- db 读写端数据库相关信息 -->
<db>
<!-- sourcedb 源端(读端)数据库配置信息 -->
<sourcedb driver="com.mysql.jdbc.Driver"
url="jdbc:mysql://192.168.217.142:3306/testdb?user=root&password=Jrf200888@&useUnicode=true&characterEncoding=utf8&characterSetResults=utf8&useSSL=false" -- 分片2节点
type="MYSQL"
user="root"
password="Jrf200888@"
port="3306"
host="192.168.217.142"
subType="mysql"
sshUser="root"
sshPassword="123456"
sshPort="22"
charset="UTF8"
columnTypeFormat="time=hh:mm:ss.SSSSSSS;date=yyyy-MM-dd;datetime=yyyy-MM-dd HH:mm:ss;smalldatetime=yyyy-MM-dd HH:mm:ss.SSSSSS;datetime2=yyyy-MM-dd HH:mm:ss.SSSSSS"
allowPrimaryKeyNull="true"
catalog="testdb"
delayTimeThreshold="0"
dyntal="true"
fetchSize="100"
isGetDBspace="true"
isMineWholeTransaction="fasle"
isOverDelayTimeThresholdExit="false"
isParallelForMysql="true"
isUseMinerCache="true"
longTxCheckIntervalTime="10"
longTxMaxWaitTime="60"
maxRecordsPerRead="200"
maxSizeOfPerRecord="1024"
mysqlSlaveId="00"
operationType="dml"
packetMaxRecord="2"
parallel="4"
startLSN="0"
timeOut="2"
timestampWithFraction="true"
transMaxCount="10000"
useAddFile="false"/>
<targetdb charset="UTF8"
type="GCLUSTER"
commitSize="100"
queueSize="20000"
timeOut="30"
user="gbase"
password="gbase20110531"
driver="com.gbase.jdbc.Driver"
catalog="testdb"
url="jdbc:gbase://192.168.217.129:5258/testdb?useOldAliasMetadataBehavior=true&rewriteBatchedStatements=true&connectTimeout=0&socketTimeout=0"/>
<!-- tables 读写端表映射关系 -->
<tables isInclude="TRUE">
<table deleteMode="NORMAL"
dyncol="true"
sourcePkColName="id"
sourceTableName="test"
targetPkColName=""
targetTableName="test">
</table>
</tables>
</db>
<full tableParallel="3" isSupportLO="true" memorySize="500" parameter="fileName=config_full_increment_mysql.properties">
<!--dataTransfer 数据传输方式，目前支持:local，即使用jdbc进行导出导入 ;remote:仅支持oracleto8a-->
<dataTransfer>
<exportTransferor type="remote" />
<importTransferor type="remote" />
</dataTransfer>
<!-- dbExport相关参数,path oracle导出用，设置数据文件路径 -->
<dbExport parameter="" threadNum="10" path="/var/www/html/"/>
<dbImport parameter="maxErrorCount=100;batchSize=500"
threadNum="10" path="http://192.168.217.140"/>
</full>
</source-target>
</mappings>
</server>
</servers>config_kafka_mysqlto8a1.properties
cp config_kafka_mysqlto8a.properties config_kafka_mysqlto8a1.properties
修改配置文件中topic.name=mysqlto8a为 topic.name=mysqlto8a,mysqlto8a1 两个文件都需要修改（3）多分片集群数据节点的表中会多出两个gid字段，迁移到gbase8a时，需要在原表基础上再创建两个id才可正常同步
评论
登录后才可以发表评论
热门帖子
- 12025-12-01浏览数：182762
- 22023-05-09浏览数：25056
- 42023-09-25浏览数：18525
- 52020-05-11浏览数：17528