外部表

外部表测试
准备条件：
配置域名所有的hadoop的节点信息写入到8a的/etc/hosts
验证hostname所有节点主机名和ip
echo '1172.16.3.233' >> /etc/hosts

namenode端口rpc 25000

datanode端口rpc  25009 
保证所有的datanode端口都需要开通

2.配置/etc/krb5.conf 拷贝到所有集群节点
[root@gbase233 ~]# cat /etc/krb5.conf
# Configuration snippets may be placed in this directory as well
includedir /etc/krb5.conf.d/

[logging]
default = FILE:/var/log/krb5libs.log
kdc = FILE:/var/log/krb5kdc.log
admin_server = FILE:/var/log/kadmind.log

[libdefaults]
dns_lookup_realm = false
ticket_lifetime = 24h
renew_lifetime = 7d
forwardable = true
rdns = false
pkinit_anchors = FILE:/etc/pki/tls/certs/ca-bundle.crt
default_realm = HADOOP.COM
default_ccache_name = FILE:/tmp/krb5cc_%{uid}

[realms]
HADOOP.COM = {
kdc=gbase233
 admin_server = gbase233
}

[domain_realm]
#.example.com = HADOOP.COM
#example.com = HADOOP.COM

3.安装krb5客户端
x86格式直接yum
arm的需要上传包

4.找到hadoop的相关信息配置集群参数如果不需要高可用则不需要设置namenode参数

set global _gbase_hdfs_rpcconfig='dfs.encrypt.data.transfer=false,dfs.block.access.token.enable=true';
set global gbase_hdfs_auth_mode='Kerberos';
set global gbase_hdfs_keytab='/home/test/hdfs.keytab';
set global gbase_hdfs_port=8020;
set global gbase_hdfs_principal='hdfs@HADOOP.COM';
set global gbase_hdfs_protocol='RPC';

外部表参数
gcluster_enable_external_table=1
gcluster_hive_external_table=1

5.

CREATE TABLE IF NOT EXISTS orc_partitioned_table ( 
name1 string,
name2 string,
name3 string,
name4 string,
name5 string,
name6 string,
name7 string,
name8 string,
name9 string,
name10 string,
name11 string,
name12 string,
name13 string,
name14 string,
name15 string,
name16 string,
name17 string,
name18 string,
name19 string,
name20 string
) PARTITIONED BY (riqi STRING)
STORED AS ORC;

CREATE  load read  EXTERNAL table t001 (
name1 varchar(100),
name2 varchar(100),
name3 varchar(100),
name4 varchar(100),
name5 varchar(100),
name6 varchar(100),
name7 varchar(100),
name8 varchar(100),
name9 varchar(100),
name10 varchar(100),
name11 varchar(100),
name12 varchar(100),
name13 varchar(100),
name14 varchar(100),
name15 varchar(100),
name16 varchar(100),
name17 varchar(100),
name18 varchar(100),
name19 varchar(100),
name20 varchar(100),
riqi varchar(30),
fileid int) -- distributed by ('name1')
LOCATION  'hdp://hdfs@gbase234:8020//user/hive/warehouse/test.db/orc_partitioned_table/*/*' informat data_format 8 
fields terminated by ',' table_fields 'name1,name2,name3,name4,name5,name6,name7,name8,name9,name10,name11,name12,name13,name14,name15,name16,name17,name18,name19,name20' outformat fields terminated by ',' ;

CREATE   table t002 (
name1 varchar(100),
name2 varchar(100),
name3 varchar(100),
name4 varchar(100),
name5 varchar(100),
name6 varchar(100),
name7 varchar(100),
name8 varchar(100),
name9 varchar(100),
name10 varchar(100),
name11 varchar(100),
name12 varchar(100),
name13 varchar(100),
name14 varchar(100),
name15 varchar(100),
name16 varchar(100),
name17 varchar(100),
name18 varchar(100),
name19 varchar(100),
name20 varchar(100),
riqi varchar(30));

load data infile  'hdp://hdfs@gbase234:8020//user/hive/warehouse/test.db/orc_partitioned_table/*/*' into table t002  data_format 8 
fields terminated by ',' table_fields 'name1,name2,name3,name4,name5,name6,name7,name8,name9,name10,name11,name12,name13,name14,name15,name16,name17,name18,name19,name20' ;

排查问题
针对hive分区表小文件 使用nosplit速度会更快。
针对hdfs文件读取使用此命令排查hdfs   --loglevel debug dfs   -get /