├── README.md ├── pom.xml └── src ├── main └── java │ └── com │ └── yaoqi │ └── hbase │ └── component │ ├── HBaseApp.java │ ├── assertion │ └── Assert.java │ ├── config │ └── HBaseConfig.java │ ├── constant │ ├── CommonConstant.java │ ├── ExceptionMessage.java │ └── KerberosConstant.java │ ├── operations │ └── HBaseOperations.java │ └── template │ └── HBaseTemplate.java └── test ├── java └── com │ └── yaoqi │ └── hbase │ └── component │ ├── assertion │ └── AssertTest.java │ └── template │ └── HBaseTemplateTest.java └── resources └── application.yml /README.md: -------------------------------------------------------------------------------- 1 | # HBase 组件接口文档 2 | 3 | ---- 4 | 5 | #### 使用说明 6 | 7 | 基本概念 8 | 9 | table: 表 10 | 11 | columnFamily:列族，一个表下可以有多个列族，但是不建议设置多个列族，HBase建议设计长窄型的表而不是短宽型。 12 | 13 | qualifier:列，一个列族下可以有多列，一个表中的列可以是不对齐的，但是这样效率不高，同一张表中的列最好是相同的。 14 | 15 | cell:一列数据下的一个单元格，一个列下可以有多个单元格，根据版本号区分，默认每次读取最新版本的数据，cell下的存储是数据本身。 16 | 17 | row: 行，多列数据组成一行，一行中有多个qualifier。 18 | 19 | rowKey: 行健，用于唯一标识一行数据，一行下有多列，行健的设计直接关系到查询的效率。 20 | 21 | ### HBase配置 22 | 23 | 以下配置为最基础配置，缺一不可。 24 | 25 | ``` yml 26 | HBase: 27 | conf: 28 | quorum: 192.168.80.234:2181,192.168.80.235:2181,192.168.80.241:2181 29 | znodeParent: /hbase-unsecure 30 | #如果有更多配置，写在config下，例如： 31 | #config: 32 | # key: value 33 | # key: value 34 | ``` 35 | 36 | 如果需要更多配置，需要在config中配置，以key-value的形式书写。 37 | 38 | 39 | ### 参数说明 40 | 41 | quorum是HBase中zookeeper的配置，znodeParent是HBase配置在zookeeper中的路径。 42 | 43 | 带kerberos认证的配置如下： 44 | 45 | ``` yml 46 | # HBase配置 47 | HBase: 48 | conf: 49 | quorum: 192.168.80.54:2181,192.168.80.62:2181,192.168.80.64:2181,192.168.80.78:2181,192.168.80.52:2181 50 | znodeParent: /hbase-secure 51 | config: 52 | #keytab地址路径 53 | user-keytab: D:\semptian\environment\security/hbase.service.keytab 54 | #如果使用kb认证请写kerberos否则不用写 55 | authMethod: kerberos 56 | #master.kerberos.principal 57 | masterPrincipal: hbase/node248@SEMPTIAN.COM 58 | #regionserver.kerberos.principal regionserverPrincipal/hbaseSitePath 二选一,一些环境下使用hbaseSitePath更简单 59 | #regionserverPrincipal: root/_HOST@SEMPTIAN.COM 60 | #hbseSite配置文件路径D:\kerberos\hbase-kb\hbase-site.xml 61 | hbaseSitePath: D:\semptian\environment\security/hbase-site.xml 62 | #coreSitePath: D:\kerberos\dev-hbase-kb\core-site.xml 63 | # 刷新认证周期，单位：小时 64 | refreshAuth: 8 65 | ``` 66 | 67 | ### 认证配置参数说明 68 | 69 | user-keytab 和 masterPrincipal 是对应的，一个对应用户的认证文件，一个对应用户。 70 | hbaseSitePath 是hbase-site.xml的文件路径。 71 | 72 | 在jdk中会自动设置认证过期时间为24小时，所以在组件中需要对认证进行刷新，重新建立连接。refreshAuth 就是配置刷新认证的周期，一般小于24小时， 73 | 建议12小时，因为重新建立连接需要消耗一些资源。 74 | 75 | ## 简单示例 76 | 77 | 引入组件jar包： 78 | 79 | ``` xml 80 | 81 | com.semptian.hbase.component 82 | hbase-component 83 | 1.0.1-SNAPSHOT 84 | 85 | ``` 86 | 87 | 在需要的地方注入HBaseOperations接口，该接口的实现类是HBaseTemplate，通过这个类来操作HBase。 88 | 89 | ``` java 90 | @Autowired 91 | private HBaseOperations hBaseDao; 92 | ``` 93 | 94 | 查询一条数据，通过rowKey查询： 95 | 96 | ``` java 97 | public void testQueryTable() { 98 | Result result = hBaseDao.queryByTableNameAndRowKey( 99 | "LBS", 9223372036854775803L); 100 | System.out.println(result.isEmpty()); 101 | result.listCells().forEach(cell -> { 102 | System.out.println( 103 | "row:" + Bytes.toLong(CellUtil.cloneRow(cell)) + 104 | ",family:"+ Bytes.toString(CellUtil.cloneFamily(cell)) + 105 | ", qualifier: " + Bytes.toString(CellUtil.cloneQualifier(cell)) + 106 | ", value:" + Bytes.toString(CellUtil.cloneValue(cell))); 107 | }); 108 | } 109 | 110 | ``` 111 | 112 | ## 表的基本操作 113 | 114 | ### 新建表 115 | 116 | 创建表通过HBaseTemplate就可以实现，HBaseTemplate类中带有这个方法。 117 | 118 | 操作示例： 119 | ``` java 120 | hBaseDao.createTable("HBASE-COMPONENT_1", "CF1", "CF2"); 121 | ``` 122 | 上述代码创建了一张表，HBASE-COMPONENT_1 是表名，CF1,CF2代表这个表有两个列族。 123 | 124 | 如果有多个列族可以往后面加，列族不建议设置很多个。 125 | 126 | ### 删除表 127 | 128 | ``` java 129 | hBaseDao.dropTable("HBASE-COMPONENT_1"); 130 | ``` 131 | 参数是表名，通过表名删除表。 132 | 133 | ### 判断表是否存在 134 | 135 | ``` java 136 | hBaseDao.tableExists("lbs"); 137 | ``` 138 | 这里的表名是区分大小写的。返回值：boolean。 139 | 140 | ### 新增数据 141 | 142 | #### 新增一条数据 143 | 144 | 需要注意的是在HBase中的存储的数据是不分格式的，都是以字节数组的形式存储，因此在存储一条数据时需要将数据都转化成字节数组。 145 | 146 | String格式的数据能直接转换为字节数组getBytes()，但是其他格式的数据需要借助工具作转换。 147 | 148 | 这里需要格外注意rowKey的格式，用什么格式存就决定了用什么格式取。 149 | ``` java 150 | hBaseDao.put("HBase-component", "1534154424340", "CF1", "test_1", Bytes.toBytes("testData")); 151 | ``` 152 | 153 | 参数说明： 154 | ``` java 155 | (1) tableName 目标数据表 156 | (2) rowName rowKey 157 | (3) familyName 列族名 158 | (4) qualifier 列名 159 | (5) data 字节数组类型的数据 160 | 161 | ``` 162 | 这里新增一条数据是填充数据到一个cell中去。 163 | 164 | #### 批量新增数据 165 | 166 | ``` java 167 | String rowKey = String.valueOf(System.currentTimeMillis()); 168 | Put put = new Put(rowKey.getBytes()); 169 | String defaultColumn = "CF1"; 170 | String column1 = "col1"; 171 | String column2 = "col2"; 172 | String column3 = "col3"; 173 | 174 | String value = "test"; 175 | put.addColumn(defaultColumn.getBytes(), column1.getBytes(), value.getBytes()); 176 | put.addColumn(defaultColumn.getBytes(), column2.getBytes(), value.getBytes()); 177 | put.addColumn(defaultColumn.getBytes(), column3.getBytes(), value.getBytes()); 178 | 179 | List putList = new ArrayList<>(); 180 | putList.add(put); 181 | putList.add(put); 182 | putList.add(put); 183 | putList.add(put); 184 | putList.add(put); 185 | 186 | hBaseDao.putBatch("HBase-component", putList); 187 | ``` 188 | 批量插入数据就是使用多个Put对象，putBatch(...)方法的参数：表名，putList(多个put的集合)。 189 | 注意批量插入数据也都是插入字节数组格式的数据。 190 | 191 | ### 删除数据 192 | 193 | #### 删除一条数据 194 | 195 | ``` java 196 | hBaseDao.delete("HBase-component", "1534210201115", "CF1", "col2"); 197 | ``` 198 | 参数说明： 199 | 200 | (1) 表名 201 | 202 | (2) rowKey 203 | 204 | (3) 列族名 205 | 206 | (4) 列名 207 | 208 | 这里删除是删除一个cell下的数据 209 | 210 | #### 批量删除数据 211 | 212 | ``` java 213 | String tableName = "HBase-component"; 214 | String rowKey1 = "1534164113922"; 215 | String rowKey2 = "1534168248328"; 216 | 217 | List deleteList = new ArrayList<>(); 218 | Delete delete = new Delete(rowKey1.getBytes()); 219 | Delete delete1 = new Delete(rowKey2.getBytes()); 220 | deleteList.add(delete); 221 | deleteList.add(delete1); 222 | hBaseDao.deleteBatch(tableName, deleteList); 223 | ``` 224 | 批量删除需要借助Delete对象。 225 | 226 | ### 查询 227 | 228 | ### 单条结果查询 229 | ``` java 230 | Result result = hBaseDao.queryByTableNameAndRowKey("LBS", 9223372036854775803L); 231 | System.out.println(result.isEmpty()); 232 | result.listCells().forEach(cell -> { 233 | System.out.println( 234 | " row:" + Bytes.toLong(CellUtil.cloneRow(cell)) + 235 | " family:"+ Bytes.toString(CellUtil.cloneFamily(cell)) + 236 | " qualifier: " + Bytes.toString(CellUtil.cloneQualifier(cell)) + 237 | " value:" + Bytes.toString(CellUtil.cloneValue(cell))); 238 | }); 239 | ``` 240 | queryByTableNameAndRowKey()该方法是通过表名和rowKey查询数据，这里的rowKey支持多种类型，Long，double，Integer几种类型。 241 | 至于这里传什么类型的参数，取决于插入数据时rowKey的类型，虽然HBase里存储的都是字节数组，但是对类型是敏感的，如果类型对不上可能会出错。 242 | 243 | ### 批量扫描 244 | 245 | ``` java 246 | // 构建scan 247 | Scan scan = new Scan(); 248 | 249 | // 设置时间戳,计算时间差 250 | Long timeDifference = 2L * 30L * 24L * 60L * 60L * 1000L; 251 | Long endTime = System.currentTimeMillis(); 252 | Long fromTime = endTime - timeDifference; 253 | 254 | // 设置时间过滤器 255 | FilterList filterList = new FilterList(); 256 | Filter startTimeFilter = new SingleColumnValueFilter( 257 | DEFAULT_COLUMN_FAMILY.getBytes(), 258 | DATA_CREATE_TIME.getBytes(), 259 | CompareFilter.CompareOp.GREATER, 260 | Bytes.toBytes(fromTime) 261 | ); 262 | 263 | Filter endTimeFilter = new SingleColumnValueFilter( 264 | DEFAULT_COLUMN_FAMILY.getBytes(), 265 | DATA_CREATE_TIME.getBytes(), 266 | CompareFilter.CompareOp.LESS, 267 | Bytes.toBytes(endTime) 268 | ); 269 | 270 | 271 | filterList.addFilter(startTimeFilter); 272 | filterList.addFilter(endTimeFilter); 273 | 274 | scan.setFilter(filterList); 275 | 276 | // 获取结果集 277 | ResultScanner resultScanner = hBaseTemplate.queryByScan(TABLE_NAME, scan); 278 | 279 | // 遍历结果集 280 | try{ 281 | if (resultScanner != null) { 282 | resultScanner.forEach(result -> { 283 | List cellList = result.listCells(); 284 | ... 285 | } 286 | } 287 | }finally{ 288 | if (resultScanner != null) { 289 | resultScanner.close(); 290 | } 291 | } 292 | 293 | ``` 294 | 295 | 批量查询可以通过queryByScan()方法实现，第一个参数是表名，第二个参数是scan，通过构建不同的scan来查询，过滤器也是在构建scan对象是添加的，可以添加多个过滤器。 296 | 297 | *需要注意的是这里的ResultScanner类，在遍历结果集时需要使用try-finally结构，在使用完resultScanner对象之后关闭该对象。HBase官方文档上强调了这一点。因此在使用ResultScanner对象时需要格外注意。* 298 | 299 | 常见过滤器： 300 | 301 | 行健过滤器：RowFilter 302 | 303 | 列族过滤器：FamilyFilter 304 | 305 | 值过滤器：ValueFilter 306 | 307 | 列过滤器：QualifierFilter 308 | 309 | 单列值过滤器：SingleColumnValueFilter(会返回满足条件的行) 310 | 311 | 单列值排除过滤器：SingleColumnExcludeFilter(返回排除了该列的结果，与单列值过滤器相反) 312 | 313 | 前缀过滤器：PrefixFilter(这个过滤器是针对行健的，在构造方法中传入字节数组形式的内容，过滤器会去匹配行健) 314 | 315 | 页数过滤器：PageFilter(使用pageFilter过滤器的时候需要注意，并不是设置了页数大小就能返回相应数目的结果) 316 | 317 | ``` java 318 | String tableName = "RECOMMEND_ENGINE_DATA_MODEL"; 319 | Scan scan = new Scan(); 320 | 321 | PageFilter pageFilter = new PageFilter(1); 322 | scan.setFilter(pageFilter); 323 | 324 | ResultScanner resultScanner = hBaseDao.queryByScan(tableName, scan); 325 | 326 | try{ 327 | resultScanner.forEach(result -> { 328 | result.listCells().forEach(cell -> { 329 | // process 330 | }); 331 | }finally{ 332 | if (resultScanner != null) { 333 | resultScanner.close(); 334 | } 335 | } 336 | ``` 337 | 338 | 上面这段代码中设置了页面大小为1，预期是返回一条数据，但是结果会返回两条数据，这时返回的结果数会取决于regionServer的数量。 339 | 340 | 如果是FilterList，FilterList的顺序会影响PageFilter的效果。 341 | 342 | 一般比较型过滤器，需要用CompareFilter.CompareOp中的比较运算符。所有的过滤器都是用Scan对象去设置。 343 | 344 | #### 多过滤器查询 345 | ``` java 346 | String tableName = "HBase-component"; 347 | Scan scan = new Scan(); 348 | PageFilter pageFilter = new PageFilter(1); 349 | 350 | SingleColumnValueFilter singleColumnValueFilter = new SingleColumnValueFilter( 351 | "CF1".getBytes(), 352 | "col1".getBytes(), 353 | CompareFilter.CompareOp.EQUAL, 354 | new SubstringComparator("group")); 355 | 356 | singleColumnValueFilter.setFilterIfMissing(true); 357 | FilterList filterList = new FilterList(); 358 | filterList.addFilter(singleColumnValueFilter); 359 | filterList.addFilter(pageFilter); 360 | 361 | scan.setFilter(filterList); 362 | ResultScanner resultScanner = hBaseDao.queryByScan(tableName, scan); 363 | 364 | try { 365 | resultScanner.forEach(result -> { 366 | result.listCells().forEach(cell -> { 367 | System.out.println( 368 | " row:" + Bytes.toString(CellUtil.cloneRow(cell)) + 369 | " family:"+ Bytes.toString(CellUtil.cloneFamily(cell)) + 370 | " qualifier: " + Bytes.toString(CellUtil.cloneQualifier(cell))+ 371 | " value:" + Bytes.toString(CellUtil.cloneValue(cell))); 372 | }); 373 | }); 374 | } finally { 375 | if (resultScanner != null) { 376 | resultScanner.close(); 377 | } 378 | } 379 | ``` 380 | 381 | 多过滤器需要用到FilterList，也是直接设置到Scan对象中。多过滤器的时候需要注意过滤器的顺序问题，例如上面代码中如果将两个过滤器调换顺序，查询的结果也是不一样的。 382 | 383 | ### 结果集的映射 384 | 385 | 在HBase中，默认所有的顺序都是按照字母序排列，例如CF1列族下有多个列：col1、col2、col3，那么在遍历结果集时，listCells()中的cell的顺序总是按照列名的字母序来排列的。 386 | 387 | 所以cellList.get(0)就是对应col1中的数据，cellList.get(1)就是对应col2中的数据，cellList.get(2)就是对应col3中的数据。 388 | 389 | 如果列名为a、b、c那分别对应的下标为cellList.get(0)、cellList.get(1)、cellList.get(2) -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | com.yaoqi.hbase.component 7 | hbase-component 8 | 1.0.2-SNAPSHOT 9 | jar 10 | 11 | org.springframework.boot 12 | spring-boot-starter-parent 13 | 1.5.10.RELEASE 14 | 15 | 16 | 17 | org.springframework.boot 18 | spring-boot-starter 19 | 20 | 21 | org.apache.hbase 22 | hbase-shaded-client 23 | 2.0.2 24 | 25 | 26 | org.apache.hadoop 27 | hadoop-common 28 | 2.6.4 29 | jar 30 | 31 | 32 | 33 | org.springframework.boot 34 | spring-boot-starter-test 35 | test 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | org.springframework.boot 44 | spring-boot-maven-plugin 45 | 46 | true 47 | true 48 | 49 | 50 | 51 | org.apache.maven.plugins 52 | maven-compiler-plugin 53 | 54 | 1.8 55 | 1.8 56 | UTF-8 57 | 58 | 59 | 60 | 61 | 62 | -------------------------------------------------------------------------------- /src/main/java/com/yaoqi/hbase/component/HBaseApp.java: -------------------------------------------------------------------------------- 1 | package com.yaoqi.hbase.component; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | 6 | /** 7 | * @Author YaoQi 8 | * Date 2018/8/13 11:35 9 | * Modified 10 | * Description 11 | */ 12 | @SpringBootApplication 13 | public class HBaseApp { 14 | public static void main(String[] args) { 15 | SpringApplication.run(HBaseApp.class); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/main/java/com/yaoqi/hbase/component/assertion/Assert.java: -------------------------------------------------------------------------------- 1 | package com.yaoqi.hbase.component.assertion; 2 | 3 | import com.yaoqi.hbase.component.constant.ExceptionMessage; 4 | import org.springframework.util.StringUtils; 5 | 6 | /** 7 | * @author YaoQi 8 | * Date 2018/8/13 17:00 9 | * Modified 10 | * Description 参数断言 11 | */ 12 | public class Assert { 13 | 14 | 15 | /** 16 | * String 类型参数校验，参数不合法直接抛出异常 17 | * 18 | * @param text 参数 19 | */ 20 | public static void hasLength(String text) { 21 | if (!StringUtils.hasLength(text)) { 22 | throw new IllegalArgumentException(ExceptionMessage.HAS_LENGTH_MSG); 23 | } 24 | } 25 | 26 | /** 27 | * 参数批量判断是否为空 28 | * 29 | * @param text 可传多个参数 30 | */ 31 | public static void hasLengthBatch(String... text) { 32 | for (int i = 0; i < text.length; i++) { 33 | hasLength(text[i]); 34 | } 35 | } 36 | 37 | /** 38 | * Object类型参数校验，参数不合法直接抛出异常 39 | * 40 | * @param object 对象类型参数 41 | */ 42 | public static void notNull(Object object) { 43 | if (object == null) { 44 | throw new IllegalArgumentException(ExceptionMessage.NOT_NULL_MSG); 45 | } 46 | } 47 | 48 | /** 49 | * 参数批量判断 50 | * 51 | * @param objects 多个参数集合 52 | */ 53 | public static void notNullBatch(Object... objects) { 54 | for (Object object : objects) { 55 | notNull(object); 56 | } 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /src/main/java/com/yaoqi/hbase/component/config/HBaseConfig.java: -------------------------------------------------------------------------------- 1 | package com.yaoqi.hbase.component.config; 2 | 3 | import com.yaoqi.hbase.component.constant.KerberosConstant; 4 | import org.apache.commons.lang.StringUtils; 5 | import org.apache.hadoop.conf.Configuration; 6 | import org.apache.hadoop.fs.Path; 7 | import org.apache.hadoop.hbase.HBaseConfiguration; 8 | import org.apache.hadoop.hbase.client.Connection; 9 | import org.apache.hadoop.hbase.client.ConnectionFactory; 10 | import org.apache.hadoop.hbase.security.User; 11 | import org.apache.hadoop.hbase.shaded.org.apache.commons.lang3.concurrent.BasicThreadFactory; 12 | import org.apache.hadoop.security.UserGroupInformation; 13 | import org.slf4j.Logger; 14 | import org.slf4j.LoggerFactory; 15 | import org.springframework.boot.context.properties.ConfigurationProperties; 16 | import org.springframework.context.annotation.ComponentScan; 17 | 18 | import javax.annotation.PostConstruct; 19 | import java.io.IOException; 20 | import java.util.Map; 21 | import java.util.concurrent.ConcurrentLinkedQueue; 22 | import java.util.concurrent.ScheduledExecutorService; 23 | import java.util.concurrent.ScheduledThreadPoolExecutor; 24 | import java.util.concurrent.TimeUnit; 25 | 26 | /** 27 | * @author YaoQi 28 | * Date 2018/8/13 11:32 29 | * Modified 30 | * Description HBase配置类 31 | */ 32 | @org.springframework.context.annotation.Configuration 33 | @ConfigurationProperties(prefix = "HBase.conf") 34 | @ComponentScan(value = "com.semptian.base.hbasecomponent") 35 | public class HBaseConfig { 36 | 37 | private static final Logger logger = LoggerFactory.getLogger(HBaseConfig.class); 38 | 39 | private String quorum; 40 | private String znodeParent; 41 | private Map config; 42 | private User user = null; 43 | private volatile Connection connection; 44 | private static ConcurrentLinkedQueue queue = new ConcurrentLinkedQueue<>(); 45 | 46 | /** 47 | * 初始化配置 48 | * 49 | * @return 50 | * @throws IOException 51 | */ 52 | private Configuration configuration() throws IOException { 53 | Configuration configuration = HBaseConfiguration.create(); 54 | logger.info("zookeeper node : {}", quorum); 55 | logger.info("znodeParent is : {}", znodeParent); 56 | configuration.set("hbase.zookeeper.quorum", quorum); 57 | configuration.set("zookeeper.znode.parent", znodeParent); 58 | if (config != null && config.containsKey(KerberosConstant.AUTH_METHOD) && KerberosConstant.KERBEROS.equals(config.get(KerberosConstant.AUTH_METHOD))) { 59 | String keytabPath = config.get("user-keytab"); 60 | String masterPrincipal = config.get("masterPrincipal"); 61 | String regionserverPrincipal = config.get("regionserverPrincipal"); 62 | String hbaseSitePath = config.get("hbaseSitePath"); 63 | String coreSitePath = config.get("coreSitePath"); 64 | configuration.set("hadoop.security.authentication", "kerberos"); 65 | configuration.set("hbase.security.authentication", "kerberos"); 66 | if (StringUtils.isNotBlank(hbaseSitePath)) { 67 | configuration.addResource(new Path(hbaseSitePath)); 68 | } 69 | if (StringUtils.isNotBlank(coreSitePath)) { 70 | configuration.addResource(new Path(coreSitePath)); 71 | } 72 | if (StringUtils.isNotBlank(masterPrincipal) && StringUtils.isNotBlank(regionserverPrincipal)) { 73 | configuration.set("hbase.master.kerberos.principal", masterPrincipal); 74 | configuration.set("hbase.regionserver.kerberos.principal", regionserverPrincipal); 75 | } 76 | UserGroupInformation.setConfiguration(configuration); 77 | UserGroupInformation.loginUserFromKeytab(masterPrincipal, keytabPath); 78 | UserGroupInformation loginUser = UserGroupInformation.getLoginUser(); 79 | user = User.create(loginUser); 80 | } 81 | // 将config中的配置加入到configuration中 82 | if (config != null && !config.isEmpty()) { 83 | config.forEach(configuration::set); 84 | } 85 | return configuration; 86 | } 87 | 88 | /** 89 | * 重新初始化连接，解决认证过期问题，默认过期时间为24h，因此reInitial() 的执行周期应该小于24h 90 | * 认证环境下才会开启 91 | */ 92 | @PostConstruct 93 | private void reInitial() { 94 | int refreshAuth = 12; 95 | boolean openReInitial = false; 96 | if (config != null && !config.isEmpty()) { 97 | String refreshAuthConfig = config.get(KerberosConstant.REFRESH_AUTH); 98 | if (StringUtils.isNotEmpty(refreshAuthConfig)) { 99 | // 如果有配置，则按照配置的时间进行刷新 100 | refreshAuth = Integer.valueOf(refreshAuthConfig); 101 | } 102 | String authMethod = config.get(KerberosConstant.AUTH_METHOD); 103 | if (KerberosConstant.KERBEROS.equals(authMethod)) { 104 | openReInitial = true; 105 | } 106 | } 107 | if (openReInitial) { 108 | // 认证环境下才会开启 109 | logger.info("create new thread refresh connection."); 110 | ScheduledExecutorService executorService = new ScheduledThreadPoolExecutor(1, 111 | new BasicThreadFactory.Builder().namingPattern("renew-connection-pool-%d").daemon(true).build()); 112 | final int finalRefreshAuth = refreshAuth; 113 | executorService.submit((Runnable) () -> { 114 | while (true) { 115 | try { 116 | TimeUnit.HOURS.sleep(finalRefreshAuth); 117 | } catch (InterruptedException e) { 118 | logger.error("reLogin kerberos error ,error info : {}", e.getMessage()); 119 | } 120 | initial(); 121 | closeOldConnection(); 122 | logger.info("renew connection:{}", connection); 123 | } 124 | }); 125 | } 126 | } 127 | 128 | /** 129 | * 获取连接 130 | * 131 | * @return 132 | */ 133 | public Connection getConnection() { 134 | if (connection == null) { 135 | connection = initial(); 136 | } 137 | return connection; 138 | } 139 | 140 | /** 141 | * 初始化连接，如果存在老的连接则加入队列 142 | * 143 | * @return 144 | */ 145 | @PostConstruct 146 | private Connection initial() { 147 | Connection newConnection; 148 | try { 149 | newConnection = ConnectionFactory.createConnection(configuration(), user); 150 | Connection oldConnection = connection; 151 | if (newConnection != null) { 152 | connection = newConnection; 153 | if (oldConnection != null) { 154 | queue.add(oldConnection); 155 | } 156 | } 157 | } catch (IOException e) { 158 | e.printStackTrace(); 159 | } 160 | return connection; 161 | } 162 | 163 | /** 164 | * 关闭老的连接 165 | */ 166 | private void closeOldConnection() { 167 | if (queue != null && !queue.isEmpty() && queue.size() > 1) { 168 | // 在创建connection的第二个周期开始清空队列 169 | Connection oldConnection = queue.poll(); 170 | if (oldConnection != null && !oldConnection.isClosed()) { 171 | try { 172 | oldConnection.close(); 173 | } catch (IOException e) { 174 | logger.error("oldConnection close failed."); 175 | e.printStackTrace(); 176 | } 177 | } 178 | } 179 | } 180 | 181 | public String getQuorum() { 182 | return quorum; 183 | } 184 | 185 | public void setQuorum(String quorum) { 186 | this.quorum = quorum; 187 | } 188 | 189 | public String getZnodeParent() { 190 | return znodeParent; 191 | } 192 | 193 | public void setZnodeParent(String znodeParent) { 194 | this.znodeParent = znodeParent; 195 | } 196 | 197 | public Map getConfig() { 198 | return config; 199 | } 200 | 201 | public void setConfig(Map config) { 202 | this.config = config; 203 | } 204 | } 205 | -------------------------------------------------------------------------------- /src/main/java/com/yaoqi/hbase/component/constant/CommonConstant.java: -------------------------------------------------------------------------------- 1 | package com.yaoqi.hbase.component.constant; 2 | 3 | /** 4 | * @author : YaoQi 5 | * Date: 2018/10/27 17:57 6 | * Modified: 7 | * Description:公共常量类 8 | */ 9 | public class CommonConstant { 10 | 11 | /** 12 | * 批量插入最大长度限制 13 | */ 14 | public static final Integer BATCH_PUT_LIMIT = 2000000; 15 | } 16 | -------------------------------------------------------------------------------- /src/main/java/com/yaoqi/hbase/component/constant/ExceptionMessage.java: -------------------------------------------------------------------------------- 1 | package com.yaoqi.hbase.component.constant; 2 | 3 | /** 4 | * @author YaoQi 5 | * Date 2018/8/13 16:52 6 | * Modified 7 | * Description 异常信息常量类 8 | */ 9 | public class ExceptionMessage { 10 | 11 | /** 12 | * 参数不为空异常信息 13 | */ 14 | public static final String NOT_NULL_MSG = "This argument is required. It must not be null!"; 15 | 16 | /** 17 | * String 参数长度异常信息 18 | */ 19 | public static final String HAS_LENGTH_MSG = "This String must have length. It must not be null or empty!"; 20 | 21 | /** 22 | * 表名不存在异常信息 23 | */ 24 | public static final String TABLE_NOT_EXISTS_MSG = "Table is not exists!"; 25 | 26 | /** 27 | * 表名已存在异常信息 28 | */ 29 | public static final String TABLE_ALREADY_EXISTS_MSG = "Table is already exists!"; 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/com/yaoqi/hbase/component/constant/KerberosConstant.java: -------------------------------------------------------------------------------- 1 | package com.yaoqi.hbase.component.constant; 2 | 3 | /** 4 | * @author YaoQi 5 | * date 2019 / 12 / 17 / 16: 12 6 | * desc 7 | **/ 8 | public class KerberosConstant { 9 | 10 | /** 11 | * 认证方式 12 | */ 13 | public static final String AUTH_METHOD = "authMethod"; 14 | /** 15 | * kerberos 认证类型 16 | */ 17 | public static final String KERBEROS = "kerberos"; 18 | 19 | /** 20 | * 认证刷新配置 21 | */ 22 | public static final String REFRESH_AUTH = "refreshAuth"; 23 | } 24 | -------------------------------------------------------------------------------- /src/main/java/com/yaoqi/hbase/component/operations/HBaseOperations.java: -------------------------------------------------------------------------------- 1 | package com.yaoqi.hbase.component.operations; 2 | 3 | import org.apache.hadoop.hbase.client.*; 4 | 5 | import java.io.IOException; 6 | import java.util.List; 7 | 8 | /** 9 | * @author YaoQi 10 | * Date 2018/8/13 14:46 11 | * Modified 12 | * Description 组件接口 13 | */ 14 | public interface HBaseOperations { 15 | 16 | /** 17 | * 创建一张表 18 | * 19 | * @param tableName 表名 20 | * @param familyName 列族名 21 | */ 22 | void createTable(final String tableName, final String... familyName); 23 | 24 | /** 25 | * 创建带有分区的表 26 | * 27 | * @param tableName 28 | * @param familyName 29 | * @param splitkeys 30 | */ 31 | void createTable(String tableName, List familyName, byte[][] splitkeys); 32 | 33 | /** 34 | * 通过表名和rowKey获取数据 35 | * 36 | * @param tableName 表名 37 | * @param rowKeyVar rowKey 泛型 可支持多种类型{String,Long,Double} 38 | * @return Result 类型 39 | */ 40 | Result queryByTableNameAndRowKey(String tableName, T rowKeyVar); 41 | 42 | /** 43 | * 自定义查询 44 | * 45 | * @param tableName 表名 46 | * @param getList 请求体 47 | * @return Result类型 48 | */ 49 | Result[] query(String tableName, List getList); 50 | 51 | /** 52 | * 判断表名是否存在 53 | * 54 | * @param tableName 表名 String ,注意这里区分大小写 55 | * @return 56 | */ 57 | boolean tableExists(String tableName); 58 | 59 | /** 60 | * 新增一条数据 61 | * 62 | * @param tableName 目标数据表 63 | * @param rowName rowKey 64 | * @param familyName 列族名 65 | * @param qualifier 列名 66 | * @param data 字节数组类型的数据 67 | */ 68 | void put(final String tableName, final String rowName, final String familyName, final String qualifier, final byte[] data); 69 | 70 | /** 71 | * 批量插入数据 72 | * 73 | * @param tableName 表名 74 | * @param putList put集合 75 | * @throws IOException 76 | */ 77 | void putBatch(final String tableName, List putList) throws IOException; 78 | 79 | /** 80 | * 删除一个列族下的数据 81 | * 82 | * @param tableName target table 83 | * @param rowName row name 84 | * @param familyName family 85 | */ 86 | void delete(final String tableName, final String rowName, final String familyName); 87 | 88 | /** 89 | * 删除某个列下的数据 90 | * 91 | * @param tableName 目标数据表 92 | * @param rowName rowKey 93 | * @param familyName 列族名 94 | * @param qualifier 列名 95 | */ 96 | void delete(final String tableName, final String rowName, final String familyName, final String qualifier); 97 | 98 | /** 99 | * 批量删除数据 100 | * 101 | * @param tableName 表名 102 | * @param deleteList 需要删除的数据 103 | */ 104 | void deleteBatch(final String tableName, List deleteList); 105 | 106 | /** 107 | * 通过scan查询数据 108 | * 109 | * @param tableName 表名 110 | * @param scan scan 111 | * @return 返回 ResultScanner 112 | */ 113 | ResultScanner queryByScan(final String tableName, Scan scan); 114 | 115 | /** 116 | * 删除表 117 | * 118 | * @param tableName 表名 119 | */ 120 | void dropTable(String tableName); 121 | 122 | /** 123 | * 批量删除表 124 | * 125 | * @param tableNames 126 | */ 127 | void dropTable(List tableNames); 128 | 129 | /** 130 | * 清空表数据 131 | * 132 | * @param tableName 133 | * @throws IOException 134 | */ 135 | void truncateTable(String tableName) throws IOException; 136 | 137 | /** 138 | * 删除指定行指定列 139 | * 140 | * @param tableName 141 | * @param family 142 | * @param rowKey 143 | * @param column 144 | * @return 145 | * @throws IOException 146 | */ 147 | boolean deleteColumn(String tableName, String family, String rowKey, String column) throws IOException; 148 | 149 | /** 150 | * 删除列簇下的某一列 151 | * 152 | * @param tableName 153 | * @param family 154 | * @param rowKey 155 | * @param columns 156 | * @return 157 | * @throws IOException 158 | */ 159 | boolean deleteColumn(String tableName, String family, String rowKey, List columns) throws IOException; 160 | 161 | /** 162 | * 获取连接对象 163 | * 164 | * @return 165 | */ 166 | Connection getConnection(); 167 | } 168 | -------------------------------------------------------------------------------- /src/main/java/com/yaoqi/hbase/component/template/HBaseTemplate.java: -------------------------------------------------------------------------------- 1 | package com.yaoqi.hbase.component.template; 2 | 3 | import com.yaoqi.hbase.component.assertion.Assert; 4 | import com.yaoqi.hbase.component.config.HBaseConfig; 5 | import com.yaoqi.hbase.component.constant.CommonConstant; 6 | import com.yaoqi.hbase.component.constant.ExceptionMessage; 7 | import com.yaoqi.hbase.component.operations.HBaseOperations; 8 | import org.apache.commons.lang.StringUtils; 9 | import org.apache.hadoop.hbase.HColumnDescriptor; 10 | import org.apache.hadoop.hbase.HTableDescriptor; 11 | import org.apache.hadoop.hbase.TableName; 12 | import org.apache.hadoop.hbase.client.*; 13 | import org.apache.hadoop.hbase.util.Bytes; 14 | import org.slf4j.Logger; 15 | import org.slf4j.LoggerFactory; 16 | import org.springframework.beans.factory.annotation.Autowired; 17 | import org.springframework.stereotype.Component; 18 | 19 | import java.io.IOException; 20 | import java.util.Arrays; 21 | import java.util.List; 22 | 23 | /** 24 | * @author YaoQi 25 | * Date 2018HBaseApp/8/13 14:05 26 | * Modified 27 | * Description 28 | */ 29 | @Component 30 | public class HBaseTemplate implements HBaseOperations { 31 | 32 | private static final Logger logger = LoggerFactory.getLogger(HBaseTemplate.class); 33 | 34 | @Autowired 35 | private HBaseConfig hBaseConfig; 36 | 37 | /** 38 | * 判断表名是否存在 39 | * 40 | * @param tableName 表名 String ,注意这里区分大小写 41 | * @return 42 | */ 43 | @Override 44 | public boolean tableExists(String tableName) { 45 | Table table = null; 46 | boolean tableExistsFlag = false; 47 | Connection connection = hBaseConfig.getConnection(); 48 | try (Admin admin = connection.getAdmin()) { 49 | table = connection.getTable(TableName.valueOf(tableName)); 50 | tableExistsFlag = admin.tableExists(table.getName()); 51 | } catch (IOException e) { 52 | logger.error("IOException : {}", e.getMessage()); 53 | } finally { 54 | closeTable(table); 55 | } 56 | return tableExistsFlag; 57 | } 58 | 59 | /** 60 | * 通过表名和rowKey获取数据,获取一条数据 61 | * 62 | * @param tableName 表名 63 | * @param rowKeyVar rowKey 泛型 可支持多种类型{String,Long,Double} 64 | * @return Result 类型 65 | */ 66 | @Override 67 | public Result queryByTableNameAndRowKey(String tableName, T rowKeyVar) { 68 | Assert.notNullBatch(tableName, rowKeyVar); 69 | Assert.hasLength(tableName); 70 | boolean tableExists = tableExists(tableName); 71 | if (!tableExists) { 72 | logger.info("{}" + ExceptionMessage.TABLE_NOT_EXISTS_MSG, tableName); 73 | return null; 74 | } 75 | byte[] rowKey = checkType(rowKeyVar); 76 | Result result = null; 77 | Table table = null; 78 | try { 79 | Connection connection = hBaseConfig.getConnection(); 80 | table = connection.getTable(TableName.valueOf(tableName)); 81 | Get get = new Get(rowKey); 82 | result = table.get(get); 83 | } catch (IOException e) { 84 | logger.error("IOException : {}", e.getMessage()); 85 | } finally { 86 | closeTable(table); 87 | } 88 | return result; 89 | } 90 | 91 | /** 92 | * 自定义查询 93 | * 94 | * @param tableName 表名 95 | * @param getList 请求体 96 | * @return Result类型 97 | */ 98 | @Override 99 | public Result[] query(String tableName, List getList) { 100 | Assert.notNullBatch(tableName, getList); 101 | Assert.hasLength(tableName); 102 | boolean tableExists = tableExists(tableName); 103 | if (!tableExists) { 104 | logger.info("{}" + ExceptionMessage.TABLE_NOT_EXISTS_MSG, tableName); 105 | return null; 106 | } 107 | Table table = null; 108 | Result[] result = null; 109 | try { 110 | Connection connection = hBaseConfig.getConnection(); 111 | table = connection.getTable(TableName.valueOf(tableName)); 112 | result = table.get(getList); 113 | } catch (IOException e) { 114 | logger.error("query error , message:{}", e.getMessage()); 115 | } finally { 116 | closeTable(table); 117 | } 118 | return result; 119 | } 120 | 121 | 122 | /** 123 | * 传入一个泛型V 判断基本类型，返回对应的byte数组 124 | * 125 | * @param rowKeyVar 泛型rowKey 126 | * @param 泛型 127 | * @return 返回格式化后的字节数组 128 | */ 129 | private byte[] checkType(V rowKeyVar) { 130 | byte[] rowKey = new byte[0]; 131 | //判断T的类型 132 | String rowKeyType = rowKeyVar.getClass().getTypeName(); 133 | logger.info("rowKey Type is : {}", rowKeyType); 134 | if (String.class.getName().equals(rowKeyType)) { 135 | rowKey = Bytes.toBytes((String) rowKeyVar); 136 | } 137 | if (Long.class.getName().equals(rowKeyType)) { 138 | rowKey = Bytes.toBytes((Long) rowKeyVar); 139 | } 140 | if (Double.class.getName().equals(rowKeyType)) { 141 | rowKey = Bytes.toBytes((Double) rowKeyVar); 142 | } 143 | if (Integer.class.getName().equals(rowKeyType)) { 144 | rowKey = Bytes.toBytes((Integer) rowKeyVar); 145 | } 146 | 147 | return rowKey; 148 | } 149 | 150 | /** 151 | * 关闭连接 152 | * 153 | * @param table 表名 154 | */ 155 | private void closeTable(Table table) { 156 | if (table != null) { 157 | try { 158 | table.close(); 159 | } catch (IOException e) { 160 | logger.error("close table {} error {}", table.getName(), e.getMessage()); 161 | } 162 | } else { 163 | logger.info("table is null"); 164 | } 165 | } 166 | 167 | /** 168 | * 创建一张表 169 | * 170 | * @param tableName 表名 171 | * @param familyName 列族名 172 | */ 173 | @Override 174 | public void createTable(String tableName, String... familyName) { 175 | createTable(tableName, Arrays.asList(familyName), null); 176 | } 177 | 178 | /** 179 | * 创建表 180 | * 181 | * @param tableName 182 | * @param familyName 183 | * @param splitKeys 184 | */ 185 | @Override 186 | public void createTable(String tableName, List familyName, byte[][] splitKeys) { 187 | Assert.notNullBatch(tableName, familyName); 188 | Assert.hasLength(tableName); 189 | TableName tableNameVar = TableName.valueOf(tableName); 190 | if (!tableExists(tableName)) { 191 | HTableDescriptor hTableDescriptor = new HTableDescriptor(tableNameVar); 192 | for (String aFamilyName : familyName) { 193 | HColumnDescriptor hColumnDescriptor = new HColumnDescriptor(aFamilyName); 194 | hColumnDescriptor.setBlockCacheEnabled(true); 195 | hColumnDescriptor.setPrefetchBlocksOnOpen(true); 196 | hTableDescriptor.addFamily(hColumnDescriptor); 197 | } 198 | Connection connection = hBaseConfig.getConnection(); 199 | try (Admin admin = connection.getAdmin()) { 200 | if (splitKeys != null) { 201 | admin.createTable(hTableDescriptor, splitKeys); 202 | } else { 203 | admin.createTable(hTableDescriptor); 204 | } 205 | } catch (IOException e) { 206 | logger.error("create failed , Exception: {}", e.getMessage()); 207 | } 208 | } else { 209 | throw new IllegalArgumentException(ExceptionMessage.TABLE_ALREADY_EXISTS_MSG); 210 | } 211 | } 212 | 213 | /** 214 | * 新增一条数据 215 | * 216 | * @param tableName 目标数据表 217 | * @param rowName rowKey 218 | * @param familyName 列族名 219 | * @param qualifier 列名 220 | * @param data 字节数组类型的数据 221 | */ 222 | @Override 223 | public void put(String tableName, String rowName, String familyName, String qualifier, byte[] data) { 224 | Assert.notNullBatch(tableName, rowName, familyName, qualifier); 225 | Assert.hasLengthBatch(tableName, rowName, familyName, qualifier); 226 | Table table = null; 227 | if (tableExists(tableName)) { 228 | try { 229 | Connection connection = hBaseConfig.getConnection(); 230 | table = connection.getTable(TableName.valueOf(tableName)); 231 | Put put = new Put(Bytes.toBytes(rowName)); 232 | put.addColumn(Bytes.toBytes(familyName), Bytes.toBytes(qualifier), data); 233 | table.put(put); 234 | } catch (IOException e) { 235 | logger.error("data put error,message: {}", e.getMessage()); 236 | } finally { 237 | closeTable(table); 238 | } 239 | } else { 240 | throw new IllegalArgumentException(ExceptionMessage.TABLE_NOT_EXISTS_MSG); 241 | } 242 | } 243 | 244 | /** 245 | * 批量插入数据 246 | * 247 | * @param tableName 表名 248 | * @param putList put集合 249 | */ 250 | @Override 251 | public void putBatch(String tableName, List putList) throws IOException { 252 | Assert.notNull(putList); 253 | Assert.hasLength(tableName); 254 | // 限制最大插入数量 255 | if (putList.size() > CommonConstant.BATCH_PUT_LIMIT) { 256 | logger.warn("putList is too large to insert, {} is limit，The real size is {}", 257 | CommonConstant.BATCH_PUT_LIMIT, putList.size()); 258 | } else { 259 | Table table = null; 260 | if (tableExists(tableName)) { 261 | try { 262 | Connection connection = hBaseConfig.getConnection(); 263 | table = connection.getTable(TableName.valueOf(tableName)); 264 | table.put(putList); 265 | } catch (IOException e) { 266 | logger.error("data put error, message:{}", e.getMessage()); 267 | throw new IOException(e); 268 | } finally { 269 | closeTable(table); 270 | } 271 | } else { 272 | throw new IllegalArgumentException(ExceptionMessage.TABLE_NOT_EXISTS_MSG); 273 | } 274 | } 275 | } 276 | 277 | /** 278 | * 删除一个列族下的数据 279 | * 280 | * @param tableName 目标数据表 281 | * @param rowName rowKey 282 | * @param familyName 列族名 283 | */ 284 | @Override 285 | public void delete(String tableName, String rowName, String familyName) { 286 | delete(tableName, rowName, familyName, null); 287 | } 288 | 289 | /** 290 | * 删除某个列下的数据 291 | * 292 | * @param tableName 目标数据表 293 | * @param rowName rowKey 294 | * @param familyName 列族名 295 | * @param qualifier 列名 296 | */ 297 | @Override 298 | public void delete(String tableName, String rowName, String familyName, String qualifier) { 299 | Assert.notNullBatch(tableName, rowName, familyName); 300 | Assert.hasLengthBatch(tableName, rowName, familyName); 301 | Table table = null; 302 | try { 303 | Connection connection = hBaseConfig.getConnection(); 304 | table = connection.getTable(TableName.valueOf(tableName)); 305 | Delete delete = new Delete(rowName.getBytes()); 306 | if (qualifier != null) { 307 | delete.addColumn(familyName.getBytes(), qualifier.getBytes()); 308 | } 309 | table.delete(delete); 310 | } catch (IOException e) { 311 | logger.error("data delete error, message:{}", e.getMessage()); 312 | } finally { 313 | closeTable(table); 314 | } 315 | } 316 | 317 | /** 318 | * 批量删除数据 319 | * 320 | * @param tableName 表名 321 | * @param deleteList 需要删除的数据 322 | */ 323 | @Override 324 | public void deleteBatch(String tableName, List deleteList) { 325 | Assert.notNull(tableName); 326 | Assert.hasLength(tableName); 327 | Table table = null; 328 | try { 329 | Connection connection = hBaseConfig.getConnection(); 330 | table = connection.getTable(TableName.valueOf(tableName)); 331 | table.delete(deleteList); 332 | } catch (IOException e) { 333 | logger.error("data delete error, message:{}", e.getMessage()); 334 | } finally { 335 | closeTable(table); 336 | } 337 | } 338 | 339 | /** 340 | * 通过scan查询数据 341 | * 342 | * @param tableName 表名 343 | * @param scan scan 344 | * @return 返回 ResultScanner 345 | */ 346 | @Override 347 | public ResultScanner queryByScan(String tableName, Scan scan) { 348 | Assert.notNullBatch(tableName, scan); 349 | Assert.hasLength(tableName); 350 | ResultScanner resultScanner = null; 351 | Table table = null; 352 | try { 353 | Connection connection = hBaseConfig.getConnection(); 354 | table = connection.getTable(TableName.valueOf(tableName)); 355 | resultScanner = table.getScanner(scan); 356 | } catch (IOException e) { 357 | logger.error("query error, message:{}", e.getMessage()); 358 | } finally { 359 | closeTable(table); 360 | } 361 | return resultScanner; 362 | } 363 | 364 | /** 365 | * 删除表 366 | * 367 | * @param tableName 表名称 368 | */ 369 | @Override 370 | public void dropTable(String tableName) { 371 | boolean tableExists = tableExists(tableName); 372 | if (tableExists) { 373 | Connection connection = hBaseConfig.getConnection(); 374 | try (Admin admin = connection.getAdmin()) { 375 | admin.disableTable(TableName.valueOf(tableName)); 376 | admin.deleteTable(TableName.valueOf(tableName)); 377 | logger.info("table {} delete successfully", tableName); 378 | } catch (IOException e) { 379 | logger.error("class:[{}] method:[{}] excuter throw exception:[{}]", this.getClass().getName(), "dropTable", e.getMessage()); 380 | } 381 | } else { 382 | logger.info("table {} is not exists", tableName); 383 | } 384 | } 385 | 386 | /** 387 | * 批量删除表 388 | * 389 | * @param tableNames 390 | */ 391 | @Override 392 | public void dropTable(List tableNames) { 393 | Connection connection = hBaseConfig.getConnection(); 394 | try (Admin admin = connection.getAdmin()) { 395 | tableNames.stream().distinct().forEach(tableName -> { 396 | boolean tableExists = tableExists(tableName); 397 | if (tableExists) { 398 | try { 399 | admin.disableTable(TableName.valueOf(tableName)); 400 | admin.deleteTable(TableName.valueOf(tableName)); 401 | logger.info("table {} delete successfully", tableName); 402 | } catch (IOException e) { 403 | } 404 | } else { 405 | logger.info("table {} is not exists", tableName); 406 | } 407 | }); 408 | } catch (IOException e) { 409 | logger.error("class:[{}] method:[{}] excuter throw exception:[{}]", this.getClass().getName(), "dropTable", e.getMessage()); 410 | } 411 | } 412 | 413 | /** 414 | * 清空表 415 | * 416 | * @param tableName 417 | * @throws IOException 418 | */ 419 | @Override 420 | public void truncateTable(String tableName) throws IOException { 421 | boolean tableExists = tableExists(tableName); 422 | if (tableExists) { 423 | Connection connection = hBaseConfig.getConnection(); 424 | Admin admin = connection.getAdmin(); 425 | admin.disableTable(TableName.valueOf(tableName)); 426 | admin.truncateTable(TableName.valueOf(tableName), false); 427 | logger.info("table {} truncate successfully", tableName); 428 | } else { 429 | logger.info("table {} is not exists", tableName); 430 | } 431 | } 432 | 433 | /** 434 | * 删除某列 435 | * 436 | * @param tableName 437 | * @param family 438 | * @param rowKey 439 | * @param column 440 | * @return 441 | * @throws IOException 442 | */ 443 | @Override 444 | public boolean deleteColumn(String tableName, String family, String rowKey, String column) throws IOException { 445 | return this.deleteColumn(tableName, family, rowKey, Arrays.asList(column)); 446 | } 447 | 448 | /** 449 | * 删除某些列 450 | * 451 | * @param tableName 452 | * @param family 453 | * @param rowKey 454 | * @param columns 455 | * @return 456 | * @throws IOException 457 | */ 458 | @Override 459 | public boolean deleteColumn(String tableName, String family, String rowKey, List columns) throws IOException { 460 | if (StringUtils.isBlank(tableName) || StringUtils.isBlank(family) || StringUtils.isBlank(rowKey)) { 461 | logger.info("parameter is error"); 462 | return false; 463 | } 464 | if (columns.isEmpty()) { 465 | logger.info("columns is empty"); 466 | return false; 467 | } 468 | boolean tableExists = tableExists(tableName); 469 | if (tableExists) { 470 | Connection connection = hBaseConfig.getConnection(); 471 | Table table = connection.getTable(TableName.valueOf(tableName)); 472 | Delete delete = new Delete(rowKey.getBytes()); 473 | for (String column : columns) { 474 | delete.addColumn(family.getBytes(), column.getBytes()); 475 | } 476 | table.delete(delete); 477 | logger.info("table {} truncate successfully", tableName); 478 | return true; 479 | } else { 480 | logger.info("table {} is not exists", tableName); 481 | return false; 482 | } 483 | } 484 | 485 | /** 486 | * 获取连接对象 487 | * 488 | * @return 489 | */ 490 | @Override 491 | public Connection getConnection() { 492 | return hBaseConfig.getConnection(); 493 | } 494 | } 495 | -------------------------------------------------------------------------------- /src/test/java/com/yaoqi/hbase/component/assertion/AssertTest.java: -------------------------------------------------------------------------------- 1 | package com.yaoqi.hbase.component.assertion; 2 | 3 | import org.junit.Test; 4 | 5 | /** 6 | * @Author YaoQi 7 | * @Date 2018/8/13 17:17 8 | * @Modified 9 | * @Description 10 | */ 11 | public class AssertTest { 12 | 13 | @Test 14 | public void hasLengthBatch() { 15 | 16 | } 17 | 18 | @Test 19 | public void notNullBatch() { 20 | Object a = null; 21 | String[] b = {"a", "b"}; 22 | Assert.notNullBatch(a, b); 23 | } 24 | 25 | @Test 26 | public void testStr() { 27 | String a = "aa"; 28 | String b = "aa"; 29 | System.out.println(b.contains(a)); 30 | } 31 | 32 | public static void main(String[] args) { 33 | for (;;){ 34 | System.out.println("fuck"); 35 | } 36 | } 37 | 38 | } -------------------------------------------------------------------------------- /src/test/java/com/yaoqi/hbase/component/template/HBaseTemplateTest.java: -------------------------------------------------------------------------------- 1 | package com.yaoqi.hbase.component.template; 2 | 3 | import com.yaoqi.hbase.component.HBaseApp; 4 | import com.yaoqi.hbase.component.operations.HBaseOperations; 5 | import org.apache.hadoop.hbase.CellUtil; 6 | import org.apache.hadoop.hbase.client.*; 7 | import org.apache.hadoop.hbase.filter.*; 8 | import org.apache.hadoop.hbase.util.Bytes; 9 | import org.junit.Test; 10 | import org.junit.runner.RunWith; 11 | import org.springframework.beans.factory.annotation.Autowired; 12 | import org.springframework.boot.test.context.SpringBootTest; 13 | import org.springframework.test.context.junit4.SpringRunner; 14 | 15 | import java.io.IOException; 16 | import java.util.ArrayList; 17 | import java.util.List; 18 | 19 | /** 20 | * @Author YaoQi 21 | * @Date 2018/8/14 11:31 22 | * @Modified 23 | * @Description 24 | */ 25 | @RunWith(SpringRunner.class) 26 | @SpringBootTest(classes = HBaseApp.class) 27 | public class HBaseTemplateTest { 28 | 29 | @Autowired 30 | private HBaseOperations hBaseDao; 31 | 32 | @Autowired 33 | private HBaseAdmin hBaseAdmin; 34 | 35 | /** 36 | * 查询测试 37 | */ 38 | @Test 39 | public void testQueryTable() { 40 | Result result = hBaseDao.queryByTableNameAndRowKey("LBS", 9223372036854775803L); 41 | System.out.println(result.isEmpty()); 42 | result.listCells().forEach(cell -> { 43 | System.out.println("row:" + Bytes.toLong(CellUtil.cloneRow(cell)) + " family:" 44 | + Bytes.toString(CellUtil.cloneFamily(cell)) + " qualifier: " + Bytes.toString(CellUtil.cloneQualifier(cell)) 45 | + " value:" + Bytes.toString(CellUtil.cloneValue(cell))); 46 | }); 47 | } 48 | 49 | @Test 50 | public void testQuery() { 51 | String tableName = "HBase-component"; 52 | String rowKey = "1534154424340"; 53 | Get get = new Get(rowKey.getBytes()); 54 | List getList = new ArrayList<>(); 55 | getList.add(get); 56 | System.out.println(hBaseDao.query(tableName, getList).length); 57 | } 58 | 59 | /** 60 | * 创建表测试 61 | */ 62 | @Test 63 | public void testCreateTable() { 64 | hBaseDao.createTable("HBase-component", "CF1", "CF2"); 65 | } 66 | 67 | /** 68 | * 删除表测试 69 | */ 70 | @Test 71 | public void deleteTable() { 72 | hBaseDao.dropTable("HBase-component_1"); 73 | } 74 | 75 | /** 76 | * 判断表是否存在 77 | */ 78 | @Test 79 | public void testTableExist() { 80 | System.out.println(hBaseDao.tableExists("lbs")); 81 | System.out.println(hBaseDao.tableExists("LBS")); 82 | System.out.println(hBaseDao.tableExists("goods")); 83 | System.out.println(hBaseDao.tableExists("GOODS")); 84 | } 85 | 86 | /** 87 | * 插入一条记录 88 | */ 89 | @Test 90 | public void putTest() { 91 | String rowKey = String.valueOf(System.currentTimeMillis()); 92 | hBaseDao.put("HBase-component", "1534154424340", "CF1", "test_1", Bytes.toBytes("testData")); 93 | } 94 | 95 | /** 96 | * 批量插入数据 97 | */ 98 | @Test 99 | public void testBatchPut() throws IOException { 100 | String rowKey = String.valueOf(System.currentTimeMillis()); 101 | Put put = new Put(rowKey.getBytes()); 102 | String defaultColumn = "CF1"; 103 | String column1 = "col1"; 104 | String column2 = "col2"; 105 | String column3 = "col3"; 106 | 107 | String value = "test"; 108 | put.addColumn(defaultColumn.getBytes(), column1.getBytes(), value.getBytes()); 109 | put.addColumn(defaultColumn.getBytes(), column2.getBytes(), value.getBytes()); 110 | put.addColumn(defaultColumn.getBytes(), column3.getBytes(), value.getBytes()); 111 | 112 | List putList = new ArrayList<>(); 113 | putList.add(put); 114 | putList.add(put); 115 | putList.add(put); 116 | putList.add(put); 117 | putList.add(put); 118 | 119 | hBaseDao.putBatch("HBase-component", putList); 120 | } 121 | 122 | @Test 123 | public void deleteTest() { 124 | hBaseDao.delete("HBase-component", "1534210201115", "CF1", "col2"); 125 | } 126 | 127 | @Test 128 | public void deleteBatchTest() { 129 | String tableName = "HBase-component"; 130 | String rowKey1 = "1534164113922"; 131 | String rowKey2 = "1534168248328"; 132 | 133 | List deleteList = new ArrayList<>(); 134 | Delete delete = new Delete(rowKey1.getBytes()); 135 | Delete delete1 = new Delete(rowKey2.getBytes()); 136 | deleteList.add(delete); 137 | deleteList.add(delete1); 138 | hBaseDao.deleteBatch(tableName, deleteList); 139 | } 140 | 141 | @Test 142 | public void testQueryScan() { 143 | 144 | String tableName = "RECOMMEND_ENGINE_DATA_MODEL"; 145 | List timestampList = new ArrayList<>(); 146 | Long timeDifference = 3L * 30L * 24L * 60L * 60L * 1000L; 147 | Long from = System.currentTimeMillis() - timeDifference; 148 | timestampList.add(from); 149 | TimestampsFilter timestampsFilter = new TimestampsFilter(timestampList); 150 | 151 | Scan scan = new Scan(); 152 | try { 153 | scan.setTimeRange(from, System.currentTimeMillis()); 154 | } catch (IOException e) { 155 | e.printStackTrace(); 156 | } 157 | 158 | ResultScanner resultScanner = hBaseDao.queryByScan(tableName, scan); 159 | 160 | resultScanner.forEach(result -> { 161 | result.listCells().forEach(cell -> { 162 | System.out.println("row:" + Bytes.toLong(CellUtil.cloneRow(cell)) + " family:" 163 | + Bytes.toString(CellUtil.cloneFamily(cell)) + " qualifier: " + Bytes.toString(CellUtil.cloneQualifier(cell)) 164 | + " value:" + Bytes.toString(CellUtil.cloneValue(cell))); 165 | 166 | }); 167 | }); 168 | } 169 | 170 | @Test 171 | public void testPageFilter() { 172 | String tableName = "HBase-component"; 173 | Scan scan = new Scan(); 174 | 175 | PageFilter pageFilter = new PageFilter(1); 176 | 177 | SingleColumnValueFilter singleColumnValueFilter = new SingleColumnValueFilter( 178 | "CF1".getBytes(), 179 | "col1".getBytes(), 180 | CompareFilter.CompareOp.EQUAL, 181 | new SubstringComparator("group")); 182 | 183 | singleColumnValueFilter.setFilterIfMissing(true); 184 | FilterList filterList = new FilterList(); 185 | filterList.addFilter(singleColumnValueFilter); 186 | filterList.addFilter(pageFilter); 187 | 188 | scan.setFilter(filterList); 189 | 190 | ResultScanner resultScanner = hBaseDao.queryByScan(tableName, scan); 191 | 192 | try { 193 | resultScanner.forEach(result -> { 194 | result.listCells().forEach(cell -> { 195 | System.out.println("row:" + Bytes.toString(CellUtil.cloneRow(cell)) + " family:" 196 | + Bytes.toString(CellUtil.cloneFamily(cell)) + " qualifier: " + Bytes.toString(CellUtil.cloneQualifier(cell)) 197 | + " value:" + Bytes.toString(CellUtil.cloneValue(cell))); 198 | 199 | }); 200 | }); 201 | } finally { 202 | if (resultScanner != null) { 203 | resultScanner.close(); 204 | } 205 | } 206 | } 207 | } -------------------------------------------------------------------------------- /src/test/resources/application.yml: -------------------------------------------------------------------------------- 1 | # HBase配置 2 | HBase: 3 | conf: 4 | quorum: 192.168.90.201:2181,192.168.90.202:2181,192.168.90.203:2181,192.168.90.204:2181 5 | znodeParent: /hbase-unsecure 6 | # config: 7 | # #keytab地址路径 8 | # user-keytab: D:\java\work\auth\hbase\test-hbase-kb\root.keytab 9 | # #如果使用kb认证请写以下kerberos否则不用写 10 | # authMethod: kerberos 11 | # #master.kerberos.principal 12 | # masterPrincipal: root/hadoop-beta-01@SEMPTIAN.COM 13 | # #regionserver.kerberos.principal regionserverPrincipal/hbaseSitePath 二选一,一些环境下使用hbaseSitePath更简单 14 | # #regionserverPrincipal: root/_HOST@SEMPTIAN.COM 15 | # #hbseSite配置文件路径D:\kerberos\hbase-kb\hbase-site.xml 16 | # hbaseSitePath: D:\java\work\auth\hbase\test-hbase-kb\hbase-site.xml 17 | # #coreSitePath: D:\kerberos\dev-hbase-kb\core-site.xml --------------------------------------------------------------------------------