├── .gitignore ├── LICENSE ├── README.md ├── pom.xml └── src ├── main ├── java │ └── org │ │ └── opentsdb │ │ └── client │ │ ├── OpenTSDBClient.java │ │ ├── OpenTSDBClientFactory.java │ │ ├── OpenTSDBConfig.java │ │ ├── bean │ │ ├── request │ │ │ ├── Api.java │ │ │ ├── LastPointQuery.java │ │ │ ├── LastPointSubQuery.java │ │ │ ├── Point.java │ │ │ ├── Query.java │ │ │ ├── SubQuery.java │ │ │ └── SuggestQuery.java │ │ └── response │ │ │ ├── DetailResult.java │ │ │ ├── ErrorResponse.java │ │ │ ├── LastPointQueryResult.java │ │ │ └── QueryResult.java │ │ ├── common │ │ └── Json.java │ │ ├── exception │ │ └── http │ │ │ └── HttpException.java │ │ ├── http │ │ ├── HttpClient.java │ │ ├── HttpClientFactory.java │ │ └── callback │ │ │ ├── BatchPutHttpResponseCallback.java │ │ │ ├── GracefulCloseFutureCallBack.java │ │ │ └── QueryHttpResponseCallback.java │ │ ├── sender │ │ ├── consumer │ │ │ ├── Consumer.java │ │ │ ├── ConsumerImpl.java │ │ │ └── ConsumerRunnable.java │ │ ├── package-info.java │ │ └── producer │ │ │ ├── Producer.java │ │ │ └── ProducerImpl.java │ │ └── util │ │ └── ResponseUtil.java └── resources │ └── log4j.properties └── test └── java └── org └── opentsdb └── client ├── ControlTest.java └── CrudTest.java /.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | .idea/ 3 | *.iml 4 | pom.xml.tag 5 | pom.xml.releaseBackup 6 | pom.xml.versionsBackup 7 | pom.xml.next 8 | release.properties 9 | dependency-reduced-pom.xml 10 | buildNumber.properties 11 | .mvn/timing.properties 12 | .mvn/wrapper/maven-wrapper.jar 13 | 14 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # opentsdb-java-sdk 2 | [http api地址](http://opentsdb.net/docs/build/html/api_http/index.html#api-endpoints) 3 | 4 | # 目前实现的功能 5 | * 查询数据，支持同步和异步 6 | * 写入数据，支持异步回调 7 | * 删除数据 8 | * 查询最新数据 9 | * 查询metric、tag_key和tag_value，支持auto_complete 10 |
11 | `源码中CrudTest类提供了一些使用说明和测试，包括并发查询测试和并发写入测试` 12 | 13 | # 快速开始 14 | ## Maven依赖 15 | ```xml 16 | 17 | com.github.eulery 18 | opentsdb-java-sdk 19 | 1.1.6 20 | 21 | ``` 22 | ## 创建连接 23 | ```java 24 | OpenTSDBConfig config = OpenTSDBConfig.address(host, port) 25 | .config(); 26 | OpenTSDBClient client = OpenTSDBClientFactory.connect(config); 27 | 28 | // 优雅关闭连接，会等待所有异步操作完成 29 | client.gracefulClose(); 30 | ``` 31 | ```java 32 | OpenTSDBConfig.address(host, port) 33 | // http连接池大小，默认100 34 | .httpConnectionPool(100) 35 | // http请求超时时间，默认100s 36 | .httpConnectTimeout(100) 37 | // 异步写入数据时，每次http提交的数据条数，默认50 38 | .batchPutSize(50) 39 | // 异步写入数据中，内部有一个队列，默认队列大小20000 40 | .batchPutBufferSize(20000) 41 | // 异步写入等待时间，如果距离上一次请求超多300ms，且有数据，则直接提交 42 | .batchPutTimeLimit(300) 43 | // 当确认这个client只用于查询时设置，可不创建内部队列从而提高效率 44 | .readonly() 45 | // 每批数据提交完成后回调 46 | .batchPutCallBack(new BatchPutHttpResponseCallback.BatchPutCallBack() { 47 | @Override 48 | public void response(List points, DetailResult result) { 49 | // 在请求完成并且response code成功时回调 50 | } 51 | 52 | @Override 53 | public void responseError(List points, DetailResult result) { 54 | // 在response code失败时回调 55 | } 56 | 57 | @Override 58 | public void failed(List points, Exception e) { 59 | // 在发生错误是回调 60 | } 61 | }) 62 | 63 | ``` 64 | ## 查询数据 65 | 具体参数使用说明查看[http api](http://opentsdb.net/docs/build/html/api_http/query/index.html) 66 | ```java 67 | Query query = Query.begin("7d-ago") 68 | .sub(SubQuery.metric("metric.test") 69 | .aggregator(SubQuery.Aggregator.NONE) 70 | .build()) 71 | .build(); 72 | // 同步查询 73 | List resultList = client.query(query); 74 | 75 | // 异步查询 76 | client.query(query, new QueryHttpResponseCallback.QueryCallback() { 77 | @Override 78 | public void response(Query query, List queryResults) { 79 | // 在请求完成并且response code成功时回调 80 | } 81 | 82 | @Override 83 | public void responseError(Query query, HttpException e) { 84 | // 在response code失败时回调 85 | } 86 | 87 | @Override 88 | public void failed(Query query, Exception e) { 89 | // 在发生错误是回调 90 | } 91 | }); 92 | ``` 93 | ## 写入数据 94 | 具体参数使用说明查看[http api](http://opentsdb.net/docs/build/html/api_http/put.html) 95 | ```java 96 | Point point = Point.metric("point") 97 | .tag("testTag", "test") 98 | .value(timestamp, 1.0) 99 | .build(); 100 | client.put(point); 101 | ``` 102 | ## 查询最新数据 103 | 具体参数使用说明查看[http api](http://opentsdb.net/docs/build/html/api_http/query/last.html) 104 | ```java 105 | LastPointQuery query = LastPointQuery.sub(LastPointSubQuery.metric("point") 106 | .tag("testTag", "test_1") 107 | .build()) 108 | // baskScan表示查询最多向前推进多少小时 109 | // 比如在5小时前写入过数据 110 | // 那么backScan(6)可以查出数据，但backScan(4)则不行 111 | .backScan(1000) 112 | .build(); 113 | List lastPointQueryResults = client.queryLast(query); 114 | ``` 115 | 116 | ## 删除数据 117 | 删除数据和查询数据使用同一个Query，查询命中的数据将会被删除 118 | ```java 119 | Query query = Query.begin("7d-ago") 120 | .sub(SubQuery.metric("metric.test") 121 | .aggregator(SubQuery.Aggregator.NONE) 122 | .build()) 123 | .build(); 124 | client.delete(query); 125 | ``` 126 | ## 查询metrics、tagk、tagv 127 | 具体参数使用说明查看[http api](http://opentsdb.net/docs/build/html/api_http/suggest.html) 128 | ```java 129 | SuggestQuery query = SuggestQuery.type(SuggestQuery.Type.METRICS) 130 | .build(); 131 | List suggests = client.querySuggest(query); 132 | ``` 133 | 134 | ## ttl 135 | 如果想为数据设置ttl(time to live)，opentsdb没有直接提供这方面的api，只能通过底层hbase的ttl来完成 136 | ```shell 137 | hbase> describe 'tsdb' 138 | Table tsdb is ENABLED 139 | tsdb, {NAME => 't', VERSIONS => 1, COMPRESSION => 'NONE', TTL => 'FOREVER'} 140 | 141 | hbase> alter ‘tsdb′, NAME => ‘t′, TTL => 8640000 142 | ``` 143 | 144 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | com.github.eulery 8 | opentsdb-java-sdk 9 | 1.1.6 10 | jar 11 | 12 | opentsdb-java-sdk 13 | sdk for opentsdb wrote by java 14 | https://github.com/eulery/opentsdb-java-sdk 15 | 16 | 17 | 2.9.8 18 | 19 | 20 | 21 | 22 | 23 | org.apache.commons 24 | commons-lang3 25 | 3.4 26 | 27 | 28 | org.apache.commons 29 | commons-collections4 30 | 4.3 31 | 32 | 33 | 34 | 35 | org.apache.httpcomponents 36 | httpasyncclient 37 | 4.1.3 38 | 39 | 40 | 41 | 42 | org.slf4j 43 | slf4j-api 44 | 1.7.25 45 | 46 | 47 | org.slf4j 48 | slf4j-log4j12 49 | 1.7.21 50 | test 51 | 52 | 53 | 54 | 55 | com.fasterxml.jackson.core 56 | jackson-core 57 | ${jacksonVersion} 58 | 59 | 60 | com.fasterxml.jackson.core 61 | jackson-databind 62 | ${jacksonVersion} 63 | 64 | 65 | com.fasterxml.jackson.core 66 | jackson-annotations 67 | ${jacksonVersion} 68 | 69 | 70 | 71 | com.fasterxml.jackson.datatype 72 | jackson-datatype-jsr310 73 | ${jacksonVersion} 74 | 75 | 76 | 77 | com.fasterxml.jackson.datatype 78 | jackson-datatype-jdk8 79 | ${jacksonVersion} 80 | 81 | 82 | com.fasterxml.jackson.module 83 | jackson-module-parameter-names 84 | ${jacksonVersion} 85 | 86 | 87 | 88 | 89 | 90 | com.google.guava 91 | guava 92 | 18.0 93 | 94 | 95 | 96 | 97 | junit 98 | junit 99 | 4.12 100 | test 101 | 102 | 103 | org.slf4j 104 | jcl-over-slf4j 105 | 1.7.21 106 | test 107 | 108 | 109 | 110 | 111 | org.projectlombok 112 | lombok 113 | 1.18.0 114 | 115 | 116 | 117 | 118 | 119 | The Apache Software License, Version 2.0 120 | http://www.apache.org/licenses/LICENSE-2.0.txt 121 | 122 | 123 | 124 | 125 | jinyao 126 | 573380618@qq.com 127 | 128 | 129 | 130 | 131 | scm:git:https://github.com/eulery/opentsdb-java-sdk.git 132 | 133 | 134 | scm:git:https://github.com/eulery/opentsdb-java-sdk.git 135 | 136 | https://github.com/eulery/opentsdb-java-sdk 137 | 138 | 139 | 140 | 141 | ossrh 142 | https://oss.sonatype.org/content/repositories/snapshots/ 143 | 144 | 145 | ossrh 146 | https://oss.sonatype.org/service/local/staging/deploy/maven2/ 147 | 148 | 149 | 150 | 151 | 152 | 153 | org.sonatype.plugins 154 | nexus-staging-maven-plugin 155 | 1.6.8 156 | true 157 | 158 | ossrh 159 | https://oss.sonatype.org/ 160 | true 161 | 162 | 163 | 164 | 165 | 166 | org.apache.maven.plugins 167 | maven-source-plugin 168 | 3.0.1 169 | 170 | 171 | attach-sources 172 | 173 | jar-no-fork 174 | 175 | 176 | 177 | 178 | 179 | 180 | 181 | org.apache.maven.plugins 182 | maven-javadoc-plugin 183 | 3.0.0 184 | 185 | false 186 | none 187 | 188 | 189 | 190 | attach-javadocs 191 | 192 | jar 193 | 194 | package 195 | 196 | 197 | 198 | 199 | 200 | org.apache.maven.plugins 201 | maven-gpg-plugin 202 | 1.5 203 | 204 | 205 | sign-artifacts 206 | verify 207 | 208 | sign 209 | 210 | 211 | 212 | 213 | 214 | 215 | org.apache.maven.plugins 216 | maven-compiler-plugin 217 | 218 | 8 219 | 8 220 | 221 | 222 | 223 | 224 | 225 | -------------------------------------------------------------------------------- /src/main/java/org/opentsdb/client/OpenTSDBClient.java: -------------------------------------------------------------------------------- 1 | package org.opentsdb.client; 2 | 3 | import com.fasterxml.jackson.core.JsonProcessingException; 4 | import com.google.common.collect.Lists; 5 | import lombok.extern.slf4j.Slf4j; 6 | import org.apache.http.HttpResponse; 7 | import org.apache.http.nio.reactor.IOReactorException; 8 | import org.opentsdb.client.bean.request.*; 9 | import org.opentsdb.client.bean.response.LastPointQueryResult; 10 | import org.opentsdb.client.bean.response.QueryResult; 11 | import org.opentsdb.client.common.Json; 12 | import org.opentsdb.client.http.HttpClient; 13 | import org.opentsdb.client.http.HttpClientFactory; 14 | import org.opentsdb.client.http.callback.BatchPutHttpResponseCallback; 15 | import org.opentsdb.client.http.callback.QueryHttpResponseCallback; 16 | import org.opentsdb.client.sender.consumer.Consumer; 17 | import org.opentsdb.client.sender.consumer.ConsumerImpl; 18 | import org.opentsdb.client.sender.producer.Producer; 19 | import org.opentsdb.client.sender.producer.ProducerImpl; 20 | import org.opentsdb.client.util.ResponseUtil; 21 | 22 | import java.io.IOException; 23 | import java.lang.reflect.Field; 24 | import java.util.List; 25 | import java.util.concurrent.*; 26 | 27 | /** 28 | * @Description: opentsdb客户端 29 | * @Author: jinyao 30 | * @CreateDate: 2019/2/21 下午9:16 31 | * @Version: 1.0 32 | */ 33 | @Slf4j 34 | public class OpenTSDBClient { 35 | 36 | private final OpenTSDBConfig config; 37 | 38 | private final HttpClient httpClient; 39 | 40 | private Producer producer; 41 | 42 | private Consumer consumer; 43 | 44 | private BlockingQueue queue; 45 | 46 | /*** 47 | * 通过反射来允许删除 48 | */ 49 | private static Field queryDeleteField; 50 | 51 | public OpenTSDBClient(OpenTSDBConfig config) throws IOReactorException { 52 | this.config = config; 53 | this.httpClient = HttpClientFactory.createHttpClient(config); 54 | this.httpClient.start(); 55 | 56 | if (!config.isReadonly()) { 57 | this.queue = new ArrayBlockingQueue<>(config.getBatchPutBufferSize()); 58 | this.producer = new ProducerImpl(queue); 59 | this.consumer = new ConsumerImpl(queue, httpClient, config); 60 | this.consumer.start(); 61 | 62 | try { 63 | queryDeleteField = Query.class.getDeclaredField("delete"); 64 | queryDeleteField.setAccessible(true); 65 | } catch (NoSuchFieldException e) { 66 | e.printStackTrace(); 67 | } 68 | } 69 | 70 | log.debug("the httpclient has started"); 71 | } 72 | 73 | /*** 74 | * 查询数据 75 | * @param query 查询对象 76 | * @return 77 | */ 78 | public List query(Query query) throws IOException, ExecutionException, InterruptedException { 79 | Future future = httpClient.post(Api.QUERY.getPath(), Json.writeValueAsString(query)); 80 | HttpResponse response = future.get(); 81 | List results = Json.readValue(ResponseUtil.getContent(response), List.class, QueryResult.class); 82 | return results; 83 | } 84 | 85 | /*** 86 | * 异步查询 87 | * @param query 查询对象 88 | * @param callback 回调 89 | */ 90 | public void query(Query query, QueryHttpResponseCallback.QueryCallback callback) throws JsonProcessingException { 91 | QueryHttpResponseCallback queryHttpResponseCallback = new QueryHttpResponseCallback(callback, query); 92 | httpClient.post(Api.QUERY.getPath(), Json.writeValueAsString(query), queryHttpResponseCallback); 93 | } 94 | 95 | /*** 96 | * 查询最新的数据 97 | * @param query 查询对象 98 | * @return 99 | */ 100 | public List queryLast(LastPointQuery query) throws IOException, ExecutionException, InterruptedException { 101 | Future future = httpClient.post(Api.LAST.getPath(), Json.writeValueAsString(query)); 102 | HttpResponse response = future.get(); 103 | List results = Json.readValue(ResponseUtil.getContent(response), List.class, LastPointQueryResult.class); 104 | return results; 105 | } 106 | 107 | /*** 108 | * 写入数据 109 | * @param point 数据点 110 | */ 111 | public void put(Point point) { 112 | if (config.isReadonly()) { 113 | throw new IllegalArgumentException("this client is readonly,can't put point"); 114 | } 115 | producer.send(point); 116 | } 117 | 118 | /*** 119 | * 同步写入 120 | * @param point 121 | */ 122 | public void putSync(Point point) throws IOException, ExecutionException, InterruptedException { 123 | this.putSync(Lists.newArrayList(point)); 124 | } 125 | 126 | /*** 127 | * 同步写入 128 | * @param points 129 | */ 130 | public void putSync(List points) throws IOException, ExecutionException, InterruptedException { 131 | Future future = httpClient.post( 132 | Api.PUT.getPath(), 133 | Json.writeValueAsString(points), 134 | new BatchPutHttpResponseCallback() 135 | ); 136 | HttpResponse httpResponse = future.get(); 137 | ResponseUtil.getContent(httpResponse); 138 | } 139 | 140 | /*** 141 | * 同步写入,将会使用创建opentsdbClient时默认的callback 142 | * @param point 143 | */ 144 | public void putSyncWithCallBack(Point point) throws IOException, ExecutionException, InterruptedException { 145 | this.putSyncWithCallBack(Lists.newArrayList(point)); 146 | } 147 | 148 | /*** 149 | * 同步写入,将会使用创建opentsdbClient时默认的callback 150 | * @param points 151 | */ 152 | public void putSyncWithCallBack(List points) throws IOException, ExecutionException, InterruptedException { 153 | Future future = httpClient.post( 154 | Api.PUT.getPath(), 155 | Json.writeValueAsString(points), 156 | new BatchPutHttpResponseCallback(config.getBatchPutCallBack(), points) 157 | ); 158 | HttpResponse httpResponse = future.get(); 159 | ResponseUtil.getContent(httpResponse); 160 | } 161 | 162 | /*** 163 | * 同步写入,使用自定义的callback 164 | * @param point 165 | */ 166 | public void putSyncWithCallBack(Point point, BatchPutHttpResponseCallback.BatchPutCallBack callBack) throws 167 | JsonProcessingException { 168 | this.putSyncWithCallBack(Lists.newArrayList(point), callBack); 169 | } 170 | 171 | /*** 172 | * 同步写入,使用自定义的callback 173 | * @param points 174 | */ 175 | public void putSyncWithCallBack(List points, BatchPutHttpResponseCallback.BatchPutCallBack callBack) throws JsonProcessingException { 176 | httpClient.post( 177 | Api.PUT.getPath(), 178 | Json.writeValueAsString(points), 179 | new BatchPutHttpResponseCallback(callBack, points) 180 | ); 181 | } 182 | 183 | /*** 184 | * 同步写入 185 | * @param point 186 | */ 187 | public void putSync(Point point, BatchPutHttpResponseCallback.BatchPutCallBack callBack) throws IOException, ExecutionException, InterruptedException { 188 | this.putSync(Lists.newArrayList(point)); 189 | } 190 | 191 | /*** 192 | * 删除数据，返回删除的数据 193 | * @param query 查询对象 194 | */ 195 | public List delete(Query query) throws IllegalAccessException, ExecutionException, InterruptedException, IOException { 196 | if (config.isReadonly()) { 197 | throw new IllegalArgumentException("this client is readonly,can't delete data"); 198 | } 199 | queryDeleteField.set(query, true); 200 | Future future = httpClient.post(Api.QUERY.getPath(), Json.writeValueAsString(query)); 201 | HttpResponse response = future.get(); 202 | List results = Json.readValue(ResponseUtil.getContent(response), List.class, QueryResult.class); 203 | return results; 204 | } 205 | 206 | /*** 207 | * 查询metric、tag_key、tag_value的信息 208 | * @param query 209 | * @return 210 | */ 211 | public List querySuggest(SuggestQuery query) throws ExecutionException, InterruptedException, IOException { 212 | Future future = httpClient.post(Api.SUGGEST.getPath(), Json.writeValueAsString(query)); 213 | HttpResponse response = future.get(); 214 | List results = Json.readValue(ResponseUtil.getContent(response), List.class, String.class); 215 | return results; 216 | } 217 | 218 | /*** 219 | * 优雅关闭链接，会等待所有消费者线程结束 220 | */ 221 | public void gracefulClose() throws IOException { 222 | if (!config.isReadonly()) { 223 | // 先停止写入 224 | this.producer.forbiddenSend(); 225 | // 等待队列被消费空 226 | this.waitEmpty(); 227 | // 关闭消费者 228 | this.consumer.gracefulStop(); 229 | } 230 | this.httpClient.gracefulClose(); 231 | } 232 | 233 | /*** 234 | * 等待队列被消费空 235 | */ 236 | private void waitEmpty() { 237 | while (!queue.isEmpty()) { 238 | try { 239 | TimeUnit.MILLISECONDS.sleep(config.getBatchPutTimeLimit()); 240 | } catch (InterruptedException e) { 241 | e.printStackTrace(); 242 | } 243 | } 244 | } 245 | 246 | /*** 247 | * 强行关闭 248 | */ 249 | public void forceClose() throws IOException { 250 | if (!config.isReadonly()) { 251 | this.consumer.forceStop(); 252 | } 253 | this.httpClient.forceClose(); 254 | } 255 | 256 | } 257 | -------------------------------------------------------------------------------- /src/main/java/org/opentsdb/client/OpenTSDBClientFactory.java: -------------------------------------------------------------------------------- 1 | package org.opentsdb.client; 2 | 3 | import org.apache.http.nio.reactor.IOReactorException; 4 | 5 | /** 6 | * @Description: openstadb客户端工厂类 7 | * @Author: jinyao 8 | * @CreateDate: 2019/2/21 下午9:13 9 | * @Version: 1.0 10 | */ 11 | public class OpenTSDBClientFactory { 12 | 13 | public static OpenTSDBClient connect(OpenTSDBConfig config) throws IOReactorException { 14 | return new OpenTSDBClient(config); 15 | } 16 | 17 | } 18 | -------------------------------------------------------------------------------- /src/main/java/org/opentsdb/client/OpenTSDBConfig.java: -------------------------------------------------------------------------------- 1 | package org.opentsdb.client; 2 | 3 | import lombok.Data; 4 | import org.opentsdb.client.http.callback.BatchPutHttpResponseCallback; 5 | 6 | /** 7 | * @Description: opentsdb通用配置 8 | * @Author: jinyao 9 | * @CreateDate: 2019/2/21 下午9:06 10 | * @Version: 1.0 11 | */ 12 | @Data 13 | public class OpenTSDBConfig { 14 | 15 | private String host; 16 | 17 | private int port; 18 | 19 | private int httpConnectionPool; 20 | 21 | private int httpConnectTimeout; 22 | 23 | private int putConsumerThreadCount; 24 | 25 | private int batchPutSize; 26 | 27 | private int batchPutBufferSize; 28 | 29 | private int batchPutTimeLimit; 30 | 31 | private boolean readonly; 32 | 33 | private BatchPutHttpResponseCallback.BatchPutCallBack batchPutCallBack; 34 | 35 | public static class Builder { 36 | 37 | private String host; 38 | 39 | private int port; 40 | 41 | /** 42 | * 每个Host分配的连接数 43 | */ 44 | private int httpConnectionPool = 100; 45 | 46 | /** 47 | * 单位：秒 48 | */ 49 | private int httpConnectTimeout = 100; 50 | 51 | /*** 52 | * 发送数据时，消费者线程 53 | */ 54 | private int putConsumerThreadCount = 2; 55 | 56 | /** 57 | * 每个http请求提交的数据大小 58 | */ 59 | private int batchPutSize = 50; 60 | 61 | /*** 62 | * 生产着消费者模式中，缓冲池的大小 63 | */ 64 | private int batchPutBufferSize = 20000; 65 | 66 | /*** 67 | * 每次提交等待的最大时间限制，单位ms 68 | */ 69 | private int batchPutTimeLimit = 300; 70 | 71 | /*** 72 | * 如果确定不写入数据，可以把这个属性设置为true，将不会开启写数据用的队列和线程池 73 | */ 74 | private boolean readonly = false; 75 | 76 | /*** 77 | * 对这个client实例的批量写入设置一个回调接口 78 | */ 79 | private BatchPutHttpResponseCallback.BatchPutCallBack batchPutCallBack; 80 | 81 | public Builder(String host, int port) { 82 | this.host = host; 83 | this.port = port; 84 | } 85 | 86 | public OpenTSDBConfig config() { 87 | OpenTSDBConfig config = new OpenTSDBConfig(); 88 | 89 | config.host = this.host; 90 | config.port = this.port; 91 | config.httpConnectTimeout = this.httpConnectTimeout; 92 | config.httpConnectionPool = this.httpConnectionPool; 93 | config.putConsumerThreadCount = this.putConsumerThreadCount; 94 | config.batchPutSize = this.batchPutSize; 95 | config.batchPutBufferSize = this.batchPutBufferSize; 96 | config.batchPutTimeLimit = this.batchPutTimeLimit; 97 | config.readonly = this.readonly; 98 | config.batchPutCallBack = this.batchPutCallBack; 99 | 100 | return config; 101 | } 102 | 103 | public Builder httpConnectionPool(int connectionPool) { 104 | if (connectionPool < 1) { 105 | throw new IllegalArgumentException("The ConnectionPool can't be less then 1"); 106 | } 107 | httpConnectionPool = connectionPool; 108 | return this; 109 | } 110 | 111 | public Builder httpConnectTimeout(int httpConnectTimeout) { 112 | if (httpConnectTimeout <= 0) { 113 | throw new IllegalArgumentException("The connectTimtout can't be less then 0"); 114 | } 115 | this.httpConnectTimeout = httpConnectTimeout; 116 | return this; 117 | } 118 | 119 | public Builder putConsumerThreadCount(int putConsumerThreadCount) { 120 | if (putConsumerThreadCount < 1) { 121 | throw new IllegalArgumentException("The threadCount can't be less then 1"); 122 | } 123 | this.putConsumerThreadCount = putConsumerThreadCount; 124 | return this; 125 | } 126 | 127 | public Builder batchPutSize(int batchPutSize) { 128 | if (batchPutSize < 1) { 129 | throw new IllegalArgumentException("The size can't be less then 1"); 130 | } 131 | this.batchPutSize = batchPutSize; 132 | return this; 133 | } 134 | 135 | public Builder batchPutBufferSize(int batchPutBufferSize) { 136 | if (batchPutBufferSize < 1) { 137 | throw new IllegalArgumentException("The size can't be less then 1"); 138 | } 139 | this.batchPutBufferSize = batchPutBufferSize; 140 | return this; 141 | } 142 | 143 | public Builder batchPutTimeLimit(int batchPutTimeLimit) { 144 | if (batchPutTimeLimit < 1) { 145 | throw new IllegalArgumentException("The time limit can't be less then 1"); 146 | } 147 | this.batchPutTimeLimit = batchPutTimeLimit; 148 | return this; 149 | } 150 | 151 | public Builder readonly() { 152 | this.readonly = true; 153 | return this; 154 | } 155 | 156 | public Builder batchPutCallBack(BatchPutHttpResponseCallback.BatchPutCallBack batchPutCallBack){ 157 | this.batchPutCallBack = batchPutCallBack; 158 | return this; 159 | } 160 | 161 | } 162 | 163 | public static Builder address(String host, int port) { 164 | return new Builder(host, port); 165 | } 166 | 167 | } 168 | -------------------------------------------------------------------------------- /src/main/java/org/opentsdb/client/bean/request/Api.java: -------------------------------------------------------------------------------- 1 | package org.opentsdb.client.bean.request; 2 | 3 | /** 4 | * api地址 5 | * 6 | * @Description: 7 | * @Author: jinyao 8 | * @CreateDate: 2019/2/23 下午12:49 9 | * @Version: 1.0 10 | */ 11 | public enum Api { 12 | 13 | /*** 14 | * path对应api地址 15 | */ 16 | PUT("/api/put"), 17 | PUT_DETAIL("/api/put?details=true"), 18 | QUERY("/api/query"), 19 | LAST("/api/query/last"), 20 | SUGGEST("/api/suggest"); 21 | 22 | private String path; 23 | 24 | Api(String path) { 25 | this.path = path; 26 | } 27 | 28 | public String getPath() { 29 | return path; 30 | } 31 | 32 | public void setPath(String path) { 33 | this.path = path; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/org/opentsdb/client/bean/request/LastPointQuery.java: -------------------------------------------------------------------------------- 1 | package org.opentsdb.client.bean.request; 2 | 3 | import lombok.Data; 4 | 5 | import java.util.ArrayList; 6 | import java.util.List; 7 | 8 | /** 9 | * @Author: jinyao 10 | * @Description: 11 | * @CreateDate: 2019/2/24 下午2:48 12 | * @Version: 1.0 13 | */ 14 | @Data 15 | public class LastPointQuery { 16 | 17 | private List queries; 18 | 19 | private boolean resolveNames; 20 | 21 | private int backScan; 22 | 23 | public static class Builder { 24 | 25 | private List queries = new ArrayList<>(); 26 | 27 | private boolean resolveNames = true; 28 | 29 | private int backScan; 30 | 31 | public LastPointQuery build() { 32 | LastPointQuery query = new LastPointQuery(); 33 | query.queries = this.queries; 34 | query.resolveNames = this.resolveNames; 35 | query.backScan = this.backScan; 36 | return query; 37 | } 38 | 39 | public Builder(LastPointSubQuery query) { 40 | this.queries.add(query); 41 | } 42 | 43 | public Builder(List queries) { 44 | this.queries = queries; 45 | } 46 | 47 | public Builder resolveNames(boolean resolveNames) { 48 | this.resolveNames = resolveNames; 49 | return this; 50 | } 51 | 52 | public Builder backScan(int backScan) { 53 | this.backScan = backScan; 54 | return this; 55 | } 56 | 57 | } 58 | 59 | public static Builder sub(LastPointSubQuery query) { 60 | return new Builder(query); 61 | } 62 | 63 | public static Builder sub(List queries) { 64 | return new Builder(queries); 65 | } 66 | 67 | } 68 | -------------------------------------------------------------------------------- /src/main/java/org/opentsdb/client/bean/request/LastPointSubQuery.java: -------------------------------------------------------------------------------- 1 | package org.opentsdb.client.bean.request; 2 | 3 | import lombok.Data; 4 | import org.apache.commons.collections4.MapUtils; 5 | import org.apache.commons.lang3.StringUtils; 6 | 7 | import java.util.HashMap; 8 | import java.util.Map; 9 | 10 | /** 11 | * @Author: jinyao 12 | * @Description: 13 | * @CreateDate: 2019/2/24 下午2:48 14 | * @Version: 1.0 15 | */ 16 | @Data 17 | public class LastPointSubQuery { 18 | 19 | private String metric; 20 | 21 | private Map tags; 22 | 23 | public static class Builder { 24 | 25 | private String metric; 26 | 27 | private Map tags = new HashMap<>(); 28 | 29 | public LastPointSubQuery build() { 30 | LastPointSubQuery query = new LastPointSubQuery(); 31 | query.metric = this.metric; 32 | query.tags = this.tags; 33 | return query; 34 | } 35 | 36 | public Builder(String metric) { 37 | this.metric = metric; 38 | } 39 | 40 | public Builder tag(String tagk, String tagv) { 41 | if (StringUtils.isNoneBlank(tagk) && StringUtils.isNoneBlank(tagv)) { 42 | this.tags.put(tagk, tagv); 43 | } 44 | return this; 45 | } 46 | 47 | public Builder tag(Map tags) { 48 | if (!MapUtils.isEmpty(tags)) { 49 | this.tags.putAll(tags); 50 | } 51 | return this; 52 | } 53 | 54 | } 55 | 56 | public static Builder metric(String metric) { 57 | return new Builder(metric); 58 | } 59 | 60 | } 61 | -------------------------------------------------------------------------------- /src/main/java/org/opentsdb/client/bean/request/Point.java: -------------------------------------------------------------------------------- 1 | package org.opentsdb.client.bean.request; 2 | 3 | import lombok.Data; 4 | import org.apache.commons.collections4.MapUtils; 5 | import org.apache.commons.lang3.StringUtils; 6 | 7 | import java.util.HashMap; 8 | import java.util.Map; 9 | import java.util.Objects; 10 | 11 | /** 12 | * 构建数据点对象 13 | * 14 | * @Author: jinyao 15 | * @Description: 16 | * @CreateDate: 2019/2/23 下午1:52 17 | * @Version: 1.0 18 | */ 19 | @Data 20 | public class Point { 21 | 22 | private String metric; 23 | 24 | private Map tags = new HashMap<>(); 25 | 26 | private Number value; 27 | 28 | private long timestamp; 29 | 30 | public static MetricBuilder metric(String metric) { 31 | return new MetricBuilder(metric); 32 | } 33 | 34 | public static class MetricBuilder { 35 | 36 | private String metric; 37 | 38 | private Map tags = new HashMap<>(); 39 | 40 | private Number value; 41 | 42 | private long timestamp; 43 | 44 | public MetricBuilder(String metric) { 45 | if (StringUtils.isBlank(metric)) { 46 | throw new IllegalArgumentException("The metric can't be empty"); 47 | } 48 | this.metric = metric; 49 | } 50 | 51 | public MetricBuilder value(long timestamp, Number value) { 52 | if (timestamp == 0) { 53 | throw new IllegalArgumentException("timestamp must gt 0"); 54 | } 55 | Objects.requireNonNull(value); 56 | this.timestamp = timestamp; 57 | this.value = value; 58 | return this; 59 | } 60 | 61 | public MetricBuilder tag(final String tagName, final String value) { 62 | if (StringUtils.isNoneBlank(tagName) && StringUtils.isNoneBlank(value)) { 63 | tags.put(tagName, value); 64 | } 65 | return this; 66 | } 67 | 68 | public MetricBuilder tag(final Map tags) { 69 | if (!MapUtils.isEmpty(tags)) { 70 | this.tags.putAll(tags); 71 | } 72 | return this; 73 | } 74 | 75 | public Point build() { 76 | Point point = new Point(); 77 | point.metric = this.metric; 78 | 79 | if (MapUtils.isEmpty(tags)) { 80 | throw new IllegalArgumentException("tags can't be empty"); 81 | } 82 | point.tags = this.tags; 83 | 84 | point.timestamp = this.timestamp; 85 | point.value = this.value; 86 | return point; 87 | } 88 | 89 | } 90 | 91 | } 92 | -------------------------------------------------------------------------------- /src/main/java/org/opentsdb/client/bean/request/Query.java: -------------------------------------------------------------------------------- 1 | package org.opentsdb.client.bean.request; 2 | 3 | import lombok.Data; 4 | import org.apache.commons.collections4.CollectionUtils; 5 | import org.apache.commons.lang3.StringUtils; 6 | 7 | import java.util.ArrayList; 8 | import java.util.List; 9 | 10 | /** 11 | * 详见http://opentsdb.net/docs/build/html/api_http/query/index.html 12 | * 13 | * @Description: 14 | * @Author: jinyao 15 | * @CreateDate: 2019/2/21 下午9:45 16 | * @Version: 1.0 17 | */ 18 | @Data 19 | public class Query { 20 | 21 | /** 22 | * 形如1h-ago 23 | */ 24 | private String start; 25 | 26 | /*** 27 | * 形如1s-ago 28 | */ 29 | private String end; 30 | 31 | private Boolean msResolution; 32 | 33 | private Boolean noAnnotations; 34 | 35 | private Boolean globalAnnotations; 36 | 37 | private Boolean showTSUIDs; 38 | 39 | private Boolean showSummary; 40 | 41 | private Boolean showStats; 42 | 43 | private Boolean showQuery; 44 | 45 | private String timezone; 46 | 47 | private Boolean useCalendar; 48 | 49 | private Boolean delete; 50 | 51 | private List queries; 52 | 53 | public static class Builder { 54 | 55 | private Long startTimestamp; 56 | 57 | private Long endTimestamp; 58 | 59 | private String start; 60 | 61 | private String end; 62 | 63 | private Boolean msResolution; 64 | 65 | private Boolean noAnnotations; 66 | 67 | private Boolean globalAnnotations; 68 | 69 | private Boolean showTSUIDs; 70 | 71 | private Boolean showSummary; 72 | 73 | private Boolean showStats; 74 | 75 | private Boolean showQuery; 76 | 77 | private String timezone; 78 | 79 | private Boolean useCalendar; 80 | 81 | private List queries = new ArrayList<>(); 82 | 83 | public Query build() { 84 | Query query = new Query(); 85 | if (this.startTimestamp == null && StringUtils.isBlank(this.start)) { 86 | throw new IllegalArgumentException("the start time must be set"); 87 | } 88 | 89 | if (CollectionUtils.isEmpty(queries)) { 90 | throw new IllegalArgumentException("the subQueries must be set"); 91 | } 92 | query.queries = this.queries; 93 | 94 | if (StringUtils.isNoneBlank(start)) { 95 | query.start = this.start; 96 | } else if (this.startTimestamp != null) { 97 | query.start = this.startTimestamp.toString(); 98 | } 99 | 100 | if (StringUtils.isNoneBlank(end)) { 101 | query.end = this.end; 102 | } else if (this.endTimestamp != null) { 103 | query.end = this.endTimestamp.toString(); 104 | } 105 | 106 | query.msResolution = this.msResolution; 107 | query.noAnnotations = this.noAnnotations; 108 | query.globalAnnotations = this.globalAnnotations; 109 | query.showTSUIDs = this.showTSUIDs; 110 | query.showSummary = this.showSummary; 111 | query.showStats = this.showStats; 112 | query.showQuery = this.showQuery; 113 | query.timezone = this.timezone; 114 | query.useCalendar = this.useCalendar; 115 | return query; 116 | } 117 | 118 | public Builder begin(Long startTimestamp) { 119 | this.startTimestamp = startTimestamp; 120 | return this; 121 | } 122 | 123 | public Builder end(Long endTimestamp) { 124 | this.endTimestamp = endTimestamp; 125 | return this; 126 | } 127 | 128 | public Builder begin(String start) { 129 | this.start = start; 130 | return this; 131 | } 132 | 133 | public Builder end(String end) { 134 | this.end = end; 135 | return this; 136 | } 137 | 138 | public Builder msResolution() { 139 | this.msResolution = true; 140 | return this; 141 | } 142 | 143 | public Builder noAnnotations() { 144 | this.noAnnotations = true; 145 | return this; 146 | } 147 | 148 | public Builder globalAnnotations() { 149 | this.globalAnnotations = true; 150 | return this; 151 | } 152 | 153 | public Builder showTSUIDs() { 154 | this.showTSUIDs = true; 155 | return this; 156 | } 157 | 158 | public Builder showSummary() { 159 | this.showSummary = true; 160 | return this; 161 | } 162 | 163 | public Builder showStats() { 164 | this.showStats = true; 165 | return this; 166 | } 167 | 168 | public Builder showQuery() { 169 | this.showQuery = true; 170 | return this; 171 | } 172 | 173 | public Builder timezone(String timezone) { 174 | this.timezone = timezone; 175 | return this; 176 | } 177 | 178 | public Builder useCalendar() { 179 | this.useCalendar = true; 180 | return this; 181 | } 182 | 183 | public Builder sub(SubQuery subQuery) { 184 | queries.add(subQuery); 185 | return this; 186 | } 187 | 188 | public Builder sub(List subQueryList) { 189 | if (!CollectionUtils.isEmpty(subQueryList)) { 190 | queries.addAll(subQueryList); 191 | } 192 | return this; 193 | } 194 | 195 | } 196 | 197 | public static Builder begin(Long startTimestamp) { 198 | return new Builder().begin(startTimestamp); 199 | } 200 | 201 | public static Builder begin(String start) { 202 | return new Builder().begin(start); 203 | } 204 | 205 | /*** 206 | * 设置私有，不允许用户设置delete属性，会在client中通过反射来设置 207 | * @param delete 208 | */ 209 | private void setDelete(Boolean delete) { 210 | this.delete = delete; 211 | } 212 | } 213 | -------------------------------------------------------------------------------- /src/main/java/org/opentsdb/client/bean/request/SubQuery.java: -------------------------------------------------------------------------------- 1 | package org.opentsdb.client.bean.request; 2 | 3 | import com.fasterxml.jackson.annotation.JsonValue; 4 | import com.google.common.collect.Maps; 5 | import lombok.Data; 6 | import org.apache.commons.collections4.CollectionUtils; 7 | import org.apache.commons.collections4.MapUtils; 8 | import org.apache.commons.lang3.StringUtils; 9 | 10 | import java.util.*; 11 | 12 | /** 13 | * 子查询，详见http://opentsdb.net/docs/build/html/api_http/query/index.html 14 | * 15 | * @Description: 16 | * @Author: jinyao 17 | * @CreateDate: 2019/2/21 下午10:37 18 | * @Version: 1.0 19 | */ 20 | @Data 21 | public class SubQuery { 22 | 23 | private Aggregator aggregator; 24 | 25 | private String metric; 26 | 27 | private String downsample; 28 | 29 | private Boolean rate; 30 | 31 | private RateOptions rateOptions; 32 | 33 | private Map tags; 34 | 35 | private Boolean explicitTags; 36 | 37 | private String rollupUsage; 38 | 39 | private List filters; 40 | 41 | public static MetricBuilder metric(String metric) { 42 | return new MetricBuilder(metric); 43 | } 44 | 45 | public static AggregatorBuilder aggregator(Aggregator aggregator) { 46 | return new AggregatorBuilder(aggregator); 47 | } 48 | 49 | /*** 50 | * subQuery builder 51 | */ 52 | public static class Builder { 53 | 54 | private Aggregator aggregator; 55 | 56 | private String metric; 57 | 58 | private String downsample; 59 | 60 | private Boolean rate; 61 | 62 | private RateOptions rateOptions; 63 | 64 | private Map tags = new HashMap<>(); 65 | 66 | private Boolean explicitTags; 67 | 68 | private String rollupUsage; 69 | 70 | private List filters = new ArrayList<>(); 71 | 72 | public Builder(String metric, Aggregator aggregator) { 73 | Objects.requireNonNull(metric, "metric"); 74 | Objects.requireNonNull(aggregator, "aggregator"); 75 | this.aggregator = aggregator; 76 | this.metric = metric; 77 | } 78 | 79 | public Builder rate() { 80 | this.rate = true; 81 | return this; 82 | } 83 | 84 | public Builder rate(RateOptions rateOptions) { 85 | this.rate = true; 86 | this.rateOptions = rateOptions; 87 | return this; 88 | } 89 | 90 | public Builder downsample(String downsample) { 91 | if (StringUtils.isNoneBlank(downsample)) { 92 | this.downsample = downsample; 93 | } 94 | return this; 95 | } 96 | 97 | public Builder rollupUsage(String rollupUsage) { 98 | if (StringUtils.isNoneBlank(rollupUsage)) { 99 | this.rollupUsage = rollupUsage; 100 | } 101 | return this; 102 | } 103 | 104 | public Builder explicitTags() { 105 | this.explicitTags = true; 106 | return this; 107 | } 108 | 109 | public Builder tag(String tagk, String tagv) { 110 | if (StringUtils.isNoneBlank(tagk) && StringUtils.isNoneBlank(tagv)) { 111 | this.tags.put(tagk, tagv); 112 | } 113 | return this; 114 | } 115 | 116 | public Builder tag(Map tags) { 117 | if (!MapUtils.isEmpty(tags)) { 118 | this.tags.putAll(tags); 119 | } 120 | return this; 121 | } 122 | 123 | public Builder filter(Filter filter) { 124 | filters.add(filter); 125 | return this; 126 | } 127 | 128 | public Builder filter(List filterList) { 129 | if (!CollectionUtils.isEmpty(filterList)) { 130 | filters.addAll(filterList); 131 | } 132 | return this; 133 | } 134 | 135 | public SubQuery build() { 136 | SubQuery subQuery = new SubQuery(); 137 | subQuery.aggregator = this.aggregator; 138 | subQuery.downsample = this.downsample; 139 | subQuery.metric = this.metric; 140 | subQuery.tags = this.tags; 141 | subQuery.filters = this.filters; 142 | subQuery.rate = this.rate; 143 | subQuery.rateOptions = this.rateOptions; 144 | subQuery.explicitTags = this.explicitTags; 145 | subQuery.rollupUsage = this.rollupUsage; 146 | return subQuery; 147 | } 148 | 149 | } 150 | 151 | /*** 152 | * metric和aggregator是必传参数 153 | */ 154 | public static class MetricBuilder { 155 | private String metric; 156 | 157 | public MetricBuilder(String metric) { 158 | this.metric = metric; 159 | } 160 | 161 | public Builder aggregator(Aggregator aggregator) { 162 | return new Builder(metric, aggregator); 163 | } 164 | } 165 | 166 | public static class AggregatorBuilder { 167 | private Aggregator aggregator; 168 | 169 | public AggregatorBuilder(Aggregator aggregator) { 170 | this.aggregator = aggregator; 171 | } 172 | 173 | public Builder metric(String metric) { 174 | return new Builder(metric, aggregator); 175 | } 176 | } 177 | 178 | /*** 179 | * 聚合方法的枚举类 180 | */ 181 | public static enum Aggregator { 182 | 183 | /*** 184 | * 聚合方法 185 | */ 186 | MULT("mult"), P90("p90"), ZIMSUM("zimsum"), MIMMAX("mimmax"), SUM("sum"), 187 | P50("p50"), NONE("none"), P95("p95"), EP99R7("ep99r7"), P75("p75"), P99("p99"), 188 | EP99R3("ep99r3"), EP95R7("ep95r7"), MIN("min"), AVG("avg"), EP75R7("ep75r7"), 189 | DEV("dev"), EP95R3("ep95r3"), EP75R3("ep75r3"), EP50R7("ep50r7"), EP90R7("ep90r7"), 190 | MIMMIN("mimmin"), P999("p999"), EP50R3("ep50r3"), EP90R3("ep90r3"), EP999R7("ep999r7"), 191 | LAST("last"), MAX("max"), COUNT("count"), EP999R3("ep999r3"), FIRST("first"); 192 | 193 | private static final Map AGG_MAP = Maps.newHashMapWithExpectedSize(Aggregator.values().length); 194 | 195 | static { 196 | for (Aggregator typeEnum : Aggregator.values()) { 197 | AGG_MAP.put(typeEnum.getName(), typeEnum); 198 | } 199 | } 200 | 201 | public static Aggregator getEnum(String name) { 202 | return AGG_MAP.get(name); 203 | } 204 | 205 | private String name; 206 | 207 | private Aggregator(String name) { 208 | this.name = name; 209 | } 210 | 211 | @JsonValue 212 | public String getName() { 213 | return name; 214 | } 215 | 216 | @Override 217 | public String toString() { 218 | return name; 219 | } 220 | 221 | } 222 | 223 | /*** 224 | * 速率 225 | */ 226 | @Data 227 | public static class RateOptions { 228 | 229 | private Boolean counter; 230 | 231 | private Boolean dropResets; 232 | 233 | private Long counterMax; 234 | 235 | private Long resetValue; 236 | 237 | public static Builder newBuilder() { 238 | return new Builder(); 239 | } 240 | 241 | public static class Builder { 242 | 243 | private Boolean counter; 244 | 245 | private Boolean dropResets; 246 | 247 | private Long counterMax; 248 | 249 | private Long resetValue; 250 | 251 | public Builder() { 252 | 253 | } 254 | 255 | public Builder counter(boolean counter) { 256 | this.counter = counter; 257 | return this; 258 | } 259 | 260 | public Builder dropResets(boolean dropResets) { 261 | this.dropResets = dropResets; 262 | return this; 263 | } 264 | 265 | public Builder counterMax(long counterMax) { 266 | this.counterMax = counterMax; 267 | return this; 268 | } 269 | 270 | public Builder resetValue(long resetValue) { 271 | this.resetValue = resetValue; 272 | return this; 273 | } 274 | 275 | public RateOptions build() { 276 | RateOptions rateOptions = new RateOptions(); 277 | if (counter != null) { 278 | rateOptions.counter = counter; 279 | } 280 | if (dropResets != null) { 281 | rateOptions.dropResets = dropResets; 282 | } 283 | if (counterMax != null) { 284 | rateOptions.counterMax = counterMax; 285 | } 286 | if (resetValue != null) { 287 | rateOptions.resetValue = resetValue; 288 | } 289 | return rateOptions; 290 | } 291 | } 292 | 293 | } 294 | 295 | /*** 296 | * 查询过滤器 297 | */ 298 | @Data 299 | public static class Filter { 300 | 301 | private FilterType type; 302 | 303 | private String tagk; 304 | 305 | private String filter; 306 | 307 | private Boolean groupBy; 308 | 309 | public static class Builder { 310 | 311 | private FilterType type; 312 | 313 | private String tagk; 314 | 315 | private String filter; 316 | 317 | private Boolean groupBy; 318 | 319 | public Builder(FilterType type, String tagk, String filter, Boolean groupBy) { 320 | super(); 321 | this.type = type; 322 | this.tagk = tagk; 323 | this.filter = filter; 324 | this.groupBy = groupBy; 325 | } 326 | 327 | public Builder(FilterType type, String tagk, String filter) { 328 | super(); 329 | this.type = type; 330 | this.tagk = tagk; 331 | this.filter = filter; 332 | } 333 | 334 | public Filter build() { 335 | Filter f = new Filter(); 336 | f.type = this.type; 337 | f.tagk = this.tagk; 338 | f.filter = this.filter; 339 | if (this.groupBy == true) { 340 | f.groupBy = this.groupBy; 341 | } 342 | 343 | return f; 344 | } 345 | 346 | } 347 | 348 | public static Builder filter(FilterType type, String tagk, String filter) { 349 | return new Builder(type, tagk, filter); 350 | } 351 | 352 | public static Builder filter(FilterType type, String tagk, String filter, Boolean groupBy) { 353 | return new Builder(type, tagk, filter, groupBy); 354 | } 355 | 356 | public static Builder filter(FilterType type, String filter) { 357 | return new Builder(type, null, filter); 358 | } 359 | 360 | /*** 361 | * 过滤类型枚举类 362 | */ 363 | public static enum FilterType { 364 | 365 | /** 366 | * 具体说明可以手动调用openTSDB的接口获取 367 | * get请求ip:port/api/config/filters 368 | */ 369 | REGEXP("regexp"), 370 | IWILDCARD("iwildcard"), 371 | ILITERAL_OR("iliteral_or"), 372 | NOT_ILITERAL_OR("not_iliteral_or"), 373 | NOT_LITERAL_OR("not_literal_or"), 374 | LITERAL_OR("literal_or"); 375 | 376 | private String name; 377 | 378 | private FilterType(String name) { 379 | this.name = name; 380 | } 381 | 382 | @JsonValue 383 | public String getName() { 384 | return name; 385 | } 386 | 387 | public void setName(String name) { 388 | this.name = name; 389 | } 390 | 391 | @Override 392 | public String toString() { 393 | return name; 394 | } 395 | 396 | } 397 | 398 | } 399 | 400 | } 401 | -------------------------------------------------------------------------------- /src/main/java/org/opentsdb/client/bean/request/SuggestQuery.java: -------------------------------------------------------------------------------- 1 | package org.opentsdb.client.bean.request; 2 | 3 | import com.fasterxml.jackson.annotation.JsonValue; 4 | import lombok.Data; 5 | import org.apache.commons.lang3.StringUtils; 6 | 7 | import java.util.Objects; 8 | 9 | /** 10 | * @Author: jinyao 11 | * @Description: 12 | * @CreateDate: 2019/3/9 下午2:45 13 | * @Version: 1.0 14 | */ 15 | @Data 16 | public class SuggestQuery { 17 | 18 | private Type type; 19 | 20 | private String q; 21 | 22 | private Integer max; 23 | 24 | public static enum Type { 25 | /*** 26 | * 所查询的元数据类型 27 | */ 28 | METRICS("metrics"), 29 | TAG_KEY("tagk"), 30 | TAG_VALUE("tagv"); 31 | 32 | private String value; 33 | 34 | Type(String value) { 35 | this.value = value; 36 | } 37 | 38 | @JsonValue 39 | public String getValue() { 40 | return value; 41 | } 42 | 43 | public void setValue(String value) { 44 | this.value = value; 45 | } 46 | } 47 | 48 | public static Builder type(Type type) { 49 | return new Builder(type); 50 | } 51 | 52 | public static class Builder { 53 | 54 | private Type type; 55 | 56 | private String q; 57 | 58 | private Integer max; 59 | 60 | public Builder(Type type) { 61 | Objects.requireNonNull(type); 62 | this.type = type; 63 | } 64 | 65 | public SuggestQuery build() { 66 | SuggestQuery suggestQuery = new SuggestQuery(); 67 | suggestQuery.type = this.type; 68 | 69 | if (StringUtils.isNoneBlank(q)) { 70 | suggestQuery.q = this.q; 71 | } 72 | if (max != null) { 73 | suggestQuery.max = this.max; 74 | } 75 | return suggestQuery; 76 | } 77 | 78 | public Builder q(String q) { 79 | this.q = q; 80 | return this; 81 | } 82 | 83 | public Builder max(Integer max) { 84 | this.max = max; 85 | return this; 86 | } 87 | 88 | } 89 | 90 | } 91 | -------------------------------------------------------------------------------- /src/main/java/org/opentsdb/client/bean/response/DetailResult.java: -------------------------------------------------------------------------------- 1 | package org.opentsdb.client.bean.response; 2 | 3 | import lombok.Data; 4 | import org.opentsdb.client.bean.request.Point; 5 | 6 | import java.util.List; 7 | 8 | /** 9 | * @Author: jinyao 10 | * @Description: 11 | * @CreateDate: 2019/2/24 下午8:07 12 | * @Version: 1.0 13 | */ 14 | @Data 15 | public class DetailResult { 16 | 17 | private List errors; 18 | 19 | private int failed; 20 | 21 | private int success; 22 | 23 | @Data 24 | public static class ErrorPoint{ 25 | 26 | private Point datapoint; 27 | 28 | private String error; 29 | 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /src/main/java/org/opentsdb/client/bean/response/ErrorResponse.java: -------------------------------------------------------------------------------- 1 | package org.opentsdb.client.bean.response; 2 | 3 | import lombok.Data; 4 | 5 | import java.text.MessageFormat; 6 | 7 | /** 8 | * 错误信息 9 | * 10 | * @Description: 11 | * @Author: jinyao 12 | * @CreateDate: 2019/2/22 下午7:50 13 | * @Version: 1.0 14 | */ 15 | @Data 16 | public class ErrorResponse { 17 | 18 | private Error error; 19 | 20 | @Override 21 | public String toString() { 22 | return MessageFormat.format( 23 | "调用OpenTSDB http api发生错误，响应码:{0},错误信息:{1}", 24 | error.getCode(), 25 | error.getMessage() 26 | ); 27 | } 28 | 29 | @Data 30 | public static class Error { 31 | 32 | private int code; 33 | 34 | private String message; 35 | 36 | } 37 | 38 | } 39 | -------------------------------------------------------------------------------- /src/main/java/org/opentsdb/client/bean/response/LastPointQueryResult.java: -------------------------------------------------------------------------------- 1 | package org.opentsdb.client.bean.response; 2 | 3 | import lombok.Data; 4 | 5 | import java.util.Map; 6 | 7 | /** 8 | * @Author: jinyao 9 | * @Description: 10 | * @CreateDate: 2019/2/24 下午3:07 11 | * @Version: 1.0 12 | */ 13 | @Data 14 | public class LastPointQueryResult { 15 | 16 | private String metric; 17 | 18 | private long timestamp; 19 | 20 | private Object value; 21 | 22 | private String tsuid; 23 | 24 | private Map tags; 25 | 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/org/opentsdb/client/bean/response/QueryResult.java: -------------------------------------------------------------------------------- 1 | package org.opentsdb.client.bean.response; 2 | 3 | import lombok.Data; 4 | 5 | import java.util.LinkedHashMap; 6 | import java.util.List; 7 | import java.util.Map; 8 | 9 | /** 10 | * @Description: 11 | * @Author: jinyao 12 | * @CreateDate: 2019/2/22 上午11:51 13 | * @Version: 1.0 14 | */ 15 | @Data 16 | public class QueryResult { 17 | 18 | private String metric; 19 | 20 | private Map tags; 21 | 22 | private List aggregateTags; 23 | 24 | private LinkedHashMap dps; 25 | 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/org/opentsdb/client/common/Json.java: -------------------------------------------------------------------------------- 1 | package org.opentsdb.client.common; 2 | 3 | import com.fasterxml.jackson.annotation.JsonInclude; 4 | import com.fasterxml.jackson.core.JsonParser; 5 | import com.fasterxml.jackson.core.JsonProcessingException; 6 | import com.fasterxml.jackson.databind.DeserializationFeature; 7 | import com.fasterxml.jackson.databind.ObjectMapper; 8 | import com.fasterxml.jackson.databind.type.CollectionType; 9 | import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; 10 | import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; 11 | import com.fasterxml.jackson.module.paramnames.ParameterNamesModule; 12 | 13 | import java.io.IOException; 14 | import java.text.SimpleDateFormat; 15 | import java.util.Collection; 16 | 17 | /** 18 | * 配置一个单例的jackson objectMapper，同时防止外部修改mapper的配置 19 | * 20 | * @Description: 21 | * @Author: jinyao 22 | * @CreateDate: 2019/2/22 下午1:21 23 | * @Version: 1.0 24 | */ 25 | public class Json { 26 | 27 | private static final ObjectMapper instance; 28 | 29 | static { 30 | instance = new ObjectMapper(); 31 | 32 | // 支持java8 33 | instance.registerModule(new JavaTimeModule()) 34 | .registerModule(new ParameterNamesModule()) 35 | .registerModule(new Jdk8Module()); 36 | instance.findAndRegisterModules(); 37 | 38 | 39 | SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); 40 | instance.setDateFormat(dateFormat); 41 | // 允许对象忽略json中不存在的属性 42 | instance.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); 43 | // 允许出现特殊字符和转义符 44 | instance.configure(JsonParser.Feature.ALLOW_UNQUOTED_CONTROL_CHARS, true); 45 | // 允许出现单引号 46 | instance.configure(JsonParser.Feature.ALLOW_SINGLE_QUOTES, true); 47 | // 忽视为空的属性 48 | instance.setSerializationInclusion(JsonInclude.Include.NON_EMPTY); 49 | 50 | } 51 | 52 | /*** 53 | * 将对象序列化为json字符串 54 | * @param value 具体对象 55 | * @return 56 | * @throws JsonProcessingException 57 | */ 58 | public static String writeValueAsString(Object value) throws JsonProcessingException { 59 | return instance.writeValueAsString(value); 60 | } 61 | 62 | /*** 63 | * 将json字符串反序列化为T类型的对象 64 | * @param content json字符串 65 | * @param valueType 数据类型 66 | * @param 67 | * @return 68 | * @throws IOException 69 | */ 70 | public static T readValue(String content, Class valueType) throws IOException { 71 | return instance.readValue(content, valueType); 72 | } 73 | 74 | /*** 75 | * 将json反序列化为集合，集合类型是collectionClass，泛型是elementClass 76 | * @param content json字符串 77 | * @param collectionClass 集合类型 78 | * @param elementClass 泛型 79 | */ 80 | public static T readValue(String content, Class collectionClass, 81 | Class elementClass) throws IOException { 82 | CollectionType collectionType = instance.getTypeFactory() 83 | .constructCollectionType(collectionClass, elementClass); 84 | return instance.readValue(content, collectionType); 85 | } 86 | 87 | } 88 | -------------------------------------------------------------------------------- /src/main/java/org/opentsdb/client/exception/http/HttpException.java: -------------------------------------------------------------------------------- 1 | package org.opentsdb.client.exception.http; 2 | 3 | import lombok.Data; 4 | import org.opentsdb.client.bean.response.ErrorResponse; 5 | 6 | /** 7 | * @Description: 8 | * @Author: jinyao 9 | * @CreateDate: 2019/2/22 下午7:40 10 | * @Version: 1.0 11 | */ 12 | @Data 13 | public class HttpException extends RuntimeException { 14 | 15 | private ErrorResponse errorResponse; 16 | 17 | public HttpException(ErrorResponse errorResponse) { 18 | super(errorResponse.toString()); 19 | this.errorResponse = errorResponse; 20 | } 21 | 22 | } 23 | -------------------------------------------------------------------------------- /src/main/java/org/opentsdb/client/http/HttpClient.java: -------------------------------------------------------------------------------- 1 | package org.opentsdb.client.http; 2 | 3 | import lombok.extern.slf4j.Slf4j; 4 | import org.apache.commons.lang3.StringUtils; 5 | import org.apache.http.HttpResponse; 6 | import org.apache.http.client.methods.HttpPost; 7 | import org.apache.http.concurrent.FutureCallback; 8 | import org.apache.http.entity.StringEntity; 9 | import org.apache.http.impl.nio.client.CloseableHttpAsyncClient; 10 | import org.opentsdb.client.OpenTSDBConfig; 11 | import org.opentsdb.client.http.callback.GracefulCloseFutureCallBack; 12 | 13 | import java.io.IOException; 14 | import java.nio.charset.Charset; 15 | import java.util.concurrent.Future; 16 | import java.util.concurrent.ScheduledExecutorService; 17 | import java.util.concurrent.atomic.AtomicInteger; 18 | 19 | /** 20 | * @Description: 21 | * @Author: jinyao 22 | * @CreateDate: 2019/2/22 下午1:29 23 | * @Version: 1.0 24 | */ 25 | @Slf4j 26 | public class HttpClient { 27 | 28 | private String host; 29 | 30 | private int port; 31 | 32 | /** 33 | * 通过这个client来完成请求 34 | */ 35 | private final CloseableHttpAsyncClient client; 36 | 37 | /** 38 | * 未完成任务数 for graceful close. 39 | */ 40 | private final AtomicInteger unCompletedTaskNum; 41 | 42 | /** 43 | * 空闲连接清理服务 44 | */ 45 | private ScheduledExecutorService connectionGcService; 46 | 47 | HttpClient(OpenTSDBConfig config, CloseableHttpAsyncClient client, ScheduledExecutorService connectionGcService) { 48 | this.host = config.getHost(); 49 | this.port = config.getPort(); 50 | this.client = client; 51 | this.connectionGcService = connectionGcService; 52 | this.unCompletedTaskNum = new AtomicInteger(0); 53 | } 54 | 55 | /*** 56 | * post请求 57 | * @param path 请求路径 58 | * @param json json格式参数 59 | * @return 60 | */ 61 | public Future post(String path, String json) { 62 | return this.post(path, json, null); 63 | } 64 | 65 | /*** 66 | * post请求 67 | * @param path 请求路径 68 | * @param json 请求内容，json格式z 69 | * @param httpCallback 回调 70 | * @return 71 | */ 72 | public Future post(String path, String json, FutureCallback httpCallback) { 73 | log.debug("发送post请求，路径:{}，请求内容:{}", path, json); 74 | HttpPost httpPost = new HttpPost(getUrl(path)); 75 | if (StringUtils.isNoneBlank(json)) { 76 | httpPost.addHeader("Content-Type", "application/json"); 77 | httpPost.setEntity(generateStringEntity(json)); 78 | } 79 | 80 | FutureCallback responseCallback = null; 81 | if (httpCallback != null) { 82 | log.debug("等待完成的任务数:{}", unCompletedTaskNum.incrementAndGet()); 83 | responseCallback = new GracefulCloseFutureCallBack(unCompletedTaskNum, httpCallback); 84 | } 85 | 86 | return client.execute(httpPost, responseCallback); 87 | } 88 | 89 | private String getUrl(String path) { 90 | return host + ":" + port + path; 91 | } 92 | 93 | private StringEntity generateStringEntity(String json) { 94 | StringEntity stringEntity = new StringEntity(json, Charset.forName("UTF-8")); 95 | return stringEntity; 96 | } 97 | 98 | public void start() { 99 | this.client.start(); 100 | } 101 | 102 | public void gracefulClose() throws IOException { 103 | this.close(false); 104 | } 105 | 106 | public void forceClose() throws IOException { 107 | this.close(true); 108 | } 109 | 110 | private void close(boolean force) throws IOException { 111 | // 关闭等待 112 | if (!force) { 113 | // 优雅关闭 114 | while (client.isRunning()) { 115 | int i = this.unCompletedTaskNum.get(); 116 | if (i == 0) { 117 | break; 118 | } else { 119 | try { 120 | // 轮询检查优雅关闭 121 | Thread.sleep(50); 122 | } catch (InterruptedException e) { 123 | log.warn("The thread {} is Interrupted", Thread.currentThread() 124 | .getName()); 125 | } 126 | } 127 | } 128 | } 129 | connectionGcService.shutdownNow(); 130 | 131 | // 关闭 132 | client.close(); 133 | } 134 | 135 | 136 | } 137 | 138 | -------------------------------------------------------------------------------- /src/main/java/org/opentsdb/client/http/HttpClientFactory.java: -------------------------------------------------------------------------------- 1 | package org.opentsdb.client.http; 2 | 3 | import lombok.extern.slf4j.Slf4j; 4 | import org.apache.http.ConnectionReuseStrategy; 5 | import org.apache.http.HeaderElement; 6 | import org.apache.http.HeaderElementIterator; 7 | import org.apache.http.HttpResponse; 8 | import org.apache.http.client.config.RequestConfig; 9 | import org.apache.http.conn.ConnectionKeepAliveStrategy; 10 | import org.apache.http.impl.nio.client.CloseableHttpAsyncClient; 11 | import org.apache.http.impl.nio.client.HttpAsyncClientBuilder; 12 | import org.apache.http.impl.nio.client.HttpAsyncClients; 13 | import org.apache.http.impl.nio.conn.PoolingNHttpClientConnectionManager; 14 | import org.apache.http.impl.nio.reactor.DefaultConnectingIOReactor; 15 | import org.apache.http.impl.nio.reactor.IOReactorConfig; 16 | import org.apache.http.message.BasicHeaderElementIterator; 17 | import org.apache.http.nio.reactor.ConnectingIOReactor; 18 | import org.apache.http.nio.reactor.IOReactorException; 19 | import org.apache.http.protocol.HTTP; 20 | import org.apache.http.protocol.HttpContext; 21 | import org.opentsdb.client.OpenTSDBConfig; 22 | 23 | import java.util.Objects; 24 | import java.util.concurrent.Executors; 25 | import java.util.concurrent.ScheduledExecutorService; 26 | import java.util.concurrent.TimeUnit; 27 | import java.util.concurrent.atomic.AtomicInteger; 28 | 29 | /** 30 | * @Description: 31 | * @Author: jinyao 32 | * @CreateDate: 2019/2/22 下午1:29 33 | * @Version: 1.0 34 | */ 35 | @Slf4j 36 | public class HttpClientFactory { 37 | 38 | private static final AtomicInteger NUM = new AtomicInteger(); 39 | 40 | /*** 41 | * 创建httpclient 42 | * @param config 配置文件 43 | * @return 44 | * @throws IOReactorException 45 | */ 46 | public static HttpClient createHttpClient(OpenTSDBConfig config) throws IOReactorException { 47 | Objects.requireNonNull(config); 48 | 49 | ConnectingIOReactor ioReactor = initIOReactorConfig(); 50 | PoolingNHttpClientConnectionManager connManager = new PoolingNHttpClientConnectionManager(ioReactor); 51 | 52 | RequestConfig requestConfig = initRequestConfig(config); 53 | CloseableHttpAsyncClient httpAsyncClient = createPoolingHttpClient(requestConfig, connManager, config); 54 | 55 | return new HttpClient(config, httpAsyncClient, initFixedCycleCloseConnection(connManager)); 56 | } 57 | 58 | /*** 59 | * 创建CPU核数的IO线程 60 | * @return 61 | * @throws IOReactorException 62 | */ 63 | private static ConnectingIOReactor initIOReactorConfig() throws IOReactorException { 64 | IOReactorConfig ioReactorConfig = IOReactorConfig.custom() 65 | .setIoThreadCount(Runtime.getRuntime() 66 | .availableProcessors()) 67 | .build(); 68 | ConnectingIOReactor ioReactor; 69 | ioReactor = new DefaultConnectingIOReactor(ioReactorConfig); 70 | return ioReactor; 71 | } 72 | 73 | /*** 74 | * 设置超时时间 75 | * @return 76 | */ 77 | private static RequestConfig initRequestConfig(OpenTSDBConfig config) { 78 | return RequestConfig.custom() 79 | // ConnectTimeout:连接超时.连接建立时间，三次握手完成时间. 80 | .setConnectTimeout(config.getHttpConnectTimeout() * 1000) 81 | // SocketTimeout:Socket请求超时.数据传输过程中数据包之间间隔的最大时间. 82 | .setSocketTimeout(config.getHttpConnectTimeout() * 1000) 83 | // ConnectionRequestTimeout:httpclient使用连接池来管理连接，这个时间就是从连接池获取连接的超时时间 84 | .setConnectionRequestTimeout(config.getHttpConnectTimeout() * 1000) 85 | .build(); 86 | } 87 | 88 | private static ConnectionKeepAliveStrategy myStrategy() { 89 | ConnectionKeepAliveStrategy myStrategy = new ConnectionKeepAliveStrategy() { 90 | @Override 91 | public long getKeepAliveDuration(HttpResponse response, HttpContext context) { 92 | HeaderElementIterator it = new BasicHeaderElementIterator 93 | (response.headerIterator(HTTP.CONN_KEEP_ALIVE)); 94 | while (it.hasNext()) { 95 | HeaderElement he = it.nextElement(); 96 | String param = he.getName(); 97 | String value = he.getValue(); 98 | if (value != null && param.equalsIgnoreCase 99 | ("timeout")) { 100 | return Long.parseLong(value) * 1000; 101 | } 102 | } 103 | return 60 * 1000;//如果没有约定，则默认定义时长为60s 104 | } 105 | }; 106 | return myStrategy; 107 | } 108 | 109 | /*** 110 | * 创建client 111 | * @param config 查询对象 112 | * @param cm 连接池管理 113 | * @param openTSDBConfig 114 | * @return 115 | */ 116 | private static CloseableHttpAsyncClient createPoolingHttpClient(RequestConfig config, 117 | PoolingNHttpClientConnectionManager cm, 118 | OpenTSDBConfig openTSDBConfig) { 119 | cm.setMaxTotal(100); 120 | cm.setDefaultMaxPerRoute(100); 121 | 122 | HttpAsyncClientBuilder httpAsyncClientBuilder = HttpAsyncClients.custom() 123 | .setConnectionManager(cm) 124 | .setDefaultRequestConfig(config); 125 | // 如果不是只读，则设置为长连接 126 | if (!openTSDBConfig.isReadonly()) { 127 | httpAsyncClientBuilder.setKeepAliveStrategy(myStrategy()); 128 | } 129 | CloseableHttpAsyncClient client = httpAsyncClientBuilder.build(); 130 | return client; 131 | } 132 | 133 | /*** 134 | * 创建定时任务线程池 135 | * @param cm 连接池管理 136 | * @return 137 | */ 138 | private static ScheduledExecutorService initFixedCycleCloseConnection(final PoolingNHttpClientConnectionManager cm) { 139 | // 通过工厂方法创建线程 140 | ScheduledExecutorService connectionGcService = Executors.newSingleThreadScheduledExecutor( 141 | (r) -> { 142 | Thread t = new Thread(r, "Fixed-Cycle-Close-Connection-" + NUM.incrementAndGet()); 143 | t.setDaemon(true); 144 | return t; 145 | } 146 | ); 147 | 148 | // 定时关闭所有空闲链接 149 | connectionGcService.scheduleAtFixedRate(new Runnable() { 150 | @Override 151 | public void run() { 152 | try { 153 | log.debug("Close idle connections, fixed cycle operation"); 154 | // 关闭30秒内不活动的链接 155 | cm.closeExpiredConnections(); 156 | cm.closeIdleConnections(30, TimeUnit.SECONDS); 157 | } catch (Exception ex) { 158 | log.error("", ex); 159 | } 160 | } 161 | }, 30, 30, TimeUnit.SECONDS); 162 | return connectionGcService; 163 | } 164 | 165 | public static class OpenTSDBConnectionKeepAliveStrategy implements ConnectionKeepAliveStrategy { 166 | 167 | private long time; 168 | 169 | public OpenTSDBConnectionKeepAliveStrategy(long time) { 170 | super(); 171 | this.time = time; 172 | } 173 | 174 | @Override 175 | public long getKeepAliveDuration(HttpResponse response, HttpContext context) { 176 | return 1000 * time; 177 | } 178 | 179 | } 180 | 181 | public static class OpenTSDBConnectionReuseStrategy implements ConnectionReuseStrategy { 182 | 183 | @Override 184 | public boolean keepAlive(HttpResponse response, HttpContext context) { 185 | return false; 186 | } 187 | 188 | } 189 | 190 | } 191 | -------------------------------------------------------------------------------- /src/main/java/org/opentsdb/client/http/callback/BatchPutHttpResponseCallback.java: -------------------------------------------------------------------------------- 1 | package org.opentsdb.client.http.callback; 2 | 3 | import lombok.extern.slf4j.Slf4j; 4 | import org.apache.http.HttpEntity; 5 | import org.apache.http.HttpResponse; 6 | import org.apache.http.concurrent.FutureCallback; 7 | import org.apache.http.util.EntityUtils; 8 | import org.opentsdb.client.bean.request.Point; 9 | import org.opentsdb.client.bean.response.DetailResult; 10 | import org.opentsdb.client.common.Json; 11 | 12 | import java.io.IOException; 13 | import java.nio.charset.Charset; 14 | import java.util.List; 15 | 16 | /** 17 | * 异步写入回调 18 | * 19 | * @Author: jinyao 20 | * @Description: 21 | * @CreateDate: 2019/2/23 下午9:58 22 | * @Version: 1.0 23 | */ 24 | @Slf4j 25 | public class BatchPutHttpResponseCallback implements FutureCallback { 26 | 27 | private BatchPutCallBack callBack; 28 | 29 | private List points; 30 | 31 | public BatchPutHttpResponseCallback() { 32 | } 33 | 34 | public BatchPutHttpResponseCallback(BatchPutCallBack callBack, List points) { 35 | this.callBack = callBack; 36 | this.points = points; 37 | } 38 | 39 | @Override 40 | public void completed(HttpResponse response) { 41 | if (callBack != null) { 42 | // 无论成功失败，response body的格式始终是DetailResult的形式 43 | HttpEntity entity = response.getEntity(); 44 | if (entity != null) { 45 | try { 46 | String content = EntityUtils.toString(entity, Charset.defaultCharset()); 47 | DetailResult detailResult = Json.readValue(content, DetailResult.class); 48 | if (detailResult.getFailed() == 0) { 49 | log.debug("批量添加错误数量为0，全部成功"); 50 | this.callBack.response(points, detailResult); 51 | } else { 52 | log.debug("批量添加出现错误，错误个数:{}", detailResult.getFailed()); 53 | this.callBack.responseError(points, detailResult); 54 | } 55 | } catch (IOException e) { 56 | e.printStackTrace(); 57 | } 58 | } 59 | } 60 | } 61 | 62 | @Override 63 | public void failed(Exception e) { 64 | if (callBack != null) { 65 | log.error("批量添加请求失败,error:{}", e.getMessage()); 66 | this.callBack.failed(points, e); 67 | } 68 | } 69 | 70 | @Override 71 | public void cancelled() { 72 | 73 | } 74 | 75 | public interface BatchPutCallBack { 76 | 77 | /*** 78 | * 在请求完成并且response code成功时回调 79 | * @param points 数据点 80 | * @param result 请求结果 81 | */ 82 | void response(List points, DetailResult result); 83 | 84 | /*** 85 | * 在response code失败时回调 86 | * @param points 数据点 87 | * @param result 请求结果 88 | */ 89 | void responseError(List points, DetailResult result); 90 | 91 | /*** 92 | * 在发生错误是回调 93 | * @param points 数据点 94 | * @param e 异常 95 | */ 96 | void failed(List points, Exception e); 97 | 98 | } 99 | 100 | } 101 | -------------------------------------------------------------------------------- /src/main/java/org/opentsdb/client/http/callback/GracefulCloseFutureCallBack.java: -------------------------------------------------------------------------------- 1 | package org.opentsdb.client.http.callback; 2 | 3 | import lombok.extern.slf4j.Slf4j; 4 | import org.apache.http.HttpResponse; 5 | import org.apache.http.concurrent.FutureCallback; 6 | 7 | import java.util.concurrent.atomic.AtomicInteger; 8 | 9 | /** 10 | * 定义一个FutureCallBack，用来对任务完成、异常、取消后进行减数 11 | * 12 | * @Author: jinyao 13 | * @Description: 14 | * @CreateDate: 2019/2/23 下午10:03 15 | * @Version: 1.0 16 | */ 17 | @Slf4j 18 | public class GracefulCloseFutureCallBack implements FutureCallback{ 19 | 20 | private final AtomicInteger unCompletedTaskNum; 21 | private final FutureCallback futureCallback; 22 | 23 | public GracefulCloseFutureCallBack(AtomicInteger unCompletedTaskNum, FutureCallback futureCallback) { 24 | super(); 25 | this.unCompletedTaskNum = unCompletedTaskNum; 26 | this.futureCallback = futureCallback; 27 | } 28 | 29 | @Override 30 | public void completed(HttpResponse result) { 31 | futureCallback.completed(result); 32 | // 任务处理完毕，再减数 33 | log.debug("等待完成的任务数:{}", unCompletedTaskNum.decrementAndGet()); 34 | } 35 | 36 | @Override 37 | public void failed(Exception ex) { 38 | futureCallback.failed(ex); 39 | // 任务处理完毕，再减数 40 | log.debug("等待完成的任务数:{}", unCompletedTaskNum.decrementAndGet()); 41 | } 42 | 43 | @Override 44 | public void cancelled() { 45 | futureCallback.cancelled(); 46 | // 任务处理完毕，再减数 47 | log.debug("等待完成的任务数:{}", unCompletedTaskNum.decrementAndGet()); 48 | } 49 | 50 | } 51 | -------------------------------------------------------------------------------- /src/main/java/org/opentsdb/client/http/callback/QueryHttpResponseCallback.java: -------------------------------------------------------------------------------- 1 | package org.opentsdb.client.http.callback; 2 | 3 | import lombok.extern.slf4j.Slf4j; 4 | import org.apache.http.HttpResponse; 5 | import org.apache.http.concurrent.FutureCallback; 6 | import org.opentsdb.client.bean.request.Query; 7 | import org.opentsdb.client.bean.response.QueryResult; 8 | import org.opentsdb.client.common.Json; 9 | import org.opentsdb.client.exception.http.HttpException; 10 | import org.opentsdb.client.util.ResponseUtil; 11 | 12 | import java.io.IOException; 13 | import java.util.List; 14 | 15 | /** 16 | * 异步查询回调 17 | * 18 | * @Author: jinyao 19 | * @Description: 20 | * @CreateDate: 2019/2/24 下午4:14 21 | * @Version: 1.0 22 | */ 23 | @Slf4j 24 | public class QueryHttpResponseCallback implements FutureCallback { 25 | 26 | private final QueryCallback callback; 27 | 28 | private final Query query; 29 | 30 | public QueryHttpResponseCallback(QueryCallback callback, Query query) { 31 | this.callback = callback; 32 | this.query = query; 33 | } 34 | 35 | @Override 36 | public void completed(HttpResponse response) { 37 | try { 38 | List results = Json.readValue(ResponseUtil.getContent(response), List.class, QueryResult.class); 39 | log.debug("请求成功"); 40 | this.callback.response(query, results); 41 | } catch (IOException e) { 42 | e.printStackTrace(); 43 | this.callback.failed(query, e); 44 | } catch (HttpException e) { 45 | log.error("请求失败，query:{},error:{}", query, e.getMessage()); 46 | e.printStackTrace(); 47 | this.callback.responseError(query, e); 48 | } 49 | } 50 | 51 | @Override 52 | public void failed(Exception e) { 53 | log.error("请求失败，query:{},error:{}", query, e.getMessage()); 54 | this.callback.failed(query, e); 55 | } 56 | 57 | @Override 58 | public void cancelled() { 59 | 60 | } 61 | 62 | /*** 63 | * 定义查询callback，需要用户自己实现逻辑 64 | */ 65 | public interface QueryCallback { 66 | 67 | /*** 68 | * 在请求完成并且response code成功时回调 69 | * @param query 查询对象 70 | * @param queryResults 查询结果 71 | */ 72 | void response(Query query, List queryResults); 73 | 74 | /*** 75 | * 在response code失败时回调 76 | * @param query 查询对象 77 | * @param e 异常 78 | */ 79 | void responseError(Query query, HttpException e); 80 | 81 | /*** 82 | * 在发生错误是回调，如果http成功complete，但response code大于400，也会调用这个方法 83 | * @param query 查询对象 84 | * @param e 异常 85 | */ 86 | void failed(Query query, Exception e); 87 | 88 | } 89 | 90 | } 91 | -------------------------------------------------------------------------------- /src/main/java/org/opentsdb/client/sender/consumer/Consumer.java: -------------------------------------------------------------------------------- 1 | package org.opentsdb.client.sender.consumer; 2 | 3 | /** 4 | * 消费者接口 5 | * 6 | * @Author: jinyao 7 | * @Description: 8 | * @CreateDate: 2019/2/23 下午4:07 9 | * @Version: 1.0 10 | */ 11 | public interface Consumer { 12 | 13 | /*** 14 | * 开始消费，启动线程池中的消费线程 15 | */ 16 | void start(); 17 | 18 | /*** 19 | * 停止消费，会等待线程池中的任务完成 20 | */ 21 | void gracefulStop(); 22 | 23 | /*** 24 | * 强制停止 25 | */ 26 | void forceStop(); 27 | 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/org/opentsdb/client/sender/consumer/ConsumerImpl.java: -------------------------------------------------------------------------------- 1 | package org.opentsdb.client.sender.consumer; 2 | 3 | import lombok.extern.slf4j.Slf4j; 4 | import org.opentsdb.client.OpenTSDBConfig; 5 | import org.opentsdb.client.bean.request.Point; 6 | import org.opentsdb.client.http.HttpClient; 7 | 8 | import java.util.concurrent.BlockingQueue; 9 | import java.util.concurrent.CountDownLatch; 10 | import java.util.concurrent.ExecutorService; 11 | import java.util.concurrent.Executors; 12 | 13 | /** 14 | * @Author: jinyao 15 | * @Description: 16 | * @CreateDate: 2019/2/23 下午4:36 17 | * @Version: 1.0 18 | */ 19 | @Slf4j 20 | public class ConsumerImpl implements Consumer { 21 | 22 | private final BlockingQueue queue; 23 | 24 | private final HttpClient httpClient; 25 | 26 | private final ExecutorService threadPool; 27 | 28 | private final int threadCount; 29 | 30 | private final OpenTSDBConfig config; 31 | 32 | private final CountDownLatch countDownLatch; 33 | 34 | public ConsumerImpl(BlockingQueue queue, HttpClient httpClient, OpenTSDBConfig config) { 35 | this.queue = queue; 36 | this.httpClient = httpClient; 37 | this.config = config; 38 | this.threadCount = config.getPutConsumerThreadCount(); 39 | final int[] i = new int[1]; 40 | this.threadPool = Executors.newFixedThreadPool( 41 | threadCount, 42 | (runnable) -> new Thread(runnable, "batch-put-thread-" + ++i[0]) 43 | ); 44 | this.countDownLatch = new CountDownLatch(threadCount); 45 | 46 | log.debug("the consumer has started"); 47 | } 48 | 49 | @Override 50 | public void start() { 51 | for (int i = 0; i < threadCount; i++) { 52 | threadPool.execute(new ConsumerRunnable(queue, httpClient, config, countDownLatch)); 53 | } 54 | } 55 | 56 | @Override 57 | public void gracefulStop() { 58 | this.stop(false); 59 | } 60 | 61 | @Override 62 | public void forceStop() { 63 | this.stop(true); 64 | } 65 | 66 | /*** 67 | * 关闭线程池 68 | * @param force 是否强制关闭 69 | */ 70 | private void stop(boolean force) { 71 | if (threadPool != null) { 72 | if (force) { 73 | // 强制退出不等待，截断消费者线程。 74 | threadPool.shutdownNow(); 75 | } else { 76 | // 截断消费者线程。 77 | while (!threadPool.isShutdown() || !threadPool.isTerminated()) { 78 | threadPool.shutdownNow(); 79 | } 80 | 81 | // 等待所有消费者线程结束。 82 | try { 83 | countDownLatch.await(); 84 | } catch (InterruptedException e) { 85 | log.error("An error occurred waiting for the consumer thread to close", e); 86 | } 87 | } 88 | } 89 | } 90 | 91 | } 92 | -------------------------------------------------------------------------------- /src/main/java/org/opentsdb/client/sender/consumer/ConsumerRunnable.java: -------------------------------------------------------------------------------- 1 | package org.opentsdb.client.sender.consumer; 2 | 3 | import lombok.extern.slf4j.Slf4j; 4 | import org.opentsdb.client.OpenTSDBConfig; 5 | import org.opentsdb.client.bean.request.Api; 6 | import org.opentsdb.client.bean.request.Point; 7 | import org.opentsdb.client.common.Json; 8 | import org.opentsdb.client.http.HttpClient; 9 | import org.opentsdb.client.http.callback.BatchPutHttpResponseCallback; 10 | 11 | import java.util.ArrayList; 12 | import java.util.List; 13 | import java.util.concurrent.BlockingQueue; 14 | import java.util.concurrent.CountDownLatch; 15 | import java.util.concurrent.TimeUnit; 16 | 17 | /** 18 | * 消费者线程具体的消费逻辑 19 | * 20 | * @Author: jinyao 21 | * @Description: 22 | * @CreateDate: 2019/2/23 下午5:14 23 | * @Version: 1.0 24 | */ 25 | @Slf4j 26 | public class ConsumerRunnable implements Runnable { 27 | 28 | private final BlockingQueue queue; 29 | 30 | private final HttpClient httpClient; 31 | 32 | private final OpenTSDBConfig config; 33 | 34 | private final CountDownLatch countDownLatch; 35 | 36 | private BatchPutHttpResponseCallback.BatchPutCallBack callBack; 37 | 38 | /** 39 | * 每批次数据点个数 40 | */ 41 | private int batchSize; 42 | 43 | /*** 44 | * 每次提交等待的时间间隔，单位ms 45 | */ 46 | private int batchPutTimeLimit; 47 | 48 | public ConsumerRunnable(BlockingQueue queue, HttpClient httpClient, OpenTSDBConfig config, CountDownLatch countDownLatch) { 49 | this.queue = queue; 50 | this.httpClient = httpClient; 51 | this.config = config; 52 | this.countDownLatch = countDownLatch; 53 | this.batchSize = config.getBatchPutSize(); 54 | this.batchPutTimeLimit = config.getBatchPutTimeLimit(); 55 | this.callBack = config.getBatchPutCallBack(); 56 | } 57 | 58 | 59 | /*** 60 | * 设计原则是接收满${batchSize}个元素就提交，或者达到时间${batchPutTimeLimit} 61 | * 当线程被打断说明cosumer执行了stop 62 | */ 63 | @Override 64 | public void run() { 65 | log.debug("thread:{} has started take point from queue", Thread.currentThread() 66 | .getName()); 67 | Point waitPoint = null; 68 | boolean readyClose = false; 69 | int waitTimeLimit = batchPutTimeLimit / 3; 70 | 71 | while (!readyClose) { 72 | long t0 = System.currentTimeMillis(); 73 | List pointList = new ArrayList<>(batchSize); 74 | if (waitPoint != null) { 75 | pointList.add(waitPoint); 76 | waitPoint = null; 77 | } 78 | 79 | for (int i = pointList.size(); i < batchSize; i++) { 80 | try { 81 | Point point = queue.poll(waitTimeLimit, TimeUnit.MILLISECONDS); 82 | if (point != null) { 83 | pointList.add(point); 84 | } 85 | long t1 = System.currentTimeMillis(); 86 | if (t1 - t0 > batchPutTimeLimit) { 87 | break; 88 | } 89 | } catch (InterruptedException e) { 90 | readyClose = true; 91 | log.info("The thread {} is interrupted", Thread.currentThread() 92 | .getName()); 93 | break; 94 | } 95 | } 96 | 97 | if (pointList.size() == 0 && !readyClose) { 98 | try { 99 | waitPoint = queue.take(); 100 | } catch (InterruptedException e) { 101 | readyClose = true; 102 | log.info("The thread {} is interrupted", Thread.currentThread() 103 | .getName()); 104 | } 105 | continue; 106 | } 107 | 108 | if (pointList.size() == 0) { 109 | continue; 110 | } 111 | 112 | sendHttp(pointList); 113 | 114 | } 115 | 116 | this.countDownLatch.countDown(); 117 | } 118 | 119 | /*** 120 | * 发送请求写入数据 121 | * @param points 数据点 122 | */ 123 | private void sendHttp(List points) { 124 | try { 125 | if (callBack == null) { 126 | httpClient.post( 127 | Api.PUT.getPath(), 128 | Json.writeValueAsString(points), 129 | new BatchPutHttpResponseCallback() 130 | ); 131 | } else { 132 | httpClient.post( 133 | Api.PUT_DETAIL.getPath(), 134 | Json.writeValueAsString(points), 135 | new BatchPutHttpResponseCallback(callBack, points) 136 | ); 137 | } 138 | } catch (Exception e) { 139 | e.printStackTrace(); 140 | } 141 | } 142 | 143 | } 144 | -------------------------------------------------------------------------------- /src/main/java/org/opentsdb/client/sender/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 这个包是为写入数据而设计，基于生产者/消费者模式 3 | * 4 | * @Author: jinyao 5 | * @Description: 6 | * @CreateDate: 2019/2/23 下午2:51 7 | * @Version: 1.0 8 | */ 9 | package org.opentsdb.client.sender; -------------------------------------------------------------------------------- /src/main/java/org/opentsdb/client/sender/producer/Producer.java: -------------------------------------------------------------------------------- 1 | package org.opentsdb.client.sender.producer; 2 | 3 | import org.opentsdb.client.bean.request.Point; 4 | 5 | /** 6 | * 生产者接口 7 | * 8 | * @Author: jinyao 9 | * @Description: 10 | * @CreateDate: 2019/2/23 下午4:07 11 | * @Version: 1.0 12 | */ 13 | public interface Producer { 14 | 15 | /*** 16 | * 写入队列 17 | * @param point 数据点 18 | */ 19 | void send(Point point); 20 | 21 | /*** 22 | * 关闭写入 23 | */ 24 | void forbiddenSend(); 25 | 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/org/opentsdb/client/sender/producer/ProducerImpl.java: -------------------------------------------------------------------------------- 1 | package org.opentsdb.client.sender.producer; 2 | 3 | import lombok.extern.slf4j.Slf4j; 4 | import org.opentsdb.client.bean.request.Point; 5 | 6 | import java.util.concurrent.BlockingQueue; 7 | import java.util.concurrent.atomic.AtomicBoolean; 8 | 9 | /** 10 | * @Author: jinyao 11 | * @Description: 12 | * @CreateDate: 2019/2/23 下午4:20 13 | * @Version: 1.0 14 | */ 15 | @Slf4j 16 | public class ProducerImpl implements Producer { 17 | 18 | private final BlockingQueue queue; 19 | 20 | private final AtomicBoolean forbiddenWrite = new AtomicBoolean(false); 21 | 22 | public ProducerImpl(BlockingQueue queue) { 23 | this.queue = queue; 24 | log.debug("the producer has started"); 25 | } 26 | 27 | @Override 28 | public void send(Point point) { 29 | if (forbiddenWrite.get()) { 30 | throw new IllegalStateException("client has been closed."); 31 | } 32 | try { 33 | // 队列满时，put方法会被阻塞 34 | queue.put(point); 35 | } catch (InterruptedException e) { 36 | log.error("Client Thread been Interrupted.", e); 37 | e.printStackTrace(); 38 | } 39 | } 40 | 41 | @Override 42 | public void forbiddenSend() { 43 | forbiddenWrite.compareAndSet(false, true); 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /src/main/java/org/opentsdb/client/util/ResponseUtil.java: -------------------------------------------------------------------------------- 1 | package org.opentsdb.client.util; 2 | 3 | import org.apache.http.HttpEntity; 4 | import org.apache.http.HttpResponse; 5 | import org.apache.http.StatusLine; 6 | import org.apache.http.util.EntityUtils; 7 | import org.opentsdb.client.bean.response.ErrorResponse; 8 | import org.opentsdb.client.common.Json; 9 | import org.opentsdb.client.exception.http.HttpException; 10 | 11 | import java.io.IOException; 12 | import java.nio.charset.Charset; 13 | 14 | /** 15 | * 响应解析工具类 16 | * 17 | * @Description: 18 | * @Author: jinyao 19 | * @CreateDate: 2019/2/22 下午7:30 20 | * @Version: 1.0 21 | */ 22 | public class ResponseUtil { 23 | 24 | /*** 25 | * 解析响应的内容 26 | * @param response 响应内容 27 | * @return 28 | * @throws IOException 29 | */ 30 | public static String getContent(HttpResponse response) throws IOException { 31 | if (checkGT400(response)) { 32 | throw new HttpException(convert(response)); 33 | } else { 34 | return getContentString(response); 35 | } 36 | } 37 | 38 | private static String getContentString(HttpResponse response) throws IOException { 39 | HttpEntity entity = response.getEntity(); 40 | if (entity != null) { 41 | return EntityUtils.toString(entity, Charset.defaultCharset()); 42 | } 43 | return null; 44 | } 45 | 46 | /*** 47 | * 判断响应码的是否为400以上，如果是，则表示出错了 48 | * @param response 查询对象 49 | * @return 50 | */ 51 | private static boolean checkGT400(HttpResponse response) { 52 | StatusLine statusLine = response.getStatusLine(); 53 | int statusCode = statusLine.getStatusCode(); 54 | if (statusCode >= 400) { 55 | return true; 56 | } 57 | return false; 58 | } 59 | 60 | /*** 61 | * 将响应内容转换成errorResponse 62 | * @param response 查询对象 63 | * @return 64 | */ 65 | private static ErrorResponse convert(HttpResponse response) throws IOException { 66 | return Json.readValue(getContentString(response), ErrorResponse.class); 67 | } 68 | 69 | 70 | } 71 | -------------------------------------------------------------------------------- /src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # priority :debug> Method: %l ]%n%p:%m%n 15 | #debug log 16 | log4j.logger.debug=debug 17 | log4j.appender.debug=org.apache.log4j.ConsoleAppender 18 | log4j.appender.debug.Threshold=DEBUG 19 | log4j.appender.debug.layout=org.apache.log4j.PatternLayout 20 | log4j.appender.debug.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss a} [Thread: %t][ Class:%c >> Method: %l ]%n%p:%m%n 21 | #warn log 22 | log4j.logger.warn=warn 23 | log4j.appender.warn=org.apache.log4j.ConsoleAppender 24 | log4j.appender.warn.Threshold=WARN 25 | log4j.appender.warn.layout=org.apache.log4j.PatternLayout 26 | log4j.appender.warn.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss a} [Thread: %t][ Class:%c >> Method: %l ]%n%p:%m%n 27 | #error 28 | log4j.logger.error=error 29 | log4j.appender.error = org.apache.log4j.ConsoleAppender 30 | log4j.appender.error.Threshold = ERROR 31 | log4j.appender.error.layout = org.apache.log4j.PatternLayout 32 | log4j.appender.error.layout.ConversionPattern = %d{yyyy-MM-dd HH:mm:ss a} [Thread: %t][ Class:%c >> Method: %l ]%n%p:%m%n -------------------------------------------------------------------------------- /src/test/java/org/opentsdb/client/ControlTest.java: -------------------------------------------------------------------------------- 1 | package org.opentsdb.client; 2 | 3 | import lombok.extern.slf4j.Slf4j; 4 | import org.apache.http.nio.reactor.IOReactorException; 5 | import org.junit.Before; 6 | import org.junit.Test; 7 | import org.opentsdb.client.bean.request.SuggestQuery; 8 | 9 | import java.util.List; 10 | 11 | /** 12 | * @Author: jinyao 13 | * @Description: 14 | * @CreateDate: 2019/3/9 下午3:18 15 | * @Version: 1.0 16 | */ 17 | @Slf4j 18 | public class ControlTest { 19 | 20 | String host = "http://127.0.0.1"; 21 | 22 | int port = 4242; 23 | 24 | OpenTSDBClient client; 25 | 26 | @Before 27 | public void config() throws IOReactorException { 28 | OpenTSDBConfig config = OpenTSDBConfig.address(host, port) 29 | .config(); 30 | OpenTSDBConfig.address(host, port) 31 | // 当确认这个client只用于查询时设置，可不创建内部队列从而提高效率 32 | .readonly() 33 | // 每批数据提交完成后回调 34 | .config(); 35 | client = OpenTSDBClientFactory.connect(config); 36 | } 37 | 38 | @Test 39 | public void suggestQuery() throws Exception { 40 | SuggestQuery query = SuggestQuery.type(SuggestQuery.Type.METRICS) 41 | .build(); 42 | List suggests = client.querySuggest(query); 43 | log.debug("suggest query result:{}", suggests); 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /src/test/java/org/opentsdb/client/CrudTest.java: -------------------------------------------------------------------------------- 1 | package org.opentsdb.client; 2 | 3 | import lombok.extern.slf4j.Slf4j; 4 | import org.apache.http.nio.reactor.IOReactorException; 5 | import org.junit.Assert; 6 | import org.junit.Before; 7 | import org.junit.Test; 8 | import org.opentsdb.client.bean.request.*; 9 | import org.opentsdb.client.bean.response.DetailResult; 10 | import org.opentsdb.client.bean.response.LastPointQueryResult; 11 | import org.opentsdb.client.bean.response.QueryResult; 12 | import org.opentsdb.client.exception.http.HttpException; 13 | import org.opentsdb.client.http.callback.BatchPutHttpResponseCallback; 14 | import org.opentsdb.client.http.callback.QueryHttpResponseCallback; 15 | 16 | import java.io.IOException; 17 | import java.util.LinkedHashMap; 18 | import java.util.List; 19 | import java.util.concurrent.*; 20 | 21 | /** 22 | * @Description: 23 | * @Author: jinyao 24 | * @CreateDate: 2019/2/22 下午3:53 25 | * @Version: 1.0 26 | */ 27 | @Slf4j 28 | public class CrudTest { 29 | 30 | String host = "http://127.0.0.1"; 31 | 32 | int port = 4242; 33 | 34 | OpenTSDBClient client; 35 | 36 | @Before 37 | public void config() throws IOReactorException { 38 | OpenTSDBConfig config = OpenTSDBConfig.address(host, port) 39 | .config(); 40 | OpenTSDBConfig.address(host, port) 41 | // http连接池大小，默认100 42 | .httpConnectionPool(100) 43 | // http请求超时时间，默认100s 44 | .httpConnectTimeout(100) 45 | // 异步写入数据时，每次http提交的数据条数，默认50 46 | .batchPutSize(50) 47 | // 异步写入数据中，内部有一个队列，默认队列大小20000 48 | .batchPutBufferSize(20000) 49 | // 异步写入等待时间，如果距离上一次请求超多300ms，且有数据，则直接提交 50 | .batchPutTimeLimit(300) 51 | // 当确认这个client只用于查询时设置，可不创建内部队列从而提高效率 52 | .readonly() 53 | // 每批数据提交完成后回调 54 | .batchPutCallBack(new BatchPutHttpResponseCallback.BatchPutCallBack() { 55 | @Override 56 | public void response(List points, DetailResult result) { 57 | // 在请求完成并且response code成功时回调 58 | } 59 | 60 | @Override 61 | public void responseError(List points, DetailResult result) { 62 | // 在response code失败时回调 63 | } 64 | 65 | @Override 66 | public void failed(List points, Exception e) { 67 | // 在发生错误是回调 68 | } 69 | }) 70 | .config(); 71 | client = OpenTSDBClientFactory.connect(config); 72 | } 73 | 74 | /*** 75 | * 单点查询测试 76 | */ 77 | @Test 78 | public void testQuery() throws InterruptedException, ExecutionException, IOException { 79 | log.debug("当前线程:{}", Thread.currentThread() 80 | .getName()); 81 | Query query = Query.begin("7d-ago") 82 | .sub(SubQuery.metric("metric.test") 83 | .aggregator(SubQuery.Aggregator.NONE) 84 | .build()) 85 | .build(); 86 | List resultList = client.query(query); 87 | client.query(query, new QueryHttpResponseCallback.QueryCallback() { 88 | @Override 89 | public void response(Query query, List queryResults) { 90 | // 在请求完成并且response code成功时回调 91 | } 92 | 93 | @Override 94 | public void responseError(Query query, HttpException e) { 95 | // 在response code失败时回调 96 | } 97 | 98 | @Override 99 | public void failed(Query query, Exception e) { 100 | // 在发生错误是回调 101 | } 102 | }); 103 | log.debug("result:{}", resultList); 104 | } 105 | 106 | /*** 107 | * 并发查询测试 108 | * 测试结果大概190个线程会出现http超时 109 | * 之后会把http超时做成参数 110 | * 111 | * 更新：已解决这个问题，目前默认超时时间100秒，并可以通过参数改变 112 | */ 113 | @Test 114 | public void testQueryConcurrent() { 115 | CountDownLatch latch = new CountDownLatch(1); 116 | int threadCount = 2000; 117 | long[] times = new long[3]; 118 | // 利用CyclicBarrier模拟并发 119 | CyclicBarrier startBarrier = new CyclicBarrier( 120 | threadCount, 121 | () -> { 122 | log.debug("所有线程已经就位"); 123 | times[0] = System.currentTimeMillis(); 124 | } 125 | ); 126 | 127 | CyclicBarrier endBarrier = new CyclicBarrier( 128 | threadCount, 129 | () -> { 130 | log.debug("所有线程已经执行完毕"); 131 | times[1] = System.currentTimeMillis(); 132 | log.debug("运行时间:{}毫秒", times[1] - times[0]); 133 | latch.countDown(); 134 | } 135 | ); 136 | 137 | 138 | ExecutorService fixedThreadPool = Executors.newFixedThreadPool( 139 | threadCount, 140 | (runnable) -> { 141 | Thread thread = new Thread(runnable, "thread-" + times[2]++); 142 | return thread; 143 | } 144 | ); 145 | while (threadCount-- > 0) { 146 | fixedThreadPool.execute(() -> { 147 | try { 148 | startBarrier.await(); 149 | // 查询 150 | testQuery(); 151 | endBarrier.await(); 152 | } catch (Exception e) { 153 | log.error("", e); 154 | e.printStackTrace(); 155 | } 156 | }); 157 | } 158 | 159 | try { 160 | latch.await(); 161 | } catch (InterruptedException e) { 162 | e.printStackTrace(); 163 | } 164 | 165 | } 166 | 167 | /** 168 | * 测试写入数据 169 | */ 170 | @Test 171 | public void put() throws Exception { 172 | Point point = Point.metric("metric.test") 173 | .tag("test", "hello") 174 | .value(System.currentTimeMillis(), 1.0) 175 | .build(); 176 | //client.put(point); 177 | client.putSync(point); 178 | client.gracefulClose(); 179 | } 180 | 181 | /*** 182 | * 并发写入测试，运行前需要先清除metric为point的数据 183 | */ 184 | @Test 185 | public void batchPut() throws Exception { 186 | /** 187 | * 删除数据 188 | *//* 189 | Query delete = Query.begin("30d-ago") 190 | .delete() 191 | .sub(SubQuery.metric("point") 192 | .aggregator(SubQuery.Aggregator.NONE) 193 | .build()) 194 | .build(); 195 | client.query(delete);*/ 196 | 197 | /*** 198 | * 使用5个线程，每个线程都写入100000条数据 199 | */ 200 | int threadCount = 5; 201 | int dataCount = 100000; 202 | CountDownLatch latch = new CountDownLatch(5); 203 | int[] ints = new int[1]; 204 | /*** 205 | * 方便测试，线程名称为1,2,3,4,5 206 | */ 207 | ExecutorService threadPool = Executors.newFixedThreadPool(5, (r) -> 208 | new Thread(r, String.valueOf(++ints[0])) 209 | ); 210 | long start = System.currentTimeMillis(); 211 | 212 | /** 213 | * 获得前10天的时间戳，测试发现openTSDB不会写入大于当前时间戳的数据，所以时间戳要前移 214 | */ 215 | long begin = start - (long) 24 * 60 * 60 * 1000 * 10; 216 | for (int a = 0; a < threadCount; a++) { 217 | threadPool.execute(() -> { 218 | for (int i = 1; i <= dataCount; i++) { 219 | Point point = Point.metric("point") 220 | .tag("testTag", "test_" + Thread.currentThread() 221 | .getName()) 222 | /** 223 | * 每秒一条数据 224 | */ 225 | .value(begin + i * 1000, i) 226 | .build(); 227 | client.put(point); 228 | } 229 | latch.countDown(); 230 | }); 231 | } 232 | 233 | latch.await(); 234 | long end = System.currentTimeMillis(); 235 | log.debug("运行时间:{}毫秒", end - start); 236 | /*** 237 | * 等待10秒，因为下面查询还需要用到client，所以这里先不关闭client，用沉睡线程的方式等待队列中所有任务完成 238 | */ 239 | TimeUnit.SECONDS.sleep(10); 240 | 241 | /*** 242 | * 断言依据是写入数据求和是否等于（1+100000）*100000/2 243 | * 用double类型，防止查询结果为科学计数法 244 | */ 245 | double should = (double) (1 + dataCount) * dataCount / 2; 246 | Query query = Query.begin("20d-ago") 247 | .sub(SubQuery.metric("point") 248 | /*** 249 | * 不采用时间线聚合，结果应该会返回5条时间线 250 | */ 251 | .aggregator(SubQuery.Aggregator.NONE) 252 | /*** 253 | * 降采样，这里0all-sum表示把所有点合并成一个点 254 | */ 255 | .downsample("0all-sum") 256 | .build()) 257 | .build(); 258 | List queryResults = client.query(query); 259 | for (QueryResult queryResult : queryResults) { 260 | LinkedHashMap dps = queryResult.getDps(); 261 | /** 262 | * 因为把所有点聚合在一个点上，所有dps只有一个值 263 | */ 264 | dps.forEach((k, v) -> { 265 | /** 266 | * 因结果数值较大，在本地测试时，已被转换成了科学计数法 267 | */ 268 | 269 | Assert.assertEquals(should, v); 270 | }); 271 | } 272 | client.gracefulClose(); 273 | } 274 | 275 | /*** 276 | * 测试查询最新数据 277 | * @throws Exception 278 | */ 279 | @Test 280 | public void testQueryLast() throws Exception { 281 | LastPointQuery query = LastPointQuery.sub(LastPointSubQuery.metric("point") 282 | .tag("testTag", "test_1") 283 | .build()) 284 | // baskScan表示查询最多向前推进多少小时 285 | // 比如在5小时前写入过数据 286 | // 那么backScan(6)可以查出数据，但backScan(4)则不行 287 | .backScan(1000) 288 | .build(); 289 | List lastPointQueryResults = client.queryLast(query); 290 | log.debug("查询最新数据:{}", lastPointQueryResults); 291 | } 292 | 293 | /*** 294 | * 测试异步查询 295 | * @throws Exception 296 | */ 297 | @Test 298 | public void testAsyncQuery() throws Exception { 299 | int[] ints = new int[1]; 300 | QueryHttpResponseCallback.QueryCallback queryCallback = new QueryHttpResponseCallback.QueryCallback() { 301 | @Override 302 | public void response(Query query, List queryResults) { 303 | log.debug("success,result:{}", queryResults); 304 | } 305 | 306 | @Override 307 | public void responseError(Query query, HttpException e) { 308 | log.debug("fail,error:{}", e.getMessage()); 309 | e.printStackTrace(); 310 | ints[0] = 1; 311 | } 312 | 313 | @Override 314 | public void failed(Query query, Exception e) { 315 | e.printStackTrace(); 316 | } 317 | }; 318 | Query query = Query.begin("20d-ago") 319 | .sub(SubQuery.metric("point") 320 | .aggregator(SubQuery.Aggregator.NONE) 321 | /** 322 | * 特意写错，会触发callback中分failed方法 323 | */ 324 | .downsample("0all-su") 325 | .build()) 326 | .build(); 327 | client.query(query, queryCallback); 328 | client.gracefulClose(); 329 | Assert.assertEquals(1, ints[0]); 330 | } 331 | 332 | /*** 333 | * 测试写入回调 334 | * @throws Exception 335 | */ 336 | @Test 337 | public void testPutCallback() throws Exception { 338 | int[] ints = new int[2]; 339 | BatchPutHttpResponseCallback.BatchPutCallBack batchPutCallBack = new BatchPutHttpResponseCallback.BatchPutCallBack() { 340 | @Override 341 | public void response(List points, DetailResult result) { 342 | log.debug("添加成功，detail:{}", result); 343 | ints[0] = 1; 344 | } 345 | 346 | @Override 347 | public void responseError(List points, DetailResult result) { 348 | log.debug("添加失败，detail:{}", result); 349 | ints[1] = 1; 350 | } 351 | 352 | @Override 353 | public void failed(List points, Exception e) { 354 | 355 | } 356 | }; 357 | OpenTSDBConfig config = OpenTSDBConfig.address(host, port) 358 | .batchPutCallBack(batchPutCallBack) 359 | .config(); 360 | OpenTSDBClient openTSDBClient = OpenTSDBClientFactory.connect(config); 361 | Point point = Point.metric("batchPutCallback") 362 | .tag("testTag", "test_1") 363 | .value(System.currentTimeMillis(), 1.0) 364 | .build(); 365 | openTSDBClient.put(point); 366 | openTSDBClient.gracefulClose(); 367 | /** 368 | * 测试response 369 | */ 370 | Assert.assertEquals(1, ints[0]); 371 | /*** 372 | * 测试error 373 | */ 374 | //Assert.assertEquals(1, ints[1]); 375 | } 376 | 377 | } 378 | --------------------------------------------------------------------------------