├── .gitignore ├── .producerrc ├── LICENSE ├── PERF_README_CN.md ├── README.md ├── README_CN.md ├── pom.xml └── src └── main ├── java └── com │ └── aliyun │ └── openservices │ └── aliyun │ └── log │ └── producer │ └── sample │ ├── SamplePerformance.java │ ├── SampleProducerWithCallback.java │ ├── SampleProducerWithFuture.java │ └── Utils.java └── resources └── logback.xml /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | local_env 3 | local_env2 4 | .local_producerrc 5 | .local_producerrc2 6 | pics/ 7 | target/ 8 | .DS_Store 9 | pom.xml.releaseBackup 10 | release.properties 11 | aliyun-log-producer-sample.iml -------------------------------------------------------------------------------- /.producerrc: -------------------------------------------------------------------------------- 1 | export PROJECT= 2 | export ENDPOINT= 3 | export ACCESS_KEY_ID= 4 | export ACCESS_KEY_SECRET= 5 | export LOG_STORE= -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /PERF_README_CN.md: -------------------------------------------------------------------------------- 1 | # SamplePerformance 运行步骤 2 | 3 | 执行以下命令编译源码、生成 jar 包。 4 | ``` 5 | mvn clean package 6 | ``` 7 | 8 | 输入以下命令运行性能测试的样例，请将 \、\、\、\、\ 替换为实际值。 9 | ``` 10 | export PROJECT= \ 11 | export LOG_STORE= \ 12 | export ENDPOINT= \ 13 | export ACCESS_KEY_ID= \ 14 | export ACCESS_KEY_SECRET= \ 15 | export SEND_THREAD_COUNT=10 \ 16 | export TIMES=20000000 \ 17 | export IO_THREAD_COUNT=16 \ 18 | export TOTAL_SIZE_IN_BYTES=104857600 \ 19 | export LEVEL=INFO && \ 20 | java -jar target/aliyun-log-producer-sample-0.0.1-SNAPSHOT.jar -Xms2g -Xmx2g 21 | ``` 22 | 23 | **注意**：运行以上命令向您的目标 logstore 写入约 115 GB 的数据，在非性能测试场景下谨慎运行。 24 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Aliyun LOG Java Producer Sample Application 2 | 3 | [![License](https://img.shields.io/badge/license-Apache2.0-blue.svg)](/LICENSE) 4 | 5 | [中文版 README](/README_CN.md) 6 | 7 | This repo provides the sample application for [Aliyun LOG Java Producer](https://github.com/aliyun/aliyun-log-producer/blob/master/README_EN.md), it will help you to master the producer library more quickly. 8 | 9 | ## Prerequisite 10 | You will need the following: 11 | * Opening [Aliyun Log Service](https://www.alibabacloud.com/product/log-service) and create target project, logstore in advancce. 12 | * Prepare the AK used by producer. 13 | * Install JDK 1.6+. 14 | * Install Maven (`brew install maven`，`sudo apt-get install maven`). 15 | 16 | If you use the sub-account AK, you need to configure the following permissions for the account to ensure it can put logs to the target logstore. 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 |
ActionResource
log:PostLogStoreLogsacs:log:${regionName}:${projectOwnerAliUid}:project/${projectName}/logstore/${logstoreName}
32 | 33 | ## Set environment variables 34 | The sample will read the value of project, endpoint, logstore and AK from environment variables. You can set the value of them in `.producerrc` and then run the command `source .producerrc`. 35 | 36 | ## Run the Sample 37 | After the above tasks have been finished, do a clean build to compile the source code and generate jar: 38 | ``` 39 | mvn clean package 40 | ``` 41 | 42 | Then run the producer with callback to send some logs to your logstore: 43 | ``` 44 | mvn exec:java -Dexec.mainClass="com.aliyun.openservices.aliyun.log.producer.sample.SampleProducerWithCallback" 45 | ``` 46 | 47 | Run the producer with future to send some logs to your logstore: 48 | ``` 49 | mvn exec:java -Dexec.mainClass="com.aliyun.openservices.aliyun.log.producer.sample.SampleProducerWithFuture" 50 | ``` 51 | 52 | 53 | 54 | 55 | -------------------------------------------------------------------------------- /README_CN.md: -------------------------------------------------------------------------------- 1 | # Aliyun LOG Java Producer 应用示例 2 | 3 | [![License](https://img.shields.io/badge/license-Apache2.0-blue.svg)](/LICENSE) 4 | 5 | [README in English](/README.md) 6 | 7 | 本 repo 提供了使用 [Aliyun LOG Java Producer](https://github.com/aliyun/aliyun-log-producer) 的样例程序，有助于快速上手 producer library。 8 | 9 | ## 前提条件 10 | 您需要准备好以下内容： 11 | * 开通[日志服务](https://cn.aliyun.com/product/sls)，创建好用于存放数据的 project、logStore。 12 | * 准备好 producer 需要的 AK。 13 | * 安装 JDK 1.6+。 14 | * 安装 Maven (`brew install maven`，`sudo apt-get install maven`)。 15 | 16 | 如果您使用的是子账号 AK，需要为该账号配置以下权限来保证它能够将日志写入目标 logStore。 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 |
ActionResource
log:PostLogStoreLogsacs:log:${regionName}:${projectOwnerAliUid}:project/${projectName}/logstore/${logstoreName}
31 | 32 | 关于如何配置权限还可以参考文章 [RAM 子用户访问](https://help.aliyun.com/document_detail/29049.html)。 33 | 34 | 35 | ## 设置环境变量 36 | 示例程序会从环境变量中读取 project、endpoint、logStore、AK 等信息，您可以将这些值填入文件 .producerrc 中，然后执行命令`source .producerrc`。 37 | 38 | ## 运行样例 39 | 准备工作完成后，执行以下命令编译源码、生成 jar 包。 40 | ``` 41 | mvn clean package 42 | ``` 43 | 44 | 译完成后，输入以下命令运行使用 callback 的样例。 45 | ``` 46 | mvn exec:java -Dexec.mainClass="com.aliyun.openservices.aliyun.log.producer.sample.SampleProducerWithCallback" 47 | ``` 48 | 49 | 译完成后，输入以下命令运行使用 future 的样例。 50 | ``` 51 | mvn exec:java -Dexec.mainClass="com.aliyun.openservices.aliyun.log.producer.sample.SampleProducerWithFuture" 52 | ``` 53 | 54 | 关于 SamplePerformance 的运行方式请参考文档 [SamplePerformance 运行步骤](https://github.com/aliyun/aliyun-log-producer-sample/blob/master/PERF_README_CN.md)。 55 | 56 | ## 相关资料 57 | * [日志上云利器 - Aliyun LOG Java Producer](https://yq.aliyun.com/articles/682762) 58 | * [Aliyun LOG Java Producer 快速入门](https://yq.aliyun.com/articles/682761) 59 | 60 | 61 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | 8 | org.sonatype.oss 9 | oss-parent 10 | 9 11 | 12 | 13 | com.aliyun.openservices 14 | aliyun-log-producer-sample 15 | 0.0.1-SNAPSHOT 16 | 17 | Aliyun LOG Java Producer Sample Application 18 | https://aliyun.com/product/sls 19 | Aliyun LOG Java Producer Sample Application 20 | 21 | 22 | 23 | 24 | repo 25 | Copyright (C) Alibaba Cloud Computing. All rights reserved. 26 | 27 | 28 | 29 | https://github.com/aliyun/aliyun-log-producer-sample 30 | scm:git:https://github.com/aliyun/aliyun-log-producer-sample.git 31 | scm:git:https://github.com/aliyun/aliyun-log-producer-sample.git 32 | 33 | 34 | 35 | 36 | aliyunproducts 37 | Aliyun Log development team 38 | aliyunsdk@aliyun.com 39 | 40 | 41 | 42 | UTF-8 43 | 1.6 44 | 1.6 45 | 0.3.9 46 | 0.6.56 47 | 2.5.0 48 | 1.2.3 49 | 50 | 51 | 52 | 53 | com.aliyun.openservices 54 | aliyun-log-producer 55 | ${aliyun-log-producer.version} 56 | 57 | 58 | com.aliyun.openservices 59 | aliyun-log 60 | ${aliyun-log.version} 61 | 62 | 63 | com.google.protobuf 64 | protobuf-java 65 | ${protobuf-java.version} 66 | 67 | 68 | ch.qos.logback 69 | logback-core 70 | ${logback.version} 71 | 72 | 73 | ch.qos.logback 74 | logback-classic 75 | ${logback.version} 76 | 77 | 78 | 79 | 80 | 81 | release-sign-artifacts 82 | 83 | 84 | performRelease 85 | true 86 | 87 | 88 | 89 | 90 | 91 | org.apache.maven.plugins 92 | maven-gpg-plugin 93 | 1.5 94 | 95 | 96 | sign-artifacts 97 | verify 98 | 99 | sign 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | org.apache.maven.plugins 113 | maven-compiler-plugin 114 | 3.5.1 115 | 116 | ${maven.compiler.source} 117 | ${maven.compiler.target} 118 | ${project.build.sourceEncoding} 119 | 120 | 121 | 122 | org.apache.maven.plugins 123 | maven-source-plugin 124 | 3.0.0 125 | 126 | 127 | attach-sources 128 | 129 | jar 130 | 131 | 132 | 133 | 134 | 135 | org.apache.maven.plugins 136 | maven-javadoc-plugin 137 | 2.10.4 138 | 139 | ${project.build.sourceEncoding} 140 | ${project.build.sourceEncoding} 141 | 142 | 143 | author 144 | X 145 | 146 | 147 | 148 | 149 | 150 | attach-javadocs 151 | 152 | jar 153 | 154 | 155 | 156 | 157 | 158 | com.coveo 159 | fmt-maven-plugin 160 | 2.6.0 161 | 162 | 163 | 164 | format 165 | 166 | 167 | 168 | 169 | 170 | org.apache.maven.plugins 171 | maven-jar-plugin 172 | 2.3.1 173 | 174 | 175 | 176 | com.aliyun.openservices.aliyun.log.producer.sample.SamplePerformance 177 | true 178 | ./lib/ 179 | false 180 | 181 | 182 | 183 | 184 | 185 | org.apache.maven.plugins 186 | maven-dependency-plugin 187 | 188 | 189 | copy 190 | package 191 | 192 | copy-dependencies 193 | 194 | 195 | ${project.build.directory}/lib 196 | false 197 | false 198 | true 199 | 200 | 201 | 202 | 203 | 204 | 205 | 206 | -------------------------------------------------------------------------------- /src/main/java/com/aliyun/openservices/aliyun/log/producer/sample/SamplePerformance.java: -------------------------------------------------------------------------------- 1 | package com.aliyun.openservices.aliyun.log.producer.sample; 2 | 3 | import com.aliyun.openservices.aliyun.log.producer.Callback; 4 | import com.aliyun.openservices.aliyun.log.producer.LogProducer; 5 | import com.aliyun.openservices.aliyun.log.producer.Producer; 6 | import com.aliyun.openservices.aliyun.log.producer.ProducerConfig; 7 | import com.aliyun.openservices.aliyun.log.producer.ProjectConfig; 8 | import com.aliyun.openservices.aliyun.log.producer.Result; 9 | import com.aliyun.openservices.aliyun.log.producer.errors.ProducerException; 10 | import com.aliyun.openservices.log.common.LogItem; 11 | import java.util.Random; 12 | import java.util.concurrent.CountDownLatch; 13 | import java.util.concurrent.ExecutorService; 14 | import java.util.concurrent.Executors; 15 | import java.util.concurrent.atomic.AtomicInteger; 16 | import org.slf4j.Logger; 17 | import org.slf4j.LoggerFactory; 18 | 19 | public class SamplePerformance { 20 | 21 | private static final Logger LOGGER = LoggerFactory.getLogger(SampleProducerWithCallback.class); 22 | 23 | private static final Random random = new Random(); 24 | 25 | public static void main(String[] args) throws InterruptedException { 26 | final String project = System.getenv("PROJECT"); 27 | final String logStore = System.getenv("LOG_STORE"); 28 | final String endpoint = System.getenv("ENDPOINT"); 29 | final String accessKeyId = System.getenv("ACCESS_KEY_ID"); 30 | final String accessKeySecret = System.getenv("ACCESS_KEY_SECRET"); 31 | int sendThreadCount = Integer.valueOf(System.getenv("SEND_THREAD_COUNT")); 32 | final int times = Integer.valueOf(System.getenv("TIMES")); 33 | int ioThreadCount = Integer.valueOf(System.getenv("IO_THREAD_COUNT")); 34 | int totalSizeInBytes = Integer.valueOf(System.getenv("TOTAL_SIZE_IN_BYTES")); 35 | LOGGER.info( 36 | "project={}, logStore={}, endpoint={}, sendThreadCount={}, times={}, ioThreadCount={}, totalSizeInBytes={}", 37 | project, 38 | logStore, 39 | endpoint, 40 | sendThreadCount, 41 | times, 42 | ioThreadCount, 43 | totalSizeInBytes); 44 | ExecutorService executorService = Executors.newFixedThreadPool(sendThreadCount); 45 | ProducerConfig producerConfig = new ProducerConfig(); 46 | producerConfig.setBatchSizeThresholdInBytes(3 * 1024 * 1024); 47 | producerConfig.setBatchCountThreshold(40960); 48 | producerConfig.setIoThreadCount(ioThreadCount); 49 | producerConfig.setTotalSizeInBytes(totalSizeInBytes); 50 | 51 | final Producer producer = new LogProducer(producerConfig); 52 | producer.putProjectConfig(new ProjectConfig(project, endpoint, accessKeyId, accessKeySecret)); 53 | 54 | final AtomicInteger successCount = new AtomicInteger(0); 55 | final CountDownLatch latch = new CountDownLatch(sendThreadCount); 56 | LOGGER.info("Test started."); 57 | long t1 = System.currentTimeMillis(); 58 | for (int i = 0; i < sendThreadCount; ++i) { 59 | executorService.submit( 60 | new Runnable() { 61 | @Override 62 | public void run() { 63 | try { 64 | for (int i = 0; i < times; ++i) { 65 | int r = random.nextInt(times); 66 | producer.send( 67 | project, 68 | logStore, 69 | generateTopic(r), 70 | generateSource(r), 71 | generateLogItem(r), 72 | new Callback() { 73 | @Override 74 | public void onCompletion(Result result) { 75 | if (result.isSuccessful()) { 76 | successCount.incrementAndGet(); 77 | } 78 | } 79 | }); 80 | } 81 | } catch (Exception e) { 82 | LOGGER.error("Failed to send log, e=", e); 83 | } finally { 84 | latch.countDown(); 85 | } 86 | } 87 | }); 88 | } 89 | latch.await(); 90 | while (true) { 91 | if (successCount.get() == sendThreadCount * times) { 92 | break; 93 | } 94 | Thread.sleep(100); 95 | } 96 | long t2 = System.currentTimeMillis(); 97 | LOGGER.info("Test end."); 98 | LOGGER.info("======Summary======"); 99 | LOGGER.info("Total count " + sendThreadCount * times + "."); 100 | long timeCost = t2 - t1; 101 | LOGGER.info("Time cost " + timeCost + " millis"); 102 | try { 103 | producer.close(); 104 | } catch (ProducerException e) { 105 | LOGGER.error("Failed to close producer, e=", e); 106 | } 107 | executorService.shutdown(); 108 | } 109 | 110 | private static LogItem generateLogItem(int r) { 111 | LogItem logItem = new LogItem(); 112 | logItem.PushBack("content_key_1", "1abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_" + r); 113 | logItem.PushBack("content_key_2", "2abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_" + r); 114 | logItem.PushBack("content_key_3", "3abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_" + r); 115 | logItem.PushBack("content_key_4", "4abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_" + r); 116 | logItem.PushBack("content_key_5", "5abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_" + r); 117 | logItem.PushBack("content_key_6", "6abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_" + r); 118 | logItem.PushBack("content_key_7", "7abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_" + r); 119 | logItem.PushBack("content_key_8", "8abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_" + r); 120 | return logItem; 121 | } 122 | 123 | private static String generateTopic(int r) { 124 | return "topic-" + r % 5; 125 | } 126 | 127 | private static String generateSource(int r) { 128 | return "source-" + r % 10; 129 | } 130 | } 131 | -------------------------------------------------------------------------------- /src/main/java/com/aliyun/openservices/aliyun/log/producer/sample/SampleProducerWithCallback.java: -------------------------------------------------------------------------------- 1 | package com.aliyun.openservices.aliyun.log.producer.sample; 2 | 3 | import com.aliyun.openservices.aliyun.log.producer.Callback; 4 | import com.aliyun.openservices.aliyun.log.producer.Producer; 5 | import com.aliyun.openservices.aliyun.log.producer.Result; 6 | import com.aliyun.openservices.aliyun.log.producer.errors.LogSizeTooLargeException; 7 | import com.aliyun.openservices.aliyun.log.producer.errors.ProducerException; 8 | import com.aliyun.openservices.aliyun.log.producer.errors.TimeoutException; 9 | import com.aliyun.openservices.log.common.LogItem; 10 | import java.util.concurrent.CountDownLatch; 11 | import java.util.concurrent.ExecutorService; 12 | import java.util.concurrent.Executors; 13 | import java.util.concurrent.atomic.AtomicLong; 14 | import org.slf4j.Logger; 15 | import org.slf4j.LoggerFactory; 16 | 17 | public class SampleProducerWithCallback { 18 | 19 | private static final Logger LOGGER = LoggerFactory.getLogger(SampleProducerWithCallback.class); 20 | 21 | private static final ExecutorService EXECUTOR_SERVICE = Executors.newFixedThreadPool(10); 22 | 23 | public static void main(String[] args) throws InterruptedException { 24 | final Producer producer = Utils.createProducer(); 25 | 26 | int nTask = 100; 27 | 28 | // The monotonically increasing sequence number we will put in the data of each log 29 | final AtomicLong sequenceNumber = new AtomicLong(0); 30 | 31 | // The number of logs that have finished (either successfully send, or failed) 32 | final AtomicLong completed = new AtomicLong(0); 33 | 34 | final CountDownLatch latch = new CountDownLatch(nTask); 35 | 36 | for (int i = 0; i < nTask; ++i) { 37 | EXECUTOR_SERVICE.submit( 38 | new Runnable() { 39 | @Override 40 | public void run() { 41 | LogItem logItem = Utils.generateLogItem(sequenceNumber.getAndIncrement()); 42 | try { 43 | String project = System.getenv("PROJECT"); 44 | String logStore = System.getenv("LOG_STORE"); 45 | producer.send( 46 | project, 47 | logStore, 48 | Utils.getTopic(), 49 | Utils.getSource(), 50 | logItem, 51 | new SampleCallback(project, logStore, logItem, completed)); 52 | } catch (InterruptedException e) { 53 | LOGGER.warn("The current thread has been interrupted during send logs."); 54 | } catch (Exception e) { 55 | if (e instanceof LogSizeTooLargeException) { 56 | LOGGER.error( 57 | "The size of log is larger than the maximum allowable size, e={}", e); 58 | } else if (e instanceof TimeoutException) { 59 | LOGGER.error( 60 | "The time taken for allocating memory for the logs has surpassed., e={}", e); 61 | } else { 62 | LOGGER.error("Failed to send log, logItem={}, e=", logItem, e); 63 | } 64 | } finally { 65 | latch.countDown(); 66 | } 67 | } 68 | }); 69 | } 70 | latch.await(); 71 | EXECUTOR_SERVICE.shutdown(); 72 | 73 | Utils.doSomething(); 74 | 75 | try { 76 | producer.close(); 77 | } catch (InterruptedException e) { 78 | LOGGER.warn("The current thread has been interrupted from close."); 79 | } catch (ProducerException e) { 80 | LOGGER.info("Failed to close producer, e=", e); 81 | } 82 | 83 | LOGGER.info("All log complete, completed={}", completed.get()); 84 | } 85 | 86 | private static final class SampleCallback implements Callback { 87 | 88 | private static final Logger LOGGER = LoggerFactory.getLogger(SampleCallback.class); 89 | 90 | private final String project; 91 | 92 | private final String logStore; 93 | 94 | private final LogItem logItem; 95 | 96 | private final AtomicLong completed; 97 | 98 | SampleCallback(String project, String logStore, LogItem logItem, AtomicLong completed) { 99 | this.project = project; 100 | this.logStore = logStore; 101 | this.logItem = logItem; 102 | this.completed = completed; 103 | } 104 | 105 | @Override 106 | public void onCompletion(Result result) { 107 | try { 108 | if (result.isSuccessful()) { 109 | LOGGER.info("Send log successfully."); 110 | } else { 111 | LOGGER.error( 112 | "Failed to send log, project={}, logStore={}, logItem={}, result={}", 113 | project, 114 | logStore, 115 | logItem.ToJsonString(), 116 | result); 117 | } 118 | } finally { 119 | completed.getAndIncrement(); 120 | } 121 | } 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /src/main/java/com/aliyun/openservices/aliyun/log/producer/sample/SampleProducerWithFuture.java: -------------------------------------------------------------------------------- 1 | package com.aliyun.openservices.aliyun.log.producer.sample; 2 | 3 | import com.aliyun.openservices.aliyun.log.producer.*; 4 | import com.aliyun.openservices.aliyun.log.producer.errors.LogSizeTooLargeException; 5 | import com.aliyun.openservices.aliyun.log.producer.errors.MaxBatchCountExceedException; 6 | import com.aliyun.openservices.aliyun.log.producer.errors.ProducerException; 7 | import com.aliyun.openservices.aliyun.log.producer.errors.ResultFailedException; 8 | import com.aliyun.openservices.aliyun.log.producer.errors.TimeoutException; 9 | import com.aliyun.openservices.log.common.LogItem; 10 | import com.google.common.util.concurrent.FutureCallback; 11 | import com.google.common.util.concurrent.Futures; 12 | import com.google.common.util.concurrent.ListenableFuture; 13 | import java.util.List; 14 | import java.util.concurrent.ExecutorService; 15 | import java.util.concurrent.Executors; 16 | import java.util.concurrent.TimeUnit; 17 | import java.util.concurrent.atomic.AtomicLong; 18 | import org.checkerframework.checker.nullness.qual.Nullable; 19 | import org.slf4j.Logger; 20 | import org.slf4j.LoggerFactory; 21 | 22 | public class SampleProducerWithFuture { 23 | 24 | private static final Logger LOGGER = LoggerFactory.getLogger(SampleProducerWithFuture.class); 25 | 26 | private static final ExecutorService EXECUTOR_SERVICE = Executors 27 | .newFixedThreadPool(Math.max(Runtime.getRuntime().availableProcessors(), 1)); 28 | 29 | public static void main(String[] args) throws InterruptedException { 30 | Producer producer = Utils.createProducer(); 31 | int n = 100; 32 | int size = 20; 33 | 34 | // The number of logs that have finished (either successfully send, or failed) 35 | final AtomicLong completed = new AtomicLong(0); 36 | 37 | for (int i = 0; i < n; ++i) { 38 | List logItems = Utils.generateLogItems(size); 39 | try { 40 | String project = System.getenv("PROJECT"); 41 | String logStore = System.getenv("LOG_STORE"); 42 | ListenableFuture f = producer.send(project, logStore, logItems); 43 | Futures.addCallback( 44 | f, new SampleFutureCallback(project, logStore, logItems, completed), EXECUTOR_SERVICE); 45 | } catch (InterruptedException e) { 46 | LOGGER.warn("The current thread has been interrupted during send logs."); 47 | } catch (Exception e) { 48 | if (e instanceof MaxBatchCountExceedException) { 49 | LOGGER.error("The logs exceeds the maximum batch count, e={}", e); 50 | } else if (e instanceof LogSizeTooLargeException) { 51 | LOGGER.error("The size of log is larger than the maximum allowable size, e={}", e); 52 | } else if (e instanceof TimeoutException) { 53 | LOGGER.error("The time taken for allocating memory for the logs has surpassed., e={}", e); 54 | } else { 55 | LOGGER.error("Failed to send logs, e=", e); 56 | } 57 | } 58 | } 59 | 60 | Utils.doSomething(); 61 | 62 | try { 63 | producer.close(); 64 | } catch (InterruptedException e) { 65 | LOGGER.warn("The current thread has been interrupted from close."); 66 | } catch (ProducerException e) { 67 | LOGGER.info("Failed to close producer, e=", e); 68 | } 69 | 70 | EXECUTOR_SERVICE.shutdown(); 71 | while (!EXECUTOR_SERVICE.isTerminated()) { 72 | EXECUTOR_SERVICE.awaitTermination(100, TimeUnit.MILLISECONDS); 73 | } 74 | LOGGER.info("All log complete, completed={}", completed.get()); 75 | } 76 | 77 | private static final class SampleFutureCallback implements FutureCallback { 78 | 79 | private static final Logger LOGGER = LoggerFactory.getLogger(SampleFutureCallback.class); 80 | 81 | private final String project; 82 | 83 | private final String logStore; 84 | 85 | private final List logItems; 86 | 87 | private final AtomicLong completed; 88 | 89 | SampleFutureCallback( 90 | String project, String logStore, List logItems, AtomicLong completed) { 91 | this.project = project; 92 | this.logStore = logStore; 93 | this.logItems = logItems; 94 | this.completed = completed; 95 | } 96 | 97 | @Override 98 | public void onSuccess(@Nullable Result result) { 99 | LOGGER.info("Send logs successfully."); 100 | completed.getAndIncrement(); 101 | } 102 | 103 | @Override 104 | public void onFailure(Throwable t) { 105 | if (t instanceof ResultFailedException) { 106 | Result result = ((ResultFailedException) t).getResult(); 107 | LOGGER.error( 108 | "Failed to send logs, project={}, logStore={}, result={}", project, logStore, result); 109 | } else { 110 | LOGGER.error("Failed to send log, e=", t); 111 | } 112 | completed.getAndIncrement(); 113 | } 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /src/main/java/com/aliyun/openservices/aliyun/log/producer/sample/Utils.java: -------------------------------------------------------------------------------- 1 | package com.aliyun.openservices.aliyun.log.producer.sample; 2 | 3 | import com.aliyun.openservices.aliyun.log.producer.*; 4 | import com.aliyun.openservices.log.common.LogItem; 5 | import java.util.ArrayList; 6 | import java.util.List; 7 | import java.util.Random; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | public class Utils { 12 | 13 | private static final Logger LOGGER = LoggerFactory.getLogger(Utils.class); 14 | 15 | private static final Random RANDOM = new Random(); 16 | 17 | private static final String CHARS = "abcdefghijklmnopqrstuvwxyz"; 18 | 19 | public static Producer createProducer() { 20 | ProducerConfig producerConfig = new ProducerConfig(); 21 | Producer producer = new LogProducer(producerConfig); 22 | producer.putProjectConfig(buildProjectConfig()); 23 | return producer; 24 | } 25 | 26 | private static ProjectConfig buildProjectConfig() { 27 | String project = System.getenv("PROJECT"); 28 | String endpoint = System.getenv("ENDPOINT"); 29 | String accessKeyId = System.getenv("ACCESS_KEY_ID"); 30 | String accessKeySecret = System.getenv("ACCESS_KEY_SECRET"); 31 | return new ProjectConfig(project, endpoint, accessKeyId, accessKeySecret); 32 | } 33 | 34 | public static LogItem generateLogItem(long id) { 35 | LogItem logItem = new LogItem(); 36 | logItem.PushBack("id", String.valueOf(id)); 37 | logItem.PushBack("key1", generateStr(10)); 38 | logItem.PushBack("key2", generateStr(10)); 39 | return logItem; 40 | } 41 | 42 | public static List generateLogItems(int n) { 43 | List logItems = new ArrayList(); 44 | for (int i = 0; i < n; ++i) { 45 | logItems.add(generateLogItem(i)); 46 | } 47 | return logItems; 48 | } 49 | 50 | private static String generateStr(int len) { 51 | StringBuilder sb = new StringBuilder(); 52 | for (int i = 0; i < len; ++i) { 53 | sb.append(CHARS.charAt(RANDOM.nextInt(26))); 54 | } 55 | return sb.toString(); 56 | } 57 | 58 | public static void doSomething() throws InterruptedException { 59 | LOGGER.info("Before doSomething"); 60 | Thread.sleep(3000); 61 | LOGGER.info("After doSomething"); 62 | } 63 | 64 | public static String getTopic() { 65 | return "topic-" + RANDOM.nextInt(5); 66 | } 67 | 68 | public static String getSource() { 69 | return "source-" + RANDOM.nextInt(10); 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | --------------------------------------------------------------------------------