├── .gitignore ├── .travis.yml ├── CHANGELOG.md ├── DIAGNOSIS_CN.md ├── FAQ_CN.md ├── LICENSE ├── Makefile ├── PERFORMANCE_CN.md ├── README.md ├── README_EN.md ├── RELEASE_CN.md ├── pom.xml └── src ├── main ├── java │ └── com │ │ └── aliyun │ │ └── openservices │ │ └── aliyun │ │ └── log │ │ └── producer │ │ ├── Attempt.java │ │ ├── Callback.java │ │ ├── LogProducer.java │ │ ├── Producer.java │ │ ├── ProducerConfig.java │ │ ├── ProjectConfig.java │ │ ├── Result.java │ │ ├── ShardHashAdjuster.java │ │ ├── errors │ │ ├── Errors.java │ │ ├── LogSizeTooLargeException.java │ │ ├── MaxBatchCountExceedException.java │ │ ├── ProducerException.java │ │ ├── ResultFailedException.java │ │ ├── RetriableErrors.java │ │ └── TimeoutException.java │ │ ├── example │ │ └── ExampleUsage.java │ │ └── internals │ │ ├── BatchHandler.java │ │ ├── ExpiredBatches.java │ │ ├── GroupKey.java │ │ ├── IOThreadPool.java │ │ ├── LogAccumulator.java │ │ ├── LogSizeCalculator.java │ │ ├── LogThread.java │ │ ├── Mover.java │ │ ├── ProducerBatch.java │ │ ├── RetryQueue.java │ │ ├── SendProducerBatchTask.java │ │ └── Utils.java └── resources │ └── META-INF │ └── LICENSE └── test ├── java └── com │ └── aliyun │ └── openservices │ └── aliyun │ └── log │ └── producer │ ├── ProducerConfigTest.java │ ├── ProducerInvalidTest.java │ ├── ProducerLargeAmountTest.java │ ├── ProducerMultiProjectTest.java │ ├── ProducerMultiShardTest.java │ ├── ProducerTest.java │ ├── ShardHashAdjusterTest.java │ └── internals │ ├── LogSizeCalculatorTest.java │ ├── ProducerBatchTest.java │ ├── RetryQueueTest.java │ └── UtilsTest.java └── resources └── logback-test.xml /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | local_env 3 | .local_producerrc 4 | pics/ 5 | target/ 6 | .DS_Store 7 | pom.xml.releaseBackup 8 | release.properties 9 | *.iml -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: java 2 | jdk: 3 | - openjdk8 4 | install: mvn install -DskipTests=true -Dmaven.javadoc.skip=true 5 | script: mvn test 6 | env: 7 | global: 8 | - LEVEL=OFF 9 | - secure: TUStOM6+RsIgJ+4Pbk8eL/c93FrNB2/8m8HS/Br454XWDCuA4XPFR3L6CkDcDnqUhXpR6EtSty50lCPSl/EXNTEmE6Dvlf6U/YnEvlOXZA2awFG1tCS7i/rewSvvawvCdYnXdYljF6GIl0wM2ZCwEXXtb2o86hCyylLEOByO8j2KYWspBPcnNY6/eeu/wgOJZ+dAW5dGYXnVDhGHm7fTbIBWu5dP+dW0p/8KshXZL5TaKxKbWwh2uu3PomFsW4s4qIs5Vf9FHO3lcwYbbdE2dM1BkzVLCIrqAz0qA7C2C7aACIHxiJICFdl2Ex+RIMAMl+SXq0DhaHx+4s2N929v73tJ38AFolxCklI43eroncKUfantUTBeYPoeATx3XULAS3v66GR+oeV+99HxNYoDSXsKlpETVG+KkEDrfUjNS4OQrL0bERuSrLYebCW/rbjd/h9/oXuV02E66qFaoJ/smx23Q6bYimmgpbe9R3z4HheCS7NB+x554OHjeqBgFJQmdCnwHv0PSq23CatNRtXD4tUhBQscbu5jnTKhOCLZ7QfpbqUqV2kU9FyIBZ6hOiEKfh4Ou2qHChAz2y8IP8s5IbLH92e1nSuKKSHeVXx/k3NPrHgS5gidUl3FHXSZWiHXnK2IwL2h7ZdRAPwpTuyVZ7545CRoRDb2xbged5MuNvU= 10 | - secure: czv2mg/Urncnv797bYVjom0ODCiBRK7w9qBt1N5Y0U3nZROEDxqni4Epl3A3d7IlWR30m42WhE0pCUBhN6W74Z6ufQxjVpzCvlBABOMor9+cfZ8QbYfqHcmnm1Jj2kh6lOPQEh25QnBoHaAaseUJlJjQG9F5zYd8zUdbDrb9C0v+2qsd0DxU4+Ygwoix0JrZ9gNO6cTk+3GVROLk8bI4t1dAiy8i0SnAT4lfO2WvlHDNEKoAMXE6sSAkWhmJMUhdxvOQz/ExQtzNaIciLpKEB2lh0TH3T9PEguIdskMTTlCmIhVSsm8x1u/QmMSMaRgOd9Q9UtOl7Vfs25js1/Nv4tBb/RuHetOKdrdX4IsYVRqE56foUhUqML896ASs5CsORoyr/oNEKIuwF3uVugbVxSiMaOnX+3N8DsLy1fw1/jscJE3cC7yJZeFeuj6Cmp0OfjR8bGD6SL9GNAKSeK5WsTt0s6ikrOmIKnHr6OLaqZ7YRDyuzqmnnMiTauh30VC80fsrg8y92p8tkxDLOQoiOefHFFrJzcfX9j2XTiihzEoHijngJr0aIYMK/qidAxOwIoF/aZBGtdC7kh9KTB4aRo4wlrjmJupsXbOYsBc78+sVFhlUD1ShFPE57/DNQOqE5d5ITOR13UfeOpAnWqZAlpEUL/+I36FoY8blVjqWTkc= 11 | - secure: lwC++UKdjkcRukgFVuS0cJUoyOftCLgX0VdOAUD5imKQGipV1BEE6VQ4mFV6XKRM0ghAmM0Gp6SIlscZjOy8lE4czyOo2aG1YcQ6VG9JG54ybRLRr79erZZtNdUe3NmxwWdsX7ECIhO8eQ8A8sbaKUE1R5ehhWj6nJ0cBN8jqqmQQ3v0tCM+TWUl5Fa9YQeScpN5JIDCtWZqYSvOIUv0S6LrP6nHgEZg/y83qm5drEn/16bOv7tB93Nzz/P9k5y59SrFCBUafgcXDsiqDQnzyjSY3cjBCGnVWKplNk3rmokImATTnOh2eDyzV85gMKByk0xIs+2+z3RnEbxxxQnwMoGDkw3P8ThgZtOmceSCU13uWHy4fjmefBEBumM3Y9ehwBGuF6QB87JQGjk5EqpSrCwNqZoYOaWMxe5RaqVmM3YX/xDOr/3P9PBcyeWiN5WMT6TXm1eTCKD6pSCjvzrVpxGfWaE8ZDaFAmzyvNEJp0Cx66ahf+iA1UvD/um0V7ZmBslPSZbI2iWJjWDAuV2RMU8StAUvf3YonUiLszb158hoF/wDL3flblMcKtc1E6A3Pe0hS+b4rBG2pUzjVxTGLtwqgiSjnKwNLw1OUMhPtC6prMn/NRdeal7iZQZ3AZnihKXHSDchrgd3eBI7GBLby/a0S+kwnZ7yLWYeKkgT71A= 12 | - secure: QI5oc6VSkX1aFmUKx5IIOcRnfYj22NuoEB9+Kamsu5kUFG5u9WMccH9OqUDec3GPDccQIzasrMW+NfVb30AKZIlcPzrVfDv3xVcqp9XkEvT2AVRwIZ/aRX1Az/loUmo32Ew3Pbl2r45ws4iw4BOw5TIE+mxy844wDs8UaFi/+dm13ecmlvUbEc12q4O8ZpXr0siWf4jGak9c946KQ5BUr1azoCYJH4yVZ4r88ZKoBdSbOYNFYep53/phGYMcYf68Dy+NlnXPYluNc7fqBh7OEjCa+RCyZlxbZ/s2Kb2y86pcCKetFzTQg3sR3g50DhZatGQSdrq+BBdk0KfrLtFAIA35oIfGzIRK3HbcEIPO4YYruRZ3wecaQKk39vKghGMVgtYBdZWPIluvPHlcXFMG9QJQLViI7VKc/e/M9oL0Vnjr/h478A/u7Lg+Sa52UwsTaFGHgetiWw0+jak5JlWBTG5vOiBUuQ4VQbShBWRERycVmCDl/4us8apbpy8NUSeHoZV1AtXcMtnFoR5Ief194Hf/OGfvwAJZKP1sghVyPVmsX2zu7lUvK/gcWgVmLL2nrG5No1Tz2y1i/+nYltEMleOWnPLZ7ZfwN+koGJiB+8aD/zmfChMoyTrI9Ja8bTGjXAPLAUQGcbynG5nr5V5cEmgvZfqMGu5j/ei1ZcHOMsI= 13 | - secure: B8dxNuqK8X2FDSAUwobaRaUx+52AEGORZknk6v+86zomLCr4YU8Mu58cMNthmK3FjvLGyHgh5nyE40ANa+/OTZNKtSmmva4ySCgYCDIzLoAayTpoO7/MBGtDI6ucQbhNC3WqMXzQHYShm99BSdgpxGie+Xcmr7s0fcqW2VWUVKfnY1hMojK6CHcADoJ8MUn13VMI9LlwrD9RVLN+Y4trjNg0pkkNplyp/7Z+Wj4NeWEFe/AqEHeqeApDo+On4yW146rS06LsXF/zaPrFStMMl+dJgyloMUY4Y6BzAM3xQ1RIIcgQbAYCrB2HuAbEQ8cfvr42LXiIkQE8QkjXbVawaWLBo65g6KsjiQ3Q31v0sb03MfGVglGyd+WsESkT5EtSmdfMUmvbCf6CdOThIIMHkdxKzuOyzl8xpwcCFICIinW6wqQcw+DVk88igeCj4UZQbE03PbKW09/HnkK5ryaChZqOuqNOJNYArgg1hToeA/9eRhTtsGtfgwqcxiu8empSk7YUuCaNAKBm+E7INacG5IWtznwfKFJFT9jFOgYObYZSEkmKBxxd+TeSGuY4wg56/5C3VkPNjg3dy472mjeIuZMoEtgftBcAyKFjcPxxarnfU5kZ0Swv9dbml3utLtII487MxUxwwTwvE6ykl7srfrnI2PM30Yr1tIPij2ErC70= 14 | - secure: PsmVmAR+SpsUORT1nSsWVUA+G0i97ed6Xz2Z4D/0TTqJh8G5nbyaRFFZJWm8QEWeMlgiPgltKf6TDHLWRhpDbMbgJxuYm4nI9RCTbGQEfrzYMF6+a5Wr1YK5Oyq9TtATuyz3cyLQvIJqOSQ4q0zJavUEZ9XXVzv8s5V6/zrSlOfgmcwYxyUdQaC2qgvXObhpa/wcaq3L3TOxxHQfAxMjdZfNW7+iZ1G8RIKa1IjSICweQP/m9/xlgarybDGKIDhdNUMpq9SeWDw4GbrscW6BUxVNJbga1Sk5D7fFBFsVmVkeIzWi8AxjKQkJGdfz87b1H7Bvvc9vOzA3MY6na5anqvZGikc6ivoKzrKaXOBRcR2QbKwhikQ2Fio0u/9glH+w7vx4tWxsQyKQW6ye1JRi+2TAimrKjsmwuu/e4SUYMlAWH3AYCDrJSu1rZnfej8lNuomFsU4y8yhH5Mg69cuVaPzjFUBcwm2ah+9aJfCth1LPPX7nSyDMDv0lUyBRkwSh5idCE1DPTbfjWRTn2cm9+sG6h98zI6z3CS/o9mJH+UQL/YX/PRuwG2WJAz3Gqt1riEZdkEgA7bNgRjE/hi+ozkUWYNstuKEh7e+yVFU+y7cPase0MVFsOhoQcaL8e6a8b+EplTNYQU+qLnxBKLlwYoI/bcNWpBYZodzRCO/hn/4= 15 | - secure: sJE5KNF/SxGvQCI0jUbB/wUrY5m9dZMhv+zoS6s4FxYFQe7YfuUaQtWSmbmEe8huCqpXeHMNmFXgSwGw/pu8OCIXBJVrwJE4SHjyLX9C7pb8v0UyD5pyw0W0wP68RAQQGySyJ/YgK9anGxicIt8M4hO3vVV9kqQE17u9ymA4cwc8nxcjDpa0k96j5HmyVIELbiMmozEHVAUiMZk8Jup1d+SDDp4k7XIAnNSH8oukJGU0nsmyJEmuGxpgZExuSeIb5rca8GkNhX9gm+IhhKpETa5gwBMY0D7WSLuI3cotcVHEvQ7yhXHAxYh/1jPx6jrHlIJad5yPsjMT/AGKuYwERwx4GcRyXIYdsw5ivsrtGazcbst1BH382E0oo5meYdMcKSHjxhSY/2VOUtCUbb23j96H+KtenWRxLDxfA+3VeqUkkUMIlZrR22DlvC+wwMx2HpFVOeniWR/Ij5IfyVvG3eh8Dg61yxadkMOtd6yCY2NdAZXlLiFJRqDxp6YaL+jSFyPR4hBqTjW185DWjeh/lr1E6/gxZA+dLih5Q9+i4H9ptTK86QMEfCds+NrqXY+MWAqDpuOfre3aWEJVGEWMDGRVWvm9Y3D3wu+0cmrhlr3yf7G3UUKrN/a4CeMW+HqUcUYVLM/0sCoVW7bsXRoQ2mnkjywMveGrSo7Gva+Wiw4= 16 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## 2023/04/24 v0.3.18 2 | ### Feature 3 | - 支持设置 proxyIp -------------------------------------------------------------------------------- /DIAGNOSIS_CN.md: -------------------------------------------------------------------------------- 1 | # 异常诊断 2 | 3 | 如果您发现数据没有写入日志服务，可通过如下步骤诊断问题。 4 | 1. 检查您项目中引入的`aliyun-log-producer`、`aliyun-log`、`protobuf-java`这三个 jar 包的版本是否和文档中[安装](https://github.com/aliyun/aliyun-log-producer#%E5%AE%89%E8%A3%85)部分列出的 jar 包版本一致，如果不一致请进行升级。 5 | 2. Producer 接口的 send 方法异步发送数据，请通过 Callback 接口或返回的 Future 获取数据发送失败的原因。 6 | 3. 如果您发现并没有回调 Callback 接口的 onCompletion 方法，请检查在您的程序退出之前是否有调用 producer.close() 方法。因为数据发送是由后台线程异步完成的，为了防止缓存在内存里的少量数据丢失，请在程序退出之前务必调用 producer.close() 方法。 7 | 4. Producer 会把运行过程中的关键行为通过日志框架 slf4j 进行输出，您可以在程序中配置好相应的日志实现框架实现并打开 DEBUG 级别的日志。重点检查是否输出了 ERROR 级别的日志。 8 | 5. 如果通过上述步骤仍然没有解决您的问题请联系我们。 9 | -------------------------------------------------------------------------------- /FAQ_CN.md: -------------------------------------------------------------------------------- 1 | # 常见问题 2 | 3 | **Q:** Aliyun LOG Java Producer 依赖 [Aliyun LOG Java SDK](https://github.com/aliyun/aliyun-log-java-sdk)，而 Aliyun LOG Java SDK 依赖了 2.5.0 版本的 protobuf 库，如果该版本的 protobuf 与用户应用程序中自身带的 protobuf 库冲突怎么办？ 4 | 5 | A: 可以使用 Aliyun LOG Java SDK 提供的一个特殊版本`jar-with-dependencies`，它包含第三方依赖库并重写了这些库的命名空间。配置方式如下： 6 | ``` 7 | 8 | com.aliyun.openservices 9 | aliyun-log 10 | ${aliyun-log.version} 11 | jar-with-dependencies 12 | 13 | 14 | com.google.protobuf 15 | protobuf-java 16 | 17 | 18 | 19 | 20 | com.aliyun.openservices 21 | aliyun-log-producer 22 | ${aliyun-log-producer.version} 23 | 24 | 25 | com.google.protobuf 26 | protobuf-java 27 | 28 | 29 | com.aliyun.openservices 30 | aliyun-log 31 | 32 | 33 | 34 | ``` 35 | 36 | **Q:** 一个进程应当创建多少个 producer 实例？ 37 | 38 | A: 创建 producer 的同时会创建一系列线程，是一个相对昂贵的操作，因此强烈建议一个进程共用一个 producer 实例。 使用方式请参考 - [Aliyun LOG Java Producer 快速入门](https://yq.aliyun.com/articles/682761)。 39 | 40 | **Q:** 调用 send 方法发送数据时抛出如下异常`failed to acquire memory within the configured max blocking time xxx ms...`？ 41 | 42 | A：Producer 中的数据最终是由 IO 线程异步发送的，抛出以上异常表明 send 方法的调用速度大于 IO 线程的发送速度。一般而言，可通过以下方式解决： 43 | 44 | 1. 检查配置的 endpoint 是不是公网 endpoint。如果是，请将 endpoint 替换为[内网 endpoint](https://help.aliyun.com/document_detail/29008.html)，因为公网带宽有限。 45 | 2. 检查目标 project、logstore 是否超过了数据写入的[流量和次数限制](https://help.aliyun.com/document_detail/92571.html)，因为这可能导致 producer 不断重试，内存数据堆积。如果是，请调整 project 的写入 quota 或分裂 logstore 的 shard。 46 | 3. 通过参数 ioThreadCount 增加 IO 线程数量，从而加快数据发送速率。 47 | 48 | **Q:** 日志写入过程中返回如下错误`com.aliyun.openservices.log.exception.LogException: denied by sts or ram, action: log:PostLogStoreLogs, resource: acs:log:${regionName}:${projectOwnerAliUid}:project/${projectName}/logstore/${logstoreName}`？ 49 | 50 | A：子账号没有目标 project、logStore 的写权限，请参考 [RAM 子账号访问](https://github.com/aliyun/aliyun-log-java-producer#ram-%E5%AD%90%E8%B4%A6%E5%8F%B7%E8%AE%BF%E9%97%AE)配置相应权限。 51 | 52 | **Q:** 应该在何时调用 producer 的 close() 方法？ 53 | 54 | A: 请在程序退出之前调用 producer 的 close() 方法，以防止缓存在内存中的数据丢失。 55 | 56 | **Q:** 调用 send() 方法为什么会抛出异常`cannot append after the log accumulator was closed`？应用程序该如何处理？ 57 | 58 | A: 说明在调用了 producer 实例的 close() 方法后，仍然尝试调用 producer 实例的 send() 方法。此时，producer 会通过抛出 IllegalStateException 提示调用者它已经处于关闭状态。推荐应用程序在调用 close() 方法后就不要继续调用 send() 方法发送数据，如果做不到这一点，可以 catch IllegalStateException 并打印相关日志。 59 | 60 | **Q:** Producer 会缓存待发送的数据，并将数据合并成 batch 后批量发往服务端。什么样的数据有机会合并在相同的 batch 里？ 61 | 62 | A: 具有相同 project，logStore，topic，source，shardHash 的数据会被合并在一起。为了让数据合并功能充分发挥作用，同时也为了节省内存，建议您控制这 5 个字段的取值范围。如果某个字段如 topic 的取值确实非常多，建议您将其加入 logItem 而不是直接使用 topic。 63 | 64 | **Q:** 分裂 shard 后，原来 shard 变成只读了，为了让数据写入新的 shard 需要重启写入程序吗？ 65 | 66 | A: 不需要。服务端会自动将待写入的数据路由到新分裂出来的可写 shard 上。 67 | 68 | **Q:** Producer 能否保证日志上传顺序？即先发送的日志先写入服务端？ 69 | 70 | A: Producer 异步多线程发送数据，无法保证日志上传顺序。但您可以通过日志中的 time 字段判断日志产生顺序。 71 | 72 | **Q:** Producer 支持通过 HTTPS 发送日志吗？ 73 | 74 | A: 支持。将 ProjectConfig.endpoint 设置成`https://`即可。 75 | 76 | **Q:** 程序运行过程中抛出如下异常`Caused by: java.lang.NoSuchMethodError: com.google.common.hash.Hashing.farmHashFingerprint64()`？ 77 | 78 | A: 这是因为项目引入的 guava lib 不包含 farmHash 函数。Producer 使用的 guava 版本为 [27.0-jre](https://github.com/aliyun/aliyun-log-java-producer/blob/master/pom.xml#L45)，请将项目引入的 guava lib 至少升级到 27.0-jre。 79 | ``` 80 | 81 | com.google.guava 82 | guava 83 | 27.0-jre 84 | 85 | ``` 86 | 87 | **Q:** 程序运行过程中抛出如下异常`java.lang.VerifyError: class com.aliyun.openservices.log.common.Logs$LogGroup overrides final method getUnknownFields.()Lcom/google/protobuf/UnknownFieldSet; 88 | at java.lang.ClassLoader.defineClass1(Native Method) 89 | ...`？ 90 | 91 | A: 检查项目中引入的 protobuf 版本是否低于 2.5.0。这些版本中，类`Logs$LogGroup`的方法`getUnknownFields`被申明为 final，无法override，从而报错。解决方法是引入 2.5.0 版本的 protobuf。 92 | ``` 93 | 94 | com.google.protobuf 95 | protobuf-java 96 | 2.5.0 97 | 98 | ``` 99 | 100 | **Q:** 程序运行过程中抛出如下异常`java.lang.NoSuchMethodError: com.aliyun.openservices.log.request.PutLogsRequest.SetTags(Ljava/util/List;)V...`？ 101 | 102 | A: 这是因为项目引入的`aliyun-log`版本过低造成的，请确保其版本至少为`0.6.33`。 103 | ``` 104 | 105 | com.aliyun.openservices 106 | aliyun-log 107 | 0.6.31 108 | 109 | ``` 110 | 111 | **Q:** 程序运行过程出现异常`Failed to get client, project=xxx`或`errorCode=ProjectConfigNotExist, errorMessage=Cannot get the projectConfig for project xxx`？ 112 | 113 | A: 没有为 project xxx 设置 projectConfig。请按如下方式进行设置。 114 | ``` 115 | Producer producer = new LogProducer(new ProducerConfig()); 116 | ProjectConfig projectConfig = new ProjectConfig("xxx", endpoint, accessKeyId, accessKeySecret); 117 | producer.putProjectConfig(projectConfig); 118 | ``` 119 | Producer 构造方式请参考样例程序 [Utils.java](https://github.com/aliyun/aliyun-log-producer-sample/blob/master/src/main/java/com/aliyun/openservices/aliyun/log/producer/sample/Utils.java#L19)。 120 | 121 | **Q:** aliyun-log-producer 相比 log-loghub-producer，shardHash 的对齐逻辑有什么变化？ 122 | 123 | A：2 者差别如下。 124 | 125 | log-loghub-producer 126 | 1. 定期轮询服务端目标 logstore 的 shard 梳理 127 | 2. 根据二分法，计算指定的 shardHash 应该落在哪个区间里 128 | 129 | 源码 130 | https://github.com/aliyun/aliyun-log-producer-java/blob/master/src/main/java/com/aliyun/openservices/log/producer/inner/ShardHashManager.java#L59 131 | 132 | aliyun-log-producer 133 | 134 | 如果 buckets 是 32，则根据 shardHash 的前 5 位决定该日志应该和哪个区间对齐。 135 | 136 | 源码 137 | https://github.com/aliyun/aliyun-log-java-producer/blob/1ca3b473b1f3a71f73c1b5d513c85b3109e03022/src/main/java/com/aliyun/openservices/aliyun/log/producer/ShardHashAdjuster.java#L23 138 | 139 | 测试代码 140 | https://github.com/aliyun/aliyun-log-java-producer/blob/master/src/test/java/com/aliyun/openservices/aliyun/log/producer/ShardHashAdjusterTest.java#L13 141 | 142 | **Q：** 调用 send() 方法为什么会抛出异常`java.lang.InterruptedException`？ 143 | 144 | A：当调用 send() 方法的线程被中断时会抛出此类异常，一般当主进程准备退出时会主动中断各个线程。 145 | 146 | 147 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: deploy 2 | deploy: 3 | mvn clean deploy -Dmaven.test.skip=true 4 | 5 | .PHONY: release 6 | release: deploy 7 | mvn -Darguments="-DskipTests" release:clean release:prepare release:perform 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /PERFORMANCE_CN.md: -------------------------------------------------------------------------------- 1 | # 性能测试 2 | 3 | ## 测试环境 4 | ### ECS 虚拟机 5 | * 规格族：计算型 c5 6 | * 实例规格：ecs.c5.4xlarge 7 | * CPU：16 vCPU，Intel Xeon(Skylake) Platinum 8163，2.5 GHz 8 | * MEM：32GB 9 | * 内网带宽：5 Gbps 10 | * OS：Linux version 3.10.0-957.1.3.el7.x86_64 11 | 12 | ### JVM 13 | OpenJDK 64-Bit Server VM (build 25.191-b12, mixed mode) 14 | 15 | ### 日志样例 16 | 测试中使用的日志包含 8 个键值对以及 topic、source 两个字段。为了模拟数据的随机性，我们给每个字段值追加了一个随机后缀。其中，topic 后缀取值范围是 \[0, 5)，source 后缀取值范围是 \[0, 10)，其余 8 个键值对后缀取值范围是 \[0, 单线程发送次数)。单条日志大小约为 550 字节，格式如下： 17 | ``` 18 | __topic__: topic- 19 | __source__: source- 20 | content_key_1: 1abcdefghijklmnopqrstuvwxyz!@#$%^&*()_0123456789- 21 | content_key_2: 2abcdefghijklmnopqrstuvwxyz!@#$%^&*()_0123456789- 22 | content_key_3: 3abcdefghijklmnopqrstuvwxyz!@#$%^&*()_0123456789- 23 | content_key_4: 4abcdefghijklmnopqrstuvwxyz!@#$%^&*()_0123456789- 24 | content_key_5: 5abcdefghijklmnopqrstuvwxyz!@#$%^&*()_0123456789- 25 | content_key_6: 6abcdefghijklmnopqrstuvwxyz!@#$%^&*()_0123456789- 26 | content_key_7: 7abcdefghijklmnopqrstuvwxyz!@#$%^&*()_0123456789- 27 | content_key_8: 8abcdefghijklmnopqrstuvwxyz!@#$%^&*()_0123456789- 28 | ``` 29 | 30 | ### Project & Logstore 31 | * Project：在 ECS 所在 region 创建目标 project 并通过 VPC 网络服务入口进行访问。 32 | * Logstore：在该 project 下创建一个分区数为 10 的 logstore（未开启索引），该 logstore 的写入流量最大为 50 MB/s，参阅[数据读写](https://help.aliyun.com/document_detail/92571.html)。 33 | 34 | ## 测试用例 35 | 36 | ### 测试程序说明 37 | * ProducerConfig.totalSizeInBytes: 具体用例中调整 38 | * ProducerConfig.batchSizeThresholdInBytes: 3 \* 1024 \* 1024 39 | * ProducerConfig.batchCountThreshold：40960 40 | * ProducerConfig.lingerMs：2000 41 | * ProducerConfig.ioThreadCount: 具体用例中调整 42 | * JVM 初始堆大小：2 GB 43 | * JVM 最大堆大小：2 GB 44 | * 调用`Producer.send()`方法的线程数量：10 45 | * 每个线程发送日志条数：20,000,000 46 | * 发送日志总大小：约 115 GB 47 | * 客户端压缩后大小：约 12 GB 48 | * 发送日志总条数：200,000,000 49 | 50 | 测试代码：[SamplePerformance.java](https://github.com/aliyun/aliyun-log-producer-sample/blob/master/src/main/java/com/aliyun/openservices/aliyun/log/producer/sample/SamplePerformance.java) 51 | 52 | 运行步骤：[SamplePerformance 运行步骤 53 | ](https://github.com/aliyun/aliyun-log-producer-sample/blob/master/PERF_README_CN.md) 54 | 55 | ### 调整 IO 线程数量 56 | 将 ProducerConfig.totalSizeInBytes 设置为默认值 104,857,600（即 100 MB），通过调整 ProducerConfig.ioThreadCount 观察程序性能。 57 | 58 | | IO 线程数量 | 原始数据吞吐量 | 压缩后数据吞吐量 | CPU 使用率 | 说明 | 59 | | -------- | -------- | -------- | -------- | -------- | 60 | | 1 | 34.296 MB/s | 3.658 MB/s | 60% | 未达 10 个 shard 写入能力里上限。 | 61 | | 2 | 74.131 MB/s | 7.907MB/s | 120% | 未达 10 个 shard 写入能力里上限。 | 62 | | 4 | 142.142 MB/s | 15.160 MB/s | 235% | 未达 10 个 shard 写入能力里上限。 | 63 | | 8 | 279.335 MB/s | 29.792 MB/s | 480% | 未达 10 个 shard 写入能力里上限。 | 64 | | 16 | 450.440 MB/s | 48.040 MB/s | 900% | 未达 10 个 shard 写入能力里上限。 | 65 | | 32 | 508.207 MB/s | 54.201 MB/s | 1350% | 达到 10 个 shard 写入能力里上限，服务端偶尔返回 Write quota exceed。 | 66 | 67 | **说明：** CPU 时间主要花费在对象的序列化和压缩上，在吞吐量较高的情况下 CPU 使用率比较高。但在日常环境中，单机数据流量均值为 100KB/S，因此造成的 CPU 消耗几乎可以忽略不计。 68 | 69 | ### 调整 totalSizeInBytes 70 | 将 ProducerConfig.ioThreadCount 设置为8，通过调整 ProducerConfig.totalSizeInBytes 观察程序性能。 71 | 72 | | TotalSizeInBytes | 原始数据吞吐量 | 压缩后数据吞吐量 | CPU 使用率 | 说明 | 73 | | -------- | -------- | -------- | -------- | -------- | 74 | | 52,428,800 | 268.159 MB/s | 28.599 MB/s | 400% | 未达 10 个 shard 写入能力里上限。 | 75 | | 209,715,200 | 218.876 MB/s | 23.344 MB/s | 630% | 未达 10 个 shard 写入能力里上限。 | 76 | | 419,430,400 | 200.331 MB/s | 21.366 MB/s | 700% | 未达 10 个 shard 写入能力里上限。 | 77 | 78 | ## 总结 79 | 1. 增加 IO 线程数量可以显著提高吞吐量，尤其是当 IO 线程数量少于可用处理器个数时。 80 | 2. 调整 totalSizeInBytes 对吞吐量影响不够显著，增加 totalSizeInBytes 会造成更多的 CPU 消耗，建议使用默认值。 81 | 82 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Aliyun LOG Java Producer 2 | 3 | [![Build Status](https://travis-ci.org/aliyun/aliyun-log-producer-java.svg?branch=master)](https://travis-ci.org/aliyun/aliyun-log-producer-java) 4 | [![License](https://img.shields.io/badge/license-Apache2.0-blue.svg)](/LICENSE) 5 | 6 | [README in English](/README_EN.md) 7 | 8 | Aliyun LOG Java Producer 是一个易于使用且高度可配置的 Java 类库，专门为运行在大数据、高并发场景下的 Java 应用量身打造。 9 | 10 | ## 功能特点 11 | 1. 线程安全 - producer 接口暴露的所有方法都是线程安全的。 12 | 2. 异步发送 - 调用 producer 的发送接口通常能够立即返回。Producer 内部会缓存并合并发送数据，然后批量发送以提高吞吐量。 13 | 3. 自动重试 - 对可重试的异常，producer 会根据用户配置的最大重试次数和重试退避时间进行重试。 14 | 4. 行为追溯 - 用户通过 callback 或 future 不仅能获取当前数据是否发送成功的信息，还可以获得该数据每次被尝试发送的信息，有利于问题追溯和行为决策。 15 | 5. 上下文还原 - 同一个 producer 实例产生的日志在同一上下文中，在服务端可以查看某条日志前后相关的日志。 16 | 6. 优雅关闭 - 保证 close 方法退时，producer 缓存的所有数据都能被处理，用户也能得到相应的通知。 17 | 18 | ## 功能优势 19 | 20 | 使用 producer 相对于直接通过 API 或 SDK 向 LogHub 写数据会有如下优势。 21 | 22 | ### 高性能 23 | 在海量数据、资源有限的前提下，写入端要达到目标吞吐量需要实现复杂的控制逻辑，包括多线程、缓存策略、批量发送等，另外还要充分考虑失败重试的场景。Producer 实现了上述功能，在为您带来性能优势的同时简化了程序开发步骤。 24 | 25 | ### 异步非阻塞 26 | 在可用内存充足的前提下，producer 会对发往 LogHub 的数据进行缓存，因此用户调用 send 方法时能够立即返回，不会阻塞，达到计算与 I/O 逻辑分离的目的。稍后，用户可以通过返回的 future 对象或传入的 callback 获得数据发送的结果。 27 | 28 | ### 资源可控制 29 | 可以通过参数控制 producer 用于缓存待发送数据的内存大小，同时还可以配置用于执行数据发送任务的线程数量。这样做一方面避免了 producer 无限制地消耗资源，另一方面可以让您根据实际情况平衡资源消耗和写入吞吐量。 30 | 31 | ## 安装 32 | 33 | ### Maven 使用者 34 | 将下列依赖加入到您项目的 pom.xml 中。 35 | ``` 36 | 37 | com.aliyun.openservices 38 | aliyun-log-producer 39 | 0.3.23 40 | 41 | 42 | com.aliyun.openservices 43 | aliyun-log 44 | 0.6.116 45 | 46 | 47 | com.google.protobuf 48 | protobuf-java 49 | 2.5.0 50 | 51 | ``` 52 | 53 | jar-with-dependency 版本，可以解决producer依赖的版本冲突 54 | ``` 55 | 56 | com.aliyun.openservices 57 | aliyun-log 58 | 0.6.116 59 | jar-with-dependencies 60 | 61 | ``` 62 | 63 | ### Gradle 使用者 64 | ``` 65 | compile 'com.aliyun.openservices:aliyun-log-producer:0.3.23' 66 | compile 'com.aliyun.openservices:aliyun-log:0.6.116' 67 | compile 'com.google.protobuf:protobuf-java:2.5.0' 68 | ``` 69 | 70 | ## RAM 子账号访问 71 | 如果您使用子账号 AK，请确保该子账号拥有目标 project、logStore 的写权限，具体请参考 [RAM 子用户访问](https://help.aliyun.com/document_detail/29049.html)。 72 | 73 | | Action | Resource | 74 | |---|---| 75 | | log:PostLogStoreLogs | acs:log:${regionName}:${projectOwnerAliUid}:project/${projectName}/logstore/${logstoreName} | 76 | 77 | ## 快速入门 78 | 79 | 参考教程 [Aliyun LOG Java Producer 快速入门](https://yq.aliyun.com/articles/682761)。 80 | 81 | ## 原理剖析 82 | 83 | 参考文章[日志上云利器 - Aliyun LOG Java Producer](https://yq.aliyun.com/articles/682762)。 84 | 85 | ## 应用示例 86 | 87 | https://github.com/aliyun/aliyun-log-producer-sample 88 | 89 | ## 异常诊断 90 | 91 | 参考文档[异常诊断](/DIAGNOSIS_CN.md)。 92 | 93 | ## 常见问题 94 | 95 | 参考文档[常见问题](/FAQ_CN.md)。 96 | 97 | ## 关于性能 98 | 99 | * [性能测试报告](/PERFORMANCE_CN.md) 100 | * [性能诊断利器 JProfiler 快速入门和最佳实践](https://yq.aliyun.com/articles/684776) 101 | 102 | ## 关于升级 103 | 104 | Aliyun LOG Java Producer 是对老版 log-loghub-producer 的全面升级，解决了上一版存在的多个问题，包括网络异常情况下 CPU 占用率过高、关闭 producer 可能出现少量数据丢失等问题。另外，在容错方面也进行了加强，即使您存在误用，在资源、吞吐、隔离等方面都有较好的保证。基于上述原因，强烈建议使用老版 producer 的用户进行升级。 105 | 106 | ## 问题反馈 107 | 如果您在使用过程中遇到了问题，可以创建 [GitHub Issue](https://github.com/aliyun/aliyun-log-producer/issues) 或者前往阿里云支持中心[提交工单](https://workorder.console.aliyun.com/#/ticket/createIndex)。 108 | -------------------------------------------------------------------------------- /README_EN.md: -------------------------------------------------------------------------------- 1 | # Aliyun LOG Java Producer 2 | 3 | [![Build Status](https://travis-ci.org/aliyun/aliyun-log-producer-java.svg?branch=master)](https://travis-ci.org/aliyun/aliyun-log-producer-java) 4 | [![License](https://img.shields.io/badge/license-Apache2.0-blue.svg)](/LICENSE) 5 | 6 | [中文版 README](/README.md) 7 | 8 | The Aliyun LOG Java Producer is an easy-to-use, highly configurable Java library that helps you send data to [Aliyun Log Service](https://www.alibabacloud.com/zh/product/log-service). It designed for Java applications which running in big data and high concurrency scenarios. 9 | 10 | ## Feature 11 | 1. Thread safety - all methods exposed by the producer are thread-safe. 12 | 2. Asynchronous - a call to the send method usually returns immediately and does not wait for the data to be sent or a response to be received from the server. 13 | 3. Automatic retry - it provides an automatic and configurable retry mechanism for retriable exception. 14 | 4. Traceability - the caller can not only obtain the information about whether the current data has been sent successfully, but also the attempts information related to the data through callback or future. 15 | 5. Context restore - the logs generated by the same producer are in the same context, and the relevant logs before and after a certain log can be viewed at the server side. 16 | 6. Graceful shutdown - when the close method falls back, it ensures that all data cached by producer can be processed and the caller can be notified accordingly. 17 | 18 | ## Advantages 19 | 20 | The following list represents some of the major advantages to using the producer library to send data to log service. 21 | 22 | ### High performance 23 | The producer library can help build high-performance producer application. To achieve the throughput needed, producers must implement complicated logic, such as batching or multithreading, in addition to retry logic. The producer library performs all of these tasks for you. 24 | 25 | ### Asynchronous & non-blocking 26 | If the available memory is sufficient, the producer will buffer the data before sending them to log service, it does not force the caller application to block and wait for a confirmation that the data has arrived at the server before continuing execution. The caller can get the result of data transmission through the returned future object or the registerd callback. 27 | 28 | ### Controllable resources usage 29 | The size of memory used by the producer can be controlled by parameters as well as the number of threads used to perform data sending tasks. This can avoid unrestricted resource consumption by producer on the one hand, and allows you to balance resource consumption and write throughput according to the actual situation on the other. 30 | 31 | ## Installation 32 | 33 | ### Maven users 34 | Add this dependency to your project's POM: 35 | ``` 36 | 37 | com.aliyun.openservices 38 | aliyun-log-producer 39 | 0.3.23 40 | 41 | 42 | com.aliyun.openservices 43 | aliyun-log 44 | 0.6.116 45 | 46 | 47 | com.google.protobuf 48 | protobuf-java 49 | 2.5.0 50 | 51 | ``` 52 | 53 | jar-with-dependency version, can resolve dependent version conflicts 54 | ``` 55 | 56 | com.aliyun.openservices 57 | aliyun-log 58 | 0.6.116 59 | jar-with-dependencies 60 | 61 | ``` 62 | 63 | ### Gradle users 64 | ``` 65 | compile 'com.aliyun.openservices:aliyun-log-producer:0.3.23' 66 | compile 'com.aliyun.openservices:aliyun-log:0.6.116' 67 | compile 'com.google.protobuf:protobuf-java:2.5.0' 68 | ``` 69 | ## RAM 70 | If you use the subaccount AK, make sure that the subaccount has write permissions for the target project and logStore, please refer to [RAM](https://www.alibabacloud.com/help/en/doc-detail/29049.htm). 71 | 72 | | Action | Resource | 73 | |---|---| 74 | | log:PostLogStoreLogs | acs:log:${regionName}:${projectOwnerAliUid}:project/${projectName}/logstore/${logstoreName} | 75 | 76 | ## Sample application 77 | 78 | https://github.com/aliyun/aliyun-log-producer-sample 79 | 80 | ## Feedback 81 | If you have further questions please open a [GitHub Issue](https://github.com/aliyun/aliyun-log-producer/issues), or [sumbit a ticket](https://workorder.console.aliyun.com/#/ticket/createIndex). 82 | -------------------------------------------------------------------------------- /RELEASE_CN.md: -------------------------------------------------------------------------------- 1 | # Aliyun LOG Java Producer 新版本发布流程 2 | 3 | ## 前提条件 4 | 确保 [master](https://github.com/aliyun/aliyun-log-producer) 分支最新 commit 中的单元测试全部通过（[链接](https://travis-ci.org/aliyun/aliyun-log-producer)）。 5 | 6 | ## 发布 7 | 1. 进入`aliyun-log-producer`项目的根目录。 8 | 2. 运行命令`make release`。 9 | 3. 确认弹出的新版本信息后，等待命令执行完成（该命令执行完成后，会自动生成下一个版本的信息）。 10 | 4. 登陆 [stagingRepositories](https://oss.sonatype.org/#stagingRepositories)，close 提交的 repository。 11 | 5. 将 close 的 repository release。 12 | 13 | ## 验证 14 | 进入 [nexus-search](https://oss.sonatype.org/index.html#nexus-search;quick~aliyun-log-producer) 查看新版本是否成功发布。 15 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | 5 | 6 | org.sonatype.oss 7 | oss-parent 8 | 9 9 | 10 | 11 | com.aliyun.openservices 12 | aliyun-log-producer 13 | 0.3.25 14 | 15 | Aliyun LOG Java Producer 16 | https://aliyun.com/product/sls 17 | Aliyun LOG Java Producer 18 | 19 | 20 | The Apache License, Version 2.0 21 | http://www.apache.org/licenses/LICENSE-2.0.txt 22 | 23 | 24 | 25 | https://github.com/aliyun/aliyun-log-producer 26 | scm:git:git@github.com:aliyun/aliyun-log-java-producer.git 27 | scm:git:git@github.com:aliyun/aliyun-log-java-producer.git 28 | 29 | 30 | 31 | sonatype-nexus-snapshots 32 | https://s01.oss.sonatype.org/content/repositories/snapshots 33 | 34 | 35 | sonatype-nexus-staging 36 | https://s01.oss.sonatype.org/service/local/staging/deploy/maven2/ 37 | 38 | 39 | 40 | 41 | aliyunproducts 42 | Aliyun Log development team 43 | aliyunsdk@aliyun.com 44 | 45 | 46 | 47 | UTF-8 48 | 1.6 49 | 1.6 50 | 4.12 51 | 1.7.15 52 | 1.2.3 53 | 27.0-jre 54 | 0.6.122 55 | 56 | 57 | 58 | 59 | junit 60 | junit 61 | ${junit.version} 62 | test 63 | 64 | 65 | org.slf4j 66 | slf4j-api 67 | ${slf4j.version} 68 | 69 | 70 | ch.qos.logback 71 | logback-core 72 | ${logback.version} 73 | test 74 | 75 | 76 | ch.qos.logback 77 | logback-classic 78 | ${logback.version} 79 | test 80 | 81 | 82 | com.google.guava 83 | guava 84 | ${guava.version} 85 | 86 | 87 | com.aliyun.openservices 88 | aliyun-log 89 | ${aliyun-log.version} 90 | 91 | 92 | 93 | 94 | 95 | 96 | org.apache.maven.plugins 97 | maven-gpg-plugin 98 | 1.6 99 | 100 | 101 | sign-artifacts 102 | verify 103 | 104 | sign 105 | 106 | 107 | 108 | 109 | 110 | org.sonatype.plugins 111 | nexus-staging-maven-plugin 112 | 1.6.3 113 | true 114 | 115 | sonatype-nexus-staging 116 | https://s01.oss.sonatype.org/ 117 | true 118 | 119 | 120 | 121 | org.apache.maven.plugins 122 | maven-compiler-plugin 123 | 3.5.1 124 | 125 | ${maven.compiler.source} 126 | ${maven.compiler.target} 127 | ${project.build.sourceEncoding} 128 | 129 | 130 | 131 | org.apache.maven.plugins 132 | maven-source-plugin 133 | 3.0.0 134 | 135 | 136 | attach-sources 137 | 138 | jar 139 | 140 | 141 | 142 | 143 | 144 | org.apache.maven.plugins 145 | maven-javadoc-plugin 146 | 2.10.4 147 | 148 | ${project.build.sourceEncoding} 149 | ${project.build.sourceEncoding} 150 | 151 | 152 | author 153 | X 154 | 155 | 156 | 157 | 158 | 159 | attach-javadocs 160 | 161 | jar 162 | 163 | 164 | 165 | 166 | 167 | org.apache.maven.plugins 168 | maven-surefire-plugin 169 | 2.20 170 | 171 | 172 | **/ProducerLargeAmountTest.java 173 | 174 | 175 | 176 | 177 | com.coveo 178 | fmt-maven-plugin 179 | 2.6.0 180 | 181 | 182 | 183 | format 184 | 185 | 186 | 187 | 188 | 189 | org.apache.maven.plugins 190 | maven-shade-plugin 191 | 3.1.1 192 | 193 | 194 | package 195 | 196 | shade 197 | 198 | 199 | 200 | 201 | com.google 202 | com.shade.google 203 | 204 | 205 | com.aliyun.openservices.log 206 | com.shade.aliyun.openservices.log 207 | 208 | 209 | net.jpountz 210 | net.shade.jpountz 211 | 212 | 213 | com.alibaba 214 | com.shade.alibaba 215 | 216 | 217 | org.apache 218 | org.shade.apache 219 | 220 | 221 | org.codehaus 222 | org.shade.codehaus 223 | 224 | 225 | org.checkerframework 226 | org.shade.checkerframework 227 | 228 | 229 | jar-with-dependencies 230 | true 231 | 232 | 233 | 234 | 235 | 236 | 237 | 238 | 239 | -------------------------------------------------------------------------------- /src/main/java/com/aliyun/openservices/aliyun/log/producer/Attempt.java: -------------------------------------------------------------------------------- 1 | package com.aliyun.openservices.aliyun.log.producer; 2 | 3 | /** 4 | * Represents one attempt at writing a batch of logs to the backend. The attempt may or may not be 5 | * successful. If unsuccessful, an error code and error message are provided. Each batch may have 6 | * multiple attempts. 7 | * 8 | * @see Result 9 | */ 10 | public class Attempt { 11 | 12 | private final boolean success; 13 | 14 | private final String requestId; 15 | 16 | private final String errorCode; 17 | 18 | private final String errorMessage; 19 | 20 | private final long timestampMs; 21 | 22 | public Attempt( 23 | boolean success, String requestId, String errorCode, String errorMessage, long timestampMs) { 24 | this.success = success; 25 | this.requestId = requestId; 26 | this.errorCode = errorCode; 27 | this.errorMessage = errorMessage; 28 | this.timestampMs = timestampMs; 29 | } 30 | 31 | /** 32 | * @return Whether the attempt was successful. If true, then the batch has been confirmed by the 33 | * backend. 34 | */ 35 | public boolean isSuccess() { 36 | return success; 37 | } 38 | 39 | /** 40 | * @return Request id associated with this attempt. Empty string if the request did not reach the 41 | * backend. 42 | */ 43 | public String getRequestId() { 44 | return requestId; 45 | } 46 | 47 | /** 48 | * @return Error code associated with this attempt. Empty string if no error (i.e. successful). 49 | */ 50 | public String getErrorCode() { 51 | return errorCode; 52 | } 53 | 54 | /** 55 | * @return Error message associated with this attempt. Empty string if no error (i.e. successful). 56 | */ 57 | public String getErrorMessage() { 58 | return errorMessage; 59 | } 60 | 61 | /** @return The time when this attempt happened. */ 62 | public long getTimestampMs() { 63 | return timestampMs; 64 | } 65 | 66 | @Override 67 | public String toString() { 68 | return "Attempt{" 69 | + "success=" 70 | + success 71 | + ", requestId='" 72 | + requestId 73 | + '\'' 74 | + ", errorCode='" 75 | + errorCode 76 | + '\'' 77 | + ", errorMessage='" 78 | + errorMessage 79 | + '\'' 80 | + ", timestampMs=" 81 | + timestampMs 82 | + '}'; 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /src/main/java/com/aliyun/openservices/aliyun/log/producer/Callback.java: -------------------------------------------------------------------------------- 1 | package com.aliyun.openservices.aliyun.log.producer; 2 | 3 | /** 4 | * A callback interface that the user can implement to allow code to execute when the request is 5 | * complete. This callback will generally execute in the background batch handler thread and it 6 | * should be fast. 7 | */ 8 | public interface Callback { 9 | 10 | /** 11 | * A callback method the user can implement to provide asynchronous handling of request 12 | * completion. This method will be called when the log(s) sent to the server has been 13 | * acknowledged. 14 | * 15 | * @param result The result of a {@link LogProducer#send} operation. 16 | */ 17 | void onCompletion(Result result); 18 | } 19 | -------------------------------------------------------------------------------- /src/main/java/com/aliyun/openservices/aliyun/log/producer/Producer.java: -------------------------------------------------------------------------------- 1 | package com.aliyun.openservices.aliyun.log.producer; 2 | 3 | import com.aliyun.openservices.aliyun.log.producer.errors.ProducerException; 4 | import com.aliyun.openservices.log.common.LogItem; 5 | import com.google.common.util.concurrent.ListenableFuture; 6 | import java.util.List; 7 | 8 | /** 9 | * The interface for the {@link LogProducer} 10 | * 11 | * @see LogProducer 12 | */ 13 | public interface Producer { 14 | 15 | /** See {@link LogProducer#send(String, String, LogItem)} */ 16 | ListenableFuture send(String project, String logStore, LogItem logItem) 17 | throws InterruptedException, ProducerException; 18 | 19 | /** See {@link LogProducer#send(String, String, List)} */ 20 | ListenableFuture send(String project, String logStore, List logItems) 21 | throws InterruptedException, ProducerException; 22 | 23 | /** See {@link LogProducer#send(String, String, String, String, LogItem)} */ 24 | ListenableFuture send( 25 | String project, String logStore, String topic, String source, LogItem logItem) 26 | throws InterruptedException, ProducerException; 27 | 28 | /** See {@link LogProducer#send(String, String, String, String, List)} */ 29 | ListenableFuture send( 30 | String project, String logStore, String topic, String source, List logItems) 31 | throws InterruptedException, ProducerException; 32 | 33 | /** See {@link LogProducer#send(String, String, String, String, String, LogItem)} */ 34 | ListenableFuture send( 35 | String project, 36 | String logStore, 37 | String topic, 38 | String source, 39 | String shardHash, 40 | LogItem logItem) 41 | throws InterruptedException, ProducerException; 42 | 43 | /** See {@link LogProducer#send(String, String, String, String, String, List)} */ 44 | ListenableFuture send( 45 | String project, 46 | String logStore, 47 | String topic, 48 | String source, 49 | String shardHash, 50 | List logItems) 51 | throws InterruptedException, ProducerException; 52 | 53 | /** See {@link LogProducer#send(String, String, LogItem, Callback)} */ 54 | ListenableFuture send(String project, String logStore, LogItem logItem, Callback callback) 55 | throws InterruptedException, ProducerException; 56 | 57 | /** See {@link LogProducer#send(String, String, List, Callback)} */ 58 | ListenableFuture send( 59 | String project, String logStore, List logItems, Callback callback) 60 | throws InterruptedException, ProducerException; 61 | 62 | /** See {@link LogProducer#send(String, String, String, String, LogItem, Callback)} */ 63 | ListenableFuture send( 64 | String project, 65 | String logStore, 66 | String topic, 67 | String source, 68 | LogItem logItem, 69 | Callback callback) 70 | throws InterruptedException, ProducerException; 71 | 72 | /** See {@link LogProducer#send(String, String, String, String, List, Callback)} */ 73 | ListenableFuture send( 74 | String project, 75 | String logStore, 76 | String topic, 77 | String source, 78 | List logItems, 79 | Callback callback) 80 | throws InterruptedException, ProducerException; 81 | 82 | /** See {@link LogProducer#send(String, String, String, String, String, LogItem, Callback)} */ 83 | ListenableFuture send( 84 | String project, 85 | String logStore, 86 | String topic, 87 | String source, 88 | String shardHash, 89 | LogItem logItem, 90 | Callback callback) 91 | throws InterruptedException, ProducerException; 92 | 93 | /** See {@link LogProducer#send(String, String, String, String, String, List, Callback)} */ 94 | ListenableFuture send( 95 | String project, 96 | String logStore, 97 | String topic, 98 | String source, 99 | String shardHash, 100 | List logItems, 101 | Callback callback) 102 | throws InterruptedException, ProducerException; 103 | 104 | /** See {@link LogProducer#close()} */ 105 | void close() throws InterruptedException, ProducerException; 106 | 107 | /** See {@link LogProducer#close(long)} */ 108 | void close(long timeoutMs) throws InterruptedException, ProducerException; 109 | 110 | /** See {@link LogProducer#getProducerConfig()} */ 111 | ProducerConfig getProducerConfig(); 112 | 113 | /** See {@link LogProducer#getBatchCount()} */ 114 | int getBatchCount(); 115 | 116 | /** See {@link LogProducer#availableMemoryInBytes()} */ 117 | int availableMemoryInBytes(); 118 | 119 | /** See {@link LogProducer#putProjectConfig(ProjectConfig)} */ 120 | void putProjectConfig(ProjectConfig projectConfig); 121 | 122 | /** See {@link LogProducer#removeProjectConfig(ProjectConfig)} */ 123 | void removeProjectConfig(ProjectConfig projectConfig); 124 | } 125 | -------------------------------------------------------------------------------- /src/main/java/com/aliyun/openservices/aliyun/log/producer/ProducerConfig.java: -------------------------------------------------------------------------------- 1 | package com.aliyun.openservices.aliyun.log.producer; 2 | 3 | import com.aliyun.openservices.log.http.signer.SignVersion; 4 | 5 | /** 6 | * Configuration for {@link LogProducer}. See each individual set method for details about each 7 | * parameter. 8 | */ 9 | public class ProducerConfig { 10 | 11 | public static final int DEFAULT_TOTAL_SIZE_IN_BYTES = 100 * 1024 * 1024; 12 | 13 | public static final long DEFAULT_MAX_BLOCK_MS = 60 * 1000L; 14 | 15 | public static final int DEFAULT_IO_THREAD_COUNT = 16 | Math.max(Runtime.getRuntime().availableProcessors(), 1); 17 | 18 | public static final int DEFAULT_BATCH_SIZE_THRESHOLD_IN_BYTES = 512 * 1024; 19 | 20 | public static final int MAX_BATCH_SIZE_IN_BYTES = 8 * 1024 * 1024; 21 | 22 | public static final int DEFAULT_BATCH_COUNT_THRESHOLD = 4096; 23 | 24 | public static final int MAX_BATCH_COUNT = 40960; 25 | 26 | public static final int DEFAULT_LINGER_MS = 2000; 27 | 28 | public static final int LINGER_MS_LOWER_LIMIT = 100; 29 | 30 | public static final int DEFAULT_RETRIES = 10; 31 | 32 | public static final long DEFAULT_BASE_RETRY_BACKOFF_MS = 100L; 33 | 34 | public static final long DEFAULT_MAX_RETRY_BACKOFF_MS = 50 * 1000L; 35 | 36 | public static final int DEFAULT_BUCKETS = 64; 37 | 38 | public static final int BUCKETS_LOWER_LIMIT = 1; 39 | 40 | public static final int BUCKETS_UPPER_LIMIT = 512; 41 | 42 | public enum LogFormat { 43 | PROTOBUF, 44 | JSON 45 | } 46 | 47 | public static final LogFormat DEFAULT_LOG_FORMAT = LogFormat.PROTOBUF; 48 | 49 | private int totalSizeInBytes = DEFAULT_TOTAL_SIZE_IN_BYTES; 50 | 51 | private long maxBlockMs = DEFAULT_MAX_BLOCK_MS; 52 | 53 | private int ioThreadCount = DEFAULT_IO_THREAD_COUNT; 54 | 55 | private int batchSizeThresholdInBytes = DEFAULT_BATCH_SIZE_THRESHOLD_IN_BYTES; 56 | 57 | private int batchCountThreshold = DEFAULT_BATCH_COUNT_THRESHOLD; 58 | 59 | private int lingerMs = DEFAULT_LINGER_MS; 60 | 61 | private int retries = DEFAULT_RETRIES; 62 | 63 | private int maxReservedAttempts = DEFAULT_RETRIES + 1; 64 | 65 | private long baseRetryBackoffMs = DEFAULT_BASE_RETRY_BACKOFF_MS; 66 | 67 | private long maxRetryBackoffMs = DEFAULT_MAX_RETRY_BACKOFF_MS; 68 | 69 | private boolean adjustShardHash = true; 70 | 71 | private int buckets = DEFAULT_BUCKETS; 72 | 73 | private LogFormat logFormat = DEFAULT_LOG_FORMAT; 74 | 75 | private SignVersion signVersion = SignVersion.V1; 76 | 77 | private String region; 78 | 79 | private String sourceIp; 80 | private String compressType; 81 | private String processor; 82 | 83 | /** 84 | * @return The total bytes of memory the producer can use to buffer logs waiting to be sent to the 85 | * server. 86 | */ 87 | public int getTotalSizeInBytes() { 88 | return totalSizeInBytes; 89 | } 90 | 91 | /** 92 | * Set the total bytes of memory the producer can use to buffer logs waiting to be sent to the 93 | * server. 94 | */ 95 | public void setTotalSizeInBytes(int totalSizeInBytes) { 96 | if (totalSizeInBytes <= 0) { 97 | throw new IllegalArgumentException( 98 | "totalSizeInBytes must be greater than 0, got " + totalSizeInBytes); 99 | } 100 | this.totalSizeInBytes = totalSizeInBytes; 101 | } 102 | 103 | /** @return How long LogProducer.send() will block. */ 104 | public long getMaxBlockMs() { 105 | return maxBlockMs; 106 | } 107 | 108 | /** Set how long LogProducer.send() will block. */ 109 | public void setMaxBlockMs(long maxBlockMs) { 110 | this.maxBlockMs = maxBlockMs; 111 | } 112 | 113 | /** @return The thread count of the background I/O thread pool. */ 114 | public int getIoThreadCount() { 115 | return ioThreadCount; 116 | } 117 | 118 | /** Set the thread count of the background I/O thread pool. */ 119 | public void setIoThreadCount(int ioThreadCount) { 120 | if (ioThreadCount <= 0) { 121 | throw new IllegalArgumentException( 122 | "ioThreadCount must be greater than 0, got " + ioThreadCount); 123 | } 124 | this.ioThreadCount = ioThreadCount; 125 | } 126 | 127 | /** @return The batch size threshold. */ 128 | public int getBatchSizeThresholdInBytes() { 129 | return batchSizeThresholdInBytes; 130 | } 131 | 132 | /** Set the batch size threshold. */ 133 | public void setBatchSizeThresholdInBytes(int batchSizeThresholdInBytes) { 134 | if (batchSizeThresholdInBytes < 1 || batchSizeThresholdInBytes > MAX_BATCH_SIZE_IN_BYTES) { 135 | throw new IllegalArgumentException( 136 | String.format( 137 | "batchSizeThresholdInBytes must be between 1 and %d, got %d", 138 | MAX_BATCH_SIZE_IN_BYTES, batchSizeThresholdInBytes)); 139 | } 140 | this.batchSizeThresholdInBytes = batchSizeThresholdInBytes; 141 | } 142 | 143 | /** @return The batch count threshold. */ 144 | public int getBatchCountThreshold() { 145 | return batchCountThreshold; 146 | } 147 | 148 | /** Set the batch count threshold. */ 149 | public void setBatchCountThreshold(int batchCountThreshold) { 150 | if (batchCountThreshold < 1 || batchCountThreshold > MAX_BATCH_COUNT) { 151 | throw new IllegalArgumentException( 152 | String.format( 153 | "batchCountThreshold must be between 1 and %d, got %d", 154 | MAX_BATCH_COUNT, batchCountThreshold)); 155 | } 156 | this.batchCountThreshold = batchCountThreshold; 157 | } 158 | 159 | /** @return The max linger time of a log. */ 160 | public int getLingerMs() { 161 | return lingerMs; 162 | } 163 | 164 | /** Set the max linger time of a log. */ 165 | public void setLingerMs(int lingerMs) { 166 | if (lingerMs < LINGER_MS_LOWER_LIMIT) { 167 | throw new IllegalArgumentException( 168 | String.format( 169 | "lingerMs must be greater than or equal to %d, got %d", 170 | LINGER_MS_LOWER_LIMIT, lingerMs)); 171 | } 172 | this.lingerMs = lingerMs; 173 | } 174 | 175 | /** @return The retry times for transient error. */ 176 | public int getRetries() { 177 | return retries; 178 | } 179 | 180 | /** 181 | * Set the retry times for transient error. Setting a value greater than zero will cause the 182 | * client to resend any log whose send fails with a potentially transient error. 183 | */ 184 | public void setRetries(int retries) { 185 | this.retries = retries; 186 | } 187 | 188 | /** @return How many {@link Attempt}s will be reserved in a {@link Result}. */ 189 | public int getMaxReservedAttempts() { 190 | return maxReservedAttempts; 191 | } 192 | 193 | /** Set how many {@link Attempt}s will be reserved in a {@link Result}. */ 194 | public void setMaxReservedAttempts(int maxReservedAttempts) { 195 | if (maxReservedAttempts <= 0) { 196 | throw new IllegalArgumentException( 197 | "maxReservedAttempts must be greater than 0, got " + maxReservedAttempts); 198 | } 199 | this.maxReservedAttempts = maxReservedAttempts; 200 | } 201 | 202 | /** 203 | * @return The amount of time to wait before attempting to retry a failed request for the first 204 | * time. 205 | */ 206 | public long getBaseRetryBackoffMs() { 207 | return baseRetryBackoffMs; 208 | } 209 | 210 | /** 211 | * Set the amount of time to wait before attempting to retry a failed request for the first time. 212 | */ 213 | public void setBaseRetryBackoffMs(long baseRetryBackoffMs) { 214 | if (baseRetryBackoffMs <= 0) { 215 | throw new IllegalArgumentException( 216 | "baseRetryBackoffMs must be greater than 0, got " + baseRetryBackoffMs); 217 | } 218 | this.baseRetryBackoffMs = baseRetryBackoffMs; 219 | } 220 | 221 | /** @return The upper limit of time to wait before attempting to retry a failed request. */ 222 | public long getMaxRetryBackoffMs() { 223 | return maxRetryBackoffMs; 224 | } 225 | 226 | /** Set the upper limit of time to wait before attempting to retry a failed request. */ 227 | public void setMaxRetryBackoffMs(long maxRetryBackoffMs) { 228 | if (maxRetryBackoffMs <= 0) { 229 | throw new IllegalArgumentException( 230 | "maxRetryBackoffMs must be greater than 0, got " + maxRetryBackoffMs); 231 | } 232 | this.maxRetryBackoffMs = maxRetryBackoffMs; 233 | } 234 | 235 | /** @return The flag of whether to adjust shard hash. */ 236 | public boolean isAdjustShardHash() { 237 | return adjustShardHash; 238 | } 239 | 240 | /** Specify whether to adjust shard hash. */ 241 | public void setAdjustShardHash(boolean adjustShardHash) { 242 | this.adjustShardHash = adjustShardHash; 243 | } 244 | 245 | /** @return The buckets of the shard hash. */ 246 | public int getBuckets() { 247 | return buckets; 248 | } 249 | 250 | /** Set the buckets of the shard hash. */ 251 | public void setBuckets(int buckets) { 252 | if (buckets < BUCKETS_LOWER_LIMIT || buckets > BUCKETS_UPPER_LIMIT) { 253 | throw new IllegalArgumentException( 254 | String.format( 255 | "buckets must be between %d and %d, got %d", 256 | BUCKETS_LOWER_LIMIT, BUCKETS_UPPER_LIMIT, buckets)); 257 | } 258 | if (!ShardHashAdjuster.isPowerOfTwo(buckets)) { 259 | throw new IllegalArgumentException("buckets must be a power of 2, got " + buckets); 260 | } 261 | this.buckets = buckets; 262 | } 263 | 264 | /** @return The content type of the request. */ 265 | public LogFormat getLogFormat() { 266 | return logFormat; 267 | } 268 | 269 | /** Set the content type of the request. */ 270 | public void setLogFormat(LogFormat logFormat) { 271 | this.logFormat = logFormat; 272 | } 273 | 274 | public SignVersion getSignVersion() { 275 | return signVersion; 276 | } 277 | 278 | public void setSignVersion(SignVersion signVersion) { 279 | this.signVersion = signVersion; 280 | } 281 | 282 | public String getRegion() { 283 | return region; 284 | } 285 | 286 | public void setRegion(String region) { 287 | this.region = region; 288 | } 289 | public String getSourceIp() { 290 | return sourceIp; 291 | } 292 | /** Set the source ip of producer. */ 293 | public void setSourceIp(String sourceIp) { 294 | this.sourceIp = sourceIp; 295 | } 296 | 297 | public String getCompressType() { 298 | return compressType; 299 | } 300 | 301 | public void setCompressType(String compressType) { 302 | this.compressType = compressType; 303 | } 304 | 305 | public String getProcessor() { 306 | return processor; 307 | } 308 | 309 | public void setProcessor(String processor) { 310 | this.processor = processor; 311 | } 312 | } 313 | -------------------------------------------------------------------------------- /src/main/java/com/aliyun/openservices/aliyun/log/producer/ProjectConfig.java: -------------------------------------------------------------------------------- 1 | package com.aliyun.openservices.aliyun.log.producer; 2 | 3 | import com.aliyun.openservices.log.common.auth.Credentials; 4 | import com.aliyun.openservices.log.common.auth.CredentialsProvider; 5 | import com.aliyun.openservices.log.common.auth.DefaultCredentials; 6 | import com.aliyun.openservices.log.common.auth.StaticCredentialsProvider; 7 | import javax.annotation.Nullable; 8 | 9 | /** 10 | * Configuration for project. It contains the service entry information of the target project and 11 | * access credentials representing the identity of the caller. 12 | */ 13 | public class ProjectConfig { 14 | 15 | public static final String DEFAULT_USER_AGENT = "aliyun-log-java-producer"; 16 | 17 | private final String project; 18 | 19 | private final String endpoint; 20 | 21 | private String accessKeyId; 22 | 23 | private String accessKeySecret; 24 | 25 | private String stsToken; 26 | 27 | private final CredentialsProvider credentialsProvider; 28 | 29 | private final String userAgent; 30 | 31 | private String proxyIp; 32 | 33 | private boolean useMetricStoreUrl; 34 | 35 | /** 36 | * @param project name of log project 37 | * @param endpoint aliyun sls service endpoint 38 | * @param credentialsProvider interface which provides credentials 39 | * @param userAgent nullable, user agent, default to aliyun-log-java-producer 40 | */ 41 | public ProjectConfig( 42 | String project, 43 | String endpoint, 44 | CredentialsProvider credentialsProvider, 45 | @Nullable String userAgent) { 46 | if (project == null) { 47 | throw new NullPointerException("project cannot be null"); 48 | } 49 | if (endpoint == null) { 50 | throw new NullPointerException("endpoint cannot be null"); 51 | } 52 | if (credentialsProvider == null) { 53 | throw new NullPointerException("credentialsProvider cannot be null"); 54 | } 55 | this.project = project; 56 | this.endpoint = endpoint; 57 | this.credentialsProvider = credentialsProvider; 58 | this.userAgent = userAgent; 59 | } 60 | 61 | public ProjectConfig( 62 | String project, String endpoint, String accessKeyId, String accessKeySecret) { 63 | this(project, endpoint, accessKeyId, accessKeySecret, null, DEFAULT_USER_AGENT); 64 | } 65 | 66 | public ProjectConfig( 67 | String project, 68 | String endpoint, 69 | String accessKeyId, 70 | String accessKeySecret, 71 | @Nullable String stsToken) { 72 | this(project, endpoint, accessKeyId, accessKeySecret, stsToken, DEFAULT_USER_AGENT); 73 | } 74 | 75 | public ProjectConfig( 76 | String project, 77 | String endpoint, 78 | String accessKeyId, 79 | String accessKeySecret, 80 | @Nullable String stsToken, 81 | @Nullable String userAgent) { 82 | if (project == null) { 83 | throw new NullPointerException("project cannot be null"); 84 | } 85 | if (endpoint == null) { 86 | throw new NullPointerException("endpoint cannot be null"); 87 | } 88 | if (accessKeyId == null) { 89 | throw new NullPointerException("accessKeyId cannot be null"); 90 | } 91 | if (accessKeySecret == null) { 92 | throw new NullPointerException("accessKeySecret cannot be null"); 93 | } 94 | this.project = project; 95 | this.endpoint = endpoint; 96 | this.accessKeyId = accessKeyId; 97 | this.accessKeySecret = accessKeySecret; 98 | this.stsToken = stsToken; 99 | this.userAgent = userAgent; 100 | this.credentialsProvider = 101 | new StaticCredentialsProvider( 102 | new DefaultCredentials(accessKeyId, accessKeySecret, stsToken)); 103 | } 104 | 105 | public String getProject() { 106 | return project; 107 | } 108 | 109 | public String getEndpoint() { 110 | return endpoint; 111 | } 112 | 113 | /** use getCredentials instead */ 114 | @Deprecated 115 | public String getAccessKeyId() { 116 | return accessKeyId; 117 | } 118 | 119 | /** use getCredentials instead */ 120 | @Deprecated 121 | public String getAccessKeySecret() { 122 | return accessKeySecret; 123 | } 124 | 125 | /** use getCredentials instead */ 126 | @Deprecated 127 | public String getStsToken() { 128 | return stsToken; 129 | } 130 | 131 | public Credentials getCredentials() { 132 | return credentialsProvider.getCredentials(); 133 | } 134 | 135 | public CredentialsProvider getCredentialsProvider() { 136 | return credentialsProvider; 137 | } 138 | 139 | public String getUserAgent() { 140 | return userAgent; 141 | } 142 | 143 | public String getProxyIp() { 144 | return proxyIp; 145 | } 146 | 147 | public void setProxyIp(String proxyIp) { 148 | this.proxyIp = proxyIp; 149 | } 150 | 151 | public boolean isUseMetricStoreUrl() { 152 | return useMetricStoreUrl; 153 | } 154 | 155 | public void setUseMetricStoreUrl(final boolean useMetricStoreUrl) { 156 | this.useMetricStoreUrl = useMetricStoreUrl; 157 | } 158 | } 159 | -------------------------------------------------------------------------------- /src/main/java/com/aliyun/openservices/aliyun/log/producer/Result.java: -------------------------------------------------------------------------------- 1 | package com.aliyun.openservices.aliyun.log.producer; 2 | 3 | import com.google.common.collect.Iterables; 4 | import java.util.List; 5 | 6 | /** 7 | * The result of a {@link LogProducer#send} operation. A list of {@link Attempt}s is provided with 8 | * details about each attempt made. 9 | * 10 | * @see Attempt 11 | */ 12 | public class Result { 13 | 14 | private final boolean successful; 15 | 16 | private final List reservedAttempts; 17 | 18 | private final int attemptCount; 19 | 20 | public Result(boolean successful, List reservedAttempts, int attemptCount) { 21 | this.successful = successful; 22 | this.reservedAttempts = reservedAttempts; 23 | this.attemptCount = attemptCount; 24 | } 25 | 26 | /** 27 | * @return Whether the send operation was successful. If true, then the log(s) has been confirmed 28 | * by the backend. 29 | */ 30 | public boolean isSuccessful() { 31 | return successful; 32 | } 33 | 34 | /** 35 | * @return List of {@link Attempt}s, in the order they were made. If the attempts exceed {@link 36 | * ProducerConfig#getMaxReservedAttempts()}, the oldest one will be removed. 37 | */ 38 | public List getReservedAttempts() { 39 | return reservedAttempts; 40 | } 41 | 42 | /** @return Attempt count for the log(s) being sent. */ 43 | public int getAttemptCount() { 44 | return attemptCount; 45 | } 46 | 47 | /** @return Error code of the last attempt. Empty string if no error (i.e. successful). */ 48 | public String getErrorCode() { 49 | Attempt lastAttempt = Iterables.getLast(reservedAttempts); 50 | return lastAttempt.getErrorCode(); 51 | } 52 | 53 | /** @return Error message of the last attempt. Empty string if no error (i.e. successful). */ 54 | public String getErrorMessage() { 55 | Attempt lastAttempt = Iterables.getLast(reservedAttempts); 56 | return lastAttempt.getErrorMessage(); 57 | } 58 | 59 | @Override 60 | public String toString() { 61 | return "Result{" 62 | + "successful=" 63 | + successful 64 | + ", reservedAttempts=" 65 | + reservedAttempts 66 | + ", attemptCount=" 67 | + attemptCount 68 | + '}'; 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /src/main/java/com/aliyun/openservices/aliyun/log/producer/ShardHashAdjuster.java: -------------------------------------------------------------------------------- 1 | package com.aliyun.openservices.aliyun.log.producer; 2 | 3 | import com.google.common.base.Strings; 4 | import com.google.common.hash.HashCode; 5 | import com.google.common.hash.Hashing; 6 | import java.math.BigInteger; 7 | 8 | public class ShardHashAdjuster { 9 | 10 | private static final int HEX_LENGTH = 32; 11 | 12 | private static final int BINARY_LENGTH = 128; 13 | 14 | private int reservedBits; 15 | 16 | public ShardHashAdjuster(int buckets) { 17 | if (!isPowerOfTwo(buckets)) { 18 | throw new IllegalArgumentException("buckets must be a power of 2, got " + buckets); 19 | } 20 | reservedBits = Integer.bitCount(buckets - 1); 21 | } 22 | 23 | public String adjust(String shardHash) { 24 | HashCode hashCode = Hashing.md5().hashBytes(shardHash.getBytes()); 25 | String binary = 26 | Strings.padStart(new BigInteger(1, hashCode.asBytes()).toString(2), BINARY_LENGTH, '0'); 27 | String adjustedBinary = Strings.padEnd(binary.substring(0, reservedBits), BINARY_LENGTH, '0'); 28 | return Strings.padStart(new BigInteger(adjustedBinary, 2).toString(16), HEX_LENGTH, '0'); 29 | } 30 | 31 | public static boolean isPowerOfTwo(int number) { 32 | return number > 0 && ((number & (number - 1)) == 0); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/com/aliyun/openservices/aliyun/log/producer/errors/Errors.java: -------------------------------------------------------------------------------- 1 | package com.aliyun.openservices.aliyun.log.producer.errors; 2 | 3 | /** Retries for the errors below are useless. */ 4 | public class Errors { 5 | 6 | public static final String PROJECT_CONFIG_NOT_EXIST = "ProjectConfigNotExist"; 7 | 8 | public static final String PROJECT_NOT_EXIST = "ProjectNotExist"; 9 | 10 | public static final String SIGNATURE_NOT_MATCH = "SignatureNotMatch"; 11 | 12 | public static final String MISS_ACCESS_KEY_ID = "MissAccessKeyId"; 13 | 14 | public static final String REQUEST_TIME_TOO_SKEWED = "RequestTimeTooSkewed"; 15 | 16 | public static final String PRODUCER_EXCEPTION = "ProducerException"; 17 | } 18 | -------------------------------------------------------------------------------- /src/main/java/com/aliyun/openservices/aliyun/log/producer/errors/LogSizeTooLargeException.java: -------------------------------------------------------------------------------- 1 | package com.aliyun.openservices.aliyun.log.producer.errors; 2 | 3 | /** The log's size is larger than the maximum allowable size. */ 4 | public class LogSizeTooLargeException extends ProducerException { 5 | 6 | public LogSizeTooLargeException() { 7 | super(); 8 | } 9 | 10 | public LogSizeTooLargeException(String message, Throwable cause) { 11 | super(message, cause); 12 | } 13 | 14 | public LogSizeTooLargeException(String message) { 15 | super(message); 16 | } 17 | 18 | public LogSizeTooLargeException(Throwable cause) { 19 | super(cause); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/main/java/com/aliyun/openservices/aliyun/log/producer/errors/MaxBatchCountExceedException.java: -------------------------------------------------------------------------------- 1 | package com.aliyun.openservices.aliyun.log.producer.errors; 2 | 3 | /** The logs exceeds the maximum batch count. */ 4 | public class MaxBatchCountExceedException extends ProducerException { 5 | 6 | public MaxBatchCountExceedException() { 7 | super(); 8 | } 9 | 10 | public MaxBatchCountExceedException(String message, Throwable cause) { 11 | super(message, cause); 12 | } 13 | 14 | public MaxBatchCountExceedException(String message) { 15 | super(message); 16 | } 17 | 18 | public MaxBatchCountExceedException(Throwable cause) { 19 | super(cause); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/main/java/com/aliyun/openservices/aliyun/log/producer/errors/ProducerException.java: -------------------------------------------------------------------------------- 1 | package com.aliyun.openservices.aliyun.log.producer.errors; 2 | 3 | /** The base class of all other producer exceptions. */ 4 | public class ProducerException extends Exception { 5 | 6 | public ProducerException(String message, Throwable cause) { 7 | super(message, cause); 8 | } 9 | 10 | public ProducerException(String message) { 11 | super(message); 12 | } 13 | 14 | public ProducerException(Throwable cause) { 15 | super(cause); 16 | } 17 | 18 | public ProducerException() { 19 | super(); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/main/java/com/aliyun/openservices/aliyun/log/producer/errors/ResultFailedException.java: -------------------------------------------------------------------------------- 1 | package com.aliyun.openservices.aliyun.log.producer.errors; 2 | 3 | import com.aliyun.openservices.aliyun.log.producer.Attempt; 4 | import com.aliyun.openservices.aliyun.log.producer.Result; 5 | import java.util.List; 6 | 7 | public class ResultFailedException extends ProducerException { 8 | 9 | private final Result result; 10 | 11 | public ResultFailedException(Result result) { 12 | this.result = result; 13 | } 14 | 15 | public Result getResult() { 16 | return result; 17 | } 18 | 19 | public String getErrorCode() { 20 | return result.getErrorCode(); 21 | } 22 | 23 | public String getErrorMessage() { 24 | return result.getErrorMessage(); 25 | } 26 | 27 | public List getReservedAttempts() { 28 | return result.getReservedAttempts(); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/com/aliyun/openservices/aliyun/log/producer/errors/RetriableErrors.java: -------------------------------------------------------------------------------- 1 | package com.aliyun.openservices.aliyun.log.producer.errors; 2 | 3 | /** The errors below are transient exceptions that if retried may succeed. */ 4 | public class RetriableErrors { 5 | 6 | public static final String REQUEST_ERROR = "RequestError"; 7 | 8 | public static final String UNAUTHORIZED = "Unauthorized"; 9 | 10 | public static final String WRITE_QUOTA_EXCEED = "WriteQuotaExceed"; 11 | 12 | public static final String SHARD_WRITE_QUOTA_EXCEED = "ShardWriteQuotaExceed"; 13 | 14 | public static final String EXCEED_QUOTA = "ExceedQuota"; 15 | 16 | public static final String INTERNAL_SERVER_ERROR = "InternalServerError"; 17 | 18 | public static final String SERVER_BUSY = "ServerBusy"; 19 | 20 | public static final String BAD_RESPONSE = "BadResponse"; 21 | 22 | public static final String PROJECT_NOT_EXISTS = "ProjectNotExists"; 23 | 24 | public static final String LOGSTORE_NOT_EXISTS = "LogstoreNotExists"; 25 | 26 | public static final String SOCKET_TIMEOUT = "SocketTimeout"; 27 | 28 | public static final String SIGNATURE_NOT_MATCH = "SignatureNotMatch"; 29 | } 30 | -------------------------------------------------------------------------------- /src/main/java/com/aliyun/openservices/aliyun/log/producer/errors/TimeoutException.java: -------------------------------------------------------------------------------- 1 | package com.aliyun.openservices.aliyun.log.producer.errors; 2 | 3 | /** Indicates that a request timed out. */ 4 | public class TimeoutException extends ProducerException { 5 | 6 | public TimeoutException() { 7 | super(); 8 | } 9 | 10 | public TimeoutException(String message, Throwable cause) { 11 | super(message, cause); 12 | } 13 | 14 | public TimeoutException(String message) { 15 | super(message); 16 | } 17 | 18 | public TimeoutException(Throwable cause) { 19 | super(cause); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/main/java/com/aliyun/openservices/aliyun/log/producer/example/ExampleUsage.java: -------------------------------------------------------------------------------- 1 | package com.aliyun.openservices.aliyun.log.producer.example; 2 | 3 | import com.aliyun.openservices.aliyun.log.producer.*; 4 | import com.aliyun.openservices.aliyun.log.producer.errors.ProducerException; 5 | import com.aliyun.openservices.log.common.LogItem; 6 | import com.aliyun.openservices.log.common.auth.CredentialsProvider; 7 | import com.aliyun.openservices.log.common.auth.DefaultCredentials; 8 | import com.aliyun.openservices.log.common.auth.StaticCredentialsProvider; 9 | import com.google.common.util.concurrent.ListenableFuture; 10 | 11 | import java.util.ArrayList; 12 | import java.util.List; 13 | import java.util.concurrent.ExecutionException; 14 | 15 | public class ExampleUsage { 16 | 17 | public static void main(String[] args) throws ProducerException, InterruptedException, ExecutionException { 18 | String project = System.getenv("PROJECT"); 19 | String endpoint = System.getenv("ENDPOINT"); 20 | String logStore = System.getenv("LOG_STORE"); 21 | // init producer 22 | Producer producer = new LogProducer(new ProducerConfig()); 23 | ProjectConfig projectConfig = new ProjectConfig(project, endpoint, 24 | getCredentialsProvider(), 25 | null); 26 | producer.putProjectConfig(projectConfig); 27 | 28 | // send logs 29 | ListenableFuture f = 30 | producer.send(project, logStore, buildLogItem()); 31 | 32 | Result result = f.get(); 33 | System.out.println(result.isSuccessful()); 34 | System.out.println(result.getErrorMessage()); 35 | 36 | producer.send( 37 | project, logStore, null, null, buildLogItem()); 38 | producer.send(System.getenv("PROJECT"), System.getenv("LOG_STORE"), "", "", buildLogItem()); 39 | 40 | f = 41 | producer.send( 42 | project, logStore, 43 | "topic", 44 | "source", 45 | buildLogItem()); 46 | result = f.get(); 47 | System.out.println(result.isSuccessful()); 48 | System.out.println(result.getErrorMessage()); 49 | 50 | producer.close(); 51 | } 52 | 53 | // support customized credentials provider 54 | private static CredentialsProvider getCredentialsProvider() { 55 | String accessKeyId = System.getenv("ACCESS_KEY_ID"); 56 | String accessKeySecret = System.getenv("ACCESS_KEY_SECRET"); 57 | return new StaticCredentialsProvider(new DefaultCredentials(accessKeyId, accessKeySecret)); 58 | } 59 | 60 | public static LogItem buildLogItem() { 61 | LogItem logItem = new LogItem(); 62 | logItem.PushBack("k1", "v1"); 63 | logItem.PushBack("k2", "v2"); 64 | return logItem; 65 | } 66 | 67 | public static List buildLogItems(int n) { 68 | List logItems = new ArrayList(); 69 | for (int i = 0; i < n; ++i) { 70 | logItems.add(buildLogItem()); 71 | } 72 | return logItems; 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /src/main/java/com/aliyun/openservices/aliyun/log/producer/internals/BatchHandler.java: -------------------------------------------------------------------------------- 1 | package com.aliyun.openservices.aliyun.log.producer.internals; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | import java.util.concurrent.BlockingQueue; 6 | import java.util.concurrent.Semaphore; 7 | import java.util.concurrent.atomic.AtomicInteger; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | public class BatchHandler extends LogThread { 12 | 13 | private static final Logger LOGGER = LoggerFactory.getLogger(BatchHandler.class); 14 | 15 | private final BlockingQueue batches; 16 | 17 | private final AtomicInteger batchCount; 18 | 19 | private final Semaphore memoryController; 20 | 21 | private volatile boolean closed; 22 | 23 | public BatchHandler( 24 | String name, 25 | BlockingQueue batches, 26 | AtomicInteger batchCount, 27 | Semaphore memoryController) { 28 | super(name, true); 29 | this.batches = batches; 30 | this.batchCount = batchCount; 31 | this.memoryController = memoryController; 32 | this.closed = false; 33 | } 34 | 35 | @Override 36 | public void run() { 37 | loopHandleBatches(); 38 | handleRemainingBatches(); 39 | } 40 | 41 | private void loopHandleBatches() { 42 | while (!closed) { 43 | try { 44 | ProducerBatch b = batches.take(); 45 | handle(b); 46 | } catch (InterruptedException e) { 47 | LOGGER.info("The batch handler has been interrupted"); 48 | } 49 | } 50 | } 51 | 52 | private void handleRemainingBatches() { 53 | List remainingBatches = new ArrayList(); 54 | batches.drainTo(remainingBatches); 55 | for (ProducerBatch b : remainingBatches) { 56 | handle(b); 57 | } 58 | } 59 | 60 | private void handle(ProducerBatch batch) { 61 | try { 62 | batch.fireCallbacksAndSetFutures(); 63 | } catch (Throwable t) { 64 | LOGGER.error("Failed to handle batch, batch={}, e=", batch, t); 65 | } finally { 66 | batchCount.decrementAndGet(); 67 | memoryController.release(batch.getCurBatchSizeInBytes()); 68 | } 69 | } 70 | 71 | public boolean isClosed() { 72 | return closed; 73 | } 74 | 75 | public void close() { 76 | this.closed = true; 77 | interrupt(); 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /src/main/java/com/aliyun/openservices/aliyun/log/producer/internals/ExpiredBatches.java: -------------------------------------------------------------------------------- 1 | package com.aliyun.openservices.aliyun.log.producer.internals; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | 6 | public class ExpiredBatches { 7 | 8 | private final List batches = new ArrayList(); 9 | 10 | private long remainingMs; 11 | 12 | public List getBatches() { 13 | return batches; 14 | } 15 | 16 | public void add(ProducerBatch producerBatch) { 17 | if (!batches.add(producerBatch)) { 18 | throw new IllegalStateException("failed to add producer batch to expired batches"); 19 | } 20 | } 21 | 22 | public long getRemainingMs() { 23 | return remainingMs; 24 | } 25 | 26 | public void setRemainingMs(long remainingMs) { 27 | this.remainingMs = remainingMs; 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/main/java/com/aliyun/openservices/aliyun/log/producer/internals/GroupKey.java: -------------------------------------------------------------------------------- 1 | package com.aliyun.openservices.aliyun.log.producer.internals; 2 | 3 | public final class GroupKey { 4 | 5 | private static final String DELIMITER = "|"; 6 | 7 | private final String key; 8 | 9 | private final String project; 10 | 11 | private final String logStore; 12 | 13 | private final String topic; 14 | 15 | private final String source; 16 | 17 | private final String shardHash; 18 | 19 | public GroupKey(String project, String logStore, String topic, String source, String shardHash) { 20 | this.project = project; 21 | this.logStore = logStore; 22 | this.topic = topic; 23 | this.source = source; 24 | this.shardHash = shardHash; 25 | this.key = 26 | project + DELIMITER + logStore + DELIMITER + topic + DELIMITER + source + DELIMITER 27 | + shardHash; 28 | } 29 | 30 | @Override 31 | public boolean equals(Object o) { 32 | if (this == o) { 33 | return true; 34 | } 35 | if (o == null || getClass() != o.getClass()) { 36 | return false; 37 | } 38 | 39 | GroupKey groupKey = (GroupKey) o; 40 | 41 | return key.equals(groupKey.key); 42 | } 43 | 44 | @Override 45 | public int hashCode() { 46 | return key.hashCode(); 47 | } 48 | 49 | @Override 50 | public String toString() { 51 | return key; 52 | } 53 | 54 | public String getKey() { 55 | return key; 56 | } 57 | 58 | public String getProject() { 59 | return project; 60 | } 61 | 62 | public String getLogStore() { 63 | return logStore; 64 | } 65 | 66 | public String getTopic() { 67 | return topic; 68 | } 69 | 70 | public String getSource() { 71 | return source; 72 | } 73 | 74 | public String getShardHash() { 75 | return shardHash; 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /src/main/java/com/aliyun/openservices/aliyun/log/producer/internals/IOThreadPool.java: -------------------------------------------------------------------------------- 1 | package com.aliyun.openservices.aliyun.log.producer.internals; 2 | 3 | import com.google.common.util.concurrent.ThreadFactoryBuilder; 4 | import java.util.concurrent.*; 5 | 6 | public class IOThreadPool { 7 | 8 | private static final String IO_THREAD_SUFFIX_FORMAT = "-io-thread-%d"; 9 | 10 | private final ExecutorService ioThreadPool; 11 | 12 | public IOThreadPool(int ioThreadCount, String prefix) { 13 | this.ioThreadPool = 14 | Executors.newFixedThreadPool( 15 | ioThreadCount, 16 | new ThreadFactoryBuilder() 17 | .setDaemon(true) 18 | .setNameFormat(prefix + IO_THREAD_SUFFIX_FORMAT) 19 | .build()); 20 | } 21 | 22 | public void submit(SendProducerBatchTask task) { 23 | ioThreadPool.submit(task); 24 | } 25 | 26 | public void shutdown() { 27 | ioThreadPool.shutdown(); 28 | } 29 | 30 | public boolean isTerminated() { 31 | return ioThreadPool.isTerminated(); 32 | } 33 | 34 | public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException { 35 | return ioThreadPool.awaitTermination(timeout, unit); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/com/aliyun/openservices/aliyun/log/producer/internals/LogAccumulator.java: -------------------------------------------------------------------------------- 1 | package com.aliyun.openservices.aliyun.log.producer.internals; 2 | 3 | import com.aliyun.openservices.aliyun.log.producer.Callback; 4 | import com.aliyun.openservices.aliyun.log.producer.ProducerConfig; 5 | import com.aliyun.openservices.aliyun.log.producer.Result; 6 | import com.aliyun.openservices.aliyun.log.producer.errors.LogSizeTooLargeException; 7 | import com.aliyun.openservices.aliyun.log.producer.errors.ProducerException; 8 | import com.aliyun.openservices.aliyun.log.producer.errors.TimeoutException; 9 | import com.aliyun.openservices.log.Client; 10 | import com.aliyun.openservices.log.common.LogItem; 11 | import com.google.common.util.concurrent.ListenableFuture; 12 | import java.util.*; 13 | import java.util.concurrent.*; 14 | import java.util.concurrent.atomic.AtomicInteger; 15 | import java.util.concurrent.atomic.AtomicLong; 16 | import org.slf4j.Logger; 17 | import org.slf4j.LoggerFactory; 18 | 19 | public final class LogAccumulator { 20 | 21 | private static final Logger LOGGER = LoggerFactory.getLogger(LogAccumulator.class); 22 | 23 | private static final AtomicLong BATCH_ID = new AtomicLong(0); 24 | 25 | private final String producerHash; 26 | 27 | private final ProducerConfig producerConfig; 28 | 29 | private final Map clientPool; 30 | 31 | private final Semaphore memoryController; 32 | 33 | private final RetryQueue retryQueue; 34 | 35 | private final BlockingQueue successQueue; 36 | 37 | private final BlockingQueue failureQueue; 38 | 39 | private final IOThreadPool ioThreadPool; 40 | 41 | private final AtomicInteger batchCount; 42 | 43 | private final ConcurrentMap batches; 44 | 45 | private final AtomicInteger appendsInProgress; 46 | 47 | private volatile boolean closed; 48 | 49 | public LogAccumulator( 50 | String producerHash, 51 | ProducerConfig producerConfig, 52 | Map clientPool, 53 | Semaphore memoryController, 54 | RetryQueue retryQueue, 55 | BlockingQueue successQueue, 56 | BlockingQueue failureQueue, 57 | IOThreadPool ioThreadPool, 58 | AtomicInteger batchCount) { 59 | this.producerHash = producerHash; 60 | this.producerConfig = producerConfig; 61 | this.clientPool = clientPool; 62 | this.memoryController = memoryController; 63 | this.retryQueue = retryQueue; 64 | this.successQueue = successQueue; 65 | this.failureQueue = failureQueue; 66 | this.ioThreadPool = ioThreadPool; 67 | this.batchCount = batchCount; 68 | this.batches = new ConcurrentHashMap(); 69 | this.appendsInProgress = new AtomicInteger(0); 70 | this.closed = false; 71 | } 72 | 73 | public ListenableFuture append( 74 | String project, 75 | String logStore, 76 | String topic, 77 | String source, 78 | String shardHash, 79 | List logItems, 80 | Callback callback) 81 | throws InterruptedException, ProducerException { 82 | appendsInProgress.incrementAndGet(); 83 | try { 84 | return doAppend(project, logStore, topic, source, shardHash, logItems, callback); 85 | } finally { 86 | appendsInProgress.decrementAndGet(); 87 | } 88 | } 89 | 90 | private ListenableFuture doAppend( 91 | String project, 92 | String logStore, 93 | String topic, 94 | String source, 95 | String shardHash, 96 | List logItems, 97 | Callback callback) 98 | throws InterruptedException, ProducerException { 99 | if (closed) { 100 | throw new IllegalStateException("cannot append after the log accumulator was closed"); 101 | } 102 | int sizeInBytes = LogSizeCalculator.calculate(logItems); 103 | ensureValidLogSize(sizeInBytes); 104 | long maxBlockMs = producerConfig.getMaxBlockMs(); 105 | LOGGER.trace( 106 | "Prepare to acquire bytes, sizeInBytes={}, maxBlockMs={}, project={}, logStore={}", 107 | sizeInBytes, 108 | maxBlockMs, 109 | project, 110 | logStore); 111 | if (maxBlockMs >= 0) { 112 | boolean acquired = 113 | memoryController.tryAcquire(sizeInBytes, maxBlockMs, TimeUnit.MILLISECONDS); 114 | if (!acquired) { 115 | LOGGER.warn( 116 | "Failed to acquire memory within the configured max blocking time {} ms, " 117 | + "requiredSizeInBytes={}, availableSizeInBytes={}", 118 | producerConfig.getMaxBlockMs(), 119 | sizeInBytes, 120 | memoryController.availablePermits()); 121 | throw new TimeoutException( 122 | "failed to acquire memory within the configured max blocking time " 123 | + producerConfig.getMaxBlockMs() 124 | + " ms"); 125 | } 126 | } else { 127 | memoryController.acquire(sizeInBytes); 128 | } 129 | try { 130 | GroupKey groupKey = new GroupKey(project, logStore, topic, source, shardHash); 131 | ProducerBatchHolder holder = getOrCreateProducerBatchHolder(groupKey); 132 | synchronized (holder) { 133 | return appendToHolder(groupKey, logItems, callback, sizeInBytes, holder); 134 | } 135 | } catch (Exception e) { 136 | memoryController.release(sizeInBytes); 137 | throw new ProducerException(e); 138 | } 139 | } 140 | 141 | private ListenableFuture appendToHolder( 142 | GroupKey groupKey, 143 | List logItems, 144 | Callback callback, 145 | int sizeInBytes, 146 | ProducerBatchHolder holder) { 147 | if (holder.producerBatch != null) { 148 | ListenableFuture f = holder.producerBatch.tryAppend(logItems, sizeInBytes, callback); 149 | if (f != null) { 150 | if (holder.producerBatch.isMeetSendCondition()) { 151 | holder.transferProducerBatch( 152 | ioThreadPool, 153 | producerConfig, 154 | clientPool, 155 | retryQueue, 156 | successQueue, 157 | failureQueue, 158 | batchCount); 159 | } 160 | return f; 161 | } else { 162 | holder.transferProducerBatch( 163 | ioThreadPool, 164 | producerConfig, 165 | clientPool, 166 | retryQueue, 167 | successQueue, 168 | failureQueue, 169 | batchCount); 170 | } 171 | } 172 | holder.producerBatch = 173 | new ProducerBatch( 174 | groupKey, 175 | Utils.generatePackageId(producerHash, BATCH_ID), 176 | producerConfig.getBatchSizeThresholdInBytes(), 177 | producerConfig.getBatchCountThreshold(), 178 | producerConfig.getMaxReservedAttempts(), 179 | System.currentTimeMillis()); 180 | ListenableFuture f = holder.producerBatch.tryAppend(logItems, sizeInBytes, callback); 181 | batchCount.incrementAndGet(); 182 | if (holder.producerBatch.isMeetSendCondition()) { 183 | holder.transferProducerBatch( 184 | ioThreadPool, 185 | producerConfig, 186 | clientPool, 187 | retryQueue, 188 | successQueue, 189 | failureQueue, 190 | batchCount); 191 | } 192 | return f; 193 | } 194 | 195 | public ExpiredBatches expiredBatches() { 196 | long nowMs = System.currentTimeMillis(); 197 | ExpiredBatches expiredBatches = new ExpiredBatches(); 198 | long remainingMs = producerConfig.getLingerMs(); 199 | for (Map.Entry entry : batches.entrySet()) { 200 | ProducerBatchHolder holder = entry.getValue(); 201 | synchronized (holder) { 202 | if (holder.producerBatch == null) { 203 | continue; 204 | } 205 | long curRemainingMs = holder.producerBatch.remainingMs(nowMs, producerConfig.getLingerMs()); 206 | if (curRemainingMs <= 0) { 207 | holder.transferProducerBatch(expiredBatches); 208 | } else { 209 | remainingMs = Math.min(remainingMs, curRemainingMs); 210 | } 211 | } 212 | } 213 | expiredBatches.setRemainingMs(remainingMs); 214 | return expiredBatches; 215 | } 216 | 217 | public List remainingBatches() { 218 | if (!closed) { 219 | throw new IllegalStateException( 220 | "cannot get the remaining batches before the log accumulator closed"); 221 | } 222 | List remainingBatches = new ArrayList(); 223 | while (appendsInProgress()) { 224 | drainTo(remainingBatches); 225 | } 226 | drainTo(remainingBatches); 227 | batches.clear(); 228 | return remainingBatches; 229 | } 230 | 231 | private int drainTo(List c) { 232 | int n = 0; 233 | for (Map.Entry entry : batches.entrySet()) { 234 | ProducerBatchHolder holder = entry.getValue(); 235 | synchronized (holder) { 236 | if (holder.producerBatch == null) { 237 | continue; 238 | } 239 | c.add(holder.producerBatch); 240 | ++n; 241 | holder.producerBatch = null; 242 | } 243 | } 244 | return n; 245 | } 246 | 247 | private void ensureValidLogSize(int sizeInBytes) throws LogSizeTooLargeException { 248 | if (sizeInBytes > ProducerConfig.MAX_BATCH_SIZE_IN_BYTES) { 249 | throw new LogSizeTooLargeException( 250 | "the logs is " 251 | + sizeInBytes 252 | + " bytes which is larger than MAX_BATCH_SIZE_IN_BYTES " 253 | + ProducerConfig.MAX_BATCH_SIZE_IN_BYTES); 254 | } 255 | if (sizeInBytes > producerConfig.getTotalSizeInBytes()) { 256 | throw new LogSizeTooLargeException( 257 | "the logs is " 258 | + sizeInBytes 259 | + " bytes which is larger than the totalSizeInBytes you specified"); 260 | } 261 | } 262 | 263 | private ProducerBatchHolder getOrCreateProducerBatchHolder(GroupKey groupKey) { 264 | ProducerBatchHolder holder = batches.get(groupKey); 265 | if (holder != null) { 266 | return holder; 267 | } 268 | holder = new ProducerBatchHolder(); 269 | ProducerBatchHolder previous = batches.putIfAbsent(groupKey, holder); 270 | if (previous == null) { 271 | return holder; 272 | } else { 273 | return previous; 274 | } 275 | } 276 | 277 | public boolean isClosed() { 278 | return closed; 279 | } 280 | 281 | public void close() { 282 | this.closed = true; 283 | } 284 | 285 | private boolean appendsInProgress() { 286 | return appendsInProgress.get() > 0; 287 | } 288 | 289 | private static final class ProducerBatchHolder { 290 | 291 | ProducerBatch producerBatch; 292 | 293 | void transferProducerBatch( 294 | IOThreadPool ioThreadPool, 295 | ProducerConfig producerConfig, 296 | Map clientPool, 297 | RetryQueue retryQueue, 298 | BlockingQueue successQueue, 299 | BlockingQueue failureQueue, 300 | AtomicInteger batchCount) { 301 | if (producerBatch == null) { 302 | return; 303 | } 304 | ioThreadPool.submit( 305 | new SendProducerBatchTask( 306 | producerBatch, 307 | producerConfig, 308 | clientPool, 309 | retryQueue, 310 | successQueue, 311 | failureQueue, 312 | batchCount)); 313 | producerBatch = null; 314 | } 315 | 316 | void transferProducerBatch(ExpiredBatches expiredBatches) { 317 | if (producerBatch == null) { 318 | return; 319 | } 320 | expiredBatches.add(producerBatch); 321 | producerBatch = null; 322 | } 323 | } 324 | } 325 | -------------------------------------------------------------------------------- /src/main/java/com/aliyun/openservices/aliyun/log/producer/internals/LogSizeCalculator.java: -------------------------------------------------------------------------------- 1 | package com.aliyun.openservices.aliyun.log.producer.internals; 2 | 3 | import com.aliyun.openservices.log.common.LogContent; 4 | import com.aliyun.openservices.log.common.LogItem; 5 | import java.util.List; 6 | 7 | public abstract class LogSizeCalculator { 8 | 9 | public static int calculate(LogItem logItem) { 10 | int sizeInBytes = 8; 11 | for (LogContent content : logItem.GetLogContents()) { 12 | sizeInBytes += 8; 13 | if (content.mKey != null) { 14 | sizeInBytes += content.mKey.length(); 15 | } 16 | if (content.mValue != null) { 17 | sizeInBytes += content.mValue.length(); 18 | } 19 | } 20 | return sizeInBytes; 21 | } 22 | 23 | public static int calculate(List logItems) { 24 | int sizeInBytes = 0; 25 | for (LogItem logItem : logItems) { 26 | sizeInBytes += calculate(logItem); 27 | } 28 | return sizeInBytes; 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/com/aliyun/openservices/aliyun/log/producer/internals/LogThread.java: -------------------------------------------------------------------------------- 1 | package com.aliyun.openservices.aliyun.log.producer.internals; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | 6 | public class LogThread extends Thread { 7 | 8 | private final Logger LOGGER = LoggerFactory.getLogger(getClass()); 9 | 10 | public static LogThread daemon(final String name, Runnable runnable) { 11 | return new LogThread(name, runnable, true); 12 | } 13 | 14 | public static LogThread nonDaemon(final String name, Runnable runnable) { 15 | return new LogThread(name, runnable, false); 16 | } 17 | 18 | public LogThread(final String name, boolean daemon) { 19 | super(name); 20 | configureThread(name, daemon); 21 | } 22 | 23 | public LogThread(final String name, Runnable runnable, boolean daemon) { 24 | super(runnable, name); 25 | configureThread(name, daemon); 26 | } 27 | 28 | private void configureThread(final String name, boolean daemon) { 29 | setDaemon(daemon); 30 | setUncaughtExceptionHandler( 31 | new UncaughtExceptionHandler() { 32 | public void uncaughtException(Thread t, Throwable e) { 33 | LOGGER.error("Uncaught error in thread, name={}, e=", name, e); 34 | } 35 | }); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/com/aliyun/openservices/aliyun/log/producer/internals/Mover.java: -------------------------------------------------------------------------------- 1 | package com.aliyun.openservices.aliyun.log.producer.internals; 2 | 3 | import com.aliyun.openservices.aliyun.log.producer.*; 4 | import com.aliyun.openservices.log.Client; 5 | import java.util.List; 6 | import java.util.Map; 7 | import java.util.concurrent.BlockingQueue; 8 | import java.util.concurrent.atomic.AtomicInteger; 9 | import org.slf4j.Logger; 10 | import org.slf4j.LoggerFactory; 11 | 12 | public class Mover extends LogThread { 13 | 14 | private static final Logger LOGGER = LoggerFactory.getLogger(ProducerBatch.class); 15 | 16 | private final ProducerConfig producerConfig; 17 | 18 | private final Map clientPool; 19 | 20 | private final LogAccumulator accumulator; 21 | 22 | private final RetryQueue retryQueue; 23 | 24 | private final BlockingQueue successQueue; 25 | 26 | private final BlockingQueue failureQueue; 27 | 28 | private final IOThreadPool ioThreadPool; 29 | 30 | private final AtomicInteger batchCount; 31 | 32 | private volatile boolean closed; 33 | 34 | public Mover( 35 | String name, 36 | ProducerConfig producerConfig, 37 | Map clientPool, 38 | LogAccumulator accumulator, 39 | RetryQueue retryQueue, 40 | BlockingQueue successQueue, 41 | BlockingQueue failureQueue, 42 | IOThreadPool ioThreadPool, 43 | AtomicInteger batchCount) { 44 | super(name, true); 45 | this.producerConfig = producerConfig; 46 | this.clientPool = clientPool; 47 | this.accumulator = accumulator; 48 | this.retryQueue = retryQueue; 49 | this.successQueue = successQueue; 50 | this.failureQueue = failureQueue; 51 | this.ioThreadPool = ioThreadPool; 52 | this.batchCount = batchCount; 53 | this.closed = false; 54 | } 55 | 56 | @Override 57 | public void run() { 58 | loopMoveBatches(); 59 | LOGGER.debug("Beginning shutdown of mover thread"); 60 | List incompleteBatches = incompleteBatches(); 61 | LOGGER.debug("Submit incomplete batches, size={}", incompleteBatches.size()); 62 | submitIncompleteBatches(incompleteBatches); 63 | LOGGER.debug("Shutdown of mover thread has completed"); 64 | } 65 | 66 | private void loopMoveBatches() { 67 | while (!closed) { 68 | try { 69 | moveBatches(); 70 | } catch (Exception e) { 71 | LOGGER.error("Uncaught exception in mover, e=", e); 72 | } 73 | } 74 | } 75 | 76 | private void moveBatches() { 77 | LOGGER.debug( 78 | "Prepare to move expired batches from accumulator and retry queue to ioThreadPool"); 79 | doMoveBatches(); 80 | LOGGER.debug("Move expired batches successfully"); 81 | } 82 | 83 | private void doMoveBatches() { 84 | ExpiredBatches expiredBatches = accumulator.expiredBatches(); 85 | LOGGER.debug( 86 | "Expired batches from accumulator, size={}, remainingMs={}", 87 | expiredBatches.getBatches().size(), 88 | expiredBatches.getRemainingMs()); 89 | for (ProducerBatch b : expiredBatches.getBatches()) { 90 | ioThreadPool.submit(createSendProducerBatchTask(b)); 91 | } 92 | List expiredRetryBatches = 93 | retryQueue.expiredBatches(expiredBatches.getRemainingMs()); 94 | LOGGER.debug("Expired batches from retry queue, size={}", expiredRetryBatches.size()); 95 | for (ProducerBatch b : expiredRetryBatches) { 96 | ioThreadPool.submit(createSendProducerBatchTask(b)); 97 | } 98 | } 99 | 100 | private List incompleteBatches() { 101 | List incompleteBatches = accumulator.remainingBatches(); 102 | incompleteBatches.addAll(retryQueue.remainingBatches()); 103 | return incompleteBatches; 104 | } 105 | 106 | private void submitIncompleteBatches(List incompleteBatches) { 107 | for (ProducerBatch b : incompleteBatches) { 108 | ioThreadPool.submit(createSendProducerBatchTask(b)); 109 | } 110 | } 111 | 112 | private SendProducerBatchTask createSendProducerBatchTask(ProducerBatch batch) { 113 | return new SendProducerBatchTask( 114 | batch, producerConfig, clientPool, retryQueue, successQueue, failureQueue, batchCount); 115 | } 116 | 117 | public void close() { 118 | this.closed = true; 119 | interrupt(); 120 | } 121 | } 122 | -------------------------------------------------------------------------------- /src/main/java/com/aliyun/openservices/aliyun/log/producer/internals/ProducerBatch.java: -------------------------------------------------------------------------------- 1 | package com.aliyun.openservices.aliyun.log.producer.internals; 2 | 3 | import com.aliyun.openservices.aliyun.log.producer.*; 4 | import com.aliyun.openservices.aliyun.log.producer.errors.ResultFailedException; 5 | import com.aliyun.openservices.log.common.LogItem; 6 | import com.google.common.collect.EvictingQueue; 7 | import com.google.common.collect.Iterables; 8 | import com.google.common.util.concurrent.ListenableFuture; 9 | import com.google.common.util.concurrent.SettableFuture; 10 | import java.util.ArrayList; 11 | import java.util.List; 12 | import java.util.concurrent.Delayed; 13 | import java.util.concurrent.TimeUnit; 14 | import javax.annotation.Nonnull; 15 | import org.slf4j.Logger; 16 | import org.slf4j.LoggerFactory; 17 | 18 | public class ProducerBatch implements Delayed { 19 | 20 | private static final Logger LOGGER = LoggerFactory.getLogger(ProducerBatch.class); 21 | 22 | private final GroupKey groupKey; 23 | 24 | private final String packageId; 25 | 26 | private final int batchSizeThresholdInBytes; 27 | 28 | private final int batchCountThreshold; 29 | 30 | private final List logItems = new ArrayList(); 31 | 32 | private final List thunks = new ArrayList(); 33 | 34 | private final long createdMs; 35 | 36 | private long nextRetryMs; 37 | 38 | private int curBatchSizeInBytes; 39 | 40 | private int curBatchCount; 41 | 42 | private final EvictingQueue reservedAttempts; 43 | 44 | private int attemptCount; 45 | 46 | public ProducerBatch( 47 | GroupKey groupKey, 48 | String packageId, 49 | int batchSizeThresholdInBytes, 50 | int batchCountThreshold, 51 | int maxReservedAttempts, 52 | long nowMs) { 53 | this.groupKey = groupKey; 54 | this.packageId = packageId; 55 | this.createdMs = nowMs; 56 | this.batchSizeThresholdInBytes = batchSizeThresholdInBytes; 57 | this.batchCountThreshold = batchCountThreshold; 58 | this.curBatchCount = 0; 59 | this.curBatchSizeInBytes = 0; 60 | this.reservedAttempts = EvictingQueue.create(maxReservedAttempts); 61 | this.attemptCount = 0; 62 | } 63 | 64 | public ListenableFuture tryAppend(LogItem item, int sizeInBytes, Callback callback) { 65 | if (!hasRoomFor(sizeInBytes, 1)) { 66 | return null; 67 | } else { 68 | SettableFuture future = SettableFuture.create(); 69 | logItems.add(item); 70 | thunks.add(new Thunk(callback, future)); 71 | curBatchCount++; 72 | curBatchSizeInBytes += sizeInBytes; 73 | return future; 74 | } 75 | } 76 | 77 | public ListenableFuture tryAppend( 78 | List items, int sizeInBytes, Callback callback) { 79 | if (!hasRoomFor(sizeInBytes, items.size())) { 80 | return null; 81 | } else { 82 | SettableFuture future = SettableFuture.create(); 83 | logItems.addAll(items); 84 | thunks.add(new Thunk(callback, future)); 85 | curBatchCount += items.size(); 86 | curBatchSizeInBytes += sizeInBytes; 87 | return future; 88 | } 89 | } 90 | 91 | public void appendAttempt(Attempt attempt) { 92 | reservedAttempts.add(attempt); 93 | this.attemptCount++; 94 | } 95 | 96 | public boolean isMeetSendCondition() { 97 | return curBatchSizeInBytes >= batchSizeThresholdInBytes || curBatchCount >= batchCountThreshold; 98 | } 99 | 100 | public long remainingMs(long nowMs, long lingerMs) { 101 | return lingerMs - createdTimeMs(nowMs); 102 | } 103 | 104 | public void fireCallbacksAndSetFutures() { 105 | List attempts = new ArrayList(reservedAttempts); 106 | Attempt attempt = Iterables.getLast(attempts); 107 | Result result = new Result(attempt.isSuccess(), attempts, attemptCount); 108 | fireCallbacks(result); 109 | setFutures(result); 110 | } 111 | 112 | public GroupKey getGroupKey() { 113 | return groupKey; 114 | } 115 | 116 | public String getPackageId() { 117 | return packageId; 118 | } 119 | 120 | public List getLogItems() { 121 | return logItems; 122 | } 123 | 124 | public long getNextRetryMs() { 125 | return nextRetryMs; 126 | } 127 | 128 | public void setNextRetryMs(long nextRetryMs) { 129 | this.nextRetryMs = nextRetryMs; 130 | } 131 | 132 | public String getProject() { 133 | return groupKey.getProject(); 134 | } 135 | 136 | public String getLogStore() { 137 | return groupKey.getLogStore(); 138 | } 139 | 140 | public String getTopic() { 141 | return groupKey.getTopic(); 142 | } 143 | 144 | public String getSource() { 145 | return groupKey.getSource(); 146 | } 147 | 148 | public String getShardHash() { 149 | return groupKey.getShardHash(); 150 | } 151 | 152 | public int getCurBatchSizeInBytes() { 153 | return curBatchSizeInBytes; 154 | } 155 | 156 | public int getRetries() { 157 | return Math.max(0, attemptCount - 1); 158 | } 159 | 160 | private boolean hasRoomFor(int sizeInBytes, int count) { 161 | return curBatchSizeInBytes + sizeInBytes <= ProducerConfig.MAX_BATCH_SIZE_IN_BYTES 162 | && curBatchCount + count <= ProducerConfig.MAX_BATCH_COUNT; 163 | } 164 | 165 | private long createdTimeMs(long nowMs) { 166 | return Math.max(0, nowMs - createdMs); 167 | } 168 | 169 | private void fireCallbacks(Result result) { 170 | for (Thunk thunk : thunks) { 171 | try { 172 | if (thunk.callback != null) { 173 | thunk.callback.onCompletion(result); 174 | } 175 | } catch (Exception e) { 176 | LOGGER.error("Failed to execute user-provided callback, groupKey={}, e=", groupKey, e); 177 | } 178 | } 179 | } 180 | 181 | private void setFutures(Result result) { 182 | for (Thunk thunk : thunks) { 183 | try { 184 | if (result.isSuccessful()) { 185 | thunk.future.set(result); 186 | } else { 187 | thunk.future.setException(new ResultFailedException(result)); 188 | } 189 | } catch (Exception e) { 190 | LOGGER.error("Failed to set future, groupKey={}, e=", groupKey, e); 191 | } 192 | } 193 | } 194 | 195 | @Override 196 | public long getDelay(@Nonnull TimeUnit unit) { 197 | return unit.convert(nextRetryMs - System.currentTimeMillis(), TimeUnit.MILLISECONDS); 198 | } 199 | 200 | @Override 201 | public int compareTo(@Nonnull Delayed o) { 202 | return (int) (nextRetryMs - ((ProducerBatch) o).getNextRetryMs()); 203 | } 204 | 205 | @Override 206 | public String toString() { 207 | return "ProducerBatch{" 208 | + "groupKey=" 209 | + groupKey 210 | + ", packageId='" 211 | + packageId 212 | + '\'' 213 | + ", batchSizeThresholdInBytes=" 214 | + batchSizeThresholdInBytes 215 | + ", batchCountThreshold=" 216 | + batchCountThreshold 217 | + ", logItems=" 218 | + logItems 219 | + ", thunks=" 220 | + thunks 221 | + ", createdMs=" 222 | + createdMs 223 | + ", nextRetryMs=" 224 | + nextRetryMs 225 | + ", curBatchSizeInBytes=" 226 | + curBatchSizeInBytes 227 | + ", curBatchCount=" 228 | + curBatchCount 229 | + ", reservedAttempts=" 230 | + reservedAttempts 231 | + ", attemptCount=" 232 | + attemptCount 233 | + '}'; 234 | } 235 | 236 | private static final class Thunk { 237 | 238 | final Callback callback; 239 | 240 | final SettableFuture future; 241 | 242 | Thunk(Callback callback, SettableFuture future) { 243 | this.callback = callback; 244 | this.future = future; 245 | } 246 | } 247 | } 248 | -------------------------------------------------------------------------------- /src/main/java/com/aliyun/openservices/aliyun/log/producer/internals/RetryQueue.java: -------------------------------------------------------------------------------- 1 | package com.aliyun.openservices.aliyun.log.producer.internals; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | import java.util.concurrent.DelayQueue; 6 | import java.util.concurrent.TimeUnit; 7 | import java.util.concurrent.atomic.AtomicInteger; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | public class RetryQueue { 12 | 13 | private static final Logger LOGGER = LoggerFactory.getLogger(RetryQueue.class); 14 | 15 | private final DelayQueue retryBatches = new DelayQueue(); 16 | 17 | private final AtomicInteger putsInProgress; 18 | 19 | private volatile boolean closed; 20 | 21 | public RetryQueue() { 22 | this.putsInProgress = new AtomicInteger(0); 23 | this.closed = false; 24 | } 25 | 26 | public void put(ProducerBatch batch) { 27 | putsInProgress.incrementAndGet(); 28 | try { 29 | if (closed) { 30 | throw new IllegalStateException("cannot put after the retry queue was closed"); 31 | } 32 | retryBatches.put(batch); 33 | } finally { 34 | putsInProgress.decrementAndGet(); 35 | } 36 | } 37 | 38 | public List expiredBatches(long timeoutMs) { 39 | long deadline = System.currentTimeMillis() + timeoutMs; 40 | List expiredBatches = new ArrayList(); 41 | retryBatches.drainTo(expiredBatches); 42 | if (!expiredBatches.isEmpty()) { 43 | return expiredBatches; 44 | } 45 | while (true) { 46 | if (timeoutMs < 0) { 47 | break; 48 | } 49 | ProducerBatch batch; 50 | try { 51 | batch = retryBatches.poll(timeoutMs, TimeUnit.MILLISECONDS); 52 | } catch (InterruptedException e) { 53 | LOGGER.info("Interrupted when poll batch from the retry batches"); 54 | break; 55 | } 56 | if (batch == null) { 57 | break; 58 | } 59 | expiredBatches.add(batch); 60 | retryBatches.drainTo(expiredBatches); 61 | if (!expiredBatches.isEmpty()) { 62 | break; 63 | } 64 | timeoutMs = deadline - System.currentTimeMillis(); 65 | } 66 | return expiredBatches; 67 | } 68 | 69 | public List remainingBatches() { 70 | if (!closed) { 71 | throw new IllegalStateException( 72 | "cannot get the remaining batches before the retry queue closed"); 73 | } 74 | while (true) { 75 | if (!putsInProgress()) { 76 | break; 77 | } 78 | } 79 | List remainingBatches = new ArrayList(retryBatches); 80 | retryBatches.clear(); 81 | return remainingBatches; 82 | } 83 | 84 | public boolean isClosed() { 85 | return closed; 86 | } 87 | 88 | public void close() { 89 | this.closed = true; 90 | } 91 | 92 | private boolean putsInProgress() { 93 | return putsInProgress.get() > 0; 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /src/main/java/com/aliyun/openservices/aliyun/log/producer/internals/SendProducerBatchTask.java: -------------------------------------------------------------------------------- 1 | package com.aliyun.openservices.aliyun.log.producer.internals; 2 | 3 | import com.aliyun.openservices.aliyun.log.producer.Attempt; 4 | import com.aliyun.openservices.aliyun.log.producer.ProducerConfig; 5 | import com.aliyun.openservices.aliyun.log.producer.errors.Errors; 6 | import com.aliyun.openservices.aliyun.log.producer.errors.RetriableErrors; 7 | import com.aliyun.openservices.log.Client; 8 | import com.aliyun.openservices.log.common.Consts; 9 | import com.aliyun.openservices.log.common.TagContent; 10 | import com.aliyun.openservices.log.exception.LogException; 11 | import com.aliyun.openservices.log.request.PutLogsRequest; 12 | import com.aliyun.openservices.log.response.PutLogsResponse; 13 | import com.google.common.math.LongMath; 14 | import java.util.ArrayList; 15 | import java.util.List; 16 | import java.util.Map; 17 | import java.util.concurrent.BlockingQueue; 18 | import java.util.concurrent.atomic.AtomicInteger; 19 | import org.slf4j.Logger; 20 | import org.slf4j.LoggerFactory; 21 | 22 | public class SendProducerBatchTask implements Runnable { 23 | 24 | private static final Logger LOGGER = LoggerFactory.getLogger(ProducerBatch.class); 25 | 26 | private static final String TAG_PACK_ID = "__pack_id__"; 27 | 28 | private final ProducerBatch batch; 29 | 30 | private final ProducerConfig producerConfig; 31 | 32 | private final Map clientPool; 33 | 34 | private final RetryQueue retryQueue; 35 | 36 | private final BlockingQueue successQueue; 37 | 38 | private final BlockingQueue failureQueue; 39 | 40 | private final AtomicInteger batchCount; 41 | 42 | public SendProducerBatchTask( 43 | ProducerBatch batch, 44 | ProducerConfig producerConfig, 45 | Map clientPool, 46 | RetryQueue retryQueue, 47 | BlockingQueue successQueue, 48 | BlockingQueue failureQueue, 49 | AtomicInteger batchCount) { 50 | this.batch = batch; 51 | this.producerConfig = producerConfig; 52 | this.clientPool = clientPool; 53 | this.retryQueue = retryQueue; 54 | this.successQueue = successQueue; 55 | this.failureQueue = failureQueue; 56 | this.batchCount = batchCount; 57 | } 58 | 59 | @Override 60 | public void run() { 61 | try { 62 | sendProducerBatch(System.currentTimeMillis()); 63 | } catch (Throwable t) { 64 | LOGGER.error( 65 | "Uncaught error in send producer batch task, project=" 66 | + batch.getProject() 67 | + ", logStore=" 68 | + batch.getLogStore() 69 | + ", e=", 70 | t); 71 | } 72 | } 73 | 74 | private void sendProducerBatch(long nowMs) throws InterruptedException { 75 | LOGGER.trace("Prepare to send producer batch, batch={}", batch); 76 | String project = batch.getProject(); 77 | Client client = getClient(project); 78 | if (client == null) { 79 | LOGGER.error("Failed to get client, project={}", project); 80 | Attempt attempt = 81 | new Attempt( 82 | false, 83 | "", 84 | Errors.PROJECT_CONFIG_NOT_EXIST, 85 | "Cannot get the projectConfig for project " + project, 86 | nowMs); 87 | batch.appendAttempt(attempt); 88 | failureQueue.put(batch); 89 | } else { 90 | PutLogsResponse response; 91 | try { 92 | PutLogsRequest request = buildPutLogsRequest(batch); 93 | if (producerConfig.getCompressType() != null) { 94 | Consts.CompressType compressType = Consts.CompressType.fromString(producerConfig.getCompressType()); 95 | if (compressType != null && compressType != Consts.CompressType.NONE) { 96 | request.setCompressType(compressType); 97 | } 98 | } 99 | response = client.PutLogs(request); 100 | } catch (Exception e) { 101 | LOGGER.error( 102 | "Failed to put logs, project=" 103 | + batch.getProject() 104 | + ", logStore=" 105 | + batch.getLogStore() 106 | + ", e=", 107 | e); 108 | Attempt attempt = buildAttempt(e, nowMs); 109 | batch.appendAttempt(attempt); 110 | if (meetFailureCondition(e)) { 111 | LOGGER.debug("Prepare to put batch to the failure queue"); 112 | failureQueue.put(batch); 113 | } else { 114 | LOGGER.debug("Prepare to put batch to the retry queue"); 115 | long retryBackoffMs = calculateRetryBackoffMs(); 116 | LOGGER.debug( 117 | "Calculate the retryBackoffMs successfully, retryBackoffMs=" + retryBackoffMs); 118 | batch.setNextRetryMs(System.currentTimeMillis() + retryBackoffMs); 119 | try { 120 | retryQueue.put(batch); 121 | } catch (IllegalStateException e1) { 122 | LOGGER.error( 123 | "Failed to put batch to the retry queue, project=" 124 | + batch.getProject() 125 | + ", logStore=" 126 | + batch.getLogStore() 127 | + ", e=", 128 | e); 129 | if (retryQueue.isClosed()) { 130 | LOGGER.info( 131 | "Prepare to put batch to the failure queue since the retry queue was closed"); 132 | failureQueue.put(batch); 133 | } 134 | } 135 | } 136 | return; 137 | } 138 | Attempt attempt = new Attempt(true, response.GetRequestId(), "", "", nowMs); 139 | batch.appendAttempt(attempt); 140 | successQueue.put(batch); 141 | LOGGER.trace("Send producer batch successfully, batch={}", batch); 142 | } 143 | } 144 | 145 | private Client getClient(String project) { 146 | return clientPool.get(project); 147 | } 148 | 149 | private PutLogsRequest buildPutLogsRequest(ProducerBatch batch) { 150 | PutLogsRequest request; 151 | if (batch.getShardHash() != null && !batch.getShardHash().isEmpty()) { 152 | request = 153 | new PutLogsRequest( 154 | batch.getProject(), 155 | batch.getLogStore(), 156 | batch.getTopic(), 157 | batch.getSource(), 158 | batch.getLogItems(), 159 | batch.getShardHash()); 160 | } else { 161 | request = 162 | new PutLogsRequest( 163 | batch.getProject(), 164 | batch.getLogStore(), 165 | batch.getTopic(), 166 | batch.getSource(), 167 | batch.getLogItems()); 168 | } 169 | List tags = new ArrayList(); 170 | tags.add(new TagContent(TAG_PACK_ID, batch.getPackageId())); 171 | request.SetTags(tags); 172 | if (producerConfig.getLogFormat() == ProducerConfig.LogFormat.PROTOBUF) { 173 | request.setContentType(Consts.CONST_PROTO_BUF); 174 | } else { 175 | request.setContentType(Consts.CONST_SLS_JSON); 176 | } 177 | request.setProcessor(producerConfig.getProcessor()); 178 | return request; 179 | } 180 | 181 | private Attempt buildAttempt(Exception e, long nowMs) { 182 | if (e instanceof LogException) { 183 | LogException logException = (LogException) e; 184 | return new Attempt( 185 | false, 186 | logException.GetRequestId(), 187 | logException.GetErrorCode(), 188 | logException.GetErrorMessage(), 189 | nowMs); 190 | } else { 191 | return new Attempt(false, "", Errors.PRODUCER_EXCEPTION, e.getMessage(), nowMs); 192 | } 193 | } 194 | 195 | private boolean meetFailureCondition(Exception e) { 196 | if (!isRetriableException(e)) { 197 | return true; 198 | } 199 | if (retryQueue.isClosed()) { 200 | return true; 201 | } 202 | return batch.getRetries() >= producerConfig.getRetries(); 203 | } 204 | 205 | private boolean isRetriableException(Exception e) { 206 | if (e instanceof LogException) { 207 | LogException logException = (LogException) e; 208 | return (logException.GetErrorCode().equals(RetriableErrors.REQUEST_ERROR) 209 | || logException.GetErrorCode().equals(RetriableErrors.UNAUTHORIZED) 210 | || logException.GetErrorCode().equals(RetriableErrors.WRITE_QUOTA_EXCEED) 211 | || logException.GetErrorCode().equals(RetriableErrors.SHARD_WRITE_QUOTA_EXCEED) 212 | || logException.GetErrorCode().equals(RetriableErrors.EXCEED_QUOTA) 213 | || logException.GetErrorCode().equals(RetriableErrors.INTERNAL_SERVER_ERROR) 214 | || logException.GetErrorCode().equals(RetriableErrors.SERVER_BUSY) 215 | || logException.GetErrorCode().equals(RetriableErrors.BAD_RESPONSE) 216 | || logException.GetErrorCode().equals(RetriableErrors.PROJECT_NOT_EXISTS) 217 | || logException.GetErrorCode().equals(RetriableErrors.LOGSTORE_NOT_EXISTS) 218 | || logException.GetErrorCode().equals(RetriableErrors.SOCKET_TIMEOUT) 219 | || logException.GetErrorCode().equals(RetriableErrors.SIGNATURE_NOT_MATCH)); 220 | } 221 | return false; 222 | } 223 | 224 | private long calculateRetryBackoffMs() { 225 | int retry = batch.getRetries(); 226 | long retryBackoffMs = producerConfig.getBaseRetryBackoffMs() * LongMath.pow(2, retry); 227 | if (retryBackoffMs <= 0) { 228 | retryBackoffMs = producerConfig.getMaxRetryBackoffMs(); 229 | } 230 | return Math.min(retryBackoffMs, producerConfig.getMaxRetryBackoffMs()); 231 | } 232 | } 233 | -------------------------------------------------------------------------------- /src/main/java/com/aliyun/openservices/aliyun/log/producer/internals/Utils.java: -------------------------------------------------------------------------------- 1 | package com.aliyun.openservices.aliyun.log.producer.internals; 2 | 3 | import com.aliyun.openservices.log.util.NetworkUtils; 4 | import com.google.common.base.Charsets; 5 | import com.google.common.hash.Hashing; 6 | import java.lang.management.ManagementFactory; 7 | import java.util.concurrent.atomic.AtomicLong; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | public abstract class Utils { 12 | 13 | private static final Logger LOGGER = LoggerFactory.getLogger(Utils.class); 14 | 15 | public static void assertArgumentNotNull(Object argument, String argumentName) { 16 | if (argument == null) { 17 | throw new IllegalArgumentException(argumentName + " cannot be null"); 18 | } 19 | } 20 | 21 | public static void assertArgumentNotNullOrEmpty(String argument, String argumentName) { 22 | assertArgumentNotNull(argument, argumentName); 23 | if (argument.isEmpty()) { 24 | throw new IllegalArgumentException(argumentName + " cannot be empty"); 25 | } 26 | } 27 | 28 | public static String generateProducerHash(int instanceId) { 29 | String ip = NetworkUtils.getLocalMachineIP(); 30 | if (ip == null) { 31 | LOGGER.warn("Failed to get local machine ip, set ip to 127.0.0.1"); 32 | ip = "127.0.0.1"; 33 | } 34 | String name = ManagementFactory.getRuntimeMXBean().getName(); 35 | String input = ip + "-" + name + "-" + instanceId; 36 | return Hashing.farmHashFingerprint64().hashString(input, Charsets.US_ASCII).toString(); 37 | } 38 | 39 | public static String generatePackageId(String producerHash, AtomicLong batchId) { 40 | return (producerHash + "-" + Long.toHexString(batchId.getAndIncrement())).toUpperCase(); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/main/resources/META-INF/LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /src/test/java/com/aliyun/openservices/aliyun/log/producer/ProducerConfigTest.java: -------------------------------------------------------------------------------- 1 | package com.aliyun.openservices.aliyun.log.producer; 2 | 3 | import org.junit.Assert; 4 | import org.junit.Rule; 5 | import org.junit.Test; 6 | import org.junit.rules.ExpectedException; 7 | 8 | public class ProducerConfigTest { 9 | 10 | @Rule public ExpectedException thrown = ExpectedException.none(); 11 | 12 | @Test 13 | public void testProducerConfigDefaultValue() { 14 | ProducerConfig producerConfig = new ProducerConfig(); 15 | Assert.assertEquals( 16 | ProducerConfig.DEFAULT_TOTAL_SIZE_IN_BYTES, producerConfig.getTotalSizeInBytes()); 17 | Assert.assertEquals(ProducerConfig.DEFAULT_MAX_BLOCK_MS, producerConfig.getMaxBlockMs()); 18 | Assert.assertEquals(ProducerConfig.DEFAULT_IO_THREAD_COUNT, producerConfig.getIoThreadCount()); 19 | Assert.assertEquals( 20 | ProducerConfig.DEFAULT_BATCH_SIZE_THRESHOLD_IN_BYTES, 21 | producerConfig.getBatchSizeThresholdInBytes()); 22 | Assert.assertEquals( 23 | ProducerConfig.DEFAULT_BATCH_COUNT_THRESHOLD, producerConfig.getBatchCountThreshold()); 24 | Assert.assertEquals(ProducerConfig.DEFAULT_LINGER_MS, producerConfig.getLingerMs()); 25 | Assert.assertEquals(ProducerConfig.DEFAULT_RETRIES, producerConfig.getRetries()); 26 | Assert.assertEquals( 27 | ProducerConfig.DEFAULT_RETRIES + 1, producerConfig.getMaxReservedAttempts()); 28 | Assert.assertEquals( 29 | ProducerConfig.DEFAULT_BASE_RETRY_BACKOFF_MS, producerConfig.getBaseRetryBackoffMs()); 30 | Assert.assertEquals( 31 | ProducerConfig.DEFAULT_MAX_RETRY_BACKOFF_MS, producerConfig.getMaxRetryBackoffMs()); 32 | Assert.assertTrue(producerConfig.isAdjustShardHash()); 33 | Assert.assertEquals(ProducerConfig.DEFAULT_BUCKETS, producerConfig.getBuckets()); 34 | Assert.assertEquals(ProducerConfig.DEFAULT_LOG_FORMAT, producerConfig.getLogFormat()); 35 | } 36 | 37 | @Test 38 | public void testInvalidTotalSizeInBytes() { 39 | ProducerConfig producerConfig = new ProducerConfig(); 40 | thrown.expect(IllegalArgumentException.class); 41 | thrown.expectMessage("totalSizeInBytes must be greater than 0, got 0"); 42 | producerConfig.setTotalSizeInBytes(0); 43 | } 44 | 45 | @Test 46 | public void testInvalidIoThreadCount() { 47 | ProducerConfig producerConfig = new ProducerConfig(); 48 | thrown.expect(IllegalArgumentException.class); 49 | thrown.expectMessage("ioThreadCount must be greater than 0, got 0"); 50 | producerConfig.setIoThreadCount(0); 51 | } 52 | 53 | @Test 54 | public void testInvalidMaxBatchSizeInBytes() { 55 | ProducerConfig producerConfig = new ProducerConfig(); 56 | thrown.expect(IllegalArgumentException.class); 57 | thrown.expectMessage( 58 | "batchSizeThresholdInBytes must be between 1 and " 59 | + ProducerConfig.MAX_BATCH_SIZE_IN_BYTES 60 | + ", got -1"); 61 | producerConfig.setBatchSizeThresholdInBytes(-1); 62 | } 63 | 64 | @Test 65 | public void testInvalidMaxBatchCount() { 66 | ProducerConfig producerConfig = new ProducerConfig(); 67 | thrown.expect(IllegalArgumentException.class); 68 | thrown.expectMessage( 69 | "batchCountThreshold must be between 1 and " 70 | + ProducerConfig.MAX_BATCH_COUNT 71 | + ", got -100"); 72 | producerConfig.setBatchCountThreshold(-100); 73 | } 74 | 75 | @Test 76 | public void testInvalidLingerMs() { 77 | ProducerConfig producerConfig = new ProducerConfig(); 78 | thrown.expect(IllegalArgumentException.class); 79 | thrown.expectMessage("lingerMs must be greater than or equal to 100, got -1"); 80 | producerConfig.setLingerMs(-1); 81 | } 82 | 83 | @Test 84 | public void testInvalidLingerMs2() { 85 | ProducerConfig producerConfig = new ProducerConfig(); 86 | thrown.expect(IllegalArgumentException.class); 87 | thrown.expectMessage("lingerMs must be greater than or equal to 100, got 99"); 88 | producerConfig.setLingerMs(99); 89 | } 90 | 91 | @Test 92 | public void testInvalidMaxReservedAttempts() { 93 | ProducerConfig producerConfig = new ProducerConfig(); 94 | thrown.expect(IllegalArgumentException.class); 95 | thrown.expectMessage("maxReservedAttempts must be greater than 0, got 0"); 96 | producerConfig.setMaxReservedAttempts(0); 97 | } 98 | 99 | @Test 100 | public void testInvalidBaseRetryBackoffMs() { 101 | ProducerConfig producerConfig = new ProducerConfig(); 102 | thrown.expect(IllegalArgumentException.class); 103 | thrown.expectMessage("baseRetryBackoffMs must be greater than 0, got 0"); 104 | producerConfig.setBaseRetryBackoffMs(0); 105 | } 106 | 107 | @Test 108 | public void testInvalidMaxRetryBackoffMs() { 109 | ProducerConfig producerConfig = new ProducerConfig(); 110 | thrown.expect(IllegalArgumentException.class); 111 | thrown.expectMessage("maxRetryBackoffMs must be greater than 0, got -1"); 112 | producerConfig.setMaxRetryBackoffMs(-1); 113 | } 114 | 115 | @Test 116 | public void testInvalidBuckets() { 117 | ProducerConfig producerConfig = new ProducerConfig(); 118 | thrown.expect(IllegalArgumentException.class); 119 | thrown.expectMessage("buckets must be between " + ProducerConfig.BUCKETS_LOWER_LIMIT + " and " + ProducerConfig.BUCKETS_UPPER_LIMIT + ", got 0"); 120 | producerConfig.setBuckets(0); 121 | } 122 | 123 | @Test 124 | public void testInvalidBuckets2() { 125 | ProducerConfig producerConfig = new ProducerConfig(); 126 | thrown.expect(IllegalArgumentException.class); 127 | thrown.expectMessage("buckets must be between " + ProducerConfig.BUCKETS_LOWER_LIMIT + " and " + ProducerConfig.BUCKETS_UPPER_LIMIT + ", got 513"); 128 | producerConfig.setBuckets(513); 129 | } 130 | 131 | @Test 132 | public void testInvalidBuckets3() { 133 | ProducerConfig producerConfig = new ProducerConfig(); 134 | thrown.expect(IllegalArgumentException.class); 135 | thrown.expectMessage("buckets must be a power of 2, got 15"); 136 | producerConfig.setBuckets(15); 137 | } 138 | } 139 | -------------------------------------------------------------------------------- /src/test/java/com/aliyun/openservices/aliyun/log/producer/ProducerInvalidTest.java: -------------------------------------------------------------------------------- 1 | package com.aliyun.openservices.aliyun.log.producer; 2 | 3 | import com.aliyun.openservices.aliyun.log.producer.errors.Errors; 4 | import com.aliyun.openservices.aliyun.log.producer.errors.LogSizeTooLargeException; 5 | import com.aliyun.openservices.aliyun.log.producer.errors.MaxBatchCountExceedException; 6 | import com.aliyun.openservices.aliyun.log.producer.errors.ProducerException; 7 | import com.aliyun.openservices.aliyun.log.producer.errors.ResultFailedException; 8 | import com.aliyun.openservices.aliyun.log.producer.errors.TimeoutException; 9 | import com.aliyun.openservices.aliyun.log.producer.internals.LogSizeCalculator; 10 | import com.aliyun.openservices.log.common.LogItem; 11 | import com.google.common.math.LongMath; 12 | import com.google.common.util.concurrent.ListenableFuture; 13 | import java.util.ArrayList; 14 | import java.util.List; 15 | import java.util.concurrent.*; 16 | import org.junit.Assert; 17 | import org.junit.Rule; 18 | import org.junit.Test; 19 | import org.junit.rules.ExpectedException; 20 | 21 | public class ProducerInvalidTest { 22 | 23 | @Rule public ExpectedException thrown = ExpectedException.none(); 24 | 25 | private static final int logSize = LogSizeCalculator.calculate(ProducerTest.buildLogItem()); 26 | @Test 27 | public void testSendWithNullProject() throws InterruptedException, ProducerException { 28 | ProducerConfig producerConfig = new ProducerConfig(); 29 | Producer producer = new LogProducer(producerConfig); 30 | producer.putProjectConfig( 31 | new ProjectConfig("project", "endpoint", "accessKeyId", "accessKeySecret")); 32 | thrown.expect(IllegalArgumentException.class); 33 | thrown.expectMessage("project cannot be null"); 34 | producer.send(null, "logStore", new LogItem()); 35 | producer.close(); 36 | ProducerTest.assertProducerFinalState(producer); 37 | } 38 | 39 | @Test 40 | public void testSendWithEmptyProject() throws InterruptedException, ProducerException { 41 | ProducerConfig producerConfig = new ProducerConfig(); 42 | Producer producer = new LogProducer(producerConfig); 43 | producer.putProjectConfig(buildProjectConfig()); 44 | thrown.expect(IllegalArgumentException.class); 45 | thrown.expectMessage("project cannot be empty"); 46 | producer.send("", "logStore", new LogItem()); 47 | producer.close(); 48 | ProducerTest.assertProducerFinalState(producer); 49 | } 50 | 51 | @Test 52 | public void testSendWithNotExistProject() throws InterruptedException, ProducerException { 53 | ProducerConfig producerConfig = new ProducerConfig(); 54 | Producer producer = new LogProducer(producerConfig); 55 | producer.putProjectConfig(buildProjectConfig()); 56 | ListenableFuture f = producer.send("projectNotExist", "logStore", new LogItem(), null); 57 | try { 58 | f.get(); 59 | } catch (ExecutionException e) { 60 | ResultFailedException resultFailedException = (ResultFailedException) e.getCause(); 61 | Result result = resultFailedException.getResult(); 62 | Assert.assertFalse(result.isSuccessful()); 63 | Assert.assertEquals(Errors.PROJECT_CONFIG_NOT_EXIST, result.getErrorCode()); 64 | Assert.assertEquals( 65 | "Cannot get the projectConfig for project projectNotExist", result.getErrorMessage()); 66 | } 67 | producer.close(); 68 | ProducerTest.assertProducerFinalState(producer); 69 | } 70 | 71 | @Test 72 | public void testSendWithNullLogStore() throws InterruptedException, ProducerException { 73 | ProducerConfig producerConfig = new ProducerConfig(); 74 | Producer producer = new LogProducer(producerConfig); 75 | producer.putProjectConfig(buildProjectConfig()); 76 | thrown.expect(IllegalArgumentException.class); 77 | thrown.expectMessage("logStore cannot be null"); 78 | producer.send("project", null, new LogItem()); 79 | producer.close(); 80 | ProducerTest.assertProducerFinalState(producer); 81 | } 82 | 83 | @Test 84 | public void testSendWithEmptyLogStore() throws InterruptedException, ProducerException { 85 | ProducerConfig producerConfig = new ProducerConfig(); 86 | Producer producer = new LogProducer(producerConfig); 87 | producer.putProjectConfig(buildProjectConfig()); 88 | thrown.expect(IllegalArgumentException.class); 89 | thrown.expectMessage("logStore cannot be empty"); 90 | producer.send("project", "", new LogItem()); 91 | producer.close(); 92 | ProducerTest.assertProducerFinalState(producer); 93 | } 94 | 95 | @Test 96 | public void testSendWithNullLogItem() throws InterruptedException, ProducerException { 97 | ProducerConfig producerConfig = new ProducerConfig(); 98 | Producer producer = new LogProducer(producerConfig); 99 | producer.putProjectConfig(buildProjectConfig()); 100 | thrown.expect(IllegalArgumentException.class); 101 | thrown.expectMessage("logItem cannot be null"); 102 | producer.send("project", "logStore", (LogItem) null); 103 | producer.close(); 104 | ProducerTest.assertProducerFinalState(producer); 105 | } 106 | 107 | @Test 108 | public void testSendWithNullLogItems() throws InterruptedException, ProducerException { 109 | ProducerConfig producerConfig = new ProducerConfig(); 110 | Producer producer = new LogProducer(producerConfig); 111 | producer.putProjectConfig(buildProjectConfig()); 112 | thrown.expect(IllegalArgumentException.class); 113 | thrown.expectMessage("logItems cannot be null"); 114 | producer.send("project", "logStore", (List) null); 115 | producer.close(); 116 | ProducerTest.assertProducerFinalState(producer); 117 | } 118 | 119 | @Test 120 | public void testSendWithEmptyLogItems() throws InterruptedException, ProducerException { 121 | ProducerConfig producerConfig = new ProducerConfig(); 122 | Producer producer = new LogProducer(producerConfig); 123 | producer.putProjectConfig(buildProjectConfig()); 124 | thrown.expect(IllegalArgumentException.class); 125 | thrown.expectMessage("logItems cannot be empty"); 126 | List logItems = new ArrayList(); 127 | producer.send("project", "logStore", logItems); 128 | producer.close(); 129 | ProducerTest.assertProducerFinalState(producer); 130 | } 131 | 132 | @Test 133 | public void testSendLogsThrownMaxBatchCountExceedException() 134 | throws InterruptedException, ProducerException { 135 | ProducerConfig producerConfig = new ProducerConfig(); 136 | Producer producer = new LogProducer(producerConfig); 137 | producer.putProjectConfig(buildProjectConfig()); 138 | thrown.expect(MaxBatchCountExceedException.class); 139 | thrown.expectMessage( 140 | "the log list size is 40961 which exceeds the MAX_BATCH_COUNT " 141 | + ProducerConfig.MAX_BATCH_COUNT); 142 | List logItems = new ArrayList(); 143 | for (int i = 0; i < ProducerConfig.MAX_BATCH_COUNT + 1; ++i) { 144 | logItems.add(ProducerTest.buildLogItem()); 145 | } 146 | producer.send("project", "logStore", logItems); 147 | } 148 | 149 | @Test 150 | public void testSendLogThrownLogSizeTooLargeException() 151 | throws InterruptedException, ProducerException { 152 | ProducerConfig producerConfig = new ProducerConfig(); 153 | producerConfig.setTotalSizeInBytes(logSize - 1); 154 | Producer producer = new LogProducer(producerConfig); 155 | producer.putProjectConfig(buildProjectConfig()); 156 | thrown.expect(LogSizeTooLargeException.class); 157 | thrown.expectMessage( 158 | "the logs is " + logSize + " bytes which is larger than the totalSizeInBytes you specified"); 159 | producer.send("project", "logStore", ProducerTest.buildLogItem()); 160 | producer.close(); 161 | ProducerTest.assertProducerFinalState(producer); 162 | } 163 | 164 | @Test 165 | public void testSendLogsThrownLogSizeTooLargeException() 166 | throws InterruptedException, ProducerException { 167 | ProducerConfig producerConfig = new ProducerConfig(); 168 | producerConfig.setTotalSizeInBytes(logSize - 1); 169 | Producer producer = new LogProducer(producerConfig); 170 | producer.putProjectConfig(buildProjectConfig()); 171 | thrown.expect(LogSizeTooLargeException.class); 172 | thrown.expectMessage( 173 | "the logs is " + logSize * 3 + " bytes which is larger than the totalSizeInBytes you specified"); 174 | List logItems = new ArrayList(); 175 | logItems.add(ProducerTest.buildLogItem()); 176 | logItems.add(ProducerTest.buildLogItem()); 177 | logItems.add(ProducerTest.buildLogItem()); 178 | producer.send("project", "logStore", logItems); 179 | producer.close(); 180 | ProducerTest.assertProducerFinalState(producer); 181 | } 182 | 183 | @Test 184 | public void testSendLogThrownTimeoutException() throws InterruptedException, ProducerException { 185 | ProducerConfig producerConfig = new ProducerConfig(); 186 | producerConfig.setTotalSizeInBytes(logSize + 1); 187 | producerConfig.setMaxBlockMs(3); 188 | Producer producer = new LogProducer(producerConfig); 189 | producer.putProjectConfig(buildProjectConfig()); 190 | thrown.expect(TimeoutException.class); 191 | producer.send("project", "logStore", ProducerTest.buildLogItem()); 192 | producer.send("project", "logStore", ProducerTest.buildLogItem()); 193 | } 194 | 195 | @Test 196 | public void testSendWithRequestError() throws InterruptedException, ProducerException { 197 | ProducerConfig producerConfig = new ProducerConfig(); 198 | int retries = 5; 199 | producerConfig.setRetries(retries); 200 | producerConfig.setMaxReservedAttempts(retries + 1); 201 | Producer producer = new LogProducer(producerConfig); 202 | producer.putProjectConfig(buildProjectConfig()); 203 | ListenableFuture f = producer.send("project", "logStore", ProducerTest.buildLogItem()); 204 | try { 205 | f.get(); 206 | } catch (ExecutionException e) { 207 | ResultFailedException resultFailedException = (ResultFailedException) e.getCause(); 208 | Result result = resultFailedException.getResult(); 209 | Assert.assertFalse(result.isSuccessful()); 210 | Assert.assertEquals("RequestError", result.getErrorCode()); 211 | Assert.assertTrue( 212 | result.getErrorMessage().startsWith("Web request failed: project.endpoint")); 213 | List attempts = result.getReservedAttempts(); 214 | Assert.assertEquals(retries + 1, attempts.size()); 215 | long t1; 216 | long t2 = -1; 217 | for (int i = 0; i < attempts.size(); ++i) { 218 | Attempt attempt = attempts.get(i); 219 | Assert.assertFalse(attempt.isSuccess()); 220 | Assert.assertEquals("RequestError", attempt.getErrorCode()); 221 | Assert.assertTrue( 222 | attempt.getErrorMessage().startsWith("Web request failed: project.endpoint")); 223 | Assert.assertEquals("", attempt.getRequestId()); 224 | t1 = t2; 225 | t2 = attempt.getTimestampMs(); 226 | if (i == 0) { 227 | continue; 228 | } 229 | long diff = t2 - t1; 230 | long retryBackoffMs = producerConfig.getBaseRetryBackoffMs() * LongMath.pow(2, i - 1); 231 | long low = retryBackoffMs - (long) (producerConfig.getBaseRetryBackoffMs() * 0.1); 232 | long high = retryBackoffMs + (long) (producerConfig.getBaseRetryBackoffMs() * 0.2); 233 | if (i == 1) { 234 | Assert.assertTrue(low <= diff); 235 | } else { 236 | Assert.assertTrue(low <= diff && diff <= high); 237 | } 238 | } 239 | } 240 | producer.close(); 241 | ProducerTest.assertProducerFinalState(producer); 242 | } 243 | 244 | @Test 245 | public void testSendWithRequestError2() throws InterruptedException, ProducerException { 246 | ProducerConfig producerConfig = new ProducerConfig(); 247 | int retries = 5; 248 | int maxReservedAttempts = 2; 249 | producerConfig.setRetries(retries); 250 | producerConfig.setMaxReservedAttempts(maxReservedAttempts); 251 | Producer producer = new LogProducer(producerConfig); 252 | producer.putProjectConfig(buildProjectConfig()); 253 | ListenableFuture f = producer.send("project", "logStore", ProducerTest.buildLogItem()); 254 | try { 255 | f.get(); 256 | } catch (ExecutionException e) { 257 | ResultFailedException resultFailedException = (ResultFailedException) e.getCause(); 258 | Result result = resultFailedException.getResult(); 259 | Assert.assertFalse(result.isSuccessful()); 260 | Assert.assertEquals("RequestError", result.getErrorCode()); 261 | Assert.assertTrue( 262 | result.getErrorMessage().startsWith("Web request failed: project.endpoint")); 263 | List attempts = result.getReservedAttempts(); 264 | Assert.assertEquals(maxReservedAttempts, attempts.size()); 265 | Assert.assertEquals(retries + 1, result.getAttemptCount()); 266 | for (Attempt attempt : attempts) { 267 | Assert.assertFalse(attempt.isSuccess()); 268 | Assert.assertEquals("RequestError", attempt.getErrorCode()); 269 | Assert.assertTrue( 270 | attempt.getErrorMessage().startsWith("Web request failed: project.endpoint")); 271 | Assert.assertEquals("", attempt.getRequestId()); 272 | } 273 | } 274 | producer.close(); 275 | ProducerTest.assertProducerFinalState(producer); 276 | } 277 | 278 | @Test 279 | public void testCloseMultiTimes() throws InterruptedException, ProducerException { 280 | ProducerConfig producerConfig = new ProducerConfig(); 281 | producerConfig.setRetries(3); 282 | Producer producer = new LogProducer(producerConfig); 283 | producer.putProjectConfig(buildProjectConfig()); 284 | int n = 1000000; 285 | int futureGetCount = 0; 286 | List futures = new ArrayList(); 287 | for (int i = 0; i < n; ++i) { 288 | ListenableFuture f = 289 | producer.send("project", "logStore", ProducerTest.buildLogItem()); 290 | futures.add(f); 291 | } 292 | for (int i = 0; i < 1000; ++i) { 293 | closeInMultiThreads(producer, 1); 294 | } 295 | for (ListenableFuture f : futures) { 296 | try { 297 | f.get(); 298 | } catch (ExecutionException e) { 299 | futureGetCount++; 300 | ResultFailedException resultFailedException = (ResultFailedException) e.getCause(); 301 | Result result = resultFailedException.getResult(); 302 | Assert.assertFalse(result.isSuccessful()); 303 | Assert.assertEquals("RequestError", result.getErrorCode()); 304 | Assert.assertTrue( 305 | result.getErrorMessage().startsWith("Web request failed: project.endpoint")); 306 | List attempts = result.getReservedAttempts(); 307 | Assert.assertTrue(attempts.size() >= 1); 308 | for (Attempt attempt : attempts) { 309 | Assert.assertFalse(attempt.isSuccess()); 310 | Assert.assertEquals("RequestError", attempt.getErrorCode()); 311 | Assert.assertTrue( 312 | attempt.getErrorMessage().startsWith("Web request failed: project.endpoint")); 313 | } 314 | } 315 | } 316 | Assert.assertEquals(n, futureGetCount); 317 | ProducerTest.assertProducerFinalState(producer); 318 | } 319 | 320 | @Test 321 | public void testCloseInMultiThreads() throws InterruptedException, ProducerException { 322 | final ProducerConfig producerConfig = new ProducerConfig(); 323 | producerConfig.setRetries(3); 324 | final Producer producer = new LogProducer(producerConfig); 325 | producer.putProjectConfig(buildProjectConfig()); 326 | int n = 1000000; 327 | int futureGetCount = 0; 328 | List futures = new ArrayList(); 329 | for (int i = 0; i < n; ++i) { 330 | ListenableFuture f = 331 | producer.send("project", "logStore", ProducerTest.buildLogItem()); 332 | futures.add(f); 333 | } 334 | closeInMultiThreads(producer, 100); 335 | for (ListenableFuture f : futures) { 336 | try { 337 | f.get(); 338 | } catch (ExecutionException e) { 339 | futureGetCount++; 340 | ResultFailedException resultFailedException = (ResultFailedException) e.getCause(); 341 | Result result = resultFailedException.getResult(); 342 | Assert.assertFalse(result.isSuccessful()); 343 | Assert.assertEquals("RequestError", result.getErrorCode()); 344 | Assert.assertTrue( 345 | result.getErrorMessage().startsWith("Web request failed: project.endpoint")); 346 | List attempts = result.getReservedAttempts(); 347 | Assert.assertTrue(attempts.size() >= 1); 348 | for (Attempt attempt : attempts) { 349 | Assert.assertFalse(attempt.isSuccess()); 350 | Assert.assertEquals("RequestError", attempt.getErrorCode()); 351 | Assert.assertTrue( 352 | attempt.getErrorMessage().startsWith("Web request failed: project.endpoint")); 353 | } 354 | } 355 | } 356 | Assert.assertEquals(n, futureGetCount); 357 | } 358 | 359 | private void closeInMultiThreads(final Producer producer, int nTasks) { 360 | ExecutorService executorService = Executors.newFixedThreadPool(nTasks); 361 | List closeFutures = new ArrayList(); 362 | for (int i = 0; i < nTasks; ++i) { 363 | Future f = 364 | executorService.submit( 365 | new Callable() { 366 | @Override 367 | public Object call() throws Exception { 368 | producer.close(); 369 | ProducerTest.assertProducerFinalState(producer); 370 | return null; 371 | } 372 | }); 373 | closeFutures.add(f); 374 | } 375 | for (Future f : closeFutures) { 376 | try { 377 | f.get(); 378 | } catch (Exception e) { 379 | throw new RuntimeException(e); 380 | } 381 | } 382 | executorService.shutdown(); 383 | } 384 | 385 | private ProjectConfig buildProjectConfig() { 386 | return new ProjectConfig("project", "endpoint", "accessKeyId", "accessKeySecret"); 387 | } 388 | } 389 | -------------------------------------------------------------------------------- /src/test/java/com/aliyun/openservices/aliyun/log/producer/ProducerLargeAmountTest.java: -------------------------------------------------------------------------------- 1 | package com.aliyun.openservices.aliyun.log.producer; 2 | 3 | import com.aliyun.openservices.aliyun.log.producer.errors.ProducerException; 4 | import com.aliyun.openservices.log.common.LogItem; 5 | import java.util.List; 6 | import java.util.Random; 7 | import java.util.concurrent.CountDownLatch; 8 | import java.util.concurrent.ExecutorService; 9 | import java.util.concurrent.Executors; 10 | import java.util.concurrent.atomic.AtomicInteger; 11 | import org.junit.Assert; 12 | import org.junit.Test; 13 | 14 | public class ProducerLargeAmountTest { 15 | 16 | private final Random random = new Random(); 17 | 18 | @Test 19 | public void testSend() throws InterruptedException, ProducerException { 20 | ProducerConfig producerConfig = new ProducerConfig(); 21 | final Producer producer = new LogProducer(producerConfig); 22 | producer.putProjectConfig(buildProjectConfig()); 23 | final AtomicInteger successCount = new AtomicInteger(0); 24 | final int nTasks = 100; 25 | final int times = 10000; 26 | ExecutorService executorService = Executors.newFixedThreadPool(nTasks); 27 | final CountDownLatch latch = new CountDownLatch(nTasks); 28 | for (int i = 0; i < nTasks; ++i) { 29 | executorService.submit( 30 | new Runnable() { 31 | @Override 32 | public void run() { 33 | try { 34 | for (int i = 0; i < times; ++i) { 35 | producer.send( 36 | System.getenv("PROJECT"), 37 | System.getenv("LOG_STORE"), 38 | getTopic(), 39 | getSource(), 40 | ProducerTest.buildLogItem(), 41 | new Callback() { 42 | @Override 43 | public void onCompletion(Result result) { 44 | if (result.isSuccessful()) { 45 | successCount.incrementAndGet(); 46 | } 47 | } 48 | }); 49 | } 50 | } catch (Exception e) { 51 | e.printStackTrace(); 52 | } 53 | latch.countDown(); 54 | } 55 | }); 56 | } 57 | latch.await(); 58 | executorService.shutdown(); 59 | Thread.sleep(producerConfig.getLingerMs() * 2); 60 | producer.close(); 61 | Assert.assertEquals(nTasks * times, successCount.get()); 62 | ProducerTest.assertProducerFinalState(producer); 63 | } 64 | 65 | @Test 66 | public void testSendLogs() throws InterruptedException, ProducerException { 67 | ProducerConfig producerConfig = new ProducerConfig(); 68 | final Producer producer = new LogProducer(producerConfig); 69 | final AtomicInteger successCount = new AtomicInteger(0); 70 | final int nTasks = 100; 71 | final int times = 1000; 72 | final List logItems = ProducerTest.buildLogItems(50); 73 | ExecutorService executorService = Executors.newFixedThreadPool(nTasks); 74 | final CountDownLatch latch = new CountDownLatch(nTasks); 75 | for (int i = 0; i < nTasks; ++i) { 76 | executorService.submit( 77 | new Runnable() { 78 | @Override 79 | public void run() { 80 | try { 81 | for (int i = 0; i < times; ++i) { 82 | producer.send( 83 | System.getenv("PROJECT"), 84 | System.getenv("LOG_STORE"), 85 | getTopic(), 86 | getSource(), 87 | logItems, 88 | new Callback() { 89 | @Override 90 | public void onCompletion(Result result) { 91 | if (result.isSuccessful()) { 92 | successCount.incrementAndGet(); 93 | } 94 | } 95 | }); 96 | } 97 | } catch (Exception e) { 98 | e.printStackTrace(); 99 | } 100 | latch.countDown(); 101 | } 102 | }); 103 | } 104 | latch.await(); 105 | executorService.shutdown(); 106 | Thread.sleep(producerConfig.getLingerMs() * 4); 107 | producer.close(); 108 | Assert.assertEquals(nTasks * times, successCount.get()); 109 | ProducerTest.assertProducerFinalState(producer); 110 | } 111 | 112 | private ProjectConfig buildProjectConfig() { 113 | String project = System.getenv("PROJECT"); 114 | String endpoint = System.getenv("ENDPOINT"); 115 | String accessKeyId = System.getenv("ACCESS_KEY_ID"); 116 | String accessKeySecret = System.getenv("ACCESS_KEY_SECRET"); 117 | return new ProjectConfig(project, endpoint, accessKeyId, accessKeySecret); 118 | } 119 | 120 | private String getTopic() { 121 | return "topic-" + random.nextInt(5); 122 | } 123 | 124 | private String getSource() { 125 | return "source-" + random.nextInt(10); 126 | } 127 | } 128 | -------------------------------------------------------------------------------- /src/test/java/com/aliyun/openservices/aliyun/log/producer/ProducerMultiProjectTest.java: -------------------------------------------------------------------------------- 1 | package com.aliyun.openservices.aliyun.log.producer; 2 | 3 | import com.aliyun.openservices.aliyun.log.producer.errors.ProducerException; 4 | import com.google.common.util.concurrent.ListenableFuture; 5 | import java.util.ArrayList; 6 | import java.util.List; 7 | import java.util.Random; 8 | import java.util.concurrent.ExecutionException; 9 | import java.util.concurrent.atomic.AtomicInteger; 10 | import org.junit.Assert; 11 | import org.junit.Before; 12 | import org.junit.Test; 13 | 14 | public class ProducerMultiProjectTest { 15 | 16 | private final Random random = new Random(); 17 | 18 | private final List shardHashList = new ArrayList(); 19 | 20 | @Before 21 | public void setUp() { 22 | shardHashList.add("5F"); 23 | shardHashList.add("8C"); 24 | } 25 | 26 | @Test 27 | public void testSend() throws InterruptedException, ProducerException, ExecutionException { 28 | ProducerConfig producerConfig = new ProducerConfig(); 29 | final Producer producer = new LogProducer(producerConfig); 30 | producer.putProjectConfig( 31 | new ProjectConfig( 32 | System.getenv("PROJECT"), 33 | System.getenv("ENDPOINT"), 34 | System.getenv("ACCESS_KEY_ID"), 35 | System.getenv("ACCESS_KEY_SECRET"))); 36 | producer.putProjectConfig( 37 | new ProjectConfig( 38 | System.getenv("OTHER_PROJECT"), 39 | System.getenv("ENDPOINT"), 40 | System.getenv("ACCESS_KEY_ID"), 41 | System.getenv("ACCESS_KEY_SECRET"))); 42 | final AtomicInteger successCount = new AtomicInteger(); 43 | int futureGetCount = 0; 44 | int n = 10000; 45 | List futures = new ArrayList(); 46 | for (int i = 0; i < n; ++i) { 47 | ListenableFuture f = 48 | producer.send( 49 | System.getenv("PROJECT"), 50 | System.getenv("LOG_STORE"), 51 | "topic", 52 | null, 53 | getShardHash(), 54 | ProducerTest.buildLogItem(), 55 | new Callback() { 56 | @Override 57 | public void onCompletion(Result result) { 58 | if (result.isSuccessful()) { 59 | successCount.incrementAndGet(); 60 | } 61 | } 62 | }); 63 | futures.add(f); 64 | f = 65 | producer.send( 66 | System.getenv("OTHER_PROJECT"), 67 | System.getenv("OTHER_LOG_STORE"), 68 | null, 69 | "source", 70 | getShardHash(), 71 | ProducerTest.buildLogItem(), 72 | new Callback() { 73 | @Override 74 | public void onCompletion(Result result) { 75 | if (result.isSuccessful()) { 76 | successCount.incrementAndGet(); 77 | } 78 | } 79 | }); 80 | futures.add(f); 81 | } 82 | for (ListenableFuture f : futures) { 83 | Result result = (Result) f.get(); 84 | Assert.assertTrue(result.isSuccessful()); 85 | futureGetCount++; 86 | } 87 | producer.close(); 88 | Assert.assertEquals(n * 2, successCount.get()); 89 | Assert.assertEquals(n * 2, futureGetCount); 90 | ProducerTest.assertProducerFinalState(producer); 91 | } 92 | 93 | private String getShardHash() { 94 | return shardHashList.get(random.nextInt(shardHashList.size())); 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /src/test/java/com/aliyun/openservices/aliyun/log/producer/ProducerMultiShardTest.java: -------------------------------------------------------------------------------- 1 | package com.aliyun.openservices.aliyun.log.producer; 2 | 3 | import com.aliyun.openservices.aliyun.log.producer.errors.ProducerException; 4 | import com.aliyun.openservices.aliyun.log.producer.errors.ResultFailedException; 5 | import com.google.common.util.concurrent.ListenableFuture; 6 | import java.util.concurrent.ExecutionException; 7 | import org.junit.Assert; 8 | import org.junit.Rule; 9 | import org.junit.Test; 10 | import org.junit.rules.ExpectedException; 11 | 12 | public class ProducerMultiShardTest { 13 | 14 | @Rule public ExpectedException thrown = ExpectedException.none(); 15 | 16 | @Test 17 | public void testSend() throws InterruptedException, ProducerException, ExecutionException { 18 | ProducerConfig producerConfig = new ProducerConfig(); 19 | final Producer producer = new LogProducer(producerConfig); 20 | producer.putProjectConfig( 21 | new ProjectConfig( 22 | System.getenv("PROJECT"), 23 | System.getenv("ENDPOINT"), 24 | System.getenv("ACCESS_KEY_ID"), 25 | System.getenv("ACCESS_KEY_SECRET"))); 26 | producer.putProjectConfig( 27 | new ProjectConfig( 28 | System.getenv("OTHER_PROJECT"), 29 | System.getenv("ENDPOINT"), 30 | System.getenv("ACCESS_KEY_ID"), 31 | System.getenv("ACCESS_KEY_SECRET"))); 32 | ListenableFuture f = 33 | producer.send( 34 | System.getenv("OTHER_PROJECT"), 35 | System.getenv("OTHER_LOG_STORE"), 36 | "", 37 | "shard3", 38 | "127.0.0.1", 39 | ProducerTest.buildLogItem()); 40 | Result result = f.get(); 41 | Assert.assertTrue(result.isSuccessful()); 42 | 43 | f = 44 | producer.send( 45 | System.getenv("OTHER_PROJECT"), 46 | System.getenv("OTHER_LOG_STORE"), 47 | null, 48 | "shard1", 49 | "192.168.0.2", 50 | ProducerTest.buildLogItem()); 51 | result = f.get(); 52 | Assert.assertTrue(result.isSuccessful()); 53 | 54 | producer.close(); 55 | ProducerTest.assertProducerFinalState(producer); 56 | } 57 | 58 | @Test 59 | public void testInvalidSend() throws InterruptedException, ProducerException { 60 | ProducerConfig producerConfig = new ProducerConfig(); 61 | producerConfig.setAdjustShardHash(false); 62 | final Producer producer = new LogProducer(producerConfig); 63 | producer.putProjectConfig( 64 | new ProjectConfig( 65 | System.getenv("PROJECT"), 66 | System.getenv("ENDPOINT"), 67 | System.getenv("ACCESS_KEY_ID"), 68 | System.getenv("ACCESS_KEY_SECRET"))); 69 | producer.putProjectConfig( 70 | new ProjectConfig( 71 | System.getenv("OTHER_PROJECT"), 72 | System.getenv("ENDPOINT"), 73 | System.getenv("ACCESS_KEY_ID"), 74 | System.getenv("ACCESS_KEY_SECRET"))); 75 | ListenableFuture f = 76 | producer.send( 77 | System.getenv("OTHER_PROJECT"), 78 | System.getenv("OTHER_LOG_STORE"), 79 | "", 80 | "", 81 | "0", 82 | ProducerTest.buildLogItem()); 83 | try { 84 | f.get(); 85 | } catch (ExecutionException e) { 86 | ResultFailedException resultFailedException = (ResultFailedException) e.getCause(); 87 | Assert.assertEquals("ParameterInvalid", resultFailedException.getErrorCode()); 88 | } 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /src/test/java/com/aliyun/openservices/aliyun/log/producer/ProducerTest.java: -------------------------------------------------------------------------------- 1 | package com.aliyun.openservices.aliyun.log.producer; 2 | 3 | import com.aliyun.openservices.aliyun.log.producer.errors.ProducerException; 4 | import com.aliyun.openservices.aliyun.log.producer.errors.ResultFailedException; 5 | import com.aliyun.openservices.aliyun.log.producer.errors.RetriableErrors; 6 | import com.aliyun.openservices.aliyun.log.producer.internals.LogSizeCalculator; 7 | import com.aliyun.openservices.log.common.LogItem; 8 | import com.aliyun.openservices.log.common.auth.DefaultCredentials; 9 | import com.aliyun.openservices.log.common.auth.StaticCredentialsProvider; 10 | import com.google.common.util.concurrent.ListenableFuture; 11 | import java.util.ArrayList; 12 | import java.util.List; 13 | import java.util.concurrent.ExecutionException; 14 | import java.util.concurrent.atomic.AtomicInteger; 15 | import org.junit.Assert; 16 | import org.junit.Test; 17 | 18 | public class ProducerTest { 19 | 20 | @Test 21 | public void testSend() throws InterruptedException, ProducerException, ExecutionException { 22 | ProducerConfig producerConfig = new ProducerConfig(); 23 | producerConfig.setSourceIp("127.0.0.1"); 24 | final Producer producer = new LogProducer(producerConfig); 25 | producer.putProjectConfig(buildProjectConfig()); 26 | ListenableFuture f = 27 | producer.send(System.getenv("PROJECT"), System.getenv("LOG_STORE"), buildLogItem()); 28 | Result result = f.get(); 29 | Assert.assertTrue(result.isSuccessful()); 30 | Assert.assertEquals("", result.getErrorCode()); 31 | Assert.assertEquals("", result.getErrorMessage()); 32 | Assert.assertEquals(1, result.getReservedAttempts().size()); 33 | Assert.assertTrue(!result.getReservedAttempts().get(0).getRequestId().isEmpty()); 34 | 35 | f = 36 | producer.send( 37 | System.getenv("PROJECT"), System.getenv("LOG_STORE"), null, null, buildLogItem()); 38 | result = f.get(); 39 | Assert.assertTrue(result.isSuccessful()); 40 | Assert.assertEquals("", result.getErrorCode()); 41 | Assert.assertEquals("", result.getErrorMessage()); 42 | Assert.assertEquals(1, result.getReservedAttempts().size()); 43 | Assert.assertTrue(!result.getReservedAttempts().get(0).getRequestId().isEmpty()); 44 | 45 | f = producer.send(System.getenv("PROJECT"), System.getenv("LOG_STORE"), "", "", buildLogItem()); 46 | result = f.get(); 47 | Assert.assertTrue(result.isSuccessful()); 48 | Assert.assertEquals("", result.getErrorCode()); 49 | Assert.assertEquals("", result.getErrorMessage()); 50 | Assert.assertEquals(1, result.getReservedAttempts().size()); 51 | Assert.assertTrue(!result.getReservedAttempts().get(0).getRequestId().isEmpty()); 52 | 53 | f = 54 | producer.send( 55 | System.getenv("PROJECT"), 56 | System.getenv("LOG_STORE"), 57 | "topic", 58 | "source", 59 | buildLogItem()); 60 | result = f.get(); 61 | Assert.assertTrue(result.isSuccessful()); 62 | Assert.assertEquals("", result.getErrorCode()); 63 | Assert.assertEquals("", result.getErrorMessage()); 64 | Assert.assertEquals(1, result.getReservedAttempts().size()); 65 | Assert.assertTrue(!result.getReservedAttempts().get(0).getRequestId().isEmpty()); 66 | 67 | producer.close(); 68 | assertProducerFinalState(producer); 69 | } 70 | 71 | @Test 72 | public void testSendWithCredentialsProvider() throws InterruptedException, ProducerException, ExecutionException { 73 | ProducerConfig producerConfig = new ProducerConfig(); 74 | final Producer producer = new LogProducer(producerConfig); 75 | producer.putProjectConfig(buildCredentialsProjectConfig()); 76 | ListenableFuture f = 77 | producer.send(System.getenv("PROJECT"), System.getenv("LOG_STORE"), buildLogItem()); 78 | Result result = f.get(); 79 | Assert.assertTrue(result.isSuccessful()); 80 | Assert.assertEquals("", result.getErrorCode()); 81 | Assert.assertEquals("", result.getErrorMessage()); 82 | Assert.assertEquals(1, result.getReservedAttempts().size()); 83 | Assert.assertTrue(!result.getReservedAttempts().get(0).getRequestId().isEmpty()); 84 | 85 | f = 86 | producer.send( 87 | System.getenv("PROJECT"), System.getenv("LOG_STORE"), null, null, buildLogItem()); 88 | result = f.get(); 89 | Assert.assertTrue(result.isSuccessful()); 90 | Assert.assertEquals("", result.getErrorCode()); 91 | Assert.assertEquals("", result.getErrorMessage()); 92 | Assert.assertEquals(1, result.getReservedAttempts().size()); 93 | Assert.assertTrue(!result.getReservedAttempts().get(0).getRequestId().isEmpty()); 94 | 95 | f = producer.send(System.getenv("PROJECT"), System.getenv("LOG_STORE"), "", "", buildLogItem()); 96 | result = f.get(); 97 | Assert.assertTrue(result.isSuccessful()); 98 | Assert.assertEquals("", result.getErrorCode()); 99 | Assert.assertEquals("", result.getErrorMessage()); 100 | Assert.assertEquals(1, result.getReservedAttempts().size()); 101 | Assert.assertTrue(!result.getReservedAttempts().get(0).getRequestId().isEmpty()); 102 | 103 | f = 104 | producer.send( 105 | System.getenv("PROJECT"), 106 | System.getenv("LOG_STORE"), 107 | "topic", 108 | "source", 109 | buildLogItem()); 110 | result = f.get(); 111 | Assert.assertTrue(result.isSuccessful()); 112 | Assert.assertEquals("", result.getErrorCode()); 113 | Assert.assertEquals("", result.getErrorMessage()); 114 | Assert.assertEquals(1, result.getReservedAttempts().size()); 115 | Assert.assertTrue(!result.getReservedAttempts().get(0).getRequestId().isEmpty()); 116 | 117 | producer.close(); 118 | assertProducerFinalState(producer); 119 | } 120 | 121 | @Test 122 | public void testSendWithCallback() 123 | throws InterruptedException, ProducerException, ExecutionException { 124 | ProducerConfig producerConfig = new ProducerConfig(); 125 | final Producer producer = new LogProducer(producerConfig); 126 | producer.putProjectConfig(buildProjectConfig()); 127 | final AtomicInteger successCount = new AtomicInteger(0); 128 | ListenableFuture f = 129 | producer.send( 130 | System.getenv("PROJECT"), 131 | System.getenv("LOG_STORE"), 132 | buildLogItem(), 133 | new Callback() { 134 | @Override 135 | public void onCompletion(Result result) { 136 | if (result.isSuccessful()) { 137 | successCount.incrementAndGet(); 138 | } 139 | } 140 | }); 141 | Result result = f.get(); 142 | Assert.assertTrue(result.isSuccessful()); 143 | Assert.assertEquals("", result.getErrorCode()); 144 | Assert.assertEquals("", result.getErrorMessage()); 145 | Assert.assertEquals(1, result.getReservedAttempts().size()); 146 | Assert.assertTrue(!result.getReservedAttempts().get(0).getRequestId().isEmpty()); 147 | 148 | f = 149 | producer.send( 150 | System.getenv("PROJECT"), 151 | System.getenv("LOG_STORE"), 152 | null, 153 | null, 154 | buildLogItem(), 155 | new Callback() { 156 | @Override 157 | public void onCompletion(Result result) { 158 | if (result.isSuccessful()) { 159 | successCount.incrementAndGet(); 160 | } 161 | } 162 | }); 163 | result = f.get(); 164 | Assert.assertTrue(result.isSuccessful()); 165 | 166 | f = 167 | producer.send( 168 | System.getenv("PROJECT"), 169 | System.getenv("LOG_STORE"), 170 | "", 171 | "", 172 | buildLogItem(), 173 | new Callback() { 174 | @Override 175 | public void onCompletion(Result result) { 176 | if (result.isSuccessful()) { 177 | successCount.incrementAndGet(); 178 | } 179 | } 180 | }); 181 | result = f.get(); 182 | Assert.assertTrue(result.isSuccessful()); 183 | Assert.assertEquals("", result.getErrorCode()); 184 | Assert.assertEquals("", result.getErrorMessage()); 185 | Assert.assertEquals(1, result.getReservedAttempts().size()); 186 | Assert.assertTrue(!result.getReservedAttempts().get(0).getRequestId().isEmpty()); 187 | 188 | f = 189 | producer.send( 190 | System.getenv("PROJECT"), 191 | System.getenv("LOG_STORE"), 192 | "topic", 193 | "source", 194 | buildLogItem(), 195 | new Callback() { 196 | @Override 197 | public void onCompletion(Result result) { 198 | if (result.isSuccessful()) { 199 | successCount.incrementAndGet(); 200 | } 201 | } 202 | }); 203 | result = f.get(); 204 | Assert.assertTrue(result.isSuccessful()); 205 | Assert.assertEquals("", result.getErrorCode()); 206 | Assert.assertEquals("", result.getErrorMessage()); 207 | Assert.assertEquals(1, result.getReservedAttempts().size()); 208 | Assert.assertTrue(!result.getReservedAttempts().get(0).getRequestId().isEmpty()); 209 | 210 | Assert.assertEquals(4, successCount.get()); 211 | 212 | producer.close(); 213 | assertProducerFinalState(producer); 214 | } 215 | 216 | @Test 217 | public void testSendWithInvalidAccessKeyId() throws InterruptedException, ProducerException { 218 | ProducerConfig producerConfig = new ProducerConfig(); 219 | producerConfig.setRetries(4); 220 | final Producer producer = new LogProducer(producerConfig); 221 | producer.putProjectConfig(buildInvalidAccessKeyIdProjectConfig()); 222 | ListenableFuture f = 223 | producer.send(System.getenv("PROJECT"), System.getenv("LOG_STORE"), buildLogItem()); 224 | Thread.sleep(1000 * 3); 225 | producer.putProjectConfig(buildProjectConfig()); 226 | try { 227 | Result result = f.get(); 228 | Assert.assertTrue(result.isSuccessful()); 229 | Assert.assertTrue(result.getErrorCode().isEmpty()); 230 | Assert.assertTrue(result.getErrorMessage().isEmpty()); 231 | List attempts = result.getReservedAttempts(); 232 | System.out.println(attempts.size()); 233 | for (int i = 0; i < attempts.size(); ++i) { 234 | Attempt attempt = attempts.get(i); 235 | if (i == attempts.size() - 1) { 236 | Assert.assertTrue(attempt.isSuccess()); 237 | Assert.assertTrue(result.getErrorCode().isEmpty()); 238 | Assert.assertTrue(result.getErrorMessage().isEmpty()); 239 | } else { 240 | Assert.assertFalse(attempt.isSuccess()); 241 | Assert.assertEquals("Unauthorized", attempt.getErrorCode()); 242 | Assert.assertFalse(attempt.getErrorMessage().isEmpty()); 243 | } 244 | } 245 | 246 | } catch (ExecutionException e) { 247 | ResultFailedException resultFailedException = (ResultFailedException) e.getCause(); 248 | Result result = resultFailedException.getResult(); 249 | Assert.assertFalse(result.isSuccessful()); 250 | Assert.assertEquals("SignatureNotMatch", result.getErrorCode()); 251 | Assert.assertTrue(!result.getErrorMessage().isEmpty()); 252 | List attempts = result.getReservedAttempts(); 253 | Assert.assertEquals(1, attempts.size()); 254 | for (Attempt attempt : attempts) { 255 | Assert.assertFalse(attempt.isSuccess()); 256 | Assert.assertEquals("SignatureNotMatch", attempt.getErrorCode()); 257 | Assert.assertTrue(!attempt.getErrorMessage().isEmpty()); 258 | Assert.assertTrue(!attempt.getRequestId().isEmpty()); 259 | } 260 | } 261 | } 262 | 263 | @Test 264 | public void testSendWithInvalidAccessKeySecret() throws InterruptedException, ProducerException { 265 | ProducerConfig producerConfig = new ProducerConfig(); 266 | final Producer producer = new LogProducer(producerConfig); 267 | producer.putProjectConfig(buildInvalidAccessKeySecretProjectConfig()); 268 | ListenableFuture f = 269 | producer.send(System.getenv("PROJECT"), System.getenv("LOG_STORE"), buildLogItem()); 270 | try { 271 | f.get(); 272 | } catch (ExecutionException e) { 273 | ResultFailedException resultFailedException = (ResultFailedException) e.getCause(); 274 | Result result = resultFailedException.getResult(); 275 | Assert.assertFalse(result.isSuccessful()); 276 | Assert.assertEquals(RetriableErrors.SIGNATURE_NOT_MATCH, result.getErrorCode()); 277 | Assert.assertTrue(!result.getErrorMessage().isEmpty()); 278 | List attempts = result.getReservedAttempts(); 279 | Assert.assertEquals(11, attempts.size()); 280 | for (Attempt attempt : attempts) { 281 | Assert.assertFalse(attempt.isSuccess()); 282 | Assert.assertEquals(RetriableErrors.SIGNATURE_NOT_MATCH, attempt.getErrorCode()); 283 | Assert.assertTrue(!attempt.getErrorMessage().isEmpty()); 284 | Assert.assertTrue(!attempt.getRequestId().isEmpty()); 285 | } 286 | } 287 | } 288 | 289 | @Test 290 | public void testClose() throws InterruptedException, ProducerException, ExecutionException { 291 | ProducerConfig producerConfig = new ProducerConfig(); 292 | final Producer producer = new LogProducer(producerConfig); 293 | producer.putProjectConfig(buildProjectConfig()); 294 | final AtomicInteger successCount = new AtomicInteger(0); 295 | int futureGetCount = 0; 296 | int n = 100000; 297 | List futures = new ArrayList(); 298 | for (int i = 0; i < n; ++i) { 299 | ListenableFuture f = 300 | producer.send( 301 | System.getenv("PROJECT"), 302 | System.getenv("LOG_STORE"), 303 | buildLogItem(), 304 | new Callback() { 305 | @Override 306 | public void onCompletion(Result result) { 307 | if (result.isSuccessful()) { 308 | successCount.incrementAndGet(); 309 | } 310 | } 311 | }); 312 | futures.add(f); 313 | } 314 | producer.close(); 315 | for (ListenableFuture f : futures) { 316 | Result result = (Result) f.get(); 317 | Assert.assertTrue(result.isSuccessful()); 318 | futureGetCount++; 319 | } 320 | Assert.assertEquals(n, successCount.get()); 321 | Assert.assertEquals(n, futureGetCount); 322 | assertProducerFinalState(producer); 323 | } 324 | 325 | @Test 326 | public void testCloseInCallback() 327 | throws InterruptedException, ProducerException, ExecutionException { 328 | ProducerConfig producerConfig = new ProducerConfig(); 329 | final Producer producer = new LogProducer(producerConfig); 330 | producer.putProjectConfig(buildProjectConfig()); 331 | final AtomicInteger successCount = new AtomicInteger(0); 332 | int futureGetCount = 0; 333 | int n = 10000; 334 | List futures = new ArrayList(); 335 | for (int i = 0; i < n; ++i) { 336 | ListenableFuture f = 337 | producer.send( 338 | System.getenv("PROJECT"), 339 | System.getenv("LOG_STORE"), 340 | buildLogItem(), 341 | new Callback() { 342 | @Override 343 | public void onCompletion(Result result) { 344 | if (result.isSuccessful()) { 345 | successCount.incrementAndGet(); 346 | } 347 | try { 348 | producer.close(); 349 | } catch (Exception e) { 350 | e.printStackTrace(); 351 | } 352 | } 353 | }); 354 | futures.add(f); 355 | } 356 | producer.close(); 357 | for (ListenableFuture f : futures) { 358 | Result result = (Result) f.get(); 359 | Assert.assertTrue(result.isSuccessful()); 360 | futureGetCount++; 361 | } 362 | Assert.assertEquals(n, successCount.get()); 363 | Assert.assertEquals(n, futureGetCount); 364 | assertProducerFinalState(producer); 365 | } 366 | 367 | @Test 368 | public void testMaxBatchSizeInBytes() throws InterruptedException, ProducerException { 369 | ProducerConfig producerConfig = new ProducerConfig(); 370 | producerConfig.setBatchSizeThresholdInBytes(27); 371 | Producer producer = new LogProducer(producerConfig); 372 | producer.putProjectConfig(buildProjectConfig()); 373 | LogItem logItem = new LogItem(); 374 | logItem.PushBack("key1", "val1"); 375 | logItem.PushBack("key2", "val2"); 376 | logItem.PushBack("key3", "val3"); 377 | int sizeInBytes = LogSizeCalculator.calculate(logItem); 378 | Assert.assertEquals(28, sizeInBytes); 379 | producer.send("project", "logStore", new LogItem()); 380 | } 381 | 382 | public static void assertProducerFinalState(Producer producer) { 383 | Assert.assertEquals(0, producer.getBatchCount()); 384 | Assert.assertEquals( 385 | producer.getProducerConfig().getTotalSizeInBytes(), producer.availableMemoryInBytes()); 386 | } 387 | 388 | public static LogItem buildLogItem() { 389 | LogItem logItem = new LogItem(); 390 | logItem.PushBack("k1", "v1"); 391 | logItem.PushBack("k2", "v2"); 392 | return logItem; 393 | } 394 | 395 | public static List buildLogItems(int n) { 396 | List logItems = new ArrayList(); 397 | for (int i = 0; i < n; ++i) { 398 | logItems.add(buildLogItem()); 399 | } 400 | return logItems; 401 | } 402 | 403 | private ProjectConfig buildProjectConfig() { 404 | String project = System.getenv("PROJECT"); 405 | String endpoint = System.getenv("ENDPOINT"); 406 | String accessKeyId = System.getenv("ACCESS_KEY_ID"); 407 | String accessKeySecret = System.getenv("ACCESS_KEY_SECRET"); 408 | return new ProjectConfig(project, endpoint, accessKeyId, accessKeySecret); 409 | } 410 | 411 | private ProjectConfig buildCredentialsProjectConfig() { 412 | String project = System.getenv("PROJECT"); 413 | String endpoint = System.getenv("ENDPOINT"); 414 | String accessKeyId = System.getenv("ACCESS_KEY_ID"); 415 | String accessKeySecret = System.getenv("ACCESS_KEY_SECRET"); 416 | return new ProjectConfig(project, endpoint, 417 | new StaticCredentialsProvider(new DefaultCredentials(accessKeyId, accessKeySecret)), 418 | null); 419 | } 420 | 421 | private ProjectConfig buildInvalidAccessKeyIdProjectConfig() { 422 | String project = System.getenv("PROJECT"); 423 | String endpoint = System.getenv("ENDPOINT"); 424 | String accessKeyId = System.getenv("ACCESS_KEY_ID") + "XXX"; 425 | String accessKeySecret = System.getenv("ACCESS_KEY_SECRET"); 426 | return new ProjectConfig(project, endpoint, accessKeyId, accessKeySecret); 427 | } 428 | 429 | private ProjectConfig buildInvalidAccessKeySecretProjectConfig() { 430 | String project = System.getenv("PROJECT"); 431 | String endpoint = System.getenv("ENDPOINT"); 432 | String accessKeyId = System.getenv("ACCESS_KEY_ID"); 433 | String accessKeySecret = System.getenv("ACCESS_KEY_SECRET") + "XXX"; 434 | return new ProjectConfig(project, endpoint, accessKeyId, accessKeySecret); 435 | } 436 | } 437 | -------------------------------------------------------------------------------- /src/test/java/com/aliyun/openservices/aliyun/log/producer/ShardHashAdjusterTest.java: -------------------------------------------------------------------------------- 1 | package com.aliyun.openservices.aliyun.log.producer; 2 | 3 | import org.junit.Assert; 4 | import org.junit.Rule; 5 | import org.junit.Test; 6 | import org.junit.rules.ExpectedException; 7 | 8 | public class ShardHashAdjusterTest { 9 | 10 | @Rule public ExpectedException thrown = ExpectedException.none(); 11 | 12 | @Test 13 | public void testAdjust() { 14 | ShardHashAdjuster adjuster = new ShardHashAdjuster(1); 15 | String adjustedShardHash = adjuster.adjust("127.0.0.1"); 16 | Assert.assertEquals("00000000000000000000000000000000", adjustedShardHash); 17 | adjustedShardHash = adjuster.adjust("192.168.0.2"); 18 | Assert.assertEquals("00000000000000000000000000000000", adjustedShardHash); 19 | 20 | adjuster = new ShardHashAdjuster(2); 21 | adjustedShardHash = adjuster.adjust("127.0.0.1"); 22 | Assert.assertEquals("80000000000000000000000000000000", adjustedShardHash); 23 | adjustedShardHash = adjuster.adjust("192.168.0.2"); 24 | Assert.assertEquals("00000000000000000000000000000000", adjustedShardHash); 25 | 26 | adjuster = new ShardHashAdjuster(4); 27 | adjustedShardHash = adjuster.adjust("127.0.0.1"); 28 | Assert.assertEquals("c0000000000000000000000000000000", adjustedShardHash); 29 | adjustedShardHash = adjuster.adjust("192.168.0.2"); 30 | Assert.assertEquals("40000000000000000000000000000000", adjustedShardHash); 31 | 32 | adjuster = new ShardHashAdjuster(8); 33 | adjustedShardHash = adjuster.adjust("127.0.0.1"); 34 | Assert.assertEquals("e0000000000000000000000000000000", adjustedShardHash); 35 | adjustedShardHash = adjuster.adjust("192.168.0.2"); 36 | Assert.assertEquals("60000000000000000000000000000000", adjustedShardHash); 37 | 38 | adjuster = new ShardHashAdjuster(16); 39 | adjustedShardHash = adjuster.adjust("127.0.0.1"); 40 | Assert.assertEquals("f0000000000000000000000000000000", adjustedShardHash); 41 | adjustedShardHash = adjuster.adjust("192.168.0.2"); 42 | Assert.assertEquals("60000000000000000000000000000000", adjustedShardHash); 43 | 44 | adjuster = new ShardHashAdjuster(32); 45 | adjustedShardHash = adjuster.adjust("127.0.0.1"); 46 | Assert.assertEquals("f0000000000000000000000000000000", adjustedShardHash); 47 | adjustedShardHash = adjuster.adjust("192.168.0.2"); 48 | Assert.assertEquals("68000000000000000000000000000000", adjustedShardHash); 49 | 50 | adjuster = new ShardHashAdjuster(64); 51 | adjustedShardHash = adjuster.adjust("127.0.0.1"); 52 | Assert.assertEquals("f4000000000000000000000000000000", adjustedShardHash); 53 | adjustedShardHash = adjuster.adjust("192.168.0.2"); 54 | Assert.assertEquals("6c000000000000000000000000000000", adjustedShardHash); 55 | 56 | adjuster = new ShardHashAdjuster(128); 57 | adjustedShardHash = adjuster.adjust("127.0.0.1"); 58 | Assert.assertEquals("f4000000000000000000000000000000", adjustedShardHash); 59 | adjustedShardHash = adjuster.adjust("192.168.0.2"); 60 | Assert.assertEquals("6e000000000000000000000000000000", adjustedShardHash); 61 | 62 | adjuster = new ShardHashAdjuster(256); 63 | adjustedShardHash = adjuster.adjust("127.0.0.1"); 64 | Assert.assertEquals("f5000000000000000000000000000000", adjustedShardHash); 65 | adjustedShardHash = adjuster.adjust("192.168.0.2"); 66 | Assert.assertEquals("6f000000000000000000000000000000", adjustedShardHash); 67 | } 68 | 69 | @Test 70 | public void testAdjustEmptyStr() { 71 | ShardHashAdjuster adjuster = new ShardHashAdjuster(1); 72 | String adjustedShardHash = adjuster.adjust(""); 73 | Assert.assertEquals("00000000000000000000000000000000", adjustedShardHash); 74 | 75 | adjuster = new ShardHashAdjuster(2); 76 | adjustedShardHash = adjuster.adjust(""); 77 | Assert.assertEquals("80000000000000000000000000000000", adjustedShardHash); 78 | 79 | adjuster = new ShardHashAdjuster(4); 80 | adjustedShardHash = adjuster.adjust(""); 81 | Assert.assertEquals("c0000000000000000000000000000000", adjustedShardHash); 82 | 83 | adjuster = new ShardHashAdjuster(8); 84 | adjustedShardHash = adjuster.adjust(""); 85 | Assert.assertEquals("c0000000000000000000000000000000", adjustedShardHash); 86 | 87 | adjuster = new ShardHashAdjuster(16); 88 | adjustedShardHash = adjuster.adjust(""); 89 | Assert.assertEquals("d0000000000000000000000000000000", adjustedShardHash); 90 | 91 | adjuster = new ShardHashAdjuster(32); 92 | adjustedShardHash = adjuster.adjust(""); 93 | Assert.assertEquals("d0000000000000000000000000000000", adjustedShardHash); 94 | 95 | adjuster = new ShardHashAdjuster(64); 96 | adjustedShardHash = adjuster.adjust(""); 97 | Assert.assertEquals("d4000000000000000000000000000000", adjustedShardHash); 98 | 99 | adjuster = new ShardHashAdjuster(128); 100 | adjustedShardHash = adjuster.adjust(""); 101 | Assert.assertEquals("d4000000000000000000000000000000", adjustedShardHash); 102 | 103 | adjuster = new ShardHashAdjuster(256); 104 | adjustedShardHash = adjuster.adjust(""); 105 | Assert.assertEquals("d4000000000000000000000000000000", adjustedShardHash); 106 | } 107 | 108 | @Test 109 | public void testConstructInvalidShardHashAdjusterWith9() { 110 | thrown.expect(IllegalArgumentException.class); 111 | thrown.expectMessage("buckets must be a power of 2, got 9"); 112 | new ShardHashAdjuster(9); 113 | } 114 | 115 | @Test 116 | public void testConstructInvalidShardHashAdjusterWith62() { 117 | thrown.expect(IllegalArgumentException.class); 118 | thrown.expectMessage("buckets must be a power of 2, got 63"); 119 | new ShardHashAdjuster(63); 120 | } 121 | } 122 | -------------------------------------------------------------------------------- /src/test/java/com/aliyun/openservices/aliyun/log/producer/internals/LogSizeCalculatorTest.java: -------------------------------------------------------------------------------- 1 | package com.aliyun.openservices.aliyun.log.producer.internals; 2 | 3 | import com.aliyun.openservices.aliyun.log.producer.ProducerTest; 4 | import com.aliyun.openservices.log.common.LogItem; 5 | import org.junit.Assert; 6 | import org.junit.Test; 7 | 8 | public class LogSizeCalculatorTest { 9 | 10 | @Test 11 | public void testCalculateLog() { 12 | ProducerTest.buildLogItem(); 13 | int sizeInBytes = LogSizeCalculator.calculate(ProducerTest.buildLogItem()); 14 | Assert.assertEquals(32, sizeInBytes); 15 | 16 | LogItem logItem = new LogItem(); 17 | logItem.PushBack("key1", "val1"); 18 | logItem.PushBack("key2", "val2"); 19 | logItem.PushBack("key3", "val3"); 20 | sizeInBytes = LogSizeCalculator.calculate(logItem); 21 | Assert.assertEquals(56, sizeInBytes); 22 | } 23 | 24 | @Test 25 | public void testCalculateLogNullKey() { 26 | LogItem logItem = new LogItem(); 27 | logItem.PushBack("key1", "val1"); 28 | logItem.PushBack("key2", "val2"); 29 | logItem.PushBack(null, "null_key"); 30 | int sizeInBytes = LogSizeCalculator.calculate(logItem); 31 | Assert.assertEquals(56, sizeInBytes); 32 | } 33 | 34 | @Test 35 | public void testCalculateLogNullValue() { 36 | LogItem logItem = new LogItem(); 37 | logItem.PushBack("key1", "val1"); 38 | logItem.PushBack("key2", "val2"); 39 | logItem.PushBack("null_value", null); 40 | int sizeInBytes = LogSizeCalculator.calculate(logItem); 41 | Assert.assertEquals(58, sizeInBytes); 42 | } 43 | 44 | @Test 45 | public void testCalculateLogNullKeyAndValue() { 46 | LogItem logItem = new LogItem(); 47 | logItem.PushBack(null, null); 48 | logItem.PushBack(null, null); 49 | logItem.PushBack(null, null); 50 | int sizeInBytes = LogSizeCalculator.calculate(logItem); 51 | Assert.assertEquals(32, sizeInBytes); 52 | } 53 | 54 | @Test 55 | public void testCalculateLogs() { 56 | ProducerTest.buildLogItem(); 57 | int sizeInBytes = LogSizeCalculator.calculate(ProducerTest.buildLogItems(4)); 58 | Assert.assertEquals(128, sizeInBytes); 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/test/java/com/aliyun/openservices/aliyun/log/producer/internals/ProducerBatchTest.java: -------------------------------------------------------------------------------- 1 | package com.aliyun.openservices.aliyun.log.producer.internals; 2 | 3 | import com.aliyun.openservices.aliyun.log.producer.Attempt; 4 | import com.aliyun.openservices.aliyun.log.producer.ProducerTest; 5 | import com.aliyun.openservices.aliyun.log.producer.Result; 6 | import com.aliyun.openservices.log.common.LogItem; 7 | import com.google.common.util.concurrent.ListenableFuture; 8 | import java.util.ArrayList; 9 | import java.util.List; 10 | import java.util.NoSuchElementException; 11 | import org.junit.Assert; 12 | import org.junit.Rule; 13 | import org.junit.Test; 14 | import org.junit.rules.ExpectedException; 15 | 16 | public class ProducerBatchTest { 17 | 18 | @Rule public ExpectedException thrown = ExpectedException.none(); 19 | 20 | @Test 21 | public void testTryAppendLog() { 22 | GroupKey groupKey = new GroupKey("project", "logStore", "topic", "source", "shardHash"); 23 | ProducerBatch batch = new ProducerBatch(groupKey, "id", 35, 10, 3, System.currentTimeMillis()); 24 | LogItem logItem = new LogItem(); 25 | logItem.PushBack("key1", "val1"); 26 | logItem.PushBack("key2", "val2"); 27 | logItem.PushBack("key3", "val3"); 28 | logItem.PushBack("key4", "val4"); 29 | int sizeInBytes = LogSizeCalculator.calculate(logItem); 30 | ListenableFuture f = batch.tryAppend(logItem, sizeInBytes, null); 31 | Assert.assertNotNull(f); 32 | Assert.assertTrue(batch.isMeetSendCondition()); 33 | } 34 | 35 | @Test 36 | public void testTryAppendLogsExceedBatchSizeThreshold() { 37 | GroupKey groupKey = new GroupKey("project", "logStore", "topic", "source", "shardHash"); 38 | ProducerBatch batch = new ProducerBatch(groupKey, "id", 20, 10, 3, System.currentTimeMillis()); 39 | List logItems = new ArrayList(); 40 | logItems.add(ProducerTest.buildLogItem()); 41 | logItems.add(ProducerTest.buildLogItem()); 42 | logItems.add(ProducerTest.buildLogItem()); 43 | int sizeInBytes = LogSizeCalculator.calculate(logItems); 44 | ListenableFuture f = batch.tryAppend(logItems, sizeInBytes, null); 45 | Assert.assertNotNull(f); 46 | Assert.assertTrue(batch.isMeetSendCondition()); 47 | } 48 | 49 | @Test 50 | public void testTryAppendLogsExceedBatchCountThreshold() { 51 | GroupKey groupKey = new GroupKey("project", "logStore", "topic", "source", "shardHash"); 52 | ProducerBatch batch = 53 | new ProducerBatch(groupKey, "id", 10000, 1, 3, System.currentTimeMillis()); 54 | List logItems = new ArrayList(); 55 | logItems.add(new LogItem()); 56 | logItems.add(new LogItem()); 57 | int sizeInBytes = LogSizeCalculator.calculate(logItems); 58 | ListenableFuture f = batch.tryAppend(logItems, sizeInBytes, null); 59 | Assert.assertNotNull(f); 60 | Assert.assertTrue(batch.isMeetSendCondition()); 61 | } 62 | 63 | @Test 64 | public void testIsMeetSendCondition() { 65 | GroupKey groupKey = new GroupKey("project", "logStore", "topic", "source", "shardHash"); 66 | ProducerBatch batch = new ProducerBatch(groupKey, "id", 8, 100, 3, System.currentTimeMillis()); 67 | List logItems = new ArrayList(); 68 | logItems.add(new LogItem()); 69 | logItems.add(new LogItem()); 70 | int sizeInBytes = LogSizeCalculator.calculate(logItems); 71 | ListenableFuture f = batch.tryAppend(logItems, sizeInBytes, null); 72 | Assert.assertNotNull(f); 73 | Assert.assertTrue(batch.isMeetSendCondition()); 74 | } 75 | 76 | @Test 77 | public void testAppendAttempt() { 78 | GroupKey groupKey = new GroupKey("project", "logStore", "topic", "source", "shardHash"); 79 | ProducerBatch batch = 80 | new ProducerBatch(groupKey, "id", 100, 100, 3, System.currentTimeMillis()); 81 | List logItems = new ArrayList(); 82 | logItems.add(new LogItem()); 83 | logItems.add(new LogItem()); 84 | logItems.add(new LogItem()); 85 | logItems.add(new LogItem()); 86 | logItems.add(new LogItem()); 87 | int sizeInBytes = LogSizeCalculator.calculate(logItems); 88 | ListenableFuture f = batch.tryAppend(logItems, sizeInBytes, null); 89 | Assert.assertNotNull(f); 90 | batch.appendAttempt(new Attempt(true, "xxx", "", "", System.currentTimeMillis())); 91 | batch.fireCallbacksAndSetFutures(); 92 | } 93 | 94 | @Test 95 | public void testFireCallbacksAndSetFutures() { 96 | GroupKey groupKey = new GroupKey("project", "logStore", "topic", "source", "shardHash"); 97 | ProducerBatch batch = 98 | new ProducerBatch(groupKey, "id", 100, 100, 3, System.currentTimeMillis()); 99 | List logItems = new ArrayList(); 100 | logItems.add(new LogItem()); 101 | logItems.add(new LogItem()); 102 | logItems.add(new LogItem()); 103 | logItems.add(new LogItem()); 104 | logItems.add(new LogItem()); 105 | int sizeInBytes = LogSizeCalculator.calculate(logItems); 106 | ListenableFuture f = batch.tryAppend(logItems, sizeInBytes, null); 107 | Assert.assertNotNull(f); 108 | thrown.expect(NoSuchElementException.class); 109 | batch.fireCallbacksAndSetFutures(); 110 | } 111 | 112 | @Test 113 | public void testRemainingMs() { 114 | GroupKey groupKey = new GroupKey("project", "logStore", "topic", "source", "shardHash"); 115 | ProducerBatch batch = new ProducerBatch(groupKey, "id", 100, 100, 3, 1000); 116 | Assert.assertEquals(200, batch.remainingMs(0, 200)); 117 | Assert.assertEquals(200, batch.remainingMs(1000, 200)); 118 | Assert.assertEquals(50, batch.remainingMs(1150, 200)); 119 | Assert.assertEquals(0, batch.remainingMs(1200, 200)); 120 | Assert.assertEquals(-100, batch.remainingMs(1300, 200)); 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /src/test/java/com/aliyun/openservices/aliyun/log/producer/internals/RetryQueueTest.java: -------------------------------------------------------------------------------- 1 | package com.aliyun.openservices.aliyun.log.producer.internals; 2 | 3 | import java.util.List; 4 | import org.junit.Assert; 5 | import org.junit.Rule; 6 | import org.junit.Test; 7 | import org.junit.rules.ExpectedException; 8 | 9 | public class RetryQueueTest { 10 | 11 | @Rule public ExpectedException thrown = ExpectedException.none(); 12 | 13 | @Test 14 | public void testPutAfterClose() { 15 | RetryQueue retryQueue = new RetryQueue(); 16 | retryQueue.put(newProducerBatch("1")); 17 | retryQueue.close(); 18 | thrown.expect(IllegalStateException.class); 19 | thrown.expectMessage("cannot put after the retry queue was closed"); 20 | retryQueue.put(newProducerBatch("2")); 21 | retryQueue.remainingBatches(); 22 | } 23 | 24 | @Test 25 | public void testExpiredBatches() throws InterruptedException { 26 | RetryQueue retryQueue = new RetryQueue(); 27 | ProducerBatch b1 = newProducerBatch("1"); 28 | ProducerBatch b2 = newProducerBatch("2"); 29 | ProducerBatch b3 = newProducerBatch("3"); 30 | ProducerBatch b4 = newProducerBatch("4"); 31 | ProducerBatch b5 = newProducerBatch("5"); 32 | long nowMs = System.currentTimeMillis(); 33 | b1.setNextRetryMs(nowMs + 3000); 34 | b2.setNextRetryMs(nowMs + 5000); 35 | b3.setNextRetryMs(nowMs + 1000); 36 | b4.setNextRetryMs(nowMs + 4000); 37 | b5.setNextRetryMs(nowMs + 2000); 38 | retryQueue.put(b1); 39 | retryQueue.put(b2); 40 | retryQueue.put(b3); 41 | retryQueue.put(b4); 42 | retryQueue.put(b5); 43 | Thread.sleep(1000); 44 | List expiredBatches = retryQueue.expiredBatches(10); 45 | Assert.assertEquals(1, expiredBatches.size()); 46 | Assert.assertEquals("3", expiredBatches.get(0).getPackageId()); 47 | Thread.sleep(1000); 48 | expiredBatches = retryQueue.expiredBatches(10); 49 | Assert.assertEquals(1, expiredBatches.size()); 50 | Assert.assertEquals("5", expiredBatches.get(0).getPackageId()); 51 | Thread.sleep(1000); 52 | expiredBatches = retryQueue.expiredBatches(10); 53 | Assert.assertEquals(1, expiredBatches.size()); 54 | Assert.assertEquals("1", expiredBatches.get(0).getPackageId()); 55 | Thread.sleep(1000); 56 | expiredBatches = retryQueue.expiredBatches(10); 57 | Assert.assertEquals(1, expiredBatches.size()); 58 | Assert.assertEquals("4", expiredBatches.get(0).getPackageId()); 59 | Thread.sleep(1000); 60 | expiredBatches = retryQueue.expiredBatches(10); 61 | Assert.assertEquals(1, expiredBatches.size()); 62 | Assert.assertEquals("2", expiredBatches.get(0).getPackageId()); 63 | } 64 | 65 | @Test 66 | public void testRemainingBatches() { 67 | RetryQueue retryQueue = new RetryQueue(); 68 | ProducerBatch b1 = newProducerBatch("1"); 69 | ProducerBatch b2 = newProducerBatch("2"); 70 | ProducerBatch b3 = newProducerBatch("3"); 71 | b1.setNextRetryMs(500); 72 | b2.setNextRetryMs(100); 73 | b3.setNextRetryMs(300); 74 | retryQueue.put(b1); 75 | retryQueue.put(b2); 76 | retryQueue.put(b3); 77 | retryQueue.close(); 78 | List remainingBatches = retryQueue.remainingBatches(); 79 | Assert.assertEquals(3, remainingBatches.size()); 80 | Assert.assertEquals("2", remainingBatches.get(0).getPackageId()); 81 | } 82 | 83 | @Test 84 | public void testInvalidRemainingBatches() { 85 | RetryQueue retryQueue = new RetryQueue(); 86 | retryQueue.put(newProducerBatch("1")); 87 | retryQueue.put(newProducerBatch("2")); 88 | retryQueue.put(newProducerBatch("3")); 89 | thrown.expect(IllegalStateException.class); 90 | thrown.expectMessage("cannot get the remaining batches before the retry queue closed"); 91 | retryQueue.remainingBatches(); 92 | } 93 | 94 | private ProducerBatch newProducerBatch(String packageId) { 95 | GroupKey groupKey = new GroupKey("project", "logStore", "", "", ""); 96 | return new ProducerBatch(groupKey, packageId, 100, 100, 3, System.currentTimeMillis()); 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /src/test/java/com/aliyun/openservices/aliyun/log/producer/internals/UtilsTest.java: -------------------------------------------------------------------------------- 1 | package com.aliyun.openservices.aliyun.log.producer.internals; 2 | 3 | import java.util.concurrent.atomic.AtomicLong; 4 | import org.junit.Assert; 5 | import org.junit.Test; 6 | 7 | public class UtilsTest { 8 | 9 | @Test 10 | public void testGenerateProducerHash() { 11 | String producerHash1 = Utils.generateProducerHash(3); 12 | String producerHash2 = Utils.generateProducerHash(4); 13 | Assert.assertTrue(!producerHash1.equals(producerHash2)); 14 | } 15 | 16 | @Test 17 | public void testGeneratePackageId() { 18 | AtomicLong batchId = new AtomicLong(0); 19 | String producerHash = Utils.generateProducerHash(3); 20 | for (int i = 0; i < 100; ++i) { 21 | String packageId = Utils.generatePackageId(producerHash, batchId); 22 | String[] pieces = packageId.split("-", 2); 23 | Assert.assertEquals(producerHash.toLowerCase(), pieces[0].toLowerCase()); 24 | Assert.assertEquals(Long.toHexString(i).toLowerCase(), pieces[1].toLowerCase()); 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/test/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | --------------------------------------------------------------------------------