├── .gitignore ├── LICENSE ├── README.md ├── doc └── eight-byte-id-structure.md ├── pom.xml ├── uniqueid-core ├── pom.xml └── src │ ├── main │ └── java │ │ └── org │ │ └── lable │ │ └── oss │ │ └── uniqueid │ │ ├── AutoRefillStack.java │ │ ├── BaseUniqueIDGenerator.java │ │ ├── Clock.java │ │ ├── GeneratorException.java │ │ ├── GeneratorIdentityHolder.java │ │ ├── IDGenerator.java │ │ ├── LocalGeneratorIdentity.java │ │ ├── LocalUniqueIDGeneratorFactory.java │ │ ├── OnePerMillisecondDecorator.java │ │ ├── ParameterUtil.java │ │ └── bytes │ │ ├── Blueprint.java │ │ ├── IDBuilder.java │ │ └── Mode.java │ └── test │ └── java │ └── org │ └── lable │ └── oss │ └── uniqueid │ ├── AutoRefillStackTest.java │ ├── BaseUniqueIDGeneratorTest.java │ ├── ByteArray.java │ ├── GeneratorExceptionTest.java │ ├── LocalUniqueIDGeneratorFactoryTest.java │ ├── LocalUniqueIDGeneratorIT.java │ ├── OnePerMillisecondDecoratorIT.java │ ├── UniqueIDGeneratorThreadSafetyIT.java │ └── bytes │ ├── BlueprintTest.java │ └── IDBuilderTest.java └── uniqueid-etcd ├── pom.xml └── src ├── main └── java │ └── org │ └── lable │ └── oss │ └── uniqueid │ └── etcd │ ├── ClusterID.java │ ├── EtcdHelper.java │ ├── ExpiringResourceClaim.java │ ├── RegistryBasedGeneratorIdentity.java │ ├── RegistryBasedResourceClaim.java │ ├── ResourceClaim.java │ ├── ResourcePair.java │ ├── SynchronizedGeneratorIdentity.java │ └── SynchronizedUniqueIDGeneratorFactory.java └── test ├── java └── org │ └── lable │ └── oss │ └── uniqueid │ └── etcd │ ├── AcquisitionTimeoutIT.java │ ├── ClusterIDIT.java │ ├── EtcdHelperIT.java │ ├── EtcdTestCluster.java │ ├── ExpiringResourceClaimIT.java │ ├── HighGeneratorCountIT.java │ ├── MultipleClusterIdsIT.java │ ├── MultipleGeneratorsIT.java │ ├── RegistryBasedGeneratorIdentityTest.java │ ├── SynchronizedUniqueIDGeneratorIT.java │ └── TestHelper.java └── resources └── log4j2.xml /.gitignore: -------------------------------------------------------------------------------- 1 | # Autogenerared OS X clutter. 2 | .DS_Store 3 | ._.DS_Store 4 | # Autogenerated Windows explorer clutter. 5 | Thumbs.db 6 | 7 | # Various text editor swap files, etc. 8 | .*.swp 9 | 10 | # All Maven output goes here, compiled classes, generated code, etc. 11 | # Ignore only folders named target in root dir and direct child dirs. 12 | /target/ 13 | */target/ 14 | 15 | 16 | # Your IDE generates project files based on the Maven POM. It is 17 | # usually a good idea to exclude these from your source code repository 18 | # as well. 19 | 20 | # Eclipse generated files. 21 | .settings 22 | .classpath 23 | .project 24 | 25 | # IntelliJ generated files. 26 | *.iml 27 | .idea 28 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | 204 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Unique ID generator 2 | =================== 3 | 4 | A unique ID generator that generates unique¹ eight byte identifiers in a distributed context. 5 | 6 | 1. Unique within the confines of the chosen computing environment. 7 | 8 | ## What is it for? 9 | 10 | When you want to assign unique identifiers to objects (e.g., database records) in a 11 | distributed computing environment that are short and guaranteed to be unique (within your 12 | data realm), this library can provide them. 13 | 14 | ## Versions 15 | 16 | For Java 8 and newer, please use the `2.x` series of releases. For Java 7 version `1.x` may be 17 | used, but this version is no longer actively maintained. 18 | 19 | ## Concepts 20 | 21 | To have multiple concurrent processes generate short unique identifiers, some form of 22 | coordination is required. The two basic premises of this library are: 23 | 24 | 1. Each process that generates identifiers must claim or be assigned a number representing 25 | its **generator-ID** and incorporate that in the identifiers it generates 26 | 2. Each process that generates identifiers must have its clock synchronised and must 27 | incorporate the current **timestamp** in the identifiers it generates 28 | 29 | Processes can generate up to 64 identifiers per millisecond; a **sequence** counter 30 | is incorporated in each identifier that represents these 64 possibilities. 31 | 32 | In addition to the generator-ID, the **cluster-ID** allows for 16 'clusters' of generators to 33 | simultaneously generate identifiers. This is useful if several groups of processes have to 34 | operate without being able to coordinate with each other in real time. Examples include 35 | processes run on data clusters in separate data centres, and maintenance processes that 36 | operate on off-line copies of data sets. 37 | 38 | The identifiers generated are composed from the components mentioned above, and have 39 | the following structure: 40 | 41 | ``` 42 | // Each letter represents one bit in the eight byte identifier. 43 | TTTTTTTT TTTTTTTT TTTTTTTT TTTTTTTT TTTTTTTT TTSSSSSS ...MGGGG GGGGCCCC 44 | ``` 45 | 46 | * `T`: Timestamp (in milliseconds, bit order depends on mode) 47 | * `S`: Sequence counter 48 | * `.`: *Reserved for future use* 49 | * `M`: Mode 50 | * `G`: Generator-ID 51 | * `C`: Cluster-ID 52 | 53 | See also: [eight byte ID structure](doc/eight-byte-id-structure.md). 54 | 55 | ### Modes 56 | 57 | The **mode** flag is used to distinguish between the two modes of generating IDs: `SPREAD` 58 | and `TIME_SEQUENTIAL`. 59 | 60 | In `SPREAD` mode the IDs generated are meant to be used as opaque identifiers. Generated 61 | IDs do not sort sequentially, but are instead 'spread out' over the eight byte address space. 62 | This is useful when these IDs are used as part of a row-key in a key-value store, because the 63 | non-sequential nature of the IDs can help prevent hot-spotting. 64 | 65 | Conversely, the `TIME_SEQUENTIAL` mode is meant for IDs that *should* sort in order of their 66 | time of creation, and can be useful in assigning time-based identifiers to objects, and prevent 67 | objects created in the same millisecond instant from overlapping. 68 | 69 | ### Coordination 70 | 71 | As long as a single identifier generating process has a claim on a unique 72 | cluster-ID/generator-ID pair, it can generate unique identifiers (within the system it is a 73 | part of). The cluster-ID is always assigned manually (the assumption being that only a very 74 | limited number of 'clusters' exists). The generator-ID can also be assigned manually, but 75 | usually it is desirable for processes to be able to exclusively claim a generator-ID 76 | automatically. 77 | 78 | This library facilitates this using [Etcd](https://etcd.io/) or [Apache ZooKeeper](http://zookeeper.apache.org/). 79 | Generators can stake a claim on a generator-ID for a short period of time (usually ten 80 | minutes), and repeat this whenever IDs are generated. 81 | 82 | ## Usage 83 | 84 | ### Local usage 85 | 86 | If you have a computing environment where you know exactly which processes may generate IDs. 87 | The simple `LocalUniqueIDGenerator` can be used. This generator assumes that you know which 88 | process may use which generator-ID at any time. 89 | 90 | For example, if you have just one process that handles the creation of new IDs (perhaps a 91 | single server that creates database records using these IDs), a generator can be used like 92 | this: 93 | 94 | ```java 95 | final int generatorID = 0; 96 | final int clusterID = 0; 97 | IDGenerator generator = LocalUniqueIDGeneratorFactory.generatorFor(generatorID, clusterID, Mode.SPREAD); 98 | 99 | // Generate IDs 100 | byte[] id = generator.generate(); 101 | ``` 102 | 103 | The `LocalUniqueIDGeneratorFactory` assumes that you can guarantee that it is the only 104 | generator with the specific generator-ID and cluster-ID combination you chose, during its 105 | lifetime. 106 | 107 | If there is a fixed number of processes that may generate IDs, you can assign one of the 108 | 256 possible generator-IDs to each one. For a more in-depth explanation of generator-IDs 109 | and cluster-IDs, see [eight byte ID structure](doc/eight-byte-id-structure.md). 110 | 111 | For a cluster of Tomcat servers in a high-availability setup, you could configure a system 112 | property on each server with a unique generator-ID, although this approach does assume that 113 | there is only one ID generating instance running on that server. 114 | 115 | For local usage the `uniqueid-core` module can be used: 116 | 117 | ```xml 118 | 119 | org.lable.oss.uniqueid 120 | uniqueid-core 121 | ${uniqueid.version} 122 | 123 | ``` 124 | 125 | ### Distributed usage 126 | 127 | If you need to generate unique IDs within a distributed environment, automatic coordination of 128 | the generator-ID is also a possibility. The acquisition of a generator-ID can be handled by a 129 | `SynchronizedGeneratorIdentity` instance, which uses [Etcd](https://etcd.io/) or 130 | [Apache ZooKeeper](http://zookeeper.apache.org/) to claim its generator-ID 131 | 132 | #### With an Etcd cluster 133 | 134 | For this Etcd the `uniqueid-etcd` module is used: 135 | 136 | ```xml 137 | 138 | org.lable.oss.uniqueid 139 | uniqueid-etcd 140 | ${uniqueid.version} 141 | 142 | ``` 143 | 144 | ##### Preparing the Etcd cluster 145 | 146 | To use this method of generator-ID acquisition, a namespace on the Etcd cluster must 147 | be chosen to hold the resource pool used by `SynchronizedGeneratorIdentity`. 148 | 149 | For example, if you choose `unique-id-generator/` as the namespace, these keys can be 150 | automatically created when the library is used: 151 | 152 | ``` 153 | unique-id-generator/cluster-id 154 | unique-id-generator/pool/0 155 | unique-id-generator/pool/1 156 | … 157 | unique-id-generator/pool/255 158 | ``` 159 | 160 | Note that if you do not create the `cluster-id` key yourself (recommended), the default 161 | value of `0` will be used. To use a different cluster ID, set the content of this key to 162 | one of the 16 permissible values (i.e., `0..15`). 163 | 164 | If you have access to the `etcdctl` command line utility you can set the 165 | cluster-ID like so: 166 | 167 | ``` 168 | etcdctl --endpoints=… put unique-id-generator/cluster-id 1 169 | ``` 170 | 171 | ##### Using the generator 172 | 173 | To use an `IDGenerator` with a negotiated generator-Id, create a new instance like this: 174 | 175 | ```java 176 | // Change the values of etcdCluster and namespace as needed: 177 | final List etcdCluster = Arrays.asList("https://etcd1:2379","https://etcd2:2379","https://etcd3:2379"); 178 | final String namespace = "unique-id-generator/"; 179 | final ByteSequence ns = ByteSequence.from(namespace, StandardCharsets.UTF_8); 180 | final Client client = Client.builder() 181 | .endpoints(etcdCluster) 182 | .namespace(ns) 183 | .build(); 184 | IDGenerator generator = SynchronizedUniqueIDGeneratorFactory.generatorFor(client, Mode.SPREAD); 185 | // ... 186 | byte[] id = generator.generate() 187 | // ... 188 | ``` 189 | 190 | If you expect that you will be using dozens of IDs in a single process, it is more 191 | efficient to generate IDs in batches: 192 | 193 | ```java 194 | Deque ids = generator.batch(500); 195 | // ... 196 | byte[] id = ids.pop(); 197 | // etc. 198 | ``` 199 | 200 | If you intend to generate more than a few IDs at a time, you can also wrap the generator in 201 | an `AutoRefillStack`, and simply call `generate()` on that whenever you need a new ID. 202 | It will grab IDs in batches from the wrapped `IDGenerator` instance for you. This is 203 | probably the simplest and safest way to use an `IDGenerator` in the default `SPREAD` mode. 204 | 205 | ```java 206 | final List etcdCluster = Arrays.asList("https://etcd1:2379","https://etcd2:2379","https://etcd3:2379"); 207 | final String namespace = "unique-id-generator/"; 208 | final ByteSequence ns = ByteSequence.from(namespace, StandardCharsets.UTF_8); 209 | final Client client = Client.builder() 210 | .endpoints(etcdCluster) 211 | .namespace(ns) 212 | .build(); 213 | IDGenerator generator = new AutoRefillStack( 214 | SynchronizedUniqueIDGeneratorFactory.generatorFor(client, Mode.SPREAD) 215 | ); 216 | // ... 217 | byte[] id = generator.generate() 218 | // ... 219 | ``` 220 | 221 | For the `TIME_SEQUENTIAL` mode the above is usually not what you want, if you intend to use 222 | the timestamp stored in the generated ID as part of your data model (the batched pre-generated 223 | IDs might have a timestamp that lies further in the past then you might want). 224 | 225 | ```java 226 | final List etcdCluster = Arrays.asList("https://etcd1:2379","https://etcd2:2379","https://etcd3:2379"); 227 | final String namespace = "unique-id-generator/"; 228 | final ByteSequence ns = ByteSequence.from(namespace, StandardCharsets.UTF_8); 229 | final Client client = Client.builder() 230 | .endpoints(etcdCluster) 231 | .namespace(ns) 232 | .build(); 233 | IDGenerator generator = SynchronizedUniqueIDGeneratorFactory.generatorFor(client, Mode.TIME_SEQUENTIAL); 234 | // ... 235 | byte[] id = generator.generate() 236 | // Extract the timestamp in the ID. 237 | long createdAt = IDBuilder.parseTimestamp(id); 238 | ``` 239 | 240 | 241 | #### With a ZooKeeper quorum 242 | 243 | For ZooKeeper the `uniqueid-zookeeper` module is used: 244 | 245 | ```xml 246 | 247 | org.lable.oss.uniqueid 248 | uniqueid-zookeeper 249 | ${uniqueid.version} 250 | 251 | ``` 252 | 253 | ##### Preparing the ZooKeeper quorum 254 | 255 | To use this method of generator-ID acquisition, a node on the ZooKeeper quorum must 256 | be chosen to hold the queue and resource pool used by `SynchronizedGeneratorIdentity`. 257 | 258 | For example, if you choose `/unique-id-generator` as the node, these child nodes will be 259 | created: 260 | 261 | ``` 262 | /unique-id-generator/ 263 | ├─ queue/ 264 | ├─ pool/ 265 | └─ cluster-id 266 | ``` 267 | 268 | Note that if you do not create the `cluster-id` node yourself (recommended), the default 269 | value of `0` will be used. To use a different cluster ID, set the content of this znode to 270 | one of the 16 permissible values (i.e., `0..15`). 271 | 272 | If you have access to the `zkcli` (or `hbase zkcli`) command line utility you can set the 273 | cluster-ID like so: 274 | 275 | ``` 276 | create /unique-id-generator/cluster-id 1 277 | ``` 278 | 279 | Or if the node already exists: 280 | 281 | ``` 282 | set /unique-id-generator/cluster-id 1 283 | ``` 284 | 285 | ##### Using the generator 286 | 287 | To use an `IDGenerator` with a negotiated generator-Id, create a new instance like this: 288 | 289 | ```java 290 | // Change the values of zookeeperQuorum and znode as needed: 291 | final String zookeeperQuorum = "zookeeper1,zookeeper2,zookeeper3"; 292 | final String znode = "/unique-id-generator"; 293 | IDGenerator generator = SynchronizedUniqueIDGeneratorFactory.generatorFor(zookeeperQuorum, znode, Mode.SPREAD); 294 | // ... 295 | byte[] id = generator.generate() 296 | // ... 297 | ``` 298 | 299 | If you expect that you will be using dozens of IDs in a single process, it is more 300 | efficient to generate IDs in batches: 301 | 302 | ```java 303 | Deque ids = generator.batch(500); 304 | // ... 305 | byte[] id = ids.pop(); 306 | // etc. 307 | ``` 308 | 309 | If you intend to generate more than a few IDs at a time, you can also wrap the generator in 310 | an `AutoRefillStack`, and simply call `generate()` on that whenever you need a new ID. 311 | It will grab IDs in batches from the wrapped `IDGenerator` instance for you. This is 312 | probably the simplest and safest way to use an `IDGenerator` in the default `SPREAD` mode. 313 | 314 | ```java 315 | final String zookeeperQuorum = "zookeeper1,zookeeper2,zookeeper3"; 316 | final String znode = "/unique-id-generator"; 317 | IDGenerator generator = new AutoRefillStack( 318 | SynchronizedUniqueIDGeneratorFactory.generatorFor(zookeeperQuorum, znode, Mode.SPREAD) 319 | ); 320 | // ... 321 | byte[] id = generator.generate() 322 | // ... 323 | ``` 324 | 325 | For the `TIME_SEQUENTIAL` mode the above is usually not what you want, if you intend to use 326 | the timestamp stored in the generated ID as part of your data model (the batched pre-generated 327 | IDs might have a timestamp that lies further in the past then you might want). 328 | 329 | ```java 330 | final String zookeeperQuorum = "zookeeper1,zookeeper2,zookeeper3"; 331 | final String znode = "/unique-id-generator"; 332 | IDGenerator generator = SynchronizedUniqueIDGeneratorFactory.generatorFor(zookeeperQuorum, znode, Mode.TIME_SEQUENTIAL); 333 | // ... 334 | byte[] id = generator.generate() 335 | // Extract the timestamp in the ID. 336 | long createdAt = IDBuilder.parseTimestamp(id); 337 | ``` 338 | -------------------------------------------------------------------------------- /doc/eight-byte-id-structure.md: -------------------------------------------------------------------------------- 1 | Structure of the eight byte unique IDs 2 | ====================================== 3 | 4 | The eight byte IDs generated by the `LocalUniqueIDGenerator` and 5 | `SynchronizedUniqueIDGenerator` classes conform to the following structure: 6 | 7 | ``` 8 | TTTTTTTT TTTTTTTT TTTTTTTT TTTTTTTT TTTTTTTT TTSSSSSS GGGMGGGG GGGGCCCC 9 | | | | || | 10 | v | | |\ | 11 | | v \ \ | 12 | Date of ID creation, measured in milliseconds | \ \ | 13 | ellapsed since 1970-01-01T00:00:00.000, | Generator \ \ | 14 | represented in reverse byte-order (in SPREAD | ID, see | | | 15 | mode) to guarantee an even spread of IDs. | below. | | | 16 | | | | | 17 | v | | | 18 | | | | 19 | Sequence. This field is incremented by the | | | 20 | generator for each ID generated during the | | | 21 | same timestamp. | | | 22 | | | | 23 | v | | 24 | | | 25 | Mode flag. | | 26 | | | 27 | v | 28 | | 29 | Generator ID, manually assigned, or acquired | 30 | through negotiation via Etcd. Limited to 2048 | 31 | per Cluster ID. | 32 | v 33 | 34 | Cluster ID. Always manually assigned 35 | or statically configured. Limited to 36 | 16 (active) clusters. 37 | ``` 38 | 39 | This approach offers a few useful features: 40 | 41 | * Possible to generate a unique ID, in a computing environment that can have up to 256 42 | processes generating IDs at the same time 43 | * Can generate IDs in isolation (e.g., on a backup database cluster, or off-line for 44 | maintenance) as long as there are no more than 16 of such isolated clusters 45 | * Can generate unique IDs at a decent rate (up to 64 IDs per millisecond, per generator) 46 | * IDs are lexicographically spread over the full byte range, making these IDs suitable for 47 | use in NoSQL or other key-value data stores as row/object keys. 48 | * Very short at only 8 bytes, and as such suitable as part of longer compound IDs, such 49 | as those used in key-value data stores. 50 | 51 | ## Structure 52 | 53 | ### Timestamp 54 | 55 | The *timestamp* serves a dual purpose. Firstly, it helps guarantee uniqueness. As long as 56 | a generator has the exclusive use of a certain generator-ID/cluster-ID pair, and it uses the 57 | current time as the basis of the ID it is generating, it can be fairly sure that the 58 | resulting ID is unique. Secondly, the timestamp is a permanent record of the date of 59 | creation of the ID, which may be useful if is used as ID for a database record the moment 60 | it is created. 61 | 62 | Because of the limited bit-size of the timestamp, only dates until the year 2109 are supported. 63 | As this library generates only 'now' timestamps, this should not be practical limitation. 64 | 65 | ### Sequence counter 66 | 67 | To prevent the generators from being limited to one ID per millisecond, the *sequence counter* 68 | adds 6 bits to the address space so up to 64 IDs per millisecond can be generated by a single 69 | generator. This is useful when a batch of IDs is pre-generated. 70 | 71 | ### *Reserved for future use* 72 | 73 | Unused bits. These are always zero in the current implementation, but may be used for 74 | future revisions. 75 | 76 | ### Mode flag 77 | 78 | Shows which mode the ID was generated in: `SPREAD` or `TIME_SEQUENTIAL`. 79 | 80 | ### Generator IDs 81 | 82 | Multiple generators can simultaneously generate IDs, as long as every generator holds 83 | a unique generator-ID/cluster-ID combination. The generator ID becomes part of the 84 | generated ID. 85 | 86 | Within a single cluster generator IDs can either be assigned statically, or managed 87 | through a resource pool. There are 256 generator IDs available per cluster. 88 | 89 | ### Cluster IDs 90 | 91 | Usually, all your data resides within a single *cluster*. That is, a single computing 92 | environment wherein ID generation can be coordinated. The cluster ID enables you to 93 | generate IDs in separate clusters (or by means of an off-line operation separate from 94 | the primary cluster) that are unique across all clusters. 95 | 96 | In contrast to the generator ID, which may be automatically assigned from a resource 97 | pool, the cluster ID is assigned statically per cluster, with a maximum of 16 98 | (simultaneously) active clusters. 99 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 19 | 20 | 4.0.0 21 | 22 | 23 | org.lable.oss 24 | parent 25 | 2.0 26 | 27 | 28 | org.lable.oss.uniqueid 29 | uniqueid 30 | 4.7-SNAPSHOT 31 | pom 32 | 33 | UniqueID 34 | A unique ID generator that specialises in small IDs. 35 | 2014 36 | https://github.com/LableOrg/java-uniqueid 37 | 38 | 39 | uniqueid-core 40 | uniqueid-etcd 41 | 42 | 43 | 44 | 4.0 45 | 0.7.6 46 | 1.7.32 47 | 1.3.2 48 | 49 | 50 | 2.17.1 51 | 1.0 52 | 53 | 54 | 55 | 56 | The Apache License, Version 2.0 57 | http://www.apache.org/licenses/LICENSE-2.0 58 | 59 | 60 | 61 | 62 | scm:git:git@github.com:LableOrg/java-uniqueid.git 63 | scm:git:git@github.com:LableOrg/java-uniqueid.git 64 | https://github.com/LableOrg/java-uniqueid 65 | v4.5 66 | 67 | 68 | 69 | 70 | jdhoek 71 | Jeroen Hoek 72 | jeroen.hoek@lable.nl 73 | Lable 74 | http://lable.nl 75 | 76 | 77 | 78 | 79 | 80 | 81 | uniqueid-core 82 | org.lable.oss.uniqueid 83 | ${project.version} 84 | 85 | 86 | 87 | 88 | 89 | 90 | org.slf4j 91 | slf4j-api 92 | ${slf4j.version} 93 | 94 | 95 | 96 | org.apache.logging.log4j 97 | log4j-slf4j-impl 98 | ${log4j.version} 99 | test 100 | 101 | 102 | org.apache.logging.log4j 103 | log4j-core 104 | ${log4j.version} 105 | test 106 | 107 | 108 | org.apache.logging.log4j 109 | log4j-web 110 | ${log4j.version} 111 | test 112 | 113 | 114 | org.slf4j 115 | log4j-over-slf4j 116 | ${slf4j.version} 117 | test 118 | 119 | 120 | com.github.stefanbirkner 121 | system-rules 122 | 1.4.0 123 | test 124 | 125 | 126 | commons-codec 127 | commons-codec 128 | 1.9 129 | test 130 | 131 | 132 | 133 | 134 | 135 | 136 | maven-failsafe-plugin 137 | 138 | 139 | 140 | com.mycila 141 | license-maven-plugin 142 | 143 | 144 | 145 | org.sonatype.central 146 | central-publishing-maven-plugin 147 | 0.7.0 148 | true 149 | 150 | central 151 | true 152 | 153 | 154 | 155 | 156 | 157 | -------------------------------------------------------------------------------- /uniqueid-core/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 19 | 20 | 21 | uniqueid 22 | org.lable.oss.uniqueid 23 | 4.7-SNAPSHOT 24 | 25 | 4.0.0 26 | 27 | uniqueid-core 28 | UniqueID :: Core 29 | 30 | 31 | 32 | javax.annotation 33 | javax.annotation-api 34 | ${javax.annotation.version} 35 | 36 | 37 | 38 | 39 | 40 | 41 | maven-failsafe-plugin 42 | 43 | 44 | 45 | org.jacoco 46 | jacoco-maven-plugin 47 | 48 | 49 | 50 | org.apache.maven.plugins 51 | maven-jar-plugin 52 | 2.6 53 | 54 | 55 | 56 | test-jar 57 | 58 | 59 | 60 | 61 | 62 | 63 | -------------------------------------------------------------------------------- /uniqueid-core/src/main/java/org/lable/oss/uniqueid/AutoRefillStack.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2014 Lable (info@lable.nl) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.lable.oss.uniqueid; 17 | 18 | import java.io.IOException; 19 | import java.util.ArrayDeque; 20 | import java.util.Deque; 21 | import java.util.NoSuchElementException; 22 | 23 | /** 24 | * A caching wrapper around an {@link IDGenerator} instance. 25 | *

26 | * This class will cache a bunch of generated IDs and automatically refill the stack when it runs out. By letting 27 | * this class handle the caching, calling classes can simply call {@link #generate()} whenever a new ID is needed, 28 | * without having to worry about any performance hit you might see when calling 29 | * {@link IDGenerator#generate()} repeatedly from a time-consuming loop. 30 | */ 31 | public class AutoRefillStack implements IDGenerator { 32 | 33 | static final int DEFAULT_BATCH_SIZE = 500; 34 | 35 | final int batchSize; 36 | final IDGenerator generator; 37 | final Deque idStack = new ArrayDeque<>(); 38 | 39 | protected AutoRefillStack(IDGenerator generator, int batchSize) { 40 | this.batchSize = batchSize; 41 | this.generator = generator; 42 | } 43 | 44 | /** 45 | * Wrap an {@link IDGenerator} in an AutoRefillStack, with a default batch-size. 46 | * 47 | * @param generator Generator to decorate. 48 | * @return The decorated generator. 49 | */ 50 | public static IDGenerator decorate(IDGenerator generator) { 51 | return new AutoRefillStack(generator, DEFAULT_BATCH_SIZE); 52 | } 53 | 54 | /** 55 | * Wrap an {@link IDGenerator} in an AutoRefillStack, with a specific batch size. 56 | * 57 | * @param generator Generator to decorate. 58 | * @param batchSize The amount of IDs to cache. 59 | * @return The decorated generator. 60 | */ 61 | public static IDGenerator decorate(IDGenerator generator, int batchSize) { 62 | return new AutoRefillStack(generator, batchSize); 63 | } 64 | 65 | @Override 66 | public void close() throws IOException { 67 | generator.close(); 68 | } 69 | 70 | /** 71 | * {@inheritDoc} 72 | */ 73 | @Override 74 | public synchronized byte[] generate() throws GeneratorException { 75 | return popOne(); 76 | } 77 | 78 | /** 79 | * {@inheritDoc} 80 | */ 81 | @Override 82 | public synchronized Deque batch(int size) throws GeneratorException { 83 | if (size < 0) { 84 | size = 0; 85 | } 86 | Deque batch = new ArrayDeque<>(size); 87 | while (size > 0) { 88 | batch.add(popOne()); 89 | size--; 90 | } 91 | return batch; 92 | } 93 | 94 | /** 95 | * Grab a single ID from the stack. If the stack is empty, load up a new batch from the wrapped generator. 96 | * 97 | * @return A single ID. 98 | */ 99 | byte[] popOne() throws GeneratorException { 100 | try { 101 | return idStack.pop(); 102 | } catch (NoSuchElementException e) { 103 | // Cached stack is empty, load up a fresh stack. 104 | idStack.addAll(generator.batch(batchSize)); 105 | return popOne(); 106 | } 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /uniqueid-core/src/main/java/org/lable/oss/uniqueid/BaseUniqueIDGenerator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2014 Lable (info@lable.nl) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.lable.oss.uniqueid; 17 | 18 | import org.lable.oss.uniqueid.bytes.Blueprint; 19 | import org.lable.oss.uniqueid.bytes.IDBuilder; 20 | import org.lable.oss.uniqueid.bytes.Mode; 21 | 22 | import java.io.IOException; 23 | import java.util.ArrayDeque; 24 | import java.util.Deque; 25 | import java.util.concurrent.TimeUnit; 26 | 27 | /** 28 | * Generate short, possibly unique IDs based on the current timestamp. 29 | *

30 | * Whether the IDs are truly unique or not depends on the scope of its use. If the combination of generator-ID and 31 | * cluster-ID passed to this class is unique — i.e., there is only one ID-generator using that specific combination of 32 | * generator-ID and cluster-ID within the confines of your computing environment at the moment you generate an ID — 33 | * then the IDs returned are unique. 34 | */ 35 | public class BaseUniqueIDGenerator implements IDGenerator { 36 | protected final GeneratorIdentityHolder generatorIdentityHolder; 37 | private final Clock clock; 38 | private final Mode mode; 39 | 40 | long previousTimestamp = 0; 41 | int sequence = 0; 42 | 43 | /** 44 | * Create a new UniqueIDGenerator instance. 45 | * 46 | * @param generatorIdentityHolder Generator identity holder. 47 | * @param mode Generator mode. 48 | */ 49 | public BaseUniqueIDGenerator(GeneratorIdentityHolder generatorIdentityHolder, 50 | Mode mode) { 51 | this(generatorIdentityHolder, null, mode); 52 | } 53 | 54 | /** 55 | * Create a new UniqueIDGenerator instance. 56 | * 57 | * @param generatorIdentityHolder Generator identity holder. 58 | * @param clock System clock (optional; useful for tests). 59 | * @param mode Generator mode. 60 | */ 61 | public BaseUniqueIDGenerator(GeneratorIdentityHolder generatorIdentityHolder, 62 | Clock clock, 63 | Mode mode) { 64 | this.generatorIdentityHolder = generatorIdentityHolder; 65 | // Fall back to the default wall clock if no alternative is passed. 66 | this.clock = clock == null ? System::currentTimeMillis : clock; 67 | this.mode = mode == null ? Mode.defaultMode() : mode; 68 | } 69 | 70 | /** 71 | * {@inheritDoc} 72 | */ 73 | @Override 74 | public synchronized byte[] generate() throws GeneratorException { 75 | return generate(0); 76 | } 77 | 78 | synchronized byte[] generate(int attempt) throws GeneratorException { 79 | // To prevent the generator from becoming stuck in a loop when the supplied clock 80 | // doesn't progress, this safety valve will trigger after waiting too long for the 81 | // next clock tick. 82 | if (attempt > 10) throw new GeneratorException("Clock supplied to generator failed to progress."); 83 | 84 | long now = clock.currentTimeMillis(); 85 | if (now == previousTimestamp) { 86 | sequence++; 87 | } else { 88 | sequence = 0; 89 | } 90 | if (sequence > Blueprint.MAX_SEQUENCE_COUNTER) { 91 | try { 92 | TimeUnit.MICROSECONDS.sleep(400); 93 | return generate(attempt + 1); 94 | } catch (InterruptedException e) { 95 | Thread.currentThread().interrupt(); 96 | } 97 | } 98 | previousTimestamp = now; 99 | 100 | Blueprint blueprint = new Blueprint( 101 | now, 102 | sequence, 103 | generatorIdentityHolder.getGeneratorId(), 104 | generatorIdentityHolder.getClusterId(), 105 | mode 106 | ); 107 | 108 | return IDBuilder.build(blueprint); 109 | } 110 | 111 | /** 112 | * {@inheritDoc} 113 | */ 114 | @Override 115 | public Deque batch(int size) throws GeneratorException { 116 | Deque stack = new ArrayDeque<>(); 117 | for (int i = 0; i < size; i++) { 118 | stack.add(generate()); 119 | } 120 | return stack; 121 | } 122 | 123 | @Override 124 | public void close() throws IOException { 125 | generatorIdentityHolder.close(); 126 | } 127 | } 128 | -------------------------------------------------------------------------------- /uniqueid-core/src/main/java/org/lable/oss/uniqueid/Clock.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2014 Lable (info@lable.nl) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.lable.oss.uniqueid; 17 | 18 | /** 19 | * Abstraction for the clock implementation. This allows for use of this library in deterministic systems and tests. 20 | *

21 | * Implementation note: clocks should at a minimum progress once every millisecond. 22 | */ 23 | @FunctionalInterface 24 | public interface Clock { 25 | /** 26 | * @return The current time in milliseconds. 27 | */ 28 | long currentTimeMillis(); 29 | } 30 | -------------------------------------------------------------------------------- /uniqueid-core/src/main/java/org/lable/oss/uniqueid/GeneratorException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2014 Lable (info@lable.nl) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.lable.oss.uniqueid; 17 | 18 | /** 19 | * General exception throwable by the public API of this project. 20 | */ 21 | public class GeneratorException extends Exception { 22 | public GeneratorException(String message) { 23 | super(message); 24 | } 25 | 26 | public GeneratorException(String message, Throwable cause) { 27 | super(message, cause); 28 | } 29 | 30 | public GeneratorException() { 31 | super(); 32 | } 33 | 34 | public GeneratorException(Throwable cause) { 35 | super(cause); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /uniqueid-core/src/main/java/org/lable/oss/uniqueid/GeneratorIdentityHolder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2014 Lable (info@lable.nl) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.lable.oss.uniqueid; 17 | 18 | import java.io.Closeable; 19 | 20 | public interface GeneratorIdentityHolder extends Closeable { 21 | int getClusterId() throws GeneratorException; 22 | int getGeneratorId() throws GeneratorException; 23 | } 24 | -------------------------------------------------------------------------------- /uniqueid-core/src/main/java/org/lable/oss/uniqueid/IDGenerator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2014 Lable (info@lable.nl) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.lable.oss.uniqueid; 17 | 18 | import java.io.Closeable; 19 | import java.util.Deque; 20 | 21 | /** 22 | * Generate short, unique IDs. 23 | */ 24 | public interface IDGenerator extends Closeable { 25 | /** 26 | * Generate a fresh ID. 27 | * 28 | * @return The generated ID. 29 | * @throws GeneratorException Thrown when an ID could not be generated. In practice, this exception is usually only 30 | * thrown by the more complex implementations of {@link IDGenerator}. 31 | */ 32 | byte[] generate() throws GeneratorException; 33 | 34 | /** 35 | * Generate a batch of IDs. This is the preferred way of generating IDs when you expect to use more than a few IDs. 36 | * 37 | * @param size How many IDs to generate, implementing classes may decide to limit the maximum number of IDs 38 | * generated at a time. 39 | * @return A stack of IDs, containing {@code size} or fewer IDs. 40 | * @throws GeneratorException Thrown when an ID could not be generated. In practice, this exception is usually only 41 | * thrown by the more complex implementations of {@link IDGenerator}. 42 | */ 43 | Deque batch(int size) throws GeneratorException; 44 | } 45 | -------------------------------------------------------------------------------- /uniqueid-core/src/main/java/org/lable/oss/uniqueid/LocalGeneratorIdentity.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2014 Lable (info@lable.nl) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.lable.oss.uniqueid; 17 | 18 | import org.lable.oss.uniqueid.bytes.Blueprint; 19 | 20 | import java.io.IOException; 21 | 22 | import static org.lable.oss.uniqueid.ParameterUtil.assertParameterWithinBounds; 23 | 24 | public class LocalGeneratorIdentity implements GeneratorIdentityHolder { 25 | private final int clusterId; 26 | private final int generatorId; 27 | private boolean closed = false; 28 | 29 | LocalGeneratorIdentity(int clusterId, int generatorId) { 30 | this.clusterId = clusterId; 31 | this.generatorId = generatorId; 32 | } 33 | 34 | public static LocalGeneratorIdentity with(int clusterId, int generatorId) { 35 | assertParameterWithinBounds("generatorId", 0, Blueprint.MAX_GENERATOR_ID, generatorId); 36 | assertParameterWithinBounds("clusterId", 0, Blueprint.MAX_CLUSTER_ID, clusterId); 37 | return new LocalGeneratorIdentity(clusterId, generatorId); 38 | } 39 | 40 | @Override 41 | public int getClusterId() { 42 | if (closed) throw new IllegalStateException("Resource was closed."); 43 | return clusterId; 44 | } 45 | 46 | @Override 47 | public int getGeneratorId() { 48 | if (closed) throw new IllegalStateException("Resource was closed."); 49 | return generatorId; 50 | } 51 | 52 | @Override 53 | public void close() throws IOException { 54 | closed = true; 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /uniqueid-core/src/main/java/org/lable/oss/uniqueid/LocalUniqueIDGeneratorFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2014 Lable (info@lable.nl) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.lable.oss.uniqueid; 17 | 18 | import org.lable.oss.uniqueid.bytes.Blueprint; 19 | import org.lable.oss.uniqueid.bytes.Mode; 20 | 21 | import java.util.HashMap; 22 | import java.util.Map; 23 | 24 | import static org.lable.oss.uniqueid.ParameterUtil.assertParameterWithinBounds; 25 | 26 | /** 27 | * Create an {@link IDGenerator} that generates short, possibly unique IDs based on the current timestamp. Whether the 28 | * IDs are truly unique or not depends on the scope of use; if the combination of generator-ID and cluster-ID passed 29 | * to this class is unique (i.e., there is only one ID-generator using that specific combination of generator-ID and 30 | * cluster-ID within the confines of your computing environment at the moment you generate an ID) then the IDs 31 | * returned are unique. 32 | */ 33 | public class LocalUniqueIDGeneratorFactory { 34 | final static Map instances = new HashMap<>(); 35 | 36 | /** 37 | * Return the UniqueIDGenerator instance for this specific generator-ID, cluster-ID combination. If one was 38 | * already created, that is returned. 39 | * 40 | * @param generatorId Generator ID to use (0 ≤ n ≤ 2047). 41 | * @param clusterId Cluster ID to use (0 ≤ n ≤ 15). 42 | * @param clock Clock implementation. 43 | * @param mode Generator mode. 44 | * @return A thread-safe UniqueIDGenerator instance. 45 | */ 46 | public synchronized static IDGenerator generatorFor(int generatorId, int clusterId, Clock clock, Mode mode) { 47 | assertParameterWithinBounds("generatorId", 0, Blueprint.MAX_GENERATOR_ID, generatorId); 48 | assertParameterWithinBounds("clusterId", 0, Blueprint.MAX_CLUSTER_ID, clusterId); 49 | String generatorAndCluster = String.format("%d_%d", generatorId, clusterId); 50 | if (!instances.containsKey(generatorAndCluster)) { 51 | GeneratorIdentityHolder identityHolder = LocalGeneratorIdentity.with(clusterId, generatorId); 52 | instances.putIfAbsent(generatorAndCluster, new BaseUniqueIDGenerator(identityHolder, clock, mode)); 53 | } 54 | return instances.get(generatorAndCluster); 55 | } 56 | 57 | /** 58 | * Return the UniqueIDGenerator instance for this specific generator-ID, cluster-ID combination. If one was 59 | * already created, that is returned. 60 | * 61 | * @param generatorId Generator ID to use (0 ≤ n ≤ 2047). 62 | * @param clusterId Cluster ID to use (0 ≤ n ≤ 15). 63 | * @param mode Generator mode. 64 | * @return A thread-safe UniqueIDGenerator instance. 65 | */ 66 | public synchronized static IDGenerator generatorFor(int generatorId, int clusterId, Mode mode) { 67 | return generatorFor(generatorId, clusterId, null, mode); 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /uniqueid-core/src/main/java/org/lable/oss/uniqueid/OnePerMillisecondDecorator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2014 Lable (info@lable.nl) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.lable.oss.uniqueid; 17 | 18 | import java.io.IOException; 19 | import java.util.ArrayDeque; 20 | import java.util.Deque; 21 | import java.util.concurrent.TimeUnit; 22 | 23 | /** 24 | * Decorator for an {@link IDGenerator} that sleeps at least a millisecond between each invocation to guarantee ID 25 | * spread. 26 | *

27 | * This is not normally necessary nor desired, but can be useful when you want to generate several IDs, but you don't 28 | * want subsequent IDs to start with the same byte. 29 | *

30 | * This class is of course significantly slower than using an undecorated generator. 31 | */ 32 | public class OnePerMillisecondDecorator implements IDGenerator { 33 | final IDGenerator generator; 34 | long previousInvocation = 0; 35 | byte[] previous = null; 36 | 37 | protected OnePerMillisecondDecorator(IDGenerator generator) { 38 | this.generator = generator; 39 | } 40 | 41 | /** 42 | * Wrap an {@link IDGenerator} in a OnePerMillisecondDecorator. 43 | * 44 | * @param generator Generator to decorate. 45 | * @return The decorated generator. 46 | */ 47 | public static IDGenerator decorate(IDGenerator generator) { 48 | return new OnePerMillisecondDecorator(generator); 49 | } 50 | 51 | @Override 52 | public void close() throws IOException { 53 | generator.close(); 54 | } 55 | 56 | @Override 57 | public byte[] generate() throws GeneratorException { 58 | // Wait a millisecond (or two) until the current timestamp is not the same as the next. 59 | // Because the first byte is the last byte (reversed) of the current timestamp, the timestamps 60 | // have to differ to guarantee a different byte there. 61 | long now = System.currentTimeMillis(); 62 | while (previousInvocation == now) { 63 | sleepAMillisecond(); 64 | now = System.currentTimeMillis(); 65 | } 66 | previousInvocation = now; 67 | 68 | // The above trick fails in rare cases, so perform an additional check to guarantee the desired 69 | // result. 70 | byte[] id = generator.generate(); 71 | if (previous != null) { 72 | while (previous[0] == id[0]) { 73 | sleepAMillisecond(); 74 | id = generator.generate(); 75 | } 76 | } 77 | 78 | previous = id; 79 | return id; 80 | } 81 | 82 | private void sleepAMillisecond() { 83 | try { 84 | TimeUnit.MILLISECONDS.sleep(1); 85 | } catch (InterruptedException e) { 86 | Thread.currentThread().interrupt(); 87 | } 88 | } 89 | 90 | @Override 91 | public Deque batch(int size) throws GeneratorException { 92 | Deque deck = new ArrayDeque<>(); 93 | for (int i = 0; i < size; i++) { 94 | deck.add(generate()); 95 | } 96 | return deck; 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /uniqueid-core/src/main/java/org/lable/oss/uniqueid/ParameterUtil.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2014 Lable (info@lable.nl) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.lable.oss.uniqueid; 17 | 18 | /** 19 | * Parameter validation helpers. 20 | */ 21 | public class ParameterUtil { 22 | /** 23 | * Throw an {@link IllegalArgumentException} when a number is not within the supplied range. 24 | * 25 | * @param name Name of the parameter to use in the Exception message. 26 | * @param lower Lower bound (inclusive). 27 | * @param upper Upper bound (inclusive). 28 | * @param parameter The parameter to test. 29 | * @throws IllegalArgumentException Thrown when the parameter is out of bounds. 30 | */ 31 | public static void assertParameterWithinBounds(String name, long lower, long upper, long parameter) { 32 | if (parameter < lower || parameter > upper) { 33 | throw new IllegalArgumentException(String.format("Invalid %s: %d (expected: %d <= n < %d)", 34 | name, parameter, lower, upper + 1)); 35 | } 36 | } 37 | 38 | /** 39 | * Thrown an {@link IllegalArgumentException} when the byte array does not contain exactly eight bytes. 40 | * 41 | * @param bytes Byte array. 42 | */ 43 | public static void assertNotNullEightBytes(byte[] bytes) { 44 | if (bytes == null) { 45 | throw new IllegalArgumentException("Expected 8 bytes, but got null."); 46 | } 47 | if (bytes.length != 8) { 48 | throw new IllegalArgumentException(String.format("Expected 8 bytes, but got: %d bytes.", bytes.length)); 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /uniqueid-core/src/main/java/org/lable/oss/uniqueid/bytes/Blueprint.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2014 Lable (info@lable.nl) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.lable.oss.uniqueid.bytes; 17 | 18 | import static org.lable.oss.uniqueid.ParameterUtil.assertParameterWithinBounds; 19 | 20 | /** 21 | * Contains all parameters required to build the ID. 22 | */ 23 | public class Blueprint { 24 | /** 25 | * Maximum timestamp, this represents a date somewhen in 2109. 26 | */ 27 | public final static long MAX_TIMESTAMP = 0x3FFFFFFFFFFL; 28 | 29 | /** 30 | * IDs using the same timestamp are limited to 64 variations. 31 | */ 32 | public final static int MAX_SEQUENCE_COUNTER = 63; 33 | 34 | /** 35 | * Upper bound (inclusive) of the generator-ID. 36 | */ 37 | public final static int MAX_GENERATOR_ID = 2047; 38 | 39 | /** 40 | * Upper bound (inclusive) of the cluster-ID. 41 | */ 42 | public final static int MAX_CLUSTER_ID = 15; 43 | 44 | final long timestamp; 45 | final int sequence; 46 | final int generatorId; 47 | final int clusterId; 48 | final Mode mode; 49 | 50 | /** 51 | * Create a blueprint for a unique ID with the default mode of {@link Mode#SPREAD}. 52 | * 53 | * @param timestamp Milliseconds since the Unix epoch. 54 | * @param sequence Sequence counter. 55 | * @param generatorId Generator ID. 56 | * @param clusterId Cluster ID. 57 | * @see #MAX_CLUSTER_ID 58 | * @see #MAX_GENERATOR_ID 59 | */ 60 | public Blueprint(long timestamp, int sequence, int generatorId, int clusterId) { 61 | this(timestamp, sequence, generatorId, clusterId, Mode.SPREAD); 62 | } 63 | 64 | /** 65 | * Create a blueprint for a unique ID. 66 | * 67 | * @param timestamp Milliseconds since the Unix epoch. 68 | * @param sequence Sequence counter. 69 | * @param generatorId Generator ID. 70 | * @param clusterId Cluster ID. 71 | * @param mode Mode to use. 72 | * @see #MAX_CLUSTER_ID 73 | * @see #MAX_GENERATOR_ID 74 | * @see Mode 75 | */ 76 | public Blueprint(long timestamp, int sequence, int generatorId, int clusterId, Mode mode) { 77 | assertParameterWithinBounds("timestamp", 0, MAX_TIMESTAMP, timestamp); 78 | assertParameterWithinBounds("sequence counter", 0, MAX_SEQUENCE_COUNTER, sequence); 79 | assertParameterWithinBounds("generator-ID", 0, MAX_GENERATOR_ID, generatorId); 80 | assertParameterWithinBounds("cluster-ID", 0, MAX_CLUSTER_ID, clusterId); 81 | 82 | this.timestamp = timestamp; 83 | this.sequence = sequence; 84 | this.generatorId = generatorId; 85 | this.clusterId = clusterId; 86 | this.mode = mode == null ? Mode.SPREAD : mode; 87 | } 88 | 89 | /** 90 | * @return The timestamp. 91 | */ 92 | public long getTimestamp() { 93 | return timestamp; 94 | } 95 | 96 | /** 97 | * @return The sequence counter, incremented in case more than one ID was generated in the same millisecond. 98 | */ 99 | public int getSequence() { 100 | return sequence; 101 | } 102 | 103 | /** 104 | * @return ID of the generating instance. 105 | */ 106 | public int getGeneratorId() { 107 | return generatorId; 108 | } 109 | 110 | /** 111 | * @return ID of the cluster this ID was generated on. 112 | */ 113 | public int getClusterId() { 114 | return clusterId; 115 | } 116 | 117 | /** 118 | * @return The ID mode chosen. 119 | */ 120 | public Mode getMode() { 121 | return mode; 122 | } 123 | 124 | @Override 125 | public String toString() { 126 | return String.format( 127 | "{\n mode: %s,\n timestamp: %d,\n sequence: %d,\n generator: %d,\n cluster: %d\n}", 128 | mode, timestamp, sequence, generatorId, clusterId 129 | ); 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /uniqueid-core/src/main/java/org/lable/oss/uniqueid/bytes/IDBuilder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2014 Lable (info@lable.nl) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.lable.oss.uniqueid.bytes; 17 | 18 | import java.nio.ByteBuffer; 19 | 20 | import static org.lable.oss.uniqueid.ParameterUtil.assertNotNullEightBytes; 21 | 22 | /** 23 | * Composes and deconstructs the special eight byte identifiers generated by this library. 24 | *

25 | * The eight byte ID is composed as follows: 26 | * 27 | * 

TTTTTTTT TTTTTTTT TTTTTTTT TTTTTTTT TTTTTTTT TTSSSSSS GGGMGGGG GGGGCCCC
28 | * 29 | *
    30 | *
  • T: Timestamp (in milliseconds, bit order depends on mode) 31 | *
  • S: Sequence counter 32 | *
  • .: Reserved for future use 33 | *
  • M: Mode 34 | *
  • G: Generator ID 35 | *
  • C: Cluster ID 36 | *
37 | * 38 | * Because only 42 bits are assigned to represent the timestamp in the generated ID, the timestamp used must take place 39 | * between the Unix epoch (1970-01-01T00:00:00.000 UTC) and 2109. 40 | */ 41 | public class IDBuilder { 42 | /** 43 | * Perform all the byte mangling needed to create the eight byte ID. 44 | * 45 | * @param blueprint Blueprint containing all needed data to work with. 46 | * @return The 8-byte ID. 47 | */ 48 | public static byte[] build(Blueprint blueprint) { 49 | // First 42 bits are the timestamp. 50 | // [0] TTTTTTTT [1] TTTTTTTT [2] TTTTTTTT [3] TTTTTTTT [4] TTTTTTTT [5] TT...... 51 | ByteBuffer bb = ByteBuffer.allocate(8); 52 | switch (blueprint.getMode()) { 53 | case SPREAD: 54 | long reverseTimestamp = Long.reverse(blueprint.getTimestamp()); 55 | bb.putLong(reverseTimestamp); 56 | break; 57 | case TIME_SEQUENTIAL: 58 | long timestamp = blueprint.getTimestamp(); 59 | bb.putLong(timestamp << 22); 60 | break; 61 | } 62 | byte[] tsBytes = bb.array(); 63 | 64 | // Last 6 bits of byte 6 are for the sequence counter. The first two bits are from the timestamp. 65 | // [5] TTSSSSSS 66 | int or = tsBytes[5] | (byte) blueprint.getSequence(); 67 | tsBytes[5] = (byte) or; 68 | 69 | // Last two bytes. The mode flag, generator ID, and cluster ID. 70 | // [6] GGGMGGGG [7] GGGGCCCC 71 | int flagGeneratorCluster = (blueprint.getGeneratorId() << 5) & 0xE000; 72 | flagGeneratorCluster += (blueprint.getGeneratorId() & 0x00FF) << 4; 73 | flagGeneratorCluster += blueprint.getClusterId(); 74 | flagGeneratorCluster += blueprint.getMode().getModeMask() << 12; 75 | 76 | tsBytes[7] = (byte) flagGeneratorCluster; 77 | flagGeneratorCluster >>>= 8; 78 | tsBytes[6] = (byte) flagGeneratorCluster; 79 | 80 | return tsBytes; 81 | } 82 | 83 | /** 84 | * Decompose a generated ID into its {@link Blueprint}. 85 | * 86 | * @param id Eight byte ID to parse. 87 | * @return A blueprint containing the four ID components. 88 | */ 89 | public static Blueprint parse(byte[] id) { 90 | assertNotNullEightBytes(id); 91 | 92 | int sequence = parseSequenceIdNoChecks(id); 93 | int generatorId = parseGeneratorIdNoChecks(id); 94 | int clusterId = parseClusterIdNoChecks(id); 95 | long timestamp = parseTimestampNoChecks(id); 96 | Mode mode = parseModeNoChecks(id); 97 | 98 | return new Blueprint(timestamp, sequence, generatorId, clusterId, mode); 99 | } 100 | 101 | /** 102 | * Find the sequence number in an identifier. 103 | * 104 | * @param id Identifier. 105 | * @return The sequence number, if {@code id} is a byte array with length eight. 106 | */ 107 | public static int parseSequenceId(byte[] id) { 108 | assertNotNullEightBytes(id); 109 | return parseSequenceIdNoChecks(id); 110 | } 111 | 112 | /** 113 | * Find the generator id in an identifier. 114 | * 115 | * @param id Identifier. 116 | * @return The generator id, if {@code id} is a byte array with length eight. 117 | */ 118 | public static int parseGeneratorId(byte[] id) { 119 | assertNotNullEightBytes(id); 120 | return parseGeneratorIdNoChecks(id); 121 | } 122 | 123 | /** 124 | * Find the cluster id in an identifier. 125 | * 126 | * @param id Identifier. 127 | * @return The cluster id, if {@code id} is a byte array with length eight. 128 | */ 129 | public static int parseClusterId(byte[] id) { 130 | assertNotNullEightBytes(id); 131 | return parseClusterIdNoChecks(id); 132 | } 133 | 134 | /** 135 | * Find the timestamp in an identifier. 136 | * 137 | * @param id Identifier. 138 | * @return The timestamp, if {@code id} is a byte array with length eight. 139 | */ 140 | public static long parseTimestamp(byte[] id) { 141 | assertNotNullEightBytes(id); 142 | return parseTimestampNoChecks(id); 143 | } 144 | 145 | /** 146 | * Find the ID mode used to construct the identifier. 147 | * 148 | * @param id Identifier. 149 | * @return The {@link Mode}, if {@code id} is a byte array with length eight. 150 | */ 151 | public static Mode parseMode(byte[] id) { 152 | assertNotNullEightBytes(id); 153 | return parseModeNoChecks(id); 154 | } 155 | 156 | // The private methods skip the null and length check on the id, because the method calling them took care of that. 157 | 158 | private static int parseSequenceIdNoChecks(byte[] id) { 159 | // [5] ..SSSSSS 160 | return id[5] & 0x3F; 161 | } 162 | 163 | private static int parseGeneratorIdNoChecks(byte[] id) { 164 | // [6] GGG.GGGG [7] GGGG.... 165 | return (id[7] >> 4 & 0x0F) | (id[6] << 3 & 0x0700) | (id[6] << 4 & 0xF0); 166 | } 167 | 168 | private static int parseClusterIdNoChecks(byte[] id) { 169 | // [7] ....CCCC 170 | return id[7] & 0x0F; 171 | } 172 | 173 | private static long parseTimestampNoChecks(byte[] id) { 174 | Mode mode = parseModeNoChecks(id); 175 | switch (mode) { 176 | case TIME_SEQUENTIAL: 177 | return parseTimestampNoChecksTime(id); 178 | case SPREAD: 179 | default: 180 | return parseTimestampNoChecksSpread(id); 181 | } 182 | } 183 | 184 | private static long parseTimestampNoChecksSpread(byte[] id) { 185 | byte[] copy = id.clone(); 186 | 187 | // Clear everything but the first 42 bits for the timestamp. 188 | // [0] TTTTTTTT [1] TTTTTTTT [2] TTTTTTTT [3] TTTTTTTT [4] TTTTTTTT [5] TT...... 189 | copy[5] = (byte) (copy[5] & 0xC0); 190 | copy[6] = 0; 191 | copy[7] = 0; 192 | 193 | ByteBuffer bb = ByteBuffer.wrap(copy); 194 | return Long.reverse(bb.getLong()); 195 | } 196 | 197 | private static long parseTimestampNoChecksTime(byte[] id) { 198 | byte[] copy = id.clone(); 199 | 200 | ByteBuffer bb = ByteBuffer.wrap(copy); 201 | long ts = bb.getLong(); 202 | ts >>>= 22; 203 | return ts; 204 | } 205 | 206 | private static Mode parseModeNoChecks(byte[] id) { 207 | // [6] ...M.... 208 | int modeMask = id[6] >> 4 & 0x01; 209 | return Mode.fromModeMask(modeMask); 210 | } 211 | } 212 | -------------------------------------------------------------------------------- /uniqueid-core/src/main/java/org/lable/oss/uniqueid/bytes/Mode.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2014 Lable (info@lable.nl) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.lable.oss.uniqueid.bytes; 17 | 18 | /** 19 | * ID generation mode. 20 | */ 21 | public enum Mode { 22 | /** 23 | * Generated IDs start with a timestamp with the bytes in reverse order to prevent hot-spotting in key-value 24 | * stores that order their records based on the key. This mode is ideally suited for generating opaque 25 | * identifiers without a predictable order. 26 | *

27 | * Generators are encouraged to cache a stack of pre-generated IDs, to reduce I/O, as there is no need to 28 | * maintain a claimed generator-ID for longer than it takes to top up the stack of IDs. 29 | */ 30 | SPREAD, 31 | /** 32 | * Generated IDs start with a timestamp in natural sorting order. The timestamps are intended to be 33 | * used actively, so generators should not cache pre-generated IDs for long periods of time (how long 34 | * depends on the application) or not at all. This may result in more I/O to maintain an active claim on a 35 | * generator-ID (if a coordination service such as ZooKeeper is used). 36 | */ 37 | TIME_SEQUENTIAL; 38 | 39 | public int getModeMask() { 40 | return ordinal(); 41 | } 42 | 43 | public static Mode defaultMode() { 44 | return SPREAD; 45 | } 46 | 47 | public static Mode fromModeMask(int modeMask) { 48 | switch (modeMask) { 49 | case 1: 50 | return Mode.TIME_SEQUENTIAL; 51 | case 0: 52 | default: 53 | return Mode.SPREAD; 54 | } 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /uniqueid-core/src/test/java/org/lable/oss/uniqueid/AutoRefillStackTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2014 Lable (info@lable.nl) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.lable.oss.uniqueid; 17 | 18 | import org.junit.Test; 19 | 20 | import java.util.ArrayDeque; 21 | import java.util.Deque; 22 | import java.util.Random; 23 | 24 | import static org.hamcrest.CoreMatchers.is; 25 | import static org.hamcrest.CoreMatchers.not; 26 | import static org.hamcrest.CoreMatchers.nullValue; 27 | import static org.hamcrest.MatcherAssert.assertThat; 28 | import static org.mockito.Mockito.*; 29 | 30 | 31 | public class AutoRefillStackTest { 32 | Random random = new Random(); 33 | 34 | @Test 35 | public void refillTest() throws GeneratorException { 36 | IDGenerator generator = mock(IDGenerator.class); 37 | Deque deck1 = new ArrayDeque<>(10); 38 | Deque deck2 = new ArrayDeque<>(10); 39 | for (int i = 0; i < 10; i++) { 40 | deck1.add(Long.toHexString(random.nextLong()).getBytes()); 41 | deck2.add(Long.toHexString(random.nextLong()).getBytes()); 42 | } 43 | when(generator.batch(10)).thenReturn(deck1).thenReturn(deck2); 44 | 45 | IDGenerator stack = AutoRefillStack.decorate(generator, 10); 46 | 47 | // Grab 9 IDs. 48 | Deque deck = stack.batch(9); 49 | assertThat(deck.size(), is(9)); 50 | 51 | byte[] id = stack.generate(); 52 | assertThat(id, is(not(nullValue()))); 53 | 54 | // This should cause the wrapped IDGenerator's #batch() to be called a second time. 55 | id = stack.generate(); 56 | assertThat(id, is(not(nullValue()))); 57 | 58 | verify(generator, times(2)).batch(10); 59 | verifyNoMoreInteractions(generator); 60 | } 61 | 62 | @Test 63 | public void defaultConstructorTest() throws GeneratorException { 64 | IDGenerator generator = mock(IDGenerator.class); 65 | Deque dummyDeck = new ArrayDeque(AutoRefillStack.DEFAULT_BATCH_SIZE); 66 | for (int i = 0; i < AutoRefillStack.DEFAULT_BATCH_SIZE; i++) { 67 | dummyDeck.add(Long.toHexString(random.nextLong()).getBytes()); 68 | } 69 | when(generator.batch(AutoRefillStack.DEFAULT_BATCH_SIZE)).thenReturn(dummyDeck); 70 | 71 | IDGenerator stack = AutoRefillStack.decorate(generator); 72 | 73 | // Call batch with a value that will cause it to return an empty list. 74 | // The wrapped generator should not be called. 75 | Deque ids = stack.batch(-1); 76 | assertThat(ids.size(), is(0)); 77 | verify(generator, never()).batch(anyInt()); 78 | 79 | // Trigger the wrapper to load up a fresh batch of IDs. 80 | stack.generate(); 81 | assertThat(((AutoRefillStack) stack).idStack.size(), is(AutoRefillStack.DEFAULT_BATCH_SIZE - 1)); 82 | verify(generator).batch(AutoRefillStack.DEFAULT_BATCH_SIZE); 83 | } 84 | } -------------------------------------------------------------------------------- /uniqueid-core/src/test/java/org/lable/oss/uniqueid/BaseUniqueIDGeneratorTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2014 Lable (info@lable.nl) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.lable.oss.uniqueid; 17 | 18 | import org.apache.commons.codec.binary.Hex; 19 | import org.junit.Test; 20 | import org.lable.oss.uniqueid.bytes.Blueprint; 21 | import org.lable.oss.uniqueid.bytes.IDBuilder; 22 | 23 | import static org.hamcrest.CoreMatchers.is; 24 | import static org.hamcrest.CoreMatchers.notNullValue; 25 | import static org.hamcrest.core.IsNot.not; 26 | import static org.hamcrest.MatcherAssert.assertThat; 27 | 28 | public class BaseUniqueIDGeneratorTest { 29 | 30 | /* 31 | * Byte mangling tests. 32 | */ 33 | 34 | 35 | } -------------------------------------------------------------------------------- /uniqueid-core/src/test/java/org/lable/oss/uniqueid/ByteArray.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2014 Lable (info@lable.nl) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.lable.oss.uniqueid; 17 | 18 | import java.util.Arrays; 19 | 20 | public class ByteArray { 21 | final byte[] value; 22 | 23 | public ByteArray(byte[] value) { 24 | this.value = value; 25 | } 26 | 27 | public byte[] getValue() { 28 | return value; 29 | } 30 | 31 | @Override 32 | public boolean equals(Object o) { 33 | if (this == o) return true; 34 | if (o == null || getClass() != o.getClass()) return false; 35 | ByteArray byteArray = (ByteArray) o; 36 | return Arrays.equals(value, byteArray.value); 37 | } 38 | 39 | @Override 40 | public int hashCode() { 41 | return Arrays.hashCode(value); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /uniqueid-core/src/test/java/org/lable/oss/uniqueid/GeneratorExceptionTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2014 Lable (info@lable.nl) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.lable.oss.uniqueid; 17 | 18 | 19 | import org.junit.Test; 20 | 21 | import java.io.IOException; 22 | 23 | import static org.hamcrest.CoreMatchers.instanceOf; 24 | import static org.hamcrest.CoreMatchers.is; 25 | import static org.hamcrest.MatcherAssert.assertThat; 26 | import static org.junit.Assert.assertThrows; 27 | 28 | public class GeneratorExceptionTest { 29 | @Test 30 | public void constructionTest() { 31 | assertThrows(GeneratorException.class, () -> { 32 | throw new GeneratorException(); 33 | }); 34 | } 35 | 36 | @Test 37 | public void constructionWithMessageTest() { 38 | assertThrows( 39 | "Hello!", 40 | GeneratorException.class, 41 | () -> { 42 | throw new GeneratorException("Hello!"); 43 | } 44 | ); 45 | 46 | } 47 | 48 | @Test 49 | public void constructionWithMessageAndThrowableTest() { 50 | GeneratorException e = assertThrows( 51 | "Hello!", 52 | GeneratorException.class, 53 | () -> { 54 | throw new GeneratorException("Hello!", new IOException("XXX")); 55 | } 56 | ); 57 | assertThat(e.getCause(), is(instanceOf(IOException.class))); 58 | } 59 | 60 | @Test 61 | public void constructionWithThrowableTest() throws GeneratorException { 62 | GeneratorException e = assertThrows( 63 | GeneratorException.class, 64 | () -> { 65 | throw new GeneratorException(new IOException("XXX")); 66 | } 67 | ); 68 | assertThat(e.getCause(), is(instanceOf(IOException.class))); 69 | } 70 | } -------------------------------------------------------------------------------- /uniqueid-core/src/test/java/org/lable/oss/uniqueid/LocalUniqueIDGeneratorFactoryTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2014 Lable (info@lable.nl) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.lable.oss.uniqueid; 17 | 18 | import org.junit.Test; 19 | import org.lable.oss.uniqueid.bytes.Mode; 20 | 21 | public class LocalUniqueIDGeneratorFactoryTest { 22 | 23 | @Test(expected = IllegalArgumentException.class) 24 | public void outOfBoundsGeneratorIDTest() { 25 | LocalUniqueIDGeneratorFactory.generatorFor(2048, 0, Mode.SPREAD); 26 | } 27 | 28 | @Test(expected = IllegalArgumentException.class) 29 | public void outOfBoundsClusterIDTest() { 30 | LocalUniqueIDGeneratorFactory.generatorFor(0, 16, Mode.SPREAD); 31 | } 32 | 33 | @Test(expected = IllegalArgumentException.class) 34 | public void outOfBoundsGeneratorIDNegativeTest() { 35 | LocalUniqueIDGeneratorFactory.generatorFor(-1, 0, Mode.SPREAD); 36 | } 37 | 38 | @Test(expected = IllegalArgumentException.class) 39 | public void outOfBoundsClusterIDNegativeTest() { 40 | LocalUniqueIDGeneratorFactory.generatorFor(0, -1, Mode.SPREAD); 41 | } 42 | } -------------------------------------------------------------------------------- /uniqueid-core/src/test/java/org/lable/oss/uniqueid/LocalUniqueIDGeneratorIT.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2014 Lable (info@lable.nl) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.lable.oss.uniqueid; 17 | 18 | 19 | import org.junit.Test; 20 | import org.lable.oss.uniqueid.bytes.Blueprint; 21 | import org.lable.oss.uniqueid.bytes.IDBuilder; 22 | import org.lable.oss.uniqueid.bytes.Mode; 23 | 24 | import java.util.Deque; 25 | 26 | import static org.hamcrest.CoreMatchers.is; 27 | import static org.hamcrest.MatcherAssert.assertThat; 28 | 29 | public class LocalUniqueIDGeneratorIT { 30 | 31 | @Test 32 | public void batchTest() throws Exception { 33 | final int GENERATOR_ID = 42; 34 | final int CLUSTER_ID = 7; 35 | final int BATCH_SIZE = 500; 36 | IDGenerator generator = LocalUniqueIDGeneratorFactory.generatorFor(GENERATOR_ID, CLUSTER_ID, Mode.SPREAD); 37 | 38 | Deque stack = generator.batch(BATCH_SIZE); 39 | assertThat(stack.size(), is(BATCH_SIZE)); 40 | 41 | Blueprint blueprint = IDBuilder.parse(stack.pop()); 42 | assertThat(blueprint.getGeneratorId(), is(GENERATOR_ID)); 43 | assertThat(blueprint.getClusterId(), is(CLUSTER_ID)); 44 | } 45 | 46 | @Test 47 | public void highGeneratorIdTest() throws Exception { 48 | final int GENERATOR_ID = 10; 49 | final int CLUSTER_ID = 15; 50 | IDGenerator generator = LocalUniqueIDGeneratorFactory.generatorFor(GENERATOR_ID, CLUSTER_ID, Mode.SPREAD); 51 | 52 | byte[] id = generator.generate(); 53 | 54 | Blueprint blueprint = IDBuilder.parse(id); 55 | assertThat(blueprint.getGeneratorId(), is(GENERATOR_ID)); 56 | assertThat(blueprint.getClusterId(), is(CLUSTER_ID)); 57 | } 58 | 59 | @Test 60 | public void clockTest() throws Exception { 61 | final int GENERATOR_ID = 20; 62 | final int CLUSTER_ID = 15; 63 | IDGenerator generator = LocalUniqueIDGeneratorFactory.generatorFor( 64 | GENERATOR_ID, 65 | CLUSTER_ID, 66 | () -> 1, 67 | Mode.SPREAD 68 | ); 69 | byte[] id = null; 70 | for (int i = 0; i < 64; i++) { 71 | id = generator.generate(); 72 | } 73 | 74 | Blueprint blueprint = IDBuilder.parse(id); 75 | assertThat(blueprint.getGeneratorId(), is(GENERATOR_ID)); 76 | assertThat(blueprint.getClusterId(), is(CLUSTER_ID)); 77 | assertThat(blueprint.getTimestamp(), is(1L)); 78 | assertThat(blueprint.getSequence(), is(63)); 79 | } 80 | 81 | @Test(expected = GeneratorException.class) 82 | public void clockTestFails() throws Exception { 83 | final int GENERATOR_ID = 30; 84 | final int CLUSTER_ID = 15; 85 | IDGenerator generator = LocalUniqueIDGeneratorFactory.generatorFor( 86 | GENERATOR_ID, 87 | CLUSTER_ID, 88 | () -> 1, 89 | Mode.SPREAD 90 | ); 91 | 92 | // If the clock doesn't progress, no more then 64 ids can be generated. 93 | for (int i = 0; i < 65; i++) { 94 | generator.generate(); 95 | } 96 | } 97 | } -------------------------------------------------------------------------------- /uniqueid-core/src/test/java/org/lable/oss/uniqueid/OnePerMillisecondDecoratorIT.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2014 Lable (info@lable.nl) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.lable.oss.uniqueid; 17 | 18 | import org.junit.Test; 19 | import org.lable.oss.uniqueid.bytes.Blueprint; 20 | import org.lable.oss.uniqueid.bytes.IDBuilder; 21 | import org.lable.oss.uniqueid.bytes.Mode; 22 | 23 | import java.util.Deque; 24 | 25 | import static org.hamcrest.CoreMatchers.is; 26 | import static org.hamcrest.Matchers.not; 27 | import static org.hamcrest.MatcherAssert.assertThat; 28 | 29 | public class OnePerMillisecondDecoratorIT { 30 | @Test 31 | public void batchTest() throws Exception { 32 | final int GENERATOR_ID = 42; 33 | final int CLUSTER_ID = 7; 34 | final int BATCH_SIZE = 500; 35 | 36 | IDGenerator generator = LocalUniqueIDGeneratorFactory.generatorFor( 37 | GENERATOR_ID, 38 | CLUSTER_ID, 39 | Mode.SPREAD 40 | ); 41 | IDGenerator decorator = OnePerMillisecondDecorator.decorate(generator); 42 | 43 | Deque stack = decorator.batch(BATCH_SIZE); 44 | assertThat(stack.size(), is(BATCH_SIZE)); 45 | 46 | byte[] first = stack.pop(); 47 | Blueprint blueprint = IDBuilder.parse(first); 48 | assertThat(blueprint.getGeneratorId(), is(GENERATOR_ID)); 49 | assertThat(blueprint.getClusterId(), is(CLUSTER_ID)); 50 | 51 | // Verify that subsequent IDs don't start with the same byte as their predecessors if generated with a 52 | // one-per-millisecond decorator wrapping the generator. It *is* possible for the bytes to be the same even 53 | // with this decorator, but not when they are generated as fast as possible. 54 | byte previous = first[0]; 55 | for (byte[] bytes : stack) { 56 | byte current = bytes[0]; 57 | assertThat(previous, is(not(current))); 58 | previous = current; 59 | } 60 | } 61 | } -------------------------------------------------------------------------------- /uniqueid-core/src/test/java/org/lable/oss/uniqueid/UniqueIDGeneratorThreadSafetyIT.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2014 Lable (info@lable.nl) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.lable.oss.uniqueid; 17 | 18 | import org.apache.commons.codec.binary.Hex; 19 | import org.junit.Test; 20 | import org.lable.oss.uniqueid.bytes.Mode; 21 | 22 | import java.util.Collections; 23 | import java.util.HashSet; 24 | import java.util.Set; 25 | import java.util.concurrent.CountDownLatch; 26 | import java.util.concurrent.TimeUnit; 27 | 28 | import static org.hamcrest.CoreMatchers.is; 29 | import static org.hamcrest.MatcherAssert.assertThat; 30 | 31 | /** 32 | * Test thread safety. 33 | */ 34 | public class UniqueIDGeneratorThreadSafetyIT { 35 | 36 | @Test 37 | public void multipleInstancesTest() throws InterruptedException { 38 | final Set ids = Collections.synchronizedSet(new HashSet<>()); 39 | final int threadCount = 20; 40 | final int iterationCount = 10000; 41 | final CountDownLatch latch = new CountDownLatch(threadCount); 42 | 43 | // Generate IDs for the same generator-ID and cluster-ID in multiple threads. 44 | // Collision of IDs is almost guaranteed if the generator doesn't handle multi-threading gracefully. 45 | 46 | for (int i = 0; i < threadCount; i++) { 47 | Thread t = new Thread(() -> { 48 | IDGenerator generator = LocalUniqueIDGeneratorFactory.generatorFor(1, 1, Mode.SPREAD); 49 | try { 50 | for (int i1 = 0; i1 < iterationCount; i1++) { 51 | byte[] id = generator.generate(); 52 | String asHex = Hex.encodeHexString(id); 53 | ids.add(asHex); 54 | } 55 | } catch (GeneratorException e) { 56 | // Test will fail due to missing IDs. 57 | e.printStackTrace(); 58 | } 59 | latch.countDown(); 60 | }); 61 | t.start(); 62 | } 63 | 64 | // Wait for all the threads to finish, or timeout. 65 | boolean successfullyUnlatched = latch.await(20, TimeUnit.SECONDS); 66 | assertThat(successfullyUnlatched, is(true)); 67 | 68 | // If the set holds fewer items than this, duplicates were generated. 69 | assertThat(ids.size(), is(threadCount * iterationCount)); 70 | } 71 | 72 | @Test 73 | public void moreThanOneGeneratorClusterIDTest() throws InterruptedException { 74 | final Set ids = Collections.synchronizedSet(new HashSet<>()); 75 | // {generatorId, clusterId} 76 | final int[][] profiles = { 77 | {0, 0}, {1, 1}, {1, 2}, {1, 3}, {1, 15}, 78 | {2, 0}, {3, 0}, {4, 0}, {5, 0}, {63, 0} 79 | }; 80 | final int iterationCount = 10000; 81 | final CountDownLatch latch = new CountDownLatch(profiles.length); 82 | 83 | // Generate IDs for different generator-IDs and cluster-IDs in multiple threads. 84 | // Collision of IDs is almost guaranteed if the generator doesn't handle multi-threading gracefully. 85 | 86 | for (final int[] profile : profiles) { 87 | Thread t = new Thread(() -> { 88 | IDGenerator generator = LocalUniqueIDGeneratorFactory.generatorFor(profile[0], profile[1], Mode.SPREAD); 89 | try { 90 | for (int i = 0; i < iterationCount; i++) { 91 | byte[] id = generator.generate(); 92 | String asHex = Hex.encodeHexString(id); 93 | ids.add(asHex); 94 | } 95 | } catch (GeneratorException e) { 96 | // Test will fail due to missing IDs. 97 | e.printStackTrace(); 98 | } 99 | latch.countDown(); 100 | }); 101 | t.start(); 102 | } 103 | 104 | // Wait for all the threads to finish, or timeout. 105 | boolean successfullyUnlatched = latch.await(20, TimeUnit.SECONDS); 106 | assertThat(successfullyUnlatched, is(true)); 107 | 108 | // If the set holds fewer items than this, duplicates were generated. 109 | assertThat(ids.size(), is(profiles.length * iterationCount)); 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /uniqueid-core/src/test/java/org/lable/oss/uniqueid/bytes/BlueprintTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2014 Lable (info@lable.nl) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.lable.oss.uniqueid.bytes; 17 | 18 | import org.junit.Test; 19 | 20 | import static org.hamcrest.CoreMatchers.is; 21 | import static org.hamcrest.CoreMatchers.notNullValue; 22 | import static org.hamcrest.core.IsNot.not; 23 | import static org.hamcrest.MatcherAssert.assertThat; 24 | 25 | public class BlueprintTest { 26 | @Test 27 | public void toStringTest() { 28 | Blueprint blueprint = 29 | new Blueprint(System.currentTimeMillis(), 0, 0, 0); 30 | assertThat(blueprint.toString(), is(notNullValue())); 31 | assertThat(blueprint.toString().length(), is(not(0))); 32 | } 33 | } -------------------------------------------------------------------------------- /uniqueid-core/src/test/java/org/lable/oss/uniqueid/bytes/IDBuilderTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2014 Lable (info@lable.nl) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.lable.oss.uniqueid.bytes; 17 | 18 | import org.apache.commons.codec.binary.Hex; 19 | import org.junit.Test; 20 | import org.lable.oss.uniqueid.ByteArray; 21 | 22 | import java.util.HashSet; 23 | import java.util.Set; 24 | 25 | import static org.hamcrest.CoreMatchers.is; 26 | import static org.hamcrest.MatcherAssert.assertThat; 27 | import static org.lable.oss.uniqueid.bytes.IDBuilder.parseGeneratorId; 28 | 29 | public class IDBuilderTest { 30 | @Test 31 | public void buildZero() { 32 | final byte[] result = IDBuilder.build(new Blueprint(0, 0, 0, 0)); 33 | final byte[] zero = new byte[8]; 34 | 35 | // Baseline check, if all ID parts are zero so is the result. 36 | assertThat(result, is(zero)); 37 | } 38 | 39 | @Test 40 | public void buildMostlyOnes() { 41 | final byte[] result = IDBuilder.build(new Blueprint( 42 | Blueprint.MAX_TIMESTAMP, 43 | Blueprint.MAX_SEQUENCE_COUNTER, 44 | Blueprint.MAX_GENERATOR_ID, 45 | Blueprint.MAX_CLUSTER_ID, 46 | Mode.SPREAD 47 | )); 48 | final String expected = "ffffffffffffefff"; 49 | 50 | // Baseline check, if all ID parts are all ones so is the result (except for the reserved bytes). 51 | assertThat(Hex.encodeHexString(result), is(expected)); 52 | } 53 | 54 | @Test 55 | public void buildTimestampOnly() { 56 | final long TEST_TS_A = 143062936275L; 57 | // This is the above long with its bytes reversed. 58 | final String TEST_A_REVERSED = "cb54ecf284000000"; 59 | 60 | // Timestamp test. 61 | final byte[] result_a = IDBuilder.build(new Blueprint(TEST_TS_A, 0, 0, 0)); 62 | assertThat(Hex.encodeHexString(result_a).toLowerCase(), is(TEST_A_REVERSED)); 63 | 64 | final long TEST_TS_B = 0x3FFFFFFFDL; 65 | // This is the above long with its bytes reversed. 66 | final String TEST_B_REVERSED = "bfffffffc0000000"; 67 | 68 | // Timestamp test. 69 | final byte[] result_b = IDBuilder.build(new Blueprint(TEST_TS_B, 0, 0, 0)); 70 | assertThat(Hex.encodeHexString(result_b).toLowerCase(), is(TEST_B_REVERSED)); 71 | } 72 | 73 | @Test 74 | public void buildSequenceCounterOnly() { 75 | // Sequence counter test. 76 | final byte[] result = IDBuilder.build(new Blueprint(0, 0x22, 0, 0)); 77 | final byte[] sixthByte = new byte[]{result[5]}; 78 | // 0x88 is 0x22 shifted left two bits. 79 | final String expected = "22"; 80 | assertThat(Hex.encodeHexString(sixthByte), is(expected)); 81 | } 82 | 83 | @Test 84 | public void buildGeneratorIdOnly() { 85 | // Generator ID test. 86 | final byte[] result = IDBuilder.build(new Blueprint(0, 0, 0x27, 0)); 87 | final byte[] lastTwoBytes = new byte[]{result[6], result[7]}; 88 | // 0x0270 is 0x0027 shifted left four bits. 89 | final String expected = "0270"; 90 | assertThat(Hex.encodeHexString(lastTwoBytes), is(expected)); 91 | } 92 | 93 | @Test 94 | public void buildClusterIdOnly() { 95 | // Cluster ID test. 96 | final byte[] result = IDBuilder.build(new Blueprint(0, 0, 0, 5)); 97 | final byte[] lastTwoBytes = new byte[]{result[6], result[7]}; 98 | final String expected = "0005"; 99 | assertThat(Hex.encodeHexString(lastTwoBytes), is(expected)); 100 | } 101 | 102 | @Test 103 | public void buildModeOnlySpread() { 104 | // Cluster ID test. 105 | final byte[] result = IDBuilder.build(new Blueprint(0, 0, 0, 0)); 106 | final byte[] lastTwoBytes = new byte[]{result[6], result[7]}; 107 | final String expected = "0000"; 108 | assertThat(Hex.encodeHexString(lastTwoBytes), is(expected)); 109 | } 110 | 111 | @Test 112 | public void buildModeOnlySpreadExplicit() { 113 | // Cluster ID test. 114 | final byte[] result = IDBuilder.build(new Blueprint(0, 0, 0, 0, Mode.SPREAD)); 115 | final byte[] lastTwoBytes = new byte[]{result[6], result[7]}; 116 | final String expected = "0000"; 117 | assertThat(Hex.encodeHexString(lastTwoBytes), is(expected)); 118 | } 119 | 120 | @Test 121 | public void buildModeOnlyTimeExplicit() { 122 | // Cluster ID test. 123 | final byte[] result = IDBuilder.build(new Blueprint(0, 0, 0, 0, Mode.TIME_SEQUENTIAL)); 124 | final byte[] lastTwoBytes = new byte[]{result[6], result[7]}; 125 | final String expected = "1000"; 126 | assertThat(Hex.encodeHexString(lastTwoBytes), is(expected)); 127 | } 128 | 129 | @Test 130 | public void parseBytes() { 131 | // Create an ID, then un-mangle it, and run the resulting blueprint through the mangler again. 132 | final long TEST_TS = 143062936275L; 133 | final byte[] resultOne = IDBuilder.build(new Blueprint(TEST_TS, 10, 1, 5)); 134 | 135 | assertThat(IDBuilder.parseGeneratorId(resultOne), is(1)); 136 | assertThat(IDBuilder.parseClusterId(resultOne), is(5)); 137 | assertThat(IDBuilder.parseSequenceId(resultOne), is(10)); 138 | assertThat(IDBuilder.parseTimestamp(resultOne), is(TEST_TS)); 139 | assertThat(IDBuilder.parseMode(resultOne), is(Mode.SPREAD)); 140 | 141 | Blueprint blueprint = IDBuilder.parse(resultOne); 142 | 143 | final byte[] result_two = IDBuilder.build(blueprint); 144 | assertThat(resultOne, is(result_two)); 145 | } 146 | 147 | @Test 148 | public void parseBytesTimeSequential() { 149 | // Create an ID, then un-mangle it, and run the resulting blueprint through the mangler again. 150 | final long TEST_TS = 143062936275L; 151 | final byte[] resultOne = IDBuilder.build(new Blueprint(TEST_TS, 10, 2, 5, Mode.TIME_SEQUENTIAL)); 152 | 153 | assertThat(IDBuilder.parseGeneratorId(resultOne), is(2)); 154 | assertThat(IDBuilder.parseClusterId(resultOne), is(5)); 155 | assertThat(IDBuilder.parseSequenceId(resultOne), is(10)); 156 | assertThat(IDBuilder.parseTimestamp(resultOne), is(TEST_TS)); 157 | assertThat(IDBuilder.parseMode(resultOne), is(Mode.TIME_SEQUENTIAL)); 158 | 159 | Blueprint blueprint = IDBuilder.parse(resultOne); 160 | 161 | final byte[] result_two = IDBuilder.build(blueprint); 162 | assertThat(resultOne, is(result_two)); 163 | } 164 | 165 | @Test 166 | public void blueprintSpread() { 167 | // Round-trip test. First generate the byte[] with mangleBytes, then back to the blueprint with Blueprint.parse. 168 | 169 | final long TEST_TS = 143062936275L; 170 | final byte[] resultOne = IDBuilder.build(new Blueprint(TEST_TS, 10, 1, 5)); 171 | final Blueprint blueprintOne = IDBuilder.parse(resultOne); 172 | final byte[] resultOneAgain = IDBuilder.build(blueprintOne); 173 | assertThat(resultOne, is(resultOneAgain)); 174 | 175 | final byte[] resultZeros = IDBuilder.build(new Blueprint(0, 0, 0, 0)); 176 | final Blueprint blueprintZeros = IDBuilder.parse(resultZeros); 177 | final byte[] resultZerosAgain = IDBuilder.build(blueprintZeros); 178 | assertThat(resultZeros, is(resultZerosAgain)); 179 | 180 | final byte[] resultMostlyOnes = IDBuilder.build(new Blueprint( 181 | Blueprint.MAX_TIMESTAMP, 182 | Blueprint.MAX_SEQUENCE_COUNTER, 183 | Blueprint.MAX_GENERATOR_ID, 184 | Blueprint.MAX_CLUSTER_ID 185 | )); 186 | final Blueprint blueprintMostlyOnes = IDBuilder.parse(resultMostlyOnes); 187 | final byte[] resultMostlyOnesAgain = IDBuilder.build(blueprintMostlyOnes); 188 | assertThat(resultMostlyOnes, is(resultMostlyOnesAgain)); 189 | } 190 | 191 | @Test 192 | public void blueprintTimeSequential() { 193 | // Round-trip test. First generate the byte[] with mangleBytes, then back to the blueprint with Blueprint.parse. 194 | 195 | final long TEST_TS = 143062936275L; 196 | final byte[] resultOne = IDBuilder.build(new Blueprint(TEST_TS, 10, 1, 5, Mode.TIME_SEQUENTIAL)); 197 | final Blueprint blueprintOne = IDBuilder.parse(resultOne); 198 | final byte[] resultOneAgain = IDBuilder.build(blueprintOne); 199 | assertThat(resultOne, is(resultOneAgain)); 200 | 201 | final byte[] resultZeros = IDBuilder.build(new Blueprint(0, 0, 0, 0, Mode.TIME_SEQUENTIAL)); 202 | final Blueprint blueprintZeros = IDBuilder.parse(resultZeros); 203 | final byte[] resultZerosAgain = IDBuilder.build(blueprintZeros); 204 | assertThat(resultZeros, is(resultZerosAgain)); 205 | 206 | final byte[] resultMostlyOnes = IDBuilder.build(new Blueprint( 207 | Blueprint.MAX_TIMESTAMP, 208 | Blueprint.MAX_SEQUENCE_COUNTER, 209 | Blueprint.MAX_GENERATOR_ID, 210 | Blueprint.MAX_CLUSTER_ID, 211 | Mode.TIME_SEQUENTIAL 212 | )); 213 | final Blueprint blueprintMostlyOnes = IDBuilder.parse(resultMostlyOnes); 214 | final byte[] resultMostlyOnesAgain = IDBuilder.build(blueprintMostlyOnes); 215 | assertThat(resultMostlyOnes, is(resultMostlyOnesAgain)); 216 | } 217 | 218 | @Test 219 | public void parseGeneratorIdTest() { 220 | byte[] id = new byte[8]; 221 | id[6] = 0x0f; 222 | id[7] = (byte) (0x0f << 4); 223 | 224 | byte[] clone = id.clone(); 225 | 226 | assertThat(parseGeneratorId(id), is(255)); 227 | 228 | assertThat(id, is(clone)); 229 | } 230 | 231 | @Test(expected = IllegalArgumentException.class) 232 | public void parseIllegalArgument() { 233 | IDBuilder.parse(new byte[0]); 234 | } 235 | 236 | @Test(expected = IllegalArgumentException.class) 237 | public void parseIllegalArgumentNull() { 238 | IDBuilder.parse(null); 239 | } 240 | 241 | @Test 242 | public void fullGeneratorSpace() { 243 | // Verify that bitwise operations in IDBuilder work. 244 | Set results = new HashSet<>(); 245 | for (int generatorId = 0; generatorId <= Blueprint.MAX_GENERATOR_ID; generatorId++) { 246 | byte[] result = IDBuilder.build(new Blueprint( 247 | Blueprint.MAX_TIMESTAMP, 248 | Blueprint.MAX_SEQUENCE_COUNTER, 249 | generatorId, 250 | Blueprint.MAX_CLUSTER_ID, 251 | Mode.SPREAD 252 | )); 253 | results.add(new ByteArray(result)); 254 | 255 | result = IDBuilder.build(new Blueprint( 256 | Blueprint.MAX_TIMESTAMP, 257 | Blueprint.MAX_SEQUENCE_COUNTER, 258 | generatorId, 259 | Blueprint.MAX_CLUSTER_ID, 260 | Mode.TIME_SEQUENTIAL 261 | )); 262 | results.add(new ByteArray(result)); 263 | } 264 | 265 | assertThat(results.size(), is(2 *(Blueprint.MAX_GENERATOR_ID + 1))); 266 | } 267 | } -------------------------------------------------------------------------------- /uniqueid-etcd/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 19 | 20 | 21 | uniqueid 22 | org.lable.oss.uniqueid 23 | 4.7-SNAPSHOT 24 | 25 | 4.0.0 26 | 27 | uniqueid-etcd 28 | UniqueID :: Etcd 29 | 30 | 31 | 32 | uniqueid-core 33 | org.lable.oss.uniqueid 34 | 35 | 36 | io.etcd 37 | jetcd-core 38 | ${jetcd-version} 39 | 40 | 41 | 42 | 43 | uniqueid-core 44 | org.lable.oss.uniqueid 45 | tests 46 | test 47 | ${project.version} 48 | 49 | 50 | io.etcd 51 | jetcd-launcher 52 | ${jetcd-version} 53 | test 54 | 55 | 56 | com.github.npathai 57 | hamcrest-optional 58 | ${hamcrest.optional} 59 | test 60 | 61 | 62 | 63 | 64 | 65 | 66 | maven-failsafe-plugin 67 | 68 | 69 | 70 | org.jacoco 71 | jacoco-maven-plugin 72 | 73 | 74 | 75 | org.apache.maven.plugins 76 | maven-shade-plugin 77 | 3.2.1 78 | 79 | 80 | package 81 | 82 | shade 83 | 84 | 85 | true 86 | shaded 87 | 88 | 89 | 95 | com.fasterxml.jackson.core:jackson-core 96 | 97 | META-INF/versions/17/** 98 | META-INF/versions/19/** 99 | 100 | 101 | 102 | 103 | 104 | com.google. 105 | com.shaded.google. 106 | 107 | 108 | io.grpc. 109 | io.shaded.grpc. 110 | 111 | 112 | io.netty. 113 | io.shaded.netty. 114 | 115 | 116 | 117 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | -------------------------------------------------------------------------------- /uniqueid-etcd/src/main/java/org/lable/oss/uniqueid/etcd/ClusterID.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2014 Lable (info@lable.nl) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.lable.oss.uniqueid.etcd; 17 | 18 | import io.etcd.jetcd.ByteSequence; 19 | import io.etcd.jetcd.Client; 20 | import io.etcd.jetcd.KeyValue; 21 | import io.etcd.jetcd.kv.GetResponse; 22 | 23 | import java.io.IOException; 24 | import java.nio.charset.StandardCharsets; 25 | import java.util.Collections; 26 | import java.util.List; 27 | import java.util.concurrent.ExecutionException; 28 | import java.util.stream.Collectors; 29 | import java.util.stream.Stream; 30 | 31 | public class ClusterID { 32 | final static ByteSequence CLUSTER_ID_KEY = ByteSequence.from("cluster-id", StandardCharsets.UTF_8); 33 | final static int DEFAULT_CLUSTER_ID = 0; 34 | 35 | /** 36 | * Retrieves the numeric cluster ID from the Etcd cluster. 37 | * 38 | * @param etcd Etcd connection. 39 | * @return The cluster ID, if configured in the cluster. 40 | * @throws IOException Thrown when retrieving the ID fails. 41 | */ 42 | public static List get(Client etcd) throws IOException { 43 | GetResponse get; 44 | try { 45 | get = etcd.getKVClient().get(CLUSTER_ID_KEY).get(); 46 | } catch (InterruptedException | ExecutionException e) { 47 | throw new IOException(e); 48 | } 49 | 50 | List ids = null; 51 | 52 | for (KeyValue kv : get.getKvs()) { 53 | if (kv.getKey().equals(CLUSTER_ID_KEY)) { 54 | // There should be only one key returned. 55 | String value = kv.getValue().toString(StandardCharsets.UTF_8); 56 | try { 57 | ids = parseIntegers(value); 58 | } catch (NumberFormatException e) { 59 | throw new IOException("Failed to parse cluster-id value `" + value + "`.", e); 60 | } 61 | break; 62 | } 63 | } 64 | 65 | if (ids == null) { 66 | ByteSequence defaultValue = ByteSequence.from(String.valueOf(DEFAULT_CLUSTER_ID).getBytes()); 67 | try { 68 | etcd.getKVClient().put(CLUSTER_ID_KEY, defaultValue).get(); 69 | return Collections.singletonList(DEFAULT_CLUSTER_ID); 70 | } catch (InterruptedException | ExecutionException e) { 71 | throw new IOException(e); 72 | } 73 | } else { 74 | return ids; 75 | } 76 | } 77 | 78 | static List parseIntegers(String serialized) { 79 | return Stream.of(serialized.split(",")) 80 | .map(String::trim) 81 | .map(Integer::parseInt) 82 | .collect(Collectors.toList()); 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /uniqueid-etcd/src/main/java/org/lable/oss/uniqueid/etcd/EtcdHelper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2014 Lable (info@lable.nl) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.lable.oss.uniqueid.etcd; 17 | 18 | import io.etcd.jetcd.ByteSequence; 19 | import io.etcd.jetcd.Client; 20 | import io.etcd.jetcd.kv.GetResponse; 21 | import io.etcd.jetcd.lease.LeaseKeepAliveResponse; 22 | import io.etcd.jetcd.support.CloseableClient; 23 | import io.grpc.stub.StreamObserver; 24 | 25 | import java.nio.charset.StandardCharsets; 26 | import java.util.Optional; 27 | import java.util.concurrent.ExecutionException; 28 | 29 | /** 30 | * Utility methods that make working with jetcd a little less verbose. 31 | */ 32 | public class EtcdHelper { 33 | final static ByteSequence UNIQUE_ID_NAMESPACE = ByteSequence.from("unique-id/", StandardCharsets.UTF_8); 34 | 35 | public static Optional getInt(Client etcd, String key) throws ExecutionException, InterruptedException { 36 | GetResponse getResponse = etcd.getKVClient().get(asByteSequence(key)).get(); 37 | 38 | if (getResponse.getCount() == 0) return Optional.empty(); 39 | 40 | String value = getResponse.getKvs().get(0).getValue().toString(StandardCharsets.UTF_8); 41 | try { 42 | return Optional.of(Integer.parseInt(value)); 43 | } catch (NumberFormatException e) { 44 | return Optional.empty(); 45 | } 46 | } 47 | 48 | public static Optional get(Client etcd, String key) throws ExecutionException, InterruptedException { 49 | GetResponse getResponse = etcd.getKVClient().get(asByteSequence(key)).get(); 50 | 51 | if (getResponse.getCount() == 0) return Optional.empty(); 52 | 53 | return Optional.of(getResponse.getKvs().get(0).getValue().toString(StandardCharsets.UTF_8)); 54 | } 55 | 56 | public static void put(Client etcd, String key, int value) throws ExecutionException, InterruptedException { 57 | etcd.getKVClient().put(asByteSequence(key), asByteSequence(value)).get(); 58 | } 59 | 60 | public static void put(Client etcd, String key) throws ExecutionException, InterruptedException { 61 | etcd.getKVClient().put(asByteSequence(key),ByteSequence.EMPTY).get(); 62 | } 63 | 64 | public static void delete(Client etcd, String key) throws ExecutionException, InterruptedException { 65 | etcd.getKVClient().delete(asByteSequence(key)).get(); 66 | } 67 | 68 | static ByteSequence asByteSequence(String value) { 69 | return ByteSequence.from(value, StandardCharsets.UTF_8); 70 | } 71 | 72 | static ByteSequence asByteSequence(int value) { 73 | return asByteSequence(String.valueOf(value)); 74 | } 75 | 76 | public static CloseableClient keepLeaseAlive(Client etcd, Long leaseId, OnRelease onRelease) { 77 | final OnRelease onReleaseCallback = onRelease == null 78 | ? () -> {} 79 | : onRelease; 80 | 81 | return etcd.getLeaseClient().keepAlive( 82 | leaseId, 83 | new StreamObserver() { 84 | @Override 85 | public void onNext(LeaseKeepAliveResponse value) { 86 | // Great! No-op. 87 | } 88 | 89 | @Override 90 | public void onError(Throwable t) { 91 | onReleaseCallback.cleanUp(); 92 | } 93 | 94 | @Override 95 | public void onCompleted() { 96 | onReleaseCallback.cleanUp(); 97 | } 98 | } 99 | ); 100 | } 101 | 102 | @FunctionalInterface 103 | public interface OnRelease { 104 | void cleanUp(); 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /uniqueid-etcd/src/main/java/org/lable/oss/uniqueid/etcd/ExpiringResourceClaim.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2014 Lable (info@lable.nl) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.lable.oss.uniqueid.etcd; 17 | 18 | import io.etcd.jetcd.Client; 19 | import org.slf4j.Logger; 20 | import org.slf4j.LoggerFactory; 21 | 22 | import java.io.IOException; 23 | import java.time.Duration; 24 | import java.util.List; 25 | import java.util.Timer; 26 | import java.util.TimerTask; 27 | 28 | /** 29 | * {@link ResourceClaim} that automatically relinquishes its hold on a resource 30 | * after a set amount of time. 31 | */ 32 | public class ExpiringResourceClaim extends ResourceClaim { 33 | private static final Logger logger = LoggerFactory.getLogger(ExpiringResourceClaim.class); 34 | 35 | public final static Duration DEFAULT_CLAIM_HOLD = Duration.ofSeconds(30); 36 | public final static Duration DEFAULT_ACQUISITION_TIMEOUT = Duration.ofMinutes(10); 37 | 38 | ExpiringResourceClaim(Client etcd, 39 | int maxGeneratorCount, 40 | List clusterIds, 41 | Duration claimHold, 42 | Duration acquisitionTimeout) throws IOException { 43 | super(etcd, maxGeneratorCount, clusterIds, acquisitionTimeout); 44 | new Timer().schedule(new TimerTask() { 45 | @Override 46 | public void run() { 47 | close(); 48 | } 49 | }, claimHold.toMillis()); 50 | } 51 | 52 | /** 53 | * Claim a resource. 54 | * 55 | * @param etcd Etcd connection to use. 56 | * @param maxGeneratorCount Maximum number of generators possible. 57 | * @return A resource claim. 58 | */ 59 | public static ResourceClaim claimExpiring(Client etcd, int maxGeneratorCount, List clusterIds) 60 | throws IOException { 61 | return claimExpiring(etcd, maxGeneratorCount, clusterIds, DEFAULT_CLAIM_HOLD, DEFAULT_ACQUISITION_TIMEOUT); 62 | } 63 | 64 | /** 65 | * Claim a resource. 66 | * 67 | * @param etcd Etcd connection to use. 68 | * @param maxGeneratorCount Maximum number of generators possible. 69 | * @param clusterIds Cluster Ids available to use. 70 | * @param claimHold How long the claim should be held. May be {@code null} for the default value of 71 | * {@link #DEFAULT_CLAIM_HOLD}. 72 | * @param acquisitionTimeout How long to keep trying to acquire a claim. May be {@code null} to keep trying 73 | * indefinitely. 74 | * @return A resource claim. 75 | */ 76 | public static ResourceClaim claimExpiring(Client etcd, 77 | int maxGeneratorCount, 78 | List clusterIds, 79 | Duration claimHold, 80 | Duration acquisitionTimeout) 81 | throws IOException { 82 | 83 | claimHold = claimHold == null ? DEFAULT_CLAIM_HOLD : claimHold; 84 | if (logger.isDebugEnabled()) { 85 | logger.debug("Preparing expiring resource-claim; will release it in {}ms.", claimHold.toMillis()); 86 | } 87 | 88 | return new ExpiringResourceClaim(etcd, maxGeneratorCount, clusterIds, claimHold, acquisitionTimeout); 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /uniqueid-etcd/src/main/java/org/lable/oss/uniqueid/etcd/RegistryBasedGeneratorIdentity.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2014 Lable (info@lable.nl) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.lable.oss.uniqueid.etcd; 17 | 18 | import io.etcd.jetcd.ByteSequence; 19 | import io.etcd.jetcd.Client; 20 | import org.lable.oss.uniqueid.GeneratorException; 21 | import org.lable.oss.uniqueid.GeneratorIdentityHolder; 22 | import org.lable.oss.uniqueid.bytes.Blueprint; 23 | import org.slf4j.Logger; 24 | import org.slf4j.LoggerFactory; 25 | 26 | import java.io.IOException; 27 | import java.nio.charset.StandardCharsets; 28 | import java.time.Duration; 29 | 30 | /** 31 | * Holder for a claimed cluster-id and generator-id that once claimed remains claimed without an active connection to 32 | * an Etcd cluster. The claim is relinquished upon calling {@link #close()} (where a new connection to Etcd will be 33 | * set up briefly). 34 | */ 35 | public class RegistryBasedGeneratorIdentity implements GeneratorIdentityHolder { 36 | private static final Logger logger = LoggerFactory.getLogger(RegistryBasedGeneratorIdentity.class); 37 | 38 | private final String endpoints; 39 | private final String namespace; 40 | private final Duration acquisitionTimeout; 41 | private final boolean waitWhenNoResourcesAvailable; 42 | private final RegistryBasedResourceClaim resourceClaim; 43 | 44 | public RegistryBasedGeneratorIdentity(String endpoints, 45 | String namespace, 46 | String registryEntry, 47 | Duration acquisitionTimeout, 48 | boolean waitWhenNoResourcesAvailable) { 49 | this.endpoints = endpoints; 50 | this.namespace = namespace; 51 | this.acquisitionTimeout = acquisitionTimeout; 52 | this.waitWhenNoResourcesAvailable = waitWhenNoResourcesAvailable; 53 | 54 | try { 55 | resourceClaim = acquireResourceClaim(registryEntry, 0); 56 | } catch (GeneratorException e) { 57 | throw new RuntimeException(e); 58 | } 59 | } 60 | 61 | public static RegistryBasedGeneratorIdentity basedOn(String endpoints, String namespace, String registryEntry) 62 | throws IOException { 63 | return new RegistryBasedGeneratorIdentity( 64 | endpoints, namespace, registryEntry, Duration.ofMinutes(5), true 65 | ); 66 | } 67 | 68 | public static RegistryBasedGeneratorIdentity basedOn(String endpoints, 69 | String namespace, 70 | String registryEntry, 71 | Duration acquisitionTimeout, 72 | boolean waitWhenNoResourcesAvailable) 73 | throws IOException { 74 | return new RegistryBasedGeneratorIdentity( 75 | endpoints, namespace, registryEntry, acquisitionTimeout, waitWhenNoResourcesAvailable 76 | ); 77 | } 78 | 79 | @Override 80 | public int getClusterId() throws GeneratorException { 81 | return resourceClaim.getClusterId(); 82 | } 83 | 84 | @Override 85 | public int getGeneratorId() throws GeneratorException { 86 | return resourceClaim.getGeneratorId(); 87 | } 88 | 89 | public String getRegistryEntry() { 90 | return resourceClaim.getRegistryEntry(); 91 | } 92 | 93 | private RegistryBasedResourceClaim acquireResourceClaim(String registryEntry, int retries) 94 | throws GeneratorException { 95 | try { 96 | return RegistryBasedResourceClaim.claim( 97 | this::getEtcdConnection, 98 | Blueprint.MAX_GENERATOR_ID + 1, 99 | registryEntry, 100 | acquisitionTimeout, 101 | waitWhenNoResourcesAvailable 102 | ); 103 | } catch (IOException e) { 104 | if (retries < 3) { 105 | logger.warn("Connection to Etcd failed, retrying claim acquisition, attempt {}.", retries + 1, e); 106 | return acquireResourceClaim(registryEntry, retries + 1); 107 | } else { 108 | logger.error("Failed to acquire resource claim after attempt {}.", retries + 1, e); 109 | throw new GeneratorException(e); 110 | } 111 | } 112 | } 113 | 114 | Client getEtcdConnection() { 115 | return Client.builder() 116 | .endpoints(endpoints.split(",")) 117 | .loadBalancerPolicy("round_robin") 118 | .namespace(ByteSequence.from(namespace, StandardCharsets.UTF_8)) 119 | .build(); 120 | } 121 | 122 | @Override 123 | public void close() throws IOException { 124 | if (resourceClaim != null) { 125 | resourceClaim.close(); 126 | } 127 | } 128 | } 129 | -------------------------------------------------------------------------------- /uniqueid-etcd/src/main/java/org/lable/oss/uniqueid/etcd/RegistryBasedResourceClaim.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2014 Lable (info@lable.nl) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.lable.oss.uniqueid.etcd; 17 | 18 | import io.etcd.jetcd.*; 19 | import io.etcd.jetcd.kv.GetResponse; 20 | import io.etcd.jetcd.kv.TxnResponse; 21 | import io.etcd.jetcd.lease.LeaseGrantResponse; 22 | import io.etcd.jetcd.lock.LockResponse; 23 | import io.etcd.jetcd.op.Cmp; 24 | import io.etcd.jetcd.op.CmpTarget; 25 | import io.etcd.jetcd.op.Op; 26 | import io.etcd.jetcd.options.GetOption; 27 | import io.etcd.jetcd.options.OptionsUtil; 28 | import io.etcd.jetcd.options.PutOption; 29 | import io.etcd.jetcd.options.WatchOption; 30 | import org.slf4j.Logger; 31 | import org.slf4j.LoggerFactory; 32 | 33 | import java.io.IOException; 34 | import java.nio.charset.StandardCharsets; 35 | import java.time.Duration; 36 | import java.time.Instant; 37 | import java.util.List; 38 | import java.util.concurrent.CountDownLatch; 39 | import java.util.concurrent.ExecutionException; 40 | import java.util.concurrent.TimeUnit; 41 | import java.util.concurrent.TimeoutException; 42 | import java.util.function.Supplier; 43 | import java.util.stream.Collectors; 44 | 45 | public class RegistryBasedResourceClaim { 46 | private static final Logger logger = LoggerFactory.getLogger(RegistryBasedResourceClaim.class); 47 | 48 | static final String REGISTRY_PREFIX = "registry/"; 49 | static final ByteSequence REGISTRY_KEY = ByteSequence.from(REGISTRY_PREFIX, StandardCharsets.UTF_8); 50 | static final ByteSequence LOCK_NAME = ByteSequence.from("unique-id-registry-lock", StandardCharsets.UTF_8); 51 | 52 | final Supplier connectToEtcd; 53 | final String registryEntry; 54 | final int clusterId; 55 | final int generatorId; 56 | 57 | final int poolSize; 58 | final KV kvClient; 59 | 60 | RegistryBasedResourceClaim(Supplier connectToEtcd, 61 | int maxGeneratorCount, 62 | String registryEntry, 63 | Duration acquisitionTimeout, 64 | boolean waitWhenNoResourcesAvailable) 65 | throws IOException { 66 | this.registryEntry = registryEntry; 67 | this.connectToEtcd = connectToEtcd; 68 | 69 | Duration timeout = acquisitionTimeout == null 70 | ? Duration.ofMinutes(5) 71 | : acquisitionTimeout; 72 | logger.info("Acquiring resource-claim; timeout is set to {}.", timeout); 73 | 74 | Client etcd = connectToEtcd.get(); 75 | 76 | // Keep the KV client around, because if we try to instantiate it during shutdown of a Java application when 77 | // the resource is likely to be released, it will fail because further down the stack an attempt is made to 78 | // register a shutdown handler, which fails because the application is already shutting down. So we instantiate 79 | // this here and keep it. 80 | kvClient = etcd.getKVClient(); 81 | 82 | List clusterIds = ClusterID.get(etcd); 83 | 84 | 85 | Instant giveUpAfter = Instant.now().plus(timeout); 86 | long timeoutSeconds = timeout.getSeconds(); 87 | 88 | this.poolSize = maxGeneratorCount; 89 | 90 | ResourcePair resourcePair; 91 | LockResponse lock; 92 | long leaseId; 93 | try { 94 | logger.debug("Acquiring lock."); 95 | // Have the lease TTL just a bit after our timeout. 96 | LeaseGrantResponse lease = etcd.getLeaseClient().grant(timeoutSeconds + 5).get(timeoutSeconds, TimeUnit.SECONDS); 97 | leaseId = lease.getID(); 98 | logger.debug("Got lease {}.", leaseId); 99 | 100 | // Acquire the lock. This makes sure we are the only process claiming a resource. 101 | try { 102 | lock = etcd.getLockClient() 103 | .lock(LOCK_NAME, leaseId) 104 | .get(timeout.toMillis(), TimeUnit.MILLISECONDS); 105 | } catch (TimeoutException e) { 106 | throw new IOException("Process timed out."); 107 | } 108 | 109 | if (logger.isDebugEnabled()) { 110 | logger.debug("Acquired lock: {}.", lock.getKey().toString(StandardCharsets.UTF_8)); 111 | } 112 | 113 | // Keep the lease alive for another period in order to safely finish claiming the resource. 114 | etcd.getLeaseClient().keepAliveOnce(leaseId).get(timeoutSeconds, TimeUnit.SECONDS); 115 | 116 | logger.debug("Lease renewed."); 117 | 118 | resourcePair = claimResource( 119 | etcd, maxGeneratorCount, clusterIds, giveUpAfter, waitWhenNoResourcesAvailable 120 | ); 121 | this.clusterId = resourcePair.clusterId; 122 | this.generatorId = resourcePair.generatorId; 123 | } catch (TimeoutException | ExecutionException e) { 124 | throw new IOException(e); 125 | } catch (InterruptedException e) { 126 | Thread.currentThread().interrupt(); 127 | throw new IOException(e); 128 | } 129 | 130 | try { 131 | // Explicitly release the lock. If this line is not reached due to exceptions raised, the lock will 132 | // automatically be removed when the lease holding it expires. 133 | etcd.getLockClient().unlock(lock.getKey()).get(timeoutSeconds, TimeUnit.SECONDS); 134 | if (logger.isDebugEnabled()) { 135 | logger.debug("Released lock: {}.", lock.getKey().toString(StandardCharsets.UTF_8)); 136 | } 137 | 138 | // Revoke the lease instead of letting it time out. 139 | etcd.getLeaseClient().revoke(leaseId).get(timeoutSeconds, TimeUnit.SECONDS); 140 | } catch (TimeoutException | ExecutionException e) { 141 | logger.warn( 142 | "Failed to release lock {} (will be released automatically by Etcd server). Resource-claims was successfully acquired though.", 143 | lock.getKey().toString(StandardCharsets.UTF_8) 144 | ); 145 | } catch (InterruptedException e) { 146 | Thread.currentThread().interrupt(); 147 | } 148 | 149 | logger.debug("Resource-claim acquired ({}/{}).", clusterId, generatorId); 150 | } 151 | 152 | /** 153 | * Claim a resource. 154 | * 155 | * @param connectToEtcd Provide a connection to Etcd. 156 | * @param maxGeneratorCount Maximum number of generators possible. 157 | * @param registryEntry Metadata stored under the Etcd key. 158 | * @param acquisitionTimeout Abort attempt to claim a resource after this duration. 159 | * @param waitWhenNoResourcesAvailable Wait for a resource to become available when all resources are claimed. 160 | * @return The resource claim, if successful. 161 | * @throws IOException Thrown when the claim could not be acquired. 162 | */ 163 | public static RegistryBasedResourceClaim claim(Supplier connectToEtcd, 164 | int maxGeneratorCount, 165 | String registryEntry, 166 | Duration acquisitionTimeout, 167 | boolean waitWhenNoResourcesAvailable) throws IOException { 168 | return new RegistryBasedResourceClaim( 169 | connectToEtcd, maxGeneratorCount, registryEntry, acquisitionTimeout, waitWhenNoResourcesAvailable 170 | ); 171 | } 172 | 173 | /** 174 | * Try to claim an available resource from the resource pool. 175 | * 176 | * @param etcd Etcd connection. 177 | * @param maxGeneratorCount Maximum number of generators possible. 178 | * @param clusterIds Cluster Ids available to use. 179 | * @param giveUpAfter Give up after this instant in time. 180 | * @param waitWhenNoResourcesAvailable Wait for a resource to become available when all resources are claimed. 181 | * @return The claimed resource. 182 | */ 183 | ResourcePair claimResource(Client etcd, 184 | int maxGeneratorCount, 185 | List clusterIds, 186 | Instant giveUpAfter, 187 | boolean waitWhenNoResourcesAvailable) 188 | throws InterruptedException, IOException, ExecutionException { 189 | 190 | logger.debug("Trying to claim a resource."); 191 | 192 | int registrySize = maxGeneratorCount * clusterIds.size(); 193 | 194 | GetOption getOptions = GetOption.builder() 195 | .withKeysOnly(true) 196 | .withRange(OptionsUtil.prefixEndOf(REGISTRY_KEY)) 197 | .build(); 198 | GetResponse get = etcd.getKVClient().get(REGISTRY_KEY, getOptions).get(); 199 | 200 | List claimedResources = get.getKvs().stream() 201 | .map(KeyValue::getKey) 202 | .collect(Collectors.toList()); 203 | 204 | if (claimedResources.size() >= registrySize) { 205 | if (!waitWhenNoResourcesAvailable) { 206 | throw new IOException( 207 | "No resources available. Giving up as requested. Registry size: " + registrySize + "." 208 | ); 209 | } 210 | logger.warn("No resources available at the moment (registry size: {}), waiting.", registrySize); 211 | // No resources available. Wait for a resource to become available. 212 | final CountDownLatch latch = new CountDownLatch(1); 213 | Watch.Watcher watcher = etcd.getWatchClient().watch( 214 | REGISTRY_KEY, 215 | WatchOption.builder() 216 | .withRange(OptionsUtil.prefixEndOf(REGISTRY_KEY)) 217 | .build(), 218 | watchResponse -> latch.countDown() 219 | ); 220 | awaitLatchUnlessItTakesTooLong(latch, giveUpAfter); 221 | watcher.close(); 222 | return claimResource(etcd, maxGeneratorCount, clusterIds, giveUpAfter, true); 223 | } 224 | 225 | // Try to claim an available resource. 226 | for (Integer clusterId : clusterIds) { 227 | for (int generatorId = 0; generatorId < maxGeneratorCount; generatorId++) { 228 | String resourcePathString = resourceKey(clusterId, generatorId); 229 | ByteSequence resourcePath = ByteSequence.from(resourcePathString, StandardCharsets.UTF_8); 230 | if (!claimedResources.contains(resourcePath)) { 231 | logger.debug("Trying to claim seemingly available resource {}.", resourcePathString); 232 | TxnResponse txnResponse = etcd.getKVClient().txn() 233 | .If( 234 | // Version == 0 means the key does not exist. 235 | new Cmp(resourcePath, Cmp.Op.EQUAL, CmpTarget.version(0)) 236 | ).Then( 237 | Op.put( 238 | resourcePath, 239 | ByteSequence.from(registryEntry, StandardCharsets.UTF_8), 240 | PutOption.builder().build() 241 | ) 242 | ).commit().get(); 243 | 244 | if (!txnResponse.isSucceeded()) { 245 | // Failed to claim this resource for some reason. 246 | continue; 247 | } 248 | 249 | logger.info("Successfully claimed resource {}.", resourcePathString); 250 | return new ResourcePair(clusterId, generatorId); 251 | } 252 | } 253 | } 254 | 255 | return claimResource(etcd, maxGeneratorCount, clusterIds, giveUpAfter, waitWhenNoResourcesAvailable); 256 | } 257 | 258 | static String resourceKey(Integer clusterId, int generatorId) { 259 | return REGISTRY_PREFIX + clusterId + ":" + generatorId; 260 | } 261 | 262 | private void awaitLatchUnlessItTakesTooLong(CountDownLatch latch, Instant giveUpAfter) 263 | throws IOException, InterruptedException { 264 | if (giveUpAfter == null) { 265 | latch.await(); 266 | } else { 267 | Instant now = Instant.now(); 268 | if (!giveUpAfter.isAfter(now)) throw new IOException("Process timed out."); 269 | 270 | boolean success = latch.await(Duration.between(now, giveUpAfter).toMillis(), TimeUnit.MILLISECONDS); 271 | if (!success) { 272 | close(); 273 | throw new IOException("Process timed out."); 274 | } 275 | } 276 | } 277 | 278 | /** 279 | * Relinquish a claimed resource. 280 | */ 281 | private void relinquishResource() { 282 | logger.debug("Relinquishing claimed registry resource {}:{}.", clusterId, generatorId); 283 | 284 | String resourcePathString = resourceKey(clusterId, generatorId); 285 | ByteSequence resourcePath = ByteSequence.from(resourcePathString, StandardCharsets.UTF_8); 286 | 287 | try { 288 | kvClient.delete(resourcePath).get(5, TimeUnit.SECONDS); 289 | } catch (InterruptedException e) { 290 | Thread.currentThread().interrupt(); 291 | } catch (ExecutionException | TimeoutException e) { 292 | logger.error("Failed to revoke Etcd lease.", e); 293 | } 294 | } 295 | 296 | public int getClusterId() { 297 | return clusterId; 298 | } 299 | 300 | public int getGeneratorId() { 301 | return generatorId; 302 | } 303 | 304 | public void close() { 305 | relinquishResource(); 306 | } 307 | 308 | public String getRegistryEntry() { 309 | return registryEntry; 310 | } 311 | } 312 | -------------------------------------------------------------------------------- /uniqueid-etcd/src/main/java/org/lable/oss/uniqueid/etcd/ResourceClaim.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2014 Lable (info@lable.nl) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.lable.oss.uniqueid.etcd; 17 | 18 | import io.etcd.jetcd.*; 19 | import io.etcd.jetcd.kv.GetResponse; 20 | import io.etcd.jetcd.kv.TxnResponse; 21 | import io.etcd.jetcd.lease.LeaseGrantResponse; 22 | import io.etcd.jetcd.lock.LockResponse; 23 | import io.etcd.jetcd.op.Cmp; 24 | import io.etcd.jetcd.op.CmpTarget; 25 | import io.etcd.jetcd.op.Op; 26 | import io.etcd.jetcd.options.GetOption; 27 | import io.etcd.jetcd.options.OptionsUtil; 28 | import io.etcd.jetcd.options.PutOption; 29 | import io.etcd.jetcd.options.WatchOption; 30 | import io.etcd.jetcd.support.CloseableClient; 31 | import io.etcd.jetcd.watch.WatchEvent; 32 | import org.slf4j.Logger; 33 | import org.slf4j.LoggerFactory; 34 | 35 | import java.io.Closeable; 36 | import java.io.IOException; 37 | import java.nio.charset.StandardCharsets; 38 | import java.time.Duration; 39 | import java.time.Instant; 40 | import java.util.ArrayList; 41 | import java.util.List; 42 | import java.util.Timer; 43 | import java.util.TimerTask; 44 | import java.util.concurrent.CountDownLatch; 45 | import java.util.concurrent.ExecutionException; 46 | import java.util.concurrent.TimeUnit; 47 | import java.util.concurrent.TimeoutException; 48 | import java.util.stream.Collectors; 49 | 50 | /** 51 | * Represents a claim on resource (represented by an int) from a finite pool of resources negotiated through a 52 | * queueing protocol facilitated by a ZooKeeper-quorum. 53 | */ 54 | public class ResourceClaim implements Closeable { 55 | static final String POOL_PREFIX = "pool/"; 56 | static final ByteSequence POOL_KEY = ByteSequence.from(POOL_PREFIX, StandardCharsets.UTF_8); 57 | static final ByteSequence LOCK_NAME = ByteSequence.from("unique-id-resource-lock", StandardCharsets.UTF_8); 58 | 59 | final static Logger logger = LoggerFactory.getLogger(ResourceClaim.class); 60 | 61 | final int clusterId; 62 | final int generatorId; 63 | 64 | final int poolSize; 65 | final Client etcd; 66 | final Lease leaseClient; 67 | 68 | long leaseId; 69 | 70 | protected State state; 71 | 72 | protected List closeables = new ArrayList<>(); 73 | 74 | ResourceClaim(Client etcd, 75 | int maxGeneratorCount, 76 | List clusterIds, 77 | Duration timeout) throws IOException { 78 | state = State.INITIALIZING; 79 | logger.debug("Acquiring resource-claim…"); 80 | 81 | timeout = timeout == null ? Duration.ofMinutes(5) : timeout; 82 | Instant giveUpAfter = Instant.now().plus(timeout); 83 | 84 | this.poolSize = maxGeneratorCount; 85 | this.etcd = etcd; 86 | 87 | // Keep the lease client around, because if we try to instantiate it during shutdown of a Java application when 88 | // the resource is likely to be released, it will fail because further down the stack an attempt is made to 89 | // register a shutdown handler, which fails because the application is already shutting down. So we instantiate 90 | // this here and keep it. 91 | this.leaseClient = etcd.getLeaseClient(); 92 | 93 | KV kvClient = etcd.getKVClient(); 94 | 95 | try { 96 | LeaseGrantResponse lease = etcd.getLeaseClient().grant(5).get(); 97 | leaseId = lease.getID(); 98 | 99 | // Keep the lease alive until we are done. 100 | CloseableClient leaseKeepAlive = EtcdHelper.keepLeaseAlive(etcd, leaseId, this::close); 101 | 102 | // Release the lease when closed. 103 | closeables.add(leaseKeepAlive::close); 104 | 105 | // Acquire the lock. This makes sure we are the only process claiming a resource. 106 | LockResponse lock; 107 | try { 108 | lock = etcd.getLockClient() 109 | .lock(LOCK_NAME, leaseId) 110 | .get(timeout.toMillis(), TimeUnit.MILLISECONDS); 111 | } catch (TimeoutException e) { 112 | close(); 113 | throw new IOException("Process timed out."); 114 | } 115 | 116 | if (logger.isDebugEnabled()) { 117 | logger.debug("Acquired lock: {}.", lock.getKey().toString(StandardCharsets.UTF_8)); 118 | } 119 | 120 | ResourcePair resourcePair = claimResource(kvClient, maxGeneratorCount, clusterIds, giveUpAfter); 121 | this.clusterId = resourcePair.clusterId; 122 | this.generatorId = resourcePair.generatorId; 123 | 124 | // Release the lock. If this line is not reached due to exceptions raised, the lock will automatically 125 | // be removed when the lease holding it expires. 126 | etcd.getLockClient().unlock(lock.getKey()).get(); 127 | } catch (ExecutionException e) { 128 | close(); 129 | throw new IOException(e); 130 | } catch (InterruptedException e) { 131 | close(); 132 | Thread.currentThread().interrupt(); 133 | throw new IOException(e); 134 | } 135 | state = State.HAS_CLAIM; 136 | 137 | logger.debug("Resource-claim acquired ({}/{}).", clusterId, generatorId); 138 | } 139 | 140 | /** 141 | * Claim a resource. 142 | * 143 | * @param etcd Etcd connection. 144 | * @param maxGeneratorCount Maximum number of generators possible. 145 | * @return A resource claim. 146 | */ 147 | public static ResourceClaim claim(Client etcd, 148 | int maxGeneratorCount, 149 | List clusterIds) throws IOException { 150 | return new ResourceClaim(etcd, maxGeneratorCount, clusterIds, Duration.ofMinutes(10)); 151 | } 152 | 153 | /** 154 | * Claim a resource. 155 | * 156 | * @param etcd Etcd connection. 157 | * @param maxGeneratorCount Maximum number of generators possible. 158 | * @param clusterIds Cluster Ids available to use. 159 | * @param timeout Time out if the process takes longer than this. 160 | * @return A resource claim. 161 | */ 162 | public static ResourceClaim claim(Client etcd, 163 | int maxGeneratorCount, 164 | List clusterIds, 165 | Duration timeout) throws IOException { 166 | return new ResourceClaim(etcd, maxGeneratorCount, clusterIds, timeout); 167 | } 168 | 169 | /** 170 | * Get the claimed resource. 171 | * 172 | * @return The resource claimed. 173 | * @throws IllegalStateException Thrown when the claim is no longer held. 174 | */ 175 | public int getClusterId() { 176 | if (state != State.HAS_CLAIM) { 177 | throw new IllegalStateException("Resource claim not held."); 178 | } 179 | return clusterId; 180 | } 181 | 182 | public int getGeneratorId() { 183 | if (state != State.HAS_CLAIM) { 184 | throw new IllegalStateException("Resource claim not held."); 185 | } 186 | return generatorId; 187 | } 188 | 189 | /** 190 | * Relinquish the claim to this resource, and release it back to the resource pool. 191 | */ 192 | public void close() { 193 | close(false); 194 | } 195 | 196 | public void close(boolean nodeAlreadyDeleted) { 197 | if (state == State.CLAIM_RELINQUISHED) { 198 | // Already relinquished nothing to do. 199 | return; 200 | } 201 | 202 | logger.debug("Closing resource-claim ({}).", resourceKey(clusterId, generatorId)); 203 | 204 | // No need to delete the node if the reason we are closing is the deletion of said node. 205 | if (nodeAlreadyDeleted) { 206 | state = State.CLAIM_RELINQUISHED; 207 | return; 208 | } 209 | 210 | if (state == State.HAS_CLAIM) { 211 | state = State.CLAIM_RELINQUISHED; 212 | // Hang on to the claimed resource without using it for a short while to facilitate clock skew. 213 | // That is, if any participant is generating IDs with a slightly skewed clock, it can generate IDs that 214 | // overlap with the ones generated by the participant who successfully claims the same resource before or 215 | // after. By hanging on to each resource for a bit a slight clock skew may be handled gracefully. 216 | new Timer().schedule(new TimerTask() { 217 | @Override 218 | public void run() { 219 | relinquishResource(); 220 | } 221 | // Two seconds seems reasonable. The NTP docs state that clocks running more than 128ms out of sync are 222 | // rare under normal conditions. 223 | }, TimeUnit.SECONDS.toMillis(2)); 224 | } else { 225 | state = State.CLAIM_RELINQUISHED; 226 | } 227 | 228 | for (Closeable closeable : closeables) { 229 | try { 230 | closeable.close(); 231 | } catch (IOException e) { 232 | logger.warn("Failed to close resource properly.", e); 233 | } 234 | } 235 | } 236 | 237 | /** 238 | * Try to claim an available resource from the resource pool. 239 | * 240 | * @param kvClient Etcd KV client. 241 | * @param maxGeneratorCount Maximum number of generators possible. 242 | * @param clusterIds Cluster Ids available to use. 243 | * @param giveUpAfter Give up after this instant in time. 244 | * @return The claimed resource. 245 | */ 246 | ResourcePair claimResource(KV kvClient, int maxGeneratorCount, List clusterIds, Instant giveUpAfter) 247 | throws InterruptedException, IOException, ExecutionException { 248 | 249 | logger.debug("Trying to claim a resource."); 250 | 251 | int poolSize = maxGeneratorCount * clusterIds.size(); 252 | 253 | GetOption getOptions = GetOption.builder() 254 | .withKeysOnly(true) 255 | .withRange(OptionsUtil.prefixEndOf(POOL_KEY)) 256 | .build(); 257 | GetResponse get = kvClient.get(POOL_KEY, getOptions).get(); 258 | 259 | List claimedResources = get.getKvs().stream() 260 | .map(KeyValue::getKey) 261 | .collect(Collectors.toList()); 262 | 263 | if (claimedResources.size() >= poolSize) { 264 | logger.debug("No resources available at the moment (pool size: {}), waiting.", poolSize); 265 | // No resources available. Wait for a resource to become available. 266 | final CountDownLatch latch = new CountDownLatch(1); 267 | Watch.Watcher watcher = etcd.getWatchClient().watch( 268 | POOL_KEY, 269 | WatchOption 270 | .builder() 271 | .withRange(OptionsUtil.prefixEndOf(POOL_KEY)) 272 | .build(), 273 | watchResponse -> latch.countDown() 274 | ); 275 | awaitLatchUnlessItTakesTooLong(latch, giveUpAfter); 276 | watcher.close(); 277 | return claimResource(kvClient, maxGeneratorCount, clusterIds, giveUpAfter); 278 | } 279 | 280 | // Try to claim an available resource. 281 | for (Integer clusterId : clusterIds) { 282 | for (int generatorId = 0; generatorId < maxGeneratorCount; generatorId++) { 283 | String resourcePathString = resourceKey(clusterId, generatorId); 284 | ByteSequence resourcePath = ByteSequence.from(resourcePathString, StandardCharsets.UTF_8); 285 | if (!claimedResources.contains(resourcePath)) { 286 | logger.debug("Trying to claim seemingly available resource {}.", resourcePathString); 287 | TxnResponse txnResponse = etcd.getKVClient().txn() 288 | .If( 289 | // Version == 0 means the key does not exist. 290 | new Cmp(resourcePath, Cmp.Op.EQUAL, CmpTarget.version(0)) 291 | ).Then( 292 | Op.put( 293 | resourcePath, 294 | ByteSequence.EMPTY, 295 | PutOption.builder().withLeaseId(leaseId).build() 296 | ) 297 | ).commit().get(); 298 | 299 | if (!txnResponse.isSucceeded()) { 300 | // Failed to claim this resource for some reason. 301 | continue; 302 | } 303 | 304 | closeables.add(etcd.getWatchClient().watch(resourcePath, watchResponse -> { 305 | for (WatchEvent event : watchResponse.getEvents()) { 306 | if (event.getEventType() == WatchEvent.EventType.DELETE) { 307 | // Invalidate our claim when the node is deleted by some other process. 308 | logger.debug("Resource-claim node unexpectedly deleted ({})", resourcePathString); 309 | close(true); 310 | } 311 | } 312 | })); 313 | 314 | logger.debug("Successfully claimed resource {}.", resourcePathString); 315 | return new ResourcePair(clusterId, generatorId); 316 | } 317 | } 318 | } 319 | 320 | return claimResource(kvClient, maxGeneratorCount, clusterIds, giveUpAfter); 321 | } 322 | 323 | /** 324 | * Relinquish a claimed resource. 325 | */ 326 | private void relinquishResource() { 327 | logger.debug("Relinquishing claimed resource {}:{}.", clusterId, generatorId); 328 | try { 329 | leaseClient.revoke(leaseId).get(); 330 | } catch (InterruptedException e) { 331 | Thread.currentThread().interrupt(); 332 | } catch (ExecutionException e) { 333 | logger.error("Failed to revoke Etcd lease.", e); 334 | } 335 | } 336 | 337 | static String resourceKey(Integer clusterId, int generatorId) { 338 | return POOL_PREFIX + clusterId + ":" + generatorId; 339 | } 340 | 341 | private void awaitLatchUnlessItTakesTooLong(CountDownLatch latch, Instant giveUpAfter) 342 | throws IOException, InterruptedException { 343 | if (giveUpAfter == null) { 344 | latch.await(); 345 | } else { 346 | Instant now = Instant.now(); 347 | if (!giveUpAfter.isAfter(now)) throw new IOException("Process timed out."); 348 | 349 | boolean success = latch.await(Duration.between(now, giveUpAfter).toMillis(), TimeUnit.MILLISECONDS); 350 | if (!success) { 351 | close(); 352 | throw new IOException("Process timed out."); 353 | } 354 | } 355 | } 356 | 357 | /** 358 | * Internal state of this ResourceClaim. 359 | */ 360 | public enum State { 361 | INITIALIZING, 362 | HAS_CLAIM, 363 | CLAIM_RELINQUISHED 364 | } 365 | } 366 | -------------------------------------------------------------------------------- /uniqueid-etcd/src/main/java/org/lable/oss/uniqueid/etcd/ResourcePair.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2014 Lable (info@lable.nl) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.lable.oss.uniqueid.etcd; 17 | 18 | class ResourcePair { 19 | int clusterId; 20 | int generatorId; 21 | 22 | public ResourcePair(Integer clusterId, int generatorId) { 23 | this.clusterId = clusterId; 24 | this.generatorId = generatorId; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /uniqueid-etcd/src/main/java/org/lable/oss/uniqueid/etcd/SynchronizedGeneratorIdentity.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2014 Lable (info@lable.nl) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.lable.oss.uniqueid.etcd; 17 | 18 | import io.etcd.jetcd.ByteSequence; 19 | import io.etcd.jetcd.Client; 20 | import org.lable.oss.uniqueid.GeneratorException; 21 | import org.lable.oss.uniqueid.GeneratorIdentityHolder; 22 | import org.lable.oss.uniqueid.bytes.Blueprint; 23 | import org.slf4j.Logger; 24 | import org.slf4j.LoggerFactory; 25 | 26 | import java.io.IOException; 27 | import java.nio.charset.StandardCharsets; 28 | import java.time.Duration; 29 | import java.util.List; 30 | import java.util.function.Supplier; 31 | 32 | public class SynchronizedGeneratorIdentity implements GeneratorIdentityHolder { 33 | private static final Logger logger = LoggerFactory.getLogger(SynchronizedGeneratorIdentity.class); 34 | 35 | private final Client client; 36 | private final List clusterIds; 37 | private final Supplier claimDurationSupplier; 38 | private final Supplier acquisitionTimeoutSupplier; 39 | 40 | ResourceClaim resourceClaim = null; 41 | 42 | public SynchronizedGeneratorIdentity(Client client, 43 | List clusterIds, 44 | Supplier claimDurationSupplier, 45 | Supplier acquisitionTimeoutSupplier) { 46 | this.client = client; 47 | this.clusterIds = clusterIds; 48 | this.claimDurationSupplier = claimDurationSupplier; 49 | this.acquisitionTimeoutSupplier = acquisitionTimeoutSupplier == null 50 | ? () -> null 51 | : acquisitionTimeoutSupplier; 52 | } 53 | 54 | /** 55 | * Create a new {@link SynchronizedGeneratorIdentity} instance. 56 | *

57 | * By using a {@link Supplier} instead of static longs for the claim duration and the acquisition timeout, these 58 | * values can be dynamically reconfigured at runtime. 59 | * 60 | * @param endpoints Addresses of the Etcd cluster (comma-separated). 61 | * @param namespace Namespace of the unique-id keys in Etcd. 62 | * @param claimDurationSupplier Provides the amount of time a claim to a generator-ID should be held. 63 | * @param acquisitionTimeoutSupplier Provides the amount of time the process of acquiring a generator-ID may take. 64 | * May be {@code null} to indicate that the process may wait indefinitely. 65 | * @return A {@link SynchronizedGeneratorIdentity} instance. 66 | */ 67 | public static SynchronizedGeneratorIdentity basedOn(String endpoints, 68 | String namespace, 69 | Supplier claimDurationSupplier, 70 | Supplier acquisitionTimeoutSupplier) 71 | throws IOException { 72 | Client client = Client.builder() 73 | .endpoints(endpoints.split(",")) 74 | .namespace(ByteSequence.from(namespace, StandardCharsets.UTF_8)) 75 | .build(); 76 | List clusterIds = ClusterID.get(client); 77 | 78 | return new SynchronizedGeneratorIdentity( 79 | client, clusterIds, claimDurationSupplier, acquisitionTimeoutSupplier 80 | ); 81 | } 82 | 83 | /** 84 | * Create a new {@link SynchronizedGeneratorIdentity} instance. 85 | * 86 | * @param endpoints Addresses of the Etcd cluster (comma-separated). 87 | * @param namespace Namespace of the unique-id keys in Etcd. 88 | * @param claimDuration How long a claim to a generator-ID should be held, in milliseconds. 89 | * @return A {@link SynchronizedGeneratorIdentity} instance. 90 | */ 91 | public static SynchronizedGeneratorIdentity basedOn(String endpoints, 92 | String namespace, 93 | Long claimDuration) 94 | throws IOException { 95 | Client client = Client.builder() 96 | .endpoints(endpoints.split(",")) 97 | .namespace(ByteSequence.from(namespace, StandardCharsets.UTF_8)) 98 | .build(); 99 | List clusterIds = ClusterID.get(client); 100 | Supplier durationSupplier = () -> Duration.ofMillis(claimDuration); 101 | 102 | return new SynchronizedGeneratorIdentity(client, clusterIds, durationSupplier, null); 103 | } 104 | 105 | @Override 106 | public int getClusterId() throws GeneratorException { 107 | acquireResourceClaim(); 108 | 109 | try { 110 | return resourceClaim.getClusterId(); 111 | } catch (IllegalStateException e) { 112 | // Claim expired? 113 | relinquishResourceClaim(); 114 | acquireResourceClaim(); 115 | return resourceClaim.getClusterId(); 116 | } 117 | } 118 | 119 | @Override 120 | public int getGeneratorId() throws GeneratorException { 121 | acquireResourceClaim(); 122 | 123 | try { 124 | return resourceClaim.getGeneratorId(); 125 | } catch (IllegalStateException e) { 126 | // Claim expired? 127 | relinquishResourceClaim(); 128 | acquireResourceClaim(); 129 | return resourceClaim.getGeneratorId(); 130 | } 131 | } 132 | 133 | 134 | public synchronized void relinquishResourceClaim() { 135 | if (resourceClaim == null) return; 136 | resourceClaim.close(); 137 | resourceClaim = null; 138 | } 139 | 140 | private synchronized void acquireResourceClaim() throws GeneratorException { 141 | if (resourceClaim != null) return; 142 | 143 | resourceClaim = acquireResourceClaim(0); 144 | } 145 | 146 | private ResourceClaim acquireResourceClaim(int retries) throws GeneratorException { 147 | try { 148 | return ExpiringResourceClaim.claimExpiring( 149 | client, 150 | Blueprint.MAX_GENERATOR_ID + 1, 151 | clusterIds, 152 | claimDurationSupplier == null ? null : claimDurationSupplier.get(), 153 | acquisitionTimeoutSupplier.get() 154 | ); 155 | } catch (IOException e) { 156 | logger.warn( 157 | "Connection to Etcd failed, retrying resource claim acquisition, attempt {}.", 158 | retries + 1 159 | ); 160 | if (retries < 3) { 161 | return acquireResourceClaim(retries + 1); 162 | } else { 163 | throw new GeneratorException(e); 164 | } 165 | } 166 | } 167 | 168 | @Override 169 | public void close() throws IOException { 170 | if (resourceClaim != null) { 171 | resourceClaim.close(); 172 | } 173 | } 174 | } 175 | -------------------------------------------------------------------------------- /uniqueid-etcd/src/main/java/org/lable/oss/uniqueid/etcd/SynchronizedUniqueIDGeneratorFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2014 Lable (info@lable.nl) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.lable.oss.uniqueid.etcd; 17 | 18 | import io.etcd.jetcd.Client; 19 | import org.lable.oss.uniqueid.BaseUniqueIDGenerator; 20 | import org.lable.oss.uniqueid.Clock; 21 | import org.lable.oss.uniqueid.GeneratorIdentityHolder; 22 | import org.lable.oss.uniqueid.IDGenerator; 23 | import org.lable.oss.uniqueid.bytes.Mode; 24 | 25 | import java.io.IOException; 26 | import java.util.List; 27 | 28 | /** 29 | * Create an {@link IDGenerator} capable of generating unique identifiers in a distributed environment with multiple 30 | * services generating them. To do this, the {@link GeneratorIdentityHolder} it uses acquires a temporary claim on a 31 | * generator ID negotiated via an Etcd cluster. 32 | *

33 | * Because claimed generator IDs are automatically returned to the pool after a set time 34 | * ({@link ExpiringResourceClaim#DEFAULT_CLAIM_HOLD}), there is no guarantee that IDs generated by the same 35 | * {@link IDGenerator} instance share the same generator ID. 36 | */ 37 | public class SynchronizedUniqueIDGeneratorFactory { 38 | /** 39 | * Get the synchronized ID generator instance. 40 | * 41 | * @param etcd Connection to the Etcd cluster. 42 | * @param clock Clock implementation. 43 | * @param mode Generator mode. 44 | * @return An instance of this class. 45 | * @throws IOException Thrown when something went wrong trying to find the cluster ID or trying to claim a 46 | * generator ID. 47 | */ 48 | public static synchronized IDGenerator generatorFor(Client etcd, Clock clock, Mode mode) 49 | throws IOException { 50 | 51 | final List clusterIds = ClusterID.get(etcd); 52 | SynchronizedGeneratorIdentity generatorIdentityHolder = 53 | new SynchronizedGeneratorIdentity(etcd, clusterIds, null, null); 54 | 55 | return generatorFor(generatorIdentityHolder, clock, mode); 56 | } 57 | 58 | /** 59 | * Get the synchronized ID generator instance. 60 | * 61 | * @param etcd Connection to the Etcd cluster. 62 | * @param mode Generator mode. 63 | * @return An instance of this class. 64 | * @throws IOException Thrown when something went wrong trying to find the cluster ID or trying to claim a 65 | * generator ID. 66 | */ 67 | public static synchronized IDGenerator generatorFor(Client etcd, Mode mode) 68 | throws IOException { 69 | return generatorFor(etcd, null, mode); 70 | } 71 | 72 | /** 73 | * Get the synchronized ID generator instance. 74 | * 75 | * @param synchronizedGeneratorIdentity An instance of {@link SynchronizedGeneratorIdentity} to (re)use for 76 | * acquiring the generator ID. 77 | * @param clock Clock implementation. 78 | * @param mode Generator mode. 79 | * @return An instance of this class. 80 | */ 81 | public static synchronized IDGenerator generatorFor(SynchronizedGeneratorIdentity synchronizedGeneratorIdentity, 82 | Clock clock, 83 | Mode mode) { 84 | return new BaseUniqueIDGenerator(synchronizedGeneratorIdentity, clock, mode); 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /uniqueid-etcd/src/test/java/org/lable/oss/uniqueid/etcd/AcquisitionTimeoutIT.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2014 Lable (info@lable.nl) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.lable.oss.uniqueid.etcd; 17 | 18 | import io.etcd.jetcd.ByteSequence; 19 | import io.etcd.jetcd.Client; 20 | import io.etcd.jetcd.support.CloseableClient; 21 | import org.junit.*; 22 | import org.junit.rules.ExpectedException; 23 | 24 | import java.io.IOException; 25 | import java.nio.charset.StandardCharsets; 26 | import java.time.Duration; 27 | import java.util.Collections; 28 | import java.util.Timer; 29 | import java.util.TimerTask; 30 | import java.util.concurrent.ExecutionException; 31 | 32 | import static org.hamcrest.CoreMatchers.is; 33 | import static org.hamcrest.MatcherAssert.assertThat; 34 | import static org.hamcrest.Matchers.greaterThanOrEqualTo; 35 | import static org.hamcrest.Matchers.lessThan; 36 | import static org.hamcrest.core.CombinableMatcher.both; 37 | import static org.lable.oss.uniqueid.etcd.ResourceClaim.LOCK_NAME; 38 | 39 | public class AcquisitionTimeoutIT { 40 | @ClassRule 41 | public static final EtcdTestCluster etcd = new EtcdTestCluster("test-etcd", 1); 42 | 43 | @Rule 44 | public ExpectedException thrown = ExpectedException.none(); 45 | 46 | static Client client; 47 | 48 | CloseableClient leaseKeepAlive; 49 | ByteSequence lockKey; 50 | 51 | @BeforeClass 52 | public static void setup() throws InterruptedException, ExecutionException { 53 | client = Client.builder() 54 | .endpoints(etcd.getClientEndpoints()) 55 | .namespace(ByteSequence.from("unique-id/", StandardCharsets.UTF_8)) 56 | .build(); 57 | 58 | TestHelper.prepareClusterID(client, 5); 59 | } 60 | 61 | @Before 62 | public void before() throws ExecutionException, InterruptedException { 63 | long leaseId = client.getLeaseClient().grant(10).get().getID(); 64 | 65 | leaseKeepAlive = EtcdHelper.keepLeaseAlive(client, leaseId, null); 66 | lockKey = client.getLockClient().lock(LOCK_NAME, leaseId).get().getKey(); 67 | } 68 | 69 | @After 70 | public void after() throws InterruptedException, ExecutionException { 71 | client.getLockClient().unlock(lockKey).get(); 72 | leaseKeepAlive.close(); 73 | } 74 | 75 | @Test 76 | public void timeoutTest() throws IOException { 77 | thrown.expect(IOException.class); 78 | thrown.expectMessage("Process timed out."); 79 | 80 | ResourceClaim claim = ExpiringResourceClaim.claimExpiring( 81 | client, 82 | 64, 83 | Collections.singletonList(5), 84 | Duration.ofSeconds(2), 85 | Duration.ofSeconds(2) 86 | ); 87 | 88 | claim.getGeneratorId(); 89 | claim.close(); 90 | } 91 | 92 | @Test 93 | public void timeoutTestNull() throws IOException { 94 | thrown = ExpectedException.none(); 95 | 96 | Timer timer = new Timer(); 97 | timer.schedule(new TimerTask() { 98 | @Override 99 | public void run() { 100 | System.out.println("TIMER"); 101 | try { 102 | client.getLockClient().unlock(lockKey).get(); 103 | } catch (InterruptedException | ExecutionException e) { 104 | e.printStackTrace(); 105 | } 106 | } 107 | }, 2000); 108 | 109 | ResourceClaim claim = ExpiringResourceClaim.claimExpiring( 110 | client, 111 | 64, 112 | Collections.singletonList(5), 113 | Duration.ofSeconds(2), 114 | Duration.ofSeconds(5) 115 | ); 116 | 117 | int resource = claim.getGeneratorId(); 118 | assertThat(claim.state, is(ResourceClaim.State.HAS_CLAIM)); 119 | assertThat(resource, is(both(greaterThanOrEqualTo(0)).and(lessThan(64)))); 120 | 121 | claim.close(); 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /uniqueid-etcd/src/test/java/org/lable/oss/uniqueid/etcd/ClusterIDIT.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2014 Lable (info@lable.nl) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.lable.oss.uniqueid.etcd; 17 | 18 | import io.etcd.jetcd.ByteSequence; 19 | import io.etcd.jetcd.Client; 20 | import org.junit.Rule; 21 | import org.junit.Test; 22 | 23 | import java.io.IOException; 24 | import java.nio.charset.StandardCharsets; 25 | import java.util.List; 26 | import java.util.concurrent.ExecutionException; 27 | 28 | import static org.hamcrest.MatcherAssert.assertThat; 29 | import static org.hamcrest.Matchers.contains; 30 | 31 | public class ClusterIDIT { 32 | @Rule 33 | public final EtcdTestCluster etcd = new EtcdTestCluster("test-etcd", 1); 34 | 35 | @Test 36 | public void defaultTest() throws IOException { 37 | ByteSequence ns = ByteSequence.from("unique-id/", StandardCharsets.UTF_8); 38 | 39 | Client client = Client.builder() 40 | .endpoints(etcd.getClientEndpoints()) 41 | .namespace(ns) 42 | .build(); 43 | 44 | List ids = ClusterID.get(client); 45 | 46 | assertThat(ids, contains(0)); 47 | } 48 | 49 | @Test 50 | public void preconfiguredTest() throws ExecutionException, InterruptedException, IOException { 51 | ByteSequence ns = ByteSequence.from("unique-id/", StandardCharsets.UTF_8); 52 | 53 | Client client = Client.builder() 54 | .endpoints(etcd.getClientEndpoints()) 55 | .namespace(ns) 56 | .build(); 57 | 58 | client.getKVClient().put(ClusterID.CLUSTER_ID_KEY, ByteSequence.from("12".getBytes())).get(); 59 | 60 | List ids = ClusterID.get(client); 61 | 62 | assertThat(ids, contains(12)); 63 | } 64 | 65 | @Test 66 | public void preconfiguredMultipleTest() throws ExecutionException, InterruptedException, IOException { 67 | ByteSequence ns = ByteSequence.from("unique-id/", StandardCharsets.UTF_8); 68 | 69 | Client client = Client.builder() 70 | .endpoints(etcd.getClientEndpoints()) 71 | .namespace(ns) 72 | .build(); 73 | 74 | client.getKVClient().put(ClusterID.CLUSTER_ID_KEY, ByteSequence.from("12, 13".getBytes())).get(); 75 | 76 | List ids = ClusterID.get(client); 77 | 78 | assertThat(ids, contains(12, 13)); 79 | } 80 | 81 | @Test(expected = IOException.class) 82 | public void invalidValueTest() throws ExecutionException, InterruptedException, IOException { 83 | ByteSequence ns = ByteSequence.from("unique-id/", StandardCharsets.UTF_8); 84 | 85 | Client client = Client.builder() 86 | .endpoints(etcd.getClientEndpoints()) 87 | .namespace(ns) 88 | .build(); 89 | 90 | client.getKVClient().put(ClusterID.CLUSTER_ID_KEY, ByteSequence.from("BOGUS".getBytes())).get(); 91 | 92 | ClusterID.get(client); 93 | } 94 | } -------------------------------------------------------------------------------- /uniqueid-etcd/src/test/java/org/lable/oss/uniqueid/etcd/EtcdHelperIT.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2014 Lable (info@lable.nl) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.lable.oss.uniqueid.etcd; 17 | 18 | import io.etcd.jetcd.Client; 19 | import org.junit.Rule; 20 | import org.junit.Test; 21 | 22 | import java.util.concurrent.ExecutionException; 23 | 24 | import static com.github.npathai.hamcrestopt.OptionalMatchers.hasValue; 25 | import static com.github.npathai.hamcrestopt.OptionalMatchers.isEmpty; 26 | import static org.hamcrest.MatcherAssert.assertThat; 27 | import static org.lable.oss.uniqueid.etcd.EtcdHelper.*; 28 | 29 | public class EtcdHelperIT { 30 | @Rule 31 | public final EtcdTestCluster etcd = new EtcdTestCluster("test-etcd", 1); 32 | 33 | @Test 34 | public void test() throws ExecutionException, InterruptedException { 35 | Client client = Client.builder().endpoints(etcd.getClientEndpoints()).build(); 36 | 37 | assertThat(getInt(client, "a"), isEmpty()); 38 | 39 | put(client, "a", -2); 40 | 41 | assertThat(getInt(client, "a"), hasValue(-2)); 42 | 43 | delete(client, "a"); 44 | 45 | assertThat(getInt(client, "a"), isEmpty()); 46 | } 47 | } -------------------------------------------------------------------------------- /uniqueid-etcd/src/test/java/org/lable/oss/uniqueid/etcd/EtcdTestCluster.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2014 Lable (info@lable.nl) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /* 18 | * Copyright © 2015 Lable (info@lable.nl) 19 | * 20 | * Licensed under the Apache License, Version 2.0 (the "License"); 21 | * you may not use this file except in compliance with the License. 22 | * You may obtain a copy of the License at 23 | * 24 | * http://www.apache.org/licenses/LICENSE-2.0 25 | * 26 | * Unless required by applicable law or agreed to in writing, software 27 | * distributed under the License is distributed on an "AS IS" BASIS, 28 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 29 | * See the License for the specific language governing permissions and 30 | * limitations under the License. 31 | */ 32 | package org.lable.oss.uniqueid.etcd; 33 | 34 | import io.etcd.jetcd.launcher.Etcd; 35 | import io.etcd.jetcd.launcher.EtcdCluster; 36 | import org.junit.rules.TestRule; 37 | import org.junit.runner.Description; 38 | import org.junit.runners.model.Statement; 39 | 40 | import java.net.URI; 41 | import java.util.List; 42 | 43 | /** 44 | * The jetcd library dropped support for JUnit4, so we wrap the cluster ourselves. 45 | */ 46 | public class EtcdTestCluster implements TestRule { 47 | 48 | private final String clusterName; 49 | private final int nodes; 50 | private final boolean ssl; 51 | private EtcdCluster cluster; 52 | 53 | public EtcdTestCluster(String clusterName, int nodes) { 54 | this(clusterName, nodes, false); 55 | } 56 | 57 | public EtcdTestCluster(String clusterName, int nodes, boolean ssl) { 58 | this.clusterName = clusterName; 59 | this.nodes = nodes; 60 | this.ssl = ssl; 61 | } 62 | 63 | @Override 64 | public Statement apply(Statement base, Description description) { 65 | return new Statement() { 66 | @Override 67 | public void evaluate() throws Throwable { 68 | cluster = Etcd.builder() 69 | .withClusterName(clusterName) 70 | .withNodes(nodes) 71 | .withSsl(ssl) 72 | .withMountedDataDirectory(false) 73 | .build(); 74 | 75 | cluster.start(); 76 | try { 77 | base.evaluate(); 78 | } finally { 79 | cluster.close(); 80 | cluster = null; 81 | } 82 | } 83 | }; 84 | } 85 | 86 | public List getClientEndpoints() { 87 | return cluster.clientEndpoints(); 88 | } 89 | } -------------------------------------------------------------------------------- /uniqueid-etcd/src/test/java/org/lable/oss/uniqueid/etcd/ExpiringResourceClaimIT.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2014 Lable (info@lable.nl) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.lable.oss.uniqueid.etcd; 17 | 18 | import io.etcd.jetcd.ByteSequence; 19 | import io.etcd.jetcd.Client; 20 | import org.junit.Before; 21 | import org.junit.Rule; 22 | import org.junit.Test; 23 | import org.junit.rules.ExpectedException; 24 | 25 | import java.io.IOException; 26 | import java.nio.charset.StandardCharsets; 27 | import java.time.Duration; 28 | import java.util.Collections; 29 | import java.util.concurrent.ExecutionException; 30 | import java.util.concurrent.TimeUnit; 31 | 32 | import static com.github.npathai.hamcrestopt.OptionalMatchers.isEmpty; 33 | import static org.hamcrest.CoreMatchers.is; 34 | import static org.hamcrest.MatcherAssert.assertThat; 35 | import static org.hamcrest.Matchers.greaterThanOrEqualTo; 36 | import static org.hamcrest.Matchers.lessThan; 37 | import static org.hamcrest.core.CombinableMatcher.both; 38 | import static org.lable.oss.uniqueid.etcd.ResourceClaim.POOL_PREFIX; 39 | import static org.lable.oss.uniqueid.etcd.ResourceClaim.resourceKey; 40 | 41 | public class ExpiringResourceClaimIT { 42 | @Rule 43 | public final EtcdTestCluster etcd = new EtcdTestCluster("test-etcd", 1); 44 | 45 | @Rule 46 | public ExpectedException thrown = ExpectedException.none(); 47 | 48 | Client client; 49 | 50 | @Before 51 | public void before() throws IOException, InterruptedException, ExecutionException { 52 | client = Client.builder() 53 | .endpoints(etcd.getClientEndpoints()) 54 | .namespace(ByteSequence.from("unique-id", StandardCharsets.UTF_8)) 55 | .build(); 56 | 57 | TestHelper.prepareClusterID(client, 5); 58 | } 59 | 60 | @Test 61 | public void expirationTest() throws IOException, InterruptedException { 62 | ResourceClaim claim = ExpiringResourceClaim.claimExpiring( 63 | client, 64 | 64, 65 | Collections.singletonList(5), 66 | Duration.ofSeconds(4), 67 | null 68 | ); 69 | int resource = claim.getGeneratorId(); 70 | assertThat(claim.state, is(ResourceClaim.State.HAS_CLAIM)); 71 | assertThat(resource, is(both(greaterThanOrEqualTo(0)).and(lessThan(64)))); 72 | 73 | TimeUnit.SECONDS.sleep(2); 74 | 75 | int resource2 = claim.getGeneratorId(); 76 | assertThat(claim.state, is(ResourceClaim.State.HAS_CLAIM)); 77 | assertThat(resource, is(resource2)); 78 | 79 | // Wait for the resource to expire. 80 | TimeUnit.SECONDS.sleep(4); 81 | 82 | assertThat(claim.state, is(ResourceClaim.State.CLAIM_RELINQUISHED)); 83 | thrown.expect(IllegalStateException.class); 84 | thrown.expectMessage("Resource claim not held."); 85 | claim.getGeneratorId(); 86 | } 87 | 88 | @Test 89 | public void deletionTest() throws IOException, InterruptedException, ExecutionException { 90 | ResourceClaim claim = ExpiringResourceClaim.claimExpiring( 91 | client, 92 | 64, 93 | Collections.singletonList(5), 94 | // Very long expiration that shouldn't interfere with this test. 95 | Duration.ofSeconds(20), 96 | null 97 | ); 98 | int resource = claim.getGeneratorId(); 99 | assertThat(claim.state, is(ResourceClaim.State.HAS_CLAIM)); 100 | assertThat(resource, is(both(greaterThanOrEqualTo(0)).and(lessThan(64)))); 101 | 102 | // Remove resource manually. Claim should get relinquished via watcher. 103 | EtcdHelper.delete(client, resourceKey(claim.getClusterId(), claim.getGeneratorId())); 104 | 105 | TimeUnit.MILLISECONDS.sleep(500); 106 | 107 | assertThat(claim.state, is(ResourceClaim.State.CLAIM_RELINQUISHED)); 108 | thrown.expect(IllegalStateException.class); 109 | thrown.expectMessage("Resource claim not held."); 110 | claim.getGeneratorId(); 111 | } 112 | 113 | @Test 114 | public void resourceRemovedTest() throws IOException, InterruptedException, ExecutionException { 115 | ResourceClaim claim = ExpiringResourceClaim.claimExpiring( 116 | client, 117 | 64, 118 | Collections.singletonList(5), 119 | // Very long expiration that shouldn't interfere with this test. 120 | Duration.ofSeconds(20), 121 | null 122 | ); 123 | int resource = claim.getGeneratorId(); 124 | assertThat(claim.state, is(ResourceClaim.State.HAS_CLAIM)); 125 | assertThat(resource, is(both(greaterThanOrEqualTo(0)).and(lessThan(64)))); 126 | 127 | claim.close(); 128 | 129 | assertThat(EtcdHelper.getInt(client, POOL_PREFIX + resource), isEmpty()); 130 | 131 | thrown.expect(IllegalStateException.class); 132 | thrown.expectMessage("Resource claim not held."); 133 | claim.getGeneratorId(); 134 | } 135 | } -------------------------------------------------------------------------------- /uniqueid-etcd/src/test/java/org/lable/oss/uniqueid/etcd/HighGeneratorCountIT.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2014 Lable (info@lable.nl) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.lable.oss.uniqueid.etcd; 17 | 18 | import io.etcd.jetcd.ByteSequence; 19 | import io.etcd.jetcd.Client; 20 | import org.junit.BeforeClass; 21 | import org.junit.ClassRule; 22 | import org.junit.Test; 23 | import org.lable.oss.uniqueid.IDGenerator; 24 | import org.lable.oss.uniqueid.bytes.Blueprint; 25 | import org.lable.oss.uniqueid.bytes.IDBuilder; 26 | import org.lable.oss.uniqueid.bytes.Mode; 27 | 28 | import java.nio.charset.StandardCharsets; 29 | import java.util.concurrent.ExecutionException; 30 | 31 | import static org.hamcrest.CoreMatchers.is; 32 | import static org.junit.Assert.assertThat; 33 | import static org.lable.oss.uniqueid.etcd.SynchronizedUniqueIDGeneratorFactory.generatorFor; 34 | 35 | public class HighGeneratorCountIT { 36 | @ClassRule 37 | public static final EtcdTestCluster etcd = new EtcdTestCluster("test-etcd", 1); 38 | 39 | final static int CLUSTER_ID_A = 4; 40 | final static int CLUSTER_ID_B = 5; 41 | 42 | static Client client; 43 | 44 | @BeforeClass 45 | public static void setup() throws InterruptedException, ExecutionException { 46 | client = Client.builder() 47 | .endpoints(etcd.getClientEndpoints()) 48 | .namespace(ByteSequence.from("unique-id/", StandardCharsets.UTF_8)) 49 | .build(); 50 | 51 | TestHelper.prepareClusterID(client, CLUSTER_ID_A, CLUSTER_ID_B); 52 | for (int i = 0; i < 2047; i++) { 53 | EtcdHelper.put(client, "pool/4:" + i); 54 | } 55 | } 56 | 57 | @Test 58 | public void above255Test() throws Exception { 59 | IDGenerator generator = generatorFor(client, Mode.TIME_SEQUENTIAL); 60 | byte[] result = generator.generate(); 61 | Blueprint blueprint = IDBuilder.parse(result); 62 | assertThat(result.length, is(8)); 63 | assertThat(blueprint.getClusterId(), is(CLUSTER_ID_A)); 64 | assertThat(blueprint.getGeneratorId(), is(2047)); 65 | 66 | IDGenerator generator2 = generatorFor(client, Mode.TIME_SEQUENTIAL); 67 | result = generator2.generate(); 68 | blueprint = IDBuilder.parse(result); 69 | assertThat(result.length, is(8)); 70 | assertThat(blueprint.getClusterId(), is(CLUSTER_ID_B)); 71 | assertThat(blueprint.getGeneratorId(), is(0)); 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /uniqueid-etcd/src/test/java/org/lable/oss/uniqueid/etcd/MultipleClusterIdsIT.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2014 Lable (info@lable.nl) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.lable.oss.uniqueid.etcd; 17 | 18 | import io.etcd.jetcd.ByteSequence; 19 | import io.etcd.jetcd.Client; 20 | import org.junit.*; 21 | import org.junit.rules.ExpectedException; 22 | import org.lable.oss.uniqueid.ByteArray; 23 | import org.lable.oss.uniqueid.GeneratorException; 24 | import org.lable.oss.uniqueid.IDGenerator; 25 | import org.lable.oss.uniqueid.bytes.Blueprint; 26 | import org.lable.oss.uniqueid.bytes.IDBuilder; 27 | import org.lable.oss.uniqueid.bytes.Mode; 28 | 29 | import java.io.IOException; 30 | import java.nio.charset.StandardCharsets; 31 | import java.util.HashMap; 32 | import java.util.HashSet; 33 | import java.util.Map; 34 | import java.util.Set; 35 | import java.util.concurrent.ConcurrentLinkedDeque; 36 | import java.util.concurrent.CountDownLatch; 37 | import java.util.concurrent.ExecutionException; 38 | 39 | import static org.hamcrest.CoreMatchers.is; 40 | import static org.hamcrest.MatcherAssert.assertThat; 41 | import static org.junit.Assert.fail; 42 | import static org.lable.oss.uniqueid.etcd.SynchronizedUniqueIDGeneratorFactory.generatorFor; 43 | 44 | public class MultipleClusterIdsIT { 45 | @ClassRule 46 | public static final EtcdTestCluster etcd = new EtcdTestCluster("test-etcd", 1); 47 | 48 | @Rule 49 | public ExpectedException thrown = ExpectedException.none(); 50 | 51 | static Client clientA; 52 | static Client clientB; 53 | 54 | @BeforeClass 55 | public static void setup() throws InterruptedException, ExecutionException { 56 | clientA = Client.builder() 57 | .endpoints(etcd.getClientEndpoints()) 58 | .namespace(ByteSequence.from("unique-id/", StandardCharsets.UTF_8)) 59 | .build(); 60 | clientB = Client.builder() 61 | .endpoints(etcd.getClientEndpoints()) 62 | .namespace(ByteSequence.from("unique-id/", StandardCharsets.UTF_8)) 63 | .build(); 64 | 65 | TestHelper.prepareClusterID(clientA, 4, 5, 6); 66 | } 67 | 68 | @Test 69 | @Ignore 70 | // This test works, but not on every computer and not always due to the high number of threads. 71 | // Run it manually if needed. 72 | public void doubleConcurrentTest() throws Exception { 73 | final int threadCount = Blueprint.MAX_GENERATOR_ID + 2; 74 | 75 | final CountDownLatch ready = new CountDownLatch(threadCount); 76 | final CountDownLatch start = new CountDownLatch(1); 77 | final CountDownLatch done = new CountDownLatch(threadCount); 78 | final ConcurrentLinkedDeque result = new ConcurrentLinkedDeque<>(); 79 | 80 | for (int i = 0; i < threadCount; i++) { 81 | final int number = 10 + i; 82 | new Thread(() -> { 83 | ready.countDown(); 84 | try { 85 | start.await(); 86 | Client client = number % 2 == 0 ? clientA : clientB; 87 | IDGenerator generator = generatorFor(client, Mode.SPREAD); 88 | result.add(new ByteArray(generator.generate())); 89 | } catch (IOException | InterruptedException | GeneratorException e) { 90 | fail(e.getMessage()); 91 | } 92 | done.countDown(); 93 | }, String.valueOf(number)).start(); 94 | } 95 | 96 | ready.await(); 97 | start.countDown(); 98 | done.await(); 99 | 100 | assertThat(result.size(), is(threadCount)); 101 | 102 | Map> clusterGeneratorIds = new HashMap<>(); 103 | 104 | for (ByteArray byteArray : result) { 105 | Blueprint blueprint = IDBuilder.parse(byteArray.getValue()); 106 | int clusterId = blueprint.getClusterId(); 107 | int generatorId = blueprint.getGeneratorId(); 108 | 109 | if (!clusterGeneratorIds.containsKey(clusterId)) clusterGeneratorIds.put(clusterId, new HashSet<>()); 110 | clusterGeneratorIds.get(clusterId).add(generatorId); 111 | } 112 | 113 | assertThat(clusterGeneratorIds.get(4).size(), is (2048)); 114 | assertThat(clusterGeneratorIds.get(5).size(), is (1)); 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /uniqueid-etcd/src/test/java/org/lable/oss/uniqueid/etcd/MultipleGeneratorsIT.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2014 Lable (info@lable.nl) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.lable.oss.uniqueid.etcd; 17 | 18 | import io.etcd.jetcd.ByteSequence; 19 | import io.etcd.jetcd.Client; 20 | import org.junit.BeforeClass; 21 | import org.junit.ClassRule; 22 | import org.junit.Rule; 23 | import org.junit.Test; 24 | import org.junit.rules.ExpectedException; 25 | import org.lable.oss.uniqueid.GeneratorException; 26 | import org.lable.oss.uniqueid.IDGenerator; 27 | import org.lable.oss.uniqueid.bytes.Mode; 28 | 29 | import java.io.IOException; 30 | import java.nio.charset.StandardCharsets; 31 | import java.util.Deque; 32 | import java.util.HashSet; 33 | import java.util.Map; 34 | import java.util.Set; 35 | import java.util.concurrent.ConcurrentHashMap; 36 | import java.util.concurrent.ConcurrentMap; 37 | import java.util.concurrent.CountDownLatch; 38 | import java.util.concurrent.ExecutionException; 39 | 40 | import static org.hamcrest.CoreMatchers.is; 41 | import static org.hamcrest.MatcherAssert.assertThat; 42 | import static org.junit.Assert.fail; 43 | import static org.lable.oss.uniqueid.etcd.SynchronizedUniqueIDGeneratorFactory.generatorFor; 44 | 45 | public class MultipleGeneratorsIT { 46 | @ClassRule 47 | public static final EtcdTestCluster etcd = new EtcdTestCluster("test-etcd", 1); 48 | 49 | @Rule 50 | public ExpectedException thrown = ExpectedException.none(); 51 | 52 | static Client clientA; 53 | static Client clientB; 54 | 55 | @BeforeClass 56 | public static void setup() throws InterruptedException, ExecutionException { 57 | clientA = Client.builder() 58 | .endpoints(etcd.getClientEndpoints()) 59 | .namespace(ByteSequence.from("unique-id-a/", StandardCharsets.UTF_8)) 60 | .build(); 61 | clientB = Client.builder() 62 | .endpoints(etcd.getClientEndpoints()) 63 | .namespace(ByteSequence.from("unique-id-b/", StandardCharsets.UTF_8)) 64 | .build(); 65 | 66 | TestHelper.prepareClusterID(clientA, 4); 67 | TestHelper.prepareClusterID(clientB, 5); 68 | } 69 | 70 | @Test 71 | public void doubleConcurrentTest() throws Exception { 72 | final int threadCount = 20; 73 | final int batchSize = 500; 74 | 75 | final CountDownLatch ready = new CountDownLatch(threadCount); 76 | final CountDownLatch start = new CountDownLatch(1); 77 | final CountDownLatch done = new CountDownLatch(threadCount); 78 | final ConcurrentMap> result = new ConcurrentHashMap<>(threadCount); 79 | 80 | for (int i = 0; i < threadCount; i++) { 81 | final Integer number = 10 + i; 82 | new Thread(() -> { 83 | ready.countDown(); 84 | try { 85 | start.await(); 86 | Client client = number % 2 == 0 ? clientA : clientB; 87 | IDGenerator generator = generatorFor(client, Mode.SPREAD); 88 | result.put(number, generator.batch(batchSize)); 89 | } catch (IOException | InterruptedException | GeneratorException e) { 90 | fail(e.getMessage()); 91 | } 92 | done.countDown(); 93 | }, String.valueOf(number)).start(); 94 | } 95 | 96 | ready.await(); 97 | start.countDown(); 98 | done.await(); 99 | 100 | assertThat(result.size(), is(threadCount)); 101 | 102 | Set allAIDs = new HashSet<>(); 103 | Set allBIDs = new HashSet<>(); 104 | for (Map.Entry> entry : result.entrySet()) { 105 | Integer number = entry.getKey(); 106 | assertThat(entry.getValue().size(), is(batchSize)); 107 | if (number % 2 == 0) { 108 | allAIDs.addAll(entry.getValue()); 109 | } else { 110 | allBIDs.addAll(entry.getValue()); 111 | } 112 | } 113 | assertThat(allAIDs.size(), is(threadCount * batchSize / 2)); 114 | assertThat(allBIDs.size(), is(threadCount * batchSize / 2)); 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /uniqueid-etcd/src/test/java/org/lable/oss/uniqueid/etcd/RegistryBasedGeneratorIdentityTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2014 Lable (info@lable.nl) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.lable.oss.uniqueid.etcd; 17 | 18 | import io.etcd.jetcd.ByteSequence; 19 | import io.etcd.jetcd.Client; 20 | import org.junit.Before; 21 | import org.junit.Rule; 22 | import org.junit.Test; 23 | import org.lable.oss.uniqueid.BaseUniqueIDGenerator; 24 | import org.lable.oss.uniqueid.GeneratorException; 25 | import org.lable.oss.uniqueid.IDGenerator; 26 | import org.lable.oss.uniqueid.bytes.Mode; 27 | 28 | import java.io.IOException; 29 | import java.net.URI; 30 | import java.nio.charset.StandardCharsets; 31 | import java.util.*; 32 | import java.util.concurrent.ConcurrentHashMap; 33 | import java.util.concurrent.ConcurrentMap; 34 | import java.util.concurrent.CountDownLatch; 35 | import java.util.concurrent.ExecutionException; 36 | import java.util.stream.Collectors; 37 | 38 | import static com.github.npathai.hamcrestopt.OptionalMatchers.hasValue; 39 | import static com.github.npathai.hamcrestopt.OptionalMatchers.isEmpty; 40 | import static org.hamcrest.CoreMatchers.is; 41 | import static org.hamcrest.MatcherAssert.assertThat; 42 | import static org.junit.Assert.fail; 43 | 44 | public class RegistryBasedGeneratorIdentityTest { 45 | @Rule 46 | public final EtcdTestCluster etcd = new EtcdTestCluster("test-etcd", 1); 47 | 48 | Client client; 49 | 50 | @Before 51 | public void before() throws IOException, InterruptedException, ExecutionException { 52 | client = Client.builder() 53 | .endpoints(etcd.getClientEndpoints()) 54 | .namespace(ByteSequence.from("unique-id", StandardCharsets.UTF_8)) 55 | .build(); 56 | 57 | TestHelper.prepareClusterID(client, 5); 58 | } 59 | 60 | @Test 61 | public void simpleTest() throws IOException, GeneratorException, ExecutionException, InterruptedException { 62 | RegistryBasedGeneratorIdentity generatorIdentity = RegistryBasedGeneratorIdentity.basedOn( 63 | etcd.getClientEndpoints().stream().map(URI::toString).collect(Collectors.joining(",")), 64 | "unique-id", 65 | "Hello!" 66 | ); 67 | 68 | int clusterId = generatorIdentity.getClusterId(); 69 | int generatorId = generatorIdentity.getGeneratorId(); 70 | 71 | Optional content = EtcdHelper.get(client, RegistryBasedResourceClaim.resourceKey(clusterId, generatorId)); 72 | assertThat(content, hasValue("Hello!")); 73 | 74 | generatorIdentity.close(); 75 | 76 | content = EtcdHelper.get(client, RegistryBasedResourceClaim.resourceKey(clusterId, generatorId)); 77 | assertThat(content, isEmpty()); 78 | } 79 | 80 | @Test 81 | public void multipleTest() throws IOException, GeneratorException, ExecutionException, InterruptedException { 82 | final int threadCount = 4; 83 | final int batchSize = 500; 84 | 85 | final CountDownLatch ready = new CountDownLatch(threadCount); 86 | final CountDownLatch start = new CountDownLatch(1); 87 | final CountDownLatch done = new CountDownLatch(threadCount); 88 | final ConcurrentMap> result = new ConcurrentHashMap<>(threadCount); 89 | final Set generatorIds = new HashSet<>(); 90 | 91 | for (int i = 0; i < threadCount; i++) { 92 | final Integer number = 10 + i; 93 | new Thread(() -> { 94 | ready.countDown(); 95 | try { 96 | start.await(); 97 | RegistryBasedGeneratorIdentity generatorIdentity = RegistryBasedGeneratorIdentity.basedOn( 98 | etcd.getClientEndpoints().stream().map(URI::toString).collect(Collectors.joining(",")), 99 | "unique-id", 100 | "Hello!" 101 | ); 102 | 103 | IDGenerator generator = new BaseUniqueIDGenerator(generatorIdentity, Mode.SPREAD); 104 | generatorIds.add(generatorIdentity.getGeneratorId()); 105 | result.put(number, generator.batch(batchSize)); 106 | } catch (IOException | InterruptedException | GeneratorException e) { 107 | fail(e.getMessage()); 108 | } 109 | done.countDown(); 110 | }, String.valueOf(number)).start(); 111 | } 112 | 113 | ready.await(); 114 | start.countDown(); 115 | done.await(); 116 | 117 | assertThat(result.size(), is(threadCount)); 118 | assertThat(generatorIds.size(), is(threadCount)); 119 | 120 | for (Map.Entry> entry : result.entrySet()) { 121 | assertThat(entry.getValue().size(), is(batchSize)); 122 | } 123 | } 124 | } -------------------------------------------------------------------------------- /uniqueid-etcd/src/test/java/org/lable/oss/uniqueid/etcd/SynchronizedUniqueIDGeneratorIT.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2014 Lable (info@lable.nl) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.lable.oss.uniqueid.etcd; 17 | 18 | import io.etcd.jetcd.ByteSequence; 19 | import io.etcd.jetcd.Client; 20 | import org.apache.commons.codec.binary.Hex; 21 | import org.junit.BeforeClass; 22 | import org.junit.ClassRule; 23 | import org.junit.Test; 24 | import org.lable.oss.uniqueid.ByteArray; 25 | import org.lable.oss.uniqueid.GeneratorException; 26 | import org.lable.oss.uniqueid.IDGenerator; 27 | import org.lable.oss.uniqueid.bytes.Blueprint; 28 | import org.lable.oss.uniqueid.bytes.IDBuilder; 29 | import org.lable.oss.uniqueid.bytes.Mode; 30 | 31 | import java.io.IOException; 32 | import java.nio.charset.StandardCharsets; 33 | import java.util.*; 34 | import java.util.concurrent.ConcurrentHashMap; 35 | import java.util.concurrent.ConcurrentMap; 36 | import java.util.concurrent.CountDownLatch; 37 | import java.util.concurrent.ExecutionException; 38 | import java.util.concurrent.atomic.AtomicLong; 39 | 40 | import static org.hamcrest.CoreMatchers.is; 41 | import static org.hamcrest.MatcherAssert.assertThat; 42 | import static org.junit.Assert.fail; 43 | import static org.lable.oss.uniqueid.etcd.SynchronizedUniqueIDGeneratorFactory.generatorFor; 44 | 45 | public class SynchronizedUniqueIDGeneratorIT { 46 | @ClassRule 47 | public static final EtcdTestCluster etcd = new EtcdTestCluster("test-etcd", 1); 48 | 49 | final static int CLUSTER_ID = 4; 50 | 51 | static Client client; 52 | 53 | @BeforeClass 54 | public static void setup() throws InterruptedException, ExecutionException { 55 | client = Client.builder() 56 | .endpoints(etcd.getClientEndpoints()) 57 | .namespace(ByteSequence.from("unique-id/", StandardCharsets.UTF_8)) 58 | .build(); 59 | 60 | TestHelper.prepareClusterID(client, CLUSTER_ID); 61 | } 62 | 63 | @Test 64 | public void simpleTest() throws Exception { 65 | IDGenerator generator = generatorFor(client, Mode.TIME_SEQUENTIAL); 66 | byte[] result = generator.generate(); 67 | Blueprint blueprint = IDBuilder.parse(result); 68 | assertThat(result.length, is(8)); 69 | assertThat(blueprint.getClusterId(), is(CLUSTER_ID)); 70 | } 71 | 72 | @Test 73 | public void timeSequentialTest() throws Exception { 74 | // Explicitly implement a clock ourselves for testing. 75 | AtomicLong time = new AtomicLong(1_500_000_000); 76 | SynchronizedGeneratorIdentity generatorIdentityHolder = new SynchronizedGeneratorIdentity( 77 | client, 78 | Collections.singletonList(0), 79 | null, 80 | null 81 | ); 82 | IDGenerator generator = generatorFor( 83 | generatorIdentityHolder, 84 | time::getAndIncrement, 85 | Mode.TIME_SEQUENTIAL 86 | ); 87 | 88 | Set ids = new HashSet<>(); 89 | for (int i = 0; i < 100_000; i++) { 90 | ids.add(new ByteArray(generator.generate())); 91 | } 92 | 93 | assertThat(ids.size(), is(100_000)); 94 | 95 | ByteArray id = ids.iterator().next(); 96 | 97 | System.out.println(Hex.encodeHex(id.getValue())); 98 | System.out.println(IDBuilder.parseTimestamp(id.getValue())); 99 | } 100 | 101 | @Test 102 | public void test() { 103 | Set s = new HashSet<>(); 104 | s.add(new ByteArray(new byte[]{0, 1})); 105 | s.add(new ByteArray(new byte[]{0, 1})); 106 | assertThat(s.size(), is(1)); 107 | 108 | } 109 | 110 | @Test 111 | public void concurrentTest() throws Exception { 112 | final int threadCount = 20; 113 | final int batchSize = 500; 114 | 115 | final CountDownLatch ready = new CountDownLatch(threadCount); 116 | final CountDownLatch start = new CountDownLatch(1); 117 | final CountDownLatch done = new CountDownLatch(threadCount); 118 | final ConcurrentMap> result = new ConcurrentHashMap<>(threadCount); 119 | 120 | for (int i = 0; i < threadCount; i++) { 121 | final Integer number = 10 + i; 122 | new Thread(() -> { 123 | ready.countDown(); 124 | try { 125 | start.await(); 126 | IDGenerator generator = generatorFor(client, Mode.SPREAD); 127 | result.put(number, generator.batch(batchSize)); 128 | } catch (IOException | InterruptedException | GeneratorException e) { 129 | fail(); 130 | } 131 | done.countDown(); 132 | }, String.valueOf(number)).start(); 133 | } 134 | 135 | ready.await(); 136 | start.countDown(); 137 | done.await(); 138 | 139 | assertThat(result.size(), is(threadCount)); 140 | 141 | Set allIDs = new HashSet<>(); 142 | for (Map.Entry> entry : result.entrySet()) { 143 | assertThat(entry.getValue().size(), is(batchSize)); 144 | entry.getValue().forEach(value -> allIDs.add(new ByteArray(value))); 145 | } 146 | assertThat(allIDs.size(), is(threadCount * batchSize)); 147 | } 148 | } 149 | -------------------------------------------------------------------------------- /uniqueid-etcd/src/test/java/org/lable/oss/uniqueid/etcd/TestHelper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2014 Lable (info@lable.nl) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.lable.oss.uniqueid.etcd; 17 | 18 | import io.etcd.jetcd.ByteSequence; 19 | import io.etcd.jetcd.Client; 20 | 21 | import java.nio.charset.StandardCharsets; 22 | import java.util.Arrays; 23 | import java.util.concurrent.ExecutionException; 24 | import java.util.stream.Collectors; 25 | 26 | import static org.lable.oss.uniqueid.etcd.ClusterID.CLUSTER_ID_KEY; 27 | 28 | public class TestHelper { 29 | 30 | public static void prepareClusterID(Client etcd, int... clusterId) throws ExecutionException, InterruptedException { 31 | String serialized = Arrays.stream(clusterId).boxed().map(String::valueOf).collect(Collectors.joining(", ")); 32 | 33 | etcd.getKVClient() 34 | .put(CLUSTER_ID_KEY, ByteSequence.from(serialized, StandardCharsets.UTF_8)) 35 | .get(); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /uniqueid-etcd/src/test/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | --------------------------------------------------------------------------------