├── .gitignore ├── LICENSE ├── README.md ├── pom.xml └── src ├── main ├── assembly │ └── dep.xml └── java │ └── com │ └── sysgears │ ├── Main.java │ ├── Queue.java │ └── TestUtils.java └── test └── java └── com └── sysgears └── UTestQueue.java /.gitignore: -------------------------------------------------------------------------------- 1 | *.class 2 | 3 | # Mobile Tools for Java (J2ME) 4 | .mtj.tmp/ 5 | 6 | # Package Files # 7 | *.jar 8 | *.war 9 | *.ear 10 | 11 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 12 | hs_err_pid* 13 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | This is free and unencumbered software released into the public domain. 2 | 3 | Anyone is free to copy, modify, publish, use, compile, sell, or 4 | distribute this software, either in source code form or as a compiled 5 | binary, for any purpose, commercial or non-commercial, and by any 6 | means. 7 | 8 | In jurisdictions that recognize copyright laws, the author or authors 9 | of this software dedicate any and all copyright interest in the 10 | software to the public domain. We make this dedication for the benefit 11 | of the public at large and to the detriment of our heirs and 12 | successors. We intend this dedication to be an overt act of 13 | relinquishment in perpetuity of all present and future rights to this 14 | software under copyright law. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 19 | IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 | OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 | ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 | OTHER DEALINGS IN THE SOFTWARE. 23 | 24 | For more information, please refer to 25 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Lightweight fast persistent queue in Java using Berkley DB 2 | 3 | [![Twitter Follow](https://img.shields.io/twitter/follow/sysgears.svg?style=social)](https://twitter.com/sysgears) 4 | 5 | My original article has appeared here first: 6 | https://sysgears.com/articles/lightweight-fast-persistent-queue-in-java-using-berkley-db/ 7 | 8 | I have uploaded the code for my article here to clarify that 9 | it has Public Domain license. 10 | 11 | ## Article 12 | 13 | Recently I had a task to develop the application which will have large 14 | work queue and which need to survive the restarts. The application need 15 | to be lightweight. After trying several different persistent engines for 16 | Java I''ve chosen to stick with Berkley DB Java edition. This persistent 17 | engine is pretty lightweight it is fast, optimized for multi-threaded 18 | usage and have no problems with reclaiming free space. 19 | 20 | As I needed the fast persistent queue at a cost of possible data loss on 21 | system crash I've chosen non-transactional API for Berkley DB. With 22 | non-transactional API the great speed can be achieved for persistent 23 | queue at a price of loss of some data at system crash. The more data you 24 | allow to be lost the greater speed of the queue you will have. Though 25 | you can opt to sync to disk each operation on the queue and in that case 26 | your data loss will be minimal. 27 | 28 | 29 | 30 | Berkley DB keeps data sorted by key in B-Tree. By default keys are 31 | sorted lexicographically byte by byte. But you can override sorting 32 | order by providing your own comparator. In this implementation of the 33 | queue keys are just big integers and sorted in ascending order. 34 | 35 | The DB allows row locking model, this is great for doing multi-threaded 36 | polling of the queue. But for multi-threaded pushing you either need to 37 | keep key counter in separate database or synchronize the push method. 38 | I've decided to choose later route. 39 | 40 | So, here is the code of the queue with described choices: 41 | ``` java 42 | 43 | package com.sysgears; 44 | 45 | import com.sleepycat.je.*; 46 | 47 | import java.io.File; 48 | import java.io.IOException; 49 | import java.io.Serializable; 50 | import java.math.BigInteger; 51 | import java.util.Comparator; 52 | 53 | /** 54 | * Key comparator for DB keys 55 | */ 56 | class KeyComparator implements Comparator, Serializable { 57 | 58 | /** 59 | * Compares two DB keys. 60 | * 61 | * @param key1 first key 62 | * @param key2 second key 63 | * 64 | * @return comparison result 65 | */ 66 | public int compare(byte[] key1, byte[] key2) { 67 | return new BigInteger(key1).compareTo(new BigInteger(key2)); 68 | } 69 | } 70 | 71 | /** 72 | * Fast queue implementation on top of Berkley DB Java Edition. 73 | * 74 | 75 | * This class is thread-safe. 76 | */ 77 | public class Queue { 78 | 79 | /** 80 | * Berkley DB environment 81 | */ 82 | private final Environment dbEnv; 83 | 84 | /** 85 | * Berkley DB instance for the queue 86 | */ 87 | private final Database queueDatabase; 88 | 89 | /** 90 | * Queue cache size - number of element operations it is allowed to loose in case of system crash. 91 | */ 92 | private final int cacheSize; 93 | 94 | /** 95 | * This queue name. 96 | */ 97 | private final String queueName; 98 | 99 | /** 100 | * Queue operation counter, which is used to sync the queue database to disk periodically. 101 | */ 102 | private int opsCounter; 103 | 104 | /** 105 | * Creates instance of persistent queue. 106 | * 107 | * @param queueEnvPath queue database environment directory path 108 | * @param queueName descriptive queue name 109 | * @param cacheSize how often to sync the queue to disk 110 | */ 111 | public Queue(final String queueEnvPath, 112 | final String queueName, 113 | final int cacheSize) { 114 | // Create parent dirs for queue environment directory 115 | new File(queueEnvPath).mkdirs(); 116 | 117 | // Setup database environment 118 | final EnvironmentConfig dbEnvConfig = new EnvironmentConfig(); 119 | dbEnvConfig.setTransactional(false); 120 | dbEnvConfig.setAllowCreate(true); 121 | this.dbEnv = new Environment(new File(queueEnvPath), 122 | dbEnvConfig); 123 | 124 | // Setup non-transactional deferred-write queue database 125 | DatabaseConfig dbConfig = new DatabaseConfig(); 126 | dbConfig.setTransactional(false); 127 | dbConfig.setAllowCreate(true); 128 | dbConfig.setDeferredWrite(true); 129 | dbConfig.setBtreeComparator(new KeyComparator()); 130 | this.queueDatabase = dbEnv.openDatabase(null, 131 | queueName, 132 | dbConfig); 133 | this.queueName = queueName; 134 | this.cacheSize = cacheSize; 135 | this.opsCounter = 0; 136 | } 137 | 138 | /** 139 | * Retrieves and returns element from the head of this queue. 140 | * 141 | * @return element from the head of the queue or null if queue is empty 142 | * 143 | * @throws IOException in case of disk IO failure 144 | */ 145 | public String poll() throws IOException { 146 | final DatabaseEntry key = new DatabaseEntry(); 147 | final DatabaseEntry data = new DatabaseEntry(); 148 | final Cursor cursor = queueDatabase.openCursor(null, null); 149 | try { 150 | cursor.getFirst(key, data, LockMode.RMW); 151 | if (data.getData() == null) 152 | return null; 153 | final String res = new String(data.getData(), "UTF-8"); 154 | cursor.delete(); 155 | opsCounter++; 156 | if (opsCounter >= cacheSize) { 157 | queueDatabase.sync(); 158 | opsCounter = 0; 159 | } 160 | return res; 161 | } finally { 162 | cursor.close(); 163 | } 164 | } 165 | 166 | /** 167 | * Pushes element to the tail of this queue. 168 | * 169 | * @param element element 170 | * 171 | * @throws IOException in case of disk IO failure 172 | */ 173 | public synchronized void push(final String element) throws IOException { 174 | DatabaseEntry key = new DatabaseEntry(); 175 | DatabaseEntry data = new DatabaseEntry(); 176 | Cursor cursor = queueDatabase.openCursor(null, null); 177 | try { 178 | cursor.getLast(key, data, LockMode.RMW); 179 | 180 | BigInteger prevKeyValue; 181 | if (key.getData() == null) { 182 | prevKeyValue = BigInteger.valueOf(-1); 183 | } else { 184 | prevKeyValue = new BigInteger(key.getData()); 185 | } 186 | BigInteger newKeyValue = prevKeyValue.add(BigInteger.ONE); 187 | 188 | try { 189 | final DatabaseEntry newKey = new DatabaseEntry( 190 | newKeyValue.toByteArray()); 191 | final DatabaseEntry newData = new DatabaseEntry( 192 | element.getBytes("UTF-8")); 193 | queueDatabase.put(null, newKey, newData); 194 | 195 | opsCounter++; 196 | if (opsCounter >= cacheSize) { 197 | queueDatabase.sync(); 198 | opsCounter = 0; 199 | } 200 | } catch (IOException e) { 201 | e.printStackTrace(); 202 | } 203 | } finally { 204 | cursor.close(); 205 | } 206 | } 207 | 208 | /** 209 | * Returns the size of this queue. 210 | * 211 | * @return the size of the queue 212 | */ 213 | public long size() { 214 | return queueDatabase.count(); 215 | } 216 | 217 | /** 218 | * Returns this queue name. 219 | * 220 | * @return this queue name 221 | */ 222 | public String getQueueName() { 223 | return queueName; 224 | } 225 | 226 | /** 227 | * Closes this queue and frees up all resources associated to it. 228 | */ 229 | public void close() { 230 | queueDatabase.close(); 231 | dbEnv.close(); 232 | } 233 | } 234 | ``` 235 | 236 | The unit tests for the queue: 237 | ``` java 238 | 239 | package com.sysgears; 240 | 241 | import org.testng.annotations.Test; 242 | 243 | import java.io.File; 244 | import java.io.IOException; 245 | import java.util.*; 246 | import java.util.concurrent.CountDownLatch; 247 | import java.util.concurrent.TimeUnit; 248 | 249 | @Test 250 | public class UTestQueue { 251 | 252 | @Test 253 | public void testCreateQueue() { 254 | File queueDir = TestUtils.createTempSubdir("test-queue"); 255 | Queue queue = new Queue(queueDir.getPath(), "test-queue", 3); 256 | try { 257 | assert Arrays.asList(queueDir.listFiles()).contains(new File(queueDir, "00000000.jdb")); 258 | } finally { 259 | queue.close(); 260 | } 261 | } 262 | 263 | @Test public void testPush() throws Throwable { 264 | File queueDir = TestUtils.createTempSubdir("test-queue"); 265 | Queue queue = new Queue(queueDir.getPath(), "test-queue", 3); 266 | try { 267 | queue.push("1"); 268 | queue.push("2"); 269 | String head = queue.poll(); 270 | 271 | assert head.equals("1"); 272 | } finally { 273 | queue.close(); 274 | } 275 | } 276 | 277 | @Test public void testQueueSurviveReopen() throws Throwable { 278 | File queueDir = TestUtils.createTempSubdir("test-queue"); 279 | Queue queue = new Queue(queueDir.getPath(), "test-queue", 3); 280 | try { 281 | queue.push("5"); 282 | } finally { 283 | queue.close(); 284 | } 285 | 286 | queue = new Queue(queueDir.getPath(), "test-queue", 3); 287 | try { 288 | String head = queue.poll(); 289 | 290 | assert head.equals("5"); 291 | } finally { 292 | queue.close(); 293 | } 294 | } 295 | 296 | @Test public void testQueuePushOrder() throws Throwable { 297 | File queueDir = TestUtils.createTempSubdir("test-queue"); 298 | final Queue queue = new Queue(queueDir.getPath(), "test-queue", 1000); 299 | try { 300 | for (int i = 0; i < 300; i++) { 301 | queue.push(Integer.toString(i)); 302 | } 303 | 304 | for (int i = 0; i < 300; i++) { 305 | String element = queue.poll(); 306 | if (!Integer.toString(i).equals(element)) { 307 | throw new AssertionError("Expected element " + i + ", but got " + element); 308 | } 309 | } 310 | } finally { 311 | queue.close(); 312 | } 313 | 314 | } 315 | 316 | @Test public void testMultiThreadedPoll() throws Throwable { 317 | File queueDir = TestUtils.createTempSubdir("test-queue"); 318 | final Queue queue = new Queue(queueDir.getPath(), "test-queue", 3); 319 | try { 320 | int threadCount = 20; 321 | for (int i = 0; i < threadCount; i++) 322 | queue.push(Integer.toString(i)); 323 | 324 | final Set set = Collections.synchronizedSet(new HashSet()); 325 | final CountDownLatch startLatch = new CountDownLatch(threadCount); 326 | final CountDownLatch latch = new CountDownLatch(threadCount); 327 | 328 | for (int i = 0; i < threadCount; i++) { 329 | new Thread() { 330 | public void run() { 331 | try { 332 | startLatch.countDown(); 333 | startLatch.await(); 334 | 335 | String val = queue.poll(); 336 | if (val != null) { 337 | set.add(val); 338 | } 339 | latch.countDown(); 340 | } catch (Throwable e) { 341 | e.printStackTrace(); 342 | } 343 | } 344 | }.start(); 345 | } 346 | 347 | latch.await(5, TimeUnit.SECONDS); 348 | 349 | assert set.size() == threadCount; 350 | } finally { 351 | queue.close(); 352 | } 353 | } 354 | 355 | @Test public void testMultiThreadedPush() throws Throwable { 356 | File queueDir = TestUtils.createTempSubdir("test-queue"); 357 | final Queue queue = new Queue(queueDir.getPath(), "test-queue", 3); 358 | try { 359 | int threadCount = 20; 360 | 361 | final CountDownLatch startLatch = new CountDownLatch(threadCount); 362 | final CountDownLatch latch = new CountDownLatch(threadCount); 363 | 364 | for (int i = 0; i < threadCount; i++) { 365 | new Thread(Integer.toString(i)) { 366 | public void run() { 367 | try { 368 | startLatch.countDown(); 369 | startLatch.await(); 370 | 371 | queue.push(getName()); 372 | latch.countDown(); 373 | } catch (Throwable e) { 374 | e.printStackTrace(); 375 | } 376 | } 377 | }.start(); 378 | } 379 | 380 | latch.await(5, TimeUnit.SECONDS); 381 | 382 | assert queue.size() == threadCount; 383 | } finally { 384 | queue.close(); 385 | } 386 | } 387 | } 388 | ``` 389 | 390 | And here is a main class which measures the queue performance: 391 | ``` java 392 | 393 | package com.sysgears; 394 | 395 | import java.io.File; 396 | 397 | public class Main { 398 | 399 | public static void main(String[] args) throws Throwable { 400 | int elementCount = 10000; 401 | File queueDir = TestUtils.createTempSubdir("test-queue"); 402 | final Queue queue = new Queue(queueDir.getPath(), "test-queue", 1000); 403 | try { 404 | long pushStart = System.currentTimeMillis(); 405 | for (int i = 0; i < elementCount; i++) { 406 | queue.push(Integer.toString(i)); 407 | } 408 | long pushEnd = System.currentTimeMillis(); 409 | System.out.println("Time to push " + elementCount + " records: " + (pushEnd - pushStart) + " ms"); 410 | 411 | long pollStart = System.currentTimeMillis(); 412 | for (int i = 0; i < elementCount; i++) { 413 | String element = queue.poll(); 414 | if (!Integer.toString(i).equals(element)) { 415 | throw new AssertionError("Expected element " + i + ", but got " + element); 416 | } 417 | } 418 | long pollEnd = System.currentTimeMillis(); 419 | System.out.println("Time to poll " + elementCount + " records: " + (pollEnd - pollStart) + " ms"); 420 | } finally { 421 | queue.close(); 422 | } 423 | } 424 | } 425 | ``` 426 | 427 | Output of running main on my PC: 428 | 
429 | 
430 | Time to push 10000 records: 2633 ms
431 | Time to poll 10000 records: 7764 ms
432 | 433 | Hope this helps, 434 | Victor Vlasenko, 435 | SysGears 436 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4 | 4.0.0 5 | com.sysgears 6 | queue 7 | 1.0 8 | Persistent queue implementation 9 | 10 | Persistent queue implementation on top of Berkley DB 11 | 12 | 13 | 14 | 15 | oracleReleases 16 | Oracle Released Java Packages 17 | http://download.oracle.com/maven 18 | default 19 | 20 | 21 | 22 | 23 | 24 | com.sleepycat 25 | je 26 | 4.1.7 27 | 28 | 29 | org.testng 30 | testng 31 | 5.1 32 | jdk15 33 | test 34 | 35 | 36 | 37 | 38 | 39 | 40 | org.apache.maven.plugins 41 | maven-compiler-plugin 42 | 43 | 1.5 44 | 1.5 45 | 46 | 47 | 48 | maven-assembly-plugin 49 | 2.2-beta-2 50 | 51 | 52 | src/main/assembly/dep.xml 53 | 54 | 55 | 56 | com.sysgears.Main 57 | 58 | 59 | 60 | 61 | 62 | package 63 | 64 | single 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | -------------------------------------------------------------------------------- /src/main/assembly/dep.xml: -------------------------------------------------------------------------------- 1 | 2 | all 3 | 4 | jar 5 | 6 | false 7 | 8 | 9 | / 10 | true 11 | runtime 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /src/main/java/com/sysgears/Main.java: -------------------------------------------------------------------------------- 1 | package com.sysgears; 2 | 3 | import java.io.File; 4 | 5 | public class Main { 6 | 7 | public static void main(String[] args) throws Throwable { 8 | int elementCount = 10000; 9 | File queueDir = TestUtils.createTempSubdir("test-queue"); 10 | final Queue queue = new Queue(queueDir.getPath(), "test-queue", 1000); 11 | try { 12 | long pushStart = System.currentTimeMillis(); 13 | for (int i = 0; i < elementCount; i++) { 14 | queue.push(Integer.toString(i)); 15 | } 16 | long pushEnd = System.currentTimeMillis(); 17 | System.out.println("Time to push " + elementCount + " records: " + (pushEnd - pushStart) + " ms"); 18 | 19 | long pollStart = System.currentTimeMillis(); 20 | for (int i = 0; i < elementCount; i++) { 21 | String element = queue.poll(); 22 | if (!Integer.toString(i).equals(element)) { 23 | throw new AssertionError("Expected element " + i + ", but got " + element); 24 | } 25 | } 26 | long pollEnd = System.currentTimeMillis(); 27 | System.out.println("Time to poll " + elementCount + " records: " + (pollEnd - pollStart) + " ms"); 28 | } finally { 29 | queue.close(); 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/main/java/com/sysgears/Queue.java: -------------------------------------------------------------------------------- 1 | package com.sysgears; 2 | 3 | import com.sleepycat.je.*; 4 | 5 | import java.io.File; 6 | import java.io.IOException; 7 | import java.io.Serializable; 8 | import java.math.BigInteger; 9 | import java.util.Comparator; 10 | 11 | /** 12 | * Key comparator for DB keys 13 | */ 14 | class KeyComparator implements Comparator, Serializable { 15 | 16 | /** 17 | * Compares two DB keys. 18 | * 19 | * @param key1 first key 20 | * @param key2 second key 21 | * 22 | * @return comparison result 23 | */ 24 | public int compare(byte[] key1, byte[] key2) { 25 | return new BigInteger(key1).compareTo(new BigInteger(key2)); 26 | } 27 | } 28 | 29 | /** 30 | * Fast queue implementation on top of Berkley DB Java Edition. 31 | *

32 | * This class is thread-safe. 33 | */ 34 | public class Queue { 35 | 36 | /** 37 | * Berkley DB environment 38 | */ 39 | private final Environment dbEnv; 40 | 41 | /** 42 | * Berkley DB instance for the queue 43 | */ 44 | private final Database queueDatabase; 45 | 46 | /** 47 | * Queue cache size - number of element operations it is allowed to loose in case of system crash. 48 | */ 49 | private final int cacheSize; 50 | 51 | /** 52 | * This queue name. 53 | */ 54 | private final String queueName; 55 | 56 | /** 57 | * Queue operation counter, which is used to sync the queue database to disk periodically. 58 | */ 59 | private int opsCounter; 60 | 61 | /** 62 | * Creates instance of persistent queue. 63 | * 64 | * @param queueEnvPath queue database environment directory path 65 | * @param queueName descriptive queue name 66 | * @param cacheSize how often to sync the queue to disk 67 | */ 68 | public Queue(final String queueEnvPath, 69 | final String queueName, 70 | final int cacheSize) { 71 | // Create parent dirs for queue environment directory 72 | new File(queueEnvPath).mkdirs(); 73 | 74 | // Setup database environment 75 | final EnvironmentConfig dbEnvConfig = new EnvironmentConfig(); 76 | dbEnvConfig.setTransactional(false); 77 | dbEnvConfig.setAllowCreate(true); 78 | this.dbEnv = new Environment(new File(queueEnvPath), 79 | dbEnvConfig); 80 | 81 | // Setup non-transactional deferred-write queue database 82 | DatabaseConfig dbConfig = new DatabaseConfig(); 83 | dbConfig.setTransactional(false); 84 | dbConfig.setAllowCreate(true); 85 | dbConfig.setDeferredWrite(true); 86 | dbConfig.setBtreeComparator(new KeyComparator()); 87 | this.queueDatabase = dbEnv.openDatabase(null, 88 | queueName, 89 | dbConfig); 90 | this.queueName = queueName; 91 | this.cacheSize = cacheSize; 92 | this.opsCounter = 0; 93 | } 94 | 95 | /** 96 | * Retrieves and returns element from the head of this queue. 97 | * 98 | * @return element from the head of the queue or null if queue is empty 99 | * 100 | * @throws IOException in case of disk IO failure 101 | */ 102 | public String poll() throws IOException { 103 | final DatabaseEntry key = new DatabaseEntry(); 104 | final DatabaseEntry data = new DatabaseEntry(); 105 | final Cursor cursor = queueDatabase.openCursor(null, null); 106 | try { 107 | cursor.getFirst(key, data, LockMode.RMW); 108 | if (data.getData() == null) 109 | return null; 110 | final String res = new String(data.getData(), "UTF-8"); 111 | cursor.delete(); 112 | opsCounter++; 113 | if (opsCounter >= cacheSize) { 114 | queueDatabase.sync(); 115 | opsCounter = 0; 116 | } 117 | return res; 118 | } finally { 119 | cursor.close(); 120 | } 121 | } 122 | 123 | /** 124 | * Pushes element to the tail of this queue. 125 | * 126 | * @param element element 127 | * 128 | * @throws IOException in case of disk IO failure 129 | */ 130 | public synchronized void push(final String element) throws IOException { 131 | DatabaseEntry key = new DatabaseEntry(); 132 | DatabaseEntry data = new DatabaseEntry(); 133 | Cursor cursor = queueDatabase.openCursor(null, null); 134 | try { 135 | cursor.getLast(key, data, LockMode.RMW); 136 | 137 | BigInteger prevKeyValue; 138 | if (key.getData() == null) { 139 | prevKeyValue = BigInteger.valueOf(-1); 140 | } else { 141 | prevKeyValue = new BigInteger(key.getData()); 142 | } 143 | BigInteger newKeyValue = prevKeyValue.add(BigInteger.ONE); 144 | 145 | try { 146 | final DatabaseEntry newKey = new DatabaseEntry( 147 | newKeyValue.toByteArray()); 148 | final DatabaseEntry newData = new DatabaseEntry( 149 | element.getBytes("UTF-8")); 150 | queueDatabase.put(null, newKey, newData); 151 | 152 | opsCounter++; 153 | if (opsCounter >= cacheSize) { 154 | queueDatabase.sync(); 155 | opsCounter = 0; 156 | } 157 | } catch (IOException e) { 158 | e.printStackTrace(); 159 | } 160 | } finally { 161 | cursor.close(); 162 | } 163 | } 164 | 165 | /** 166 | * Returns the size of this queue. 167 | * 168 | * @return the size of the queue 169 | */ 170 | public long size() { 171 | return queueDatabase.count(); 172 | } 173 | 174 | /** 175 | * Returns this queue name. 176 | * 177 | * @return this queue name 178 | */ 179 | public String getQueueName() { 180 | return queueName; 181 | } 182 | 183 | /** 184 | * Closes this queue and frees up all resources associated to it. 185 | */ 186 | public void close() { 187 | queueDatabase.close(); 188 | dbEnv.close(); 189 | } 190 | } 191 | -------------------------------------------------------------------------------- /src/main/java/com/sysgears/TestUtils.java: -------------------------------------------------------------------------------- 1 | package com.sysgears; 2 | 3 | import java.io.File; 4 | 5 | public class TestUtils { 6 | 7 | /** 8 | * Creates temporary subdirectory 9 | * 10 | * @param subdir subdirectory to create 11 | * 12 | * @return created subdirectory 13 | */ 14 | static public File createTempSubdir(String subdir) { 15 | File subDir = new File(TestUtils.getTempDirectory(), subdir); 16 | if (!TestUtils.createEmptyDirectory(subDir)) { 17 | throw new RuntimeException("Error creating temporary subdirectory"); 18 | } 19 | return subDir; 20 | } 21 | 22 | /** 23 | * Returns OS-specific temp directory 24 | * 25 | * @return temp directory 26 | */ 27 | static public File getTempDirectory() { 28 | return new File(System.getProperty("java.io.tmpdir")); 29 | } 30 | 31 | /** 32 | * Creates empty directory. 33 | * 34 | * @param path path to create 35 | * 36 | * @return whether operation was successful 37 | */ 38 | static public boolean createEmptyDirectory(File path) { 39 | return deleteDirectory(path) && path.mkdirs(); 40 | } 41 | 42 | /** 43 | * Deletes directory recursively. 44 | * 45 | * @param path directory to delete 46 | * 47 | * @return whether operation was successful 48 | */ 49 | static public boolean deleteDirectory(File path) { 50 | if (path.exists()) { 51 | File[] files = path.listFiles(); 52 | if (files != null) { 53 | for (File file : files) { 54 | deleteDirectory(file); 55 | } 56 | } 57 | return path.delete(); 58 | } 59 | return true; 60 | } 61 | 62 | } 63 | -------------------------------------------------------------------------------- /src/test/java/com/sysgears/UTestQueue.java: -------------------------------------------------------------------------------- 1 | package com.sysgears; 2 | 3 | import org.testng.annotations.Test; 4 | 5 | import java.io.File; 6 | import java.io.IOException; 7 | import java.util.*; 8 | import java.util.concurrent.CountDownLatch; 9 | import java.util.concurrent.TimeUnit; 10 | 11 | @Test 12 | public class UTestQueue { 13 | 14 | @Test 15 | public void testCreateQueue() { 16 | File queueDir = TestUtils.createTempSubdir("test-queue"); 17 | Queue queue = new Queue(queueDir.getPath(), "test-queue", 3); 18 | try { 19 | assert Arrays.asList(queueDir.listFiles()).contains(new File(queueDir, "00000000.jdb")); 20 | } finally { 21 | queue.close(); 22 | } 23 | } 24 | 25 | @Test public void testPush() throws Throwable { 26 | File queueDir = TestUtils.createTempSubdir("test-queue"); 27 | Queue queue = new Queue(queueDir.getPath(), "test-queue", 3); 28 | try { 29 | queue.push("1"); 30 | queue.push("2"); 31 | String head = queue.poll(); 32 | 33 | assert head.equals("1"); 34 | } finally { 35 | queue.close(); 36 | } 37 | } 38 | 39 | @Test public void testQueueSurviveReopen() throws Throwable { 40 | File queueDir = TestUtils.createTempSubdir("test-queue"); 41 | Queue queue = new Queue(queueDir.getPath(), "test-queue", 3); 42 | try { 43 | queue.push("5"); 44 | } finally { 45 | queue.close(); 46 | } 47 | 48 | queue = new Queue(queueDir.getPath(), "test-queue", 3); 49 | try { 50 | String head = queue.poll(); 51 | 52 | assert head.equals("5"); 53 | } finally { 54 | queue.close(); 55 | } 56 | } 57 | 58 | @Test public void testQueuePushOrder() throws Throwable { 59 | File queueDir = TestUtils.createTempSubdir("test-queue"); 60 | final Queue queue = new Queue(queueDir.getPath(), "test-queue", 1000); 61 | try { 62 | for (int i = 0; i < 300; i++) { 63 | queue.push(Integer.toString(i)); 64 | } 65 | 66 | for (int i = 0; i < 300; i++) { 67 | String element = queue.poll(); 68 | if (!Integer.toString(i).equals(element)) { 69 | throw new AssertionError("Expected element " + i + ", but got " + element); 70 | } 71 | } 72 | } finally { 73 | queue.close(); 74 | } 75 | 76 | } 77 | 78 | @Test public void testMultiThreadedPoll() throws Throwable { 79 | File queueDir = TestUtils.createTempSubdir("test-queue"); 80 | final Queue queue = new Queue(queueDir.getPath(), "test-queue", 3); 81 | try { 82 | int threadCount = 20; 83 | for (int i = 0; i < threadCount; i++) 84 | queue.push(Integer.toString(i)); 85 | 86 | final Set set = Collections.synchronizedSet(new HashSet()); 87 | final CountDownLatch startLatch = new CountDownLatch(threadCount); 88 | final CountDownLatch latch = new CountDownLatch(threadCount); 89 | 90 | for (int i = 0; i < threadCount; i++) { 91 | new Thread() { 92 | public void run() { 93 | try { 94 | startLatch.countDown(); 95 | startLatch.await(); 96 | 97 | String val = queue.poll(); 98 | if (val != null) { 99 | set.add(val); 100 | } 101 | latch.countDown(); 102 | } catch (Throwable e) { 103 | e.printStackTrace(); 104 | } 105 | } 106 | }.start(); 107 | } 108 | 109 | latch.await(5, TimeUnit.SECONDS); 110 | 111 | assert set.size() == threadCount; 112 | } finally { 113 | queue.close(); 114 | } 115 | } 116 | 117 | @Test public void testMultiThreadedPush() throws Throwable { 118 | File queueDir = TestUtils.createTempSubdir("test-queue"); 119 | final Queue queue = new Queue(queueDir.getPath(), "test-queue", 3); 120 | try { 121 | int threadCount = 20; 122 | 123 | final CountDownLatch startLatch = new CountDownLatch(threadCount); 124 | final CountDownLatch latch = new CountDownLatch(threadCount); 125 | 126 | for (int i = 0; i < threadCount; i++) { 127 | new Thread(Integer.toString(i)) { 128 | public void run() { 129 | try { 130 | startLatch.countDown(); 131 | startLatch.await(); 132 | 133 | queue.push(getName()); 134 | latch.countDown(); 135 | } catch (Throwable e) { 136 | e.printStackTrace(); 137 | } 138 | } 139 | }.start(); 140 | } 141 | 142 | latch.await(5, TimeUnit.SECONDS); 143 | 144 | assert queue.size() == threadCount; 145 | } finally { 146 | queue.close(); 147 | } 148 | } 149 | } 150 | --------------------------------------------------------------------------------