├── .gitignore ├── CHANGELOG ├── README.md ├── docs └── journal.md ├── project ├── Build.scala ├── build.properties ├── plugins.sbt └── release.properties └── src ├── main └── scala │ └── com │ └── twitter │ └── libkestrel │ ├── BlockingQueue.scala │ ├── BookmarkFile.scala │ ├── ConcurrentBlockingQueue.scala │ ├── DeadlineWaitQueue.scala │ ├── ItemIdList.scala │ ├── Journal.scala │ ├── JournalFile.scala │ ├── JournaledBlockingQueue.scala │ ├── JournaledQueue.scala │ ├── MemoryMappedFile.scala │ ├── PeriodicSyncFile.scala │ ├── QueueItem.scala │ ├── ReadWriteLock.scala │ ├── Record.scala │ ├── RecordReader.scala │ ├── RecordWriter.scala │ ├── SimpleBlockingQueue.scala │ ├── SimpleFile.scala │ ├── config │ └── JournaledQueueConfig.scala │ └── tools │ └── QueueDumper.scala ├── scripts ├── qdumper └── qtest └── test └── scala └── com └── twitter └── libkestrel ├── BookmarkFileSpec.scala ├── ConcurrentBlockingQueueSpec.scala ├── DeadlineWaitQueueSpec.scala ├── ItemIdListSpec.scala ├── JournalFileSpec.scala ├── JournalReaderSpec.scala ├── JournalSpec.scala ├── JournaledBlockingQueueSpec.scala ├── JournaledQueueSpec.scala ├── MemoryMappedFileSpec.scala ├── PeriodicSyncFileSpec.scala ├── ResourceCheckingSuite.scala ├── TempFolder.scala ├── TestLogging.scala └── load ├── FloodTest.scala ├── LoadTesting.scala ├── PutTest.scala ├── QTest.scala └── TimeoutTest.scala /.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | kestrel.tmproj 3 | dist 4 | *.class 5 | bin 6 | .manager 7 | .DS_Store 8 | *.sw? 9 | ignore/ 10 | *.iml 11 | .idea/ 12 | .classpath 13 | .project 14 | .scala_dependencies 15 | .#* 16 | *~ 17 | [#]*[#] 18 | -------------------------------------------------------------------------------- /CHANGELOG: -------------------------------------------------------------------------------- 1 | 2.0.0 2 | ----- 3 | release: november 2012 4 | 5 | - rewrite JournaledQueue to avoid in-memory queue and remove use 6 | of Serialized, which can exhibit catastrophic behavior 7 | - fix discardExpired semantics to match kestrel 8 | - ability to evict waiters to shutdown more quickly 9 | - compute queue age from earliest add time 10 | - total/canceled transaction counters 11 | - exclude directories from queue names generated by directory listing 12 | - scala 2.9.2, util 5.3.13 13 | 14 | 1.2.0 15 | ----- 16 | release: 6 july 2012 17 | 18 | - avoid creation of timer tasks when caller is willing to wait "forever" 19 | 20 | 1.1.0 21 | ----- 22 | release: 29 may 2012 23 | 24 | - upgrade to util-core 5.0.3 25 | 26 | 1.0.4 27 | ----- 28 | release: 15 may 2012 29 | 30 | - interim release that uses util-core 4.0.1 31 | 32 | 1.0.3 33 | ----- 34 | release: 7 may 2012 35 | 36 | - interim release that uses the (unreleased yet) util-core 4.0.0 37 | 38 | 1.0.2 39 | ----- 40 | release: 19 april 2012 41 | 42 | - convert to sbt 0.11 43 | - make ConcurrentBlockingQueue#maxItems a hard limit instead of a sloppy limit 44 | - track putBytes, getHitCount, and getMissCount in JournaledQueue 45 | 46 | 1.0.1 47 | ----- 48 | release: 26 march 2012 49 | 50 | - add TransactionalBlockingQueue wrapped for JournaledQueue. Provides 51 | simplified API for single-reader applications. 52 | 53 | 1.0.0 54 | ----- 55 | release: 20 march 2012 56 | 57 | - initial release! 58 | 59 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # libkestrel 3 | 4 | Libkestrel is a library for scala/java containing: 5 | 6 | - ConcurrentBlockingQueue - a lock-free queue that allows readers to block 7 | (wait) for items 8 | 9 | - JournaledQueue - a queue with potentially many read-pointers, where state 10 | of the queue and its readers is saved to disk 11 | 12 | These are variants and improvements of the building blocks of the kestrel 13 | distributed queue server project. The intent (as of November 2011) is to make 14 | kestrel use this library in its next major release. 15 | 16 | 17 | ## Build 18 | 19 | $ sbt clean update package-dist 20 | 21 | SBT must presently be version 0.11.2 or you'll get build errors (due to SBT 22 | plugin versioning). Note also that if the script you use to run SBT ("sbt" 23 | in the example) enables assertions (via java -ea or java -enableassertions) 24 | the build will fail due to a Scala compiler bug. 25 | 26 | ## Community 27 | 28 | Come talk to us on the kestrel mailing list! 29 | 30 | http://groups.google.com/group/kestrel-talk 31 | 32 | 33 | ## ConcurrentBlockingQueue 34 | 35 | ConcurrentBlockingQueue extends the idea of java's `ConcurrentLinkedQueue` to 36 | allow consumers to block, indefinitely or with a timeout, until items arrive. 37 | 38 | It works by having one `ConcurrentLinkedQueue` for the queue itself, and 39 | another one to track waiting consumers. Each time an item is put into the 40 | queue, it's handed off to the next waiting consumer, like an airport taxi 41 | line. 42 | 43 | The handoff occurs in a serialized block (like the `Serialized` trait in 44 | util-core), so when there's no contention, a new item is handed directly from 45 | the producer to the consumer. When there is contention, producers increment a 46 | counter representing how many pent-up handoffs there are, and the producer 47 | that got into the serialized block first will do each handoff in order until 48 | the count is zero again. This way, no producer is ever blocked. 49 | 50 | Consumers receive a future that will eventually be fulfilled either with 51 | `Some(item)` if an item arrived before the requested timeout, or `None` if the 52 | request timed out. If an item was available immediately, the future will be 53 | fulfilled before the consumer receives it. This way, no consumer is ever 54 | blocked. 55 | 56 | 57 | ## JournaledQueue 58 | 59 | A JournaledQueue is a journaled queue that may have multiple "readers", each of 60 | which may have multiple consumers. 61 | 62 | ### Puts 63 | 64 | When an item is added to a queue, it's journaled and notifiation is passed on 65 | to any readers. There is always at least one reader, and the reader knows its 66 | current location in the memory-mapped journal file. If there are multiple 67 | readers, they behave as multiple independent queues, each receiving a copy of 68 | each item added to the `JournaledQueue`, but sharing a single journal. They may 69 | have different policies on queue size limits, item expiration, and error 70 | handling. 71 | 72 | ### Gets 73 | 74 | Items are read only from readers. When an item is available, it's set aside as 75 | an "open read", but not committed to the journal. A separate call is made to 76 | either commit the item or abort it. Aborting an item returns it to the head of 77 | the queue to be given to the next consumer. 78 | 79 | Periodically each reader records its state in a separate checkpoint journal. 80 | When initialized, if a journal already exists for a queue and its readers, each 81 | reader restores itself from this saved state. If the queues were not shutdown 82 | cleanly, the state files may be out of date and items may be replayed. Care is 83 | taken never to let any of the journal files be corrupted or in a 84 | non-recoverable state. In case of error, the choice is always made to possibly 85 | replay items instead of losing them. 86 | 87 | ### Infinite scroll 88 | 89 | The writer journal is treated as an "infinite scroll" of put operations. Each 90 | journal file is created with the current timestamp in its filename, and after 91 | it gets "full" (configurable, but 16MB by default), that file is closed and a 92 | new one is opened. If no readers ever consumed items from the queue, these 93 | files would sit around forever. 94 | 95 | Once all readers have moved their read-pointer (the head of their queue) past 96 | the end of journal file, that file is archived. By default, that just means 97 | the file is deleted, but one of the configuration parameters allows you to 98 | have the dead files moved to a different folder instead. 99 | 100 | There are several advantages to splitting the journals into a single 101 | (multi-file) writer journal and several reader checkpoint files: 102 | 103 | - Fan-out queues (multiple read-pointers into the same queue) are free, and 104 | all the readers share a single journal, saving disk space and bandwidth. 105 | Disk bandwidth is now almost entirely based on write throughput, not the 106 | number of readers. 107 | 108 | - The journals never have to be "packed" to save disk space, the way they did 109 | in kestrel 2.x. Packing creates more disk I/O at the very time a server 110 | might be struggling to keep up with existing load. 111 | 112 | - Archiving old queue items is trivial, and allows you to do some 113 | meta-analysis of load offline. 114 | 115 | 116 | ## Tests 117 | 118 | Some load tests are included. You can run them with 119 | 120 | $ ./dist/libkestrel/scripts/qtest 121 | 122 | which will list the available tests. Each test responds to "`--help`". 123 | 124 | 125 | ## File by file overview 126 | 127 | - BlockingQueue - interface for any blocking queue 128 | 129 | - SimpleBlockingQueue - a simple queue using "synchronized", based on the one 130 | in kestrel 2.1 131 | 132 | - ConcurrentBlockingQueue - a lock-free BlockingQueue (see above) 133 | 134 | - PeriodicSyncFile - a writable file that `fsync`s on a schedule 135 | 136 | - ItemIdList - a simple Set[Long] implementation optimized for small sets 137 | 138 | - QueueItem - an item plus associated metadata (item ID, expiration time) 139 | 140 | - JournalFile - a single on-disk journal file, encapsulating reading & writing 141 | journal records 142 | 143 | - Journal - representation of a collection of files (the writer files and a 144 | file for each reader) 145 | 146 | - JournaledQueue - a `Journal` based queue implementation 147 | 148 | - JournaledBlockingQueue - JournaledQueue wrappers that provide a simplified 149 | interface for users that only use a single reader (with or without 150 | transactions) 151 | 152 | ## Improvement targets 153 | 154 | - Trustin suggested that the read-pointer files could be, instead of an id, a 155 | filename and position. That would save us from reading the first half of a 156 | journal file on startup (to find the id). 157 | 158 | - Nick suggested that writing all of the readers into the same file could 159 | reduce disk I/O by writing fewer blocks during reader checkpointing. 160 | -------------------------------------------------------------------------------- /docs/journal.md: -------------------------------------------------------------------------------- 1 | 2 | # Journal file format 3 | 4 | There are two types of journal file: writer and reader. A writer file is a 5 | sequence of "put" operations -- an infinite scroll of all items ever added to 6 | the queue. A reader file represents the state of a single reader: its current 7 | head position, and any items that have been confirmed out-of-order. Each type 8 | of file uses the same low-level format, differentiated only by their header 9 | bytes. 10 | 11 | ## Filenames 12 | 13 | All filenames for a queue begin with that queue name followed by a dot. For 14 | example, the journal files for the queue "jobs" will all begin with "`jobs.`". 15 | 16 | Reader files are followed by "`read.`" and the name of the reader, which might 17 | be an empty string if there is only one reader. The default (empty-string) 18 | reader for "jobs" would be "`jobs.read.`". For a reader named "indexer", it 19 | would be "`jobs.read.indexer`". 20 | 21 | Writer files are followed by an always-incrementing number, usually the 22 | current time in milliseconds. An example journal file for the "jobs" queue is 23 | "`jobs.1321401903634`". These journal files are always sorted numerically, 24 | with smaller numbers being older segments of the journal. 25 | 26 | ## Low-level format 27 | 28 | Each file contains a 4-byte identifying header followed by a sequence of records. 29 | 30 | Each record is: 31 | 32 | - command (1 byte) 33 | - header bytes (optional) 34 | - data bytes (optional) 35 | 36 | The command byte is made up of 8 bits: 37 | 38 | . 7 6 5 4 3 2 1 0 39 | +---+---+---+---+---+---+---+---+ 40 | | command | header size | 41 | +---+---+---+---+---+---+---+---+ 42 | 43 | The header size is the number of 32-bit header words following the command 44 | byte. In other words, it's the number of header bytes divided by 4, so there 45 | may be from 0 to 64 bytes of header. 46 | 47 | Commands with the high bit set (8 - 15) are followed by a block of data. In 48 | this case, there is always at least 4 bytes of header, and the first 32 bits 49 | of header is a count of the number of data bytes to follow. 50 | 51 | For example, command byte 0x73 represents command 7, which has no data block 52 | and 12 bytes of header. 53 | 54 | All header data is in little-endian format. 55 | 56 | All enqueued items have an ID, which is a non-zero 64-bit number. 57 | 58 | ## Write journal 59 | 60 | header: 27 64 26 03 61 | 62 | ### PUT (8) 63 | 64 | An item was added to the queue. This is the only entry in the writer files. 65 | The header size is either 6 or 8. 66 | 67 | Header: 68 | 69 | - i32 data size 70 | - i32 error_count 71 | - i64 item_id 72 | - i64 add_time (msec, absolute epoch time) 73 | - i64 expire_time (msec, absolute epoch time) [optional] 74 | 75 | Data: 76 | 77 | - (bytes) 78 | 79 | ## Read checkpoint 80 | 81 | header: 26 3C 26 03 82 | 83 | ### READ_HEAD (0) 84 | 85 | The id of the current queue head for this reader. The header size is always 86 | 2. 87 | 88 | Header: 89 | 90 | - i64 id (0 for empty queue) 91 | 92 | ### READ_DONE (9) 93 | 94 | A set of item ids for items that have been confirmed as removed from the queue 95 | out-of-order. These ids will always be greater than the current head id. The 96 | header size is always 1, and the data block is a sequence of 64-bit ids. 97 | 98 | Header: 99 | 100 | - i32 count 101 | 102 | Data: 103 | 104 | - i64* xid 105 | 106 | -------------------------------------------------------------------------------- /project/Build.scala: -------------------------------------------------------------------------------- 1 | import sbt._ 2 | import Keys._ 3 | import com.twitter.sbt._ 4 | 5 | object Libkestrel extends Build { 6 | val utilVersion = "5.3.14" 7 | 8 | lazy val root = Project( 9 | id = "libkestrel", 10 | base = file("."), 11 | settings = Project.defaultSettings ++ 12 | StandardProject.newSettings ++ 13 | SubversionPublisher.newSettings 14 | ).settings( 15 | name := "libkestrel", 16 | organization := "com.twitter", 17 | version := "2.0.1-SNAPSHOT", 18 | scalaVersion := "2.9.2", 19 | 20 | // time-based tests cannot be run in parallel 21 | logBuffered in Test := false, 22 | parallelExecution in Test := false, 23 | 24 | libraryDependencies ++= Seq( 25 | "com.twitter" % "util-core" % utilVersion, 26 | "com.twitter" % "util-logging" % utilVersion, 27 | 28 | // for tests only: 29 | "org.scalatest" %% "scalatest" % "1.8" % "test", 30 | "com.github.scopt" %% "scopt" % "2.1.0" % "test", 31 | "com.twitter" % "scalatest-mixins_2.9.1" % "1.1.0" % "test" 32 | ), 33 | 34 | scalacOptions += "-deprecation", 35 | SubversionPublisher.subversionRepository := Some("https://svn.twitter.biz/maven-public"), 36 | publishArtifact in Test := true 37 | ) 38 | } 39 | -------------------------------------------------------------------------------- /project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=0.11.2 2 | 3 | -------------------------------------------------------------------------------- /project/plugins.sbt: -------------------------------------------------------------------------------- 1 | name := "plugins" 2 | 3 | sbtResolver <<= (sbtResolver) { r => 4 | Option(System.getenv("SBT_PROXY_REPO")) map { x => 5 | Resolver.url("proxy repo for sbt", url(x))(Resolver.ivyStylePatterns) 6 | } getOrElse r 7 | } 8 | 9 | resolvers <<= (resolvers) { r => 10 | (Option(System.getenv("SBT_PROXY_REPO")) map { url => 11 | Seq("proxy-repo" at url) 12 | } getOrElse { 13 | r ++ Seq( 14 | "twitter.com" at "http://maven.twttr.com/", 15 | "scala-tools" at "http://scala-tools.org/repo-releases/", 16 | "maven" at "http://repo1.maven.org/maven2/", 17 | "freemarker" at "http://freemarker.sourceforge.net/maven2/" 18 | ) 19 | }) ++ Seq("local" at ("file:" + System.getProperty("user.home") + "/.m2/repository/")) 20 | } 21 | 22 | externalResolvers <<= (resolvers) map identity 23 | 24 | addSbtPlugin("com.twitter" % "sbt-package-dist" % "1.0.6") 25 | -------------------------------------------------------------------------------- /project/release.properties: -------------------------------------------------------------------------------- 1 | #Automatically generated by ReleaseManagement 2 | #Mon Mar 26 11:59:34 PDT 2012 3 | version=1.0.1 4 | sha1=0281a70c78f99f21276389f1b3e7ffb05eba053f 5 | -------------------------------------------------------------------------------- /src/main/scala/com/twitter/libkestrel/BlockingQueue.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2011 Twitter, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | * not use this file except in compliance with the License. You may obtain 6 | * a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twitter.libkestrel 18 | 19 | import com.twitter.util.{Future, Time} 20 | 21 | sealed abstract class Deadline 22 | case class Before(deadline: Time) extends Deadline 23 | case object Forever extends Deadline 24 | 25 | trait BlockingQueue[A <: AnyRef] { 26 | def put(item: A): Boolean 27 | def putHead(item: A) 28 | def size: Int 29 | def get(): Future[Option[A]] 30 | def get(deadline: Deadline): Future[Option[A]] 31 | def poll(): Future[Option[A]] 32 | def pollIf(predicate: A => Boolean): Future[Option[A]] 33 | def flush() 34 | def toDebug: String 35 | def close() 36 | def waiterCount: Int 37 | def evictWaiters() 38 | } 39 | 40 | trait Transaction[A <: AnyRef] { 41 | def item: A 42 | def commit(): Unit 43 | def rollback(): Unit 44 | } 45 | 46 | trait TransactionalBlockingQueue[A <: AnyRef] { 47 | def put(item: A): Boolean 48 | def size: Int 49 | def get(): Future[Option[Transaction[A]]] 50 | def get(deadline: Deadline): Future[Option[Transaction[A]]] 51 | def poll(): Future[Option[Transaction[A]]] 52 | def flush() 53 | def toDebug: String 54 | def close() 55 | def waiterCount: Int 56 | def evictWaiters() 57 | } 58 | -------------------------------------------------------------------------------- /src/main/scala/com/twitter/libkestrel/BookmarkFile.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Twitter, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | * not use this file except in compliance with the License. You may obtain 6 | * a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twitter.libkestrel 18 | 19 | import java.io.{File, FileInputStream, IOException} 20 | import java.nio.ByteBuffer 21 | 22 | object BookmarkFile { 23 | val HEADER_BOOKMARK = 0x263C2603 24 | 25 | def open(file: File) = { 26 | val reader = new BookmarkFileReader(file) 27 | if (reader.open() != HEADER_BOOKMARK) { 28 | reader.close() 29 | throw new IOException("Not a bookmark file: " + file) 30 | } 31 | reader 32 | } 33 | 34 | def create(file: File) = { 35 | val writer = new BookmarkFileWriter(file) 36 | writer.create(HEADER_BOOKMARK) 37 | writer 38 | } 39 | } 40 | 41 | class BookmarkFileReader(val file: File) extends RecordReader { 42 | var reader: ByteBuffer = null 43 | 44 | def open(): Int = { 45 | val channel = new FileInputStream(file).getChannel 46 | if (channel.size > Int.MaxValue.toLong) throw new IOException("Bookmark file too large: " + file) 47 | 48 | reader = ByteBuffer.allocate(channel.size.toInt) 49 | channel.read(reader) 50 | channel.close 51 | reader.flip() 52 | 53 | readMagic() 54 | } 55 | 56 | def close() { 57 | reader = null 58 | } 59 | } 60 | 61 | class BookmarkFileWriter(val file: File) extends RecordWriter { 62 | protected[this] val writer = new SimpleFile(file) 63 | 64 | def create(magic: Int) { 65 | writeMagic(magic) 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /src/main/scala/com/twitter/libkestrel/ConcurrentBlockingQueue.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2011 Twitter, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | * not use this file except in compliance with the License. You may obtain 6 | * a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twitter.libkestrel 18 | 19 | import com.twitter.util._ 20 | import java.util.concurrent.{ConcurrentHashMap, ConcurrentLinkedQueue} 21 | import java.util.concurrent.atomic.AtomicInteger 22 | import scala.collection.JavaConverters._ 23 | 24 | object ConcurrentBlockingQueue { 25 | /** What to do when the queue is full and a `put` is attempted (for the constructor). */ 26 | abstract sealed class FullPolicy 27 | 28 | object FullPolicy { 29 | /** When the queue is full, a `put` attempt returns `false`. */ 30 | case object RefusePuts extends FullPolicy 31 | 32 | /** When the queue is full, a `put` attempt will throw away the head item. */ 33 | case object DropOldest extends FullPolicy 34 | } 35 | 36 | /** 37 | * Make a queue with no effective size limit. 38 | */ 39 | def apply[A <: AnyRef](implicit timer: Timer) = { 40 | new ConcurrentBlockingQueue[A](Long.MaxValue, FullPolicy.RefusePuts, timer) 41 | } 42 | 43 | /** 44 | * Make a queue with a fixed maximum item count and a policy for what to do when the queue is 45 | * full and a `put` is attempted. 46 | */ 47 | def apply[A <: AnyRef](maxItems: Long, fullPolicy: FullPolicy)(implicit timer: Timer) = { 48 | new ConcurrentBlockingQueue[A](maxItems, fullPolicy, timer) 49 | } 50 | } 51 | 52 | /** 53 | * A lock-free blocking queue that supports timeouts. 54 | * 55 | * It works by having one `ConcurrentLinkedQueue` for the queue itself, and 56 | * another one to track waiting consumers. Each time an item is put into the 57 | * queue, it's handed off to the next waiting consumer, like an airport taxi 58 | * line. 59 | * 60 | * The handoff occurs in a serialized block (like the `Serialized` trait in 61 | * util-core), so when there's no contention, a new item is handed directly from 62 | * the producer to the consumer. When there is contention, producers increment a 63 | * counter representing how many pent-up handoffs there are, and the producer 64 | * that got into the serialized block first will do each handoff in order until 65 | * the count is zero again. This way, no producer is ever blocked. 66 | * 67 | * Consumers receive a future that will eventually be fulfilled either with 68 | * `Some(item)` if an item arrived before the requested timeout, or `None` if the 69 | * request timed out. If an item was available immediately, the future will be 70 | * fulfilled before the consumer receives it. This way, no consumer is ever 71 | * blocked. 72 | * 73 | * @param maxItems maximum allowed size of the queue (use `Long.MaxValue` for infinite size) 74 | * @param fullPolicy what to do when the queue is full and a `put` is attempted 75 | * @param timer a Timer to use for triggering timeouts 76 | */ 77 | final class ConcurrentBlockingQueue[A <: AnyRef]( 78 | maxItems: Long, 79 | fullPolicy: ConcurrentBlockingQueue.FullPolicy, 80 | timer: Timer 81 | ) extends BlockingQueue[A] { 82 | import ConcurrentBlockingQueue._ 83 | 84 | /** 85 | * The actual queue of items. 86 | * We assume that normally there are more readers than writers, so the queue is normally empty. 87 | * But when nobody is waiting, we degenerate into a non-blocking queue, and this queue comes 88 | * into play. 89 | */ 90 | private[this] val queue = new ConcurrentLinkedQueue[A] 91 | 92 | /** 93 | * Items "returned" to the head of this queue. Usually this has zero or only a few items. 94 | */ 95 | private[this] val headQueue = new ConcurrentLinkedQueue[A] 96 | 97 | /** 98 | * A queue of readers, some waiting with a timeout, others polling. 99 | * `consumers` tracks the order for fairness, but `waiterSet` and `pollerSet` are 100 | * the definitive sets: a waiter/poller may be in the queue, but not in the set, which 101 | * just means that they had a timeout set and gave up or were rejected due to an 102 | * empty queue. 103 | */ 104 | abstract sealed class Consumer { 105 | def promise: Promise[Option[A]] 106 | def apply(item: A): Boolean 107 | } 108 | private[this] val consumers = new ConcurrentLinkedQueue[Consumer] 109 | 110 | /** 111 | * A queue of readers waiting to retrieve an item. See `consumers`. 112 | */ 113 | case class Waiter(promise: Promise[Option[A]], timerTask: Option[TimerTask]) extends Consumer { 114 | def apply(item: A) = { 115 | timerTask.foreach { _.cancel() } 116 | promise.setValue(Some(item)) 117 | true 118 | } 119 | } 120 | private[this] val waiterSet = new ConcurrentHashMap[Promise[Option[A]], Promise[Option[A]]] 121 | 122 | /** 123 | * A queue of pollers just checking in to see if anything is immediately available. 124 | * See `consumers`. 125 | */ 126 | case class Poller(promise: Promise[Option[A]], predicate: A => Boolean) extends Consumer { 127 | def apply(item: A) = { 128 | if (predicate(item)) { 129 | promise.setValue(Some(item)) 130 | true 131 | } else { 132 | promise.setValue(None) 133 | false 134 | } 135 | } 136 | } 137 | private[this] val truth: A => Boolean = { _ => true } 138 | private[this] val pollerSet = new ConcurrentHashMap[Promise[Option[A]], Promise[Option[A]]] 139 | 140 | /** 141 | * An estimate of the queue size, tracked for each put/get. 142 | */ 143 | private[this] val elementCount = new AtomicInteger(0) 144 | 145 | /** 146 | * Sequential lock used to serialize access to handoffOne(). 147 | */ 148 | private[this] val triggerLock = new AtomicInteger(0) 149 | 150 | /** 151 | * Count of items dropped because the queue was full. 152 | */ 153 | val droppedCount = new AtomicInteger(0) 154 | 155 | /** 156 | * Inserts the specified element into this queue if it is possible to do so immediately without 157 | * violating capacity restrictions, returning `true` upon success and `false` if no space is 158 | * currently available. 159 | */ 160 | def put(item: A): Boolean = { 161 | if (elementCount.incrementAndGet() > maxItems && fullPolicy == FullPolicy.RefusePuts) { 162 | elementCount.decrementAndGet() 163 | false 164 | } else { 165 | queue.add(item) 166 | handoff() 167 | true 168 | } 169 | } 170 | 171 | /** 172 | * Inserts the specified element into this queue at the head, without checking capacity 173 | * restrictions. This is used to "return" items to a queue. 174 | */ 175 | def putHead(item: A) { 176 | headQueue.add(item) 177 | elementCount.incrementAndGet() 178 | handoff() 179 | } 180 | 181 | /** 182 | * Return the size of the queue as it was at some (recent) moment in time. 183 | */ 184 | def size: Int = elementCount.get() 185 | 186 | /** 187 | * Return the number of consumers waiting for an item. 188 | */ 189 | def waiterCount: Int = waiterSet.size 190 | 191 | /** 192 | * Get the next item from the queue, waiting forever if necessary. 193 | */ 194 | def get(): Future[Option[A]] = get(Forever) 195 | 196 | /** 197 | * Get the next item from the queue if it arrives before a timeout. 198 | */ 199 | def get(deadline: Deadline): Future[Option[A]] = { 200 | val promise = new Promise[Option[A]] 201 | waiterSet.put(promise, promise) 202 | val timerTask = 203 | deadline match { 204 | case Before(time) => 205 | val timerTask = timer.schedule(time) { 206 | if (waiterSet.remove(promise) ne null) { 207 | promise.setValue(None) 208 | } 209 | } 210 | Some(timerTask) 211 | case Forever => None 212 | } 213 | consumers.add(Waiter(promise, timerTask)) 214 | promise.onCancellation { 215 | waiterSet.remove(promise) 216 | timerTask.foreach { _.cancel() } 217 | } 218 | if (!queue.isEmpty || !headQueue.isEmpty) handoff() 219 | promise 220 | } 221 | 222 | 223 | 224 | /** 225 | * Get the next item from the queue if one is immediately available. 226 | */ 227 | def poll(): Future[Option[A]] = pollIf(truth) 228 | 229 | /** 230 | * Get the next item from the queue if it satisfies a predicate. 231 | */ 232 | def pollIf(predicate: A => Boolean): Future[Option[A]] = { 233 | if (queue.isEmpty && headQueue.isEmpty) { 234 | Future.value(None) 235 | } else { 236 | val promise = new Promise[Option[A]] 237 | pollerSet.put(promise, promise) 238 | consumers.add(Poller(promise, predicate)) 239 | handoff() 240 | promise 241 | } 242 | } 243 | 244 | def flush() { 245 | queue.clear() 246 | headQueue.clear() 247 | } 248 | 249 | /** 250 | * This is the only code path allowed to remove an item from `queue` or `consumers`. 251 | */ 252 | private[this] def handoff() { 253 | if (triggerLock.getAndIncrement() == 0) { 254 | do { 255 | handoffOne() 256 | } while (triggerLock.decrementAndGet() > 0) 257 | } 258 | } 259 | 260 | private[this] def handoffOne() { 261 | if (fullPolicy == FullPolicy.DropOldest) { 262 | // make sure we aren't over the max queue size. 263 | while (elementCount.get > maxItems) { 264 | droppedCount.getAndIncrement() 265 | queue.poll() 266 | elementCount.decrementAndGet() 267 | } 268 | } 269 | 270 | var fromHead = false 271 | val item = { 272 | val x = headQueue.peek() 273 | if (x ne null) { 274 | fromHead = true 275 | x 276 | } else queue.peek() 277 | } 278 | if (item ne null) { 279 | var consumer: Consumer = null 280 | var invalid = true 281 | do { 282 | consumer = consumers.poll() 283 | invalid = consumer match { 284 | case null => false 285 | case Waiter(promise, _) => waiterSet.remove(promise) eq null 286 | case Poller(promise, _) => pollerSet.remove(promise) eq null 287 | } 288 | } while (invalid) 289 | 290 | if ((consumer ne null) && consumer(item)) { 291 | if (fromHead) headQueue.poll() else queue.poll() 292 | if (elementCount.decrementAndGet() == 0) { 293 | dumpPollerSet() 294 | } 295 | } 296 | } else { 297 | // empty -- dump outstanding pollers 298 | dumpPollerSet() 299 | } 300 | } 301 | 302 | private[this] def dumpWaiterSet() { 303 | waiterSet.keySet.asScala.toArray.foreach { waiter => 304 | waiter.setValue(None) 305 | waiterSet.remove(waiter) 306 | } 307 | } 308 | 309 | private[this] def dumpPollerSet() { 310 | pollerSet.keySet.asScala.toArray.foreach { poller => 311 | poller.setValue(None) 312 | pollerSet.remove(poller) 313 | } 314 | } 315 | 316 | def toDebug: String = { 317 | "".format( 318 | elementCount.get, consumers.size, waiterSet.size, pollerSet.size) 319 | } 320 | 321 | def close() { 322 | queue.clear() 323 | headQueue.clear() 324 | waiterSet.asScala.keys.foreach { _.setValue(None) } 325 | pollerSet.asScala.keys.foreach { _.setValue(None) } 326 | } 327 | 328 | def peekOldest: Option[A] = { 329 | Option(queue.peek()) match { 330 | case s@Some(_) => s 331 | case None => Option(headQueue.peek()) // check for returned items 332 | } 333 | } 334 | 335 | def evictWaiters() { 336 | dumpWaiterSet() 337 | dumpPollerSet() 338 | } 339 | } 340 | -------------------------------------------------------------------------------- /src/main/scala/com/twitter/libkestrel/DeadlineWaitQueue.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2011 Twitter, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | * not use this file except in compliance with the License. You may obtain 6 | * a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twitter.libkestrel 18 | 19 | import java.util.LinkedHashSet 20 | import scala.collection.JavaConversions 21 | import com.twitter.util.{Time, Timer, TimerTask} 22 | 23 | sealed trait Waiter { 24 | def awaken: () => Unit 25 | def timeout: () => Unit 26 | def cancel(): Unit 27 | } 28 | 29 | case class InfiniteWaiter(val awaken: () => Unit, val timeout: () => Unit) extends Waiter { 30 | def cancel() { } 31 | } 32 | 33 | case class DeadlineWaiter(var timerTask: TimerTask, val awaken: () => Unit, val timeout: () => Unit) 34 | extends Waiter { 35 | def cancel() { 36 | if (timerTask ne null) timerTask.cancel() 37 | } 38 | } 39 | 40 | /** 41 | * A wait queue where each item has a timeout. 42 | * On each `trigger()`, one waiter is awoken (the awaken function is called). If the timeout is 43 | * triggered by the Timer, the timeout function will be called instead. The queue promises that 44 | * exactly one of the functions will be called, never both. 45 | */ 46 | final class DeadlineWaitQueue(timer: Timer) { 47 | 48 | private val queue = JavaConversions.asScalaSet(new LinkedHashSet[Waiter]) 49 | 50 | def add(deadline: Deadline, awaken: () => Unit, timeout: () => Unit) = { 51 | val waiter: Waiter = 52 | deadline match { 53 | case Before(time) => 54 | val deadlineWaiter = DeadlineWaiter(null, awaken, timeout) 55 | val timerTask = timer.schedule(time) { 56 | if (synchronized { queue.remove(deadlineWaiter) }) deadlineWaiter.timeout() 57 | } 58 | deadlineWaiter.timerTask = timerTask 59 | deadlineWaiter 60 | case Forever => InfiniteWaiter(awaken, timeout) 61 | } 62 | 63 | synchronized { queue.add(waiter) } 64 | waiter 65 | } 66 | 67 | def remove(waiter: Waiter) { 68 | synchronized { queue.remove(waiter) } 69 | waiter.cancel() 70 | } 71 | 72 | def trigger() { 73 | synchronized { 74 | queue.headOption.map { waiter => 75 | queue.remove(waiter) 76 | waiter 77 | } 78 | }.foreach { waiter => 79 | waiter.cancel() 80 | waiter.awaken() 81 | } 82 | } 83 | 84 | def triggerAll() { 85 | synchronized { 86 | val rv = queue.toArray 87 | queue.clear() 88 | rv 89 | }.foreach { waiter => 90 | waiter.cancel() 91 | waiter.awaken() 92 | } 93 | } 94 | 95 | def evictAll() { 96 | synchronized { 97 | val rv = queue.toArray 98 | queue.clear() 99 | rv 100 | }.foreach { waiter => 101 | waiter.cancel() 102 | waiter.timeout() 103 | } 104 | } 105 | 106 | def size() = { 107 | synchronized { queue.size } 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /src/main/scala/com/twitter/libkestrel/ItemIdList.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2011 Twitter, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | * not use this file except in compliance with the License. You may obtain 6 | * a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.twitter.libkestrel 17 | 18 | import scala.annotation.tailrec 19 | import scala.collection.mutable 20 | import scala.collection.Set 21 | 22 | /** 23 | * Set of ids that maintains insert order. 24 | * 25 | * The "set" property is not enforced but is assumed because these are meant to be item ids. 26 | * Good performance assumes that operations are usually add, pop, popAll, or a remove of most 27 | * or all of the items. (Remove is O(n) so if there are many items and only a few are being 28 | * removed, performance will be bad.) This is tuned toward the general case where a client will 29 | * have either very few items open, or will have many items open but will remove them all at 30 | * once. 31 | */ 32 | class ItemIdList { 33 | private[this] var _ids = new Array[Long](16) 34 | private[this] var head = 0 35 | private[this] var tail = 0 36 | 37 | def pop(count: Int): Seq[Long] = { 38 | if (count > tail - head) { 39 | Seq() 40 | } else { 41 | val rv = _ids.slice(head, head + count) 42 | head += count 43 | rv 44 | } 45 | } 46 | 47 | def pop(): Option[Long] = pop(1).headOption 48 | 49 | def add(id: Long) { 50 | if (head > 0) { 51 | compact() 52 | } 53 | if (tail == _ids.size) { 54 | val bigger = new Array[Long](_ids.size * 4) 55 | System.arraycopy(_ids, 0, bigger, 0, _ids.size) 56 | _ids = bigger 57 | } 58 | _ids(tail) = id 59 | tail += 1 60 | } 61 | 62 | def add(ids: Seq[Long]) { 63 | ids.foreach(add) 64 | } 65 | 66 | def size: Int = tail - head 67 | 68 | def popAll(): Seq[Long] = { 69 | val rv = _ids.slice(head, tail) 70 | head = 0 71 | tail = 0 72 | rv 73 | } 74 | 75 | def remove(ids: Set[Long]): Set[Long] = { 76 | var n = head 77 | val removed = new mutable.HashSet[Long] 78 | while (n < tail) { 79 | if (ids contains _ids(n)) { 80 | removed += _ids(n) 81 | _ids(n) = 0 82 | } 83 | n += 1 84 | } 85 | compact() 86 | removed.toSet 87 | } 88 | 89 | private[this] def find(id: Long, remove: Boolean): Boolean = { 90 | var n = head 91 | while (n < tail) { 92 | if (_ids(n) == id) { 93 | if (remove) { 94 | _ids(n) = 0 95 | compact() 96 | } 97 | return true 98 | } 99 | n += 1 100 | } 101 | false 102 | } 103 | 104 | def remove(id: Long) = find(id, true) 105 | 106 | def contains(id: Long) = find(id, false) 107 | 108 | // for tests: 109 | def toSeq: Seq[Long] = _ids.slice(head, tail) 110 | 111 | def compact() { compact(0, head) } 112 | 113 | override def toString() = "".format(toSeq.sorted) 114 | 115 | @tailrec 116 | private[this] def compact(start: Int, h1: Int) { 117 | if (h1 == tail) { 118 | // string of zeros. flatten tail and done. 119 | head = 0 120 | tail = start 121 | } else { 122 | // now find string of filled items. 123 | var h2 = h1 124 | while (h2 < tail && _ids(h2) != 0) h2 += 1 125 | if (h1 != start && h2 != h1) System.arraycopy(_ids, h1, _ids, start, h2 - h1) 126 | val newStart = start + h2 - h1 127 | while (h2 < tail && _ids(h2) == 0) h2 += 1 128 | compact(newStart, h2) 129 | } 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /src/main/scala/com/twitter/libkestrel/JournalFile.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2011 Twitter, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | * not use this file except in compliance with the License. You may obtain 6 | * a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twitter.libkestrel 18 | 19 | import com.twitter.util._ 20 | import java.io.{File, FileInputStream, IOException} 21 | import java.nio.{ByteBuffer, ByteOrder, MappedByteBuffer} 22 | import java.nio.channels.FileChannel 23 | import java.util.concurrent.ScheduledExecutorService 24 | 25 | case class CorruptedJournalException(lastValidPosition: Long, file: File, message: String) extends IOException(message) 26 | 27 | object JournalFile { 28 | val HEADER_WRITER = 0x27642603 29 | 30 | def append(file: File, scheduler: ScheduledExecutorService, syncJournal: Duration, 31 | maxFileSize: StorageUnit) = { 32 | val position = file.length() 33 | val writer = new JournalFileWriter(file, scheduler, syncJournal, maxFileSize) 34 | writer.openForAppend(position) 35 | writer 36 | } 37 | 38 | def create(file: File, scheduler: ScheduledExecutorService, syncJournal: Duration, 39 | maxFileSize: StorageUnit) = { 40 | val writer = new JournalFileWriter(file, scheduler, syncJournal, maxFileSize) 41 | writer.create(HEADER_WRITER) 42 | writer 43 | } 44 | 45 | def open(file: File) = { 46 | val reader = new JournalFileReader(file) 47 | if (reader.open() != HEADER_WRITER) { 48 | reader.close() 49 | throw new IOException("Not a journal file") 50 | } 51 | 52 | reader 53 | } 54 | } 55 | 56 | class JournalFileReader(val file: File) extends RecordReader 57 | { 58 | private[this] val memMappedFile = MemoryMappedFile.readOnlyMap(file) 59 | private[this] var _reader: ByteBuffer = memMappedFile.buffer() 60 | 61 | def open(): Int = { 62 | try { 63 | readMagic() 64 | } catch { 65 | case e => 66 | close() 67 | throw e 68 | } 69 | } 70 | 71 | def reader = _reader 72 | 73 | def close() { 74 | _reader = null 75 | memMappedFile.close() 76 | } 77 | } 78 | 79 | class JournalFileWriter(val file: File, scheduler: ScheduledExecutorService, syncJournal: Duration, 80 | val maxFileSize: StorageUnit) 81 | extends RecordWriter 82 | { 83 | protected[this] var writer: WritableFile = null 84 | 85 | def openForAppend(position: Long) { 86 | writer = PeriodicSyncFile.append(file, scheduler, syncJournal, maxFileSize) 87 | writer.position = position 88 | } 89 | 90 | def create(header: Int) { 91 | writer = PeriodicSyncFile.create(file, scheduler, syncJournal, maxFileSize) 92 | writer.position = 0 93 | 94 | val b = buffer(4) 95 | b.order(ByteOrder.BIG_ENDIAN) 96 | b.putInt(header) 97 | b.flip() 98 | writer.write(b) 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /src/main/scala/com/twitter/libkestrel/JournaledBlockingQueue.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2011 Twitter, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | * not use this file except in compliance with the License. You may obtain 6 | * a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twitter.libkestrel 18 | 19 | import com.twitter.util.{Future, Time} 20 | import java.nio.ByteBuffer 21 | 22 | trait Codec[A] { 23 | def encode(item: A): ByteBuffer 24 | def decode(data: ByteBuffer): A 25 | } 26 | 27 | private trait JournaledBlockingQueueMixin[A] { 28 | def queue: JournaledQueue 29 | def codec: Codec[A] 30 | 31 | val reader = queue.reader("") 32 | 33 | def put(item: A) = { 34 | val rv = queue.put(codec.encode(item), Time.now, None) 35 | rv.isDefined && { rv.get.get(); true } 36 | } 37 | 38 | def putHead(item: A) { 39 | throw new Exception("Unsupported operation") 40 | } 41 | 42 | def size: Int = reader.items 43 | 44 | def flush() { 45 | queue.flush() 46 | } 47 | 48 | def toDebug: String = queue.toDebug 49 | 50 | def close() { 51 | queue.close() 52 | } 53 | 54 | def waiterCount = reader.waiterCount 55 | 56 | def evictWaiters { 57 | reader.evictWaiters() 58 | } 59 | } 60 | 61 | private[libkestrel] class JournaledBlockingQueue[A <: AnyRef](val queue: JournaledQueue, val codec: Codec[A]) 62 | extends BlockingQueue[A] with JournaledBlockingQueueMixin[A] { 63 | 64 | def get(): Future[Option[A]] = get(Forever) 65 | 66 | def get(deadline: Deadline): Future[Option[A]] = { 67 | reader.get(Some(deadline)).map { optItem => 68 | optItem.map { item => 69 | reader.commit(item.id) 70 | codec.decode(item.data) 71 | } 72 | } 73 | } 74 | 75 | def poll(): Future[Option[A]] = { 76 | reader.get(None).map { optItem => 77 | optItem.map { item => 78 | reader.commit(item.id) 79 | codec.decode(item.data) 80 | } 81 | } 82 | } 83 | 84 | def pollIf(predicate: A => Boolean): Future[Option[A]] = { 85 | throw new Exception("Unsupported operation") 86 | } 87 | } 88 | 89 | private[libkestrel] class TransactionalJournaledBlockingQueue[A <: AnyRef]( 90 | val queue: JournaledQueue, val codec: Codec[A]) 91 | extends TransactionalBlockingQueue[A] with JournaledBlockingQueueMixin[A] { 92 | 93 | def get(): Future[Option[Transaction[A]]] = get(Forever) 94 | 95 | def get(deadline: Deadline): Future[Option[Transaction[A]]] = { 96 | reader.get(Some(deadline)).map { optItem => 97 | optItem.map { queueItem => 98 | new Transaction[A] { 99 | val item = codec.decode(queueItem.data) 100 | def commit() { reader.commit(queueItem.id) } 101 | def rollback() { reader.unget(queueItem.id) } 102 | } 103 | } 104 | } 105 | } 106 | 107 | def poll(): Future[Option[Transaction[A]]] = { 108 | reader.get(None).map { optItem => 109 | optItem.map { queueItem => 110 | new Transaction[A] { 111 | val item = codec.decode(queueItem.data) 112 | def commit() { reader.commit(queueItem.id) } 113 | def rollback() { reader.unget(queueItem.id) } 114 | } 115 | } 116 | } 117 | } 118 | } 119 | -------------------------------------------------------------------------------- /src/main/scala/com/twitter/libkestrel/JournaledQueue.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2011 Twitter, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | * not use this file except in compliance with the License. You may obtain 6 | * a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twitter.libkestrel 18 | 19 | import com.twitter.conversions.storage._ 20 | import com.twitter.conversions.time._ 21 | import com.twitter.logging.Logger 22 | import com.twitter.util._ 23 | import java.io.File 24 | import java.nio.ByteBuffer 25 | import java.util.concurrent.{ConcurrentHashMap, ScheduledExecutorService} 26 | import java.util.concurrent.atomic.{AtomicInteger,AtomicLong} 27 | import scala.annotation.tailrec 28 | import scala.collection.immutable 29 | import scala.collection.JavaConverters._ 30 | import config._ 31 | 32 | /** 33 | * A journaled queue built on top of `ConcurrentBlockingQueue` that may have multiple "readers". 34 | * 35 | * When an item is added to a queue, it's journaled and passed on to any readers. There is always 36 | * at least one reader, and the reader contains the actual queue. If there are multiple readers, 37 | * they behave as multiple independent queues, each receiving a copy of each item added, but 38 | * sharing a single journal. They may have different policies on memory use, queue size limits, 39 | * and item expiration. 40 | * 41 | * Items are read only from readers. When an item is available, it's set aside as an "open read", 42 | * but not committed to the journal. A separate call is made to either commit the item or abort 43 | * it. Aborting an item returns it to the head of the queue to be given to the next consumer. 44 | * 45 | * Periodically each reader records its state in a separate checkpoint journal. When initialized, 46 | * if a journal already exists for a queue and its readers, each reader restores itself from this 47 | * saved state. If the queues were not shutdown cleanly, the state files may be out of date and 48 | * items may be replayed. Care is taken never to let any of the journal files be corrupted or in a 49 | * non-recoverable state. In case of error, the choice is always made to possibly replay items 50 | * instead of losing them. 51 | * 52 | * @param config a set of configuration parameters for the queue 53 | * @param path the folder to store journals in 54 | * @param timer a Timer to use for triggering timeouts on reads 55 | * @param scheduler a service to use for scheduling periodic disk syncs 56 | */ 57 | class JournaledQueue( 58 | val config: JournaledQueueConfig, path: File, timer: Timer, scheduler: ScheduledExecutorService 59 | ) { 60 | private[this] val log = Logger.get(getClass) 61 | 62 | private[this] val NAME_REGEX = """[^A-Za-z0-9:_-]""".r 63 | if (NAME_REGEX.findFirstIn(config.name).isDefined) { 64 | throw new Exception("Illegal queue name: " + config.name) 65 | } 66 | 67 | private[JournaledQueue] val journal = 68 | new Journal(path, config.name, config.journalSize, scheduler, config.syncJournal, 69 | config.saveArchivedJournals) 70 | 71 | @volatile private[this] var closed = false 72 | 73 | @volatile private[this] var readerMap = immutable.Map.empty[String, Reader] 74 | journal.readerMap.foreach { case (name, _) => reader(name) } 75 | 76 | // checkpoint readers on a schedule. 77 | timer.schedule(config.checkpointTimer) { checkpoint() } 78 | 79 | val name = config.name 80 | 81 | /** 82 | * A read-only view of all the current `Reader` objects for this queue. 83 | */ 84 | def readers = readerMap.values 85 | 86 | /** 87 | * Total number of items across every reader queue being fed by this queue. 88 | */ 89 | def items = readerMap.values.foldLeft(0L) { _ + _.items } 90 | 91 | /** 92 | * Total number of bytes of data across every reader queue being fed by this queue. 93 | */ 94 | def bytes = readerMap.values.foldLeft(0L) { _ + _.bytes } 95 | 96 | /** 97 | * Total number of bytes of data used by the on-disk journal. 98 | */ 99 | def journalBytes = journal.journalSize 100 | 101 | /** 102 | * Get the named reader. If this is a normal (single reader) queue, the default reader is named 103 | * "". If any named reader is created, the default reader is converted to that name and there is 104 | * no longer a default reader. 105 | */ 106 | def reader(name: String): Reader = { 107 | readerMap.get(name).getOrElse { 108 | synchronized { 109 | readerMap.get(name).getOrElse { 110 | if (readerMap.size >= 1 && name == "") { 111 | throw new Exception("Fanout queues don't have a default reader") 112 | } 113 | val readerConfig = config.readerConfigs.get(name).getOrElse(config.defaultReaderConfig) 114 | log.info("Creating reader queue %s+%s", config.name, name) 115 | val reader = new Reader(name, readerConfig) 116 | reader.loadFromJournal(journal.reader(name)) 117 | readerMap += (name -> reader) 118 | reader.catchUp() 119 | 120 | if (name != "") { 121 | readerMap.get("") foreach { r => 122 | // kill the default reader. 123 | readerMap -= "" 124 | journal.dropReader("") 125 | } 126 | } 127 | 128 | reader 129 | } 130 | } 131 | } 132 | } 133 | 134 | def dropReader(name: String) { 135 | synchronized { 136 | readerMap.get(name) foreach { r => 137 | log.info("Destroying reader queue %s+%s", config.name, name) 138 | readerMap -= name 139 | r.close() 140 | journal.dropReader(name) 141 | } 142 | } 143 | } 144 | 145 | /** 146 | * Save the state of all readers. 147 | */ 148 | def checkpoint() { 149 | journal.checkpoint() 150 | } 151 | 152 | /** 153 | * Close this queue. Any further operations will fail. 154 | */ 155 | def close() { 156 | synchronized { 157 | closed = true 158 | readerMap.values.foreach { _.close() } 159 | journal.close() 160 | readerMap = Map() 161 | } 162 | } 163 | 164 | /** 165 | * Close this queue and also erase any journal files. 166 | */ 167 | def erase() { 168 | synchronized { 169 | closed = true 170 | readerMap.values.foreach { _.close() } 171 | journal.erase() 172 | readerMap = Map() 173 | } 174 | } 175 | 176 | /** 177 | * Erase any items in any reader queue. 178 | */ 179 | def flush() { 180 | readerMap.values.foreach { _.flush() } 181 | } 182 | 183 | /** 184 | * Do a sweep of each reader, discarding all expired items up to the reader's configured 185 | * `maxExpireSweep`. Returns the total number of items expired across all readers. 186 | */ 187 | def discardExpired(): Int = discardExpired(true) 188 | 189 | /** 190 | * Do a sweep of each reader, discarding expired items at the head of the reader's queue. If 191 | * applyMaxExpireSweep is true, the reader's currently configured `maxExpireSweep` limit is 192 | * enforced, otherwise expiration continues until there are no more expired items at the head 193 | * of the queue. Returns the total number of items expired across all readers. 194 | */ 195 | def discardExpired(applyMaxExpireSweep: Boolean): Int = { 196 | readerMap.values.foldLeft(0) { _ + _.discardExpired(applyMaxExpireSweep) } 197 | } 198 | 199 | /** 200 | * Put an item into the queue. If the put fails, `None` is returned. On success, the item has 201 | * been added to every reader, and the returned future will trigger when the journal has been 202 | * written to disk. 203 | */ 204 | def put( 205 | data: ByteBuffer, addTime: Time, expireTime: Option[Time] = None, errorCount: Int = 0 206 | ): Option[Future[Unit]] = { 207 | if (closed) return None 208 | if (data.remaining > config.maxItemSize.inBytes) { 209 | log.debug("Rejecting put to %s: item too large (%s).", config.name, data.remaining.bytes.toHuman) 210 | return None 211 | } 212 | // if any reader is rejecting puts, the put is rejected. 213 | if (readerMap.values.exists { r => !r.canPut }) { 214 | log.debug("Rejecting put to %s: reader is full.", config.name) 215 | return None 216 | } 217 | 218 | val (_, syncFuture) = journal.put(data, addTime, expireTime, errorCount) 219 | 220 | readerMap.values.foreach { _.postPut() } 221 | 222 | Some(syncFuture) 223 | } 224 | 225 | /** 226 | * Put an item into the queue. If the put fails, `None` is returned. On success, the item has 227 | * been added to every reader, and the returned future will trigger when the journal has been 228 | * written to disk. 229 | * 230 | * This method really just calls the other `put` method with the parts of the queue item, 231 | * ignoring the given item id. The written item will have a new id. 232 | */ 233 | def put(item: QueueItem): Option[Future[Unit]] = { 234 | put(item.data, item.addTime, item.expireTime, item.errorCount) 235 | } 236 | 237 | /** 238 | * Generate a debugging string (python-style) for this queue, with key stats. 239 | */ 240 | def toDebug: String = { 241 | "".format( 242 | config.name, items, bytes, journalBytes, readerMap.values.map { _.toDebug }.mkString(", "), 243 | (if (closed) " closed" else "")) 244 | } 245 | 246 | /** 247 | * Create a wrapper object for this queue that implements the `BlockingQueue` trait. Not all 248 | * operations are supported: specifically, `putHead` and `pollIf` throw an exception if called. 249 | */ 250 | def toBlockingQueue[A <: AnyRef](implicit codec: Codec[A]): BlockingQueue[A] = 251 | new JournaledBlockingQueue(this, codec) 252 | 253 | /** 254 | * Create a wrapper object for this queue that implements the `TransactionalBlockingQueue` trait. 255 | */ 256 | def toTransactionalBlockingQueue[A <: AnyRef](implicit codec: Codec[A]): TransactionalBlockingQueue[A] = 257 | new TransactionalJournaledBlockingQueue(this, codec) 258 | 259 | /** 260 | * A reader for this queue, which has its own head pointer and list of open reads. 261 | */ 262 | class Reader(val name: String, val readerConfig: JournaledQueueReaderConfig) { 263 | val journalReader = journal.reader(name) 264 | 265 | private val openReads = new ConcurrentHashMap[Long, QueueItem]() 266 | 267 | /** 268 | * The current value of `itemCount`. 269 | */ 270 | def items = journalReader.itemCount.get 271 | 272 | /** 273 | * The current value of `byteCount`. 274 | */ 275 | def bytes = journalReader.byteCount.get 276 | 277 | /** 278 | * When was this reader created? 279 | */ 280 | val createTime = Time.now 281 | 282 | /** 283 | * Number of open (uncommitted) reads. 284 | */ 285 | def openItems = openReads.values.size 286 | 287 | /** 288 | * Byte count of open (uncommitted) reads. 289 | */ 290 | def openBytes = openReads.values.asScala.foldLeft(0L) { _ + _.dataSize } 291 | 292 | /** 293 | * Total number of items ever added to this queue. 294 | */ 295 | def putCount = journalReader.putCount 296 | 297 | /** 298 | * Total number of bytes ever added to this queue. 299 | */ 300 | def putBytes = journalReader.putBytes 301 | 302 | /** 303 | * Total number of items ever successfully fetched from this queue. 304 | */ 305 | val getHitCount = new AtomicLong(0) 306 | 307 | /** 308 | * Total number of times a fetch from this queue failed because no item was available. 309 | */ 310 | val getMissCount = new AtomicLong(0) 311 | 312 | /** 313 | * Total number of items ever expired from this queue. 314 | */ 315 | val expiredCount = new AtomicLong(0) 316 | 317 | /** 318 | * Total number of items ever discarded from this queue. Discards occur when the queue 319 | * reaches its configured maximum size or maximum number of items. 320 | */ 321 | val discardedCount = new AtomicLong(0) 322 | 323 | /** 324 | * Total number of reads opened (transactions) on this queue. 325 | */ 326 | val openedItemCount = new AtomicLong(0) 327 | 328 | /** 329 | * Total number of committed reads (transactions) on this queue. 330 | */ 331 | val committedItemCount = new AtomicLong(0) 332 | 333 | /** 334 | * Total number of canceled reads (transactions) on this queue. 335 | */ 336 | val canceledItemCount = new AtomicLong(0) 337 | 338 | /** 339 | * Number of consumers waiting for an item to arrive. 340 | */ 341 | def waiterCount: Int = waiters.size 342 | 343 | /** 344 | * Number of times this queue has been flushed. 345 | */ 346 | val flushCount = new AtomicLong(0) 347 | 348 | /** 349 | * FQDN for this reader, which is usually of the form "queue_name+reader_name", but will just 350 | * be "queue_name" for the default reader. 351 | */ 352 | def fullname: String = { 353 | if (name == "") config.name else (config.name + "+" + name) 354 | } 355 | 356 | def writer: JournaledQueue = JournaledQueue.this 357 | 358 | val waiters = new DeadlineWaitQueue(timer) 359 | 360 | /* 361 | * in order to reload the contents of a queue at startup, we need to: 362 | * - read the last saved state (head id, done ids) 363 | * - start at the head, and read in items (ignoring ones already done) until we hit the 364 | * current item and then: 365 | * - count the items left in the current journal file 366 | * - add in the summarized counts from the remaining journal files, if any. 367 | */ 368 | private[libkestrel] def loadFromJournal(j: Journal#Reader) { 369 | log.info("Restoring state of %s+%s", config.name, name) 370 | j.open() 371 | } 372 | 373 | /** 374 | * Age of the oldest item in this queue or 0 if the queue is empty. 375 | */ 376 | def age: Duration = 377 | journalReader.peekOldest() match { 378 | case Some(item) => 379 | Time.now - item.addTime 380 | case None => 0.nanoseconds 381 | } 382 | 383 | def catchUp() { 384 | journalReader.catchUp() 385 | } 386 | 387 | def checkpoint() { 388 | journalReader.checkpoint() 389 | } 390 | 391 | def canPut: Boolean = { 392 | readerConfig.fullPolicy == ConcurrentBlockingQueue.FullPolicy.DropOldest || 393 | (items < readerConfig.maxItems && bytes < readerConfig.maxSize.inBytes) 394 | } 395 | 396 | @tailrec 397 | private[libkestrel] final def dropOldest() { 398 | if (readerConfig.fullPolicy == ConcurrentBlockingQueue.FullPolicy.DropOldest && 399 | (items > readerConfig.maxItems || bytes > readerConfig.maxSize.inBytes)) { 400 | journalReader.next() match { 401 | case None => () 402 | case Some(item) => 403 | commitItem(item) 404 | discardedCount.getAndIncrement() 405 | dropOldest() 406 | } 407 | } 408 | } 409 | 410 | private [libkestrel] def postPut() { 411 | dropOldest() 412 | discardExpiredWithoutLimit() 413 | waiters.trigger() 414 | } 415 | 416 | private[this] def hasExpired(startTime: Time, expireTime: Option[Time], now: Time): Boolean = { 417 | val adjusted = if (readerConfig.maxAge.isDefined) { 418 | val maxExpiry = startTime + readerConfig.maxAge.get 419 | if (expireTime.isDefined) Some(expireTime.get min maxExpiry) else Some(maxExpiry) 420 | } else { 421 | expireTime 422 | } 423 | adjusted.isDefined && adjusted.get <= now 424 | } 425 | 426 | def discardExpiredWithoutLimit() = discardExpired(false) 427 | 428 | def discardExpired(applyMaxExpireSweep: Boolean = true): Int = { 429 | val max = if (applyMaxExpireSweep) readerConfig.maxExpireSweep else Int.MaxValue 430 | discardExpired(max) 431 | } 432 | 433 | // check the queue and discard anything that's expired. 434 | private[this] def discardExpired(max: Int): Int = { 435 | var numExpired = 0 436 | var remainingAttempts = max 437 | while(remainingAttempts > 0) { 438 | journalReader.nextIf { item => hasExpired(item.addTime, item.expireTime, Time.now) } match { 439 | case Some(item) => 440 | readerConfig.processExpiredItem(item) 441 | expiredCount.getAndIncrement() 442 | commitItem(item) 443 | remainingAttempts -= 1 444 | numExpired += 1 445 | case None => 446 | remainingAttempts = 0 447 | } 448 | } 449 | 450 | numExpired 451 | } 452 | 453 | private[this] def waitNext(deadline: Deadline, peeking: Boolean): Future[Option[QueueItem]] = { 454 | val startTime = Time.now 455 | val promise = new Promise[Option[QueueItem]] 456 | waitNext(startTime, deadline, promise, peeking) 457 | promise 458 | } 459 | 460 | private[this] def waitNext(startTime: Time, 461 | deadline: Deadline, 462 | promise: Promise[Option[QueueItem]], 463 | peeking: Boolean) { 464 | val item = if (peeking) journalReader.peekOldest() else journalReader.next() 465 | if (item.isDefined || closed) { 466 | promise.setValue(item) 467 | } else { 468 | // checking future.isCancelled is a race, we assume that the caller will either commit 469 | // or unget the item if we miss the cancellation 470 | def onTrigger() = { 471 | if (promise.isCancelled) { 472 | promise.setValue(None) 473 | waiters.trigger() 474 | } else { 475 | // if we get woken up, try again with the same deadline. 476 | waitNext(startTime, deadline, promise, peeking) 477 | } 478 | } 479 | def onTimeout() { 480 | promise.setValue(None) 481 | } 482 | val w = waiters.add(deadline, onTrigger, onTimeout) 483 | promise.onCancellation { waiters.remove(w) } 484 | } 485 | } 486 | 487 | /** 488 | * Remove and return an item from the queue, if there is one. 489 | * If no deadline is given, an item is only returned if one is immediately available. 490 | */ 491 | def get(deadline: Option[Deadline], peeking: Boolean = false): Future[Option[QueueItem]] = { 492 | if (closed) return Future.value(None) 493 | discardExpiredWithoutLimit() 494 | val startTime = Time.now 495 | 496 | val future = deadline match { 497 | case Some(d) => waitNext(d, peeking) 498 | case None if !peeking => Future.value(journalReader.next()) 499 | case None => Future.value(journalReader.peekOldest()) 500 | } 501 | future.flatMap { optItem => 502 | optItem match { 503 | case None => { 504 | readerConfig.timeoutLatency(this, Time.now - startTime) 505 | getMissCount.getAndIncrement() 506 | Future.value(None) 507 | } 508 | case s @ Some(item) => { 509 | if (hasExpired(item.addTime, item.expireTime, Time.now)) { 510 | // try again. 511 | get(deadline, peeking) 512 | } else { 513 | readerConfig.deliveryLatency(this, Time.now - item.addTime) 514 | getHitCount.getAndIncrement() 515 | if (!peeking) { 516 | openedItemCount.incrementAndGet 517 | openReads.put(item.id, item) 518 | item.data.mark() 519 | } 520 | Future.value(s) 521 | } 522 | } 523 | } 524 | } 525 | } 526 | 527 | /** 528 | * Commit an item that was previously fetched via `get`. 529 | * This is required to actually remove an item from the queue. 530 | */ 531 | def commit(id: Long) { 532 | if (closed) return 533 | val item = openReads.remove(id) 534 | if (item eq null) { 535 | log.error("Tried to commit unknown item %d on %s+%s", id, config.name, name) 536 | return 537 | } 538 | 539 | commitItem(item) 540 | } 541 | 542 | private[this] def commitItem(item: QueueItem) { 543 | journalReader.commit(item) 544 | committedItemCount.getAndIncrement() 545 | } 546 | 547 | /** 548 | * Return an uncommited "get" to the head of the queue and give it to someone else. 549 | */ 550 | def unget(id: Long) { 551 | if (closed) return 552 | val item = openReads.remove(id) 553 | if (item eq null) { 554 | log.error("Tried to uncommit unknown item %d on %s+%s", id, config.name, name) 555 | return 556 | } 557 | canceledItemCount.incrementAndGet 558 | item.data.reset() 559 | val newItem = item.copy(errorCount = item.errorCount + 1) 560 | if (readerConfig.errorHandler(newItem)) { 561 | commitItem(newItem) 562 | } else { 563 | journalReader.unget(newItem) 564 | waiters.trigger() 565 | } 566 | } 567 | 568 | /** 569 | * Peek at the head item in the queue, if there is one. 570 | */ 571 | def peek(deadline: Option[Deadline]): Future[Option[QueueItem]] = { 572 | get(deadline, true) 573 | } 574 | 575 | /** 576 | * Drain all items from this reader. 577 | */ 578 | def flush() { 579 | journalReader.flush() 580 | flushCount.getAndIncrement() 581 | } 582 | 583 | def close() { 584 | journalReader.checkpoint() 585 | journalReader.close() 586 | } 587 | 588 | def evictWaiters() { 589 | waiters.evictAll() 590 | } 591 | 592 | /** 593 | * Check if this Queue is eligible for expiration by way of it being empty 594 | * and its age being greater than or equal to maxQueueAge 595 | */ 596 | def isReadyForExpiration: Boolean = { 597 | readerConfig.maxQueueAge.map { age => 598 | Time.now > createTime + age && journalReader.isCaughtUp 599 | }.getOrElse(false) 600 | } 601 | 602 | def toDebug: String = { 603 | "".format( 604 | name, items, bytes, age, openReads.keys.asScala.toList.sorted, putCount.get, discardedCount.get, 605 | expiredCount.get) 606 | } 607 | } 608 | } 609 | -------------------------------------------------------------------------------- /src/main/scala/com/twitter/libkestrel/MemoryMappedFile.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Twitter, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | * not use this file except in compliance with the License. You may obtain 6 | * a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twitter.libkestrel 18 | 19 | import com.twitter.conversions.storage._ 20 | import com.twitter.util.StorageUnit 21 | import java.io.{File, IOException, RandomAccessFile} 22 | import java.nio.{ByteBuffer, MappedByteBuffer} 23 | import java.nio.channels.FileChannel 24 | import java.util.concurrent.atomic.AtomicInteger 25 | import scala.collection.mutable.HashMap 26 | import scala.ref.WeakReference 27 | 28 | object MemoryMappedFile { 29 | private val mappedFiles = new HashMap[String, WeakReference[WritableMemoryMappedFile]] 30 | 31 | def map(file: File, size: StorageUnit, truncate: Boolean = false): MemoryMappedFile = { 32 | getOrCreateMappedByteBuffer(file, size, truncate = truncate) 33 | } 34 | 35 | def readOnlyMap(file: File): MemoryMappedFile = { 36 | getOrCreateMappedByteBuffer(file, file.length.bytes, truncate = false, readOnly = true) 37 | } 38 | 39 | // for testing 40 | def openFiles() = { 41 | mappedFiles.synchronized { 42 | mappedFiles.keys.toList 43 | } 44 | } 45 | 46 | // for testing 47 | def reset() = { 48 | mappedFiles.synchronized { 49 | mappedFiles.clear() 50 | } 51 | } 52 | 53 | private[this] def getOrCreateMappedByteBuffer[A](file: File, 54 | size: StorageUnit, 55 | truncate: Boolean = false, 56 | readOnly: Boolean = false): MemoryMappedFile = { 57 | val canonicalFile = file.getCanonicalFile 58 | val canonicalPath = canonicalFile.getPath 59 | mappedFiles.synchronized { 60 | val weakRef = mappedFiles get(canonicalPath) flatMap { _.get } 61 | weakRef match { 62 | case Some(mappedFile) => 63 | if (readOnly) { 64 | new ReadOnlyMemoryMappedFileView(mappedFile) 65 | } else { 66 | throw new IOException("multiple writers on " + file) 67 | } 68 | case None => 69 | val mappedFile = new WritableMemoryMappedFile(canonicalFile, size, truncate) 70 | mappedFiles(canonicalPath) = new WeakReference(mappedFile) 71 | if (readOnly) { 72 | val readOnlyMapping = new ReadOnlyMemoryMappedFileView(mappedFile) 73 | mappedFile.close() 74 | readOnlyMapping 75 | } else { 76 | mappedFile 77 | } 78 | } 79 | } 80 | } 81 | } 82 | 83 | trait MemoryMappedFile { 84 | def file: File 85 | def size: StorageUnit 86 | 87 | val MMAP_LIMIT = 2.gigabytes 88 | 89 | protected[this] var _buffer: ByteBuffer = null 90 | 91 | def buffer() = _buffer.slice() 92 | 93 | def force(): Unit 94 | 95 | def close(): Unit 96 | 97 | protected[this] def mappedFiles = MemoryMappedFile.mappedFiles 98 | 99 | protected def open(mode: FileChannel.MapMode, truncate: Boolean): MappedByteBuffer = { 100 | val modeString = mode match { 101 | case FileChannel.MapMode.READ_WRITE => "rw" 102 | case FileChannel.MapMode.READ_ONLY if truncate => throw new IOException("cannot truncate read-only mapping") 103 | case FileChannel.MapMode.READ_ONLY => "r" 104 | case _ => throw new IllegalArgumentException("unknown map mode: " + mode) 105 | } 106 | 107 | if (size > MMAP_LIMIT) throw new IOException("exceeded %s mmap limit".format(MMAP_LIMIT.toHuman)) 108 | 109 | val channel = new RandomAccessFile(file, modeString).getChannel 110 | try { 111 | if (truncate) channel.truncate(0) 112 | channel.map(mode, 0, size.inBytes) 113 | } finally { 114 | channel.close() 115 | } 116 | } 117 | 118 | protected override def finalize() { 119 | if (_buffer ne null) { 120 | close() 121 | } 122 | } 123 | } 124 | 125 | class WritableMemoryMappedFile(val file: File, val size: StorageUnit, truncate: Boolean) extends MemoryMappedFile { 126 | _buffer = open(FileChannel.MapMode.READ_WRITE, truncate) 127 | 128 | private[this] val refs = new AtomicInteger(1) 129 | 130 | def force() { 131 | _buffer.asInstanceOf[MappedByteBuffer].force() 132 | } 133 | 134 | def close() { 135 | mappedFiles.synchronized { 136 | if (removeReference()) { 137 | mappedFiles.remove(file.getPath) 138 | } 139 | } 140 | } 141 | 142 | def addReference() { 143 | refs.incrementAndGet 144 | } 145 | 146 | def removeReference(): Boolean = { 147 | if (_buffer eq null) { 148 | throw new IOException("already closed") 149 | } 150 | 151 | if (refs.decrementAndGet() == 0) { 152 | _buffer = null 153 | true 154 | } else { 155 | false 156 | } 157 | } 158 | } 159 | 160 | trait MemoryMappedFileView extends MemoryMappedFile { 161 | protected[this] def underlyingMap: MemoryMappedFile 162 | 163 | def file = underlyingMap.file 164 | def size = underlyingMap.size 165 | 166 | def close() { 167 | if (_buffer eq null) throw new IOException("already closed") 168 | 169 | _buffer = null 170 | underlyingMap.close() 171 | } 172 | } 173 | 174 | class WritableMemoryMappedFileView(val underlyingMap: WritableMemoryMappedFile) extends MemoryMappedFileView { 175 | _buffer = underlyingMap.buffer() 176 | underlyingMap.addReference() 177 | 178 | def force() { underlyingMap.force() } 179 | } 180 | 181 | class ReadOnlyMemoryMappedFileView(val underlyingMap: WritableMemoryMappedFile) extends MemoryMappedFileView { 182 | _buffer = underlyingMap.buffer().asReadOnlyBuffer() 183 | underlyingMap.addReference() 184 | 185 | def force() { 186 | throw new UnsupportedOperationException("read-only memory mapped file") 187 | } 188 | } 189 | -------------------------------------------------------------------------------- /src/main/scala/com/twitter/libkestrel/PeriodicSyncFile.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.libkestrel 2 | 3 | import com.twitter.conversions.storage._ 4 | import com.twitter.conversions.time._ 5 | import com.twitter.util._ 6 | import java.io.{IOException, File, FileOutputStream, RandomAccessFile} 7 | import java.nio.channels.FileChannel 8 | import java.nio.ByteBuffer 9 | import java.util.concurrent.{ConcurrentLinkedQueue, ScheduledExecutorService, ScheduledFuture, TimeUnit} 10 | 11 | abstract class PeriodicSyncTask(val scheduler: ScheduledExecutorService, initialDelay: Duration, period: Duration) 12 | extends Runnable { 13 | @volatile private[this] var scheduledFsync: Option[ScheduledFuture[_]] = None 14 | 15 | def start() { 16 | synchronized { 17 | if (scheduledFsync.isEmpty && period > 0.seconds) { 18 | val handle = scheduler.scheduleWithFixedDelay(this, initialDelay.inMilliseconds, period.inMilliseconds, 19 | TimeUnit.MILLISECONDS) 20 | scheduledFsync = Some(handle) 21 | } 22 | } 23 | } 24 | 25 | def stop() { 26 | synchronized { _stop() } 27 | } 28 | 29 | def stopIf(f: => Boolean) { 30 | synchronized { 31 | if (f) _stop() 32 | } 33 | } 34 | 35 | private[this] def _stop() { 36 | scheduledFsync.foreach { _.cancel(false) } 37 | scheduledFsync = None 38 | } 39 | } 40 | 41 | object PeriodicSyncFile { 42 | def create(file: File, scheduler: ScheduledExecutorService, period: Duration, maxFileSize: StorageUnit) = { 43 | new PeriodicSyncFile(file, scheduler, period, maxFileSize, true) 44 | } 45 | 46 | def append(file: File, scheduler: ScheduledExecutorService, period: Duration, maxFileSize: StorageUnit) = { 47 | new PeriodicSyncFile(file, scheduler, period, maxFileSize, false) 48 | } 49 | 50 | // override me to track fsync delay metrics 51 | var addTiming: Duration => Unit = { _ => } 52 | } 53 | 54 | class MMappedSyncFileWriter(file: File, size: StorageUnit, truncate: Boolean) { 55 | private[this] val memMappedFile = MemoryMappedFile.map(file, size, truncate) 56 | var writer = memMappedFile.buffer() 57 | 58 | def force() { memMappedFile.force() } 59 | 60 | def position = writer.position.toLong 61 | 62 | // overflow -> IllegalArgumentException 63 | def position(p: Long) { writer.position(p.toInt) } 64 | 65 | def write(buffer: ByteBuffer) { 66 | // Write the first byte last so that readers using the same memory mapped 67 | // file will not see a non-zero first byte until the remainder of the 68 | // buffer has been written. 69 | val startPos = writer.position 70 | writer.put(0.toByte) 71 | val startByte = buffer.get 72 | writer.put(buffer) 73 | writer.put(startPos, startByte) 74 | } 75 | 76 | def truncate() { 77 | val f = new RandomAccessFile(file, "rw") 78 | f.setLength(writer.position) 79 | f.close() 80 | } 81 | 82 | def close() { 83 | writer = null 84 | memMappedFile.close() 85 | } 86 | } 87 | 88 | /** 89 | * Open a file for writing, and fsync it on a schedule. The period may be 0 to force an fsync 90 | * after every write, or `Duration.MaxValue` to never fsync. 91 | */ 92 | class PeriodicSyncFile(file: File, scheduler: ScheduledExecutorService, period: Duration, maxFileSize: StorageUnit, truncate: Boolean) 93 | extends WritableFile { 94 | // pre-completed future for writers who are behaving synchronously. 95 | private final val DONE = Future(()) 96 | 97 | case class TimestampedPromise(val promise: Promise[Unit], val time: Time) 98 | 99 | val writer = new MMappedSyncFileWriter(file, maxFileSize, truncate) 100 | 101 | val promises = new ConcurrentLinkedQueue[TimestampedPromise]() 102 | val periodicSyncTask = new PeriodicSyncTask(scheduler, period, period) { 103 | override def run() { 104 | if (!closed && !promises.isEmpty) fsync() 105 | } 106 | } 107 | 108 | @volatile var closed = false 109 | 110 | private def fsync() { 111 | // race: we could underestimate the number of completed writes. that's okay. 112 | val completed = promises.size 113 | val fsyncStart = Time.now 114 | try { 115 | writer.force() 116 | } catch { 117 | case e: IOException => { 118 | for (i <- 0 until completed) { 119 | promises.poll().promise.setException(e) 120 | } 121 | return 122 | } 123 | } 124 | 125 | for (i <- 0 until completed) { 126 | val timestampedPromise = promises.poll() 127 | timestampedPromise.promise.setValue(()) 128 | val delaySinceWrite = fsyncStart - timestampedPromise.time 129 | val durationBehind = if (delaySinceWrite > period) delaySinceWrite - period else 0.seconds 130 | PeriodicSyncFile.addTiming(durationBehind) 131 | } 132 | 133 | periodicSyncTask.stopIf { promises.isEmpty } 134 | } 135 | 136 | def write(buffer: ByteBuffer): Future[Unit] = { 137 | writer.write(buffer) 138 | if (period == 0.seconds) { 139 | try { 140 | writer.force() 141 | DONE 142 | } catch { 143 | case e: IOException => 144 | Future.exception(e) 145 | } 146 | } else if (period == Duration.MaxValue) { 147 | // not fsync'ing. 148 | DONE 149 | } else { 150 | val promise = new Promise[Unit]() 151 | promises.add(TimestampedPromise(promise, Time.now)) 152 | periodicSyncTask.start() 153 | promise 154 | } 155 | } 156 | 157 | def flush() { 158 | fsync() 159 | } 160 | 161 | /** 162 | * No locking is done here. Be sure you aren't doing concurrent writes or they may be lost 163 | * forever, and nobody will cry. 164 | */ 165 | def close() { 166 | closed = true 167 | periodicSyncTask.stop() 168 | fsync() 169 | writer.truncate() 170 | writer.close() 171 | } 172 | 173 | def position: Long = writer.position 174 | 175 | def position_=(p: Long) { 176 | writer.position(p) 177 | } 178 | } 179 | -------------------------------------------------------------------------------- /src/main/scala/com/twitter/libkestrel/QueueItem.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2011 Twitter, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | * not use this file except in compliance with the License. You may obtain 6 | * a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twitter.libkestrel 18 | 19 | import com.twitter.util.Time 20 | import java.nio.ByteBuffer 21 | 22 | case class QueueItem( 23 | id: Long, 24 | addTime: Time, 25 | expireTime: Option[Time], 26 | data: ByteBuffer, 27 | errorCount: Int = 0 28 | ) { 29 | val dataSize = data.remaining 30 | 31 | override def equals(other: Any) = { 32 | other match { 33 | case x: QueueItem => { 34 | (x eq this) || 35 | (x.id == id && x.addTime == addTime && x.expireTime == expireTime && 36 | x.data.duplicate.rewind == data.duplicate.rewind) 37 | } 38 | case _ => false 39 | } 40 | } 41 | 42 | override def toString = { 43 | val limit = data.limit min (data.position + 64) 44 | val hex = (data.position until limit).map { i => "%x".format(data.get(i)) }.mkString(" ") 45 | "QueueItem(id=%d, addTime=%s, expireTime=%s, data=%s, errors=%s)".format( 46 | id, addTime.inNanoseconds, expireTime.map { _.inNanoseconds }, hex, errorCount 47 | ) 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/main/scala/com/twitter/libkestrel/ReadWriteLock.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2011 Twitter, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | * not use this file except in compliance with the License. You may obtain 6 | * a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twitter.libkestrel 18 | 19 | import java.util.concurrent.locks.ReentrantReadWriteLock 20 | 21 | trait ReadWriteLock { 22 | private[this] val lock = new ReentrantReadWriteLock(true) 23 | 24 | def withWriteLock[A](f: => A) = { 25 | val writeLock = lock.writeLock 26 | writeLock.lock() 27 | try { 28 | f 29 | } finally { 30 | writeLock.unlock() 31 | } 32 | } 33 | 34 | def withDowngradeableWriteLock[A](f: (Function0[Unit]) => A) = { 35 | val writeLock = lock.writeLock 36 | val readLock = lock.readLock 37 | val downgrade = () => { 38 | readLock.lock() 39 | writeLock.unlock() 40 | } 41 | 42 | writeLock.lock() 43 | try { 44 | f(downgrade) 45 | } finally { 46 | if (writeLock.isHeldByCurrentThread) { 47 | // did not downgrade 48 | writeLock.unlock() 49 | } else { 50 | readLock.unlock() 51 | } 52 | } 53 | } 54 | 55 | def withReadLock[A](f: => A) = { 56 | val readLock = lock.readLock 57 | readLock.lock() 58 | try { 59 | f 60 | } finally { 61 | readLock.unlock() 62 | } 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/main/scala/com/twitter/libkestrel/Record.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Twitter, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | * not use this file except in compliance with the License. You may obtain 6 | * a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twitter.libkestrel 18 | 19 | import com.twitter.conversions.storage._ 20 | 21 | sealed trait Record 22 | object Record { 23 | // throw an error if any queue item is larger than this: 24 | val LARGEST_DATA = 16.megabytes 25 | 26 | private[libkestrel] val READ_HEAD = 0 27 | private[libkestrel] val PUT = 8 28 | private[libkestrel] val READ_DONE = 9 29 | 30 | case class Put(item: QueueItem) extends Record 31 | case class ReadHead(id: Long) extends Record 32 | case class ReadDone(ids: Seq[Long]) extends Record 33 | case class Unknown(command: Int) extends Record 34 | } 35 | -------------------------------------------------------------------------------- /src/main/scala/com/twitter/libkestrel/RecordReader.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Twitter, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | * not use this file except in compliance with the License. You may obtain 6 | * a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twitter.libkestrel 18 | 19 | import com.twitter.util.Time 20 | import java.io.File 21 | import java.nio.{ByteBuffer, ByteOrder} 22 | 23 | abstract class RecordReader extends Iterable[Record] { 24 | import Record._ 25 | 26 | def file: File 27 | def reader: ByteBuffer 28 | 29 | def readMagic(): Int = { 30 | if (reader.remaining < 4) throw new CorruptedJournalException(0, file, "No header in journal file: " + file) 31 | reader.order(ByteOrder.BIG_ENDIAN) 32 | val magic = reader.getInt 33 | reader.order(ByteOrder.LITTLE_ENDIAN) 34 | magic 35 | } 36 | 37 | def mark: Int = reader.position 38 | 39 | def rewind(position: Int) { 40 | reader.position(position) 41 | } 42 | 43 | def readNext(): Option[Record] = { 44 | val lastPosition = reader.position 45 | 46 | if (reader.remaining < 1) { 47 | return None 48 | } 49 | val commandAndHeaderSize = reader.get 50 | if (commandAndHeaderSize == 0) { 51 | reader.position(lastPosition) 52 | return None 53 | } 54 | 55 | val command = (commandAndHeaderSize >> 4) & 0xf 56 | val headerSize = (commandAndHeaderSize & 0xf) 57 | 58 | if (reader.remaining < headerSize * 4) { 59 | throw new CorruptedJournalException(lastPosition, file, "truncated header: %d left".format(reader.remaining)) 60 | } 61 | 62 | var dataBuffer: ByteBuffer = null 63 | var dataSize = 0 64 | if (command >= 8) { 65 | dataSize = reader.getInt 66 | if (dataSize > LARGEST_DATA.inBytes) { 67 | throw new CorruptedJournalException(lastPosition, file, "item too large") 68 | } 69 | 70 | val remainingHeaderSize = (headerSize - 1) * 4 71 | if (reader.remaining < dataSize + remainingHeaderSize) { 72 | throw new CorruptedJournalException(lastPosition, file, "truncated entry") 73 | } 74 | } 75 | 76 | Some(command match { 77 | case READ_HEAD => { 78 | if (reader.remaining < 8) { 79 | throw new CorruptedJournalException(lastPosition, file, "corrupted READ_HEAD") 80 | } 81 | val id = reader.getLong() 82 | Record.ReadHead(id) 83 | } 84 | case PUT => { 85 | if (reader.remaining < 20) { 86 | throw new CorruptedJournalException(lastPosition, file, "corrupted PUT") 87 | } 88 | val errorCount = reader.getInt() 89 | val id = reader.getLong() 90 | val addTime = reader.getLong() 91 | val expireTime = if (headerSize > 6) Some(reader.getLong()) else None 92 | 93 | // record current position; modify limit to the end of data; make slice; restore reader 94 | reader.limit(reader.position + dataSize) 95 | dataBuffer = reader.slice() 96 | reader.limit(reader.capacity) 97 | 98 | // advance reader past data 99 | reader.position(reader.position + dataSize) 100 | Record.Put(QueueItem(id, Time.fromMilliseconds(addTime), expireTime.map { t => 101 | Time.fromMilliseconds(t) 102 | }, dataBuffer, errorCount)) 103 | } 104 | case READ_DONE => { 105 | if (dataSize % 8 != 0) { 106 | throw new CorruptedJournalException(lastPosition, file, "corrupted READ_DONE") 107 | } 108 | val ids = new Array[Long](dataSize / 8) 109 | for (i <- 0 until ids.size) { ids(i) = reader.getLong() } 110 | Record.ReadDone(ids) 111 | } 112 | case _ => { 113 | // this is okay. we can skip the ones we don't know. 114 | reader.position(lastPosition + headerSize * 4 + dataSize + 1) 115 | Record.Unknown(command) 116 | } 117 | }) 118 | } 119 | 120 | def iterator: Iterator[Record] = { 121 | def next(): Stream[Record] = { 122 | readNext() match { 123 | case Some(entry) => new Stream.Cons(entry, next()) 124 | case None => Stream.Empty 125 | } 126 | } 127 | next().iterator 128 | } 129 | 130 | def position = reader.position 131 | } 132 | 133 | -------------------------------------------------------------------------------- /src/main/scala/com/twitter/libkestrel/RecordWriter.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Twitter, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | * not use this file except in compliance with the License. You may obtain 6 | * a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twitter.libkestrel 18 | 19 | import com.twitter.util.Future 20 | import java.io.{File, IOException} 21 | import java.nio.{ByteBuffer, ByteOrder} 22 | 23 | abstract class RecordWriter { 24 | import Record._ 25 | 26 | def file: File 27 | 28 | protected def writer: WritableFile 29 | 30 | private[this] val BUFFER_SIZE = 128 31 | 32 | private[this] def newBuffer(size: Int) = { 33 | val byteBuffer = ByteBuffer.allocate(size) 34 | byteBuffer.order(ByteOrder.LITTLE_ENDIAN) 35 | byteBuffer 36 | } 37 | 38 | private[this] val localBuffer = new ThreadLocal[ByteBuffer] { 39 | override def initialValue: ByteBuffer = newBuffer(BUFFER_SIZE) 40 | } 41 | 42 | def buffer: ByteBuffer = buffer(BUFFER_SIZE) 43 | 44 | // thread-local for buffering journal items 45 | def buffer(size: Int): ByteBuffer = { 46 | val b = localBuffer.get() 47 | if (b.limit < size) { 48 | localBuffer.set(newBuffer(size)) 49 | buffer(size) 50 | } else { 51 | b.clear() 52 | b 53 | } 54 | } 55 | 56 | def writeMagic(header: Int) { 57 | writer.position = 0 58 | val b = buffer(4) 59 | b.order(ByteOrder.BIG_ENDIAN) 60 | b.putInt(header) 61 | b.flip() 62 | writer.write(b) 63 | } 64 | 65 | def put(item: QueueItem): Future[Unit] = { 66 | if (item.dataSize > LARGEST_DATA.inBytes) { 67 | throw new IOException("item too large") 68 | } 69 | val b = buffer(storageSizeOf(item)) 70 | val size = if (item.expireTime.isDefined) 8 else 6 71 | b.put((PUT << 4 | size).toByte) 72 | b.putInt(item.dataSize) 73 | b.putInt(item.errorCount) 74 | b.putLong(item.id) 75 | b.putLong(item.addTime.inMilliseconds) 76 | item.expireTime.foreach { t => b.putLong(t.inMilliseconds) } 77 | item.data.mark 78 | b.put(item.data) 79 | item.data.reset 80 | write(b) 81 | } 82 | 83 | def readHead(id: Long): Future[Unit] = { 84 | val b = buffer(9) 85 | b.put((READ_HEAD << 4 | 2).toByte) 86 | b.putLong(id) 87 | write(b) 88 | } 89 | 90 | def readDone(ids: Seq[Long]): Future[Unit] = { 91 | val b = buffer(8 * ids.size + 5) 92 | b.put((READ_DONE << 4 | 1).toByte) 93 | b.putInt(8 * ids.size) 94 | ids.foreach { b.putLong(_) } 95 | write(b) 96 | } 97 | 98 | private[this] def write(b: ByteBuffer) = { 99 | b.flip() 100 | writer.write(b) 101 | } 102 | 103 | def position = writer.position 104 | 105 | def close() { 106 | writer.flush() 107 | writer.close() 108 | } 109 | 110 | def storageSizeOf(item: QueueItem): Int = { 111 | (if (item.expireTime.isDefined) 8 else 6) * 4 + 1 + item.dataSize 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /src/main/scala/com/twitter/libkestrel/SimpleBlockingQueue.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.libkestrel 2 | 3 | import java.util.LinkedHashSet 4 | import scala.collection.mutable 5 | import scala.collection.JavaConverters._ 6 | import com.twitter.util.{Duration, Future, Promise, Time, TimeoutException, Timer, TimerTask} 7 | 8 | object SimpleBlockingQueue { 9 | def apply[A <: AnyRef](implicit timer: Timer) = { 10 | new SimpleBlockingQueue[A](Long.MaxValue, ConcurrentBlockingQueue.FullPolicy.RefusePuts, timer) 11 | } 12 | 13 | def apply[A <: AnyRef](maxItems: Long, fullPolicy: ConcurrentBlockingQueue.FullPolicy)(implicit timer: Timer) = { 14 | new SimpleBlockingQueue[A](maxItems, fullPolicy, timer) 15 | } 16 | } 17 | 18 | /** 19 | * Simple reproduction of the queue from kestrel 2.x. 20 | * 21 | * Puts and gets are done within synchronized blocks, and a DeadlineWaitQueue is used to arbitrate 22 | * timeouts and handoffs. 23 | */ 24 | final class SimpleBlockingQueue[A <: AnyRef]( 25 | maxItems: Long, 26 | fullPolicy: ConcurrentBlockingQueue.FullPolicy, 27 | timer: Timer 28 | ) extends BlockingQueue[A] { 29 | private var queue = new mutable.Queue[A] 30 | private val waiters = new DeadlineWaitQueue(timer) 31 | 32 | /** 33 | * Add a value to the end of the queue, transactionally. 34 | */ 35 | def put(item: A): Boolean = { 36 | synchronized { 37 | while (queue.size >= maxItems) { 38 | if (fullPolicy == ConcurrentBlockingQueue.FullPolicy.RefusePuts) return false 39 | get() 40 | } 41 | queue += item 42 | } 43 | waiters.trigger() 44 | true 45 | } 46 | 47 | def putHead(item: A) { 48 | synchronized { 49 | item +=: queue 50 | } 51 | waiters.trigger() 52 | } 53 | 54 | def size: Int = queue.size 55 | 56 | def get(): Future[Option[A]] = get(Forever) 57 | 58 | def get(deadline: Deadline): Future[Option[A]] = { 59 | val promise = new Promise[Option[A]] 60 | waitFor(promise, deadline) 61 | promise 62 | } 63 | 64 | private def waitFor(promise: Promise[Option[A]], deadline: Deadline) { 65 | val item = poll()() 66 | item match { 67 | case s @ Some(x) => promise.setValue(s) 68 | case None => { 69 | val w = waiters.add( 70 | deadline, 71 | { () => waitFor(promise, deadline) }, 72 | { () => promise.setValue(None) } 73 | ) 74 | promise.onCancellation { waiters.remove(w) } 75 | } 76 | } 77 | } 78 | 79 | def poll(): Future[Option[A]] = { 80 | synchronized { 81 | Future.value(if (queue.isEmpty) None else Some(queue.dequeue())) 82 | } 83 | } 84 | 85 | def pollIf(predicate: A => Boolean): Future[Option[A]] = { 86 | synchronized { 87 | Future.value(if (queue.isEmpty || !predicate(queue.head)) None else Some(queue.dequeue())) 88 | } 89 | } 90 | 91 | def flush() { 92 | synchronized { 93 | queue.clear() 94 | } 95 | } 96 | 97 | def toDebug: String = { 98 | synchronized { 99 | "".format(queue.size, waiters.size) 100 | } 101 | } 102 | 103 | def close() { 104 | queue.clear() 105 | waiters.triggerAll() 106 | } 107 | 108 | /** 109 | * Return the number of consumers waiting for an item. 110 | */ 111 | def waiterCount: Int = waiters.size 112 | 113 | def evictWaiters() { 114 | waiters.evictAll() 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /src/main/scala/com/twitter/libkestrel/SimpleFile.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.libkestrel 2 | 3 | import com.twitter.util._ 4 | import java.io.{File, FileOutputStream} 5 | import java.nio.ByteBuffer 6 | 7 | 8 | trait WritableFile { 9 | def write(buffer: ByteBuffer): Future[Unit] 10 | 11 | def flush(): Unit 12 | def close(): Unit 13 | 14 | def position: Long 15 | def position_=(p: Long): Unit 16 | } 17 | 18 | class SimpleFile(file: File) extends WritableFile { 19 | private val writer = new FileOutputStream(file, false).getChannel 20 | 21 | def position = writer.position 22 | def position_=(p: Long) { writer.position(p) } 23 | 24 | def write(buffer: ByteBuffer) = { 25 | do { 26 | writer.write(buffer) 27 | } while(buffer.position < buffer.limit) 28 | 29 | Future.Unit 30 | } 31 | 32 | def flush() { writer.force(false) } 33 | 34 | def close() { writer.close() } 35 | } 36 | -------------------------------------------------------------------------------- /src/main/scala/com/twitter/libkestrel/config/JournaledQueueConfig.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2011 Twitter, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | * not use this file except in compliance with the License. You may obtain 6 | * a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twitter.libkestrel 18 | package config 19 | 20 | import com.twitter.conversions.storage._ 21 | import com.twitter.conversions.time._ 22 | import com.twitter.util._ 23 | import java.io.File 24 | 25 | /** 26 | * Configuration for a queue reader. Each JournaledQueue has at least one reader. Fanout queues 27 | * have multiple readers. Readers hold an in-memory buffer of the enqueued items (up to a limit) 28 | * and enforce policy on maximum queue size and item expiration. 29 | * 30 | * @param maxItems Set a hard limit on the number of items this queue can hold. When the queue is 31 | * full, `fullPolicy` dictates the behavior when a client attempts to add another item. 32 | * @param maxSize Set a hard limit on the number of bytes (of data in queued items) this queue can 33 | * hold. When the queue is full, `fullPolicy` dictates the behavior when a client attempts 34 | * to add another item. 35 | * @param maxAge Expiration time for items on this queue. Any item that has been sitting on the 36 | * queue longer than this duration will be discarded. Clients may also attach an expiration time 37 | * when adding items to a queue, in which case the item expires at the earlier of the two 38 | * expiration times. 39 | * @param fullPolicy What to do if a client attempts to add items to a queue that's reached its 40 | * maxItems or maxSize. 41 | * @param processExpiredItem What to do with items that are expired from this queue. This can be 42 | * used to implement special processing for expired items, such as moving them to another queue 43 | * or writing them into a logfile. 44 | * @param errorHandler Any special action to take when an item is given to a client and the client 45 | * aborts it. The `errorCount` of the item will already be incremented. Return `false` if the 46 | * normal action (give the item to another client) should happen. Return `true` if libkestrel 47 | * should consider the matter taken care of, and not do anything more. 48 | * @param maxExpireSweep Maximum number of expired items to process at once. 49 | * @param maxQueueAge If the queue is empty and has existed at least this long, it will be deleted. 50 | * @param deliveryLatency Code to execute if you wish to track delivery latency (the time between 51 | * an item being added and a client being ready to receive it). Normally you would hook this up 52 | * to a stats collection library like ostrich. 53 | * @param timeoutLatency Code to execute if you wish to track timeout latency (the time a client 54 | * actually spent waiting for an item to arrive before it timed out). Normally you would hook 55 | * this up to a stats collection library like ostrich. 56 | */ 57 | case class JournaledQueueReaderConfig( 58 | maxItems: Int = Int.MaxValue, 59 | maxSize: StorageUnit = Long.MaxValue.bytes, 60 | maxAge: Option[Duration] = None, 61 | fullPolicy: ConcurrentBlockingQueue.FullPolicy = ConcurrentBlockingQueue.FullPolicy.RefusePuts, 62 | processExpiredItem: (QueueItem) => Unit = { _ => }, 63 | errorHandler: (QueueItem) => Boolean = { _ => false }, 64 | maxExpireSweep: Int = Int.MaxValue, 65 | maxQueueAge: Option[Duration] = None, 66 | deliveryLatency: (JournaledQueue#Reader, Duration) => Unit = { (_, _) => }, 67 | timeoutLatency: (JournaledQueue#Reader, Duration) => Unit = { (_, _) => } 68 | ) { 69 | override def toString() = { 70 | ("maxItems=%d maxSize=%s maxAge=%s fullPolicy=%s maxExpireSweep=%d " + 71 | "maxQueueAge=%s").format( 72 | maxItems, maxSize, maxAge, fullPolicy, maxExpireSweep, maxQueueAge) 73 | } 74 | } 75 | 76 | /** 77 | * Configuration for a journaled queue. All of the parameters have reasonable defaults, but can be 78 | * overridden. 79 | * 80 | * @param name Name of the queue being configured. 81 | * @param maxItemSize Set a hard limit on the number of bytes a single queued item can contain. A 82 | * put request for an item larger than this will be rejected. 83 | * @param journalSize Maximum size of an individual journal file before libkestrel moves to a new 84 | * file. In the (normal) state where a queue is usually empty, this is the amount of disk space 85 | * a queue should consume before moving to a new file and erasing the old one. 86 | * @param syncJournal How often to sync the journal file. To sync after every write, set this to 87 | * `0.milliseconds`. To never sync, set it to `Duration.MaxValue`. Syncing the journal will 88 | * reduce the maximum throughput of the server in exchange for a lower chance of losing data. 89 | * @param saveArchivedJournals Optionally move "retired" journal files to this folder. Normally, 90 | * once a journal file only refers to items that have all been removed, it's erased. 91 | * @param checkpointTimer How often to checkpoint the journal and readers associated with this queue. 92 | * Checkpointing stores each reader's position on disk and causes old journal files to be deleted 93 | * (provided that all extant readers are finished with them). 94 | * @param readerConfigs Configuration to use for readers of this queue. 95 | * @param defaultReaderConfig Configuration to use for readers of this queue when the reader name 96 | * isn't in readerConfigs. 97 | */ 98 | case class JournaledQueueConfig( 99 | name: String, 100 | maxItemSize: StorageUnit = Long.MaxValue.bytes, 101 | journalSize: StorageUnit = 16.megabytes, 102 | syncJournal: Duration = Duration.MaxValue, 103 | saveArchivedJournals: Option[File] = None, 104 | checkpointTimer: Duration = 1.second, 105 | 106 | readerConfigs: Map[String, JournaledQueueReaderConfig] = Map.empty, 107 | defaultReaderConfig: JournaledQueueReaderConfig = new JournaledQueueReaderConfig() 108 | ) { 109 | override def toString() = { 110 | ("name=%s maxItemSize=%s journalSize=%s syncJournal=%s " + 111 | "saveArchivedJournals=%s checkpointTimer=%s").format( 112 | name, maxItemSize, journalSize, syncJournal, saveArchivedJournals, 113 | checkpointTimer) 114 | } 115 | 116 | def readersToStrings(): Seq[String] = { 117 | List(": " + defaultReaderConfig.toString) ++ readerConfigs.map { case (k, v) => 118 | "%s: %s".format(k, v) 119 | } 120 | } 121 | } 122 | 123 | -------------------------------------------------------------------------------- /src/main/scala/com/twitter/libkestrel/tools/QueueDumper.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2011 Twitter, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | * not use this file except in compliance with the License. You may obtain 6 | * a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twitter.libkestrel 18 | package tools 19 | 20 | import java.io.{File, FileNotFoundException, IOException} 21 | import scala.collection.mutable 22 | import com.twitter.conversions.storage._ 23 | import com.twitter.conversions.time._ 24 | import com.twitter.util.{Duration, Time} 25 | 26 | sealed trait DumpMode 27 | case object DumpString extends DumpMode 28 | case object DumpRaw extends DumpMode 29 | case object DumpHex extends DumpMode 30 | case object DumpBase64 extends DumpMode 31 | 32 | class QueueDumper(filename: String, quiet: Boolean, dumpMode: Option[DumpMode], reader: Boolean) { 33 | var operations = 0L 34 | var bytes = 0L 35 | var firstId = 0L 36 | var lastId = 0L 37 | 38 | def verbose(s: String, args: Any*) { 39 | if (!quiet) { 40 | print(s.format(args: _*)) 41 | } 42 | } 43 | 44 | def apply() { 45 | try { 46 | val file: RecordReader = if (reader) { 47 | BookmarkFile.open(new File(filename)) 48 | } else { 49 | JournalFile.open(new File(filename)) 50 | } 51 | var lastDisplay = 0L 52 | 53 | val dumpRaw = dumpMode match { 54 | case Some(DumpRaw) => true 55 | case _ => false 56 | } 57 | var position = file.position 58 | file.foreach { record => 59 | operations += 1 60 | dumpItem(position, record) 61 | if (quiet && !dumpRaw && file.position - lastDisplay > 1024 * 1024) { 62 | print("\rReading journal: %-6s".format(file.position.bytes.toHuman)) 63 | Console.flush() 64 | lastDisplay = file.position 65 | } 66 | position = file.position 67 | } 68 | if (!dumpRaw) { 69 | print("\r" + (" " * 30) + "\r") 70 | } 71 | 72 | if (!dumpRaw) { 73 | println() 74 | println("Journal size: %d operations, %d bytes.".format(operations, file.position)) 75 | if (firstId > 0) println("Ids %d - %d.".format(firstId, lastId)) 76 | } 77 | } catch { 78 | case e: FileNotFoundException => 79 | Console.err.println("Can't open journal file: " + filename) 80 | case e: IOException => 81 | Console.err.println("Exception reading journal file: " + filename) 82 | e.printStackTrace(Console.err) 83 | } 84 | } 85 | 86 | def encodeData(data: Array[Byte]): String = { 87 | dumpMode match { 88 | case Some(DumpHex) => 89 | val builder = 90 | data.map { byte => 91 | "%02x".format(byte.toInt & 0xFF) 92 | }.foldLeft(new StringBuilder(data.length * 3)) { (b, s) => 93 | b.append(s).append(" ") 94 | b 95 | } 96 | builder.toString.trim 97 | case Some(DumpBase64) => new sun.misc.BASE64Encoder().encode(data) 98 | case _ => new String(data, "ISO-8859-1") // raw, string, none 99 | } 100 | } 101 | 102 | def dumpItem(position: Long, record: Record) { 103 | val now = Time.now 104 | verbose("%08x ", position & 0xffffffffL) 105 | record match { 106 | case Record.Put(item) => 107 | if (firstId == 0) firstId = item.id 108 | lastId = item.id 109 | if (!quiet) { 110 | verbose("PUT %-6d @ %s id=%d", item.dataSize, item.addTime, item.id) 111 | if (item.expireTime.isDefined) { 112 | if (item.expireTime.get - now < 0.milliseconds) { 113 | verbose(" expired") 114 | } else { 115 | verbose(" exp=%s", item.expireTime.get - now) 116 | } 117 | } 118 | if (item.errorCount > 0) verbose(" errors=%d", item.errorCount) 119 | verbose("\n") 120 | } 121 | val data = new Array[Byte](item.dataSize) 122 | item.data.get(data) 123 | dumpMode match { 124 | case Some(DumpRaw) => print(encodeData(data)) 125 | case None => () 126 | case _ => println(" " + encodeData(data)) 127 | } 128 | case Record.ReadHead(id) => 129 | verbose("HEAD %d\n", id) 130 | case Record.ReadDone(ids) => 131 | verbose("DONE %s\n", ids.sorted.mkString("(", ", ", ")")) 132 | case x => 133 | verbose(x.toString) 134 | } 135 | } 136 | } 137 | 138 | 139 | object QueueDumper { 140 | val filenames = new mutable.ListBuffer[String] 141 | var quiet = false 142 | var dump = false 143 | var dumpRaw = false 144 | var dumpHex = false 145 | var dumpBase64 = false 146 | var reader = false 147 | 148 | def usage() { 149 | println() 150 | println("usage: queuedumper ") 151 | println(" describe the contents of a kestrel journal file") 152 | println() 153 | println("options:") 154 | println(" -q quiet: don't describe every line, just the summary") 155 | println(" -d dump contents of added items") 156 | println(" -A dump only the raw contents of added items") 157 | println(" -x dump contents as hex") 158 | println(" -64 dump contents as base 64") 159 | println(" -R file is a reader pointer") 160 | println() 161 | } 162 | 163 | def parseArgs(args: List[String]) { 164 | args match { 165 | case Nil => 166 | case "--help" :: xs => 167 | usage() 168 | System.exit(0) 169 | case "-q" :: xs => 170 | quiet = true 171 | parseArgs(xs) 172 | case "-d" :: xs => 173 | dump = true 174 | parseArgs(xs) 175 | case "-A" :: xs => 176 | dumpRaw = true 177 | quiet = true 178 | parseArgs(xs) 179 | case "-x" :: xs => 180 | dumpHex = true 181 | parseArgs(xs) 182 | case "-64" :: xs => 183 | dumpBase64 = true 184 | parseArgs(xs) 185 | case "-R" :: xs => 186 | reader = true 187 | parseArgs(xs) 188 | case x :: xs => 189 | filenames += x 190 | parseArgs(xs) 191 | } 192 | } 193 | 194 | def main(args: Array[String]) { 195 | parseArgs(args.toList) 196 | if (filenames.size == 0) { 197 | usage() 198 | System.exit(0) 199 | } 200 | 201 | val dumpMode = 202 | if (dumpRaw) Some(DumpRaw) 203 | else if (dumpHex) Some(DumpHex) 204 | else if (dumpBase64) Some(DumpBase64) 205 | else if (dump) Some(DumpString) 206 | else None 207 | 208 | for (filename <- filenames) { 209 | if (!quiet) println("Queue: " + filename) 210 | new QueueDumper(filename, quiet, dumpMode, reader)() 211 | } 212 | } 213 | } 214 | -------------------------------------------------------------------------------- /src/scripts/qdumper: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | DIST_HOME="$(dirname $0)/.." 3 | java -server -classpath @DIST_CLASSPATH@ com.twitter.libkestrel.tools.QueueDumper "$@" 4 | -------------------------------------------------------------------------------- /src/scripts/qtest: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | DIST_HOME="$(dirname $(readlink $0 || echo $0))/.." 3 | TEST_JAR=$DIST_HOME/@DIST_NAME@-@VERSION@-test.jar 4 | java -server -classpath @TEST_CLASSPATH@:$TEST_JAR com.twitter.libkestrel.load.QTest "$@" 5 | 6 | -------------------------------------------------------------------------------- /src/test/scala/com/twitter/libkestrel/BookmarkFileSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Twitter, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | * not use this file except in compliance with the License. You may obtain 6 | * a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twitter.libkestrel 18 | 19 | import com.twitter.conversions.storage._ 20 | import com.twitter.io.Files 21 | import com.twitter.util._ 22 | import java.io._ 23 | import java.nio.ByteBuffer 24 | import org.scalatest.{AbstractSuite, FunSpec, Suite} 25 | import org.scalatest.matchers.{Matcher, MatchResult, ShouldMatchers} 26 | 27 | class BookmarkFileSpec extends FunSpec with ResourceCheckingSuite with ShouldMatchers with TempFolder with TestLogging { 28 | import FileHelper._ 29 | 30 | describe("BookmarkFile") { 31 | describe("readHead") { 32 | val readHeadData = "26 3c 26 3 2 ff 1 0 0 0 0 0 0" 33 | val readHead = Record.ReadHead(511) 34 | 35 | it("write") { 36 | val testFile = new File(testFolder, "a1") 37 | val j = BookmarkFile.create(testFile) 38 | j.readHead(511) 39 | j.close() 40 | assert(hex(readFile(testFile)) === readHeadData) 41 | } 42 | 43 | it("read") { 44 | val testFile = new File(testFolder, "a1") 45 | writeFile(testFile, unhex(readHeadData)) 46 | val j = BookmarkFile.open(testFile) 47 | assert(j.readNext() === Some(readHead)) 48 | } 49 | 50 | it("read corrupted") { 51 | val testFile = new File(testFolder, "a1") 52 | val data = unhex(readHeadData) 53 | (0 until data.size).foreach { size => 54 | writeFile(testFile, data.slice(0, size)) 55 | if (size != 4) { 56 | intercept[IOException] { 57 | BookmarkFile.open(testFile).readNext() 58 | } 59 | } 60 | } 61 | } 62 | } 63 | 64 | describe("readDone") { 65 | val readDoneData = "26 3c 26 3 91 18 0 0 0 a 0 0 0 0 0 0 0 14 0 0 0 0 0 0 0 1e 0 0 0 0 0 0 0" 66 | val readDone = Record.ReadDone(Array(10, 20, 30)) 67 | 68 | it("write") { 69 | val testFile = new File(testFolder, "a1") 70 | val j = BookmarkFile.create(testFile) 71 | j.readDone(List(10, 20, 30)) 72 | j.close() 73 | assert(hex(readFile(testFile)) === readDoneData) 74 | } 75 | 76 | it("read") { 77 | val testFile = new File(testFolder, "a1") 78 | writeFile(testFile, unhex(readDoneData)) 79 | val j = BookmarkFile.open(testFile) 80 | assert(j.readNext() === Some(readDone)) 81 | } 82 | 83 | it("read corrupted") { 84 | val testFile = new File(testFolder, "a1") 85 | val data = unhex(readDoneData) 86 | (0 until data.size).foreach { size => 87 | writeFile(testFile, data.slice(0, size)) 88 | if (size != 4) { 89 | intercept[IOException] { 90 | BookmarkFile.open(testFile).readNext() 91 | } 92 | } 93 | } 94 | } 95 | } 96 | 97 | it("whole read file") { 98 | val fileData = "26 3c 26 3 91 8 0 0 0 2 0 1 0 0 0 0 0 2 0 0 1 0 0 0 0 0" 99 | val testFile = new File(testFolder, "a1") 100 | writeFile(testFile, unhex(fileData)) 101 | 102 | val j = BookmarkFile.open(testFile) 103 | assert(j.readNext() === Some(Record.ReadDone(Array(65538)))) 104 | assert(j.readNext() === Some(Record.ReadHead(65536))) 105 | assert(j.readNext() === None) 106 | } 107 | 108 | it("be okay with commands it doesn't know") { 109 | val fileData = "26 3c 26 3 f3 4 0 0 0 1 1 1 1 2 2 2 2 ff ff ff ff 2 0 40 0 0 0 0 0 0" 110 | val testFile = new File(testFolder, "a1") 111 | writeFile(testFile, unhex(fileData)) 112 | 113 | val j = BookmarkFile.open(testFile) 114 | assert(j.readNext() === Some(Record.Unknown(15))) 115 | assert(j.readNext() === Some(Record.ReadHead(16384))) 116 | assert(j.readNext() === None) 117 | } 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /src/test/scala/com/twitter/libkestrel/ConcurrentBlockingQueueSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2011 Twitter, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | * not use this file except in compliance with the License. You may obtain 6 | * a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twitter.libkestrel 18 | 19 | import com.twitter.conversions.time._ 20 | import com.twitter.util._ 21 | import org.scalatest.{AbstractSuite, FunSpec, Suite} 22 | import org.scalatest.matchers.{Matcher, MatchResult, ShouldMatchers} 23 | import scala.collection.mutable 24 | 25 | class ConcurrentBlockingQueueSpec extends FunSpec with ResourceCheckingSuite with ShouldMatchers with TempFolder with TestLogging { 26 | implicit var timer: MockTimer = null 27 | 28 | trait QueueBuilder { 29 | def newQueue(): BlockingQueue[String] 30 | def newQueue(maxItems: Int, fullPolicy: ConcurrentBlockingQueue.FullPolicy): BlockingQueue[String] 31 | } 32 | 33 | def tests(builder: QueueBuilder) { 34 | import builder._ 35 | 36 | it("add and remove items") { 37 | val queue = newQueue() 38 | assert(queue.size === 0) 39 | assert(queue.put("first")) 40 | assert(queue.size === 1) 41 | assert(queue.put("second")) 42 | assert(queue.size === 2) 43 | assert(queue.get()() === Some("first")) 44 | assert(queue.size === 1) 45 | assert(queue.get()() === Some("second")) 46 | assert(queue.size === 0) 47 | } 48 | 49 | it("poll items") { 50 | val queue = newQueue() 51 | assert(queue.size === 0) 52 | assert(queue.poll()() === None) 53 | assert(queue.put("first")) 54 | assert(queue.size === 1) 55 | assert(queue.poll()() === Some("first")) 56 | assert(queue.poll()() === None) 57 | } 58 | 59 | it("conditionally poll items") { 60 | val queue = newQueue() 61 | assert(queue.size === 0) 62 | assert(queue.poll()() === None) 63 | assert(queue.put("first") === true) 64 | assert(queue.put("second") === true) 65 | assert(queue.put("third") === true) 66 | assert(queue.size === 3) 67 | assert(queue.pollIf(_ contains "t")() === Some("first")) 68 | assert(queue.pollIf(_ contains "t")() === None) 69 | assert(queue.pollIf(_ contains "c")() === Some("second")) 70 | assert(queue.pollIf(_ contains "t")() === Some("third")) 71 | assert(queue.pollIf(_ contains "t")() === None) 72 | } 73 | 74 | describe("putHead") { 75 | it("with items") { 76 | val queue = newQueue() 77 | assert(queue.size === 0) 78 | assert(queue.put("hi")) 79 | assert(queue.size === 1) 80 | queue.putHead("bye") 81 | assert(queue.size === 2) 82 | assert(queue.get()() == Some("bye")) 83 | assert(queue.size === 1) 84 | assert(queue.get()() == Some("hi")) 85 | assert(queue.size === 0) 86 | } 87 | 88 | it("get with no items") { 89 | val queue = newQueue() 90 | assert(queue.size === 0) 91 | queue.putHead("foo") 92 | assert(queue.size === 1) 93 | assert(queue.get()() == Some("foo")) 94 | assert(queue.size === 0) 95 | } 96 | 97 | it("poll with no items") { 98 | val queue = newQueue() 99 | assert(queue.size === 0) 100 | queue.putHead("foo") 101 | assert(queue.size === 1) 102 | assert(queue.poll()() == Some("foo")) 103 | assert(queue.size === 0) 104 | } 105 | } 106 | 107 | describe("honor the max size") { 108 | it("by refusing new puts") { 109 | val queue = newQueue(5, ConcurrentBlockingQueue.FullPolicy.RefusePuts) 110 | (0 until 5).foreach { i => 111 | assert(queue.put(i.toString)) 112 | } 113 | assert(queue.size === 5) 114 | assert(!queue.put("5")) 115 | assert(queue.size === 5) 116 | (0 until 5).foreach { i => 117 | assert(queue.get()() === Some(i.toString)) 118 | } 119 | } 120 | 121 | it("by dropping old items") { 122 | val queue = newQueue(5, ConcurrentBlockingQueue.FullPolicy.DropOldest) 123 | (0 until 5).foreach { i => 124 | assert(queue.put(i.toString)) 125 | } 126 | assert(queue.size === 5) 127 | assert(queue.put("5")) 128 | assert(queue.size === 5) 129 | (0 until 5).foreach { i => 130 | assert(queue.get()() === Some((i + 1).toString)) 131 | } 132 | } 133 | } 134 | 135 | it("fill in empty promises as items arrive") { 136 | val queue = newQueue() 137 | val futures = (0 until 10).map { i => queue.get() }.toList 138 | futures.foreach { f => assert(!f.isDefined) } 139 | 140 | (0 until 10).foreach { i => 141 | if (i % 2 == 0) queue.put(i.toString) else queue.putHead(i.toString) 142 | } 143 | (0 until 10).foreach { i => 144 | assert(futures(i).isDefined) 145 | assert(futures(i)() === Some(i.toString)) 146 | } 147 | } 148 | 149 | it("timeout") { 150 | Time.withCurrentTimeFrozen { timeMutator => 151 | val queue = newQueue() 152 | val future = queue.get(Before(10.milliseconds.fromNow)) 153 | 154 | timeMutator.advance(10.milliseconds) 155 | timer.tick() 156 | 157 | assert(future.isDefined) 158 | assert(future() === None) 159 | } 160 | } 161 | 162 | it("fulfill gets before they timeout") { 163 | Time.withCurrentTimeFrozen { timeMutator => 164 | val queue = newQueue() 165 | val future1 = queue.get(Before(10.milliseconds.fromNow)) 166 | val future2 = queue.get(Before(10.milliseconds.fromNow)) 167 | queue.put("surprise!") 168 | 169 | timeMutator.advance(10.milliseconds) 170 | timer.tick() 171 | 172 | assert(future1.isDefined) 173 | assert(future2.isDefined) 174 | assert(future1() === Some("surprise!")) 175 | assert(future2() === None) 176 | } 177 | } 178 | 179 | describe("really long timeout is canceled") { 180 | val deadline = Before(7.days.fromNow) 181 | 182 | it("when an item arrives") { 183 | val queue = newQueue() 184 | val future = queue.get(deadline) 185 | assert(timer.tasks.size === 1) 186 | 187 | queue.put("hello!") 188 | assert(future() === Some("hello!")) 189 | timer.tick() 190 | assert(timer.tasks.size === 0) 191 | } 192 | 193 | it("when the future is canceled") { 194 | val queue = newQueue() 195 | val future = queue.get(deadline) 196 | assert(timer.tasks.size === 1) 197 | 198 | future.cancel() 199 | timer.tick() 200 | assert(timer.tasks.size === 0) 201 | assert(queue.waiterCount === 0) 202 | } 203 | } 204 | 205 | describe("infinitely long timeout is never created") { 206 | val deadline = Forever 207 | 208 | it("but allows items to be retrieved") { 209 | val queue = newQueue() 210 | val future = queue.get(deadline) 211 | assert(timer.tasks.size === 0) 212 | 213 | queue.put("hello!") 214 | assert(future() === Some("hello!")) 215 | assert(queue.waiterCount === 0) 216 | } 217 | 218 | it("but allows future to be canceled") { 219 | val queue = newQueue() 220 | val future = queue.get(deadline) 221 | assert(timer.tasks.size === 0) 222 | 223 | future.cancel() 224 | assert(queue.waiterCount === 0) 225 | } 226 | } 227 | 228 | describe("waiters eviction") { 229 | it("should evict waiters with reasonable timeouts") { 230 | val queue = newQueue() 231 | val futures = (0 until 10).map { i => queue.get(Before(10.seconds.fromNow)) }.toList 232 | futures.foreach { f => assert(!f.isDefined) } 233 | 234 | queue.evictWaiters() 235 | 236 | futures.foreach { f => 237 | assert(f.isDefined) 238 | assert(f() == None) 239 | } 240 | } 241 | 242 | it("should evict waiters with infinite timeouts") { 243 | val queue = newQueue() 244 | val futures = (0 until 10).map { i => queue.get(Forever) }.toList 245 | futures.foreach { f => assert(!f.isDefined) } 246 | 247 | queue.evictWaiters() 248 | 249 | futures.foreach { f => 250 | assert(f.isDefined) 251 | assert(f() == None) 252 | } 253 | } 254 | } 255 | 256 | it("remain calm in the presence of a put-storm") { 257 | val count = 100 258 | val queue = newQueue() 259 | val futures = (0 until count).map { i => queue.get() }.toList 260 | val threads = (0 until count).map { i => 261 | new Thread() { 262 | override def run() { 263 | queue.put(i.toString) 264 | } 265 | } 266 | }.toList 267 | threads.foreach { _.start() } 268 | threads.foreach { _.join() } 269 | 270 | val collected = new mutable.HashSet[String] 271 | futures.foreach { f => 272 | collected += f().get 273 | } 274 | assert((0 until count).map { _.toString }.toSet === collected) 275 | } 276 | 277 | } 278 | 279 | describe("ConcurrentBlockingQueue") { 280 | tests(new QueueBuilder { 281 | def newQueue() = { 282 | timer = new MockTimer() 283 | ConcurrentBlockingQueue[String] 284 | } 285 | 286 | def newQueue(maxItems: Int, fullPolicy: ConcurrentBlockingQueue.FullPolicy) = { 287 | timer = new MockTimer() 288 | ConcurrentBlockingQueue[String](maxItems, fullPolicy) 289 | } 290 | }) 291 | } 292 | 293 | describe("SimpleBlockingQueue") { 294 | tests(new QueueBuilder { 295 | def newQueue() = { 296 | timer = new MockTimer() 297 | SimpleBlockingQueue[String] 298 | } 299 | 300 | def newQueue(maxItems: Int, fullPolicy: ConcurrentBlockingQueue.FullPolicy) = { 301 | timer = new MockTimer() 302 | SimpleBlockingQueue[String](maxItems, fullPolicy) 303 | } 304 | }) 305 | } 306 | } 307 | -------------------------------------------------------------------------------- /src/test/scala/com/twitter/libkestrel/DeadlineWaitQueueSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Twitter, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | * not use this file except in compliance with the License. You may obtain 6 | * a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twitter.libkestrel 18 | 19 | import com.twitter.conversions.time._ 20 | import com.twitter.util._ 21 | import java.util.concurrent.atomic.AtomicInteger 22 | import org.scalatest.{BeforeAndAfter, FunSpec} 23 | 24 | class DeadlineWaitQueueSpec extends FunSpec with BeforeAndAfter { 25 | describe("DeadlineWaitQueue") { 26 | val timer = new MockTimer 27 | var timeouts = new AtomicInteger(0) 28 | var awakens = new AtomicInteger(0) 29 | 30 | val deadlineWaitQueue = new DeadlineWaitQueue(timer) 31 | 32 | before { 33 | timeouts.set(0) 34 | awakens.set(0) 35 | } 36 | 37 | def defaultAwakens() { 38 | awakens.incrementAndGet() 39 | } 40 | 41 | def defaultTimeout() { 42 | timeouts.incrementAndGet() 43 | } 44 | 45 | it("should invoke timeout function when deadline expires") { 46 | Time.withCurrentTimeFrozen { tc => 47 | deadlineWaitQueue.add(Before(10.seconds.fromNow), defaultAwakens, defaultTimeout) 48 | 49 | tc.advance(5.seconds) 50 | timer.tick() 51 | assert(timeouts.get === 0) 52 | assert(awakens.get === 0) 53 | 54 | tc.advance(5.seconds + 1.millisecond) 55 | timer.tick() 56 | assert(timeouts.get === 1) 57 | assert(awakens.get === 0) 58 | } 59 | } 60 | 61 | it("should remove waiters after timeout") { 62 | Time.withCurrentTimeFrozen { tc => 63 | deadlineWaitQueue.add(Before(10.seconds.fromNow), defaultAwakens, defaultTimeout) 64 | assert(deadlineWaitQueue.size === 1) 65 | tc.advance(11.seconds) 66 | timer.tick() 67 | assert(deadlineWaitQueue.size === 0) 68 | } 69 | } 70 | 71 | it("should not use the timer for infinite deadlines") { 72 | val mockTimer = new Timer { 73 | def schedule(when: Time)(f: => Unit): TimerTask = { 74 | throw new UnsupportedOperationException 75 | } 76 | def schedule(when: Time, period: Duration)(f: => Unit): TimerTask = { 77 | throw new UnsupportedOperationException 78 | } 79 | def stop() { } 80 | } 81 | 82 | val deadlineWaitQueue = new DeadlineWaitQueue(mockTimer) 83 | deadlineWaitQueue.add(Forever, defaultAwakens, defaultTimeout) 84 | } 85 | 86 | it("should invoke the awakens function when triggered") { 87 | Time.withCurrentTimeFrozen { tc => 88 | deadlineWaitQueue.add(Before(10.seconds.fromNow), defaultAwakens, defaultTimeout) 89 | 90 | tc.advance(5.seconds) 91 | timer.tick() 92 | assert(timeouts.get === 0) 93 | assert(awakens.get === 0) 94 | 95 | deadlineWaitQueue.trigger 96 | assert(timeouts.get === 0) 97 | assert(awakens.get === 1) 98 | } 99 | } 100 | 101 | it("should remove waiters after trigger") { 102 | Time.withCurrentTimeFrozen { tc => 103 | deadlineWaitQueue.add(Before(10.seconds.fromNow), defaultAwakens, defaultTimeout) 104 | assert(deadlineWaitQueue.size === 1) 105 | deadlineWaitQueue.trigger 106 | assert(deadlineWaitQueue.size === 0) 107 | } 108 | } 109 | 110 | it("should awaken only a single waiter at a time") { 111 | Time.withCurrentTimeFrozen { tc => 112 | deadlineWaitQueue.add(Before(10.seconds.fromNow), defaultAwakens, defaultTimeout) 113 | deadlineWaitQueue.add(Before(10.seconds.fromNow), defaultAwakens, defaultTimeout) 114 | assert(timeouts.get === 0) 115 | assert(awakens.get === 0) 116 | assert(deadlineWaitQueue.size === 2) 117 | 118 | deadlineWaitQueue.trigger 119 | assert(timeouts.get === 0) 120 | assert(awakens.get === 1) 121 | assert(deadlineWaitQueue.size === 1) 122 | 123 | 124 | deadlineWaitQueue.trigger 125 | assert(timeouts.get === 0) 126 | assert(awakens.get === 2) 127 | assert(deadlineWaitQueue.size === 0) 128 | } 129 | } 130 | 131 | it("should awaken all waiters when requested") { 132 | Time.withCurrentTimeFrozen { tc => 133 | deadlineWaitQueue.add(Before(10.seconds.fromNow), defaultAwakens, defaultTimeout) 134 | deadlineWaitQueue.add(Before(10.seconds.fromNow), defaultAwakens, defaultTimeout) 135 | assert(timeouts.get === 0) 136 | assert(awakens.get === 0) 137 | 138 | deadlineWaitQueue.triggerAll 139 | assert(timeouts.get === 0) 140 | assert(awakens.get === 2) 141 | } 142 | } 143 | 144 | it("should remove waiters after triggering all") { 145 | Time.withCurrentTimeFrozen { tc => 146 | deadlineWaitQueue.add(Before(10.seconds.fromNow), defaultAwakens, defaultTimeout) 147 | deadlineWaitQueue.add(Before(10.seconds.fromNow), defaultAwakens, defaultTimeout) 148 | assert(deadlineWaitQueue.size === 2) 149 | deadlineWaitQueue.triggerAll 150 | assert(deadlineWaitQueue.size === 0) 151 | } 152 | } 153 | 154 | it("should explicitly remove a waiter without awakening or timing out") { 155 | Time.withCurrentTimeFrozen { tc => 156 | val waiter = deadlineWaitQueue.add(Before(10.seconds.fromNow), defaultAwakens, defaultTimeout) 157 | assert(deadlineWaitQueue.size === 1) 158 | assert(timeouts.get === 0) 159 | assert(awakens.get === 0) 160 | deadlineWaitQueue.remove(waiter) 161 | assert(deadlineWaitQueue.size === 0) 162 | assert(timeouts.get === 0) 163 | assert(awakens.get === 0) 164 | } 165 | } 166 | 167 | it("should evict waiters and cancel their timer tasks") { 168 | Time.withCurrentTimeFrozen { tc => 169 | deadlineWaitQueue.add(Before(10.seconds.fromNow), defaultAwakens, defaultTimeout) 170 | deadlineWaitQueue.add(Forever, defaultAwakens, defaultTimeout) 171 | assert(deadlineWaitQueue.size === 2) 172 | assert(timeouts.get === 0) 173 | assert(awakens.get === 0) 174 | 175 | deadlineWaitQueue.evictAll() 176 | assert(deadlineWaitQueue.size === 0) 177 | assert(timeouts.get === 2) 178 | assert(awakens.get === 0) 179 | 180 | tc.advance(11.seconds) 181 | timer.tick() 182 | assert(deadlineWaitQueue.size === 0) 183 | assert(timeouts.get === 2) 184 | assert(awakens.get === 0) 185 | } 186 | } 187 | } 188 | } 189 | -------------------------------------------------------------------------------- /src/test/scala/com/twitter/libkestrel/ItemIdListSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Twitter, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | * not use this file except in compliance with the License. You may obtain 6 | * a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twitter.libkestrel 18 | 19 | import org.scalatest.{BeforeAndAfter, FunSpec} 20 | 21 | class ItemIdListSpec extends FunSpec with ResourceCheckingSuite with BeforeAndAfter{ 22 | describe("ItemIdList") { 23 | var iil = new ItemIdList() 24 | 25 | before { 26 | iil = new ItemIdList() 27 | } 28 | 29 | it("should add an Integer to the list") { 30 | iil.add(3L) 31 | assert(iil.size === 1) 32 | } 33 | 34 | it("should add a sequence of Integers to the list") { 35 | iil.add(Seq(1L, 2L, 3L, 4L)) 36 | assert(iil.size === 4) 37 | } 38 | 39 | it("should pop one item at a time") { 40 | iil.add(Seq(90L, 99L)) 41 | assert(iil.pop() === Some(90L)) 42 | assert(iil.pop() === Some(99L)) 43 | } 44 | 45 | it("should pop None when there's nothing to pop") { 46 | assert(iil.pop() === None) 47 | } 48 | 49 | it("should pop all items from an index upward") { 50 | iil.add(Seq(100L, 200L, 300L, 400L)) 51 | val expected = Seq(100L, 200L) 52 | val actual = iil.pop(2) 53 | assert(expected === actual) 54 | } 55 | 56 | it("should pop all items from the list") { 57 | val seq = Seq(12L, 13L, 14L) 58 | iil.add(seq) 59 | assert(seq === iil.popAll()) 60 | } 61 | 62 | it("should return empty seq when pop's count is invalid") { 63 | assert(iil.pop(1) === Seq()) 64 | } 65 | 66 | it("should remove a set of items from the list") { 67 | iil.add(Seq(19L, 7L, 20L, 22L)) 68 | val expected = Set(7L, 20L, 22L) 69 | assert(expected === iil.remove(expected)) 70 | } 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /src/test/scala/com/twitter/libkestrel/JournalFileSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2011 Twitter, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | * not use this file except in compliance with the License. You may obtain 6 | * a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twitter.libkestrel 18 | 19 | import com.twitter.conversions.storage._ 20 | import com.twitter.io.Files 21 | import com.twitter.util._ 22 | import java.io._ 23 | import java.nio.ByteBuffer 24 | import org.scalatest.{AbstractSuite, FunSpec, Suite} 25 | import org.scalatest.matchers.{Matcher, MatchResult, ShouldMatchers} 26 | 27 | object FileHelper { 28 | def readFile(file: File, limit: Int = Int.MaxValue) = { 29 | val f = new FileInputStream(file).getChannel 30 | val length = limit min file.length.toInt 31 | val bytes = new Array[Byte](length) 32 | val buffer = ByteBuffer.wrap(bytes) 33 | var x = 0 34 | do { 35 | x = f.read(buffer) 36 | } while (buffer.position < buffer.limit && x > 0) 37 | f.close() 38 | bytes 39 | } 40 | 41 | def writeFile(file: File, data: Array[Byte]) { 42 | val f = new FileOutputStream(file).getChannel 43 | val buffer = ByteBuffer.wrap(data) 44 | do { 45 | f.write(buffer) 46 | } while (buffer.position < buffer.limit) 47 | f.close() 48 | } 49 | 50 | def hex(bytes: Array[Byte]) = { 51 | bytes.map { b => "%x".format(b) }.mkString(" ") 52 | } 53 | 54 | def unhex(s: String): Array[Byte] = { 55 | s.split(" ").map { x => Integer.parseInt(x, 16).toByte }.toArray 56 | } 57 | } 58 | 59 | class JournalFileSpec extends FunSpec with ResourceCheckingSuite with ShouldMatchers with TempFolder with TestLogging { 60 | import FileHelper._ 61 | 62 | describe("JournalFile") { 63 | describe("put") { 64 | val putData = "27 64 26 3 86 5 0 0 0 1 0 0 0 64 0 0 0 0 0 0 0 ff 0 0 0 0 0 0 0 68 65 6c 6c 6f" 65 | val putItem = QueueItem(100, Time.fromMilliseconds(255), None, ByteBuffer.wrap("hello".getBytes), 1) 66 | 67 | it("write") { 68 | val testFile = new File(testFolder, "a1") 69 | val j = JournalFile.create(testFile, null, Duration.MaxValue, 16.kilobytes) 70 | j.put(putItem) 71 | j.close() 72 | assert(hex(readFile(testFile)) === putData) 73 | } 74 | 75 | it("read") { 76 | val testFile = new File(testFolder, "a1") 77 | writeFile(testFile, unhex(putData)) 78 | val j = JournalFile.open(testFile) 79 | assert(j.readNext() === Some(Record.Put(putItem))) 80 | j.close() 81 | } 82 | 83 | it("read corrupted") { 84 | val testFile = new File(testFolder, "a1") 85 | val data = unhex(putData) 86 | (0 until data.size).foreach { size => 87 | writeFile(testFile, data.slice(0, size)) 88 | if (size != 4) { 89 | intercept[IOException] { 90 | val j = JournalFile.open(testFile) 91 | try { 92 | j.readNext() 93 | } finally { 94 | j.close() 95 | } 96 | } 97 | } 98 | } 99 | } 100 | 101 | it("read several") { 102 | val headerData = "27 64 26 3" 103 | val putData1 = "86 5 0 0 0 3 0 0 0 64 0 0 0 0 0 0 0 ff 0 0 0 0 0 0 0 68 65 6c 6c 6f" 104 | val putItem1 = QueueItem(100, Time.fromMilliseconds(255), None, ByteBuffer.wrap("hello".getBytes), 3) 105 | val putData2 = "86 7 0 0 0 2 0 0 0 65 0 0 0 0 0 0 0 84 3 0 0 0 0 0 0 67 6f 6f 64 62 79 65" 106 | val putItem2 = QueueItem(101, Time.fromMilliseconds(900), None, ByteBuffer.wrap("goodbye".getBytes), 2) 107 | 108 | val testFile = new File(testFolder, "a1") 109 | writeFile(testFile, unhex(headerData + " " + putData1 + " " + putData2)) 110 | val j = JournalFile.open(testFile) 111 | assert(j.readNext() === Some(Record.Put(putItem1))) 112 | assert(j.readNext() === Some(Record.Put(putItem2))) 113 | assert(j.readNext() === None) 114 | j.close() 115 | } 116 | 117 | it("append") { 118 | val headerData = "27 64 26 3" 119 | val putData1 = "86 5 0 0 0 0 1 0 0 64 0 0 0 0 0 0 0 ff 0 0 0 0 0 0 0 68 65 6c 6c 6f" 120 | val putItem1 = QueueItem(100, Time.fromMilliseconds(255), None, ByteBuffer.wrap("hello".getBytes), 256) 121 | val putData2 = "86 7 0 0 0 0 0 0 0 65 0 0 0 0 0 0 0 84 3 0 0 0 0 0 0 67 6f 6f 64 62 79 65" 122 | val putItem2 = QueueItem(101, Time.fromMilliseconds(900), None, ByteBuffer.wrap("goodbye".getBytes), 0) 123 | 124 | val testFile = new File(testFolder, "a1") 125 | val j = JournalFile.create(testFile, null, Duration.MaxValue, 16.kilobytes) 126 | j.put(putItem1) 127 | j.close() 128 | val j2 = JournalFile.append(testFile, null, Duration.MaxValue, 16.kilobytes) 129 | j2.put(putItem2) 130 | j2.close() 131 | 132 | assert(hex(readFile(testFile)) === headerData + " " + putData1 + " " + putData2) 133 | } 134 | } 135 | 136 | it("refuse to deal with items too large") { 137 | val testFile = new File(testFolder, "a1") 138 | writeFile(testFile, unhex("27 64 26 3 81 ff ff ff 7f")) 139 | 140 | val j = JournalFile.open(testFile) 141 | val e = intercept[CorruptedJournalException] { j.readNext() } 142 | assert(e.message === "item too large") 143 | j.close() 144 | 145 | val item = QueueItem(100, Time.fromMilliseconds(0), None, 146 | ByteBuffer.allocate(Record.LARGEST_DATA.inBytes.toInt + 1)) 147 | val j2 = JournalFile.create(testFile, null, Duration.MaxValue, 16.kilobytes) 148 | val e2 = intercept[IOException] { j2.put(item) } 149 | assert(e2.getMessage === "item too large") 150 | j2.close() 151 | } 152 | } 153 | } 154 | -------------------------------------------------------------------------------- /src/test/scala/com/twitter/libkestrel/JournalReaderSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2011 Twitter, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | * not use this file except in compliance with the License. You may obtain 6 | * a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twitter.libkestrel 18 | 19 | import com.twitter.conversions.storage._ 20 | import com.twitter.conversions.time._ 21 | import com.twitter.util._ 22 | import java.io._ 23 | import java.nio.ByteBuffer 24 | import org.scalatest.{AbstractSuite, FunSpec, Suite} 25 | import org.scalatest.matchers.{Matcher, MatchResult, ShouldMatchers} 26 | 27 | class JournalReaderSpec extends FunSpec with ResourceCheckingSuite with ShouldMatchers with TempFolder with TestLogging { 28 | 29 | implicit def stringToBuffer(s: String): ByteBuffer = ByteBuffer.wrap(s.getBytes) 30 | 31 | def makeJournal(name: String, maxFileSize: StorageUnit): Journal = 32 | new Journal(testFolder, name, maxFileSize, null, Duration.MaxValue, None) 33 | 34 | def makeJournal(name: String): Journal = makeJournal(name, 16.megabytes) 35 | 36 | def makeFiles() { 37 | val jf1 = JournalFile.create(new File(testFolder, "test.1"), null, Duration.MaxValue, 16.kilobytes) 38 | jf1.put(QueueItem(100L, Time.now, None, "100")) 39 | jf1.put(QueueItem(101L, Time.now, None, "101")) 40 | jf1.close() 41 | 42 | val jf2 = JournalFile.create(new File(testFolder, "test.2"), null, Duration.MaxValue, 16.kilobytes) 43 | jf2.put(QueueItem(102L, Time.now, None, "102")) 44 | jf2.put(QueueItem(103L, Time.now, None, "103")) 45 | jf2.close() 46 | 47 | val jf3 = JournalFile.create(new File(testFolder, "test.3"), null, Duration.MaxValue, 16.kilobytes) 48 | jf3.put(QueueItem(104L, Time.now, None, "104")) 49 | jf3.put(QueueItem(105L, Time.now, None, "105")) 50 | jf3.close() 51 | } 52 | 53 | def queueItem(id: Long) = QueueItem(id, Time.now, None, "blah") 54 | 55 | describe("Journal#Reader") { 56 | it("created with a checkpoint file") { 57 | val j = makeJournal("test") 58 | j.reader("hello") 59 | 60 | assert(new File(testFolder, "test.read.hello").exists) 61 | j.close() 62 | } 63 | 64 | it("write a checkpoint") { 65 | val file = new File(testFolder, "a1") 66 | val j = makeJournal("test") 67 | val reader = new j.Reader("1", file) 68 | reader.head = 123L 69 | reader.commit(queueItem(125L)) 70 | reader.commit(queueItem(130L)) 71 | reader.checkpoint() 72 | 73 | assert(BookmarkFile.open(file).toList === List( 74 | Record.ReadHead(123L), 75 | Record.ReadDone(Array(125L, 130L)) 76 | )) 77 | 78 | j.close() 79 | } 80 | 81 | it("read a checkpoint") { 82 | val file = new File(testFolder, "a1") 83 | val bf = BookmarkFile.create(file) 84 | bf.readHead(900L) 85 | bf.readDone(Array(902L, 903L)) 86 | bf.close() 87 | 88 | val jf = JournalFile.create(new File(testFolder, "test.1"), null, Duration.MaxValue, 16.kilobytes) 89 | jf.put(queueItem(890L)) 90 | jf.put(queueItem(910L)) 91 | jf.close() 92 | 93 | val j = makeJournal("test") 94 | val reader = new j.Reader("1", file) 95 | reader.open() 96 | assert(reader.head === 900L) 97 | assert(reader.doneSet.toList.sorted === List(902L, 903L)) 98 | reader.close() 99 | j.close() 100 | } 101 | 102 | it("track committed items") { 103 | val file = new File(testFolder, "a1") 104 | val j = makeJournal("test") 105 | val reader = new j.Reader("1", file) 106 | reader.head = 123L 107 | 108 | reader.commit(queueItem(124L)) 109 | assert(reader.head === 124L) 110 | assert(reader.doneSet.toList.sorted === List()) 111 | 112 | reader.commit(queueItem(126L)) 113 | reader.commit(queueItem(127L)) 114 | reader.commit(queueItem(129L)) 115 | assert(reader.head === 124L) 116 | assert(reader.doneSet.toList.sorted === List(126L, 127L, 129L)) 117 | 118 | reader.commit(queueItem(125L)) 119 | assert(reader.head === 127L) 120 | assert(reader.doneSet.toList.sorted === List(129L)) 121 | 122 | reader.commit(queueItem(130L)) 123 | reader.commit(queueItem(128L)) 124 | assert(reader.head === 130L) 125 | assert(reader.doneSet.toList.sorted === List()) 126 | 127 | j.close() 128 | } 129 | 130 | it("flush all items") { 131 | val j = makeJournal("test") 132 | val (item1, future1) = j.put(ByteBuffer.wrap("hi".getBytes), Time.now, None) 133 | val reader = j.reader("1") 134 | reader.open() 135 | reader.commit(item1) 136 | val (item2, future2) = j.put(ByteBuffer.wrap("bye".getBytes), Time.now, None) 137 | 138 | assert(reader.head === item1.id) 139 | reader.flush() 140 | assert(reader.head === item2.id) 141 | 142 | j.close() 143 | } 144 | 145 | describe("file boundaries") { 146 | def createJournalFiles(journalName: String, startId: Long, idsPerFile: Int, files: Int, head: Long) { 147 | for (fileNum <- 1 to files) { 148 | val name = "%s.%d".format(journalName, fileNum) 149 | val jf = JournalFile.create(new File(testFolder, name), null, Duration.MaxValue, 16.kilobytes) 150 | for(idNum <- 1 to idsPerFile) { 151 | val id = startId + ((fileNum - 1) * idsPerFile) + (idNum - 1) 152 | jf.put(QueueItem(id, Time.now, None, ByteBuffer.wrap(id.toString.getBytes))) 153 | } 154 | jf.close() 155 | } 156 | 157 | val name = "%s.read.client".format(journalName) 158 | val bf = BookmarkFile.create(new File(testFolder, name)) 159 | bf.readHead(head) 160 | bf.close() 161 | } 162 | 163 | it("should start at a file edge") { 164 | createJournalFiles("test", 100L, 2, 2, 101L) 165 | val j = makeJournal("test") 166 | val reader = j.reader("client") 167 | reader.open() 168 | 169 | assert(reader.next().map { _.id } === Some(102L)) 170 | 171 | j.close() 172 | } 173 | 174 | it("should cross files") { 175 | createJournalFiles("test", 100L, 2, 2, 100L) 176 | val j = makeJournal("test") 177 | val reader = j.reader("client") 178 | reader.open() 179 | 180 | assert(reader.next().map { _.id } === Some(101L)) 181 | assert(reader.next().map { _.id } === Some(102L)) 182 | 183 | j.close() 184 | } 185 | 186 | it("should peek at leading file edge") { 187 | createJournalFiles("test", 100L, 2, 2, 101L) 188 | val j = makeJournal("test") 189 | val reader = j.reader("client") 190 | reader.open() 191 | 192 | assert(reader.peekOldest().map { _.id } === Some(102L)) 193 | assert(reader.peekOldest().map { _.id } === Some(102L)) 194 | 195 | j.close() 196 | } 197 | 198 | it("should peek at trailing file edge") { 199 | createJournalFiles("test", 100L, 2, 2, 100L) 200 | val j = makeJournal("test") 201 | val reader = j.reader("client") 202 | reader.open() 203 | 204 | assert(reader.peekOldest().map { _.id } === Some(101L)) 205 | assert(reader.peekOldest().map { _.id } === Some(101L)) 206 | 207 | j.close() 208 | } 209 | 210 | it("should conditionally get at leading file edge") { 211 | createJournalFiles("test", 100L, 2, 2, 101L) 212 | val j = makeJournal("test") 213 | val reader = j.reader("client") 214 | reader.open() 215 | 216 | assert(reader.nextIf { _ => false } === None) 217 | assert(reader.nextIf { _ => true }.map { _.id } === Some(102L)) 218 | assert(reader.nextIf { _ => true }.map { _.id } === Some(103L)) 219 | 220 | j.close() 221 | } 222 | 223 | it("should conditionally get at trailing file edge") { 224 | createJournalFiles("test", 100L, 2, 2, 100L) 225 | val j = makeJournal("test") 226 | val reader = j.reader("client") 227 | reader.open() 228 | 229 | assert(reader.nextIf { _ => false } === None) 230 | assert(reader.nextIf { _ => true }.map { _.id } === Some(101L)) 231 | assert(reader.nextIf { _ => true }.map { _.id } === Some(102L)) 232 | 233 | j.close() 234 | } 235 | } 236 | 237 | it("fileInfosAfter") { 238 | makeFiles() 239 | val j = makeJournal("test") 240 | val reader = j.reader("client") 241 | reader.head = 100L 242 | 243 | assert(j.fileInfosAfter(101L).toList === List( 244 | FileInfo(new File(testFolder, "test.2"), 102L, 103L, 2, 6), 245 | FileInfo(new File(testFolder, "test.3"), 104L, 105L, 2, 6) 246 | )) 247 | j.close() 248 | } 249 | } 250 | } 251 | -------------------------------------------------------------------------------- /src/test/scala/com/twitter/libkestrel/JournalSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2011 Twitter, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | * not use this file except in compliance with the License. You may obtain 6 | * a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twitter.libkestrel 18 | 19 | import com.twitter.conversions.storage._ 20 | import com.twitter.conversions.time._ 21 | import com.twitter.util._ 22 | import java.io._ 23 | import java.nio.ByteBuffer 24 | import org.scalatest.{AbstractSuite, FunSpec, Suite} 25 | import org.scalatest.matchers.{Matcher, MatchResult, ShouldMatchers} 26 | 27 | class JournalSpec extends FunSpec with ResourceCheckingSuite with ShouldMatchers with TempFolder with TestLogging { 28 | def makeJournal(name: String, maxFileSize: StorageUnit): Journal = 29 | new Journal(testFolder, name, maxFileSize, null, Duration.MaxValue, None) 30 | 31 | def makeJournal(name: String): Journal = makeJournal(name, 16.megabytes) 32 | 33 | def addItem(j: Journal, size: Int) = { 34 | val now = Time.now.inMilliseconds 35 | j.put(ByteBuffer.allocate(size), Time.now, None) 36 | now 37 | } 38 | 39 | def queueItem(id: Long) = QueueItem(id, Time.now, None, "blah") 40 | 41 | implicit def stringToBuffer(s: String): ByteBuffer = ByteBuffer.wrap(s.getBytes) 42 | 43 | describe("Journal") { 44 | describe("#getQueueNamesFromFolder") { 45 | it("should identify valid queue names") { 46 | new FileOutputStream(testFolder + "/j1").close() 47 | new FileOutputStream(testFolder + "/j2").close() 48 | assert(Journal.getQueueNamesFromFolder(testFolder) === Set("j1", "j2")) 49 | } 50 | 51 | it("should handle queues with archived journals") { 52 | new FileOutputStream(testFolder + "/j1").close() 53 | new FileOutputStream(testFolder + "/j1.1000").close() 54 | new FileOutputStream(testFolder + "/j1.2000").close() 55 | new FileOutputStream(testFolder + "/j2").close() 56 | assert(Journal.getQueueNamesFromFolder(testFolder) === Set("j1", "j2")) 57 | } 58 | 59 | it("should ignore queues with journals being packed") { 60 | new FileOutputStream(testFolder + "/j1").close() 61 | new FileOutputStream(testFolder + "/j2").close() 62 | new FileOutputStream(testFolder + "/j2~~").close() 63 | assert(Journal.getQueueNamesFromFolder(testFolder) === Set("j1", "j2")) 64 | } 65 | 66 | it("should ignore subdirectories") { 67 | new FileOutputStream(testFolder + "/j1").close() 68 | new FileOutputStream(testFolder + "/j2").close() 69 | new File(testFolder, "subdir").mkdirs() 70 | assert(Journal.getQueueNamesFromFolder(testFolder) === Set("j1", "j2")) 71 | } 72 | } 73 | 74 | it("find reader/writer files") { 75 | Time.withCurrentTimeFrozen { timeMutator => 76 | List( 77 | "test.read.client1", "test.read.client2", "test.read.client1~~", "test.readmenot", 78 | "test.read." 79 | ).foreach { name => 80 | BookmarkFile.create(new File(testFolder, name)).close() 81 | } 82 | List( 83 | "test.901", "test.8000", "test.3leet", "test.1", "test.5005" 84 | ).foreach { name => 85 | val jf = JournalFile.create(new File(testFolder, name), null, Duration.MaxValue, 16.kilobytes) 86 | jf.put(QueueItem(1L, Time.now, None, ByteBuffer.allocate(1))) 87 | jf.close() 88 | } 89 | 90 | val j = makeJournal("test") 91 | assert(j.writerFiles().map { _.getName }.toSet === 92 | Set("test.901", "test.8000", "test.1", "test.5005")) 93 | assert(j.readerFiles().map { _.getName }.toSet === 94 | Set("test.read.client1", "test.read.client2")) 95 | j.close() 96 | 97 | new File(testFolder, "test.read.client1").delete() 98 | new File(testFolder, "test.read.client2").delete() 99 | val j2 = makeJournal("test") 100 | assert(j2.readerFiles().map { _.getName }.toSet === 101 | Set("test.read.")) 102 | j2.close() 103 | } 104 | } 105 | 106 | it("erase old temporary files") { 107 | List("test.1", "test.read.1", "test.read.1~~").foreach { name => 108 | new File(testFolder, name).createNewFile() 109 | } 110 | 111 | val j = makeJournal("test") 112 | assert(!new File(testFolder, "test.read.1~~").exists) 113 | j.close() 114 | } 115 | 116 | it("erase all journal files") { 117 | List("test.1", "test.read.1", "test.read.1~~", "testbad").foreach { name => 118 | new File(testFolder, name).createNewFile() 119 | } 120 | 121 | val j = makeJournal("test") 122 | j.erase() 123 | 124 | assert(testFolder.list.toList === List("testbad")) 125 | } 126 | 127 | it("report size correctly") { 128 | val jf1 = JournalFile.create(new File(testFolder, "test.1"), null, Duration.MaxValue, 16.megabytes) 129 | jf1.put(QueueItem(101L, Time.now, None, ByteBuffer.allocate(1000))) 130 | jf1.close() 131 | val jf2 = JournalFile.create(new File(testFolder, "test.2"), null, Duration.MaxValue, 16.megabytes) 132 | jf2.put(QueueItem(102L, Time.now, None, ByteBuffer.allocate(1000))) 133 | jf2.close() 134 | 135 | val j = makeJournal("test") 136 | assert(j.journalSize === 2058L) 137 | j.close() 138 | } 139 | 140 | describe("fileForId") { 141 | it("startup") { 142 | List( 143 | ("test.901", 901), 144 | ("test.8000", 8000), 145 | ("test.1", 1), 146 | ("test.5005", 5005) 147 | ).foreach { case (name, id) => 148 | val jf = JournalFile.create(new File(testFolder, name), null, Duration.MaxValue, 16.kilobytes) 149 | jf.put(QueueItem(id, Time.now, None, ByteBuffer.allocate(5))) 150 | jf.close() 151 | } 152 | 153 | val j = makeJournal("test") 154 | assert(j.fileInfoForId(1) === Some(FileInfo(new File(testFolder, "test.1"), 1, 1, 1, 5))) 155 | assert(j.fileInfoForId(0) === None) 156 | assert(j.fileInfoForId(555) === Some(FileInfo(new File(testFolder, "test.1"), 1, 1, 1, 5))) 157 | assert(j.fileInfoForId(900) === Some(FileInfo(new File(testFolder, "test.1"), 1, 1, 1, 5))) 158 | assert(j.fileInfoForId(901) === Some(FileInfo(new File(testFolder, "test.901"), 901, 901, 1, 5))) 159 | assert(j.fileInfoForId(902) === Some(FileInfo(new File(testFolder, "test.901"), 901, 901, 1, 5))) 160 | assert(j.fileInfoForId(6666) === Some(FileInfo(new File(testFolder, "test.5005"), 5005, 5005, 1, 5))) 161 | assert(j.fileInfoForId(9999) === Some(FileInfo(new File(testFolder, "test.8000"), 8000, 8000, 1, 5))) 162 | j.close() 163 | } 164 | 165 | it("during journal rotation") { 166 | Time.withCurrentTimeFrozen { timeMutator => 167 | val j = makeJournal("test", 1078.bytes) 168 | j.put(ByteBuffer.allocate(512), Time.now, None) 169 | timeMutator.advance(1.millisecond) 170 | j.put(ByteBuffer.allocate(512), Time.now, None) 171 | timeMutator.advance(1.millisecond) 172 | j.put(ByteBuffer.allocate(512), Time.now, None) 173 | timeMutator.advance(1.millisecond) 174 | j.put(ByteBuffer.allocate(512), Time.now, None) 175 | timeMutator.advance(1.millisecond) 176 | j.put(ByteBuffer.allocate(512), Time.now, None) 177 | 178 | val file1 = new File(testFolder, "test." + 4.milliseconds.ago.inMilliseconds) 179 | val file2 = new File(testFolder, "test." + 2.milliseconds.ago.inMilliseconds) 180 | val file3 = new File(testFolder, "test." + Time.now.inMilliseconds) 181 | 182 | assert(j.fileInfoForId(1) === Some(FileInfo(file1, 1, 2, 2, 1024))) 183 | assert(j.fileInfoForId(2) === Some(FileInfo(file1, 1, 2, 2, 1024))) 184 | assert(j.fileInfoForId(3) === Some(FileInfo(file2, 3, 4, 2, 1024))) 185 | assert(j.fileInfoForId(4) === Some(FileInfo(file2, 3, 4, 2, 1024))) 186 | assert(j.fileInfoForId(5) === Some(FileInfo(file3, 5, 0, 0, 0))) 187 | j.close() 188 | } 189 | } 190 | } 191 | 192 | it("checkpoint readers") { 193 | List("test.read.client1", "test.read.client2").foreach { name => 194 | val bf = BookmarkFile.create(new File(testFolder, name)) 195 | bf.readHead(100L) 196 | bf.readDone(Array(102L)) 197 | bf.close() 198 | } 199 | val jf = JournalFile.create(new File(testFolder, "test.1"), null, Duration.MaxValue, 16.kilobytes) 200 | jf.put(QueueItem(100L, Time.now, None, "hi")) 201 | jf.put(QueueItem(105L, Time.now, None, "hi")) 202 | jf.close() 203 | 204 | val j = makeJournal("test") 205 | j.reader("client1").commit(queueItem(101L)) 206 | j.reader("client2").commit(queueItem(103L)) 207 | j.checkpoint() 208 | j.close() 209 | 210 | assert(BookmarkFile.open(new File(testFolder, "test.read.client1")).toList === List( 211 | Record.ReadHead(102L), 212 | Record.ReadDone(Array[Long]()) 213 | )) 214 | 215 | assert(BookmarkFile.open(new File(testFolder, "test.read.client2")).toList === List( 216 | Record.ReadHead(100L), 217 | Record.ReadDone(Array(102L, 103L)) 218 | )) 219 | } 220 | 221 | it("doesn't checkpoint readers that haven't changed") { 222 | val j = makeJournal("test") 223 | j.reader("client1").commit(queueItem(1L)) 224 | j.reader("client1").checkpoint() 225 | assert(new File(testFolder, "test.read.client1").exists) 226 | 227 | new File(testFolder, "test.read.client1").delete() 228 | assert(!new File(testFolder, "test.read.client1").exists) 229 | j.reader("client1").checkpoint() 230 | assert(!new File(testFolder, "test.read.client1").exists) 231 | j.close() 232 | } 233 | 234 | it("make new reader") { 235 | val j = makeJournal("test") 236 | var r = j.reader("new") 237 | r.head = 100L 238 | r.commit(queueItem(101L)) 239 | r.checkpoint() 240 | j.close() 241 | 242 | assert(BookmarkFile.open(new File(testFolder, "test.read.new")).toList === List( 243 | Record.ReadHead(101L), 244 | Record.ReadDone(Array[Long]()) 245 | )) 246 | } 247 | 248 | it("create a default reader when no others exist") { 249 | val j = makeJournal("test") 250 | j.close() 251 | 252 | assert(BookmarkFile.open(new File(testFolder, "test.read.")).toList === List( 253 | Record.ReadHead(0L), 254 | Record.ReadDone(Array[Long]()) 255 | )) 256 | } 257 | 258 | it("convert the default reader to a named reader when one is created") { 259 | val j = makeJournal("test") 260 | val reader = j.reader("") 261 | reader.head = 100L 262 | reader.checkpoint() 263 | 264 | assert(new File(testFolder, "test.read.").exists) 265 | 266 | j.reader("hello") 267 | assert(!new File(testFolder, "test.read.").exists) 268 | assert(new File(testFolder, "test.read.hello").exists) 269 | 270 | assert(BookmarkFile.open(new File(testFolder, "test.read.hello")).toList === List( 271 | Record.ReadHead(100L), 272 | Record.ReadDone(Array[Long]()) 273 | )) 274 | 275 | j.close() 276 | } 277 | 278 | describe("recover a reader") { 279 | /* 280 | * rationale: 281 | * this can happen if a write journal is corrupted/truncated, losing the last few items, and 282 | * a reader had already written a state file out claiming to have finished processing the 283 | * items that are now lost. 284 | */ 285 | it("with a head id in the future") { 286 | // create main journal 287 | val jf1 = JournalFile.create(new File(testFolder, "test.1"), null, Duration.MaxValue, 16.kilobytes) 288 | jf1.put(QueueItem(390L, Time.now, None, "hi")) 289 | jf1.put(QueueItem(400L, Time.now, None, "hi")) 290 | jf1.close() 291 | 292 | // create bookmarks with impossible ids 293 | val bf1 = BookmarkFile.create(new File(testFolder, "test.read.1")) 294 | bf1.readHead(402L) 295 | bf1.close() 296 | val bf2 = BookmarkFile.create(new File(testFolder, "test.read.2")) 297 | bf2.readHead(390L) 298 | bf2.readDone(Array(395L, 403L)) 299 | bf2.close() 300 | 301 | val j = makeJournal("test") 302 | val r1 = j.reader("1") 303 | assert(r1.head === 400L) 304 | assert(r1.doneSet === Set()) 305 | val r2 = j.reader("2") 306 | assert(r2.head === 390L) 307 | assert(r2.doneSet === Set(395L)) 308 | 309 | j.close() 310 | } 311 | 312 | it("with a head id that doesn't exist anymore") { 313 | val jf1 = JournalFile.create(new File(testFolder, "test.1"), null, Duration.MaxValue, 16.kilobytes) 314 | jf1.put(QueueItem(800L, Time.now, None, "hi")) 315 | jf1.close() 316 | val bf = BookmarkFile.create(new File(testFolder, "test.read.1")) 317 | bf.readHead(600L) 318 | bf.readDone(Array[Long]()) 319 | bf.close() 320 | 321 | val j = makeJournal("test") 322 | assert(j.reader("1").head === 799L) 323 | 324 | j.close() 325 | } 326 | } 327 | 328 | it("start with an empty journal") { 329 | Time.withCurrentTimeFrozen { timeMutator => 330 | val roundedTime = Time.fromMilliseconds(Time.now.inMilliseconds) 331 | val j = makeJournal("test") 332 | val (item, future) = j.put("hi", Time.now, None) 333 | assert(item.id === 1L) 334 | j.close() 335 | 336 | val file = new File(testFolder, "test." + Time.now.inMilliseconds) 337 | val jf = JournalFile.open(file) 338 | assert(jf.readNext() === 339 | Some(Record.Put(QueueItem(1L, roundedTime, None, "hi")))) 340 | jf.close() 341 | } 342 | } 343 | 344 | it("append new items to the end of the last journal") { 345 | Time.withCurrentTimeFrozen { timeMutator => 346 | val roundedTime = Time.fromMilliseconds(Time.now.inMilliseconds) 347 | val file1 = new File(testFolder, "test.1") 348 | val jf1 = JournalFile.create(file1, null, Duration.MaxValue, 16.kilobytes) 349 | jf1.put(QueueItem(101L, Time.now, None, "101")) 350 | jf1.close() 351 | val file2 = new File(testFolder, "test.2") 352 | val jf2 = JournalFile.create(file2, null, Duration.MaxValue, 16.kilobytes) 353 | jf2.put(QueueItem(102L, Time.now, None, "102")) 354 | jf2.close() 355 | 356 | val j = makeJournal("test") 357 | val (item, future) = j.put("hi", Time.now, None) 358 | assert(item.id === 103L) 359 | j.close() 360 | 361 | val jf3 = JournalFile.open(file2) 362 | assert(jf3.readNext() === 363 | Some(Record.Put(QueueItem(102L, roundedTime, None, "102")))) 364 | assert(jf3.readNext() === 365 | Some(Record.Put(QueueItem(103L, roundedTime, None, "hi")))) 366 | assert(jf3.readNext() === None) 367 | jf3.close() 368 | } 369 | } 370 | 371 | it("truncate corrupted journal") { 372 | Time.withCurrentTimeFrozen { timeMutator => 373 | val roundedTime = Time.fromMilliseconds(Time.now.inMilliseconds) 374 | 375 | // write 2 valid entries, but truncate the last one to make it corrupted. 376 | val file = new File(testFolder, "test.1") 377 | val jf = JournalFile.create(file, null, Duration.MaxValue, 16.kilobytes) 378 | jf.put(QueueItem(101L, Time.now, None, "101")) 379 | jf.put(QueueItem(102L, Time.now, None, "102")) 380 | jf.close() 381 | 382 | val raf = new RandomAccessFile(file, "rw") 383 | raf.setLength(file.length - 1) 384 | raf.close() 385 | 386 | val j = makeJournal("test") 387 | val (item, future) = j.put("hi", Time.now, None) 388 | assert(item.id === 102L) 389 | j.close() 390 | 391 | val jf2 = JournalFile.open(file) 392 | assert(jf2.readNext() === 393 | Some(Record.Put(QueueItem(101L, roundedTime, None, "101")))) 394 | assert(jf2.readNext() === 395 | Some(Record.Put(QueueItem(102L, roundedTime, None, "hi")))) 396 | assert(jf2.readNext() === None) 397 | jf2.close() 398 | } 399 | } 400 | 401 | it("rotate journal files") { 402 | Time.withCurrentTimeFrozen { timeMutator => 403 | val j = makeJournal("test", 1.kilobyte) 404 | 405 | val time1 = addItem(j, 475) 406 | timeMutator.advance(1.millisecond) 407 | val time2 = addItem(j, 475) 408 | timeMutator.advance(1.millisecond) 409 | val time3 = addItem(j, 475) 410 | j.close() 411 | 412 | val file1 = new File(testFolder, "test." + time1) 413 | val file2 = new File(testFolder, "test." + time3) 414 | val defaultReader = new File(testFolder, "test.read.") 415 | assert(testFolder.list.sorted.toList === List(file1, file2, defaultReader).map { _.getName() }) 416 | val jf1 = JournalFile.open(file1) 417 | assert(jf1.toList === List( 418 | Record.Put(QueueItem(1L, Time.fromMilliseconds(time1), None, ByteBuffer.allocate(475))), 419 | Record.Put(QueueItem(2L, Time.fromMilliseconds(time2), None, ByteBuffer.allocate(475))) 420 | )) 421 | jf1.close() 422 | val jf2 = JournalFile.open(file2) 423 | assert(jf2.toList === List( 424 | Record.Put(QueueItem(3L, Time.fromMilliseconds(time3), None, ByteBuffer.allocate(475))) 425 | )) 426 | jf2.close() 427 | } 428 | } 429 | 430 | describe("clean up any dead files behind it") { 431 | it("when a client catches up") { 432 | Time.withCurrentTimeFrozen { timeMutator => 433 | val j = makeJournal("test", 1.kilobyte) 434 | val reader = j.reader("client") 435 | 436 | val time1 = addItem(j, 475) 437 | timeMutator.advance(1.millisecond) 438 | val time2 = addItem(j, 475) 439 | timeMutator.advance(1.millisecond) 440 | val time3 = addItem(j, 475) 441 | timeMutator.advance(1.millisecond) 442 | 443 | assert(new File(testFolder, "test." + time1).exists) 444 | assert(new File(testFolder, "test." + time3).exists) 445 | 446 | reader.commit(queueItem(1L)) 447 | reader.commit(queueItem(2L)) 448 | j.checkpoint() 449 | 450 | assert(!new File(testFolder, "test." + time1).exists) 451 | assert(new File(testFolder, "test." + time3).exists) 452 | 453 | j.close() 454 | } 455 | } 456 | 457 | it("when the journal moves to a new file") { 458 | Time.withCurrentTimeFrozen { timeMutator => 459 | val j = makeJournal("test", 1.kilobyte) 460 | val reader = j.reader("client") 461 | val time0 = Time.now.inMilliseconds 462 | timeMutator.advance(1.millisecond) 463 | 464 | val time1 = addItem(j, 512) 465 | timeMutator.advance(1.millisecond) 466 | val time2 = addItem(j, 512) 467 | timeMutator.advance(1.millisecond) 468 | reader.commit(queueItem(1L)) 469 | reader.commit(queueItem(2L)) 470 | reader.checkpoint() 471 | 472 | assert(new File(testFolder, "test." + time0).exists) 473 | assert(new File(testFolder, "test." + time2).exists) 474 | 475 | val time3 = addItem(j, 512) 476 | timeMutator.advance(1.millisecond) 477 | val time4 = addItem(j, 512) 478 | 479 | assert(!new File(testFolder, "test." + time0).exists) 480 | assert(new File(testFolder, "test." + time3).exists) 481 | assert(new File(testFolder, "test." + time4).exists) 482 | 483 | j.close() 484 | } 485 | } 486 | } 487 | } 488 | } 489 | -------------------------------------------------------------------------------- /src/test/scala/com/twitter/libkestrel/JournaledBlockingQueueSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2011 Twitter, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | * not use this file except in compliance with the License. You may obtain 6 | * a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twitter.libkestrel 18 | 19 | import com.twitter.conversions.time._ 20 | import com.twitter.libkestrel.config._ 21 | import com.twitter.util.JavaTimer 22 | import java.nio.ByteBuffer 23 | import java.util.concurrent.ScheduledThreadPoolExecutor 24 | import org.scalatest.FunSpec 25 | import org.scalatest.matchers.ShouldMatchers 26 | 27 | class JournaledBlockingQueueSpec extends FunSpec with ResourceCheckingSuite with ShouldMatchers with TempFolder with TestLogging { 28 | val config = new JournaledQueueConfig(name = "test") 29 | def makeReaderConfig() = new JournaledQueueReaderConfig() 30 | 31 | val timer = new JavaTimer(isDaemon = true) 32 | val scheduler = new ScheduledThreadPoolExecutor(1) 33 | 34 | def makeQueue( 35 | config: JournaledQueueConfig = config, 36 | readerConfig: JournaledQueueReaderConfig = makeReaderConfig() 37 | ) = { 38 | new JournaledQueue(config.copy(defaultReaderConfig = readerConfig), testFolder, timer, scheduler) 39 | } 40 | 41 | def makeCodec() = { 42 | new Codec[String] { 43 | def encode(item: String) = ByteBuffer.wrap(item.getBytes) 44 | def decode(data: ByteBuffer): String = { 45 | val bytes = new Array[Byte](data.remaining) 46 | data.mark 47 | data.get(bytes) 48 | data.reset 49 | new String(bytes) 50 | } 51 | } 52 | } 53 | 54 | describe("JournaledBlockingQueue") { 55 | def makeBlockingQueue(queue: JournaledQueue, codec: Codec[String] = makeCodec()) = { 56 | val bq = queue.toBlockingQueue(codec) 57 | 58 | bq.put("first") 59 | bq.put("second") 60 | bq.put("third") 61 | 62 | bq 63 | } 64 | 65 | it("auto-commits on get") { 66 | val queue = makeQueue() 67 | val reader = queue.reader("") 68 | val blockingQueue = makeBlockingQueue(queue) 69 | 70 | assert(reader.items === 3) 71 | assert(reader.openItems === 0) 72 | 73 | assert(blockingQueue.get(Before(100.seconds.fromNow))() === Some("first")) 74 | assert(blockingQueue.get()() === Some("second")) 75 | 76 | assert(reader.items === 1) 77 | assert(reader.openItems === 0) 78 | 79 | blockingQueue.close() 80 | } 81 | 82 | it("auto-commits on poll") { 83 | val queue = makeQueue() 84 | val reader = queue.reader("") 85 | val blockingQueue = makeBlockingQueue(queue) 86 | 87 | assert(reader.items === 3) 88 | assert(reader.openItems === 0) 89 | 90 | assert(blockingQueue.poll()() === Some("first")) 91 | 92 | assert(reader.items === 2) 93 | assert(reader.openItems === 0) 94 | 95 | blockingQueue.close() 96 | } 97 | } 98 | 99 | describe("TransactionalJournaledBlockingQueue") { 100 | def makeTransactionalBlockingQueue(queue: JournaledQueue, codec: Codec[String] = makeCodec()) = { 101 | val bq = queue.toTransactionalBlockingQueue(codec) 102 | 103 | bq.put("first") 104 | bq.put("second") 105 | bq.put("third") 106 | 107 | bq 108 | } 109 | 110 | it("allows open transactions to be committed after get") { 111 | val queue = makeQueue() 112 | val reader = queue.reader("") 113 | val blockingQueue = makeTransactionalBlockingQueue(queue) 114 | 115 | assert(reader.items === 3) 116 | assert(reader.openItems === 0) 117 | 118 | val txn1 = blockingQueue.get(Before(100.seconds.fromNow))().get 119 | val txn2 = blockingQueue.get()().get 120 | 121 | assert(txn1.item === "first") 122 | assert(txn2.item === "second") 123 | 124 | assert(reader.items === 3) 125 | assert(reader.openItems === 2) 126 | 127 | txn1.commit() 128 | 129 | assert(reader.items === 2) 130 | assert(reader.openItems === 1) 131 | 132 | txn2.commit() 133 | 134 | assert(reader.items === 1) 135 | assert(reader.openItems === 0) 136 | 137 | blockingQueue.close() 138 | } 139 | 140 | it("allows open transactions to be rolled back after get") { 141 | val queue = makeQueue() 142 | val reader = queue.reader("") 143 | val blockingQueue = makeTransactionalBlockingQueue(queue) 144 | 145 | assert(reader.items === 3) 146 | assert(reader.openItems === 0) 147 | 148 | val txn1 = blockingQueue.get(Before(100.seconds.fromNow))().get 149 | val txn2 = blockingQueue.get()().get 150 | 151 | assert(txn1.item === "first") 152 | assert(txn2.item === "second") 153 | 154 | assert(reader.items === 3) 155 | assert(reader.openItems === 2) 156 | 157 | txn1.rollback() 158 | 159 | assert(reader.items === 3) 160 | assert(reader.openItems === 1) 161 | 162 | txn2.commit() 163 | 164 | assert(reader.items === 2) 165 | assert(reader.openItems === 0) 166 | 167 | val txn3 = blockingQueue.get()().get 168 | 169 | assert(txn3.item === "first") 170 | txn3.commit() 171 | 172 | blockingQueue.close() 173 | } 174 | 175 | it("allows open transactions to be committed after poll") { 176 | val queue = makeQueue() 177 | val reader = queue.reader("") 178 | val blockingQueue = makeTransactionalBlockingQueue(queue) 179 | 180 | assert(reader.items === 3) 181 | assert(reader.openItems === 0) 182 | 183 | val txn = blockingQueue.poll()().get 184 | 185 | assert(txn.item === "first") 186 | 187 | assert(reader.items === 3) 188 | assert(reader.openItems === 1) 189 | 190 | txn.commit() 191 | 192 | assert(reader.items === 2) 193 | assert(reader.openItems === 0) 194 | 195 | blockingQueue.close() 196 | } 197 | 198 | it("allows open transactions to be rolled back after poll") { 199 | val queue = makeQueue() 200 | val reader = queue.reader("") 201 | val blockingQueue = makeTransactionalBlockingQueue(queue) 202 | 203 | assert(reader.items === 3) 204 | assert(reader.openItems === 0) 205 | 206 | val txn1 = blockingQueue.poll()().get 207 | 208 | assert(txn1.item === "first") 209 | 210 | assert(reader.items === 3) 211 | assert(reader.openItems === 1) 212 | 213 | txn1.rollback() 214 | 215 | assert(reader.items === 3) 216 | assert(reader.openItems === 0) 217 | 218 | val txn2 = blockingQueue.get()().get 219 | 220 | assert(txn2.item === "first") 221 | txn2.commit() 222 | 223 | blockingQueue.close() 224 | } 225 | } 226 | } 227 | -------------------------------------------------------------------------------- /src/test/scala/com/twitter/libkestrel/MemoryMappedFileSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Twitter, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | * not use this file except in compliance with the License. You may obtain 6 | * a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twitter.libkestrel 18 | 19 | import com.twitter.conversions.storage._ 20 | import com.twitter.util.StorageUnit 21 | import java.io.{File, IOException} 22 | import java.nio.ByteBuffer 23 | import org.scalatest.FunSpec 24 | import org.scalatest.matchers.{Matcher, MatchResult, ShouldMatchers} 25 | 26 | class MemoryMappedFileSpec extends FunSpec with ResourceCheckingSuite with ShouldMatchers with TempFolder with TestLogging { 27 | import FileHelper._ 28 | 29 | val mmapSize = 16.kilobytes 30 | 31 | def writeMappedFile(file: File, size: StorageUnit, data: Int) { 32 | val mmap = MemoryMappedFile.map(file, size) 33 | writeMappedFile(mmap, data) 34 | } 35 | 36 | def writeMappedFile(mmap: MemoryMappedFile, data: Int, limitOption: Option[StorageUnit] = None) { 37 | var limit = limitOption map { _.inBytes.toInt } orElse { Some(Int.MaxValue) } get 38 | val buffer = mmap.buffer() 39 | try { 40 | while (buffer.remaining >= 4 && limit > 0) { 41 | buffer.putInt(data) 42 | limit -= 4 43 | } 44 | mmap.force() 45 | } finally { 46 | mmap.close() 47 | } 48 | } 49 | 50 | describe("MemoryMappedFile") { 51 | describe("read/write") { 52 | it("mapping can be created") { 53 | val testFile = new File(testFolder, "create") 54 | val mmap = MemoryMappedFile.map(testFile, mmapSize) 55 | try { 56 | testFile.exists should equal(true) 57 | testFile.length should equal(mmapSize.inBytes) 58 | } finally { 59 | mmap.close() 60 | } 61 | } 62 | 63 | it("mapping can be modified") { 64 | val testFile = new File(testFolder, "modify") 65 | writeMappedFile(testFile, mmapSize, 0xdeadbeef) 66 | 67 | val bytes = readFile(testFile) 68 | val readBuffer = ByteBuffer.wrap(bytes) 69 | readBuffer.remaining should equal(mmapSize.inBytes) 70 | while (readBuffer.remaining >= 4) readBuffer.getInt() should equal(0xdeadbeef) 71 | } 72 | 73 | it("truncates existing files") { 74 | val testFile = new File(testFolder, "truncate") 75 | writeMappedFile(testFile, mmapSize, 0xdeadbeef) 76 | testFile.length should equal(mmapSize.inBytes) 77 | 78 | val mmap = MemoryMappedFile.map(testFile, 8.kilobytes, true) 79 | writeMappedFile(mmap, 0xabadcafe) 80 | 81 | val bytes = readFile(testFile) 82 | val readBuffer = ByteBuffer.wrap(bytes) 83 | readBuffer.remaining should equal(8.kilobytes.inBytes) 84 | while (readBuffer.remaining >= 4) readBuffer.getInt() should equal(0xabadcafe) 85 | } 86 | 87 | it("leaves unwritten space zeroed") { 88 | val testFile = new File(testFolder, "zero") 89 | val mmap = MemoryMappedFile.map(testFile, mmapSize) 90 | writeMappedFile(mmap, 0xdeadbeef, Some(8.kilobytes)) 91 | 92 | val bytes = readFile(testFile) 93 | val readBuffer = ByteBuffer.wrap(bytes) 94 | readBuffer.remaining should equal(mmapSize.inBytes) 95 | val deadbeefSlice = readBuffer.slice() 96 | deadbeefSlice.limit(8.kilobytes.inBytes.toInt) 97 | while (deadbeefSlice.remaining >= 4) deadbeefSlice.getInt() should equal(0xdeadbeef) 98 | 99 | val zeroedSlice = readBuffer.slice() 100 | zeroedSlice.position(8.kilobytes.inBytes.toInt) 101 | while (zeroedSlice.remaining >= 4) zeroedSlice.getInt() should equal(0) 102 | } 103 | 104 | it("should return different mapping for a file opened at different times") { 105 | val testFile = new File(testFolder, "open-close-open") 106 | 107 | val mmap1 = MemoryMappedFile.map(testFile, mmapSize) 108 | mmap1.close() 109 | 110 | val mmap2 = MemoryMappedFile.map(testFile, mmapSize) 111 | 112 | try { 113 | mmap1 should not be theSameInstanceAs (mmap2) 114 | } finally { 115 | mmap2.close() 116 | } 117 | } 118 | 119 | it("should throw if the same file is mapped twice") { 120 | val testFile = new File(testFolder, "multi-open") 121 | val mmap = MemoryMappedFile.map(testFile, mmapSize) 122 | 123 | try { 124 | evaluating { MemoryMappedFile.map(testFile, 8.kilobytes) } should produce [IOException] 125 | } finally { 126 | mmap.close() 127 | } 128 | } 129 | 130 | it("should throw if buffer is accessed after close") { 131 | val testFile = new File(testFolder, "closed") 132 | val mmap = MemoryMappedFile.map(testFile, mmapSize) 133 | mmap.close() 134 | 135 | evaluating { mmap.buffer() } should produce [NullPointerException] 136 | } 137 | 138 | it("should throw when closing the same file multiple times") { 139 | val testFile = new File(testFolder, "double-close") 140 | val mmap = MemoryMappedFile.map(testFile, mmapSize) 141 | mmap.close() 142 | 143 | evaluating { mmap.close() } should produce [IOException] 144 | } 145 | 146 | it("should preventing mapping files larger than 2 GB") { 147 | val testFile = new File(testFolder, "huge") 148 | val size = (2.gigabytes.inBytes + 1).bytes 149 | evaluating { MemoryMappedFile.map(testFile, size) } should produce [IOException] 150 | } 151 | } 152 | 153 | describe("read-only maps") { 154 | def createTestFile(name: String, size: StorageUnit): File = { 155 | val testFile = new File(testFolder, name) 156 | val bytes = new Array[Byte](size.inBytes.toInt) 157 | Some(ByteBuffer.wrap(bytes)).foreach { b => while (b.remaining >= 4) b.putInt(0xdeadbeef) } 158 | writeFile(testFile, bytes) 159 | testFile 160 | } 161 | 162 | it("can be created and read") { 163 | val testFile = createTestFile("create-ro", mmapSize) 164 | val mmap = MemoryMappedFile.readOnlyMap(testFile) 165 | val buffer = mmap.buffer() 166 | try { 167 | buffer.remaining should equal(mmapSize.inBytes) 168 | while (buffer.remaining >= 4) buffer.getInt() should equal(0xdeadbeef) 169 | } finally { 170 | mmap.close() 171 | } 172 | } 173 | 174 | it("should not return the same mapping for a file mapped by multiple readers") { 175 | val testFile = createTestFile("create-ro-multi", mmapSize) 176 | val mmap1 = MemoryMappedFile.readOnlyMap(testFile) 177 | val mmap2 = MemoryMappedFile.readOnlyMap(testFile) 178 | 179 | try { 180 | mmap1 should not be theSameInstanceAs (mmap2) 181 | } finally { 182 | mmap1.close() 183 | mmap2.close() 184 | } 185 | } 186 | 187 | it("should throw if buffer is accessed after close") { 188 | val testFile = createTestFile("closed-ro", mmapSize) 189 | val mmap = MemoryMappedFile.readOnlyMap(testFile) 190 | mmap.close() 191 | 192 | evaluating { mmap.buffer() } should produce [NullPointerException] 193 | } 194 | 195 | it("should throw when closing the same file multiple times") { 196 | val testFile = createTestFile("double-close-ro", mmapSize) 197 | val mmap = MemoryMappedFile.readOnlyMap(testFile) 198 | mmap.close() 199 | 200 | evaluating { mmap.close() } should produce [IOException] 201 | } 202 | 203 | it("should allow a file to be mapped for writing and reading simultaneously") { 204 | val testFile = createTestFile("read-write", mmapSize) 205 | val writeMapping = MemoryMappedFile.map(testFile, mmapSize) 206 | try { 207 | val writeBuffer = writeMapping.buffer() 208 | val readMapping = MemoryMappedFile.readOnlyMap(testFile) 209 | try { 210 | val readBuffer = readMapping.buffer() 211 | readBuffer.getInt(0) should equal(0xdeadbeef) 212 | 213 | writeBuffer.putInt(0x55555555) 214 | writeMapping.force() 215 | 216 | readBuffer.getInt(0) should equal(0x55555555) 217 | } finally { 218 | readMapping.close() 219 | } 220 | } finally { 221 | writeMapping.close() 222 | } 223 | } 224 | 225 | it("should throw if the file is opened for read before write") { 226 | val testFile = createTestFile("read-before-write", mmapSize) 227 | val readMapping = MemoryMappedFile.readOnlyMap(testFile) 228 | try { 229 | evaluating { MemoryMappedFile.map(testFile, mmapSize) } should produce [IOException] 230 | } finally { 231 | readMapping.close() 232 | } 233 | } 234 | } 235 | } 236 | } 237 | -------------------------------------------------------------------------------- /src/test/scala/com/twitter/libkestrel/PeriodicSyncFileSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2011 Twitter, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | * not use this file except in compliance with the License. You may obtain 6 | * a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twitter.libkestrel 18 | 19 | import com.twitter.conversions.time._ 20 | import com.twitter.util.Duration 21 | import java.util.concurrent._ 22 | import java.util.concurrent.atomic.AtomicInteger 23 | import org.scalatest.{AbstractSuite, BeforeAndAfter, FunSpec, Suite} 24 | import org.scalatest.matchers.{Matcher, MatchResult, ShouldMatchers} 25 | 26 | class PeriodicSyncFileSpec extends FunSpec with ResourceCheckingSuite with ShouldMatchers with TempFolder with TestLogging with BeforeAndAfter { 27 | describe("PeriodicSyncTask") { 28 | var scheduler: ScheduledThreadPoolExecutor = null 29 | var invocations: AtomicInteger = null 30 | var syncTask: PeriodicSyncTask = null 31 | 32 | before { 33 | scheduler = new ScheduledThreadPoolExecutor(4) 34 | invocations = new AtomicInteger(0) 35 | syncTask = new PeriodicSyncTask(scheduler, 0.milliseconds, 20.milliseconds) { 36 | override def run() { 37 | invocations.incrementAndGet 38 | } 39 | } 40 | } 41 | 42 | after { 43 | scheduler.shutdown() 44 | scheduler.awaitTermination(5, TimeUnit.SECONDS) 45 | } 46 | 47 | it("only starts once") { 48 | val (_, duration) = Duration.inMilliseconds { 49 | syncTask.start() 50 | syncTask.start() 51 | Thread.sleep(100) 52 | syncTask.stop() 53 | } 54 | 55 | val expectedInvocations = duration.inMilliseconds / 20 56 | assert(invocations.get <= expectedInvocations * 3 / 2) 57 | } 58 | 59 | it("stops") { 60 | syncTask.start() 61 | Thread.sleep(100) 62 | syncTask.stop() 63 | val invocationsPostTermination = invocations.get 64 | Thread.sleep(100) 65 | assert(invocations.get === invocationsPostTermination) 66 | } 67 | 68 | it("stop given a condition") { 69 | syncTask.start() 70 | Thread.sleep(100) 71 | 72 | val invocationsPreStop = invocations.get 73 | syncTask.stopIf { false } 74 | Thread.sleep(100) 75 | 76 | val invocationsPostIgnoredStop = invocations.get 77 | syncTask.stopIf { true } 78 | Thread.sleep(100) 79 | 80 | val invocationsPostStop = invocations.get 81 | Thread.sleep(100) 82 | 83 | assert(invocationsPreStop > 0) // did something 84 | assert(invocationsPostIgnoredStop > invocationsPreStop) // kept going 85 | assert(invocationsPostStop >= invocationsPostIgnoredStop) // maybe did more 86 | assert(invocations.get === invocationsPostStop) // stopped 87 | } 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /src/test/scala/com/twitter/libkestrel/ResourceCheckingSuite.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Twitter, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | * not use this file except in compliance with the License. You may obtain 6 | * a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twitter.libkestrel 18 | 19 | import org.scalatest._ 20 | 21 | trait ResourceCheckingSuite extends AbstractSuite { self: Suite => 22 | abstract override def withFixture(test: NoArgTest) { 23 | try { 24 | super.withFixture(test) 25 | assert(MemoryMappedFile.openFiles.isEmpty, 26 | "MemoryMappedFile.openFiles is not empty: %s (did you forget to close a JournaledQueue?)".format(MemoryMappedFile.openFiles.mkString(", "))) 27 | } finally { 28 | MemoryMappedFile.reset() 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/test/scala/com/twitter/libkestrel/TempFolder.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2011 Twitter, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | * not use this file except in compliance with the License. You may obtain 6 | * a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twitter.libkestrel 18 | 19 | import java.io.File 20 | import org.scalatest._ 21 | import com.twitter.io.Files 22 | 23 | trait TempFolder extends AbstractSuite { self: Suite => 24 | var testFolder: File = _ 25 | 26 | abstract override def withFixture(test: NoArgTest) { 27 | val tempFolder = System.getProperty("java.io.tmpdir") 28 | var folder: File = null 29 | do { 30 | folder = new File(tempFolder, "scala-test-" + System.currentTimeMillis) 31 | } while (! folder.mkdir) 32 | testFolder = folder 33 | try { 34 | super.withFixture(test) 35 | } finally { 36 | Files.delete(testFolder) 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/test/scala/com/twitter/libkestrel/TestLogging.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2011 Twitter, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | * not use this file except in compliance with the License. You may obtain 6 | * a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twitter.libkestrel 18 | 19 | import com.twitter.logging._ 20 | import java.util.{logging => jlogging} 21 | import org.scalatest._ 22 | 23 | trait TestLogging extends AbstractSuite { self: Suite => 24 | val logLevel = Logger.levelNames(Option[String](System.getenv("log")).getOrElse("FATAL").toUpperCase) 25 | 26 | private val rootLog = Logger.get("") 27 | private var oldLevel: jlogging.Level = _ 28 | 29 | abstract override def withFixture(test: NoArgTest) { 30 | oldLevel = rootLog.getLevel() 31 | rootLog.setLevel(logLevel) 32 | rootLog.addHandler(new ConsoleHandler(new Formatter(), None)) 33 | try { 34 | super.withFixture(test) 35 | } finally { 36 | rootLog.clearHandlers() 37 | rootLog.setLevel(oldLevel) 38 | } 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/test/scala/com/twitter/libkestrel/load/FloodTest.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2011 Twitter, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | * not use this file except in compliance with the License. You may obtain 6 | * a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twitter.libkestrel 18 | package load 19 | 20 | import java.util.concurrent.{CountDownLatch, ConcurrentHashMap} 21 | import java.util.concurrent.atomic.{AtomicInteger, AtomicLongArray, AtomicIntegerArray} 22 | import scala.collection.JavaConverters._ 23 | import com.twitter.conversions.time._ 24 | import com.twitter.util.Time 25 | 26 | object FloodTest extends LoadTesting { 27 | val description = "put & get items to/from a queue as fast as possible" 28 | 29 | var writerThreadCount = Runtime.getRuntime.availableProcessors() 30 | var readerThreadCount = Runtime.getRuntime.availableProcessors() 31 | var testTime = 10.seconds 32 | var pollPercent = 25 33 | var maxItems = 10000 34 | var minBytes = 0 35 | var validate = false 36 | 37 | val parser = new CommonOptionParser("qtest flood") { 38 | common() 39 | opt("w", "writers", "", "set number of writer threads (default: %d)".format(writerThreadCount), { x: String => 40 | writerThreadCount = x.toInt 41 | }) 42 | opt("r", "readers", "", "set number of reader threads (default: %d)".format(readerThreadCount), { x: String => 43 | readerThreadCount = x.toInt 44 | }) 45 | opt("t", "time", "", "run test for specified time in milliseconds (default: %d)".format(testTime.inMilliseconds), { x: String => 46 | testTime = x.toInt.milliseconds 47 | }) 48 | opt("p", "percent", "", "set percentage of time to use poll instead of get (default: %d)".format(pollPercent), { x: String => 49 | pollPercent = x.toInt 50 | }) 51 | opt("x", "target", "", "slow down the writer threads a bit when the queue reaches this size (default: %d)".format(maxItems), { x: String => 52 | maxItems = x.toInt 53 | }) 54 | opt("b", "bytes", "", "size in bytes of payload (default: %d == variable, just big enough for validation)".format(minBytes), { x: String => 55 | minBytes = x.toInt 56 | }) 57 | opt("V", "validate", "validate items afterwards (makes it much slower)", { validate = true; () }) 58 | } 59 | 60 | def apply(args: List[String]) { 61 | setup() 62 | if (!parser.parse(args)) { 63 | System.exit(1) 64 | } 65 | val queue = makeQueue() 66 | 67 | println("flood: writers=%d, readers=%d, item_limit=%d, run=%s, poll_percent=%d, max_items=%d, validate=%s, queue=%s".format( 68 | writerThreadCount, readerThreadCount, itemLimit, testTime, pollPercent, maxItems, validate, queue.toDebug 69 | )) 70 | maybeSleep() 71 | 72 | val startLatch = new CountDownLatch(1) 73 | val lastId = new AtomicIntegerArray(writerThreadCount) 74 | val writerDone = new AtomicIntegerArray(writerThreadCount) 75 | val deadline = testTime.fromNow 76 | 77 | val messageFormat = minBytes match { 78 | case b if b <= 3 => "%d/%d" 79 | case b if b <= 15 => "%d/%0" + (b - 2) + "d" 80 | case b => val spaces = ("%-" + (b - 15) + "s").format(" "); "%d/%010d/[" + spaces + "]" 81 | } 82 | 83 | val writers = (0 until writerThreadCount).map { threadId => 84 | new Thread() { 85 | setName("writer-%d".format(threadId)) 86 | override def run() { 87 | var id = 0 88 | while (deadline > Time.now) { 89 | queue.put(messageFormat.format(threadId, id)) 90 | id += 1 91 | if (queue.size > maxItems) Thread.sleep(5) 92 | } 93 | lastId.set(threadId, id) 94 | writerDone.set(threadId, 1) 95 | } 96 | } 97 | }.toList 98 | 99 | val random = new XorRandom() 100 | val received = (0 until writerThreadCount).map { i => new ConcurrentHashMap[Int, AtomicInteger] }.toArray 101 | val readCounts = new AtomicIntegerArray(readerThreadCount) 102 | val readTimings = new AtomicLongArray(readerThreadCount) 103 | val readPolls = new AtomicIntegerArray(readerThreadCount) 104 | 105 | def writersDone(): Boolean = { 106 | (0 until writerThreadCount).foreach { i => 107 | if (writerDone.get(i) != 1) return false 108 | } 109 | true 110 | } 111 | 112 | val readers = (0 until readerThreadCount).map { threadId => 113 | new Thread() { 114 | setName("reader-%d".format(threadId)) 115 | override def run() { 116 | startLatch.await() 117 | val startTime = System.nanoTime 118 | var count = 0 119 | var polls = 0 120 | while (deadline > Time.now || queue.size > 0 || !writersDone()) { 121 | val item = if (random() % 100 < pollPercent) { 122 | polls += 1 123 | queue.poll()() 124 | } else { 125 | queue.get(Before(1.millisecond.fromNow))() 126 | } 127 | if (item.isDefined) count += 1 128 | if (validate) { 129 | item.map { x => 130 | x.split("/").slice(0, 2).map { _.toInt }.toList match { 131 | case List(tid, id) => 132 | received(tid).putIfAbsent(id, new AtomicInteger) 133 | received(tid).get(id).incrementAndGet() 134 | case _ => 135 | println("*** GIBBERISH RECEIVED") 136 | } 137 | } 138 | } 139 | } 140 | val timing = System.nanoTime - startTime 141 | readCounts.set(threadId, count) 142 | readTimings.set(threadId, timing) 143 | readPolls.set(threadId, polls) 144 | } 145 | } 146 | }.toList 147 | 148 | writers.foreach { _.start() } 149 | readers.foreach { _.start() } 150 | startLatch.countDown() 151 | 152 | while (deadline > Time.now) { 153 | Thread.sleep(1000) 154 | println(queue.toDebug) 155 | } 156 | 157 | queue.evictWaiters() 158 | readers.foreach { _.join() } 159 | writers.foreach { _.join() } 160 | 161 | (0 until readerThreadCount).foreach { threadId => 162 | val t = readTimings.get(threadId).toDouble / readCounts.get(threadId) 163 | val pollPercent = readPolls.get(threadId).toDouble * 100 / readCounts.get(threadId) 164 | println("%3d: %5.0f nsec/read (%3.0f%% polls)".format(threadId, t, pollPercent)) 165 | } 166 | 167 | if (validate) { 168 | var ok = true 169 | (0 until writerThreadCount).foreach { threadId => 170 | if (received(threadId).size != lastId.get(threadId)) { 171 | println("*** Mismatched count for writer %d: wrote=%d read=%d".format( 172 | threadId, lastId.get(threadId), received(threadId).size 173 | )) 174 | (0 until lastId.get(threadId)).foreach { id => 175 | val atom = received(threadId).get(id) 176 | if (atom eq null) { 177 | print("%d(0) ".format(id)) 178 | } else if (atom.get() != 1) { 179 | print("%d(%d) ".format(id, atom.get())) 180 | } 181 | } 182 | println() 183 | ok = false 184 | } else { 185 | println("writer %d wrote %d".format(threadId, lastId.get(threadId))) 186 | } 187 | received(threadId).asScala.foreach { case (id, count) => 188 | if (count.get() != 1) { 189 | println("*** Writer %d's item %d expected 1 receive, got %d".format( 190 | threadId, id, count.get() 191 | )) 192 | ok = false 193 | } 194 | } 195 | } 196 | if (ok) println("All good. :)") 197 | } else { 198 | val totalRead = (0 until readerThreadCount).foldLeft(0) { (total, threadId) => 199 | total + readCounts.get(threadId) 200 | } 201 | val totalWritten = (0 until writerThreadCount).foldLeft(0) { (total, threadId) => 202 | total + lastId.get(threadId) 203 | } 204 | println("Writer wrote %d, readers received %d".format(totalWritten, totalRead)) 205 | if (totalRead == totalWritten) { 206 | println("All good. :)") 207 | } else { 208 | println("*** counts did not match") 209 | } 210 | } 211 | 212 | queue.close() 213 | } 214 | } 215 | -------------------------------------------------------------------------------- /src/test/scala/com/twitter/libkestrel/load/LoadTesting.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2011 Twitter, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | * not use this file except in compliance with the License. You may obtain 6 | * a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twitter.libkestrel 18 | package load 19 | 20 | import com.twitter.logging.{ConsoleHandler, FileHandler, Formatter, Logger, Policy} 21 | import com.twitter.util.{JavaTimer, Timer} 22 | import java.io.{File, FilenameFilter} 23 | import java.nio.ByteBuffer 24 | import java.util.concurrent.ScheduledThreadPoolExecutor 25 | import scopt.OptionParser 26 | import config._ 27 | 28 | trait LoadTesting { 29 | implicit val javaTimer: Timer = new JavaTimer() 30 | val scheduler = new ScheduledThreadPoolExecutor(1) 31 | implicit val stringCodec: Codec[String] = new Codec[String] { 32 | def encode(item: String) = ByteBuffer.wrap(item.getBytes) 33 | def decode(data: ByteBuffer) = { 34 | val bytes = new Array[Byte](data.remaining) 35 | data.get(bytes) 36 | new String(bytes) 37 | } 38 | } 39 | 40 | sealed trait QueueType 41 | object QueueType { 42 | case object Simple extends QueueType 43 | case object Concurrent extends QueueType 44 | case object Journaled extends QueueType 45 | } 46 | 47 | var queueType: QueueType = QueueType.Concurrent 48 | var itemLimit = 10000 49 | var sleep = 0 50 | var preClean = false 51 | 52 | class CommonOptionParser(name: String) extends OptionParser(name) { 53 | def common() { 54 | help(None, "help", "show this help screen") 55 | opt("S", "simple", "use old simple synchronized-based queue", { queueType = QueueType.Simple; () }) 56 | opt("J", "journal", "use journaled queue in /tmp", { queueType = QueueType.Journaled; () }) 57 | opt("L", "limit", "", "limit total queue size (default: %d)".format(itemLimit), { x: String => 58 | itemLimit = x.toInt 59 | }) 60 | opt("z", "sleep", "number of seconds to sleep before starting (for profiling) (default: %d)".format(sleep), { x: String => 61 | sleep = x.toInt 62 | }) 63 | opt("c", "clean", "delete any stale queue files ahead of starting the test (default off)", { 64 | preClean = true 65 | }) 66 | } 67 | } 68 | 69 | def makeQueue(): BlockingQueue[String] = { 70 | queueType match { 71 | case QueueType.Simple => { 72 | SimpleBlockingQueue[String](itemLimit, ConcurrentBlockingQueue.FullPolicy.DropOldest) 73 | } 74 | case QueueType.Concurrent => { 75 | ConcurrentBlockingQueue[String](itemLimit, ConcurrentBlockingQueue.FullPolicy.DropOldest) 76 | } 77 | case QueueType.Journaled => { 78 | val dir = new File("/tmp") 79 | val queueName = "test" 80 | if (preClean) { 81 | dir.listFiles(new FilenameFilter { 82 | val prefix = queueName + "." 83 | def accept(dir: File, name: String) = name.startsWith(prefix) 84 | }).foreach { _.delete() } 85 | } 86 | 87 | new JournaledQueue(new JournaledQueueConfig( 88 | name = queueName, 89 | defaultReaderConfig = new JournaledQueueReaderConfig( 90 | maxItems = itemLimit, 91 | fullPolicy = ConcurrentBlockingQueue.FullPolicy.DropOldest 92 | ) 93 | ), dir, javaTimer, scheduler).toBlockingQueue[String] 94 | } 95 | } 96 | } 97 | 98 | def maybeSleep() { 99 | if (sleep > 0) { 100 | println("Sleeping %d seconds...".format(sleep)) 101 | Thread.sleep(sleep * 1000) 102 | println("Okay.") 103 | } 104 | } 105 | 106 | def setup() { 107 | val logLevel = Logger.levelNames(Option[String](System.getenv("log")).getOrElse("FATAL").toUpperCase) 108 | val rootLog = Logger.get("") 109 | rootLog.setLevel(logLevel) 110 | System.getenv("logfile") match { 111 | case null => { 112 | rootLog.addHandler(new ConsoleHandler(new Formatter(), None)) 113 | } 114 | case filename => { 115 | rootLog.addHandler(new FileHandler(filename, Policy.Never, true, 0, new Formatter(), None)) 116 | } 117 | } 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /src/test/scala/com/twitter/libkestrel/load/PutTest.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2011 Twitter, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | * not use this file except in compliance with the License. You may obtain 6 | * a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twitter.libkestrel 18 | package load 19 | 20 | import java.io.File 21 | import java.util.concurrent.CountDownLatch 22 | import java.util.concurrent.atomic.AtomicReferenceArray 23 | import scala.collection.mutable 24 | import com.twitter.conversions.time._ 25 | 26 | object PutTest extends LoadTesting { 27 | val description = "write as many items as we can into a queue, concurrently, and time it" 28 | 29 | var threadCount = Runtime.getRuntime.availableProcessors() * 2 30 | var itemCount = 10000 31 | var cycles = 10 32 | 33 | val parser = new CommonOptionParser("qtest put") { 34 | common() 35 | opt("t", "threads", "", "set number of writer threads (default: %d)".format(threadCount), { x: String => 36 | threadCount = x.toInt 37 | }) 38 | opt("n", "items", "", "set number of items to write in each thread (default: %d)".format(itemCount), { x: String => 39 | itemCount = x.toInt 40 | }) 41 | opt("C", "cycles", "", "set number of test runs (for jit warmup) (default: %d)".format(cycles), { x: String => 42 | cycles = x.toInt 43 | }) 44 | } 45 | 46 | def cycle(queue: BlockingQueue[String]) { 47 | val startLatch = new CountDownLatch(1) 48 | val timings = new AtomicReferenceArray[Long](threadCount) 49 | val threads = (0 until threadCount).map { threadId => 50 | new Thread() { 51 | override def run() { 52 | startLatch.await() 53 | val startTime = System.nanoTime 54 | (0 until itemCount).foreach { id => 55 | queue.put(threadId + "/" + id) 56 | } 57 | val elapsed = System.nanoTime - startTime 58 | timings.set(threadId, elapsed) 59 | } 60 | } 61 | }.toList 62 | 63 | threads.foreach { _.start() } 64 | startLatch.countDown() 65 | threads.foreach { _.join() } 66 | 67 | val totalTime = (0 until threadCount).map { tid => timings.get(tid) }.sum 68 | val totalItems = threadCount * itemCount 69 | println("%6.2f nsec/item".format(totalTime.toDouble / totalItems)) 70 | 71 | println(" " + queue.toDebug) 72 | 73 | // drain the queue and verify that items look okay and have a loose ordering. 74 | val itemSets = (0 until threadCount).map { i => new mutable.HashSet[Int] }.toArray 75 | while (queue.size > 0) { 76 | queue.get()().get.split("/").map { _.toInt }.toList match { 77 | case List(tid, id) => 78 | itemSets(tid) += id 79 | case _ => 80 | println("*** GIBBERISH RECEIVED") 81 | } 82 | } 83 | itemSets.indices.foreach { tid => 84 | val list = itemSets(tid).toList.sorted 85 | // with a large number of threads, some starvation will occur. 86 | if (list.size > 0) { 87 | if (list.head + list.size - 1 != list.last) { 88 | println("*** Thread %d contains [%d,%d) size %d".format(tid, list.head, list.last + 1, list.size)) 89 | } 90 | if (list.last != itemCount - 1) { 91 | println("*** Thread %d tail item %d is not %d".format(tid, list.last, itemCount - 1)) 92 | } 93 | } 94 | } 95 | } 96 | 97 | def apply(args: List[String]) { 98 | setup() 99 | if (!parser.parse(args)) { 100 | System.exit(1) 101 | } 102 | val queue = makeQueue() 103 | 104 | println("put: writers=%d, items=%d, item_limit=%d, cycles=%d, queue=%s".format( 105 | threadCount, itemCount, itemLimit, cycles, queue.toDebug 106 | )) 107 | (0 until cycles).foreach { n => cycle(queue) } 108 | queue.close() 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /src/test/scala/com/twitter/libkestrel/load/QTest.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.libkestrel.load 2 | 3 | import scala.collection.JavaConverters._ 4 | import com.twitter.conversions.time._ 5 | import java.util.concurrent.atomic._ 6 | import java.util.concurrent.ConcurrentHashMap 7 | import com.twitter.util.{JavaTimer, Timer, Time, TimeoutException} 8 | 9 | class XorRandom { 10 | var seed: Int = (System.nanoTime / 1000).toInt 11 | def apply(): Int = { 12 | seed ^= (seed << 13) 13 | seed ^= (seed >> 17) 14 | seed ^= (seed << 5) 15 | seed & 0x7fffffff 16 | } 17 | } 18 | 19 | object QTest { 20 | val version = "20111116" 21 | 22 | def usage() { 23 | Console.println() 24 | Console.println("usage: qtest [options]") 25 | Console.println(" run concurrency load tests against ConcurrentBlockingQueue") 26 | Console.println() 27 | Console.println("tests:") 28 | Console.println(" timeout") 29 | Console.println(" %s".format(TimeoutTest.description)) 30 | Console.println(" put") 31 | Console.println(" %s".format(PutTest.description)) 32 | Console.println(" flood") 33 | Console.println(" %s".format(FloodTest.description)) 34 | Console.println() 35 | Console.println("use 'qtest --help' to see options for a specific test") 36 | Console.println() 37 | Console.println("version %s".format(version)) 38 | } 39 | 40 | def main(args: Array[String]) = { 41 | args.toList match { 42 | case "--help" :: xs => 43 | usage() 44 | case "timeout" :: xs => 45 | TimeoutTest(xs) 46 | case "put" :: xs => 47 | PutTest(xs) 48 | case "flood" :: xs => 49 | FloodTest(xs) 50 | case _ => 51 | usage() 52 | } 53 | System.exit(0) 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/test/scala/com/twitter/libkestrel/load/TimeoutTest.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2011 Twitter, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | * not use this file except in compliance with the License. You may obtain 6 | * a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twitter.libkestrel 18 | package load 19 | 20 | import com.twitter.conversions.time._ 21 | import com.twitter.util._ 22 | import java.util.concurrent.ConcurrentHashMap 23 | import java.util.concurrent.atomic.{AtomicInteger, AtomicIntegerArray} 24 | import scala.collection.JavaConverters._ 25 | 26 | object TimeoutTest extends LoadTesting { 27 | val description = "write items into a queue at a slow rate while a bunch of reader threads stampede" 28 | 29 | var writerThreadCount = 1 30 | var readerThreadCount = 100 31 | var writeRate = 10.milliseconds 32 | var readTimeoutLow = 5.milliseconds 33 | var readTimeoutHigh = 15.milliseconds 34 | var testTime = 10.seconds 35 | 36 | val parser = new CommonOptionParser("qtest timeout") { 37 | common() 38 | opt("w", "writers", "", "set number of writer threads (default: %d)".format(writerThreadCount), { x: String => 39 | writerThreadCount = x.toInt 40 | }) 41 | opt("r", "readers", "", "set number of reader threads (default: %d)".format(readerThreadCount), { x: String => 42 | readerThreadCount = x.toInt 43 | }) 44 | opt("d", "delay", "", "delay between writes (default: %d)".format(writeRate.inMilliseconds), { x: String => 45 | writeRate = x.toInt.milliseconds 46 | }) 47 | opt("L", "low", "", "low end of the random reader timeout (default: %d)".format(readTimeoutLow.inMilliseconds), { x: String => 48 | readTimeoutLow = x.toInt.milliseconds 49 | }) 50 | opt("H", "high", "", "high end of the random reader timeout (default: %d)".format(readTimeoutHigh.inMilliseconds), { x: String => 51 | readTimeoutHigh = x.toInt.milliseconds 52 | }) 53 | opt("t", "timeout", "", "run test for this long (default: %d)".format(testTime.inMilliseconds), { x: String => 54 | testTime = x.toInt.milliseconds 55 | }) 56 | } 57 | 58 | def apply(args: List[String]) { 59 | setup() 60 | if (!parser.parse(args)) { 61 | System.exit(1) 62 | } 63 | val queue = makeQueue() 64 | 65 | println("timeout: writers=%d, readers=%d, item_limit=%d, write_rate=%s, read_timeout=(%s, %s), run=%s, queue=%s".format( 66 | writerThreadCount, readerThreadCount, itemLimit, writeRate, readTimeoutLow, readTimeoutHigh, testTime, queue.toDebug 67 | )) 68 | 69 | val lastId = new AtomicIntegerArray(writerThreadCount) 70 | val deadline = testTime.fromNow 71 | val readerDeadline = deadline + readTimeoutHigh * 2 72 | 73 | val writers = (0 until writerThreadCount).map { threadId => 74 | new Thread() { 75 | override def run() { 76 | var id = 0 77 | while (deadline > Time.now) { 78 | Thread.sleep(writeRate.inMilliseconds) 79 | if (deadline > Time.now) { 80 | queue.put(threadId + "/" + id) 81 | id += 1 82 | } 83 | } 84 | lastId.set(threadId, id) 85 | } 86 | } 87 | }.toList 88 | 89 | val random = new XorRandom() 90 | val range = (readTimeoutHigh.inMilliseconds - readTimeoutLow.inMilliseconds).toInt 91 | val received = (0 until writerThreadCount).map { i => new ConcurrentHashMap[Int, AtomicInteger] }.toArray 92 | 93 | val readers = (0 until readerThreadCount).map { threadId => 94 | new Thread() { 95 | override def run() { 96 | while (readerDeadline > Time.now) { 97 | val timeout = readTimeoutHigh + random() % (range + 1) 98 | val optItem = queue.get(Before(timeout.milliseconds.fromNow))() 99 | optItem match { 100 | case None => 101 | case Some(item) => { 102 | item.split("/").map { _.toInt }.toList match { 103 | case List(tid, id) => 104 | received(tid).putIfAbsent(id, new AtomicInteger) 105 | received(tid).get(id).incrementAndGet() 106 | case _ => 107 | println("*** GIBBERISH RECEIVED") 108 | } 109 | } 110 | } 111 | } 112 | } 113 | } 114 | }.toList 115 | 116 | readers.foreach { _.start() } 117 | writers.foreach { _.start() } 118 | 119 | while (deadline > Time.now) { 120 | Thread.sleep(1000) 121 | println(queue.toDebug) 122 | } 123 | 124 | readers.foreach { _.join() } 125 | writers.foreach { _.join() } 126 | 127 | var ok = true 128 | (0 until writerThreadCount).foreach { threadId => 129 | if (received(threadId).size != lastId.get(threadId)) { 130 | println("*** Mismatched count for writer %d: wrote=%d read=%d".format( 131 | threadId, lastId.get(threadId), received(threadId).size 132 | )) 133 | ok = false 134 | } else { 135 | println("writer %d wrote %d".format(threadId, lastId.get(threadId))) 136 | } 137 | received(threadId).asScala.foreach { case (id, count) => 138 | if (count.get() != 1) { 139 | println("*** Writer %d item %d expected 1 receive, got %d".format( 140 | threadId, id, count.get() 141 | )) 142 | ok = false 143 | } 144 | } 145 | } 146 | if (ok) println("All good. :)") 147 | } 148 | } 149 | --------------------------------------------------------------------------------