├── .gitignore ├── config ├── kestrel-test.conf └── kestrel.conf ├── src ├── scripts │ ├── qdump.rb │ └── kestrel.sh ├── main │ └── scala │ │ └── net │ │ └── lag │ │ └── kestrel │ │ ├── Time.scala │ │ ├── IoHandlerActorAdapter.scala │ │ ├── Kestrel.scala │ │ ├── memcache │ │ └── Decoder.scala │ │ ├── QueueCollection.scala │ │ ├── KestrelHandler.scala │ │ ├── Journal.scala │ │ └── PersistentQueue.scala └── test │ └── scala │ └── net │ └── lag │ ├── TestRunner.scala │ ├── TestHelper.scala │ ├── kestrel │ ├── TestClient.scala │ ├── load │ │ ├── PutMany.scala │ │ └── ManyClients.scala │ ├── QueueCollectionSpec.scala │ ├── memcache │ │ └── MemCacheCodecSpec.scala │ ├── ServerSpec.scala │ └── PersistentQueueSpec.scala │ └── FilterableSpecsFileRunner.scala ├── ant ├── clean.xml ├── docs.xml ├── test.xml ├── prepare.xml ├── bootstrap.xml ├── package.xml └── compile.xml ├── ivy ├── ivysettings.xml └── ivy.xml ├── LICENSE ├── tests.xml └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | kestrel.tmproj 3 | dist 4 | -------------------------------------------------------------------------------- /config/kestrel-test.conf: -------------------------------------------------------------------------------- 1 | 2 | 3 | filename = "/tmp/kestrel/kestrel.log" 4 | roll = "daily" 5 | # level = "debug" 6 | level = "info" 7 | 8 | 9 | # where to listen for connections: 10 | port = 22133 11 | host = "0.0.0.0" 12 | 13 | queue_path = "/tmp/kestrel" 14 | 15 | timeout = 10 16 | 17 | -------------------------------------------------------------------------------- /src/scripts/qdump.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | CMD_ADD = "\000" 4 | CMD_REMOVE = "\001" 5 | CMD_ADDX = "\002" 6 | 7 | f = File.open(ARGV[0], "r") 8 | while !f.eof 9 | b = f.read(1) 10 | if b == CMD_ADD 11 | len = f.read(4).unpack("I")[0] 12 | data = f.read(len) 13 | expire = data.unpack("I")[0] 14 | puts "add #{len} expire #{expire}" 15 | elsif b == CMD_REMOVE 16 | puts "remove" 17 | elsif b == CMD_ADDX 18 | len = f.read(4).unpack("I")[0] 19 | data = f.read(len) 20 | add_time, expire = data.unpack("QQ") 21 | puts "addx #{len} expire #{expire}" 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /ant/clean.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /config/kestrel.conf: -------------------------------------------------------------------------------- 1 | # kestrel config for a production system 2 | 3 | # where to listen for connections: 4 | port = 22133 5 | host = "0.0.0.0" 6 | 7 | 8 | filename = "/var/log/kestrel/kestrel.log" 9 | roll = "daily" 10 | level = "debug" 11 | 12 | 13 | queue_path = "/var/spool/kestrel" 14 | 15 | # when to timeout clients (seconds; 0 = never) 16 | timeout = 0 17 | 18 | # when a queue's journal reaches this size, the queue will wait until it 19 | # is empty, and will then rotate the journal. 20 | max_journal_size = 16277216 21 | 22 | # per-queue config 23 | 24 | 25 | # throw away any weather update that's been waiting in the queue for 1800 26 | # seconds (30 mins). if a client uses a shorter expiry, that's honored 27 | # instead. 28 | max_age = 1800 29 | 30 | # refuse SET operations when the queue would exceed this many items. 31 | max_items = 1500000 32 | 33 | 34 | -------------------------------------------------------------------------------- /ivy/ivysettings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | This software is licensed under the MIT license, quoted below. 2 | 3 | Copyright (c) 2008 Robey Pointer 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a 6 | copy of this software and associated documentation files (the "Software"), 7 | to deal in the Software without restriction, including without limitation 8 | the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 | and/or sell copies of the Software, and to permit persons to whom the 10 | Software is furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 | THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 | DEALINGS IN THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /ant/docs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /ivy/ivy.xml: -------------------------------------------------------------------------------- 1 | 2 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /src/main/scala/net/lag/kestrel/Time.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2008 Robey Pointer 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a 5 | * copy of this software and associated documentation files (the "Software"), 6 | * to deal in the Software without restriction, including without limitation 7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 | * and/or sell copies of the Software, and to permit persons to whom the 9 | * Software is furnished to do so, subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in 12 | * all copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 20 | * DEALINGS IN THE SOFTWARE. 21 | */ 22 | 23 | package net.lag.kestrel 24 | 25 | // stolen from nick's ruby version. 26 | // this lets unit tests muck around with temporality without calling Thread.sleep(). 27 | object Time { 28 | private var offset: Long = 0 29 | 30 | final def now() = System.currentTimeMillis + offset 31 | final def advance(msec: Int) = offset += msec 32 | } 33 | -------------------------------------------------------------------------------- /src/test/scala/net/lag/TestRunner.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2008 Robey Pointer 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a 5 | * copy of this software and associated documentation files (the "Software"), 6 | * to deal in the Software without restriction, including without limitation 7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 | * and/or sell copies of the Software, and to permit persons to whom the 9 | * Software is furnished to do so, subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in 12 | * all copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 20 | * DEALINGS IN THE SOFTWARE. 21 | */ 22 | 23 | package net.lag 24 | 25 | import net.lag.configgy.Configgy 26 | import net.lag.logging.Logger 27 | 28 | 29 | object TestRunner extends FilterableSpecsFileRunner("src/test/scala/**/*.scala") { 30 | // Configgy.configure("src/resources/test.conf") 31 | if (System.getProperty("debugtrace") == null) { 32 | Logger.get("").setLevel(Logger.FATAL) 33 | } else { 34 | Logger.get("").setLevel(Logger.TRACE) 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /ant/test.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 43 | 44 | 46 | 47 | 48 | 49 | 50 | -------------------------------------------------------------------------------- /ant/prepare.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | -------------------------------------------------------------------------------- /src/test/scala/net/lag/TestHelper.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2008 Robey Pointer 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a 5 | * copy of this software and associated documentation files (the "Software"), 6 | * to deal in the Software without restriction, including without limitation 7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 | * and/or sell copies of the Software, and to permit persons to whom the 9 | * Software is furnished to do so, subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in 12 | * all copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 20 | * DEALINGS IN THE SOFTWARE. 21 | */ 22 | 23 | package net.lag 24 | 25 | import java.io.File 26 | 27 | 28 | trait TestHelper { 29 | private val _folderName = new ThreadLocal[File] 30 | 31 | /** 32 | * Recursively delete a folder. Should be built in; bad java. 33 | */ 34 | def deleteFolder(folder: File): Unit = { 35 | val files = folder.listFiles 36 | if (files != null) { 37 | for (val f <- files) { 38 | if (f.isDirectory) { 39 | deleteFolder(f) 40 | } else { 41 | f.delete 42 | } 43 | } 44 | } 45 | folder.delete 46 | } 47 | 48 | def withTempFolder(f: => Any): Unit = { 49 | val tempFolder = System.getProperty("java.io.tmpdir") 50 | var folder: File = null 51 | do { 52 | folder = new File(tempFolder, "scala-test-" + System.currentTimeMillis) 53 | } while (! folder.mkdir) 54 | _folderName.set(folder) 55 | 56 | try { 57 | f 58 | } finally { 59 | deleteFolder(folder) 60 | } 61 | } 62 | 63 | def folderName = { _folderName.get.getPath } 64 | 65 | def canonicalFolderName = { _folderName.get.getCanonicalPath } 66 | } 67 | -------------------------------------------------------------------------------- /tests.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | Kestrel load tests 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | -------------------------------------------------------------------------------- /ant/bootstrap.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | -------------------------------------------------------------------------------- /src/scripts/kestrel.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # kestrel init.d script. 4 | # 5 | 6 | QUEUE_PATH="/var/spool/kestrel" 7 | KESTREL_HOME="/usr/local/kestrel" 8 | AS_USER="daemon" 9 | VERSION="0.5" 10 | DAEMON="/usr/local/bin/daemon" 11 | 12 | daemon_args="--name kestrel --pidfile /var/run/kestrel.pid" 13 | HEAP_OPTS="-Xmx2048m -Xms1024m -XX:NewSize=256m" 14 | # -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=9999 -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false 15 | JAVA_OPTS="-server -verbosegc -XX:+PrintGCDetails -XX:+UseConcMarkSweepGC -XX:+UseParNewGC $HEAP_OPTS" 16 | 17 | 18 | function running() { 19 | $DAEMON $daemon_args --running 20 | } 21 | 22 | function find_java() { 23 | if [ ! -z "$JAVA_HOME" ]; then 24 | return 25 | fi 26 | potential=$(ls -r1d /opt/jdk /System/Library/Frameworks/JavaVM.framework/Versions/CurrentJDK/Home /usr/java/default /usr/java/j* 2>/dev/null) 27 | for p in $potential; do 28 | if [ -x $p/bin/java ]; then 29 | JAVA_HOME=$p 30 | break 31 | fi 32 | done 33 | } 34 | 35 | 36 | # dirs under /var/run can go away between reboots. 37 | for p in /var/run/kestrel /var/log/kestrel $QUEUE_PATH; do 38 | if [ ! -d $p ]; then 39 | mkdir -p $p 40 | chmod 775 $p 41 | chown $AS_USER $p >/dev/null 2>&1 || true 42 | fi 43 | done 44 | 45 | find_java 46 | 47 | 48 | case "$1" in 49 | start) 50 | echo -n "Starting kestrel... " 51 | 52 | if [ ! -r $KESTREL_HOME/kestrel-$VERSION.jar ]; then 53 | echo "FAIL" 54 | echo "*** kestrel jar missing - not starting" 55 | exit 1 56 | fi 57 | if [ ! -x $JAVA_HOME/bin/java ]; then 58 | echo "FAIL" 59 | echo "*** $JAVA_HOME/bin/java doesn't exist -- check JAVA_HOME?" 60 | exit 1 61 | fi 62 | if running; then 63 | echo "already running." 64 | exit 0 65 | fi 66 | 67 | ulimit -n 8192 || echo -n " (no ulimit)" 68 | $DAEMON $daemon_args --user $AS_USER --stdout=/var/log/kestrel/stdout --stderr=/var/log/kestrel/error -- ${JAVA_HOME}/bin/java ${JAVA_OPTS} -jar ${KESTREL_HOME}/kestrel-${VERSION}.jar 69 | tries=0 70 | while ! running; do 71 | tries=$((tries + 1)) 72 | if [ $tries -ge 5 ]; then 73 | echo "FAIL" 74 | exit 1 75 | fi 76 | sleep 1 77 | done 78 | echo "done." 79 | ;; 80 | 81 | stop) 82 | echo -n "Stopping kestrel... " 83 | if ! running; then 84 | echo "wasn't running." 85 | exit 0 86 | fi 87 | 88 | (echo "shutdown"; sleep 2) | telnet localhost 22133 >/dev/null 2>&1 89 | tries=0 90 | while running; do 91 | tries=$((tries + 1)) 92 | if [ $tries -ge 5 ]; then 93 | echo "FAIL" 94 | exit 1 95 | fi 96 | sleep 1 97 | done 98 | echo "done." 99 | ;; 100 | 101 | status) 102 | if running; then 103 | echo "kestrel is running." 104 | else 105 | echo "kestrel is NOT running." 106 | fi 107 | ;; 108 | 109 | restart) 110 | $0 stop 111 | sleep 2 112 | $0 start 113 | ;; 114 | 115 | *) 116 | echo "Usage: /etc/init.d/kestrel {start|stop|restart|status}" 117 | exit 1 118 | ;; 119 | esac 120 | 121 | exit 0 122 | -------------------------------------------------------------------------------- /src/main/scala/net/lag/kestrel/IoHandlerActorAdapter.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2008 Robey Pointer 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a 5 | * copy of this software and associated documentation files (the "Software"), 6 | * to deal in the Software without restriction, including without limitation 7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 | * and/or sell copies of the Software, and to permit persons to whom the 9 | * Software is furnished to do so, subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in 12 | * all copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 20 | * DEALINGS IN THE SOFTWARE. 21 | */ 22 | 23 | package net.lag.kestrel 24 | 25 | import scala.actors.Actor 26 | import org.apache.mina.common._ 27 | import net.lag.logging.Logger 28 | 29 | 30 | // Actor messages for the Mina "events" 31 | abstract sealed class MinaMessage 32 | object MinaMessage { 33 | case object SessionOpened extends MinaMessage 34 | case class MessageReceived(message: AnyRef) extends MinaMessage 35 | case class MessageSent(message: AnyRef) extends MinaMessage 36 | case class ExceptionCaught(cause: Throwable) extends MinaMessage 37 | case class SessionIdle(status: IdleStatus) extends MinaMessage 38 | case object SessionClosed extends MinaMessage 39 | } 40 | 41 | 42 | class IoHandlerActorAdapter(val actorFactory: (IoSession) => Actor) extends IoHandler { 43 | 44 | private val log = Logger.get 45 | private val ACTOR_KEY = "scala.mina.actor" 46 | 47 | private def actorFor(session: IoSession) = session.getAttribute(ACTOR_KEY).asInstanceOf[Actor] 48 | 49 | def sessionCreated(session: IoSession) = { 50 | val actor = actorFactory(session) 51 | session.setAttribute(ACTOR_KEY, actor) 52 | } 53 | 54 | def sessionOpened(session: IoSession) = actorFor(session) ! MinaMessage.SessionOpened 55 | def messageReceived(session: IoSession, message: AnyRef) = actorFor(session) ! new MinaMessage.MessageReceived(message) 56 | def messageSent(session: IoSession, message: AnyRef) = actorFor(session) ! new MinaMessage.MessageSent(message) 57 | 58 | def exceptionCaught(session: IoSession, cause: Throwable) = { 59 | actorFor(session) match { 60 | case null => 61 | // weird bad: an exception happened but i guess it wasn't associated with any existing session. 62 | log.error(cause, "Exception inside mina!") 63 | case actor: Actor => actor ! new MinaMessage.ExceptionCaught(cause) 64 | } 65 | } 66 | 67 | def sessionIdle(session: IoSession, status: IdleStatus) = actorFor(session) ! new MinaMessage.SessionIdle(status) 68 | 69 | def sessionClosed(session: IoSession) = { 70 | val actor = actorFor(session) 71 | session.removeAttribute(ACTOR_KEY) 72 | actor ! MinaMessage.SessionClosed 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /ant/package.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 37 | 38 | 39 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 71 | 72 | 73 | -------------------------------------------------------------------------------- /src/test/scala/net/lag/kestrel/TestClient.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2008 Robey Pointer 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a 5 | * copy of this software and associated documentation files (the "Software"), 6 | * to deal in the Software without restriction, including without limitation 7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 | * and/or sell copies of the Software, and to permit persons to whom the 9 | * Software is furnished to do so, subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in 12 | * all copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 20 | * DEALINGS IN THE SOFTWARE. 21 | */ 22 | 23 | package net.lag.kestrel 24 | 25 | import java.io._ 26 | import java.net.Socket 27 | import scala.collection.Map 28 | import scala.collection.mutable 29 | 30 | 31 | class TestClient(host: String, port: Int) { 32 | 33 | var socket: Socket = null 34 | var out: OutputStream = null 35 | var in: DataInputStream = null 36 | 37 | connect 38 | 39 | 40 | def connect = { 41 | socket = new Socket(host, port) 42 | out = socket.getOutputStream 43 | in = new DataInputStream(socket.getInputStream) 44 | } 45 | 46 | def disconnect = { 47 | socket.close 48 | } 49 | 50 | private def readline = { 51 | // this isn't meant to be efficient, just simple. 52 | val out = new StringBuilder 53 | var done = false 54 | while (!done) { 55 | val ch: Int = in.read 56 | if ((ch < 0) || (ch == 10)) { 57 | done = true 58 | } else if (ch != 13) { 59 | out += ch.toChar 60 | } 61 | } 62 | out.toString 63 | } 64 | 65 | def set(key: String, value: String): String = { 66 | out.write(("set " + key + " 0 0 " + value.length + "\r\n" + value + "\r\n").getBytes) 67 | readline 68 | } 69 | 70 | def set(key: String, value: String, expiry: Int) = { 71 | out.write(("set " + key + " 0 " + expiry + " " + value.length + "\r\n" + value + "\r\n").getBytes) 72 | readline 73 | } 74 | 75 | def get(key: String): String = { 76 | out.write(("get " + key + "\r\n").getBytes) 77 | val line = readline 78 | if (line == "END") { 79 | return "" 80 | } 81 | // VALUE 82 | val len = line.split(" ")(3).toInt 83 | val buffer = new Array[Byte](len) 84 | in.readFully(buffer) 85 | readline 86 | readline // "END" 87 | new String(buffer) 88 | } 89 | 90 | def add(key: String, value: String) = { 91 | out.write(("add " + key + " 0 0 " + value.length + "\r\n" + value + "\r\n").getBytes) 92 | readline 93 | } 94 | 95 | def stats: Map[String, String] = { 96 | out.write("stats\r\n".getBytes) 97 | var done = false 98 | val map = new mutable.HashMap[String, String] 99 | while (!done) { 100 | val line = readline 101 | if (line startsWith "STAT") { 102 | val args = line.split(" ") 103 | map(args(1)) = args(2) 104 | } else if (line == "END") { 105 | done = true 106 | } 107 | } 108 | map 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /src/test/scala/net/lag/FilterableSpecsFileRunner.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2008 Robey Pointer 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a 5 | * copy of this software and associated documentation files (the "Software"), 6 | * to deal in the Software without restriction, including without limitation 7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 | * and/or sell copies of the Software, and to permit persons to whom the 9 | * Software is furnished to do so, subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in 12 | * all copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 20 | * DEALINGS IN THE SOFTWARE. 21 | */ 22 | 23 | package net.lag 24 | 25 | import org.specs.Specification 26 | import org.specs.runner.SpecsFileRunner 27 | import org.specs.specification.Sut 28 | import scala.collection.mutable 29 | import java.util.regex.Pattern 30 | 31 | 32 | /** 33 | * Runs specs against a console, finding all Specification objects in a given path, and 34 | * optionally filtering the executed Specifications/examples against a regex provided via 35 | * system properties. 36 | * 37 | * The spec property will be used to filter Specifications, and 38 | * example will filter examples within those Specifications. Because of 39 | * limitations in the current layout of the specs library, examples can only be filtered at 40 | * the top level (nested examples are not filtered). 41 | * 42 | * Substring regex matching is used, so the simple case of -Dspec=Timeline (a 43 | * substring with no special regex characters) should do what you expect. 44 | */ 45 | class FilterableSpecsFileRunner(path: String) extends SpecsFileRunner(path, ".*") { 46 | var specList = new mutable.ListBuffer[Specification] 47 | 48 | // load the specs from class files in the given path. 49 | private def loadSpecs = { 50 | for (className <- specificationNames(path, ".*")) { 51 | createSpecification(className) match { 52 | case Some(s) => specList += s 53 | case None => //println("Could not load " + className) 54 | } 55 | } 56 | 57 | // filter specs by name, if given. 58 | System.getProperty("spec") match { 59 | case null => 60 | case filterString => 61 | val re = Pattern.compile(filterString) 62 | for (spec <- specList) { 63 | spec.suts = spec.suts filter { sut => re.matcher(sut.description).find } 64 | } 65 | } 66 | 67 | // also filter examples by name, if given. 68 | System.getProperty("example") match { 69 | case null => 70 | case filterString => 71 | val re = Pattern.compile(filterString) 72 | for (spec <- specList; sut <- spec.suts) { 73 | val examples = sut.examples filter { example => re.matcher(example.description).find } 74 | sut.examples.clear 75 | sut.examples ++= examples 76 | } 77 | } 78 | 79 | // remove any now-empty specs so we don't clutter the output. 80 | for (spec <- specList) { 81 | spec.suts = spec.suts filter { sut => sut.examples.length > 0 } 82 | } 83 | for (empty <- specList filter { spec => spec.suts.length == 0 }) { 84 | specList -= empty 85 | } 86 | } 87 | 88 | override def reportSpecs = { 89 | loadSpecs 90 | 91 | // this specification is added for better reporting 92 | object totalSpecification extends Specification { 93 | new java.io.File(path).getAbsolutePath isSpecifiedBy(specList: _*) 94 | } 95 | super.report(List(totalSpecification)) 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /src/test/scala/net/lag/kestrel/load/PutMany.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2008 Robey Pointer 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a 5 | * copy of this software and associated documentation files (the "Software"), 6 | * to deal in the Software without restriction, including without limitation 7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 | * and/or sell copies of the Software, and to permit persons to whom the 9 | * Software is furnished to do so, subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in 12 | * all copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 20 | * DEALINGS IN THE SOFTWARE. 21 | */ 22 | 23 | package net.lag.kestrel.load 24 | 25 | import java.net._ 26 | import java.nio._ 27 | import java.nio.channels._ 28 | import net.lag.extensions._ 29 | 30 | 31 | /** 32 | * Spam a kestrel server with 1M copies of a pop song lyric, to see how 33 | * quickly it can absorb them. 34 | */ 35 | object PutMany { 36 | private val LYRIC = 37 | "crossed off, but never forgotten\n" + 38 | "misplaced, but never losing hold\n" + 39 | "these are the moments that bind us\n" + 40 | "repressed, but never erased\n" + 41 | "knocked down, but never giving up\n" + 42 | "locked up where no one can find us\n" + 43 | "we'll survive in here til the end\n" + 44 | "\n" + 45 | "there are no more fights to fight\n" + 46 | "my trophies are the scars that will never heal\n" + 47 | "but i get carried away sometimes\n" + 48 | "i wake up in the night swinging at the ceiling\n" + 49 | "it's hard to leave old ways behind\n" + 50 | "but harder when you think that's all there is\n" + 51 | "don't look at me that way\n" + 52 | "\n" + 53 | "ignored, when your whole world's collapsed\n" + 54 | "dismissed, before you speak a word\n" + 55 | "these are the moments that bind you\n" + 56 | "come clean, but everything's wrong\n" + 57 | "sustained, but barely holding on\n" + 58 | "run down, with no one to find you\n" + 59 | "we're survivors, here til the end" 60 | 61 | private val EXPECT = ByteBuffer.wrap("STORED\r\n".getBytes) 62 | 63 | def put(socket: SocketChannel, queueName: String, n: Int) = { 64 | val spam = ByteBuffer.wrap(("set " + queueName + " 0 0 " + LYRIC.length + "\r\n" + LYRIC + "\r\n").getBytes) 65 | val buffer = ByteBuffer.allocate(8) 66 | for (i <- 0 until n) { 67 | spam.rewind 68 | while (spam.position < spam.limit) { 69 | socket.write(spam) 70 | } 71 | buffer.rewind 72 | while (buffer.position < buffer.limit) { 73 | socket.read(buffer) 74 | } 75 | buffer.rewind 76 | EXPECT.rewind 77 | if (buffer != EXPECT) { 78 | // the "!" is important. 79 | throw new Exception("Unexpected response at " + i + "!") 80 | } 81 | } 82 | } 83 | 84 | def main(args: Array[String]) = { 85 | if (args.length < 1) { 86 | Console.println("usage: put-many ") 87 | Console.println(" spin up N clients and put 10k items spread across N queues") 88 | System.exit(1) 89 | } 90 | 91 | val clientCount = args(0).toInt 92 | val totalCount = 10000 / clientCount * clientCount 93 | 94 | var threadList: List[Thread] = Nil 95 | val startTime = System.currentTimeMillis 96 | 97 | for (i <- 0 until clientCount) { 98 | val t = new Thread { 99 | override def run = { 100 | val socket = SocketChannel.open(new InetSocketAddress("localhost", 22133)) 101 | put(socket, "spam", 10000 / clientCount) 102 | } 103 | } 104 | threadList = t :: threadList 105 | t.start 106 | } 107 | for (t <- threadList) { 108 | t.join 109 | } 110 | 111 | val duration = System.currentTimeMillis - startTime 112 | Console.println("Finished in %d msec (%.1f usec/put).".format(duration, duration * 1000.0 / totalCount)) 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | Kestrel 3 | ======= 4 | 5 | Kestrel is a port of Blaine Cook's "starling" message queue 6 | system from ruby to scala: 7 | 8 | In Blaine's words: 9 | 10 | > Starling is a powerful but simple messaging server that enables reliable 11 | > distributed queuing with an absolutely minimal overhead. It speaks the 12 | > MemCache protocol for maximum cross-platform compatibility. Any language 13 | > that speaks MemCache can take advantage of Starling's queue facilities. 14 | 15 | The concept of starling is to have a single server handle reliable, ordered 16 | message queues. When you put a cluster of these servers together, 17 | *with no cross communication*, and pick a server at random whenever you do a 18 | `set` or `get`, you end up with a reliable, *loosely ordered* message queue. 19 | 20 | In many situations, loose ordering is sufficient. Dropping the requirement on 21 | cross communication makes it horizontally scale to infinity and beyond: no 22 | multicast, no clustering, no "elections", no coordination at all. No talking! 23 | Shhh! 24 | 25 | Kestrel adds several additional features, like ginormous queues, reliable 26 | fetch, and blocking/timeout fetch -- as well as the scalability offered by 27 | actors and the JVM. 28 | 29 | Features 30 | -------- 31 | 32 | Kestrel is: 33 | 34 | - fast 35 | 36 | It runs on the JVM so it can take advantage of the hard work people have 37 | put into java performance. 38 | 39 | - small 40 | 41 | Currently about 1.5K lines of scala (including comments), because it relies 42 | on Apache Mina (a rough equivalent of Danger's ziggurat or Ruby's 43 | EventMachine) and actors -- and frankly because Scala is extremely 44 | expressive. 45 | 46 | - durable 47 | 48 | Queues are stored in memory for speed, but logged into a journal on disk 49 | so that servers can be shutdown or moved without losing any data. 50 | 51 | - reliable 52 | 53 | A client can ask to "tentatively" fetch an item from a queue, and if that 54 | client disconnects from kestrel before confirming ownership of the item, 55 | the item is handed to another client. In this way, crashing clients don't 56 | cause lost messages. 57 | 58 | Anti-Features 59 | ------------- 60 | 61 | Kestrel is not: 62 | 63 | - strongly ordered 64 | 65 | While each queue is strongly ordered on each machine, a cluster will 66 | appear "loosely ordered" because clients pick a machine at random for 67 | each operation. The end result should be "mostly fair". 68 | 69 | - transactional 70 | 71 | This is not a database. Item ownership is transferred with acknowledgement, 72 | but kestrel does not concern itself with what happens to an item after a 73 | client has accepted it. 74 | 75 | 76 | Use 77 | --- 78 | 79 | Building from source is easy: 80 | 81 | $ ant 82 | 83 | Scala libraries and dependencies will be downloaded from maven repositories 84 | the first time you do a build. The finished distribution will be in `dist`. 85 | 86 | A sample startup script is included, or you may run the jar directly. All 87 | configuration is loaded from `kestrel.conf`. 88 | 89 | 90 | Performance 91 | ----------- 92 | 93 | All of the below timings are on my 2GHz 2006-model macbook pro. 94 | 95 | Since starling uses eventmachine in a single-thread single-process form, it 96 | has similar results for all access types (and will never use more than one 97 | core). 98 | 99 | ========= ================= ========== 100 | # Clients Pushes per client Total time 101 | ========= ================= ========== 102 | 1 10,000 3.8s 103 | 10 1,000 2.9s 104 | 100 100 3.1s 105 | ========= ================= ========== 106 | 107 | Kestrel uses N+1 I/O processor threads (where N = the number of available CPU 108 | cores), and a pool of worker threads for handling actor events. Therefore it 109 | handles more poorly for small numbers of heavy-use clients, and better for 110 | large numbers of clients. 111 | 112 | ========= ================= ========== 113 | # Clients Pushes per client Total time 114 | ========= ================= ========== 115 | 1 10,000 3.8s 116 | 10 1,000 2.4s 117 | 100 100 1.6s 118 | ========= ================= ========== 119 | 120 | 121 | Robey Pointer <> 122 | -------------------------------------------------------------------------------- /src/test/scala/net/lag/kestrel/QueueCollectionSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2008 Robey Pointer 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a 5 | * copy of this software and associated documentation files (the "Software"), 6 | * to deal in the Software without restriction, including without limitation 7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 | * and/or sell copies of the Software, and to permit persons to whom the 9 | * Software is furnished to do so, subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in 12 | * all copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 20 | * DEALINGS IN THE SOFTWARE. 21 | */ 22 | 23 | package net.lag.kestrel 24 | 25 | import scala.util.Sorting 26 | import java.io.{File, FileInputStream} 27 | import net.lag.configgy.Config 28 | import org.specs._ 29 | 30 | 31 | object QueueCollectionSpec extends Specification with TestHelper { 32 | 33 | private var qc: QueueCollection = null 34 | 35 | private def sorted[T <% Ordered[T]](list: List[T]): List[T] = { 36 | val dest = list.toArray 37 | Sorting.quickSort(dest) 38 | dest.toList 39 | } 40 | 41 | 42 | "QueueCollection" should { 43 | 44 | doAfter { 45 | qc.shutdown 46 | } 47 | 48 | "create a queue" in { 49 | withTempFolder { 50 | qc = new QueueCollection(folderName, Config.fromMap(Map.empty)) 51 | qc.queueNames mustEqual Nil 52 | 53 | qc.add("work1", "stuff".getBytes) 54 | qc.add("work2", "other stuff".getBytes) 55 | 56 | sorted(qc.queueNames) mustEqual List("work1", "work2") 57 | qc.currentBytes mustEqual 16 58 | qc.currentItems mustEqual 2 59 | qc.totalAdded mustEqual 2 60 | 61 | new String(qc.receive("work1").get) mustEqual "stuff" 62 | qc.receive("work1") mustEqual None 63 | new String(qc.receive("work2").get) mustEqual "other stuff" 64 | qc.receive("work2") mustEqual None 65 | 66 | qc.currentBytes mustEqual 0 67 | qc.currentItems mustEqual 0 68 | qc.totalAdded mustEqual 2 69 | } 70 | } 71 | 72 | "load from journal" in { 73 | withTempFolder { 74 | qc = new QueueCollection(folderName, Config.fromMap(Map.empty)) 75 | qc.add("ducklings", "huey".getBytes) 76 | qc.add("ducklings", "dewey".getBytes) 77 | qc.add("ducklings", "louie".getBytes) 78 | qc.queueNames mustEqual List("ducklings") 79 | qc.currentBytes mustEqual 14 80 | qc.currentItems mustEqual 3 81 | qc.shutdown 82 | 83 | qc = new QueueCollection(folderName, Config.fromMap(Map.empty)) 84 | qc.queueNames mustEqual Nil 85 | new String(qc.receive("ducklings").get) mustEqual "huey" 86 | // now the queue should be suddenly instantiated: 87 | qc.currentBytes mustEqual 10 88 | qc.currentItems mustEqual 2 89 | } 90 | } 91 | 92 | "queue hit/miss tracking" in { 93 | withTempFolder { 94 | qc = new QueueCollection(folderName, Config.fromMap(Map.empty)) 95 | qc.add("ducklings", "ugly1".getBytes) 96 | qc.add("ducklings", "ugly2".getBytes) 97 | qc.queueHits mustEqual 0 98 | qc.queueMisses mustEqual 0 99 | 100 | new String(qc.receive("ducklings").get) mustEqual "ugly1" 101 | qc.queueHits mustEqual 1 102 | qc.queueMisses mustEqual 0 103 | qc.receive("zombie") mustEqual None 104 | qc.queueHits mustEqual 1 105 | qc.queueMisses mustEqual 1 106 | 107 | new String(qc.receive("ducklings").get) mustEqual "ugly2" 108 | qc.queueHits mustEqual 2 109 | qc.queueMisses mustEqual 1 110 | qc.receive("ducklings") mustEqual None 111 | qc.queueHits mustEqual 2 112 | qc.queueMisses mustEqual 2 113 | qc.receive("ducklings") mustEqual None 114 | qc.queueHits mustEqual 2 115 | qc.queueMisses mustEqual 3 116 | } 117 | } 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /src/test/scala/net/lag/kestrel/memcache/MemCacheCodecSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2008 Robey Pointer 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a 5 | * copy of this software and associated documentation files (the "Software"), 6 | * to deal in the Software without restriction, including without limitation 7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 | * and/or sell copies of the Software, and to permit persons to whom the 9 | * Software is furnished to do so, subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in 12 | * all copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 20 | * DEALINGS IN THE SOFTWARE. 21 | */ 22 | 23 | package net.lag.kestrel.memcache 24 | 25 | import org.apache.mina.common.{ByteBuffer, IoSession} 26 | import org.apache.mina.common.support.{BaseIoSession} 27 | import org.apache.mina.filter.codec._ 28 | import org.specs._ 29 | 30 | 31 | object MemCacheCodecSpec extends Specification { 32 | 33 | private val fakeSession = new BaseIoSession { 34 | override def updateTrafficMask = {} 35 | override def getServiceAddress = null 36 | override def getLocalAddress = null 37 | override def getRemoteAddress = null 38 | override def getTransportType = null 39 | override def getFilterChain = null 40 | override def getConfig = null 41 | override def getHandler = null 42 | override def getServiceConfig = null 43 | override def getService = null 44 | } 45 | 46 | private val fakeDecoderOutput = new ProtocolDecoderOutput { 47 | override def flush = {} 48 | override def write(obj: AnyRef) = { 49 | written = obj :: written 50 | } 51 | } 52 | 53 | private var written: List[AnyRef] = Nil 54 | 55 | 56 | "Memcache Decoder" should { 57 | doBefore { 58 | written = Nil 59 | } 60 | 61 | 62 | "'get' request chunked various ways" in { 63 | val decoder = new Decoder 64 | 65 | decoder.decode(fakeSession, ByteBuffer.wrap("get foo\r\n".getBytes), fakeDecoderOutput) 66 | written mustEqual List(Request(List("GET", "foo"), None)) 67 | written = Nil 68 | 69 | decoder.decode(fakeSession, ByteBuffer.wrap("get f".getBytes), fakeDecoderOutput) 70 | written mustEqual Nil 71 | decoder.decode(fakeSession, ByteBuffer.wrap("oo\r\n".getBytes), fakeDecoderOutput) 72 | written mustEqual List(Request(List("GET", "foo"), None)) 73 | written = Nil 74 | 75 | decoder.decode(fakeSession, ByteBuffer.wrap("g".getBytes), fakeDecoderOutput) 76 | written mustEqual Nil 77 | decoder.decode(fakeSession, ByteBuffer.wrap("et foo\r".getBytes), fakeDecoderOutput) 78 | written mustEqual Nil 79 | decoder.decode(fakeSession, ByteBuffer.wrap("\nget ".getBytes), fakeDecoderOutput) 80 | written mustEqual List(Request(List("GET", "foo"), None)) 81 | decoder.decode(fakeSession, ByteBuffer.wrap("bar\r\n".getBytes), fakeDecoderOutput) 82 | written mustEqual List(Request(List("GET", "bar"), None), Request(List("GET", "foo"), None)) 83 | } 84 | 85 | "'set' request chunked various ways" in { 86 | val decoder = new Decoder 87 | 88 | decoder.decode(fakeSession, ByteBuffer.wrap("set foo 0 0 5\r\nhello\r\n".getBytes), fakeDecoderOutput) 89 | written.mkString(",") mustEqual "" 90 | written = Nil 91 | 92 | decoder.decode(fakeSession, ByteBuffer.wrap("set foo 0 0 5\r\n".getBytes), fakeDecoderOutput) 93 | written mustEqual Nil 94 | decoder.decode(fakeSession, ByteBuffer.wrap("hello\r\n".getBytes), fakeDecoderOutput) 95 | written.mkString(",") mustEqual "" 96 | written = Nil 97 | 98 | decoder.decode(fakeSession, ByteBuffer.wrap("set foo 0 0 5".getBytes), fakeDecoderOutput) 99 | written mustEqual Nil 100 | decoder.decode(fakeSession, ByteBuffer.wrap("\r\nhell".getBytes), fakeDecoderOutput) 101 | written mustEqual Nil 102 | decoder.decode(fakeSession, ByteBuffer.wrap("o\r\n".getBytes), fakeDecoderOutput) 103 | written.mkString(",") mustEqual "" 104 | written = Nil 105 | } 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /src/main/scala/net/lag/kestrel/Kestrel.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2008 Robey Pointer 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a 5 | * copy of this software and associated documentation files (the "Software"), 6 | * to deal in the Software without restriction, including without limitation 7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 | * and/or sell copies of the Software, and to permit persons to whom the 9 | * Software is furnished to do so, subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in 12 | * all copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 20 | * DEALINGS IN THE SOFTWARE. 21 | */ 22 | 23 | package net.lag.kestrel 24 | 25 | import java.net.InetSocketAddress 26 | import java.util.Properties 27 | import java.util.concurrent.{CountDownLatch, Executors, ExecutorService, TimeUnit} 28 | import scala.actors.{Actor, Scheduler} 29 | import scala.actors.Actor._ 30 | import scala.collection.mutable 31 | import org.apache.mina.common._ 32 | import org.apache.mina.filter.codec.ProtocolCodecFilter 33 | import org.apache.mina.transport.socket.nio.{SocketAcceptor, SocketAcceptorConfig, SocketSessionConfig} 34 | import net.lag.configgy.{Config, ConfigMap, Configgy, RuntimeEnvironment} 35 | import net.lag.logging.Logger 36 | 37 | 38 | class Counter { 39 | private var value: Long = 0 40 | 41 | def get = synchronized { value } 42 | def set(n: Int) = synchronized { value = n } 43 | def incr = synchronized { value += 1; value } 44 | def incr(n: Int) = synchronized { value += n; value } 45 | def decr = synchronized { value -= 1; value } 46 | override def toString = synchronized { value.toString } 47 | } 48 | 49 | 50 | object KestrelStats { 51 | val bytesRead = new Counter 52 | val bytesWritten = new Counter 53 | val sessions = new Counter 54 | val totalConnections = new Counter 55 | val getRequests = new Counter 56 | val setRequests = new Counter 57 | val sessionID = new Counter 58 | } 59 | 60 | 61 | object Kestrel { 62 | private val log = Logger.get 63 | val runtime = new RuntimeEnvironment(getClass) 64 | 65 | var queues: QueueCollection = null 66 | 67 | private val _expiryStats = new mutable.HashMap[String, Int] 68 | private val _startTime = Time.now 69 | 70 | ByteBuffer.setUseDirectBuffers(false) 71 | ByteBuffer.setAllocator(new SimpleByteBufferAllocator()) 72 | 73 | var acceptorExecutor: ExecutorService = null 74 | var acceptor: IoAcceptor = null 75 | 76 | private val deathSwitch = new CountDownLatch(1) 77 | 78 | 79 | def main(args: Array[String]): Unit = { 80 | runtime.load(args) 81 | startup(Configgy.config) 82 | } 83 | 84 | def configure(c: Option[ConfigMap]) = { 85 | for (config <- c) { 86 | PersistentQueue.maxJournalSize = config.getInt("max_journal_size", 16 * 1024 * 1024) 87 | } 88 | } 89 | 90 | def startup(config: Config) = { 91 | val listenAddress = config.getString("host", "0.0.0.0") 92 | val listenPort = config.getInt("port", 22122) 93 | queues = new QueueCollection(config.getString("queue_path", "/tmp"), config.configMap("queues")) 94 | configure(Some(config)) 95 | config.subscribe(configure _) 96 | 97 | acceptorExecutor = Executors.newCachedThreadPool() 98 | acceptor = new SocketAcceptor(Runtime.getRuntime().availableProcessors() + 1, acceptorExecutor) 99 | 100 | // mina garbage: 101 | acceptor.getDefaultConfig.setThreadModel(ThreadModel.MANUAL) 102 | val saConfig = new SocketAcceptorConfig 103 | saConfig.setReuseAddress(true) 104 | saConfig.setBacklog(1000) 105 | saConfig.getSessionConfig.setTcpNoDelay(true) 106 | saConfig.getFilterChain.addLast("codec", new ProtocolCodecFilter(new memcache.Encoder, new memcache.Decoder)) 107 | acceptor.bind(new InetSocketAddress(listenAddress, listenPort), new IoHandlerActorAdapter((session: IoSession) => new KestrelHandler(session, config)), saConfig) 108 | 109 | log.info("Kestrel started.") 110 | 111 | // make sure there's always one actor running so scala 272rc6 doesn't kill off the actors library. 112 | actor { 113 | deathSwitch.await 114 | } 115 | } 116 | 117 | def shutdown = { 118 | log.info("Shutting down!") 119 | queues.shutdown 120 | acceptor.unbindAll 121 | Scheduler.shutdown 122 | acceptorExecutor.shutdown 123 | // the line below causes a 1s pause in unit tests. :( 124 | acceptorExecutor.awaitTermination(5, TimeUnit.SECONDS) 125 | deathSwitch.countDown 126 | } 127 | 128 | def uptime = (Time.now - _startTime) / 1000 129 | } 130 | -------------------------------------------------------------------------------- /ant/compile.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | 159 | 162 | 163 | 164 | -------------------------------------------------------------------------------- /src/test/scala/net/lag/kestrel/load/ManyClients.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2008 Robey Pointer 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a 5 | * copy of this software and associated documentation files (the "Software"), 6 | * to deal in the Software without restriction, including without limitation 7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 | * and/or sell copies of the Software, and to permit persons to whom the 9 | * Software is furnished to do so, subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in 12 | * all copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 20 | * DEALINGS IN THE SOFTWARE. 21 | */ 22 | 23 | package net.lag.kestrel.load 24 | 25 | import java.net._ 26 | import java.nio._ 27 | import java.nio.channels._ 28 | import java.util.concurrent.atomic._ 29 | import net.lag.extensions._ 30 | 31 | 32 | /** 33 | * Have one producer trickle in pop lyrics at a steady but slow rate, while 34 | * many clients clamor for each one. This is similar to how queues operate in 35 | * some typical production environments. 36 | */ 37 | object ManyClients { 38 | private val SLEEP = 100 39 | private val COUNT = 100 40 | 41 | private val LYRIC = 42 | "crossed off, but never forgotten\n" + 43 | "misplaced, but never losing hold\n" + 44 | "these are the moments that bind us\n" + 45 | "repressed, but never erased\n" + 46 | "knocked down, but never giving up\n" + 47 | "locked up where no one can find us\n" + 48 | "we'll survive in here til the end\n" + 49 | "\n" + 50 | "there are no more fights to fight\n" + 51 | "my trophies are the scars that will never heal\n" + 52 | "but i get carried away sometimes\n" + 53 | "i wake up in the night swinging at the ceiling\n" + 54 | "it's hard to leave old ways behind\n" + 55 | "but harder when you think that's all there is\n" + 56 | "don't look at me that way\n" + 57 | "\n" + 58 | "ignored, when your whole world's collapsed\n" + 59 | "dismissed, before you speak a word\n" + 60 | "these are the moments that bind you\n" + 61 | "come clean, but everything's wrong\n" + 62 | "sustained, but barely holding on\n" + 63 | "run down, with no one to find you\n" + 64 | "we're survivors, here til the end" 65 | 66 | private val EXPECT = ByteBuffer.wrap("STORED\r\n".getBytes) 67 | 68 | private val got = new AtomicInteger(0) 69 | 70 | 71 | def put(socket: SocketChannel, queueName: String, n: Int) = { 72 | val spam = ByteBuffer.wrap(("set " + queueName + " 0 0 " + LYRIC.length + "\r\n" + LYRIC + "\r\n").getBytes) 73 | val buffer = ByteBuffer.allocate(8) 74 | for (i <- 0 until n) { 75 | spam.rewind 76 | while (spam.position < spam.limit) { 77 | socket.write(spam) 78 | } 79 | buffer.rewind 80 | while (buffer.position < buffer.limit) { 81 | socket.read(buffer) 82 | } 83 | buffer.rewind 84 | if (buffer != EXPECT) { 85 | // the "!" is important. 86 | throw new Exception("Unexpected response at " + i + "!") 87 | } 88 | Thread.sleep(SLEEP) 89 | } 90 | } 91 | 92 | def getStuff(socket: SocketChannel, queueName: String) = { 93 | val req = ByteBuffer.wrap(("get " + queueName + "/t=1000\r\n").getBytes) 94 | val expectEnd = ByteBuffer.wrap("END\r\n".getBytes) 95 | val expectLyric = ByteBuffer.wrap(("VALUE " + queueName + " 0 " + LYRIC.length + "\r\n" + LYRIC + "\r\n").getBytes) 96 | val buffer = ByteBuffer.allocate(expectLyric.capacity) 97 | expectLyric.rewind 98 | 99 | while (got.get < COUNT) { 100 | req.rewind 101 | while (req.position < req.limit) { 102 | socket.write(req) 103 | } 104 | buffer.rewind 105 | while (buffer.position < expectEnd.limit) { 106 | socket.read(buffer) 107 | } 108 | val oldpos = buffer.position 109 | buffer.flip 110 | expectEnd.rewind 111 | if (buffer == expectEnd) { 112 | // ok. :( 113 | } else { 114 | buffer.position(oldpos) 115 | buffer.limit(buffer.capacity) 116 | while (buffer.position < expectLyric.limit) { 117 | socket.read(buffer) 118 | } 119 | buffer.rewind 120 | expectLyric.rewind 121 | if (buffer != expectLyric) { 122 | throw new Exception("Unexpected response!") 123 | } 124 | println("" + got.incrementAndGet) 125 | } 126 | } 127 | } 128 | 129 | def main(args: Array[String]) = { 130 | if (args.length < 1) { 131 | Console.println("usage: many-clients ") 132 | Console.println(" spin up N clients and have them do timeout reads on a queue while a") 133 | Console.println(" single producer trickles out.") 134 | System.exit(1) 135 | } 136 | 137 | val clientCount = args(0).toInt 138 | 139 | var threadList: List[Thread] = Nil 140 | val startTime = System.currentTimeMillis 141 | 142 | for (i <- 0 until clientCount) { 143 | val t = new Thread { 144 | override def run = { 145 | val socket = SocketChannel.open(new InetSocketAddress("localhost", 22133)) 146 | getStuff(socket, "spam") 147 | } 148 | } 149 | threadList = t :: threadList 150 | t.start 151 | } 152 | val t = new Thread { 153 | override def run = { 154 | val socket = SocketChannel.open(new InetSocketAddress("localhost", 22133)) 155 | put(socket, "spam", COUNT) 156 | } 157 | } 158 | threadList = t :: threadList 159 | t.start 160 | for (t <- threadList) { 161 | t.join 162 | } 163 | 164 | val duration = System.currentTimeMillis - startTime 165 | Console.println("Finished in %d msec.".format(duration)) 166 | } 167 | } 168 | -------------------------------------------------------------------------------- /src/main/scala/net/lag/kestrel/memcache/Decoder.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2008 Robey Pointer 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a 5 | * copy of this software and associated documentation files (the "Software"), 6 | * to deal in the Software without restriction, including without limitation 7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 | * and/or sell copies of the Software, and to permit persons to whom the 9 | * Software is furnished to do so, subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in 12 | * all copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 20 | * DEALINGS IN THE SOFTWARE. 21 | */ 22 | 23 | package net.lag.kestrel.memcache 24 | 25 | import scala.collection.mutable 26 | import org.apache.mina.common.{ByteBuffer, IoSession} 27 | import org.apache.mina.filter.codec._ 28 | import net.lag.extensions._ 29 | 30 | 31 | case class Request(line: List[String], data: Option[Array[Byte]]) { 32 | override def toString = { 33 | " "" 35 | case Some(x) => ": " + x.hexlify 36 | }) + ">" 37 | } 38 | } 39 | 40 | case class Response(data: ByteBuffer) 41 | 42 | class ProtocolException(desc: String) extends Exception(desc) 43 | 44 | 45 | /** 46 | * Protocol encoder for a memcache server. 47 | * Inspired by jmemcached. 48 | */ 49 | class Encoder extends ProtocolEncoder { 50 | def encode(session: IoSession, message: AnyRef, out: ProtocolEncoderOutput) = { 51 | val buffer = message.asInstanceOf[Response].data 52 | KestrelStats.bytesWritten.incr(buffer.remaining) 53 | out.write(buffer) 54 | } 55 | 56 | def dispose(session: IoSession): Unit = { 57 | // nothing. 58 | } 59 | } 60 | 61 | 62 | /** 63 | * Protocol decoder for a memcache server. 64 | * Inspired by jmemcached. 65 | */ 66 | class Decoder extends ProtocolDecoder { 67 | private class State { 68 | var buffer = ByteBuffer.allocate(1024) 69 | buffer.setAutoExpand(true) 70 | 71 | var line: Option[Array[String]] = None 72 | var dataBytes = 0 73 | 74 | def reset = { 75 | line = None 76 | dataBytes = 0 77 | resetBuffer 78 | } 79 | 80 | // truncate the buffer, deleting what's already been read. 81 | // go into write mode. 82 | def resetBuffer = { 83 | if (buffer.position == 0) { 84 | // leave it alone 85 | } else if (buffer.position < buffer.limit) { 86 | val remainingBytes = new Array[Byte](buffer.limit - buffer.position) 87 | buffer.get(remainingBytes) 88 | buffer.clear 89 | buffer.put(remainingBytes) 90 | } else { 91 | buffer.clear 92 | } 93 | } 94 | 95 | // prepare a buffer for writes. 96 | def unflipBuffer = { 97 | buffer.position(buffer.limit) 98 | buffer.limit(buffer.capacity) 99 | } 100 | } 101 | 102 | 103 | private val STATE_KEY = "scala.mina.memcache.state" 104 | private val KNOWN_COMMANDS = List("GET", "SET", "STATS", "SHUTDOWN", "RELOAD") 105 | private val DATA_COMMANDS = List("SET") 106 | 107 | 108 | def dispose(session: IoSession): Unit = { 109 | session.removeAttribute(STATE_KEY) 110 | } 111 | 112 | def finishDecode(session: IoSession, out: ProtocolDecoderOutput): Unit = { 113 | // um, no. :) 114 | } 115 | 116 | def decode(session: IoSession, in: ByteBuffer, out: ProtocolDecoderOutput): Unit = { 117 | var state = session.getAttribute(STATE_KEY).asInstanceOf[State] 118 | if (state == null) { 119 | state = new State 120 | session.setAttribute(STATE_KEY, state) 121 | } 122 | 123 | KestrelStats.bytesRead.incr(in.remaining) 124 | state.buffer.put(in) 125 | state.buffer.flip 126 | 127 | state.line match { 128 | case None => decodeLine(state, out) 129 | case Some(_) => decodeData(state, out) 130 | } 131 | } 132 | 133 | private def decodeLine(state: State, out: ProtocolDecoderOutput): Unit = { 134 | val lf = bufferIndexOf(state.buffer, '\n') 135 | if (lf < 0) { 136 | state.unflipBuffer 137 | return 138 | } 139 | 140 | var end = lf 141 | if ((end > 0) && (state.buffer.get(end - 1) == '\r')) { 142 | end -= 1 143 | } 144 | 145 | // pull off the line into a string 146 | val lineBytes = new Array[Byte](end) 147 | state.buffer.get(lineBytes) 148 | val line = new String(lineBytes, "ISO-8859-1") 149 | state.buffer.position(state.buffer.position + (lf - end) + 1) 150 | 151 | val segments = line.split(" ") 152 | segments(0) = segments(0).toUpperCase 153 | 154 | state.line = Some(segments) 155 | val command = segments(0) 156 | if (! KNOWN_COMMANDS.contains(command)) { 157 | throw new ProtocolException("Invalid command: " + command) 158 | } 159 | 160 | if (DATA_COMMANDS.contains(command)) { 161 | if (state.line.get.length < 5) { 162 | throw new ProtocolException("Malformed request line") 163 | } 164 | state.dataBytes = segments(4).toInt + 2 165 | state.resetBuffer 166 | state.buffer.flip 167 | decodeData(state, out) 168 | } else { 169 | out.write(Request(state.line.get.toList, None)) 170 | state.reset 171 | } 172 | } 173 | 174 | private def decodeData(state: State, out: ProtocolDecoderOutput): Unit = { 175 | if (state.buffer.remaining < state.dataBytes) { 176 | // still need more. 177 | state.unflipBuffer 178 | return 179 | } 180 | 181 | // final 2 bytes are just "\r\n" mandated by protocol. 182 | val bytes = new Array[Byte](state.dataBytes - 2) 183 | state.buffer.get(bytes) 184 | state.buffer.position(state.buffer.position + 2) 185 | 186 | out.write(Request(state.line.get.toList, Some(bytes))) 187 | state.reset 188 | } 189 | 190 | private def bufferIndexOf(buffer: ByteBuffer, b: Byte): Int = { 191 | var i = buffer.position 192 | while (i < buffer.limit) { 193 | if (buffer.get(i) == b) { 194 | return i - buffer.position 195 | } 196 | i += 1 197 | } 198 | return -1 199 | } 200 | } 201 | -------------------------------------------------------------------------------- /src/main/scala/net/lag/kestrel/QueueCollection.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2008 Robey Pointer 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a 5 | * copy of this software and associated documentation files (the "Software"), 6 | * to deal in the Software without restriction, including without limitation 7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 | * and/or sell copies of the Software, and to permit persons to whom the 9 | * Software is furnished to do so, subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in 12 | * all copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 20 | * DEALINGS IN THE SOFTWARE. 21 | */ 22 | 23 | package net.lag.kestrel 24 | 25 | import java.io.File 26 | import java.util.concurrent.CountDownLatch 27 | import scala.collection.mutable 28 | import net.lag.configgy.{Config, ConfigMap} 29 | import net.lag.logging.Logger 30 | 31 | 32 | class InaccessibleQueuePath extends Exception("Inaccessible queue path: Must be a directory and writable") 33 | 34 | 35 | class QueueCollection(private val queueFolder: String, private var queueConfigs: ConfigMap) { 36 | private val log = Logger.get 37 | 38 | private val path = new File(queueFolder) 39 | 40 | /** 41 | * TODO: Use File.mkdirs if possible. 42 | */ 43 | if (! path.isDirectory || ! path.canWrite) { 44 | throw new InaccessibleQueuePath 45 | } 46 | 47 | private val queues = new mutable.HashMap[String, PersistentQueue] 48 | private var shuttingDown = false 49 | 50 | // total of all data in all queues 51 | private var _currentBytes: Long = 0 52 | 53 | // total of all items in all queues 54 | private var _currentItems: Long = 0 55 | 56 | // total items added since the server started up. 57 | private var _totalAdded: Long = 0 58 | 59 | // hits/misses on removing items from the queue 60 | private var _queueHits: Long = 0 61 | private var _queueMisses: Long = 0 62 | 63 | // reader accessors: 64 | def currentBytes: Long = _currentBytes 65 | def currentItems: Long = _currentItems 66 | def totalAdded: Long = _totalAdded 67 | def queueHits: Long = _queueHits 68 | def queueMisses: Long = _queueMisses 69 | 70 | queueConfigs.subscribe { c => 71 | synchronized { 72 | queueConfigs = c.getOrElse(new Config) 73 | } 74 | } 75 | 76 | 77 | def queueNames: List[String] = synchronized { 78 | queues.keys.toList 79 | } 80 | 81 | /** 82 | * Get a named queue, creating it if necessary. 83 | * Exposed only to unit tests. 84 | */ 85 | private[kestrel] def queue(name: String): Option[PersistentQueue] = { 86 | var setup = false 87 | var queue: Option[PersistentQueue] = None 88 | 89 | synchronized { 90 | if (shuttingDown) { 91 | return None 92 | } 93 | 94 | queue = queues.get(name) match { 95 | case q @ Some(_) => q 96 | case None => 97 | setup = true 98 | val q = new PersistentQueue(path.getPath, name, queueConfigs.configMap(name)) 99 | queues(name) = q 100 | Some(q) 101 | } 102 | } 103 | 104 | if (setup) { 105 | /* race is handled by having PersistentQueue start up with an 106 | * un-initialized flag that blocks all operations until this 107 | * method is called and completed: 108 | */ 109 | queue.get.setup 110 | synchronized { 111 | _currentBytes += queue.get.bytes 112 | _currentItems += queue.get.length 113 | } 114 | } 115 | queue 116 | } 117 | 118 | /** 119 | * Add an item to a named queue. Will not return until the item has been 120 | * synchronously added and written to the queue journal file. 121 | * 122 | * @return true if the item was added; false if the server is shutting 123 | * down 124 | */ 125 | def add(key: String, item: Array[Byte], expiry: Int): Boolean = { 126 | queue(key) match { 127 | case None => false 128 | case Some(q) => 129 | val now = Time.now 130 | val normalizedExpiry: Long = if (expiry == 0) { 131 | 0 132 | } else if (expiry < 1000000) { 133 | now + expiry * 1000 134 | } else { 135 | expiry * 1000 136 | } 137 | val result = q.add(item, normalizedExpiry) 138 | if (result) { 139 | synchronized { 140 | _currentBytes += item.length 141 | _currentItems += 1 142 | _totalAdded += 1 143 | } 144 | } 145 | result 146 | } 147 | } 148 | 149 | def add(key: String, item: Array[Byte]): Boolean = add(key, item, 0) 150 | 151 | /** 152 | * Retrieve an item from a queue and pass it to a continuation. If no item is available within 153 | * the requested time, or the server is shutting down, None is passed. 154 | */ 155 | def remove(key: String, timeout: Int, transaction: Boolean)(f: Option[QItem] => Unit): Unit = { 156 | queue(key) match { 157 | case None => 158 | synchronized { _queueMisses += 1 } 159 | f(None) 160 | case Some(q) => 161 | q.remove(if (timeout == 0) timeout else Time.now + timeout, transaction) { 162 | case None => 163 | synchronized { _queueMisses += 1 } 164 | f(None) 165 | case Some(item) => 166 | synchronized { 167 | _queueHits += 1 168 | _currentBytes -= item.data.length 169 | _currentItems -= 1 170 | } 171 | f(Some(item)) 172 | } 173 | } 174 | } 175 | 176 | // for testing. 177 | def receive(key: String): Option[Array[Byte]] = { 178 | var rv: Option[Array[Byte]] = None 179 | val latch = new CountDownLatch(1) 180 | remove(key, 0, false) { 181 | case None => 182 | rv = None 183 | latch.countDown 184 | case Some(v) => 185 | rv = Some(v.data) 186 | latch.countDown 187 | } 188 | latch.await 189 | rv 190 | } 191 | 192 | def unremove(key: String, xid: Int): Unit = { 193 | queue(key) match { 194 | case None => 195 | case Some(q) => 196 | q.unremove(xid) 197 | } 198 | } 199 | 200 | def confirmRemove(key: String, xid: Int): Unit = { 201 | queue(key) match { 202 | case None => 203 | case Some(q) => 204 | q.confirmRemove(xid) 205 | } 206 | } 207 | 208 | case class Stats(items: Long, bytes: Long, totalItems: Long, journalSize: Long, 209 | totalExpired: Long, currentAge: Long, memoryItems: Long, memoryBytes: Long) 210 | 211 | def stats(key: String): Stats = { 212 | queue(key) match { 213 | case None => Stats(0, 0, 0, 0, 0, 0, 0, 0) 214 | case Some(q) => Stats(q.length, q.bytes, q.totalItems, q.journalSize, q.totalExpired, 215 | q.currentAge, q.memoryLength, q.memoryBytes) 216 | } 217 | } 218 | 219 | /** 220 | * Shutdown this queue collection. All actors are asked to exit, and 221 | * any future queue requests will fail. 222 | */ 223 | def shutdown: Unit = synchronized { 224 | if (shuttingDown) { 225 | return 226 | } 227 | shuttingDown = true 228 | for ((name, q) <- queues) { 229 | // synchronous, so the journals are all officially closed before we return. 230 | q.close 231 | } 232 | queues.clear 233 | } 234 | } 235 | -------------------------------------------------------------------------------- /src/main/scala/net/lag/kestrel/KestrelHandler.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2008 Robey Pointer 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a 5 | * copy of this software and associated documentation files (the "Software"), 6 | * to deal in the Software without restriction, including without limitation 7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 | * and/or sell copies of the Software, and to permit persons to whom the 9 | * Software is furnished to do so, subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in 12 | * all copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 20 | * DEALINGS IN THE SOFTWARE. 21 | */ 22 | 23 | package net.lag.kestrel 24 | 25 | import java.net.InetSocketAddress 26 | import java.nio.ByteOrder 27 | import scala.actors.Actor 28 | import scala.actors.Actor._ 29 | import scala.collection.mutable 30 | import net.lag.configgy.{Config, Configgy, RuntimeEnvironment} 31 | import net.lag.logging.Logger 32 | import net.lag.kestrel.memcache.ProtocolException 33 | import org.apache.mina.common._ 34 | import org.apache.mina.transport.socket.nio.SocketSessionConfig 35 | 36 | 37 | class KestrelHandler(val session: IoSession, val config: Config) extends Actor { 38 | private val log = Logger.get 39 | 40 | private val IDLE_TIMEOUT = 60 41 | private val sessionID = KestrelStats.sessionID.incr 42 | private val remoteAddress = session.getRemoteAddress.asInstanceOf[InetSocketAddress] 43 | 44 | private var pendingTransaction: Option[(String, Int)] = None 45 | 46 | // used internally to indicate a client error: tried to close a transaction on the wrong queue. 47 | private class MismatchedQueueException extends Exception 48 | 49 | 50 | if (session.getTransportType == TransportType.SOCKET) { 51 | session.getConfig.asInstanceOf[SocketSessionConfig].setReceiveBufferSize(2048) 52 | } 53 | 54 | // config can be null in unit tests 55 | val idleTimeout = if (config == null) IDLE_TIMEOUT else config.getInt("timeout", IDLE_TIMEOUT) 56 | if (idleTimeout > 0) { 57 | session.setIdleTime(IdleStatus.BOTH_IDLE, idleTimeout) 58 | } 59 | 60 | KestrelStats.sessions.incr 61 | KestrelStats.totalConnections.incr 62 | log.debug("New session %d from %s:%d", sessionID, remoteAddress.getHostName, remoteAddress.getPort) 63 | start 64 | 65 | def act = { 66 | loop { 67 | react { 68 | case MinaMessage.SessionOpened => 69 | 70 | case MinaMessage.MessageReceived(msg) => handle(msg.asInstanceOf[memcache.Request]) 71 | 72 | case MinaMessage.MessageSent(msg) => 73 | 74 | case MinaMessage.ExceptionCaught(cause) => { 75 | cause.getCause match { 76 | case _: ProtocolException => writeResponse("CLIENT_ERROR\r\n") 77 | case _ => 78 | log.error(cause, "Exception caught on session %d: %s", sessionID, cause.getMessage) 79 | writeResponse("ERROR\r\n") 80 | } 81 | session.close 82 | } 83 | 84 | case MinaMessage.SessionClosed => 85 | log.debug("End of session %d", sessionID) 86 | abortAnyTransaction 87 | KestrelStats.sessions.decr 88 | exit() 89 | 90 | case MinaMessage.SessionIdle(status) => 91 | log.debug("Idle timeout on session %s", session) 92 | session.close 93 | } 94 | } 95 | } 96 | 97 | private def writeResponse(out: String) = { 98 | val bytes = out.getBytes 99 | session.write(new memcache.Response(ByteBuffer.wrap(bytes))) 100 | } 101 | 102 | private def writeResponse(out: String, data: Array[Byte]) = { 103 | val bytes = out.getBytes 104 | val buffer = ByteBuffer.allocate(bytes.length + data.length + 7) 105 | buffer.put(bytes) 106 | buffer.put(data) 107 | buffer.put("\r\nEND\r\n".getBytes) 108 | buffer.flip 109 | KestrelStats.bytesWritten.incr(buffer.capacity) 110 | session.write(new memcache.Response(buffer)) 111 | } 112 | 113 | private def handle(request: memcache.Request) = { 114 | request.line(0) match { 115 | case "GET" => get(request.line(1)) 116 | case "SET" => 117 | try { 118 | set(request.line(1), request.line(2).toInt, request.line(3).toInt, request.data.get) 119 | } catch { 120 | case e: NumberFormatException => 121 | throw new memcache.ProtocolException("bad request: " + request) 122 | } 123 | case "STATS" => stats 124 | case "SHUTDOWN" => shutdown 125 | case "RELOAD" => 126 | Configgy.reload 127 | session.write("Reloaded config.\r\n") 128 | } 129 | } 130 | 131 | private def get(name: String): Unit = { 132 | var key = name 133 | var timeout = 0 134 | var closing = false 135 | var opening = false 136 | if (name contains '/') { 137 | val options = name.split("/") 138 | key = options(0) 139 | for (i <- 1 until options.length) { 140 | val opt = options(i) 141 | if (opt startsWith "t=") { 142 | timeout = opt.substring(2).toInt 143 | } 144 | if (opt == "close") closing = true 145 | if (opt == "open") opening = true 146 | } 147 | } 148 | log.debug("get q=%s t=%d open=%s close=%s", key, timeout, opening, closing) 149 | 150 | try { 151 | if (closing) { 152 | if (!closeTransaction(key)) { 153 | log.warning("Attempt to close a non-existent transaction on '%s' (sid %d, %s:%d)", 154 | key, sessionID, remoteAddress.getHostName, remoteAddress.getPort) 155 | writeResponse("ERROR\r\n") 156 | session.close 157 | } else if (!opening) { 158 | writeResponse("END\r\n") 159 | } 160 | } 161 | if (opening || !closing) { 162 | if (pendingTransaction.isDefined) { 163 | log.warning("Attempt to perform a non-transactional fetch with an open transaction on " + 164 | " '%s' (sid %d, %s:%d)", key, sessionID, remoteAddress.getHostName, 165 | remoteAddress.getPort) 166 | writeResponse("ERROR\r\n") 167 | session.close 168 | return 169 | } 170 | KestrelStats.getRequests.incr 171 | Kestrel.queues.remove(key, timeout, opening) { 172 | case None => 173 | writeResponse("END\r\n") 174 | case Some(item) => 175 | log.debug("get %s", item) 176 | if (opening) pendingTransaction = Some((key, item.xid)) 177 | writeResponse("VALUE " + key + " 0 " + item.data.length + "\r\n", item.data) 178 | } 179 | } 180 | } catch { 181 | case e: MismatchedQueueException => 182 | log.warning("Attempt to close a transaction on the wrong queue '%s' (sid %d, %s:%d)", 183 | key, sessionID, remoteAddress.getHostName, remoteAddress.getPort) 184 | writeResponse("ERROR\r\n") 185 | session.close 186 | } 187 | } 188 | 189 | // returns true if a transaction was actually closed. 190 | private def closeTransaction(name: String): Boolean = { 191 | pendingTransaction match { 192 | case None => false 193 | case Some((qname, xid)) => 194 | if (qname != name) { 195 | throw new MismatchedQueueException 196 | } else { 197 | Kestrel.queues.confirmRemove(qname, xid) 198 | pendingTransaction = None 199 | } 200 | true 201 | } 202 | } 203 | 204 | private def abortAnyTransaction() = { 205 | pendingTransaction match { 206 | case None => 207 | case Some((qname, xid)) => 208 | Kestrel.queues.unremove(qname, xid) 209 | pendingTransaction = None 210 | } 211 | } 212 | 213 | private def set(name: String, flags: Int, expiry: Int, data: Array[Byte]) = { 214 | KestrelStats.setRequests.incr 215 | if (Kestrel.queues.add(name, data, expiry)) { 216 | writeResponse("STORED\r\n") 217 | } else { 218 | writeResponse("NOT_STORED\r\n") 219 | } 220 | } 221 | 222 | private def stats = { 223 | var report = new mutable.ArrayBuffer[(String, String)] 224 | report += (("uptime", Kestrel.uptime.toString)) 225 | report += (("time", (Time.now / 1000).toString)) 226 | report += (("version", Kestrel.runtime.jarVersion)) 227 | report += (("curr_items", Kestrel.queues.currentItems.toString)) 228 | report += (("total_items", Kestrel.queues.totalAdded.toString)) 229 | report += (("bytes", Kestrel.queues.currentBytes.toString)) 230 | report += (("curr_connections", KestrelStats.sessions.toString)) 231 | report += (("total_connections", KestrelStats.totalConnections.toString)) 232 | report += (("cmd_get", KestrelStats.getRequests.toString)) 233 | report += (("cmd_set", KestrelStats.setRequests.toString)) 234 | report += (("get_hits", Kestrel.queues.queueHits.toString)) 235 | report += (("get_misses", Kestrel.queues.queueMisses.toString)) 236 | report += (("bytes_read", KestrelStats.bytesRead.toString)) 237 | report += (("bytes_written", KestrelStats.bytesWritten.toString)) 238 | report += (("limit_maxbytes", "0")) // ??? 239 | 240 | for (qName <- Kestrel.queues.queueNames) { 241 | val s = Kestrel.queues.stats(qName) 242 | report += (("queue_" + qName + "_items", s.items.toString)) 243 | report += (("queue_" + qName + "_bytes", s.bytes.toString)) 244 | report += (("queue_" + qName + "_total_items", s.totalItems.toString)) 245 | report += (("queue_" + qName + "_logsize", s.journalSize.toString)) 246 | report += (("queue_" + qName + "_expired_items", s.totalExpired.toString)) 247 | report += (("queue_" + qName + "_mem_items", s.memoryItems.toString)) 248 | report += (("queue_" + qName + "_mem_bytes", s.memoryBytes.toString)) 249 | report += (("queue_" + qName + "_age", s.currentAge.toString)) 250 | } 251 | 252 | val summary = { 253 | for ((key, value) <- report) yield "STAT %s %s".format(key, value) 254 | }.mkString("", "\r\n", "\r\nEND\r\n") 255 | writeResponse(summary) 256 | } 257 | 258 | private def shutdown = { 259 | Kestrel.shutdown 260 | } 261 | } 262 | -------------------------------------------------------------------------------- /src/test/scala/net/lag/kestrel/ServerSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2008 Robey Pointer 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a 5 | * copy of this software and associated documentation files (the "Software"), 6 | * to deal in the Software without restriction, including without limitation 7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 | * and/or sell copies of the Software, and to permit persons to whom the 9 | * Software is furnished to do so, subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in 12 | * all copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 20 | * DEALINGS IN THE SOFTWARE. 21 | */ 22 | 23 | package net.lag.kestrel 24 | 25 | import java.io.File 26 | import java.net.Socket 27 | import scala.collection.Map 28 | import net.lag.configgy.Config 29 | import net.lag.logging.Logger 30 | import org.specs._ 31 | 32 | 33 | object ServerSpec extends Specification with TestHelper { 34 | 35 | var config: Config = null 36 | 37 | def makeServer = { 38 | config = new Config 39 | config("host") = "localhost" 40 | config("port") = 22122 41 | config("queue_path") = canonicalFolderName 42 | config("max_journal_size") = 16 * 1024 43 | config("log.console") = true 44 | config("log.level") = "debug" 45 | config("log.filename") = "/tmp/foo" 46 | 47 | // make a queue specify max_items and max_age 48 | config("queues.weather_updates.max_items") = 1500000 49 | config("queues.weather_updates.max_age") = 1800 50 | 51 | Kestrel.startup(config) 52 | } 53 | 54 | 55 | "Server" should { 56 | doAfter { 57 | Kestrel.shutdown 58 | } 59 | 60 | "configure per-queue" in { 61 | withTempFolder { 62 | makeServer 63 | Kestrel.queues.queue("starship").map(_.maxItems) mustEqual Some(Math.MAX_INT) 64 | Kestrel.queues.queue("starship").map(_.maxAge) mustEqual Some(0) 65 | Kestrel.queues.queue("weather_updates").map(_.maxItems) mustEqual Some(1500000) 66 | Kestrel.queues.queue("weather_updates").map(_.maxAge) mustEqual Some(1800) 67 | config("queues.starship.max_items") = 9999 68 | Kestrel.queues.queue("starship").map(_.maxItems) mustEqual Some(9999) 69 | } 70 | } 71 | 72 | "set and get one entry" in { 73 | withTempFolder { 74 | makeServer 75 | val v = (Math.random * 0x7fffffff).toInt 76 | val client = new TestClient("localhost", 22122) 77 | client.get("test_one_entry") mustEqual "" 78 | client.set("test_one_entry", v.toString) mustEqual "STORED" 79 | client.get("test_one_entry") mustEqual v.toString 80 | client.get("test_one_entry") mustEqual "" 81 | } 82 | } 83 | 84 | "set with expiry" in { 85 | withTempFolder { 86 | makeServer 87 | val v = (Math.random * 0x7fffffff).toInt 88 | val client = new TestClient("localhost", 22122) 89 | client.get("test_set_with_expiry") mustEqual "" 90 | client.set("test_set_with_expiry", (v + 2).toString, (Time.now / 1000).toInt) mustEqual "STORED" 91 | client.set("test_set_with_expiry", v.toString) mustEqual "STORED" 92 | Time.advance(1000) 93 | client.get("test_set_with_expiry") mustEqual v.toString 94 | } 95 | } 96 | 97 | "commit a transactional get" in { 98 | withTempFolder { 99 | makeServer 100 | val v = (Math.random * 0x7fffffff).toInt 101 | val client = new TestClient("localhost", 22122) 102 | client.set("commit", v.toString) mustEqual "STORED" 103 | 104 | val client2 = new TestClient("localhost", 22122) 105 | val client3 = new TestClient("localhost", 22122) 106 | var stats = client3.stats 107 | stats("queue_commit_items") mustEqual "1" 108 | stats("queue_commit_total_items") mustEqual "1" 109 | stats("queue_commit_bytes") mustEqual v.toString.length.toString 110 | 111 | client2.get("commit/open") mustEqual v.toString 112 | stats = client3.stats 113 | stats("queue_commit_items") mustEqual "0" 114 | stats("queue_commit_total_items") mustEqual "1" 115 | stats("queue_commit_bytes") mustEqual "0" 116 | 117 | client2.get("commit/close") mustEqual "" 118 | stats = client3.stats 119 | stats("queue_commit_items") mustEqual "0" 120 | stats("queue_commit_total_items") mustEqual "1" 121 | stats("queue_commit_bytes") mustEqual "0" 122 | 123 | client2.disconnect 124 | Thread.sleep(10) 125 | stats = client3.stats 126 | stats("queue_commit_items") mustEqual "0" 127 | stats("queue_commit_total_items") mustEqual "1" 128 | stats("queue_commit_bytes") mustEqual "0" 129 | } 130 | } 131 | 132 | "auto-rollback a transaction on disconnect" in { 133 | withTempFolder { 134 | makeServer 135 | val v = (Math.random * 0x7fffffff).toInt 136 | val client = new TestClient("localhost", 22122) 137 | client.set("auto-rollback", v.toString) mustEqual "STORED" 138 | 139 | val client2 = new TestClient("localhost", 22122) 140 | client2.get("auto-rollback/open") mustEqual v.toString 141 | val client3 = new TestClient("localhost", 22122) 142 | client3.get("auto-rollback") mustEqual "" 143 | var stats = client3.stats 144 | stats("queue_auto-rollback_items") mustEqual "0" 145 | stats("queue_auto-rollback_total_items") mustEqual "1" 146 | stats("queue_auto-rollback_bytes") mustEqual "0" 147 | 148 | // oops, client2 dies before committing! 149 | client2.disconnect 150 | Thread.sleep(10) 151 | stats = client3.stats 152 | stats("queue_auto-rollback_items") mustEqual "1" 153 | stats("queue_auto-rollback_total_items") mustEqual "1" 154 | stats("queue_auto-rollback_bytes") mustEqual v.toString.length.toString 155 | 156 | // subsequent fetch must get the same data item back. 157 | client3.get("auto-rollback/open") mustEqual v.toString 158 | stats = client3.stats 159 | stats("queue_auto-rollback_items") mustEqual "0" 160 | stats("queue_auto-rollback_total_items") mustEqual "1" 161 | stats("queue_auto-rollback_bytes") mustEqual "0" 162 | } 163 | } 164 | 165 | "auto-commit cycles of transactional gets" in { 166 | withTempFolder { 167 | makeServer 168 | val v = (Math.random * 0x7fffffff).toInt 169 | val client = new TestClient("localhost", 22122) 170 | client.set("auto-commit", v.toString) mustEqual "STORED" 171 | client.set("auto-commit", (v + 1).toString) mustEqual "STORED" 172 | client.set("auto-commit", (v + 2).toString) mustEqual "STORED" 173 | 174 | val client2 = new TestClient("localhost", 22122) 175 | client2.get("auto-commit/open") mustEqual v.toString 176 | client2.get("auto-commit/close/open") mustEqual (v + 1).toString 177 | client2.get("auto-commit/close/open") mustEqual (v + 2).toString 178 | client2.disconnect 179 | Thread.sleep(10) 180 | 181 | val client3 = new TestClient("localhost", 22122) 182 | client3.get("auto-commit") mustEqual (v + 2).toString 183 | 184 | var stats = client3.stats 185 | stats("queue_auto-commit_items") mustEqual "0" 186 | stats("queue_auto-commit_total_items") mustEqual "3" 187 | stats("queue_auto-commit_bytes") mustEqual "0" 188 | } 189 | } 190 | 191 | "age" in { 192 | withTempFolder { 193 | makeServer 194 | val client = new TestClient("localhost", 22122) 195 | client.set("test_age", "nibbler") mustEqual "STORED" 196 | Time.advance(1000) 197 | client.get("test_age") mustEqual "nibbler" 198 | client.stats.contains("queue_test_age_age") mustEqual true 199 | client.stats("queue_test_age_age").toInt >= 1000 mustEqual true 200 | } 201 | } 202 | 203 | "rotate logs" in { 204 | withTempFolder { 205 | makeServer 206 | var v = "x" 207 | for (val i <- 1.to(13)) { v = v + v } // 8192 208 | 209 | val client = new TestClient("localhost", 22122) 210 | 211 | client.set("test_log_rotation", v) mustEqual "STORED" 212 | new File(folderName + "/test_log_rotation").length mustEqual 8192 + 16 + 5 213 | client.get("test_log_rotation") mustEqual v 214 | new File(folderName + "/test_log_rotation").length mustEqual 8192 + 16 + 5 + 1 215 | 216 | client.get("test_log_rotation") mustEqual "" 217 | new File(folderName + "/test_log_rotation").length mustEqual 8192 + 16 + 5 + 1 218 | 219 | client.set("test_log_rotation", v) mustEqual "STORED" 220 | new File(folderName + "/test_log_rotation").length mustEqual 2 * (8192 + 16 + 5) + 1 221 | client.get("test_log_rotation") mustEqual v 222 | new File(folderName + "/test_log_rotation").length mustEqual 5 223 | new File(folderName).listFiles.length mustEqual 1 224 | } 225 | } 226 | 227 | "collect stats" in { 228 | withTempFolder { 229 | makeServer 230 | val client = new TestClient("localhost", 22122) 231 | val stats = client.stats 232 | val basicStats = Array("bytes", "time", "limit_maxbytes", "cmd_get", "version", 233 | "bytes_written", "cmd_set", "get_misses", "total_connections", 234 | "curr_connections", "curr_items", "uptime", "get_hits", "total_items", 235 | "bytes_read") 236 | for (val key <- basicStats) { stats contains key mustEqual true } 237 | } 238 | } 239 | 240 | "return a valid response for an unknown command" in { 241 | withTempFolder { 242 | makeServer 243 | new TestClient("localhost", 22122).add("cheese", "swiss") mustEqual "CLIENT_ERROR" 244 | } 245 | } 246 | 247 | "disconnect and reconnect correctly" in { 248 | withTempFolder { 249 | makeServer 250 | val v = (Math.random * 0x7fffffff).toInt 251 | val client = new TestClient("localhost", 22122) 252 | client.set("disconnecting", v.toString) 253 | client.disconnect 254 | client.connect 255 | client.get("disconnecting") mustEqual v.toString 256 | } 257 | } 258 | } 259 | } 260 | -------------------------------------------------------------------------------- /src/main/scala/net/lag/kestrel/Journal.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2008 Robey Pointer 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a 5 | * copy of this software and associated documentation files (the "Software"), 6 | * to deal in the Software without restriction, including without limitation 7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 | * and/or sell copies of the Software, and to permit persons to whom the 9 | * Software is furnished to do so, subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in 12 | * all copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 20 | * DEALINGS IN THE SOFTWARE. 21 | */ 22 | 23 | package net.lag.kestrel 24 | 25 | import net.lag.logging.Logger 26 | import java.io._ 27 | import java.nio.{ByteBuffer, ByteOrder} 28 | import java.nio.channels.FileChannel 29 | 30 | 31 | // returned from journal replay 32 | abstract case class JournalItem 33 | object JournalItem { 34 | case class Add(item: QItem) extends JournalItem 35 | case object Remove extends JournalItem 36 | case object RemoveTentative extends JournalItem 37 | case class SavedXid(xid: Int) extends JournalItem 38 | case class Unremove(xid: Int) extends JournalItem 39 | case class ConfirmRemove(xid: Int) extends JournalItem 40 | case object EndOfFile extends JournalItem 41 | } 42 | 43 | 44 | /** 45 | * Codes for working with the journal file for a PersistentQueue. 46 | */ 47 | class Journal(queuePath: String) { 48 | 49 | /* in theory, you might want to sync the file after each 50 | * transaction. however, the original starling doesn't. 51 | * i think if you can cope with a truncated journal file, 52 | * this is fine, because a non-synced file only matters on 53 | * catastrophic disk/machine failure. 54 | */ 55 | 56 | private val log = Logger.get 57 | 58 | private var writer: FileChannel = null 59 | private var reader: Option[FileChannel] = None 60 | private var replayer: Option[FileChannel] = None 61 | 62 | var size: Long = 0 63 | 64 | // small temporary buffer for formatting operations into the journal: 65 | private val buffer = new Array[Byte](16) 66 | private val byteBuffer = ByteBuffer.wrap(buffer) 67 | byteBuffer.order(ByteOrder.LITTLE_ENDIAN) 68 | 69 | private val CMD_ADD = 0 70 | private val CMD_REMOVE = 1 71 | private val CMD_ADDX = 2 72 | private val CMD_REMOVE_TENTATIVE = 3 73 | private val CMD_SAVE_XID = 4 74 | private val CMD_UNREMOVE = 5 75 | private val CMD_CONFIRM_REMOVE = 6 76 | 77 | 78 | def open(): Unit = { 79 | writer = new FileOutputStream(queuePath, true).getChannel 80 | } 81 | 82 | def roll(): Unit = { 83 | writer.close 84 | val backupFile = new File(queuePath + "." + Time.now) 85 | new File(queuePath).renameTo(backupFile) 86 | open 87 | size = 0 88 | backupFile.delete 89 | } 90 | 91 | def close(): Unit = { 92 | writer.close 93 | for (r <- reader) r.close 94 | reader = None 95 | } 96 | 97 | def inReadBehind(): Boolean = reader.isDefined 98 | 99 | def add(item: QItem) = { 100 | val blob = ByteBuffer.wrap(pack(item)) 101 | byteBuffer.clear 102 | byteBuffer.put(CMD_ADDX.toByte) 103 | byteBuffer.putInt(blob.limit) 104 | byteBuffer.flip 105 | do { 106 | writer.write(byteBuffer) 107 | } while (byteBuffer.position < byteBuffer.limit) 108 | do { 109 | writer.write(blob) 110 | } while (blob.position < blob.limit) 111 | size += (5 + blob.limit) 112 | } 113 | 114 | def remove() = { 115 | byteBuffer.clear 116 | byteBuffer.put(CMD_REMOVE.toByte) 117 | byteBuffer.flip 118 | while (byteBuffer.position < byteBuffer.limit) { 119 | writer.write(byteBuffer) 120 | } 121 | size += 1 122 | } 123 | 124 | def removeTentative() = { 125 | byteBuffer.clear 126 | byteBuffer.put(CMD_REMOVE_TENTATIVE.toByte) 127 | byteBuffer.flip 128 | while (byteBuffer.position < byteBuffer.limit) { 129 | writer.write(byteBuffer) 130 | } 131 | size += 1 132 | } 133 | 134 | def saveXid(xid: Int) = { 135 | byteBuffer.clear 136 | byteBuffer.put(CMD_SAVE_XID.toByte) 137 | byteBuffer.putInt(xid) 138 | byteBuffer.flip 139 | while (byteBuffer.position < byteBuffer.limit) { 140 | writer.write(byteBuffer) 141 | } 142 | size += 5 143 | } 144 | 145 | def unremove(xid: Int) = { 146 | byteBuffer.clear 147 | byteBuffer.put(CMD_UNREMOVE.toByte) 148 | byteBuffer.putInt(xid) 149 | byteBuffer.flip 150 | while (byteBuffer.position < byteBuffer.limit) { 151 | writer.write(byteBuffer) 152 | } 153 | size += 5 154 | } 155 | 156 | def confirmRemove(xid: Int) = { 157 | byteBuffer.clear 158 | byteBuffer.put(CMD_CONFIRM_REMOVE.toByte) 159 | byteBuffer.putInt(xid) 160 | byteBuffer.flip 161 | while (byteBuffer.position < byteBuffer.limit) { 162 | writer.write(byteBuffer) 163 | } 164 | size += 5 165 | } 166 | 167 | def startReadBehind(): Unit = { 168 | val pos = if (replayer.isDefined) replayer.get.position else writer.position 169 | val rj = new FileInputStream(queuePath).getChannel 170 | rj.position(pos) 171 | reader = Some(rj) 172 | } 173 | 174 | def fillReadBehind(f: QItem => Unit): Unit = { 175 | val pos = if (replayer.isDefined) replayer.get.position else writer.position 176 | for (rj <- reader) { 177 | if (rj.position == pos) { 178 | // we've caught up. 179 | rj.close 180 | reader = None 181 | } else { 182 | readJournalEntry(rj, false) match { 183 | case JournalItem.Add(item) => f(item) 184 | case _ => 185 | } 186 | } 187 | } 188 | } 189 | 190 | def replay(name: String)(f: JournalItem => Unit): Unit = { 191 | size = 0 192 | try { 193 | val in = new FileInputStream(queuePath).getChannel 194 | replayer = Some(in) 195 | var done = false 196 | do { 197 | readJournalEntry(in, true) match { 198 | case JournalItem.EndOfFile => done = true 199 | case x: JournalItem => f(x) 200 | } 201 | } while (!done) 202 | } catch { 203 | case e: FileNotFoundException => 204 | log.info("No transaction journal for '%s'; starting with empty queue.", name) 205 | case e: IOException => 206 | log.error(e, "Exception replaying journal for '%s'", name) 207 | log.error("DATA MAY HAVE BEEN LOST!") 208 | // this can happen if the server hardware died abruptly in the middle 209 | // of writing a journal. not awesome but we should recover. 210 | } 211 | replayer = None 212 | } 213 | 214 | private def readJournalEntry(in: FileChannel, replaying: Boolean): JournalItem = { 215 | byteBuffer.rewind 216 | byteBuffer.limit(1) 217 | var x: Int = 0 218 | do { 219 | x = in.read(byteBuffer) 220 | } while (byteBuffer.position < byteBuffer.limit && x >= 0) 221 | 222 | if (x < 0) { 223 | JournalItem.EndOfFile 224 | } else { 225 | buffer(0) match { 226 | case CMD_ADD => 227 | readBlock(in) match { 228 | case None => JournalItem.EndOfFile 229 | case Some(data) => 230 | if (replaying) size += 5 + data.length 231 | JournalItem.Add(unpackOldAdd(data)) 232 | } 233 | case CMD_REMOVE => 234 | if (replaying) size += 1 235 | JournalItem.Remove 236 | case CMD_ADDX => 237 | readBlock(in) match { 238 | case None => JournalItem.EndOfFile 239 | case Some(data) => 240 | if (replaying) size += 5 + data.length 241 | JournalItem.Add(unpack(data)) 242 | } 243 | case CMD_REMOVE_TENTATIVE => 244 | if (replaying) size += 1 245 | JournalItem.RemoveTentative 246 | case CMD_SAVE_XID => 247 | readInt(in) match { 248 | case None => JournalItem.EndOfFile 249 | case Some(xid) => 250 | if (replaying) size += 5 251 | JournalItem.SavedXid(xid) 252 | } 253 | case CMD_UNREMOVE => 254 | readInt(in) match { 255 | case None => JournalItem.EndOfFile 256 | case Some(xid) => 257 | if (replaying) size += 5 258 | JournalItem.Unremove(xid) 259 | } 260 | case CMD_CONFIRM_REMOVE => 261 | readInt(in) match { 262 | case None => JournalItem.EndOfFile 263 | case Some(xid) => 264 | if (replaying) size += 5 265 | JournalItem.ConfirmRemove(xid) 266 | } 267 | case n => 268 | throw new IOException("invalid opcode in journal: " + n.toInt) 269 | } 270 | } 271 | } 272 | 273 | private def readBlock(in: FileChannel): Option[Array[Byte]] = { 274 | readInt(in) match { 275 | case None => None 276 | case Some(size) => 277 | val data = new Array[Byte](size) 278 | val dataBuffer = ByteBuffer.wrap(data) 279 | var x: Int = 0 280 | do { 281 | x = in.read(dataBuffer) 282 | } while (dataBuffer.position < dataBuffer.limit && x >= 0) 283 | if (x < 0) { 284 | None 285 | } else { 286 | Some(data) 287 | } 288 | } 289 | } 290 | 291 | private def readInt(in: FileChannel): Option[Int] = { 292 | byteBuffer.rewind 293 | byteBuffer.limit(4) 294 | var x: Int = 0 295 | do { 296 | x = in.read(byteBuffer) 297 | } while (byteBuffer.position < byteBuffer.limit && x >= 0) 298 | if (x < 0) { 299 | None 300 | } else { 301 | byteBuffer.rewind 302 | Some(byteBuffer.getInt()) 303 | } 304 | } 305 | 306 | private def pack(item: QItem): Array[Byte] = { 307 | val bytes = new Array[Byte](item.data.length + 16) 308 | val buffer = ByteBuffer.wrap(bytes) 309 | buffer.order(ByteOrder.LITTLE_ENDIAN) 310 | buffer.putLong(item.addTime) 311 | buffer.putLong(item.expiry) 312 | buffer.put(item.data) 313 | bytes 314 | } 315 | 316 | private def unpack(data: Array[Byte]): QItem = { 317 | val buffer = ByteBuffer.wrap(data) 318 | val bytes = new Array[Byte](data.length - 16) 319 | buffer.order(ByteOrder.LITTLE_ENDIAN) 320 | val addTime = buffer.getLong 321 | val expiry = buffer.getLong 322 | buffer.get(bytes) 323 | return QItem(addTime, expiry, bytes, 0) 324 | } 325 | 326 | private def unpackOldAdd(data: Array[Byte]): QItem = { 327 | val buffer = ByteBuffer.wrap(data) 328 | val bytes = new Array[Byte](data.length - 4) 329 | buffer.order(ByteOrder.LITTLE_ENDIAN) 330 | val expiry = buffer.getInt 331 | buffer.get(bytes) 332 | return QItem(Time.now, if (expiry == 0) 0 else expiry * 1000, bytes, 0) 333 | } 334 | } 335 | -------------------------------------------------------------------------------- /src/main/scala/net/lag/kestrel/PersistentQueue.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2008 Robey Pointer 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a 5 | * copy of this software and associated documentation files (the "Software"), 6 | * to deal in the Software without restriction, including without limitation 7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 | * and/or sell copies of the Software, and to permit persons to whom the 9 | * Software is furnished to do so, subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in 12 | * all copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 20 | * DEALINGS IN THE SOFTWARE. 21 | */ 22 | 23 | package net.lag.kestrel 24 | 25 | import java.io._ 26 | import java.nio.{ByteBuffer, ByteOrder} 27 | import java.nio.channels.FileChannel 28 | import java.util.concurrent.CountDownLatch 29 | import scala.actors.{Actor, TIMEOUT} 30 | import scala.collection.mutable 31 | import net.lag.configgy.{Config, Configgy, ConfigMap} 32 | import net.lag.logging.Logger 33 | 34 | 35 | case class QItem(addTime: Long, expiry: Long, data: Array[Byte], var xid: Int) 36 | 37 | 38 | class PersistentQueue(private val persistencePath: String, val name: String, 39 | val config: ConfigMap) { 40 | 41 | private case class Waiter(actor: Actor) 42 | private case object ItemArrived 43 | 44 | 45 | private val log = Logger.get 46 | 47 | // current size of all data in the queue: 48 | private var queueSize: Long = 0 49 | 50 | // # of items EVER added to the queue: 51 | private var _totalItems: Long = 0 52 | 53 | // # of items that were expired by the time they were read: 54 | private var _totalExpired: Long = 0 55 | 56 | // age (in milliseconds) of the last item read from the queue: 57 | private var _currentAge: Long = 0 58 | 59 | // # of items in the queue (including those not in memory) 60 | private var queueLength: Long = 0 61 | 62 | private var queue = new mutable.Queue[QItem] { 63 | // scala's Queue doesn't (yet?) have a way to put back. 64 | def unget(item: QItem) = prependElem(item) 65 | } 66 | private var _memoryBytes: Long = 0 67 | private var journal = new Journal(new File(persistencePath, name).getCanonicalPath) 68 | 69 | // force get/set operations to block while we're replaying any existing journal 70 | private val initialized = new CountDownLatch(1) 71 | private var closed = false 72 | 73 | // attempting to add an item after the queue reaches this size will fail. 74 | var maxItems = Math.MAX_INT 75 | 76 | // maximum expiration time for this queue (seconds). 77 | var maxAge = 0 78 | 79 | // clients waiting on an item in this queue 80 | private val waiters = new mutable.ArrayBuffer[Waiter] 81 | 82 | // track tentative removals 83 | private var xidCounter: Int = 0 84 | private val openTransactions = new mutable.HashMap[Int, QItem] 85 | 86 | def length: Long = synchronized { queueLength } 87 | 88 | def totalItems: Long = synchronized { _totalItems } 89 | 90 | def bytes: Long = synchronized { queueSize } 91 | 92 | def journalSize: Long = synchronized { journal.size } 93 | 94 | def totalExpired: Long = synchronized { _totalExpired } 95 | 96 | def currentAge: Long = synchronized { _currentAge } 97 | 98 | // mostly for unit tests. 99 | def memoryLength: Long = synchronized { queue.size } 100 | def memoryBytes: Long = synchronized { _memoryBytes } 101 | def inReadBehind = synchronized { journal.inReadBehind } 102 | 103 | 104 | config.subscribe(configure _) 105 | configure(Some(config)) 106 | 107 | def configure(c: Option[ConfigMap]) = synchronized { 108 | for (config <- c) { 109 | maxItems = config("max_items", Math.MAX_INT) 110 | maxAge = config("max_age", 0) 111 | } 112 | } 113 | 114 | private final def adjustExpiry(startingTime: Long, expiry: Long): Long = { 115 | if (maxAge > 0) { 116 | val maxExpiry = startingTime + maxAge 117 | if (expiry > 0) (expiry min maxExpiry) else maxExpiry 118 | } else { 119 | expiry 120 | } 121 | } 122 | 123 | /** 124 | * Add a value to the end of the queue, transactionally. 125 | */ 126 | def add(value: Array[Byte], expiry: Long): Boolean = { 127 | initialized.await 128 | synchronized { 129 | if (closed || queueLength >= maxItems) { 130 | false 131 | } else { 132 | val now = Time.now 133 | val item = QItem(now, adjustExpiry(now, expiry), value, 0) 134 | if (!journal.inReadBehind && queueSize >= PersistentQueue.maxMemorySize) { 135 | log.info("Dropping to read-behind for queue '%s' (%d bytes)", name, queueSize) 136 | journal.startReadBehind 137 | } 138 | _add(item) 139 | journal.add(item) 140 | if (waiters.size > 0) { 141 | waiters.remove(0).actor ! ItemArrived 142 | } 143 | true 144 | } 145 | } 146 | } 147 | 148 | def add(value: Array[Byte]): Boolean = add(value, 0) 149 | 150 | /** 151 | * Remove an item from the queue. If no item is available, an empty byte 152 | * array is returned. 153 | * 154 | * @param transaction true if this should be considered the first part 155 | * of a transaction, to be committed or rolled back (put back at the 156 | * head of the queue) 157 | */ 158 | def remove(transaction: Boolean): Option[QItem] = { 159 | initialized.await 160 | synchronized { 161 | if (closed || queueLength == 0) { 162 | None 163 | } else { 164 | val item = _remove(transaction) 165 | if (transaction) journal.removeTentative() else journal.remove() 166 | 167 | if ((queueLength == 0) && (journal.size >= PersistentQueue.maxJournalSize) && 168 | (openTransactions.size == 0)) { 169 | log.info("Rolling journal file for '%s'", name) 170 | journal.roll 171 | journal.saveXid(xidCounter) 172 | } 173 | item 174 | } 175 | } 176 | } 177 | 178 | /** 179 | * Remove an item from the queue. If no item is available, an empty byte 180 | * array is returned. 181 | */ 182 | def remove(): Option[QItem] = remove(false) 183 | 184 | def remove(timeoutAbsolute: Long, transaction: Boolean)(f: Option[QItem] => Unit): Unit = { 185 | synchronized { 186 | val item = remove(transaction) 187 | if (item.isDefined) { 188 | f(item) 189 | } else if (timeoutAbsolute == 0) { 190 | f(None) 191 | } else { 192 | val w = Waiter(Actor.self) 193 | waiters += w 194 | Actor.self.reactWithin((timeoutAbsolute - Time.now) max 0) { 195 | case ItemArrived => remove(timeoutAbsolute, transaction)(f) 196 | case TIMEOUT => synchronized { 197 | waiters -= w 198 | // race: someone could have done an add() between the timeout and grabbing the lock. 199 | Actor.self.reactWithin(0) { 200 | case ItemArrived => f(remove(transaction)) 201 | case TIMEOUT => f(remove(transaction)) 202 | } 203 | } 204 | } 205 | } 206 | } 207 | } 208 | 209 | /** 210 | * Return a transactionally-removed item to the queue. This is a rolled- 211 | * back transaction. 212 | */ 213 | def unremove(xid: Int): Unit = { 214 | initialized.await 215 | synchronized { 216 | if (!closed) { 217 | journal.unremove(xid) 218 | _unremove(xid) 219 | if (waiters.size > 0) { 220 | waiters.remove(0).actor ! ItemArrived 221 | } 222 | } 223 | } 224 | } 225 | 226 | def confirmRemove(xid: Int): Unit = { 227 | initialized.await 228 | synchronized { 229 | if (!closed) { 230 | journal.confirmRemove(xid) 231 | openTransactions.removeKey(xid) 232 | } 233 | } 234 | } 235 | 236 | /** 237 | * Close the queue's journal file. Not safe to call on an active queue. 238 | */ 239 | def close = synchronized { 240 | closed = true 241 | journal.close() 242 | } 243 | 244 | def setup(): Unit = synchronized { 245 | queueSize = 0 246 | replayJournal 247 | initialized.countDown 248 | } 249 | 250 | private final def nextXid(): Int = { 251 | do { 252 | xidCounter += 1 253 | } while (openTransactions contains xidCounter) 254 | xidCounter 255 | } 256 | 257 | private final def fillReadBehind(): Unit = { 258 | // if we're in read-behind mode, scan forward in the journal to keep memory as full as 259 | // possible. this amortizes the disk overhead across all reads. 260 | while (journal.inReadBehind && _memoryBytes < PersistentQueue.maxMemorySize) { 261 | journal.fillReadBehind { item => 262 | queue += item 263 | _memoryBytes += item.data.length 264 | } 265 | if (!journal.inReadBehind) { 266 | log.info("Coming out of read-behind for queue '%s'", name) 267 | } 268 | } 269 | } 270 | 271 | def replayJournal(): Unit = { 272 | log.info("Replaying transaction journal for '%s'", name) 273 | xidCounter = 0 274 | 275 | journal.replay(name) { 276 | case JournalItem.Add(item) => 277 | _add(item) 278 | // when processing the journal, this has to happen after: 279 | if (!journal.inReadBehind && queueSize >= PersistentQueue.maxMemorySize) { 280 | log.info("Dropping to read-behind for queue '%s' (%d bytes)", name, queueSize) 281 | journal.startReadBehind 282 | } 283 | case JournalItem.Remove => _remove(false) 284 | case JournalItem.RemoveTentative => _remove(true) 285 | case JournalItem.SavedXid(xid) => xidCounter = xid 286 | case JournalItem.Unremove(xid) => _unremove(xid) 287 | case JournalItem.ConfirmRemove(xid) => openTransactions.removeKey(xid) 288 | case x => log.error("Unexpected item in journal: %s", x) 289 | } 290 | log.info("Finished transaction journal for '%s' (%d items, %d bytes)", name, queueLength, 291 | journal.size) 292 | journal.open 293 | } 294 | 295 | 296 | // ----- internal implementations 297 | 298 | private def _add(item: QItem): Unit = { 299 | if (!journal.inReadBehind) { 300 | queue += item 301 | _memoryBytes += item.data.length 302 | } 303 | _totalItems += 1 304 | queueSize += item.data.length 305 | queueLength += 1 306 | discardExpired 307 | } 308 | 309 | private def _remove(transaction: Boolean): Option[QItem] = { 310 | discardExpired 311 | if (queue.isEmpty) return None 312 | 313 | val now = Time.now 314 | val item = queue.dequeue 315 | val len = item.data.length 316 | queueSize -= len 317 | _memoryBytes -= len 318 | queueLength -= 1 319 | val xid = if (transaction) nextXid else 0 320 | 321 | fillReadBehind 322 | _currentAge = now - item.addTime 323 | if (transaction) { 324 | item.xid = xid 325 | openTransactions(xid) = item 326 | } 327 | Some(item) 328 | } 329 | 330 | private final def discardExpired(): Unit = { 331 | if (!queue.isEmpty) { 332 | val realExpiry = adjustExpiry(queue.first.addTime, queue.first.expiry) 333 | if ((realExpiry != 0) && (realExpiry < Time.now)) { 334 | _totalExpired += 1 335 | val len = queue.dequeue.data.length 336 | queueSize -= len 337 | _memoryBytes -= len 338 | queueLength -= 1 339 | fillReadBehind 340 | discardExpired 341 | } 342 | } 343 | } 344 | 345 | private def _unremove(xid: Int) = { 346 | val item = openTransactions.removeKey(xid).get 347 | queueLength += 1 348 | queueSize += item.data.length 349 | queue unget item 350 | _memoryBytes += item.data.length 351 | } 352 | } 353 | 354 | 355 | object PersistentQueue { 356 | @volatile var maxJournalSize: Long = 16 * 1024 * 1024 357 | @volatile var maxMemorySize: Long = 128 * 1024 * 1024 358 | } 359 | -------------------------------------------------------------------------------- /src/test/scala/net/lag/kestrel/PersistentQueueSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2008 Robey Pointer 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a 5 | * copy of this software and associated documentation files (the "Software"), 6 | * to deal in the Software without restriction, including without limitation 7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 | * and/or sell copies of the Software, and to permit persons to whom the 9 | * Software is furnished to do so, subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in 12 | * all copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 20 | * DEALINGS IN THE SOFTWARE. 21 | */ 22 | 23 | package net.lag.kestrel 24 | 25 | import java.io.{File, FileInputStream} 26 | import java.util.concurrent.CountDownLatch 27 | import scala.actors.Actor.actor 28 | import net.lag.configgy.Config 29 | import org.specs._ 30 | 31 | 32 | object PersistentQueueSpec extends Specification with TestHelper { 33 | 34 | def dumpJournal(folderName: String, qname: String): String = { 35 | var rv = List[JournalItem]() 36 | new Journal(new File(folderName, qname).getCanonicalPath).replay(qname) { item => rv = item :: rv } 37 | rv.reverse map { 38 | case JournalItem.Add(item) => "add(%s)".format(new String(item.data)) 39 | case JournalItem.Remove => "remove" 40 | case JournalItem.RemoveTentative => "remove-tentative" 41 | case JournalItem.SavedXid(xid) => "xid(%d)".format(xid) 42 | case JournalItem.Unremove(xid) => "unremove(%d)".format(xid) 43 | case JournalItem.ConfirmRemove(xid) => "confirm-remove(%d)".format(xid) 44 | } mkString ", " 45 | } 46 | 47 | 48 | "PersistentQueue" should { 49 | "add and remove one item" in { 50 | withTempFolder { 51 | val q = new PersistentQueue(folderName, "work", Config.fromMap(Map.empty)) 52 | q.setup 53 | 54 | q.length mustEqual 0 55 | q.totalItems mustEqual 0 56 | q.bytes mustEqual 0 57 | q.journalSize mustEqual 0 58 | 59 | q.add("hello kitty".getBytes) 60 | 61 | q.length mustEqual 1 62 | q.totalItems mustEqual 1 63 | q.bytes mustEqual 11 64 | q.journalSize mustEqual 32 65 | 66 | new String(q.remove.get.data) mustEqual "hello kitty" 67 | 68 | q.length mustEqual 0 69 | q.totalItems mustEqual 1 70 | q.bytes mustEqual 0 71 | q.journalSize mustEqual 33 72 | 73 | q.close 74 | 75 | val f = new FileInputStream(new File(folderName, "work")) 76 | val data = new Array[Byte](33) 77 | f.read(data) 78 | for (i <- 5 until 13) data(i) = 3 79 | data.mkString(":") mustEqual "2:27:0:0:0:3:3:3:3:3:3:3:3:0:0:0:0:0:0:0:0:104:101:108:108:111:32:107:105:116:116:121:1" 80 | } 81 | } 82 | 83 | "rotate journals" in { 84 | withTempFolder { 85 | val q = new PersistentQueue(folderName, "rolling", Config.fromMap(Map.empty)) 86 | q.setup 87 | PersistentQueue.maxJournalSize = 64 88 | 89 | q.add(new Array[Byte](32)) 90 | q.add(new Array[Byte](64)) 91 | q.length mustEqual 2 92 | q.totalItems mustEqual 2 93 | q.bytes mustEqual 32 + 64 94 | q.journalSize mustEqual 32 + 64 + 16 + 16 + 5 + 5 95 | new File(folderName, "rolling").length mustEqual 32 + 64 + 16 + 16 + 5 + 5 96 | 97 | q.remove 98 | q.length mustEqual 1 99 | q.totalItems mustEqual 2 100 | q.bytes mustEqual 64 101 | q.journalSize mustEqual 32 + 64 + 16 + 16 + 5 + 5 + 1 102 | new File(folderName, "rolling").length mustEqual 32 + 64 + 16 + 16 + 5 + 5 + 1 103 | 104 | // now it should rotate: 105 | q.remove 106 | q.length mustEqual 0 107 | q.totalItems mustEqual 2 108 | q.bytes mustEqual 0 109 | q.journalSize mustEqual 5 // saved xid. 110 | new File(folderName, "rolling").length mustEqual 5 111 | 112 | PersistentQueue.maxJournalSize = 16 * 1024 * 1024 113 | } 114 | } 115 | 116 | "recover the journal after a restart" in { 117 | withTempFolder { 118 | val q = new PersistentQueue(folderName, "rolling", Config.fromMap(Map.empty)) 119 | q.setup 120 | q.add("first".getBytes) 121 | q.add("second".getBytes) 122 | new String(q.remove.get.data) mustEqual "first" 123 | q.journalSize mustEqual 5 + 6 + 16 + 16 + 5 + 5 + 1 124 | q.close 125 | 126 | val q2 = new PersistentQueue(folderName, "rolling", Config.fromMap(Map.empty)) 127 | q2.setup 128 | q2.journalSize mustEqual 5 + 6 + 16 + 16 + 5 + 5 + 1 129 | new String(q2.remove.get.data) mustEqual "second" 130 | q2.journalSize mustEqual 5 + 6 + 16 + 16 + 5 + 5 + 1 + 1 131 | q2.length mustEqual 0 132 | q2.close 133 | 134 | val q3 = new PersistentQueue(folderName, "rolling", Config.fromMap(Map.empty)) 135 | q3.setup 136 | q3.journalSize mustEqual 5 + 6 + 16 + 16 + 5 + 5 + 1 + 1 137 | q3.length mustEqual 0 138 | } 139 | } 140 | 141 | "honor max_items" in { 142 | withTempFolder { 143 | val q = new PersistentQueue(folderName, "weather_updates", Config.fromMap(Map("max_items" -> "1"))) 144 | q.setup 145 | q.add("sunny".getBytes) mustEqual true 146 | q.add("rainy".getBytes) mustEqual false 147 | q.length mustEqual 1 148 | } 149 | } 150 | 151 | "honor max_age" in { 152 | withTempFolder { 153 | val config = Config.fromMap(Map("max_age" -> "1")) 154 | val q = new PersistentQueue(folderName, "weather_updates", config) 155 | q.setup 156 | q.add("sunny".getBytes) mustEqual true 157 | q.length mustEqual 1 158 | Time.advance(1000) 159 | q.remove mustEqual None 160 | 161 | config("max_age") = 60 162 | q.add("rainy".getBytes) mustEqual true 163 | config("max_age") = 1 164 | Time.advance(1000) 165 | q.remove mustEqual None 166 | } 167 | } 168 | 169 | "drop into read-behind mode" in { 170 | withTempFolder { 171 | PersistentQueue.maxMemorySize = 1024 172 | val q = new PersistentQueue(folderName, "things", Config.fromMap(Map.empty)) 173 | q.setup 174 | for (i <- 0 until 10) { 175 | val data = new Array[Byte](128) 176 | data(0) = i.toByte 177 | q.add(data) 178 | q.inReadBehind mustEqual (i >= 8) 179 | } 180 | q.inReadBehind mustBe true 181 | q.length mustEqual 10 182 | q.bytes mustEqual 1280 183 | q.memoryLength mustEqual 8 184 | q.memoryBytes mustEqual 1024 185 | 186 | // read 1 item. queue should pro-actively read the next item in from disk. 187 | val d0 = q.remove.get.data 188 | d0(0) mustEqual 0 189 | q.inReadBehind mustBe true 190 | q.length mustEqual 9 191 | q.bytes mustEqual 1152 192 | q.memoryLength mustEqual 8 193 | q.memoryBytes mustEqual 1024 194 | 195 | // adding a new item should be ok 196 | val w10 = new Array[Byte](128) 197 | w10(0) = 10.toByte 198 | q.add(w10) 199 | q.inReadBehind mustBe true 200 | q.length mustEqual 10 201 | q.bytes mustEqual 1280 202 | q.memoryLength mustEqual 8 203 | q.memoryBytes mustEqual 1024 204 | 205 | // read again. 206 | val d1 = q.remove.get.data 207 | d1(0) mustEqual 1 208 | q.inReadBehind mustBe true 209 | q.length mustEqual 9 210 | q.bytes mustEqual 1152 211 | q.memoryLength mustEqual 8 212 | q.memoryBytes mustEqual 1024 213 | 214 | // and again. 215 | val d2 = q.remove.get.data 216 | d2(0) mustEqual 2 217 | q.inReadBehind mustBe true 218 | q.length mustEqual 8 219 | q.bytes mustEqual 1024 220 | q.memoryLength mustEqual 8 221 | q.memoryBytes mustEqual 1024 222 | 223 | for (i <- 3 until 11) { 224 | val d = q.remove.get.data 225 | d(0) mustEqual i 226 | q.inReadBehind mustBe false 227 | q.length mustEqual 10 - i 228 | q.bytes mustEqual 128 * (10 - i) 229 | q.memoryLength mustEqual 10 - i 230 | q.memoryBytes mustEqual 128 * (10 - i) 231 | } 232 | } 233 | } 234 | 235 | "drop into read-behind mode on startup" in { 236 | withTempFolder { 237 | PersistentQueue.maxMemorySize = 1024 238 | val q = new PersistentQueue(folderName, "things", Config.fromMap(Map.empty)) 239 | q.setup 240 | for (i <- 0 until 10) { 241 | val data = new Array[Byte](128) 242 | data(0) = i.toByte 243 | q.add(data) 244 | q.inReadBehind mustEqual (i >= 8) 245 | } 246 | q.inReadBehind mustBe true 247 | q.length mustEqual 10 248 | q.bytes mustEqual 1280 249 | q.memoryLength mustEqual 8 250 | q.memoryBytes mustEqual 1024 251 | q.close 252 | 253 | val q2 = new PersistentQueue(folderName, "things", Config.fromMap(Map.empty)) 254 | q2.setup 255 | 256 | q2.inReadBehind mustBe true 257 | q2.length mustEqual 10 258 | q2.bytes mustEqual 1280 259 | q2.memoryLength mustEqual 8 260 | q2.memoryBytes mustEqual 1024 261 | 262 | for (i <- 0 until 10) { 263 | val d = q2.remove.get.data 264 | d(0) mustEqual i 265 | q2.inReadBehind mustEqual (i < 2) 266 | q2.length mustEqual 9 - i 267 | q2.bytes mustEqual 128 * (9 - i) 268 | q2.memoryLength mustEqual (if (i < 2) 8 else 9 - i) 269 | q2.memoryBytes mustEqual (if (i < 2) 1024 else 128 * (9 - i)) 270 | } 271 | } 272 | } 273 | 274 | "drop into read-behind mode during journal processing, then return to ordinary times" in { 275 | withTempFolder { 276 | PersistentQueue.maxMemorySize = 1024 277 | val q = new PersistentQueue(folderName, "things", Config.fromMap(Map.empty)) 278 | q.setup 279 | for (i <- 0 until 10) { 280 | val data = new Array[Byte](128) 281 | data(0) = i.toByte 282 | q.add(data) 283 | q.inReadBehind mustEqual (i >= 8) 284 | } 285 | q.inReadBehind mustBe true 286 | q.length mustEqual 10 287 | q.bytes mustEqual 1280 288 | q.memoryLength mustEqual 8 289 | q.memoryBytes mustEqual 1024 290 | for (i <- 0 until 10) { 291 | q.remove 292 | } 293 | q.inReadBehind mustBe false 294 | q.length mustEqual 0 295 | q.bytes mustEqual 0 296 | q.memoryLength mustEqual 0 297 | q.memoryBytes mustEqual 0 298 | q.close 299 | 300 | val q2 = new PersistentQueue(folderName, "things", Config.fromMap(Map.empty)) 301 | q2.setup 302 | q2.inReadBehind mustBe false 303 | q2.length mustEqual 0 304 | q2.bytes mustEqual 0 305 | q2.memoryLength mustEqual 0 306 | q2.memoryBytes mustEqual 0 307 | } 308 | } 309 | 310 | "handle timeout reads" in { 311 | withTempFolder { 312 | PersistentQueue.maxMemorySize = 1024 313 | val q = new PersistentQueue(folderName, "things", Config.fromMap(Map.empty)) 314 | q.setup 315 | 316 | actor { 317 | Thread.sleep(100) 318 | q.add("hello".getBytes) 319 | } 320 | 321 | var rv: String = null 322 | val latch = new CountDownLatch(1) 323 | actor { 324 | q.remove(Time.now + 250, false) { item => 325 | rv = new String(item.get.data) 326 | latch.countDown 327 | } 328 | } 329 | latch.await 330 | rv mustEqual "hello" 331 | } 332 | } 333 | 334 | "correctly interleave transactions in the journal" in { 335 | withTempFolder { 336 | PersistentQueue.maxMemorySize = 1024 337 | val q = new PersistentQueue(folderName, "things", Config.fromMap(Map.empty)) 338 | q.setup 339 | q.add("house".getBytes) 340 | q.add("cat".getBytes) 341 | q.journalSize mustEqual 2 * 21 + 8 342 | 343 | val house = q.remove(true).get 344 | new String(house.data) mustEqual "house" 345 | house.xid mustEqual 1 346 | q.journalSize mustEqual 2 * 21 + 8 + 1 347 | 348 | val cat = q.remove(true).get 349 | new String(cat.data) mustEqual "cat" 350 | cat.xid mustEqual 2 351 | q.journalSize mustEqual 2 * 21 + 8 + 1 + 1 352 | 353 | q.unremove(house.xid) 354 | q.journalSize mustEqual 2 * 21 + 8 + 1 + 1 + 5 355 | 356 | q.confirmRemove(cat.xid) 357 | q.journalSize mustEqual 2 * 21 + 8 + 1 + 1 + 5 + 5 358 | q.length mustEqual 1 359 | q.bytes mustEqual 5 360 | 361 | new String(q.remove.get.data) mustEqual "house" 362 | q.length mustEqual 0 363 | q.bytes mustEqual 0 364 | 365 | q.close 366 | dumpJournal(folderName, "things") mustEqual 367 | "add(house), add(cat), remove-tentative, remove-tentative, unremove(1), confirm-remove(2), remove" 368 | 369 | // and journal is replayed correctly. 370 | val q2 = new PersistentQueue(folderName, "things", Config.fromMap(Map.empty)) 371 | q2.setup 372 | q2.length mustEqual 0 373 | q2.bytes mustEqual 0 374 | } 375 | } 376 | } 377 | } 378 | --------------------------------------------------------------------------------