├── .travis.yml ├── doc └── leader_election_flowchart.png ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── .idea ├── vcs.xml └── compiler.xml ├── src ├── main │ ├── kotlin │ │ └── org │ │ │ └── botellier │ │ │ ├── main.kt │ │ │ ├── serializer │ │ │ ├── Serializer.kt │ │ │ ├── JsonSerializer.kt │ │ │ └── ByteSerializer.kt │ │ │ ├── node │ │ │ ├── main.kt │ │ │ ├── Path.kt │ │ │ └── Node.kt │ │ │ ├── server │ │ │ ├── socket.kt │ │ │ ├── Server.kt │ │ │ ├── Request.kt │ │ │ └── Client.kt │ │ │ ├── store │ │ │ ├── PersistentStore.kt │ │ │ ├── Store.kt │ │ │ └── StoreTransaction.kt │ │ │ ├── value │ │ │ ├── ValueParser.kt │ │ │ ├── Parser.kt │ │ │ ├── Lexer.kt │ │ │ └── value.kt │ │ │ ├── log │ │ │ ├── util.kt │ │ │ ├── Entry.kt │ │ │ ├── SegmentHeader.kt │ │ │ ├── Log.kt │ │ │ └── Segment.kt │ │ │ └── command │ │ │ ├── CParameter.kt │ │ │ ├── CommandParser.kt │ │ │ ├── CValue.kt │ │ │ ├── Command.kt │ │ │ └── commands.kt │ └── proto │ │ └── org │ │ └── botellier │ │ └── log │ │ ├── SegmentHeader.proto │ │ └── Entry.proto └── test │ └── kotlin │ └── org │ └── botellier │ ├── log │ ├── SegmentHeader.kt │ ├── Entry.kt │ ├── support.kt │ ├── Log.kt │ └── Segment.kt │ ├── store │ └── StoreTransaction.kt │ ├── value │ ├── Parser.kt │ ├── value.kt │ └── Lexer.kt │ ├── serializer │ └── ByteSerializer.kt │ └── command │ ├── CommandParser.kt │ └── commands.kt ├── settings.gradle ├── LICENSE ├── README.md ├── gradlew.bat ├── .gitignore └── gradlew /.travis.yml: -------------------------------------------------------------------------------- 1 | language: java 2 | jdk: oraclejdk8 3 | -------------------------------------------------------------------------------- /doc/leader_election_flowchart.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danielrs/botellier/HEAD/doc/leader_election_flowchart.png -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danielrs/botellier/HEAD/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /src/main/kotlin/org/botellier/main.kt: -------------------------------------------------------------------------------- 1 | package org.botellier 2 | 3 | import org.botellier.server.Server 4 | 5 | fun main(args: Array) { 6 | val server = Server(password = "password") 7 | server.start() 8 | } -------------------------------------------------------------------------------- /src/main/kotlin/org/botellier/serializer/Serializer.kt: -------------------------------------------------------------------------------- 1 | package org.botellier.serializer 2 | 3 | import org.botellier.value.StoreType 4 | 5 | interface Serializer { 6 | val value: StoreType? 7 | fun serialize(): ByteArray 8 | } 9 | -------------------------------------------------------------------------------- /src/main/proto/org/botellier/log/SegmentHeader.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package org.botellier.log; 3 | option java_outer_classname = "SegmentHeaderProtos"; 4 | 5 | message SegmentHeader { 6 | uint32 id = 1; 7 | string checksum = 2; 8 | uint32 total_entries = 3; 9 | } -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | #Sun Jul 02 19:48:05 CDT 2017 2 | distributionBase=GRADLE_USER_HOME 3 | distributionPath=wrapper/dists 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | distributionUrl=https\://services.gradle.org/distributions/gradle-3.5-all.zip 7 | -------------------------------------------------------------------------------- /.idea/compiler.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /src/main/kotlin/org/botellier/node/main.kt: -------------------------------------------------------------------------------- 1 | package org.botellier.node 2 | 3 | fun newNode(): Node { 4 | val node = Node("127.0.0.1:2181") 5 | node.bootstrap() 6 | node.register() 7 | node.countReplicas() 8 | node.runForLeader() 9 | return node 10 | } 11 | 12 | fun main(args: Array) { 13 | for (i in 1..60) { 14 | newNode() 15 | } 16 | readLine() 17 | } -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * This settings file was generated by the Gradle 'init' task. 3 | * 4 | * The settings file is used to specify which projects to include in your build. 5 | * In a single project build this file can be empty or even removed. 6 | * 7 | * Detailed information about configuring a multi-project build in Gradle can be found 8 | * in the user guide at https://docs.gradle.org/3.5/userguide/multi_project_builds.html 9 | */ 10 | 11 | /* 12 | // To declare projects as part of a multi-project build use the 'include' method 13 | include 'shared' 14 | include 'api' 15 | include 'services:webservice' 16 | */ 17 | 18 | rootProject.name = 'botellier' 19 | -------------------------------------------------------------------------------- /src/main/kotlin/org/botellier/node/Path.kt: -------------------------------------------------------------------------------- 1 | package org.botellier.node 2 | 3 | /** 4 | * This class helps to get paths at the znode 5 | * service. 6 | */ 7 | class Path { 8 | companion object { 9 | const val leader = "/leader" 10 | const val leader_name = "$leader/name" 11 | val leader_version = "$leader/version" 12 | 13 | const val synced = "/synced" 14 | const val synced_name = "$synced/name" 15 | const val synced_version = "$synced/version" 16 | 17 | const val replicas = "/replicas" 18 | fun replicaPath(name: String): String = "$replicas/$name" 19 | 20 | const val changes = "/changes" 21 | fun changePath(name: String): String = "$changes/$name" 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/main/kotlin/org/botellier/server/socket.kt: -------------------------------------------------------------------------------- 1 | package org.botellier.server 2 | 3 | import java.io.BufferedInputStream 4 | import java.net.Socket 5 | 6 | /** 7 | * Waits for the InputStream until ready by given timeout. A timeout of '0' indicates to wait indefinitely. 8 | * @param timeout the time to wait for input before continuing. 9 | * @return BufferedInputStream the input stream ready to be read. 10 | */ 11 | fun Socket.waitInput(timeout: Int = 0): BufferedInputStream { 12 | val prevTimeout = this.soTimeout 13 | val stream = this.getInputStream().buffered() 14 | 15 | this.soTimeout = timeout 16 | stream.mark(1) 17 | stream.read() 18 | stream.reset() 19 | this.soTimeout = prevTimeout 20 | 21 | return stream 22 | } 23 | -------------------------------------------------------------------------------- /src/main/proto/org/botellier/log/Entry.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package org.botellier.log; 3 | option java_outer_classname = "EntryProtos"; 4 | 5 | message Entry { 6 | uint32 id = 1; 7 | oneof entry_type { 8 | DeleteEntry delete_entry = 2; 9 | SetEntry set_entry = 3; 10 | BeginTransactionEntry begin_trasaction_entry = 4; 11 | EndTransactionEntry end_transaction_entry = 5; 12 | } 13 | } 14 | 15 | // ---------------- 16 | // Entry types. 17 | // ---------------- 18 | 19 | message DeleteEntry { 20 | string key = 1; 21 | } 22 | 23 | message SetEntry { 24 | string key = 1; 25 | bytes before = 2; 26 | bytes after = 3; 27 | } 28 | 29 | message BeginTransactionEntry { 30 | } 31 | 32 | message EndTransactionEntry { 33 | } 34 | 35 | -------------------------------------------------------------------------------- /src/test/kotlin/org/botellier/log/SegmentHeader.kt: -------------------------------------------------------------------------------- 1 | package org.botellier.log 2 | 3 | import org.junit.Test 4 | import org.junit.Assert 5 | import java.io.ByteArrayOutputStream 6 | import java.security.MessageDigest 7 | 8 | class SegmentHeaderTest { 9 | @Test 10 | fun writesFixedSizeHeader() { 11 | val md = MessageDigest.getInstance("MD5") 12 | val header = SegmentHeader(md) 13 | header.update(md) 14 | 15 | val output = ByteArrayOutputStream() 16 | header.writeTo(output) 17 | 18 | val recovered = SegmentHeader.parseFrom(output.toByteArray()) 19 | Assert.assertEquals(-1, recovered.id) 20 | Assert.assertEquals(md.digest().toHexString(), recovered.checksum) 21 | Assert.assertEquals(1, recovered.totalEntries) 22 | } 23 | 24 | @Test 25 | fun messageDigestToHexString() { 26 | val md = MessageDigest.getInstance("MD5") 27 | Assert.assertEquals("d41d8cd98f00b204e9800998ecf8427e", md.digest().toHexString()) 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/main/kotlin/org/botellier/store/PersistentStore.kt: -------------------------------------------------------------------------------- 1 | package org.botellier.store 2 | 3 | import org.botellier.log.DeleteEntry 4 | import org.botellier.log.Log 5 | import org.botellier.log.SetEntry 6 | import org.botellier.value.parseValue 7 | import kotlin.system.exitProcess 8 | 9 | /** 10 | * Store that can be persisted to disk by 11 | * using org.botellier.log. 12 | * @param root the basedir for the logs. 13 | */ 14 | class PersistentStore(root: String) : Store() { 15 | val log = Log(root) 16 | 17 | init { 18 | val transaction = transaction() 19 | 20 | for (entry in log) { 21 | when (entry) { 22 | is DeleteEntry -> { 23 | transaction.begin { 24 | delete(entry.key) 25 | } 26 | } 27 | is SetEntry -> { 28 | transaction.begin { 29 | val value = parseValue(entry.after) 30 | set(entry.key, value) 31 | } 32 | } 33 | } 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/main/kotlin/org/botellier/value/ValueParser.kt: -------------------------------------------------------------------------------- 1 | package org.botellier.value 2 | 3 | import java.io.BufferedInputStream 4 | 5 | fun parseValue(input: BufferedInputStream): StoreValue { 6 | return ValueParser(input).parse() 7 | } 8 | 9 | fun parseValue(input: ByteArray): StoreValue { 10 | return ValueParser(input.inputStream().buffered()).parse() 11 | } 12 | 13 | class ValueParser(val input: BufferedInputStream) { 14 | fun parse(): StoreValue { 15 | val token = Lexer(input).lex() 16 | return parseToken(token) 17 | } 18 | 19 | private fun parseToken(token: Lexer.Token): StoreValue { 20 | return when (token) { 21 | is Lexer.IntToken -> IntValue(token.value) 22 | is Lexer.FloatToken -> FloatValue(token.value) 23 | is Lexer.StringToken -> StringValue(token.value) 24 | is Lexer.ListToken -> parseListToken(token) 25 | else -> NilValue() 26 | } 27 | } 28 | 29 | private fun parseListToken(token: Lexer.ListToken): ListValue { 30 | return token.value.map { parseToken(it) as StorePrimitive }.toValue() 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/main/kotlin/org/botellier/server/Server.kt: -------------------------------------------------------------------------------- 1 | package org.botellier.server 2 | 3 | import org.botellier.store.PersistentStore 4 | import java.net.ServerSocket 5 | import java.util.concurrent.Executor 6 | 7 | class Server(val port: Int = 6679, val password: String? = null, dbTotal: Int = 15) { 8 | val dbs = List(dbTotal, { PersistentStore("./run/db-$it") }) 9 | 10 | private val executor = HandlerExecutor() 11 | private val dispatcher = RequestDispatcher(this) 12 | 13 | fun start() { 14 | val serverSocket = ServerSocket(port) 15 | println("Server running on port $port.") 16 | 17 | while (true) { 18 | val client = Client(serverSocket.accept()) 19 | println("Client connected: ${client.socket.inetAddress.hostAddress}") 20 | executor.execute(ClientHandler(client, dispatcher)) 21 | } 22 | } 23 | 24 | fun requiresPassword(): Boolean = password != null 25 | 26 | // Inner classes. 27 | class HandlerExecutor : Executor { 28 | override fun execute(command: Runnable?) { 29 | Thread(command).start() 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Daniel Rivas 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /src/test/kotlin/org/botellier/log/Entry.kt: -------------------------------------------------------------------------------- 1 | package org.botellier.log 2 | 3 | import com.google.protobuf.ByteString 4 | import org.junit.Assert 5 | import org.junit.Test 6 | 7 | class EntryTest { 8 | @Test 9 | fun buildingDeleteEntry() { 10 | val entry = buildDeleteEntry(0) { 11 | this.key = "key" 12 | } 13 | Assert.assertTrue(entry is DeleteEntry) 14 | } 15 | 16 | @Test 17 | fun buildingSetEntry() { 18 | val entry = buildSetEntry(0) { 19 | this.key = "key" 20 | this.before = ByteString.copyFrom(byteArrayOf(48, 49, 50)) 21 | this.after = ByteString.copyFrom(byteArrayOf(51, 52, 53)) 22 | } 23 | Assert.assertTrue(entry is SetEntry) 24 | } 25 | 26 | @Test 27 | fun buildingBeginTransactionEntry() { 28 | val entry = buildBeginTransactionEntry(0) {} 29 | Assert.assertTrue(entry is BeginTransactionEntry) 30 | } 31 | 32 | @Test 33 | fun buildingEndTransactionEntry() { 34 | val entry = buildEndTransactionEntry(0) {} 35 | Assert.assertTrue(entry is EndTransactionEntry) 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/test/kotlin/org/botellier/log/support.kt: -------------------------------------------------------------------------------- 1 | package org.botellier.log 2 | 3 | import java.io.File 4 | import java.util.* 5 | 6 | /** 7 | * Calls the callback function with the folder where 8 | * the dummy files are contained. 9 | * @param n the number of dummy segment files to include in the directory. 10 | */ 11 | fun withDummy(n: Int = 0, cb: (File) -> Unit) { 12 | var folder = getFolderName() 13 | folder.mkdir() 14 | 15 | try { 16 | for (i in 0..n - 1) { 17 | File(folder, "test-segment-$i").createNewFile() 18 | } 19 | cb(folder) 20 | } catch (e: Throwable) { 21 | throw e 22 | } finally { 23 | folder.deleteRecursively() 24 | } 25 | } 26 | 27 | /** 28 | * Gets a folder with the name segments-SUFFIX where suffix 29 | * is a random number. 30 | */ 31 | private fun getFolderName(): File { 32 | val folder = File("./segments-${Math.abs(Random().nextInt())}") 33 | if (folder.exists()) { 34 | return getFolderName() 35 | } else { 36 | return folder 37 | } 38 | } 39 | 40 | /** 41 | * Deletes folder recursively. 42 | */ 43 | private fun File.deleteRecursively() { 44 | for (f in this.listFiles()) { 45 | if (f.isDirectory) { 46 | f.deleteRecursively() 47 | } else if (f.isFile) { 48 | f.delete() 49 | } 50 | } 51 | this.delete() 52 | } 53 | -------------------------------------------------------------------------------- /src/main/kotlin/org/botellier/log/util.kt: -------------------------------------------------------------------------------- 1 | package org.botellier.log 2 | 3 | import java.security.MessageDigest 4 | 5 | /** 6 | * Digests the MessageDigest without reset. 7 | * @param returns the digested data. 8 | */ 9 | fun MessageDigest.tryDigest(): ByteArray { 10 | return (this.clone() as MessageDigest).digest() 11 | } 12 | 13 | 14 | /** 15 | * Convers the given long to a byte array of 8 bytes. 16 | */ 17 | fun Long.toByteArray(): ByteArray { 18 | return byteArrayOf( 19 | (this shr 56).toByte(), 20 | (this shr 48).toByte(), 21 | (this shr 40).toByte(), 22 | (this shr 32).toByte(), 23 | (this shr 24).toByte(), 24 | (this shr 16).toByte(), 25 | (this shr 8).toByte(), 26 | (this shr 0).toByte() 27 | ) 28 | } 29 | 30 | /** 31 | * Convers the given integer to a byte array of 4 bytes. 32 | */ 33 | fun Int.toByteArray(): ByteArray { 34 | return byteArrayOf( 35 | (this shr 24).toByte(), 36 | (this shr 16).toByte(), 37 | (this shr 8).toByte(), 38 | (this shr 0).toByte() 39 | ) 40 | } 41 | 42 | /** 43 | * Converts the given byte array to an integer. 44 | */ 45 | fun ByteArray.toInt(): Int { 46 | return this.take(4).fold(0, { acc, byte -> 47 | acc or byte.toInt() 48 | }) 49 | } 50 | 51 | /** 52 | * Converts the given byte array to a long. 53 | */ 54 | fun ByteArray.toLong(): Long { 55 | return this.take(8).fold(0.toLong(), { acc, byte -> 56 | acc or byte.toLong() 57 | }) 58 | } -------------------------------------------------------------------------------- /src/main/kotlin/org/botellier/command/CParameter.kt: -------------------------------------------------------------------------------- 1 | package org.botellier.command 2 | 3 | import kotlin.reflect.KMutableProperty 4 | import kotlin.reflect.full.createType 5 | import kotlin.reflect.full.isSubtypeOf 6 | 7 | class CParameter(private val command: Command, private val property: KMutableProperty, val isOptional: Boolean) { 8 | val name: String = property.name 9 | 10 | fun get(): Any { 11 | return property.getter.call(command) 12 | } 13 | 14 | fun set(value: CValue) { 15 | property.setter.call(command, value) 16 | } 17 | 18 | // Checking functions. 19 | val type = property.returnType 20 | 21 | val isInt: Boolean = property.returnType.isSubtypeOf(CValue.Primitive.Int::class.createType()) 22 | val isFloat: Boolean = property.returnType.isSubtypeOf(CValue.Primitive.Float::class.createType()) 23 | val isString: Boolean = property.returnType.isSubtypeOf(CValue.Primitive.String::class.createType()) 24 | val isAny: Boolean = property.returnType.isSubtypeOf(CValue.Primitive::class.createType()) 25 | 26 | val isIntArray: Boolean = property.returnType.isSubtypeOf(CValue.Array.Int::class.createType()) 27 | val isFloatArray: Boolean = property.returnType.isSubtypeOf(CValue.Array.Float::class.createType()) 28 | val isStringArray: Boolean = property.returnType.isSubtypeOf(CValue.Array.String::class.createType()) 29 | val isPairArray: Boolean = property.returnType.isSubtypeOf(CValue.Array.Pair::class.createType()) 30 | val isAnyArray: Boolean = property.returnType.isSubtypeOf(CValue.Array::class.createType()) 31 | } 32 | -------------------------------------------------------------------------------- /src/main/kotlin/org/botellier/store/Store.kt: -------------------------------------------------------------------------------- 1 | package org.botellier.store 2 | 3 | import org.botellier.value.MapValue 4 | import org.botellier.value.NilValue 5 | import org.botellier.value.StoreValue 6 | 7 | /** 8 | * Read only store. 9 | */ 10 | interface ReadStore { 11 | val keys: Set 12 | val size: Int 13 | fun get(key: String): StoreValue 14 | } 15 | 16 | /** 17 | * Write store. 18 | */ 19 | interface WriteStore { 20 | fun transaction(): StoreTransaction 21 | } 22 | 23 | /** 24 | * Store class with getter method for key/values. The only 25 | * way to add/modify the stored values is through a transaction. 26 | * @see StoreTransaction 27 | */ 28 | open class Store(initialMap: MapValue = MapValue()) : ReadStore, WriteStore { 29 | protected val map = initialMap 30 | override val keys get() = map.unwrap().keys 31 | override val size get() = map.size 32 | 33 | /** 34 | * Gets the value of the given key. 35 | * @param key the key to lookup. 36 | * @returns the value or NilValue if key is not found. 37 | */ 38 | override fun get(key: String): StoreValue { 39 | return map.unwrap().get(key) ?: NilValue() 40 | } 41 | 42 | /** 43 | * Returns a StoreTransaction instance that allows 44 | * modification of this store. 45 | * @returns the StoreTransaction instance. 46 | */ 47 | override fun transaction(): StoreTransaction { 48 | return StoreTransaction(map) 49 | } 50 | } 51 | 52 | // ---------------- 53 | // Exceptions 54 | // ---------------- 55 | 56 | sealed class StoreException(msg: String) : Throwable(msg) { 57 | class InvalidTypeException(key: String, type: String) 58 | : StoreException("Invalid key type: '$key' is '$type'.") 59 | } 60 | -------------------------------------------------------------------------------- /src/test/kotlin/org/botellier/store/StoreTransaction.kt: -------------------------------------------------------------------------------- 1 | package org.botellier.store 2 | 3 | import org.botellier.value.IntValue 4 | import org.botellier.value.NilValue 5 | import org.junit.Test 6 | import org.junit.Assert 7 | 8 | class StoreTransactionTest { 9 | @Test 10 | fun cachingDuringTransaction() { 11 | val store = Store() 12 | val transaction = store.transaction() 13 | transaction.begin { 14 | set("one", IntValue(1)) 15 | Assert.assertEquals(IntValue(1), get("one")) 16 | Assert.assertEquals(NilValue(), store.get("one")) 17 | } 18 | Assert.assertEquals(IntValue(1), store.get("one")) 19 | } 20 | 21 | @Test 22 | fun updatingExistingValues() { 23 | val store = Store() 24 | val transaction = store.transaction() 25 | 26 | transaction.begin { 27 | set("one", IntValue(1)) 28 | set("two", IntValue(2)) 29 | set("three", IntValue(3)) 30 | } 31 | 32 | transaction.begin { 33 | update("one") { IntValue(it.unwrap() * 10) } 34 | update("two") { IntValue(it.unwrap() * 10) } 35 | update("three") { IntValue(it.unwrap() * 10) } 36 | } 37 | 38 | Assert.assertEquals(IntValue(10), store.get("one")) 39 | Assert.assertEquals(IntValue(20), store.get("two")) 40 | Assert.assertEquals(IntValue(30), store.get("three")) 41 | } 42 | 43 | @Test 44 | fun updatingUnexistingValues() { 45 | val store = Store() 46 | val transaction = store.transaction() 47 | 48 | transaction.begin { 49 | mupdate("one") { IntValue((it?.unwrap() ?: 9) * 10) } 50 | } 51 | 52 | Assert.assertEquals(IntValue(90), store.get("one")) 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/test/kotlin/org/botellier/value/Parser.kt: -------------------------------------------------------------------------------- 1 | package org.botellier.value 2 | 3 | import org.botellier.value.parse 4 | import org.junit.Assert 5 | import org.junit.Test 6 | 7 | class ParserTest { 8 | 9 | @Test 10 | fun parseInt() { 11 | val message = "$3\r\n100\r\n" 12 | var value = 0 13 | 14 | parse(message) { 15 | value = int() 16 | } 17 | 18 | Assert.assertEquals(value, 100) 19 | } 20 | 21 | @Test 22 | fun parseFloat() { 23 | val message = "$4\r\n10.1\r\n" 24 | var value = 0.0 25 | 26 | parse(message) { 27 | value = float() 28 | } 29 | 30 | Assert.assertEquals(value, 10.1, 0.001) 31 | } 32 | 33 | @Test 34 | fun parseString() { 35 | val message = "$3\r\none\r\n" 36 | var value = "" 37 | 38 | parse(message) { 39 | value = string() 40 | } 41 | 42 | Assert.assertEquals(value, "one") 43 | } 44 | 45 | @Test 46 | fun parseList() { 47 | val message = "*3\r\n$3\r\none\r\n$3\r\ntwo\r\n$5\r\nthree\r\n" 48 | val values = mutableListOf() 49 | 50 | parse(message) { 51 | while (true) { 52 | try { 53 | values.add(string()) 54 | } catch(e: Throwable) { 55 | break 56 | } 57 | } 58 | } 59 | 60 | Assert.assertEquals(listOf("one", "two", "three"), values) 61 | } 62 | 63 | @Test 64 | fun parseMixed() { 65 | val message = "*3\r\n$3\r\n100\r\n$7\r\n200.200\r\n$13\r\nthree-hundred\r\n" 66 | var int = 0 67 | var float = 0.0 68 | var string = "" 69 | 70 | parse(message) { 71 | int = int() 72 | float = float() 73 | string = string() 74 | } 75 | 76 | Assert.assertEquals(100, int) 77 | Assert.assertEquals(200.200, float, 0.001) 78 | Assert.assertEquals("three-hundred", string) 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [zookeeper]: https://github.com/apache/zookeeper 2 | 3 | [![Build Status](https://travis-ci.org/danielrs/botellier.svg?branch=replication)](https://travis-ci.org/danielrs/botellier) 4 | 5 | ## Botellier 6 | 7 | A distributed key-value data store. It aims to be a simple Redis clone for the JVM. 8 | 9 | ### Replication Scheme 10 | 11 | To keep things simple, Botellier uses single-leader replication. Each replica is a whole copy of the leader, and no sharding/partitioning is used. 12 | 13 | All the nodes that form the quorum can be either three types: `leader`, `synced replica` or `replica`. All writes request go through the leader, who waits for the synced replica to proccess the same requests before returning to the client. All other replicas process the requests asynchronously. 14 | 15 | #### Election process 16 | 17 | Initially, all nodes are of type `replica` which then try to become the `synced replica`, usually the most up-to-date node is the one to do it. After that, the only node that can become a `leader` is a synced replica; so in the event that the leader dies, the synced replica automatically becomes the leader, and the next most up-to-date common replica takes its place. 18 | 19 | Here's a flowchart that shows the process: 20 | 21 | ![Flowchart](https://raw.githubusercontent.com/danielrs/botellier/replication/doc/leader_election_flowchart.png) 22 | 23 | Coordination between proccesses is done using [Zookeper][zookeeper]. 24 | 25 | ### Command support 26 | 27 | As of now, most commonly used commands are supported, such as connection, strings, lists and general ones like delete, 28 | rename, etc. For a complete list of supported commands check [here](https://github.com/danielrs/botellier/blob/master/src/main/kotlin/org/botellier/command/commands.kt). 29 | 30 | ### Testing 31 | 32 | The project includes an entry point that starts a server on the default port (6679) with the 33 | password set to 'password'. After starting the server try [corkscrew](https://github.com/danielrs/corkscrew) for 34 | connecting to it and executing commands. Here's an example: 35 | 36 | [![asciicast](https://asciinema.org/a/b5yhrwnsu8v4rkna08yoa3wre.png)](https://asciinema.org/a/b5yhrwnsu8v4rkna08yoa3wre) 37 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @if "%DEBUG%" == "" @echo off 2 | @rem ########################################################################## 3 | @rem 4 | @rem Gradle startup script for Windows 5 | @rem 6 | @rem ########################################################################## 7 | 8 | @rem Set local scope for the variables with windows NT shell 9 | if "%OS%"=="Windows_NT" setlocal 10 | 11 | set DIRNAME=%~dp0 12 | if "%DIRNAME%" == "" set DIRNAME=. 13 | set APP_BASE_NAME=%~n0 14 | set APP_HOME=%DIRNAME% 15 | 16 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 17 | set DEFAULT_JVM_OPTS= 18 | 19 | @rem Find java.exe 20 | if defined JAVA_HOME goto findJavaFromJavaHome 21 | 22 | set JAVA_EXE=java.exe 23 | %JAVA_EXE% -version >NUL 2>&1 24 | if "%ERRORLEVEL%" == "0" goto init 25 | 26 | echo. 27 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 28 | echo. 29 | echo Please set the JAVA_HOME variable in your environment to match the 30 | echo location of your Java installation. 31 | 32 | goto fail 33 | 34 | :findJavaFromJavaHome 35 | set JAVA_HOME=%JAVA_HOME:"=% 36 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 37 | 38 | if exist "%JAVA_EXE%" goto init 39 | 40 | echo. 41 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 42 | echo. 43 | echo Please set the JAVA_HOME variable in your environment to match the 44 | echo location of your Java installation. 45 | 46 | goto fail 47 | 48 | :init 49 | @rem Get command-line arguments, handling Windows variants 50 | 51 | if not "%OS%" == "Windows_NT" goto win9xME_args 52 | 53 | :win9xME_args 54 | @rem Slurp the command line arguments. 55 | set CMD_LINE_ARGS= 56 | set _SKIP=2 57 | 58 | :win9xME_args_slurp 59 | if "x%~1" == "x" goto execute 60 | 61 | set CMD_LINE_ARGS=%* 62 | 63 | :execute 64 | @rem Setup the command line 65 | 66 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 67 | 68 | @rem Execute Gradle 69 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% 70 | 71 | :end 72 | @rem End local scope for the variables with windows NT shell 73 | if "%ERRORLEVEL%"=="0" goto mainEnd 74 | 75 | :fail 76 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 77 | rem the _cmd.exe /c_ return code! 78 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 79 | exit /b 1 80 | 81 | :mainEnd 82 | if "%OS%"=="Windows_NT" endlocal 83 | 84 | :omega 85 | -------------------------------------------------------------------------------- /src/main/kotlin/org/botellier/server/Request.kt: -------------------------------------------------------------------------------- 1 | package org.botellier.server 2 | 3 | import org.botellier.command.* 4 | import org.botellier.serializer.toByteArray 5 | import org.botellier.store.StoreException 6 | 7 | data class Request(val client: Client, val command: Command) 8 | 9 | class RequestDispatcher(val server: Server) { 10 | fun dispatch(request: Request) { 11 | val writer = request.client.socket.getOutputStream() 12 | val isAuthenticated = !server.requiresPassword() || request.client.isAuthenticated 13 | try { 14 | when { 15 | request.command is AuthCommand -> { 16 | val result = request.command.execute(server, request.client) 17 | writer.write(result.toByteArray()) 18 | } 19 | request.command is QuitCommand -> { 20 | val result = request.command.execute(server, request.client) 21 | writer.write(result.toByteArray()) 22 | request.client.socket.close() 23 | } 24 | isAuthenticated -> { 25 | when (request.command) { 26 | is ConnCommand -> { 27 | val result = request.command.execute(server, request.client) 28 | writer.write(result.toByteArray()) 29 | } 30 | is ReadStoreCommand -> { 31 | val store = server.dbs[request.client.dbIndex] 32 | val result = request.command.execute(store) 33 | writer.write(result.toByteArray()) 34 | } 35 | is StoreCommand -> { 36 | val store = server.dbs[request.client.dbIndex] 37 | val result = request.command.execute(store) 38 | writer.write(result.toByteArray()) 39 | } 40 | else -> { 41 | throw CommandException.RuntimeException("Invalid commands.") 42 | } 43 | } 44 | } 45 | else -> 46 | throw CommandException.RuntimeException("Not authenticated. Use AUTH command.") 47 | } 48 | } 49 | catch (e: StoreException.InvalidTypeException) { 50 | writer.write("-WRONGTYPE ${e.message}\r\n".toByteArray()) 51 | } 52 | catch (e: Throwable) { 53 | writer.write("-ERROR ${e.message}\r\n".toByteArray()) 54 | } 55 | } 56 | } -------------------------------------------------------------------------------- /src/main/kotlin/org/botellier/serializer/JsonSerializer.kt: -------------------------------------------------------------------------------- 1 | package org.botellier.serializer 2 | 3 | import org.botellier.value.* 4 | 5 | class JsonSerializer(override val value: StoreType, 6 | var pretty: Boolean = true, 7 | var indent: String = " ") : Serializer { 8 | 9 | override fun serialize(): ByteArray = print().toByteArray() 10 | 11 | fun print(): String { 12 | val builder = StringBuilder() 13 | render(builder, "", value) 14 | return builder.toString() 15 | } 16 | 17 | private fun render(builder: StringBuilder, indent: String, value: StoreType) { 18 | when (value) { 19 | is IntValue -> builder.append(value.unwrap()) 20 | is FloatValue -> builder.append(value.unwrap()) 21 | is StringValue -> builder.append("\"$value\"") 22 | is NilValue -> {} // skips nil 23 | is ListValue -> renderList(builder, indent, value) 24 | is SetValue -> renderList(builder, indent, value.map(String::toValue)) 25 | is MapValue -> renderMap(builder, indent, value) 26 | } 27 | } 28 | 29 | private fun renderList(builder: StringBuilder, indent: String, list: Iterable) { 30 | builder.append('[') 31 | val it = list.iterator() 32 | while (it.hasNext()) { 33 | val v = it.next() 34 | render(builder, indent, v) 35 | if (it.hasNext()) { 36 | builder.append(",") 37 | } 38 | } 39 | builder.append(']') 40 | } 41 | 42 | private fun renderMap(builder: StringBuilder, indent: String, map: Iterable>) { 43 | val newline = if (pretty) "\n" else "" 44 | val braceOpen = "{$newline" 45 | val braceClose = if (pretty) "$newline$indent}" else "}" 46 | 47 | builder.append(braceOpen) 48 | val it = map.iterator() 49 | while (it.hasNext()) { 50 | val v = it.next() 51 | renderKeyValue(builder, indent + this.indent, v) 52 | if (it.hasNext()) { 53 | builder.append(if (pretty) ",\n" else ",") 54 | } 55 | } 56 | builder.append(braceClose) 57 | } 58 | 59 | private fun renderKeyValue(builder: StringBuilder, indent: String, keyValue: Map.Entry) { 60 | builder.append("${if (pretty) indent else ""}\"${keyValue.key}\": ") 61 | render(builder, indent, keyValue.value) 62 | } 63 | } 64 | 65 | // Extensions. 66 | fun StoreType.toJson(pretty: Boolean = true, indent: String = " "): String = 67 | JsonSerializer(this, pretty, indent).print() 68 | -------------------------------------------------------------------------------- /src/test/kotlin/org/botellier/serializer/ByteSerializer.kt: -------------------------------------------------------------------------------- 1 | package org.botellier.serializer 2 | 3 | import org.botellier.value.* 4 | import org.junit.Assert 5 | import org.junit.Test 6 | 7 | class ByteSerializerTest { 8 | @Test 9 | fun renderInt() { 10 | val value = IntValue(1) 11 | Assert.assertArrayEquals(value.toByteArray(), ":1\r\n".toByteArray()) 12 | } 13 | 14 | @Test 15 | fun renderFloat() { 16 | val value = FloatValue(2.0) 17 | Assert.assertArrayEquals(value.toByteArray(), ";2.0\r\n".toByteArray()) 18 | } 19 | 20 | @Test 21 | fun renderString() { 22 | val value = StringValue("Hello, World!") 23 | Assert.assertArrayEquals(value.toByteArray(), "$13\r\nHello, World!\r\n".toByteArray()) 24 | } 25 | 26 | @Test 27 | fun renderRaw() { 28 | val bytes = "Hello, World!".toByteArray() 29 | val value = RawValue(bytes) 30 | Assert.assertArrayEquals(value.toByteArray(), "$13\r\nHello, World!\r\n".toByteArray()) 31 | } 32 | 33 | @Test 34 | fun renderEmptyString() { 35 | val value = StringValue("") 36 | Assert.assertArrayEquals(value.toByteArray(), "$0\r\n\r\n".toByteArray()) 37 | } 38 | 39 | @Test 40 | fun renderList() { 41 | val value = ListValue(listOf(IntValue(1), FloatValue(2.0), StringValue("three"))) 42 | Assert.assertArrayEquals( 43 | value.toByteArray(), 44 | "*3\r\n:1\r\n;2.0\r\n$5\r\nthree\r\n".toByteArray() 45 | ) 46 | } 47 | 48 | @Test 49 | fun renderEmptyList() { 50 | val value = ListValue() 51 | Assert.assertArrayEquals(value.toByteArray(), "*0\r\n".toByteArray()) 52 | } 53 | 54 | @Test 55 | fun renderSet() { 56 | val value = SetValue(listOf("one", "two", "three").toSet()) 57 | Assert.assertArrayEquals( 58 | value.toByteArray(), 59 | "&3\r\n$3\r\none\r\n$3\r\ntwo\r\n$5\r\nthree\r\n".toByteArray() 60 | ) 61 | } 62 | 63 | @Test 64 | fun renderEmptySet() { 65 | val value = SetValue() 66 | Assert.assertArrayEquals(value.toByteArray(), "&0\r\n".toByteArray()) 67 | } 68 | 69 | @Test 70 | fun renderMap() { 71 | val value = MapValue(mapOf("one" to IntValue(1), "two" to FloatValue(2.0), "three" to StringValue("three"))) 72 | Assert.assertArrayEquals( 73 | value.toByteArray(), 74 | "#3\r\n$3\r\none\r\n:1\r\n$3\r\ntwo\r\n;2.0\r\n$5\r\nthree\r\n$5\r\nthree\r\n".toByteArray() 75 | ) 76 | } 77 | 78 | @Test 79 | fun renderEmptyMap() { 80 | val value = MapValue() 81 | Assert.assertArrayEquals(value.toByteArray(), "#0\r\n".toByteArray()) 82 | } 83 | } -------------------------------------------------------------------------------- /src/main/kotlin/org/botellier/server/Client.kt: -------------------------------------------------------------------------------- 1 | package org.botellier.server 2 | 3 | import org.botellier.command.CommandParser 4 | import org.botellier.value.Lexer 5 | import org.botellier.value.LexerException 6 | import org.botellier.value.ParserException 7 | import org.botellier.value.toList 8 | import java.net.Socket 9 | import java.net.SocketException 10 | import java.net.SocketTimeoutException 11 | 12 | /** 13 | * Container class for client information. 14 | */ 15 | data class Client(val socket: Socket, var dbIndex: Int = 0, var isAuthenticated: Boolean = false) 16 | 17 | /** 18 | * Class for handling a new client connection. It reads the input, 19 | * tries to parse a command, and then sends back the constructed 20 | * request using the provided callback. 21 | * @property db the current db the client is connected to. 22 | */ 23 | class ClientHandler(val client: Client, val dispatcher: RequestDispatcher) : Runnable { 24 | var readTimeout: Int = 1000 25 | 26 | override fun run() { 27 | println("Handling client ${client.socket.inetAddress.hostAddress}") 28 | loop@while (true) { 29 | try { 30 | val stream = client.socket.waitInput() 31 | 32 | client.socket.soTimeout = readTimeout 33 | val tokens = Lexer(stream).lex().toList() 34 | client.socket.soTimeout = 0 35 | 36 | val command = CommandParser.parse(tokens) 37 | dispatcher.dispatch(Request(client, command)) 38 | } 39 | catch (e: SocketException) { 40 | break@loop 41 | } 42 | catch (e: Throwable) { 43 | println(e.message) 44 | val writer = client.socket.getOutputStream().bufferedWriter() 45 | when (e) { 46 | // Exception for Lexer waiting too much. 47 | is SocketTimeoutException -> 48 | writer.write("-ERR Command read timeout\r\n") 49 | 50 | // Exception regarding the serialized data. 51 | is LexerException -> 52 | writer.write("-COMMANDERR Unable to read command\r\n") 53 | 54 | // Exception regarding the structure of the data. 55 | is ParserException -> 56 | writer.write("-COMMANDERR Unable to parse command\r\n") 57 | 58 | // Exception regarding unknown command. 59 | is CommandParser.UnknownCommandException -> 60 | writer.write("-COMMANDERR ${e.message}\r\n") 61 | 62 | // Exception that we don't know how to handle. 63 | else -> { 64 | client.socket.close() 65 | break@loop 66 | } 67 | } 68 | writer.flush() 69 | } 70 | } 71 | println("Dropping client ${client.socket.inetAddress.hostAddress}") 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /src/main/kotlin/org/botellier/command/CommandParser.kt: -------------------------------------------------------------------------------- 1 | package org.botellier.command 2 | 3 | import org.botellier.value.Lexer 4 | import kotlin.reflect.full.createInstance 5 | 6 | class CommandParser { 7 | companion object Factory { 8 | fun parse(tokens: List): Command { 9 | val firstToken = tokens.first() 10 | 11 | if (firstToken is Lexer.StringToken) { 12 | val commandClass = COMMANDS[firstToken.value.toUpperCase()] 13 | if (commandClass != null) { 14 | val command = commandClass.createInstance() 15 | 16 | if (command.parameters.isNotEmpty()) { 17 | org.botellier.value.parse(tokens.drop(1)) { 18 | for (p in command.parameters) { 19 | when { 20 | p.isInt -> p.set(CValue.Primitive.Int(int())) 21 | p.isFloat -> p.set(CValue.Primitive.Float(float())) 22 | p.isString -> p.set(CValue.Primitive.String(string())) 23 | p.isAny -> p.set(CValue.primitive(any())) 24 | p.isIntArray -> { 25 | p.set(CValue.Array.Int( 26 | many { CValue.Primitive.Int(int()) } 27 | )) 28 | } 29 | p.isFloatArray -> { 30 | p.set(CValue.Array.Float( 31 | many { CValue.Primitive.Float(float()) } 32 | )) 33 | } 34 | p.isStringArray -> { 35 | p.set(CValue.Array.String( 36 | many { CValue.Primitive.String(string()) } 37 | )) 38 | } 39 | p.isPairArray -> { 40 | p.set(CValue.Array.Pair( 41 | many { CValue.Pair(string(), CValue.primitive(any())) } 42 | )) 43 | } 44 | p.isAnyArray -> { 45 | p.set(CValue.Array.Any( 46 | many(this::any).map { CValue.primitive(it) } 47 | )) 48 | } 49 | } 50 | } 51 | } 52 | } 53 | 54 | return command 55 | } 56 | } 57 | 58 | throw UnknownCommandException(firstToken.toString()) 59 | } 60 | } 61 | 62 | // Exceptions. 63 | class UnknownCommandException(command: String) 64 | : Throwable("Command not recognized: $command") 65 | } 66 | -------------------------------------------------------------------------------- /src/main/kotlin/org/botellier/serializer/ByteSerializer.kt: -------------------------------------------------------------------------------- 1 | package org.botellier.serializer 2 | 3 | import org.botellier.value.* 4 | import java.io.ByteArrayOutputStream 5 | 6 | private val CR: Byte = '\r'.toByte() 7 | private val LF: Byte = '\n'.toByte() 8 | private val NEWLINE: ByteArray = byteArrayOf(CR, LF) 9 | 10 | class ByteSerializer(override val value: StoreType?) : Serializer { 11 | override fun serialize(): ByteArray { 12 | val bos = ByteArrayOutputStream() 13 | render(bos, value) 14 | return bos.toByteArray() 15 | } 16 | 17 | private fun render(bos: ByteArrayOutputStream, value: StoreType?) { 18 | when (value) { 19 | is IntValue -> renderInt(bos, value) 20 | is FloatValue -> renderFloat(bos, value) 21 | is StringValue -> renderString(bos, value) 22 | is RawValue -> renderRaw(bos, value) 23 | is NilValue -> renderNil(bos) 24 | is ListValue -> renderList(bos, value) 25 | is SetValue -> renderSet(bos, value) 26 | is MapValue -> renderMap(bos, value) 27 | else -> renderNil(bos) 28 | } 29 | } 30 | 31 | private fun renderInt(bos: ByteArrayOutputStream, value: IntValue) { 32 | val bytes = value.toString().toByteArray() 33 | bos.write(':'.toInt()) 34 | bos.write(bytes) 35 | bos.write(NEWLINE) 36 | } 37 | 38 | private fun renderFloat(bos: ByteArrayOutputStream, value: FloatValue) { 39 | val bytes = value.toString().toByteArray() 40 | bos.write(';'.toInt()) 41 | bos.write(bytes) 42 | bos.write(NEWLINE) 43 | } 44 | 45 | private fun renderString(bos: ByteArrayOutputStream, value: StringValue) { 46 | val bytes = value.unwrap().toByteArray() 47 | bos.write('$'.toInt()) 48 | bos.write(bytes.size.toString().toByteArray()) 49 | bos.write(NEWLINE) 50 | bos.write(bytes) 51 | bos.write(NEWLINE) 52 | } 53 | 54 | private fun renderRaw(bos: ByteArrayOutputStream, value: RawValue) { 55 | bos.write('$'.toInt()) 56 | bos.write(value.value.size.toString().toByteArray()) 57 | bos.write(NEWLINE) 58 | bos.write(value.value) 59 | bos.write(NEWLINE) 60 | } 61 | 62 | private fun renderNil(bos: ByteArrayOutputStream) { 63 | bos.write("$-1".toByteArray()) 64 | bos.write(NEWLINE) 65 | } 66 | 67 | private fun renderList(bos: ByteArrayOutputStream, list: ListValue) { 68 | bos.write('*'.toInt()) 69 | bos.write(list.size.toString().toByteArray()) 70 | bos.write(NEWLINE) 71 | for (value in list) { 72 | render(bos, value) 73 | } 74 | } 75 | 76 | private fun renderSet(bos: ByteArrayOutputStream, set: SetValue) { 77 | bos.write('&'.toInt()) 78 | bos.write(set.size.toString().toByteArray()) 79 | bos.write(NEWLINE) 80 | for (value in set) { 81 | render(bos, StringValue(value)) 82 | } 83 | } 84 | 85 | private fun renderMap(bos: ByteArrayOutputStream, map: MapValue) { 86 | bos.write('#'.toInt()) 87 | bos.write(map.size.toString().toByteArray()) 88 | bos.write(NEWLINE) 89 | for ((key, value) in map) { 90 | render(bos, StringValue(key)) 91 | render(bos, value) 92 | } 93 | } 94 | } 95 | 96 | // Extensions. 97 | fun StoreType?.toByteArray(): ByteArray = ByteSerializer(this).serialize() 98 | -------------------------------------------------------------------------------- /src/main/kotlin/org/botellier/command/CValue.kt: -------------------------------------------------------------------------------- 1 | package org.botellier.command 2 | 3 | import org.botellier.value.* 4 | 5 | fun CValue.Primitive.toValue(): StorePrimitive { 6 | return when (this) { 7 | is CValue.Primitive.Int -> this.value.toValue() 8 | is CValue.Primitive.Float -> this.value.toValue() 9 | is CValue.Primitive.String -> this.value.toValue() 10 | is CValue.Primitive.Any -> throw CValue.InvalidPrimitiveException(this) 11 | } 12 | } 13 | 14 | // Types allowed in command parameters. 15 | sealed class CValue { 16 | companion object { 17 | fun primitive(value: Any): CValue.Primitive { 18 | return when(value) { 19 | is Int -> CValue.Primitive.Int(value) 20 | is Float -> CValue.Primitive.Float(value.toDouble()) 21 | is Double -> CValue.Primitive.Float(value) 22 | is String -> CValue.Primitive.String(value) 23 | else -> throw InvalidPrimitiveException(value) 24 | } 25 | } 26 | } 27 | 28 | sealed class Primitive : CValue() { 29 | data class Int(val value: kotlin.Int) : Primitive() 30 | data class Float(val value: kotlin.Double) : Primitive() 31 | data class String(val value: kotlin.String) : Primitive() 32 | class Any : Primitive() 33 | 34 | override final fun equals(other: kotlin.Any?): Boolean { 35 | return when { 36 | this is Int && other is Int -> 37 | this.value == other.value 38 | this is Int && other is IntValue -> 39 | this.value == other.unwrap() 40 | this is Float && other is Float -> 41 | this.value == other.value 42 | this is Float && other is FloatValue -> 43 | this.value == other.unwrap() 44 | this is String && other is String -> 45 | this.value == other.value 46 | this is String && other is StringValue -> 47 | this.value == other.unwrap() 48 | else -> false 49 | } 50 | } 51 | } 52 | 53 | data class Pair(val first: String, val second: Primitive) : CValue() 54 | 55 | sealed class Array : CValue() { 56 | data class Int(val value: List) : Array() 57 | data class Float(val value: List) : Array() 58 | data class String(val value: List) : Array() 59 | data class Pair(val value: List) : Array() 60 | data class Any(val value: List) : Array() 61 | } 62 | 63 | // TODO: Fix this values to print arrays of any(s) and pair(s). 64 | override final fun toString(): String { 65 | return when(this) { 66 | is Primitive.Int -> this.value.toString() 67 | is Primitive.Float -> this.value.toString() 68 | is Primitive.String -> this.value 69 | is Primitive.Any -> "nil" 70 | is Pair -> "(${this.first}, ${this.second})" 71 | is Array.Int -> this.value.toString() 72 | is Array.Float -> this.value.toString() 73 | is Array.String -> this.value.toString() 74 | is Array.Pair -> this.value.toString() 75 | is Array.Any -> this.value.toString() 76 | } 77 | } 78 | 79 | // Exceptions. 80 | class InvalidPrimitiveException(value: Any) 81 | : Throwable("Cannot construct a command primitive using $value.") 82 | } 83 | -------------------------------------------------------------------------------- /src/main/kotlin/org/botellier/log/Entry.kt: -------------------------------------------------------------------------------- 1 | package org.botellier.log 2 | 3 | import java.io.InputStream 4 | import java.io.OutputStream 5 | 6 | /** 7 | * Wrapper for EntryProtos. 8 | */ 9 | open class Entry(val protos: EntryProtos.Entry) { 10 | companion object { 11 | /** 12 | * Returns an Entry based on the type of EntryProtos.Entry. 13 | * @param protos the generated protobuf class EntryProtos.Entry. 14 | * @returns The wrapper for the protobuf class. 15 | */ 16 | fun fromProtos(protos: EntryProtos.Entry): Entry { 17 | when (protos.entryTypeCase) { 18 | EntryProtos.Entry.EntryTypeCase.DELETE_ENTRY -> { 19 | return DeleteEntry(protos) 20 | } 21 | EntryProtos.Entry.EntryTypeCase.SET_ENTRY -> { 22 | return SetEntry(protos) 23 | } 24 | EntryProtos.Entry.EntryTypeCase.BEGIN_TRASACTION_ENTRY -> { 25 | return BeginTransactionEntry(protos) 26 | } 27 | EntryProtos.Entry.EntryTypeCase.END_TRANSACTION_ENTRY -> { 28 | return EndTransactionEntry(protos) 29 | } 30 | else -> { 31 | return Entry(protos) 32 | } 33 | } 34 | } 35 | 36 | // Parsing. 37 | fun parseFrom(input: InputStream): Entry { 38 | return fromProtos(EntryProtos.Entry.parseFrom(input)) 39 | } 40 | 41 | fun parseFrom(input: ByteArray): Entry { 42 | return fromProtos(EntryProtos.Entry.parseFrom(input)) 43 | } 44 | } 45 | 46 | val id: Int get() = protos.id 47 | 48 | fun writeTo(output: OutputStream) { 49 | protos.writeTo(output) 50 | } 51 | 52 | // Overloads. 53 | override fun toString(): String = protos.toString() 54 | } 55 | 56 | // ---------------- 57 | // Entry types. 58 | // ---------------- 59 | 60 | class DeleteEntry(protos: EntryProtos.Entry) : Entry(protos) { 61 | val key: String get() = protos.deleteEntry.key 62 | } 63 | 64 | class SetEntry(protos: EntryProtos.Entry) : Entry(protos) { 65 | val key: String get() = protos.setEntry.key 66 | val before: ByteArray by lazy { protos.setEntry.before.toByteArray() } 67 | val after: ByteArray by lazy { protos.setEntry.after.toByteArray() } 68 | } 69 | 70 | class BeginTransactionEntry(protos: EntryProtos.Entry) : Entry(protos) 71 | 72 | class EndTransactionEntry(protos: EntryProtos.Entry) : Entry(protos) 73 | 74 | // ---------------- 75 | // Builders. 76 | // ---------------- 77 | 78 | fun buildDeleteEntry(id: Int, init: EntryProtos.DeleteEntry.Builder.() -> Unit): Entry { 79 | val entry = EntryProtos.Entry.newBuilder() 80 | val deleteEntry = EntryProtos.DeleteEntry.newBuilder() 81 | 82 | deleteEntry.init() 83 | entry.id = id 84 | entry.deleteEntry = deleteEntry.build() 85 | 86 | return Entry.fromProtos(entry.build()) 87 | } 88 | 89 | fun buildSetEntry(id: Int, init: EntryProtos.SetEntry.Builder.() -> Unit): Entry { 90 | val entry = EntryProtos.Entry.newBuilder() 91 | val setEntry = EntryProtos.SetEntry.newBuilder() 92 | 93 | setEntry.init() 94 | entry.id = id 95 | entry.setEntry = setEntry.build() 96 | 97 | return Entry.fromProtos(entry.build()) 98 | } 99 | 100 | fun buildBeginTransactionEntry(id: Int, init: EntryProtos.BeginTransactionEntry.Builder.() -> Unit = {}): Entry { 101 | val entry = EntryProtos.Entry.newBuilder() 102 | val transactionEntry = EntryProtos.BeginTransactionEntry.newBuilder() 103 | 104 | transactionEntry.init() 105 | entry.id = id 106 | entry.beginTrasactionEntry = transactionEntry.build() 107 | 108 | return Entry.fromProtos(entry.build()) 109 | } 110 | 111 | fun buildEndTransactionEntry(id: Int, init: EntryProtos.EndTransactionEntry.Builder.() -> Unit = {}): Entry { 112 | val entry = EntryProtos.Entry.newBuilder() 113 | val transactionEntry = EntryProtos.EndTransactionEntry.newBuilder() 114 | 115 | transactionEntry.init() 116 | entry.id = id 117 | entry.endTransactionEntry = transactionEntry.build() 118 | 119 | return Entry.fromProtos(entry.build()) 120 | } 121 | -------------------------------------------------------------------------------- /src/main/kotlin/org/botellier/log/SegmentHeader.kt: -------------------------------------------------------------------------------- 1 | package org.botellier.log 2 | 3 | import java.io.ByteArrayOutputStream 4 | import java.io.EOFException 5 | import java.io.InputStream 6 | import java.io.OutputStream 7 | import java.security.MessageDigest 8 | 9 | 10 | const val HEADER_SIZE = 44 11 | 12 | /** 13 | * Class with data that should be present at the beginning of each segment file. It contains 14 | * important information such as the id of the first entry (useful for lookups) and the 15 | * checksum of the segment (useful for integrity validation). 16 | * 17 | * The size of the serialized header is always 28 bytes. As the first 4 bytes are used 18 | * for specifying the size of the underlying protocol buffer (which can be variable size); 19 | * the rest of bytes are padded to meet the 28 bytes. 20 | * 21 | * Note that this class supposed to be directly manipulated by the Segment class. No other purpose 22 | * is intended. 23 | */ 24 | class SegmentHeader private constructor(initialId: Int, initialChecksum: String, initialEntries: Int) { 25 | companion object { 26 | fun parseFrom(input: InputStream): SegmentHeader { 27 | try { 28 | val protos = SegmentHeaderProtos.SegmentHeader.parseDelimitedFrom(input) 29 | protos ?: throw EOFException() 30 | return SegmentHeader(protos.id, protos.checksum, protos.totalEntries) 31 | } catch (e: Throwable) { 32 | return SegmentHeader(MessageDigest.getInstance("MD5")) 33 | } 34 | } 35 | 36 | fun parseFrom(data: ByteArray): SegmentHeader { 37 | return parseFrom(data.inputStream()) 38 | } 39 | } 40 | 41 | var id = initialId 42 | var checksum = initialChecksum; private set 43 | var totalEntries = initialEntries; private set 44 | 45 | /** 46 | * Creates a new SegmentHeader. 47 | * @param md the MessageDigest to use for initial checksum. 48 | */ 49 | constructor(md: MessageDigest) : this(-1, "", 0) { 50 | checksum = md.tryDigest().toHexString() 51 | } 52 | 53 | /** 54 | * Updates the checksum and number of entries indicated in 55 | * the header. 56 | * @param md the MessageDigest instance to use for obtaining the checksum. 57 | */ 58 | fun update(md: MessageDigest) { 59 | checksum = md.tryDigest().toHexString() 60 | totalEntries++ 61 | } 62 | 63 | /** 64 | * Converts this instance to a SegmentHeaderProtos.SegmentHeader, 65 | * which can be easily serialized. 66 | */ 67 | fun toProtos(): SegmentHeaderProtos.SegmentHeader { 68 | val builder = SegmentHeaderProtos.SegmentHeader.newBuilder() 69 | builder.id = id 70 | builder.checksum = checksum 71 | builder.totalEntries = totalEntries 72 | return builder.build() 73 | } 74 | 75 | /** 76 | * Writes the padded protocol buffer to the beginning of the 77 | * given output stream. 78 | */ 79 | fun writeTo(output: OutputStream) { 80 | output.write(toByteArray()) 81 | } 82 | 83 | /** 84 | * Serializes the SegmentHeader adding the required padding. 85 | * @returns the padded bytes of the segment header data. 86 | */ 87 | fun toByteArray(): ByteArray { 88 | val protos = toProtos() 89 | val buffer = ByteArrayOutputStream(HEADER_SIZE) 90 | protos.writeDelimitedTo(buffer) 91 | val paddingSize = HEADER_SIZE - buffer.size() 92 | buffer.write(ByteArray(paddingSize, { 0 })) 93 | return buffer.toByteArray() 94 | } 95 | } 96 | 97 | // ---------------- 98 | // Useful extensions. 99 | // ---------------- 100 | 101 | private val hexDigits = setOf( 102 | '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 103 | 'a', 'b' ,'c' ,'d', 'e', 'f', 104 | 'A', 'B' ,'C' ,'D', 'E', 'F' 105 | ) 106 | 107 | /** 108 | * Converts the contained bytes to 109 | * an hexadecimal string. 110 | */ 111 | fun ByteArray.toHexString(): String { 112 | val buffer = StringBuffer() 113 | for (b in this) { 114 | val hex = Integer.toHexString(b.toInt() and 0xff) 115 | if (hex.length < 2) { buffer.append('0') } 116 | buffer.append(hex) 117 | } 118 | return buffer.toString() 119 | } -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by https://www.gitignore.io/api/vim,java,gradle,eclipse,netbeans,intellij,intellij+iml 2 | 3 | ### Eclipse ### 4 | 5 | .metadata 6 | bin/ 7 | tmp/ 8 | *.tmp 9 | *.bak 10 | *.swp 11 | *~.nib 12 | local.properties 13 | .settings/ 14 | .loadpath 15 | .recommenders 16 | 17 | # Eclipse Core 18 | .project 19 | 20 | # External tool builders 21 | .externalToolBuilders/ 22 | 23 | # Locally stored "Eclipse launch configurations" 24 | *.launch 25 | 26 | # PyDev specific (Python IDE for Eclipse) 27 | *.pydevproject 28 | 29 | # CDT-specific (C/C++ Development Tooling) 30 | .cproject 31 | 32 | # JDT-specific (Eclipse Java Development Tools) 33 | .classpath 34 | 35 | # Java annotation processor (APT) 36 | .factorypath 37 | 38 | # PDT-specific (PHP Development Tools) 39 | .buildpath 40 | 41 | # sbteclipse plugin 42 | .target 43 | 44 | # Tern plugin 45 | .tern-project 46 | 47 | # TeXlipse plugin 48 | .texlipse 49 | 50 | # STS (Spring Tool Suite) 51 | .springBeans 52 | 53 | # Code Recommenders 54 | .recommenders/ 55 | 56 | # Scala IDE specific (Scala & Java development for Eclipse) 57 | .cache-main 58 | .scala_dependencies 59 | .worksheet 60 | 61 | ### Intellij ### 62 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm 63 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 64 | 65 | # User-specific stuff: 66 | .idea/**/workspace.xml 67 | .idea/**/tasks.xml 68 | .idea/dictionaries 69 | 70 | # Sensitive or high-churn files: 71 | .idea/**/dataSources/ 72 | .idea/**/dataSources.ids 73 | .idea/**/dataSources.xml 74 | .idea/**/dataSources.local.xml 75 | .idea/**/sqlDataSources.xml 76 | .idea/**/dynamic.xml 77 | .idea/**/uiDesigner.xml 78 | 79 | # Gradle: 80 | .idea/**/gradle.xml 81 | .idea/**/libraries 82 | 83 | # Mongo Explorer plugin: 84 | .idea/**/mongoSettings.xml 85 | 86 | ## File-based project format: 87 | *.iws 88 | 89 | ## Plugin-specific files: 90 | 91 | # IntelliJ 92 | /out/ 93 | 94 | # mpeltonen/sbt-idea plugin 95 | .idea_modules/ 96 | 97 | # JIRA plugin 98 | atlassian-ide-plugin.xml 99 | 100 | # Cursive Clojure plugin 101 | .idea/replstate.xml 102 | 103 | # Crashlytics plugin (for Android Studio and IntelliJ) 104 | com_crashlytics_export_strings.xml 105 | crashlytics.properties 106 | crashlytics-build.properties 107 | fabric.properties 108 | 109 | ### Intellij Patch ### 110 | # Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 111 | 112 | # *.iml 113 | # modules.xml 114 | # .idea/misc.xml 115 | # *.ipr 116 | 117 | # Sonarlint plugin 118 | .idea/sonarlint 119 | 120 | ### Intellij+iml ### 121 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm 122 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 123 | 124 | # User-specific stuff: 125 | 126 | # Sensitive or high-churn files: 127 | 128 | # Gradle: 129 | 130 | # Mongo Explorer plugin: 131 | 132 | ## File-based project format: 133 | 134 | ## Plugin-specific files: 135 | 136 | # IntelliJ 137 | 138 | # mpeltonen/sbt-idea plugin 139 | 140 | # JIRA plugin 141 | 142 | # Cursive Clojure plugin 143 | 144 | # Crashlytics plugin (for Android Studio and IntelliJ) 145 | 146 | ### Intellij+iml Patch ### 147 | # Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-249601023 148 | 149 | *.iml 150 | modules.xml 151 | .idea/misc.xml 152 | *.ipr 153 | 154 | ### Java ### 155 | # Compiled class file 156 | *.class 157 | 158 | # Log file 159 | *.log 160 | 161 | # BlueJ files 162 | *.ctxt 163 | 164 | # Mobile Tools for Java (J2ME) 165 | .mtj.tmp/ 166 | 167 | # Package Files # 168 | *.jar 169 | *.war 170 | *.ear 171 | *.zip 172 | *.tar.gz 173 | *.rar 174 | 175 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 176 | hs_err_pid* 177 | 178 | ### NetBeans ### 179 | nbproject/private/ 180 | build/ 181 | nbbuild/ 182 | dist/ 183 | nbdist/ 184 | .nb-gradle/ 185 | 186 | ### Vim ### 187 | # swap 188 | [._]*.s[a-v][a-z] 189 | [._]*.sw[a-p] 190 | [._]s[a-v][a-z] 191 | [._]sw[a-p] 192 | # session 193 | Session.vim 194 | # temporary 195 | .netrwhist 196 | *~ 197 | # auto-generated tag files 198 | tags 199 | 200 | ### Gradle ### 201 | .gradle 202 | /build/ 203 | 204 | # Ignore Gradle GUI config 205 | gradle-app.setting 206 | 207 | # Avoid ignoring Gradle wrapper jar file (.jar files are usually ignored) 208 | !gradle-wrapper.jar 209 | 210 | # Cache of project 211 | .gradletasknamecache 212 | 213 | # # Work around https://youtrack.jetbrains.com/issue/IDEA-116898 214 | # gradle/wrapper/gradle-wrapper.properties 215 | 216 | # End of https://www.gitignore.io/api/vim,java,gradle,eclipse,netbeans,intellij,intellij+iml 217 | 218 | ### Custom ### 219 | /run/ 220 | -------------------------------------------------------------------------------- /src/test/kotlin/org/botellier/command/CommandParser.kt: -------------------------------------------------------------------------------- 1 | package org.botellier.command 2 | 3 | import org.botellier.serializer.toByteArray 4 | import org.botellier.value.Lexer 5 | import org.botellier.value.toList 6 | import org.botellier.value.toValue 7 | 8 | import org.junit.Assert 9 | import org.junit.Test 10 | 11 | class CommandParserTest { 12 | 13 | /** 14 | * Success cases 15 | */ 16 | 17 | @Test 18 | fun appendCommand() { 19 | val tokens = toTokens("APPEND", "key", "10") 20 | val command = CommandParser.parse(tokens) 21 | Assert.assertTrue(command is AppendCommand) 22 | } 23 | 24 | @Test 25 | fun decrCommand() { 26 | val tokens = toTokens("DECR", "key") 27 | val command = CommandParser.parse(tokens) 28 | Assert.assertTrue(command is DecrCommand) 29 | } 30 | 31 | @Test 32 | fun decrbyCommand() { 33 | val tokens = toTokens("DECRBY", "key", "10") 34 | val command = CommandParser.parse(tokens) 35 | Assert.assertTrue(command is DecrbyCommand) 36 | } 37 | 38 | @Test 39 | fun getCommand() { 40 | val tokens = toTokens("GET", "key") 41 | val command = CommandParser.parse(tokens) 42 | Assert.assertTrue(command is GetCommand) 43 | } 44 | 45 | @Test 46 | fun incrCommand() { 47 | val tokens = toTokens("INCR", "key") 48 | val command = CommandParser.parse(tokens) 49 | Assert.assertTrue(command is IncrCommand) 50 | } 51 | 52 | @Test 53 | fun incrbyCommand() { 54 | val tokens = toTokens("INCRBY", "key", "10") 55 | val command = CommandParser.parse(tokens) 56 | Assert.assertTrue(command is IncrbyCommand) 57 | } 58 | 59 | @Test 60 | fun incrbyfloatCommand() { 61 | val tokens = toTokens("INCRBYFLOAT", "key", "10.0") 62 | val command = CommandParser.parse(tokens) 63 | Assert.assertTrue(command is IncrbyfloatCommand) 64 | } 65 | 66 | @Test 67 | fun msetCommand() { 68 | val tokens = toTokens("MSET", "key", "10", "key1", "20", "key0", "20") 69 | val command = CommandParser.parse(tokens) 70 | Assert.assertTrue(command is MSetCommand) 71 | } 72 | 73 | @Test 74 | fun setCommand() { 75 | val tokens = toTokens("SET", "key", "10") 76 | val command = CommandParser.parse(tokens) 77 | Assert.assertTrue(command is SetCommand) 78 | } 79 | 80 | @Test 81 | fun strlenCommand() { 82 | val tokens = toTokens("STRLEN", "key") 83 | val command = CommandParser.parse(tokens) 84 | Assert.assertTrue(command is StrlenCommand) 85 | } 86 | 87 | /** 88 | * Fail cases. 89 | */ 90 | 91 | @Test 92 | fun appendCommandFail() { 93 | commandShouldFail("APPEND", "KEY") 94 | } 95 | 96 | @Test 97 | fun decrCommandFail() { 98 | commandShouldFail("DECR") 99 | } 100 | 101 | @Test 102 | fun decrbyCommandFail() { 103 | commandShouldFail("DECRBY", "key") 104 | } 105 | 106 | @Test 107 | fun getCommandFail() { 108 | commandShouldFail("GET") 109 | } 110 | 111 | @Test 112 | fun getCommandKeyFail() { 113 | commandShouldFail("GET", "1") 114 | } 115 | 116 | @Test 117 | fun incrCommandFail() { 118 | commandShouldFail("INCR") 119 | } 120 | 121 | @Test 122 | fun incrbyCommandFail() { 123 | commandShouldFail("INCRBY", "key") 124 | } 125 | 126 | @Test 127 | fun incrbyfloatCommandFail() { 128 | commandShouldFail("INCRBYFLOAT", "key") 129 | } 130 | 131 | @Test 132 | fun incrbyfloatCommandIntFail() { 133 | commandShouldFail("INCRBYFLOAT", "key", "10") 134 | } 135 | 136 | @Test 137 | fun setCommandFail() { 138 | commandShouldFail("SET") 139 | } 140 | 141 | @Test 142 | fun setCommandIntFail() { 143 | commandShouldFail("SET", "1") 144 | } 145 | 146 | @Test 147 | fun strlenCommandFail() { 148 | commandShouldFail("STRLEN") 149 | } 150 | 151 | /** 152 | * Utility functions. 153 | */ 154 | 155 | private fun toTokens(items: List): List { 156 | val listValue = items.map { it.toValue() }.toValue() 157 | return Lexer(String(listValue.toByteArray())).lex().toList() 158 | } 159 | 160 | private fun toTokens(vararg items: String): List = toTokens(items.toList()) 161 | 162 | private fun commandShouldFail(vararg items: String) { 163 | val tokens = toTokens(items.toList()) 164 | try { 165 | CommandParser.parse(tokens) 166 | Assert.fail("Invalid command $items parsed.") 167 | } 168 | catch (e: Throwable) {} 169 | } 170 | } 171 | -------------------------------------------------------------------------------- /src/test/kotlin/org/botellier/value/value.kt: -------------------------------------------------------------------------------- 1 | package org.botellier.value 2 | 3 | import org.junit.Assert 4 | import org.junit.Test 5 | 6 | class StoreValueTest { 7 | @Test 8 | fun builtinConversionsWork() { 9 | Assert.assertTrue(1.toValue() is IntValue) 10 | Assert.assertTrue(1.0f.toValue() is FloatValue) 11 | Assert.assertTrue(1.0.toValue() is FloatValue) 12 | Assert.assertTrue("".toValue() is StringValue) 13 | Assert.assertTrue(listOf(1, 2, 3).map(Int::toValue).toValue() is ListValue) 14 | Assert.assertTrue(setOf("one").toValue() is SetValue) 15 | Assert.assertTrue(mapOf("one" to 1.toValue()).toValue() is MapValue) 16 | } 17 | 18 | @Test 19 | fun primitivesComparisons() { 20 | val int = IntValue(1) 21 | val float = FloatValue(1.0) 22 | val string = StringValue("one") 23 | 24 | Assert.assertTrue(int < IntValue(2)) 25 | Assert.assertTrue(int > IntValue(0)) 26 | 27 | Assert.assertTrue(float < FloatValue(2.0)) 28 | Assert.assertTrue(float > FloatValue(0.0)) 29 | 30 | Assert.assertTrue(string < StringValue("two")) 31 | Assert.assertTrue(string > StringValue("abc")) 32 | } 33 | 34 | @Test 35 | fun rawValue() { 36 | val bytes = "ONE, TWO, THREE".toByteArray() 37 | val raw = RawValue(bytes) 38 | Assert.assertArrayEquals(bytes, raw.value) 39 | } 40 | 41 | @Test 42 | fun listAndSetCloneNotSameAsOriginal() { 43 | val list = ListValue(listOf(1, 2).map(Int::toValue)) 44 | val set = SetValue(setOf("one", "two")) 45 | 46 | val listClone = list.copy { it.rpush(3.toValue()) } 47 | val setClone = set.copy { it.add("three") } 48 | 49 | Assert.assertNotEquals(list.size, listClone.size) 50 | Assert.assertNotEquals(set.size, setClone.size) 51 | } 52 | 53 | @Test 54 | fun modifyingListClone() { 55 | val list = listOf(1, 2, 3).map(Int::toValue).toValue() 56 | val clone = list.copy { it.lpop() } 57 | 58 | Assert.assertEquals(list.size, 3) 59 | Assert.assertEquals(clone.size, 2) 60 | } 61 | 62 | @Test 63 | fun modifyingSetClone() { 64 | val set = setOf("one", "two", "three").toValue() 65 | val clone = set.copy { it.remove("one") } 66 | 67 | Assert.assertEquals(set.size, 3) 68 | Assert.assertEquals(clone.size, 2) 69 | } 70 | 71 | @Test 72 | fun iteratingList() { 73 | val list = listOf(1, 2, 3).map(Int::toValue).toValue() 74 | var res = 0 75 | for (value in list) { 76 | when (value) { 77 | is IntValue -> res += value.unwrap() 78 | } 79 | } 80 | Assert.assertEquals(res, 6) 81 | } 82 | 83 | @Test 84 | fun iteratingSet() { 85 | val set = setOf("one", "two").toValue() 86 | var res = "" 87 | for (value in set) { 88 | res += value 89 | } 90 | Assert.assertEquals(res, "onetwo") 91 | } 92 | 93 | @Test 94 | fun iteratingMap() { 95 | val map = mapOf("one" to 1, "two" to 2).mapValues { it.value.toValue() }.toValue() 96 | var res = 0 97 | for ((_, value) in map) { 98 | when (value) { 99 | is IntValue -> res += value.unwrap() 100 | } 101 | } 102 | Assert.assertEquals(res, 3) 103 | } 104 | 105 | @Test 106 | fun settingAndGettingList() { 107 | val list = listOf(1, 2, 3).map(Int::toValue).toValue().copy { 108 | it[1] = 2.0.toValue() 109 | it[2] = "3".toValue() 110 | } 111 | 112 | Assert.assertEquals((list.unwrap().get(0) as IntValue).unwrap(), 1) 113 | Assert.assertEquals((list.unwrap().get(1) as FloatValue).unwrap(), 2.0, 0.001) 114 | Assert.assertEquals((list.unwrap().get(2) as StringValue).unwrap(), "3") 115 | } 116 | 117 | @Test 118 | fun removingFromList() { 119 | val list = listOf(1, 2, 3, 4, 5, 6, 7, 8, 9).map(Int::toValue).toValue().copy { 120 | it.removeAt(0) 121 | it.remove(listOf(0, 0, 1, 1, 2, 3, 4, 5, 6)) 122 | } 123 | Assert.assertEquals(1, list.size) 124 | Assert.assertEquals(IntValue(9), list.unwrap().first()) 125 | } 126 | 127 | @Test 128 | fun slicingList() { 129 | val list = listOf(1, 2, 3).map(Int::toValue).toValue() 130 | Assert.assertEquals(list.unwrap().slice(0, 2).toList().map{ (it as IntValue).unwrap() }, listOf(1, 2, 3)) 131 | Assert.assertEquals(list.unwrap().slice(0, 1).toList().map{ (it as IntValue).unwrap() }, listOf(1, 2)) 132 | Assert.assertEquals(list.unwrap().slice(-3, 2).toList().map{ (it as IntValue).unwrap() }, listOf(1, 2, 3)) 133 | } 134 | } -------------------------------------------------------------------------------- /src/test/kotlin/org/botellier/value/Lexer.kt: -------------------------------------------------------------------------------- 1 | package org.botellier.value 2 | 3 | import org.junit.Assert 4 | import org.junit.Test 5 | 6 | class LexerTest { 7 | @Test 8 | fun singleString() { 9 | val expected = listOf<(Lexer.Token) -> Boolean>( 10 | { intToken(it, 1) } 11 | ) 12 | val result = Lexer("*1\r\n$1\r\n1\r\n").lex().toList() 13 | assertTokens(result, expected) 14 | } 15 | 16 | @Test 17 | fun specialString() { 18 | val string = "特è" 19 | val byteCount = string.toByteArray().size 20 | val expected = listOf<(Lexer.Token) -> Boolean>( 21 | { stringToken(it, string) } 22 | ) 23 | val result = Lexer("*1\r\n$$byteCount\r\n$string\r\n").lex().toList() 24 | assertTokens(result, expected) 25 | } 26 | 27 | @Test 28 | fun multipleStrings() { 29 | val expected = listOf<(Lexer.Token) -> Boolean>( 30 | { stringToken(it, "SET") }, 31 | { stringToken(it, "counter") }, 32 | { stringToken(it, "one") } 33 | ) 34 | val result = Lexer("*3\r\n$3\r\nSET\r\n$7\r\ncounter\r\n$3\r\none\r\n").lex().toList() 35 | assertTokens(result, expected) 36 | } 37 | 38 | @Test 39 | fun integer() { 40 | val expected = listOf<(Lexer.Token) -> Boolean>( 41 | { stringToken(it, "LPUSH") }, 42 | { stringToken(it, "list") }, 43 | { intToken(it, 1000) } 44 | ) 45 | val result = Lexer("*3\r\n$5\r\nLPUSH\r\n$4\r\nlist\r\n$4\r\n1000\r\n").lex().toList() 46 | assertTokens(result, expected) 47 | } 48 | 49 | @Test 50 | fun float() { 51 | val expected = listOf<(Lexer.Token) -> Boolean>( 52 | { stringToken(it, "LPUSH") }, 53 | { stringToken(it, "list") }, 54 | { floatToken(it, 104.1) } 55 | ) 56 | val result = Lexer("*3\r\n$5\r\nLPUSH\r\n$4\r\nlist\r\n$5\r\n104.1\r\n").lex().toList() 57 | assertTokens(result, expected) 58 | } 59 | 60 | @Test 61 | fun pointFloat() { 62 | val expected = listOf<(Lexer.Token) -> Boolean>( 63 | { stringToken(it, "LPUSH") }, 64 | { stringToken(it, "list") }, 65 | { floatToken(it, 0.1) } 66 | ) 67 | val result = Lexer("*3\r\n$5\r\nLPUSH\r\n$4\r\nlist\r\n$2\r\n.1\r\n").lex().toList() 68 | assertTokens(result, expected) 69 | } 70 | 71 | @Test 72 | fun integerAndFloat() { 73 | val expected = listOf<(Lexer.Token) -> Boolean>( 74 | { stringToken(it, "HSET") }, 75 | { stringToken(it, "a") }, 76 | { intToken(it, 10) }, 77 | { stringToken(it, "b") }, 78 | { floatToken(it, 10.1) } 79 | ) 80 | val result = Lexer("*5\r\n$4\r\nHSET\r\n$1\r\na\r\n$2\r\n10\r\n$1\r\nb\r\n$4\r\n10.1\r\n").lex().toList() 81 | assertTokens(result, expected) 82 | } 83 | 84 | @Test 85 | fun floatAndInteger() { 86 | val expected = listOf<(Lexer.Token) -> Boolean>( 87 | { stringToken(it, "HSET") }, 88 | { stringToken(it, "a") }, 89 | { floatToken(it, 10.1) }, 90 | { stringToken(it, "b") }, 91 | { intToken(it, 10) } 92 | ) 93 | val result = Lexer("*5\r\n$4\r\nHSET\r\n$1\r\na\r\n$4\r\n10.1\r\n$1\r\nb\r\n$2\r\n10\r\n").lex().toList() 94 | assertTokens(result, expected) 95 | } 96 | 97 | @Test 98 | fun emptyString() { 99 | try { 100 | Lexer("").lex() 101 | Assert.fail("Passed empty string.") 102 | } 103 | catch (_: Throwable) {} 104 | } 105 | 106 | @Test 107 | fun incompleteLength() { 108 | try { 109 | val tokens = Lexer("$\r\n").lex() 110 | Assert.fail("Invalid length passed") 111 | } 112 | catch (_: Throwable) {} 113 | } 114 | 115 | /** 116 | * Utilities. 117 | */ 118 | 119 | private fun intToken(token: Lexer.Token, value: Int): Boolean { 120 | return token is Lexer.IntToken && token.value == value 121 | } 122 | 123 | private fun floatToken(token: Lexer.Token, value: Double): Boolean { 124 | return token is Lexer.FloatToken && token.value == value 125 | } 126 | 127 | private fun stringToken(token: Lexer.Token, value: String): Boolean { 128 | return token is Lexer.StringToken && token.value == value 129 | } 130 | 131 | private fun assertTokens(tokens: List, expected: List<(Lexer.Token) -> Boolean>) { 132 | Assert.assertEquals(tokens.size, expected.size) 133 | for (i in tokens.indices) { 134 | Assert.assertTrue(expected[i](tokens[i])) 135 | } 136 | } 137 | } 138 | -------------------------------------------------------------------------------- /src/test/kotlin/org/botellier/log/Log.kt: -------------------------------------------------------------------------------- 1 | package org.botellier.log 2 | 3 | import org.junit.Test 4 | import org.junit.Assert 5 | import java.io.File 6 | import java.util.* 7 | 8 | class LogTest { 9 | fun log(root: String, segmentSize: Int, clear: Boolean = false, f: Log.() -> Unit) { 10 | val log = Log(root, "test-segment-", segmentSize, clear) 11 | log.f() 12 | log.clear() 13 | } 14 | 15 | @Test 16 | fun segmentSize() { 17 | val before = "before".toByteArray() 18 | val after = "after".toByteArray() 19 | withDummy { log(it.toString(), 10) { 20 | set("key", before, after) 21 | set("key", before, after) 22 | set("key", before, after) 23 | Assert.assertTrue(File(segments[1].path.toUri()).exists()) 24 | Assert.assertTrue(File(segments[2].path.toUri()).exists()) 25 | }} 26 | } 27 | 28 | @Test 29 | fun iteratingEntries() { 30 | val before = "before".toByteArray() 31 | val after = "after".toByteArray() 32 | 33 | withDummy { log(it.toString(),10) { 34 | set("key", before, after) 35 | set("key", before, after) 36 | set("key", before, after) 37 | 38 | val res = fold(Pair("", ""), { (keys, datas), entry -> 39 | if (entry is SetEntry) { 40 | Pair(keys + entry.key, datas + String(entry.before) + String(entry.after)) 41 | } else { 42 | Pair(keys, datas) 43 | } 44 | }) 45 | 46 | Assert.assertEquals(Pair("keykeykey", "beforeafterbeforeafterbeforeafter"), res) 47 | }} 48 | } 49 | 50 | @Test 51 | fun findSegments() { 52 | withDummy(3) { log(it.toString(), 10) { 53 | Assert.assertEquals(3, segments.size) 54 | Assert.assertTrue(toList().isEmpty()) 55 | }} 56 | } 57 | 58 | @Test 59 | fun clearSegments() { 60 | withDummy(3) { log(it.toString(), 10, true) { 61 | Assert.assertEquals(1, segments.size) 62 | Assert.assertTrue(toList().isEmpty()) 63 | }} 64 | } 65 | 66 | @Test 67 | fun skippingEntriesOnEmptyLog() { 68 | withDummy { log(it.toString(), 10, true) { 69 | val res = query(87) 70 | Assert.assertEquals(0, res.toList().size) 71 | }} 72 | } 73 | 74 | @Test 75 | fun skippingEntriesOnSingleSegmentLog() { 76 | withDummy { log(it.toString(), 1024*1024, true) { 77 | for (i in 0..100) { 78 | create("$i", "$i".toByteArray()) 79 | } 80 | 81 | val res = query(87).fold("", { acc, entry -> 82 | acc + entry.id 83 | }) 84 | 85 | Assert.assertEquals(1, segments.size) 86 | Assert.assertEquals("87888990919293949596979899100", res) 87 | }} 88 | } 89 | 90 | @Test 91 | fun skippingEntriesOnMultipleSegmentLog() { 92 | withDummy { log(it.toString(), 10, true) { 93 | for (i in 0..100) { 94 | create("$i", "$i".toByteArray()) 95 | } 96 | 97 | val res = query(95).fold("", { acc, entry -> 98 | acc + entry.id 99 | }) 100 | 101 | Assert.assertEquals(101, segments.size) 102 | Assert.assertEquals("9596979899100", res) 103 | }} 104 | } 105 | 106 | @Test 107 | fun extendingWithValidSequence() { 108 | withDummy { 109 | val baseLog = Log(it.toString(), "test-segment-base-") 110 | baseLog.create("zero", "0".toByteArray()) 111 | baseLog.create("one", "1".toByteArray()) 112 | baseLog.create("two", "2".toByteArray()) 113 | baseLog.create("three", "3".toByteArray()) 114 | baseLog.create("four", "4".toByteArray()) 115 | 116 | val extendedLog = Log(it.toString(), "test-segment-ext-") 117 | extendedLog.create("zero", "0".toByteArray()) 118 | extendedLog.extend(baseLog.query(1)) 119 | 120 | val baseRes = baseLog.fold("", { acc, entry -> 121 | acc + (entry as SetEntry).key 122 | }) 123 | 124 | val extendedRes = extendedLog.fold("", { acc, entry -> 125 | acc + (entry as SetEntry).key 126 | }) 127 | 128 | Assert.assertEquals(baseRes, extendedRes) 129 | } 130 | } 131 | 132 | @Test(expected = LogException.ExtendException::class) 133 | fun extendingWithInvalidSequence() { 134 | withDummy { 135 | val baseLog = Log(it.toString(), "test-segment-base-") 136 | baseLog.create("zero", "0".toByteArray()) 137 | baseLog.create("one", "1".toByteArray()) 138 | baseLog.create("two", "2".toByteArray()) 139 | baseLog.create("three", "3".toByteArray()) 140 | baseLog.create("four", "4".toByteArray()) 141 | 142 | val extendedLog = Log(it.toString(), "test-segment-ext-") 143 | extendedLog.create("zero", "0".toByteArray()) 144 | extendedLog.extend(baseLog.query(2)) 145 | } 146 | 147 | } 148 | } 149 | -------------------------------------------------------------------------------- /src/test/kotlin/org/botellier/log/Segment.kt: -------------------------------------------------------------------------------- 1 | package org.botellier.log 2 | 3 | import org.junit.Test 4 | import org.junit.Assert 5 | import java.io.File 6 | import java.io.RandomAccessFile 7 | import java.security.MessageDigest 8 | 9 | class SegmentTest { 10 | fun segment(maxSize: Int = 1*1024*1024, f: (Segment) -> Unit) { 11 | withDummy { 12 | val s = Segment(it.toString(), 0, "test-tmp-segment-", maxSize) 13 | try { 14 | f(s) 15 | } catch (e: Throwable) { 16 | throw e 17 | } finally { 18 | s.clear() 19 | } 20 | } 21 | } 22 | 23 | @Test(expected = SegmentException::class) 24 | fun maxSize() { 25 | val before = "before".toByteArray() 26 | val after = "after".toByteArray() 27 | segment(24) { 28 | it.set(0, "none", before, after) 29 | it.set(0, "none", before, after) 30 | } 31 | } 32 | 33 | @Test 34 | fun iterating() { 35 | segment(250) { 36 | val oneBytes = "one".toByteArray() 37 | val twoBytes = "two".toByteArray() 38 | val threeBytes = "three".toByteArray() 39 | 40 | it.set(0, "one", oneBytes, oneBytes) 41 | it.delete(1, "one") 42 | it.set(2, "two", twoBytes, twoBytes) 43 | it.delete(3, "two") 44 | it.set(4, "three", threeBytes, threeBytes) 45 | it.delete(5, "three") 46 | 47 | val res = it.fold(Pair(0, ""), { (sum, str), entry -> 48 | var key = "" 49 | 50 | if (entry is DeleteEntry) { 51 | key = entry.key 52 | } else if (entry is SetEntry) { 53 | key = entry.key 54 | } 55 | 56 | Pair(sum + entry.id, str + key) 57 | }) 58 | 59 | Assert.assertEquals(Pair(15, "oneonetwotwothreethree"), res) 60 | } 61 | } 62 | 63 | @Test 64 | fun iteratingEmpty() { 65 | segment { 66 | val res = it.fold(0, { sum, entry -> 67 | sum + entry.id 68 | }) 69 | Assert.assertEquals(0, res) 70 | } 71 | } 72 | 73 | @Test 74 | fun createAndSetEntries() { 75 | segment(200) { 76 | it.create(0, "zero", "0".toByteArray()) 77 | it.create(1, "one", "1".toByteArray()) 78 | it.set(2, "zero", "0".toByteArray(), "zero".toByteArray()) 79 | it.set(3, "one", "1".toByteArray(), "one".toByteArray()) 80 | 81 | val res = it.fold("", { str, entry -> 82 | if (entry is SetEntry) { 83 | str + String(entry.before) + String(entry.after) 84 | } else { 85 | str 86 | } 87 | }) 88 | 89 | Assert.assertEquals("010zero1one", res) 90 | } 91 | } 92 | 93 | @Test 94 | fun checksumValidation() { 95 | withDummy { 96 | val segment = Segment(it.toString(), 0, "test-segment-") 97 | segment.create(0, "zero", "0".toByteArray()) 98 | segment.create(1, "one", "1".toByteArray()) 99 | segment.create(2, "two", "2".toByteArray()) 100 | 101 | Segment(it.toString(), 0, "test-segment-") 102 | } 103 | } 104 | 105 | @Test 106 | fun totalEntries() { 107 | segment { 108 | Assert.assertEquals(0, it.totalEntries) 109 | it.create(0, "one", "1".toByteArray()) 110 | it.create(1, "two", "2".toByteArray()) 111 | it.create(2, "three", "3".toByteArray()) 112 | Assert.assertEquals(3, it.totalEntries) 113 | it.beginTransaction(3) 114 | it.set(4, "one", "1".toByteArray(), "one".toByteArray()) 115 | it.endTransaction(5) 116 | Assert.assertEquals(6, it.totalEntries) 117 | } 118 | } 119 | 120 | @Test 121 | fun segmentId() { 122 | segment { 123 | Assert.assertEquals(-1, it.id) 124 | it.create(99, "one", "1".toByteArray()) 125 | Assert.assertEquals(99, it.id) 126 | it.create(100, "two", "2".toByteArray()) 127 | Assert.assertEquals(99, it.id) 128 | } 129 | } 130 | 131 | @Test 132 | fun messageDigest() { 133 | val md0 = MessageDigest.getInstance("MD5") 134 | val md1 = MessageDigest.getInstance("MD5") 135 | 136 | md0.update(byteArrayOf(0, 1, 2, 3)) 137 | 138 | md1.update(byteArrayOf(0, 1)) 139 | md1.update(byteArrayOf(2)) 140 | md1.update(byteArrayOf(3)) 141 | 142 | Assert.assertArrayEquals(md0.digest(), md1.digest()) 143 | } 144 | 145 | @Test (expected = Throwable::class) 146 | fun corruptedHeader() { 147 | withDummy { 148 | // Create segment. 149 | val segment0 = Segment(it.toString(), 0, "test-segment-") 150 | segment0.create(0, "one", "1".toByteArray()) 151 | 152 | // Writes invalid data to header. 153 | val file = File(segment0.path.toUri()) 154 | val raf = RandomAccessFile(file, "rw") 155 | raf.seek(0) 156 | raf.write(byteArrayOf(0, 0, 0, 0)) 157 | 158 | // Tries to read segment again. 159 | val segment1 = Segment(it.toString(), 0, "test-segment-") 160 | } 161 | } 162 | } 163 | -------------------------------------------------------------------------------- /gradlew: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | ############################################################################## 4 | ## 5 | ## Gradle start up script for UN*X 6 | ## 7 | ############################################################################## 8 | 9 | # Attempt to set APP_HOME 10 | # Resolve links: $0 may be a link 11 | PRG="$0" 12 | # Need this for relative symlinks. 13 | while [ -h "$PRG" ] ; do 14 | ls=`ls -ld "$PRG"` 15 | link=`expr "$ls" : '.*-> \(.*\)$'` 16 | if expr "$link" : '/.*' > /dev/null; then 17 | PRG="$link" 18 | else 19 | PRG=`dirname "$PRG"`"/$link" 20 | fi 21 | done 22 | SAVED="`pwd`" 23 | cd "`dirname \"$PRG\"`/" >/dev/null 24 | APP_HOME="`pwd -P`" 25 | cd "$SAVED" >/dev/null 26 | 27 | APP_NAME="Gradle" 28 | APP_BASE_NAME=`basename "$0"` 29 | 30 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 31 | DEFAULT_JVM_OPTS="" 32 | 33 | # Use the maximum available, or set MAX_FD != -1 to use that value. 34 | MAX_FD="maximum" 35 | 36 | warn ( ) { 37 | echo "$*" 38 | } 39 | 40 | die ( ) { 41 | echo 42 | echo "$*" 43 | echo 44 | exit 1 45 | } 46 | 47 | # OS specific support (must be 'true' or 'false'). 48 | cygwin=false 49 | msys=false 50 | darwin=false 51 | nonstop=false 52 | case "`uname`" in 53 | CYGWIN* ) 54 | cygwin=true 55 | ;; 56 | Darwin* ) 57 | darwin=true 58 | ;; 59 | MINGW* ) 60 | msys=true 61 | ;; 62 | NONSTOP* ) 63 | nonstop=true 64 | ;; 65 | esac 66 | 67 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 68 | 69 | # Determine the Java command to use to start the JVM. 70 | if [ -n "$JAVA_HOME" ] ; then 71 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 72 | # IBM's JDK on AIX uses strange locations for the executables 73 | JAVACMD="$JAVA_HOME/jre/sh/java" 74 | else 75 | JAVACMD="$JAVA_HOME/bin/java" 76 | fi 77 | if [ ! -x "$JAVACMD" ] ; then 78 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 79 | 80 | Please set the JAVA_HOME variable in your environment to match the 81 | location of your Java installation." 82 | fi 83 | else 84 | JAVACMD="java" 85 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 86 | 87 | Please set the JAVA_HOME variable in your environment to match the 88 | location of your Java installation." 89 | fi 90 | 91 | # Increase the maximum file descriptors if we can. 92 | if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then 93 | MAX_FD_LIMIT=`ulimit -H -n` 94 | if [ $? -eq 0 ] ; then 95 | if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then 96 | MAX_FD="$MAX_FD_LIMIT" 97 | fi 98 | ulimit -n $MAX_FD 99 | if [ $? -ne 0 ] ; then 100 | warn "Could not set maximum file descriptor limit: $MAX_FD" 101 | fi 102 | else 103 | warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" 104 | fi 105 | fi 106 | 107 | # For Darwin, add options to specify how the application appears in the dock 108 | if $darwin; then 109 | GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" 110 | fi 111 | 112 | # For Cygwin, switch paths to Windows format before running java 113 | if $cygwin ; then 114 | APP_HOME=`cygpath --path --mixed "$APP_HOME"` 115 | CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` 116 | JAVACMD=`cygpath --unix "$JAVACMD"` 117 | 118 | # We build the pattern for arguments to be converted via cygpath 119 | ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` 120 | SEP="" 121 | for dir in $ROOTDIRSRAW ; do 122 | ROOTDIRS="$ROOTDIRS$SEP$dir" 123 | SEP="|" 124 | done 125 | OURCYGPATTERN="(^($ROOTDIRS))" 126 | # Add a user-defined pattern to the cygpath arguments 127 | if [ "$GRADLE_CYGPATTERN" != "" ] ; then 128 | OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" 129 | fi 130 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 131 | i=0 132 | for arg in "$@" ; do 133 | CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` 134 | CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option 135 | 136 | if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition 137 | eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` 138 | else 139 | eval `echo args$i`="\"$arg\"" 140 | fi 141 | i=$((i+1)) 142 | done 143 | case $i in 144 | (0) set -- ;; 145 | (1) set -- "$args0" ;; 146 | (2) set -- "$args0" "$args1" ;; 147 | (3) set -- "$args0" "$args1" "$args2" ;; 148 | (4) set -- "$args0" "$args1" "$args2" "$args3" ;; 149 | (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; 150 | (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; 151 | (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; 152 | (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; 153 | (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; 154 | esac 155 | fi 156 | 157 | # Escape application args 158 | save ( ) { 159 | for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done 160 | echo " " 161 | } 162 | APP_ARGS=$(save "$@") 163 | 164 | # Collect all arguments for the java command, following the shell quoting and substitution rules 165 | eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" 166 | 167 | # by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong 168 | if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then 169 | cd "$(dirname "$0")" 170 | fi 171 | 172 | exec "$JAVACMD" "$@" 173 | -------------------------------------------------------------------------------- /src/main/kotlin/org/botellier/command/Command.kt: -------------------------------------------------------------------------------- 1 | package org.botellier.command 2 | 3 | import org.botellier.server.Client 4 | import org.botellier.server.Server 5 | import org.botellier.store.* 6 | import org.botellier.value.StoreValue 7 | import kotlin.reflect.KMutableProperty 8 | import kotlin.reflect.full.declaredMemberProperties 9 | 10 | @Target(AnnotationTarget.CLASS) 11 | annotation class WithCommand(val name: String) 12 | 13 | @Target(AnnotationTarget.FIELD) 14 | annotation class Parameter(val order: Int, val optional: Boolean = false) 15 | 16 | abstract class Command { 17 | val name: String by lazy { 18 | val withCommand = this::class.annotations.find { it is WithCommand } as? WithCommand 19 | withCommand?.name?.toUpperCase() ?: throw CommandException.InvalidCommandDeclarationException() 20 | } 21 | 22 | val parameters by lazy { 23 | this::class.declaredMemberProperties 24 | .filter { 25 | val field = this::class.java.getDeclaredField(it.name) 26 | 27 | if (field.javaClass.isInstance(CValue::class)) { 28 | throw CommandException.InvalidPropertyException(this::class.toString(), it.name, "must be a CValue") 29 | } 30 | if (it !is KMutableProperty<*>) { 31 | throw CommandException.InvalidPropertyException(this::class.toString(), it.name, "must be mutable") 32 | } 33 | 34 | field.getAnnotation(Parameter::class.java) != null 35 | } 36 | .filterIsInstance>() 37 | .sortedBy { 38 | val field = this::class.java.getDeclaredField(it.name) 39 | val parameter = field.getAnnotation(Parameter::class.java) 40 | parameter?.order ?: 0 41 | } 42 | .map { 43 | val field = this::class.java.getDeclaredField(it.name) 44 | val parameter = field.getAnnotation(Parameter::class.java) 45 | CParameter(this, it, parameter?.optional ?: false) 46 | } 47 | } 48 | 49 | // Values for initializing parameters in child classes. 50 | protected val intValue = CValue.Primitive.Int(0) 51 | protected val floatValue = CValue.Primitive.Float(0.0) 52 | protected val stringValue = CValue.Primitive.String("") 53 | protected val anyValue: CValue.Primitive = CValue.Primitive.Any() 54 | 55 | protected val intArrayValue = CValue.Array.Int(emptyList()) 56 | protected val floatArrayValue = CValue.Array.Float(emptyList()) 57 | protected val stringArrayValue = CValue.Array.String(emptyList()) 58 | protected val anyArrayValue = CValue.Array.Any(emptyList()) 59 | protected val pairArrayValue = CValue.Array.Pair(emptyList()) 60 | 61 | // Other. 62 | override fun toString(): String { 63 | val builder = StringBuilder() 64 | builder.append("$name ") 65 | var it = parameters.iterator() 66 | while (it.hasNext()) { 67 | builder.append(it.next().get().toString()) 68 | if (it.hasNext()) { 69 | builder.append(" ") 70 | } 71 | } 72 | return builder.toString() 73 | } 74 | } 75 | 76 | // ---------------- 77 | // Command types. 78 | // ---------------- 79 | 80 | /** 81 | * Commands that need access to the server. 82 | */ 83 | abstract class ConnCommand : Command() { 84 | open fun run(server: Server, client: Client): StoreValue { 85 | throw CommandException.CommandDisabledException(name) 86 | } 87 | fun execute(server: Server, client: Client) = run(server, client) 88 | } 89 | 90 | /** 91 | * Commands that has read-only access to the store. 92 | */ 93 | abstract class ReadStoreCommand : Command() { 94 | open fun run(store: ReadStore): StoreValue { 95 | throw CommandException.CommandDisabledException(name) 96 | } 97 | fun execute(store: ReadStore) = run(store) 98 | } 99 | 100 | /** 101 | * Commands related to node-level functions such as replication. 102 | * These type of commands return binary data, and they are not 103 | * intended to be used by the end-user. 104 | */ 105 | abstract class NodeCommand : Command() { 106 | open fun run(server: Server, client: Client): ByteArray { 107 | throw CommandException.CommandDisabledException(name) 108 | } 109 | fun execute(server: Server, client: Client) = run(server, client) 110 | } 111 | 112 | // TODO: Add logging to mutator functions. 113 | /** 114 | * Commands that have full access to the store, therefore, 115 | * all their actions must be logged. The parameter to run 116 | * is now a StoreTransaction that the command can use for making 117 | * changes to the store. 118 | */ 119 | abstract class StoreCommand : Command() { 120 | var transaction : StoreTransaction? = null 121 | 122 | open fun run(transaction: StoreTransaction): StoreValue { 123 | throw CommandException.CommandDisabledException(name) 124 | } 125 | 126 | fun transaction(block: StoreTransaction.() -> StoreValue): StoreValue { 127 | return this.transaction!!.block() 128 | } 129 | 130 | fun execute(store: Store): StoreValue { 131 | this.transaction = store.transaction() 132 | val ret = run(this.transaction!!) 133 | 134 | if (store is PersistentStore) { 135 | this.transaction!!.commit(store.log) 136 | } else { 137 | this.transaction!!.commit() 138 | } 139 | 140 | return ret 141 | } 142 | } 143 | 144 | // ---------------- 145 | // Exceptions. 146 | // ---------------- 147 | 148 | sealed class CommandException(msg: String) : Throwable(msg) { 149 | // Declaration exceptions. 150 | class InvalidCommandDeclarationException 151 | : CommandException("Command must declare a name using @WIthComand annotation.") 152 | 153 | class InvalidPropertyException(className: String, paramName: String, msg: String) 154 | : CommandException("Property '$paramName` from [$className]: $msg") 155 | 156 | // Execution exceptions. 157 | class RuntimeException(msg: String) 158 | : CommandException(msg) 159 | 160 | class CommandDisabledException(name: String) 161 | : CommandException("Command '$name' is currently disabled.") 162 | 163 | class WrongTypeException(key: String, currentType: String) 164 | : CommandException("Invalid operation on '$key' of type '$currentType'.") 165 | } 166 | -------------------------------------------------------------------------------- /src/main/kotlin/org/botellier/value/Parser.kt: -------------------------------------------------------------------------------- 1 | package org.botellier.value 2 | 3 | /** 4 | * Takes a list of tokens and an extension lambda to parse the given tokens. 5 | * @param tokens the list of Lexer.Token values. 6 | * @param init the extension lambda. 7 | * @return A parser that contains the given tokens. 8 | */ 9 | fun parse(tokens: List, init: Parser.() -> Unit): Parser { 10 | val parser = Parser(tokens) 11 | parser.parse(init) 12 | return parser 13 | } 14 | 15 | fun parse(string: String, init: Parser.() -> Unit): Parser = parse(Lexer(string).lex().toList(), init) 16 | 17 | /** 18 | * Parser. 19 | */ 20 | 21 | @DslMarker 22 | annotation class ParserMarker 23 | 24 | /** 25 | * Parser class for parsing a list of Lexer tokens. You tell the parser what 26 | * you expect, and it throws an exception if it finds something else. 27 | */ 28 | @ParserMarker 29 | open class Parser(val tokens: List) { 30 | var index: Int = 0 31 | private set 32 | public get 33 | 34 | /** 35 | * Function used for parsing the token list from the beginning. 36 | * The given init lambda has access to all the public parsing 37 | * functions. 38 | * @param init the lambda function for parsing. 39 | */ 40 | fun parse(init: Parser.() -> Unit) { 41 | index = 0 42 | this.init() 43 | if (index < tokens.size) { 44 | throw ParserException.ParsingException("Not all input was consumed.") 45 | } 46 | } 47 | 48 | /** 49 | * If the next token in the list is an IntToken, return its value. Throws 50 | * an exception otherwise. 51 | * @returns the value of the IntToken. 52 | * @throws ParserException.ParsingException if the value is not the correct token. 53 | */ 54 | fun int(): Int { 55 | val token = nextToken() 56 | if (token is Lexer.IntToken) { 57 | return token.value 58 | } 59 | else { 60 | throw ParserException.ParsingException("Expected IntToken, found $token") 61 | } 62 | } 63 | 64 | /** 65 | * If the next token in the list is a FloatTOken, return its value. Throws 66 | * an exception otherwise. 67 | * @returns the value of the FloatToken. 68 | * @throws ParserException.ParsingException if the value is not the correct token. 69 | */ 70 | fun float(): Double { 71 | val token = nextToken() 72 | if (token is Lexer.FloatToken) { 73 | return token.value 74 | } 75 | else { 76 | throw ParserException.ParsingException("Expected FloatToken, found $token") 77 | } 78 | } 79 | 80 | /** 81 | * If the next token in the list is a IntToken or FloatTOken, return its value as 82 | * a floating point value. Throws an exception otherwise. 83 | * @returns the numeric value as a floating point value. 84 | * @throws ParserException.ParsingException if the value is not the correct token. 85 | */ 86 | fun number(): Double { 87 | val token = nextToken() 88 | if (token is Lexer.IntToken) { 89 | return token.value.toDouble() 90 | } 91 | else if (token is Lexer.FloatToken) { 92 | return token.value 93 | } 94 | else { 95 | throw ParserException.ParsingException("Expected number token, found $token") 96 | } 97 | } 98 | 99 | /** 100 | * If the next token in the list is a StringToken, return its value. Throws 101 | * an exception otherwise. 102 | * @param expected lets the function also check for equality to the given value. 103 | * @returns the value of the StringToken. 104 | * @throws ParserException.ParsingException if the value is not the correct token. 105 | */ 106 | fun string(expected: String? = null): String { 107 | val token = nextToken() 108 | if (token is Lexer.StringToken) { 109 | if (expected == null) { 110 | return token.value 111 | } 112 | else if (expected == token.value) { 113 | return token.value 114 | } 115 | else { 116 | throw ParserException.ParsingException("Expected StringToken \"$expected\", found \"${token.value}\"") 117 | } 118 | } 119 | else { 120 | throw ParserException.ParsingException("Expected StringToken, found $token") 121 | } 122 | } 123 | 124 | /** 125 | * Accepts any token. 126 | * @returns the value of the token as 'Any'. 127 | * @throws ParserException.ParsingException if the token is invalid. 128 | */ 129 | fun any(): Any { 130 | val token = nextToken() 131 | return when (token) { 132 | is Lexer.IntToken -> token.value 133 | is Lexer.FloatToken -> token.value 134 | is Lexer.StringToken -> token.value 135 | else -> throw ParserException.ParsingException("Invalid token in any().") 136 | } 137 | } 138 | 139 | /** 140 | * Skips the given amount of tokens. 141 | * @param n the amount of tokens to skip. 142 | */ 143 | fun skip(n: Int = 0) { 144 | if (n <= 0) index = tokens.size else index += n 145 | } 146 | 147 | /** 148 | * Returns the next token in the list without any 149 | * type checks. 150 | * @returns the next token in the list. 151 | * @throws ParserException.EOFException 152 | */ 153 | private fun nextToken(): Lexer.Token { 154 | if (index < tokens.size) { 155 | return tokens.get(index++) 156 | } 157 | else { 158 | throw ParserException.EOFException() 159 | } 160 | } 161 | 162 | // Special combinators. 163 | 164 | inline fun many(which: () -> T): List { 165 | val array = arrayListOf() 166 | var prev: Int 167 | while (true) { 168 | try { 169 | prev = index 170 | array.add(which()) 171 | } 172 | catch (e: ParserException.EOFException) { 173 | break 174 | } 175 | if (prev == index) { 176 | throw ParserException.ParsingException("Combinator function must consume input.") 177 | } 178 | } 179 | return array.toList() 180 | } 181 | 182 | inline fun many1(which: () -> T): List { 183 | val array = many(which) 184 | if (array.isEmpty()) { 185 | throw ParserException.EOFException() 186 | } 187 | return array 188 | } 189 | } 190 | 191 | sealed class ParserException(msg: String) : Throwable(msg) { 192 | class ParsingException(msg: String) : ParserException(msg) 193 | class EOFException : ParserException("Unexpected end of input.") 194 | } 195 | -------------------------------------------------------------------------------- /src/main/kotlin/org/botellier/log/Log.kt: -------------------------------------------------------------------------------- 1 | package org.botellier.log 2 | 3 | import java.io.File 4 | import java.nio.file.Path 5 | import java.nio.file.Paths 6 | 7 | /** 8 | * Creates a new log handler with at the specified base dir. The size 9 | * of each segment file can also be specified. 10 | * 11 | * This class is basically an abstraction on top of Segment. It offers automatic creation 12 | * of new segments and querying of entries by range, which is useful for getting only a subset of 13 | * all the entries. 14 | * 15 | * @param root the base dir to use for logs. 16 | * @param clear if set to true, it will clear all existing segments and start from scratch. 17 | * @property segmentPrefix the prefix string to use for filenames. 18 | * @property segmentSize the size (in bytes) of each segment. 19 | * @property path the base directory that holds all the logs. 20 | * @property id an monotonic increasing value that changes each time an entry is added. 21 | * @property segments the list of segments for this log. 22 | */ 23 | class Log(root: String = "./", val segmentPrefix: String = "segment-", val segmentSize: Int = 2*1024*1024, clear: Boolean = false) 24 | : Iterable { 25 | val path: Path 26 | var id: Int private set 27 | val segments: MutableList 28 | 29 | init { 30 | // Initializes path. 31 | val givenPath = Paths.get(root) 32 | val file = File(givenPath.toUri()) 33 | 34 | if (file.isDirectory) { 35 | path = givenPath.toAbsolutePath().normalize() 36 | } else if (file.isFile) { 37 | path = givenPath.toAbsolutePath().parent.normalize() 38 | } else { 39 | file.mkdirs() 40 | path = givenPath.toAbsolutePath().normalize() 41 | } 42 | 43 | // Initializes segment. 44 | segments = findSegments().toMutableList() 45 | id = findId(segments) 46 | 47 | if (clear) { this.clear() } 48 | if (segments.size <= 0) { 49 | segments.add(Segment(path.toString(), 0, segmentPrefix, segmentSize)) 50 | } 51 | } 52 | 53 | /** 54 | * Clears the segment files for this log. 55 | */ 56 | fun clear() { 57 | segments.map { it.clear() } 58 | segments.clear() 59 | } 60 | 61 | /** 62 | * Finds existing segments in the given path. 63 | */ 64 | fun findSegments(): List { 65 | val regex = Regex("^($segmentPrefix)(\\d+)$") 66 | val folder = File(path.toUri()) 67 | return folder.listFiles() 68 | .filter { it.isFile && regex.matches(it.name) } 69 | .map { regex.find(it.name)!! } 70 | .map { Pair(it.groupValues[1], it.groupValues[2].toInt()) } 71 | .sortedBy { it.second } 72 | .map { Segment(path.toString(), it.second, segmentPrefix, segmentSize) } 73 | } 74 | 75 | /** 76 | * Finds the best id value based on the given segments. 77 | */ 78 | private fun findId(segments: List): Int { 79 | if (segments.isEmpty()) { 80 | return 0 81 | } else { 82 | return 0 83 | } 84 | } 85 | 86 | // ---------------- 87 | // Entry operations. 88 | // ---------------- 89 | 90 | private fun logOperation(f: (Int, Segment) -> Unit) { 91 | try { 92 | f(id, segments.last()) 93 | id++ 94 | } catch (e: SegmentException) { 95 | segments.add(segments.last().nextSegment()) 96 | logOperation(f) 97 | } 98 | } 99 | 100 | /** 101 | * Appends a deletion entry to the underlying segment. 102 | * @see Segment.delete 103 | */ 104 | fun delete(key: String) { 105 | logOperation { id, segment -> 106 | segment.delete(id, key) 107 | } 108 | } 109 | 110 | /** 111 | * Appends a data entry to the underlying segment. 112 | * @see Segment.set 113 | */ 114 | fun set(key: String, before: ByteArray, after: ByteArray) { 115 | logOperation { id, segment -> 116 | segment.set(id, key, before, after) 117 | } 118 | } 119 | 120 | /** 121 | * Appends a data entry to the underlying segment. 122 | * @see Segment.create 123 | */ 124 | fun create(key: String, data: ByteArray) { 125 | logOperation { id, segment -> 126 | segment.create(id, key, data) 127 | } 128 | } 129 | 130 | /** 131 | * Appends a new entry to the log indicating 132 | * the beginning of a transaction. 133 | * @see Segment.beginTransaction 134 | */ 135 | fun beginTransaction() { 136 | logOperation { id, segment -> 137 | segment.beginTransaction(id) 138 | } 139 | } 140 | 141 | /** 142 | * Appends a new entry to the log indicating 143 | * the end of a transaction. 144 | * @see Segment.endTransaction 145 | */ 146 | fun endTransaction() { 147 | logOperation { id, segment -> 148 | segment.endTransaction(id) 149 | } 150 | } 151 | 152 | // ---------------- 153 | // Querying and extending. 154 | // ---------------- 155 | 156 | /** 157 | * Returns an iterator that traverses the entries starting with the entry that has the given id. 158 | * @param start the id of the first entry. 159 | * @returns a Sequence containing the entries. 160 | */ 161 | fun query(start: Int): Sequence { 162 | val logIterator = LogIterator(segments.dropWhile { it.id + it.totalEntries < start }) 163 | return logIterator 164 | .asSequence() 165 | .dropWhile { it.id < start } 166 | } 167 | 168 | /** 169 | * Reads the given entry and writes to the current log as is. That means the ID needs to correspond 170 | * to next ID in this log's sequence. 171 | * @param entry the entry to read. 172 | */ 173 | fun extend(entry: Entry) { 174 | if (id == entry.id) { 175 | logOperation { id, segment -> 176 | segment.segmentOperation(id, { entry }) 177 | } 178 | } else { 179 | throw LogException.ExtendException(id, entry.id) 180 | } 181 | } 182 | 183 | /** 184 | * Just like extend but takes a sequence of entries instead of a single one. 185 | * @param entries the sequence of entries to read. 186 | */ 187 | fun extend(entries: Sequence) { 188 | entries.toList().map { this.extend(it) } 189 | } 190 | 191 | // ---------------- 192 | // Iterable. 193 | // ---------------- 194 | 195 | override fun iterator(): Iterator = LogIterator(segments) 196 | } 197 | 198 | class LogIterator(segments: List) : Iterator { 199 | var iterators = segments.map { it.iterator() } 200 | 201 | override fun hasNext(): Boolean { 202 | if (iterators.isNotEmpty() && iterators.first().hasNext()) { 203 | return true 204 | } else if (iterators.isNotEmpty()) { 205 | iterators = iterators.drop(1) 206 | return hasNext() 207 | } else { 208 | return false 209 | } 210 | } 211 | 212 | override fun next(): Entry { 213 | return iterators.first().next() 214 | } 215 | } 216 | 217 | sealed class LogException(msg: String) : Throwable(msg) { 218 | class ExtendException(id: Int, tried: Int) 219 | : LogException("Invalid extend id ($tried); has $id.") 220 | } 221 | -------------------------------------------------------------------------------- /src/main/kotlin/org/botellier/value/Lexer.kt: -------------------------------------------------------------------------------- 1 | package org.botellier.value 2 | 3 | import java.io.BufferedInputStream 4 | import java.io.ByteArrayOutputStream 5 | import java.io.EOFException 6 | import java.io.InputStream 7 | 8 | private val CR = '\r'.toInt() 9 | private val LF = '\n'.toInt() 10 | private val COLON = ':'.toInt() 11 | private val SEMICOLON = ';'.toInt() 12 | private val DOLLAR = '$'.toInt() 13 | private val STAR = '*'.toInt() 14 | 15 | // check: http://stackoverflow.com/questions/17848207/making-a-lexical-analyzer 16 | // check: http://llvm.org/docs/tutorial/index.html 17 | /** 18 | * Lexer class takes an input stream and lexes byte by byte. Useful for 19 | * reading information that is being received through a socket. 20 | * @param bufferedInputStream the buffered input stream (required for peeking values 21 | * and such). 22 | */ 23 | class Lexer(bufferedInputStream: BufferedInputStream) { 24 | private var index = 0 25 | private val reader = bufferedInputStream 26 | 27 | constructor(string: String) : this(string.byteInputStream().buffered()) 28 | 29 | /** 30 | * Returns a token. 31 | * @return a token. 32 | * @throws LexerException if the string format is invalid. 33 | */ 34 | fun lex(): Token { 35 | index = 0 36 | return token() 37 | } 38 | 39 | // Lexing functions. 40 | private fun token(): Token { 41 | val next = reader.peek() 42 | return when(next) { 43 | COLON -> intToken() 44 | SEMICOLON -> floatToken() 45 | DOLLAR -> stringToken() 46 | STAR -> listToken() 47 | else -> throw LexerException.LexingException(index, "Expected one of ':', ';', '$', '*'; found '$next'.") 48 | } 49 | } 50 | 51 | private fun primitiveToken(): PrimitiveToken { 52 | val next = reader.peek() 53 | return when(next) { 54 | COLON -> intToken() 55 | SEMICOLON -> floatToken() 56 | DOLLAR -> stringToken() 57 | else -> throw LexerException.LexingException(index, "Expected one of ':', ';', '$'; found '$next'.") 58 | } 59 | } 60 | 61 | private fun intToken(): PrimitiveToken { 62 | reader.expect(COLON) 63 | return IntToken(int()) 64 | } 65 | 66 | private fun floatToken(): PrimitiveToken { 67 | reader.expect(SEMICOLON) 68 | return FloatToken(float()) 69 | } 70 | 71 | private fun stringToken(): PrimitiveToken { 72 | reader.expect(DOLLAR) 73 | val length = int() 74 | val bulk = bulkString(length) 75 | return castToken(bulk) 76 | } 77 | 78 | private fun listToken(): ListToken { 79 | reader.expect(STAR) 80 | val length = int() 81 | val array = ArrayList(length) 82 | for (i in 0..length - 1) { 83 | array.add(primitiveToken()) 84 | } 85 | return ListToken(array.toList()) 86 | } 87 | 88 | private fun int(): Int { 89 | val number = reader.readWhile { it != CR } 90 | reader.expect(CR) 91 | reader.expect(LF) 92 | try { 93 | return String(number).toInt() 94 | } catch (e: NumberFormatException) { 95 | throw LexerException.LexingException(index, "Invalid integer number.") 96 | } 97 | } 98 | 99 | private fun float(): Double { 100 | val number = reader.readWhile { it != CR } 101 | reader.expect(CR) 102 | reader.expect(LF) 103 | try { 104 | return String(number).toDouble() 105 | } catch (e: NumberFormatException) { 106 | throw LexerException.LexingException(index, "Invalid floating point number.") 107 | } 108 | } 109 | 110 | private fun bulkString(length: Int): ByteArray { 111 | if (length <= 0) { 112 | throw LexerException.LexingException(index, "Bulk strings must have positive length.") 113 | } 114 | 115 | val string = reader.readBytes(length) 116 | reader.expect(CR) 117 | reader.expect(LF) 118 | 119 | return string 120 | } 121 | 122 | private fun castToken(bytes: ByteArray): PrimitiveToken { 123 | val string = String(bytes) 124 | try { 125 | if (string.matches(Regex("^[-+]?[0-9]+$"))) { 126 | return IntToken(string.toInt()) 127 | } else if (string.matches(Regex("^[-+]?[0-9]*\\.[0-9]+$"))) { 128 | return FloatToken(string.toDouble()) 129 | } else { 130 | return StringToken(string) 131 | } 132 | } 133 | catch(e: NumberFormatException) { 134 | throw LexerException.LexingException(index, "Unable to cast \"$string\" to a token.") 135 | } 136 | } 137 | 138 | // ---------------- 139 | // Extension functions for BufferedInputStream. 140 | // ---------------- 141 | 142 | private fun BufferedInputStream.readByte(): Int { 143 | val byte = this.read() 144 | if (byte != -1) { 145 | index++ 146 | return byte 147 | } 148 | else { 149 | throw LexerException.EOFException(index) 150 | } 151 | } 152 | 153 | private fun BufferedInputStream.peek(): Int { 154 | this.mark(1) 155 | val byte = this.read() 156 | this.reset() 157 | if (byte != -1) { 158 | return byte 159 | } 160 | else { 161 | throw LexerException.EOFException(index) 162 | } 163 | } 164 | 165 | private fun BufferedInputStream.readBytes(length: Int): ByteArray { 166 | val stream = ByteArrayOutputStream() 167 | var length = length 168 | while (length > 0) { 169 | stream.write(this.readByte()) 170 | length-- 171 | } 172 | return stream.toByteArray() 173 | } 174 | 175 | private fun BufferedInputStream.readWhile(pred: (Int) -> Boolean): ByteArray { 176 | val stream = ByteArrayOutputStream() 177 | 178 | var byte = this.peek() 179 | while (pred(byte)) { 180 | stream.write(this.readByte()) 181 | byte = this.peek() 182 | } 183 | 184 | return stream.toByteArray() 185 | } 186 | 187 | private fun BufferedInputStream.expect(expected: Int): Int { 188 | val actual = this.readByte() 189 | if (actual == expected) { 190 | return actual 191 | } 192 | else { 193 | throw LexerException.LexingException(index, "Expected '$expected'; found '$actual'.") 194 | } 195 | } 196 | 197 | // ---------------- 198 | // Inner classes. 199 | // ---------------- 200 | 201 | interface Token 202 | interface PrimitiveToken : Token 203 | 204 | data class IntToken(val value: Int) : PrimitiveToken 205 | data class FloatToken(val value: Double) : PrimitiveToken 206 | data class StringToken(val value: String) : PrimitiveToken 207 | data class ListToken(val value: List) : Token 208 | 209 | } 210 | 211 | fun Lexer.Token.toList(): List { 212 | return when (this) { 213 | is Lexer.ListToken -> this.value 214 | else -> listOf(this) 215 | } 216 | } 217 | 218 | sealed class LexerException(index: Int, msg: String) : Throwable("(At [$index]) $msg") { 219 | class LexingException(index: Int, msg: String) : LexerException(index, msg) 220 | class EOFException(index: Int) : LexerException(index, "Unexpected end of input") 221 | } 222 | -------------------------------------------------------------------------------- /src/main/kotlin/org/botellier/store/StoreTransaction.kt: -------------------------------------------------------------------------------- 1 | package org.botellier.store 2 | 3 | import org.botellier.log.Log 4 | import org.botellier.serializer.toByteArray 5 | import org.botellier.value.* 6 | 7 | /** 8 | * Class for managing transactions in the given store (a.k.a. MapValue). All the functions 9 | * (except commit) are not applied directly, they are instead saved to a cache and applied 10 | * when calling commit. 11 | */ 12 | class StoreTransaction(initialStore: MapValue) { 13 | private val changes = mutableListOf() 14 | private val map = initialStore.unwrap() 15 | private val cache = initialStore.clone().unwrap() 16 | 17 | /** 18 | * Returns the value at the given key. This function doesn't really make any changes, 19 | * it is there just for convenience. 20 | * @param key the key to lookup for. 21 | * @returns the StoreValue in the underlying store. 22 | */ 23 | fun get(key: String): StoreValue { 24 | return cache.get(key) ?: map.get(key) ?: NilValue() 25 | } 26 | 27 | /** 28 | * Sets the value of the given key to the given value. If the key already 29 | * exists throws an exception. 30 | * @param key the key to set. 31 | * @param value the value to use. 32 | * @returns 33 | */ 34 | fun set(key: String, value: StoreValue) { 35 | val current = cache.get(key) ?: NilValue() 36 | dispatch(key, current, value) 37 | } 38 | 39 | /** 40 | * Updates an existing key, sending the current value to the lambda 41 | * function and using its return value as the new value. Updating 42 | * and setting a key to a NilValue is the same as deleting it. 43 | * @param key the key to update. 44 | * @param block the lambda function that receives the old value and returns the new value. 45 | * @throws StoreException.InvalidTypeException if the key to update doesn't exists or if the requested value is a 46 | * different type. 47 | */ 48 | inline fun update(key: String, block: (T) -> StoreValue): StoreValue { 49 | val value = get(key) 50 | if (value !is NilValue) { 51 | if (value is T) { 52 | val nextValue = block(value) 53 | dispatch(key, value, nextValue) 54 | return nextValue 55 | } else { 56 | throw StoreException.InvalidTypeException(key, value::class.qualifiedName!!) 57 | } 58 | } else { 59 | throw StoreException.InvalidTypeException(key, NilValue::class.qualifiedName!!) 60 | } 61 | } 62 | 63 | /** 64 | * Works just like update, the only difference is that that the key to update 65 | * may or may not exist already. 66 | * @param key the key to update. 67 | * @param block the lambda function that receives the nullable old value and returns the new value. 68 | * @throws StoreException.InvalidTypeException if the requested value is a different type. 69 | */ 70 | inline fun mupdate(key: String, block: (T?) -> StoreValue): StoreValue { 71 | val value = get(key) 72 | if (value !is NilValue) { 73 | if (value is T) { 74 | val nextValue = block(value) 75 | dispatch(key, value, nextValue) 76 | return nextValue 77 | } else { 78 | throw StoreException.InvalidTypeException(key, value::class.qualifiedName!!) 79 | } 80 | } else { 81 | val nextValue = block(null) 82 | dispatch(key, value, nextValue) 83 | return nextValue 84 | } 85 | } 86 | 87 | /** 88 | * Deletes the value at the given key. 89 | * @param key the key to delete. 90 | * @returns the value of the key to be deleted. 91 | */ 92 | fun delete(key: String): StoreValue { 93 | val value = cache.remove(key) ?: NilValue() 94 | dispatch(key, value, NilValue()) 95 | return value 96 | } 97 | 98 | /** 99 | * Begins a transaction block that is cleared before starting, 100 | * and committed before returning. Useful for expressions like 101 | * the following: 102 | * 103 | * ``` 104 | * transaction.begin { 105 | * set("one", IntValue(1)) 106 | * set("two", IntValue(2)) 107 | * } 108 | * ``` 109 | * 110 | * Instead of: 111 | * 112 | * ``` 113 | * transaction.rollback() 114 | * transaction.set("one", IntValue(1)) 115 | * transaction.set("two", IntValue(2)) 116 | * transaction.commit() 117 | * ``` 118 | * 119 | * @param block the extension lambda 120 | * @returns a reference to the same transaction in use. 121 | */ 122 | fun begin(block: StoreTransaction.() -> Unit) { 123 | this.rollback() 124 | this.block() 125 | this.commit() 126 | } 127 | 128 | /** 129 | * Commits the changes generated by this transaction. Logging all the changes 130 | * to the underlying store's log. 131 | * @param log the optional Log that can be used to log the changes. 132 | */ 133 | fun commit(log: Log? = null) { 134 | // Logs changes. 135 | if (log != null) { 136 | for ((action, key, before, after) in changes) { 137 | when (action) { 138 | StoreChange.Action.SET -> { 139 | log.set(key, before.toByteArray(), after.toByteArray()) 140 | } 141 | StoreChange.Action.DELETE -> { 142 | log.delete(key) 143 | } 144 | } 145 | } 146 | } 147 | 148 | // Applies changes to store. 149 | for ((action, key, before, after) in changes) { 150 | when (action) { 151 | StoreChange.Action.SET -> { 152 | map.set(key, after) 153 | } 154 | StoreChange.Action.DELETE -> { 155 | map.remove(key) 156 | } 157 | } 158 | } 159 | 160 | changes.clear() 161 | } 162 | 163 | /** 164 | * Clears all the changes in the transaction. 165 | */ 166 | fun rollback() { 167 | this.changes.clear() 168 | this.cache.clear() 169 | } 170 | 171 | /** 172 | * Decides whenever to set or delete a key based on next value. 173 | * @param key the key to take the action on. 174 | * @param before the data that was in the key before. 175 | * @param after the new data of the key. 176 | */ 177 | fun dispatch(key: String, before: StoreValue, after: StoreValue) { 178 | if (after is NilValue || after.isEmpty()) { 179 | cache.remove(key) 180 | changes.add(StoreChange( 181 | StoreChange.Action.DELETE, 182 | key, 183 | before, 184 | NilValue() 185 | )) 186 | } else { 187 | cache.set(key, after) 188 | changes.add(StoreChange( 189 | StoreChange.Action.SET, 190 | key, 191 | before, 192 | after 193 | )) 194 | } 195 | } 196 | } 197 | 198 | /** 199 | * Class that represents a single action on a key (SET or DELETE). Note that 200 | * a StoreChange's SET can be used for both setting or updating a value. 201 | */ 202 | data class StoreChange(val action: Action, val key: String, val before: StoreValue, val after: StoreValue) { 203 | enum class Action { 204 | SET, DELETE 205 | } 206 | } 207 | 208 | /** 209 | * Checks if the given value is empty (empty lists, empty strings, etc). 210 | * @param value the value to check. 211 | * @returns true if the value is empty. 212 | */ 213 | private fun StoreType.isEmpty(): Boolean { 214 | return when (this) { 215 | is StringValue -> this.unwrap().isEmpty() 216 | is ListValue -> this.unwrap().isEmpty() 217 | is SetValue -> this.unwrap().isEmpty() 218 | is MapValue -> this.unwrap().isEmpty() 219 | else -> false 220 | } 221 | } 222 | -------------------------------------------------------------------------------- /src/main/kotlin/org/botellier/value/value.kt: -------------------------------------------------------------------------------- 1 | package org.botellier.value 2 | 3 | import java.util.concurrent.ConcurrentHashMap 4 | 5 | /** 6 | * Interface types, they serve the following purposes: 7 | * 8 | * - StoreType: The base type for primitives and collections. 9 | * - StoreWrapper: Interface for types that wrap around another type. 10 | * - StoreValue: It is used to represent types that can be stored 11 | * in a map. Maps themselves are not children of StoreValue to 12 | * prevent nested collections. 13 | * - StorePrimitive: Is either a number, string or nil; they are immutable. 14 | * - StoreNumber: Is a primitive that is either an int or a floating point number. 15 | * - StoreCollection: Can be either a list, set or map; they are mutable. 16 | * 17 | * Only stores values can be stored inside a store collection (i.e. no nested maps or list are 18 | * allowed). 19 | */ 20 | 21 | interface StoreType { 22 | /** 23 | * Clones this value (new object). 24 | */ 25 | fun clone(): StoreType 26 | } 27 | 28 | interface StoreWrapper { 29 | fun unwrap(): T 30 | } 31 | 32 | interface StoreValue: StoreType { 33 | override fun clone(): StoreValue 34 | } 35 | 36 | interface StorePrimitive : StoreValue { 37 | override fun clone(): StorePrimitive 38 | } 39 | 40 | interface StoreNumber : StorePrimitive { 41 | override fun clone(): StoreNumber 42 | } 43 | 44 | interface StoreCollection : StoreType, Iterable { 45 | val size: Int 46 | override fun clone(): StoreCollection 47 | } 48 | 49 | // ---------------- 50 | // Primitives. 51 | // ---------------- 52 | 53 | /** 54 | * Class that is inherited by all primitive types. It basically holds the field 55 | * for the underlying *immutable* value. 56 | */ 57 | abstract class PrimitiveValue(initialValue: T) : StoreWrapper, StorePrimitive, Comparable> 58 | where T : Comparable { 59 | private val value: T = initialValue 60 | 61 | override abstract fun clone(): PrimitiveValue 62 | 63 | override fun unwrap() = value 64 | 65 | override fun equals(other: Any?): Boolean { 66 | return when (other) { 67 | is PrimitiveValue<*> -> other.value == value 68 | else -> other == value 69 | } 70 | } 71 | 72 | override fun compareTo(other: PrimitiveValue): Int { 73 | return value.compareTo(other.value) 74 | } 75 | } 76 | 77 | class IntValue(initialValue: Int = 0) : StoreNumber, PrimitiveValue(initialValue) { 78 | override fun clone() = IntValue(unwrap()) 79 | override fun toString() = unwrap().toString() 80 | } 81 | 82 | class FloatValue(initialValue: Double = 0.0) : StoreNumber, PrimitiveValue(initialValue) { 83 | override fun clone() = FloatValue(unwrap()) 84 | override fun toString() = unwrap().toString() 85 | } 86 | 87 | class StringValue(initialValue: String = "") : PrimitiveValue(initialValue) { 88 | override fun clone() = StringValue(unwrap()) 89 | override fun toString() = unwrap() 90 | } 91 | 92 | class RawValue(val value: ByteArray = byteArrayOf()) : StorePrimitive { 93 | override fun clone(): StorePrimitive = RawValue(value.clone()) 94 | } 95 | 96 | class NilValue : StorePrimitive { 97 | override fun clone() = NilValue() 98 | override fun toString() = "nil" 99 | override fun equals(other: Any?) = other == null || other is NilValue 100 | } 101 | 102 | // ---------------- 103 | // Collections. 104 | // ---------------- 105 | 106 | /** 107 | * Wrapper over immutable List. 108 | */ 109 | class ListValue(private val list: List = listOf()): StoreWrapper>, StoreValue, StoreCollection { 110 | /** 111 | * Creates a mutable clone of the immutable list and passes it to the given function. The return 112 | * value of that function is then used to create a new ListValue. Note that internal values 113 | * are not copied. 114 | * @param block the callback to pass the mutable list to. 115 | * @returns the modified ListValue. 116 | */ 117 | fun copy(block: (MutableList) -> Unit = {}): ListValue { 118 | val next = list.toMutableList() 119 | block(next) 120 | return ListValue(next) 121 | } 122 | 123 | override val size: Int get() = list.size 124 | override fun clone(): ListValue = ListValue(list) 125 | override fun unwrap() = list 126 | override fun iterator(): Iterator = list.iterator() 127 | override fun toString() = list.joinToString(prefix = "[", postfix = "]") 128 | } 129 | 130 | // Useful extensions. 131 | 132 | fun MutableList.remove(indices: List): List { 133 | val indices = indices.sortedDescending().distinct() 134 | val removed = mutableListOf() 135 | indices.map { removed.add(this.removeAt(it)) } 136 | return removed.toList() 137 | } 138 | 139 | fun MutableList.lpush(value: T) { 140 | this.add(0, value) 141 | } 142 | 143 | fun MutableList.rpush(value: T) { 144 | this.add(this.lastIndex + 1, value) 145 | } 146 | 147 | fun MutableList.lpop(): T { 148 | return this.removeAt(0) 149 | } 150 | 151 | fun MutableList.rpop(): T { 152 | return this.removeAt(this.lastIndex) 153 | } 154 | 155 | fun MutableList.trim(start: Int, endInclusive: Int) { 156 | val start = if (start < 0) (start % size + size) % size else start 157 | val endInclusive = if (endInclusive < 0) (endInclusive % size + size) % size else endInclusive 158 | if (start > endInclusive) { 159 | this.clear() 160 | } else { 161 | this.remove((start..endInclusive).toList()) 162 | } 163 | } 164 | 165 | fun List.slice(start: Int, endInclusive: Int): List { 166 | val start = if (start < 0) (start % size + size) % size else start 167 | val endInclusive = if (endInclusive < 0) (endInclusive % size + size) % size else endInclusive 168 | return this.slice(start..endInclusive) 169 | } 170 | 171 | /** 172 | * Wrapper over immutable Set. 173 | */ 174 | class SetValue(private val set: Set = setOf()) : StoreWrapper>, StoreValue, StoreCollection { 175 | /** 176 | * Creates a mutable clone of the immutable set and passes it to the given function. The return 177 | * value of that function is then used to create a new SetValue. 178 | * @param block the callback to pass the mutable set to. 179 | * @returns the modified SetValue. 180 | */ 181 | fun copy(block: (MutableSet) -> Unit = {}): SetValue { 182 | val next = set.toMutableSet() 183 | block(next) 184 | return SetValue(next) 185 | } 186 | 187 | override val size get() = set.size 188 | override fun clone() = SetValue(set.toSet()) 189 | override fun unwrap() = set 190 | override fun iterator() = set.iterator() 191 | override fun toString() = set.joinToString(prefix = "[", postfix = "]") 192 | } 193 | 194 | /** 195 | * Wrapper over concurrent map. 196 | */ 197 | class MapValue(initialValues: Map = mapOf()) : StoreWrapper>, StoreCollection> { 198 | private val map = ConcurrentHashMap(initialValues) 199 | 200 | /** 201 | * Similar to ListValue and SetValue use, except that the map passed 202 | * to the parameter is not a clone, it is a reference to the underlying 203 | * map. 204 | * @param block the callback to pass the map reference to. 205 | * @see ListValue.use 206 | */ 207 | fun use(block: (MutableMap) -> Unit) { 208 | block(map) 209 | } 210 | 211 | override val size get() = map.size 212 | override fun clone() = MapValue(map) 213 | override fun unwrap() = map 214 | override fun iterator() = map.iterator() 215 | override fun toString(): String { 216 | val str = StringBuilder() 217 | 218 | str.append("{") 219 | str.append(map.map { it.key + ": " + it.value.toString() }.joinToString()) 220 | str.append("}") 221 | 222 | return str.toString() 223 | } 224 | } 225 | 226 | // ---------------- 227 | // Extension for converting built-in types to store types. 228 | // ---------------- 229 | 230 | fun Int.toValue(): IntValue = IntValue(this) 231 | fun Float.toValue(): FloatValue = FloatValue(this.toDouble()) 232 | fun Double.toValue(): FloatValue = FloatValue(this) 233 | fun String.toValue(): StringValue = StringValue(this) 234 | fun ByteArray.toValue(): RawValue = RawValue(this) 235 | fun List.toValue(): ListValue = ListValue(this) 236 | fun Set.toValue(): SetValue = SetValue(this) 237 | fun Map.toValue(): MapValue = MapValue(this) 238 | -------------------------------------------------------------------------------- /src/main/kotlin/org/botellier/log/Segment.kt: -------------------------------------------------------------------------------- 1 | package org.botellier.log 2 | 3 | import com.google.protobuf.ByteString 4 | import java.io.* 5 | import java.nio.file.Path 6 | import java.nio.file.Paths 7 | import java.security.MessageDigest 8 | import java.util.* 9 | 10 | /** 11 | * This file contains code for manipulating segment files. Manipulations such as 12 | * appending, deleting and clearing. The format of each segment file is simple, it 13 | * is composed of a series of entries; each entry begins with the size (in bytes) 14 | * of the entry, followed by the entry itself (protobuf encoding). 15 | */ 16 | 17 | /** 18 | * Handler for segment files. 19 | * @property path the base directory of the segment file. 20 | * @property sequence the number to use as postfix os the segment file-name. 21 | * @property prefix the prefix to use in the segment file-name. 22 | * @property maxSize the maximum size of the segment (in bytes). 23 | */ 24 | class Segment(val root: String, val sequence: Int, val prefix: String = "segment-", val maxSize: Int = 1*1024*1024) 25 | : Iterable { 26 | 27 | val path: Path 28 | private val file: File 29 | private val md: MessageDigest 30 | private val header: SegmentHeader 31 | 32 | init { 33 | // Initialize props. 34 | path = Paths.get(root, name()).toAbsolutePath().normalize() 35 | file = File(path.toUri()) 36 | md = computeDigest() 37 | header = if (file.exists()) SegmentHeader.parseFrom(file.inputStream()) else SegmentHeader(md) 38 | 39 | // Validates checksum. 40 | val checksum = md.tryDigest().toHexString() 41 | if (header.checksum != checksum) { 42 | throw SegmentException.ChecksumException(header.checksum, checksum) 43 | } 44 | } 45 | 46 | /** 47 | * Returns the name of the segments. 48 | * @returns the name of the segment. 49 | */ 50 | fun name() = "$prefix$sequence" 51 | 52 | /** 53 | * Returns the next segment in the sequence. 54 | */ 55 | fun nextSegment() = Segment(root,sequence + 1, prefix, maxSize) 56 | 57 | /** 58 | * Deletes the segment file. 59 | */ 60 | fun clear() = file.delete() 61 | 62 | /** 63 | * Computes the current message digest of the segment file, which implies that all the data is going to be 64 | * traversed. 65 | * @returns the current MessageDigest of the file. 66 | */ 67 | fun computeDigest(): MessageDigest { 68 | val md = MessageDigest.getInstance("MD5") 69 | rawIterator().forEach { md.update(it.size.toByteArray()); md.update(it) } 70 | return md 71 | } 72 | 73 | // ---------------- 74 | // Information from header. 75 | // ---------------- 76 | 77 | val id: Int get() = header.id 78 | val checksum: String get() = header.checksum 79 | val totalEntries: Int get() = header.totalEntries 80 | 81 | // ---------------- 82 | // Operations on segment 83 | // ---------------- 84 | 85 | /** 86 | * Appends a new entry to the log file indicating 87 | * the deletion of 'key'. 88 | * @param id the id to use for the entry. 89 | * @param key the key to be marked as deleted. 90 | * @throws SegmentException if the file is full. 91 | * @see nextSegment for getting a new valid segment. 92 | */ 93 | fun delete(id: Int, key: String) { 94 | segmentOperation(id) { 95 | buildDeleteEntry(id) { 96 | this.key = key 97 | } 98 | } 99 | } 100 | 101 | /** 102 | * Appends a new entry to the log file indicating 103 | * the key and its data. 104 | * @param id the id to use for the entry. 105 | * @param key the key to use for the data. 106 | * @param before the bytes to use as old data in the log. 107 | * @param after the bytes to use as new data in the log. 108 | * @throws SegmentException if the file is full. 109 | * @see nextSegment for getting a new valid segment. 110 | */ 111 | fun set(id: Int, key: String, before: ByteArray, after: ByteArray) { 112 | segmentOperation(id) { 113 | buildSetEntry(id) { 114 | this.key = key 115 | this.before = ByteString.copyFrom(before) 116 | this.after = ByteString.copyFrom(after) 117 | } 118 | } 119 | } 120 | 121 | /** 122 | * Appends a create operation to the log. A create operation 123 | * has the exact same representation as a set operation, the 124 | * only difference is that the create operation has no data 125 | * for the 'before' field. 126 | * @param id the id to use for the entry. 127 | * @param key the key to use for the data. 128 | * @param data the bytes to use for the 'after' field. 129 | * @throws SegmentException if the file is full. 130 | * @see nextSegment for getting a new valid segment. 131 | */ 132 | fun create(id: Int, key: String, data: ByteArray) { 133 | segmentOperation(id) { 134 | buildSetEntry(id) { 135 | this.key = key 136 | this.before = ByteString.EMPTY 137 | this.after = ByteString.copyFrom(data) 138 | } 139 | } 140 | } 141 | 142 | /** 143 | * Appends a new entry to the log indicating 144 | * the beginning of a transaction. 145 | */ 146 | fun beginTransaction(id: Int) { 147 | segmentOperation(id) { 148 | buildBeginTransactionEntry(id) 149 | } 150 | } 151 | 152 | /** 153 | * Appends a new entry to the log indicating 154 | * the end of a transaction. 155 | */ 156 | fun endTransaction(id: Int) { 157 | segmentOperation(id) { 158 | buildEndTransactionEntry(id) 159 | } 160 | } 161 | 162 | // ---------------- 163 | // Misc functions. 164 | // ---------------- 165 | 166 | /** 167 | * Appends the given entry to the segment. Updating the checksum 168 | * and header accordingly. 169 | */ 170 | fun segmentOperation(id: Int, block: () -> Entry) { 171 | if (file.length() >= maxSize) { 172 | throw SegmentException.SizeException() 173 | } else { 174 | val entry = block() 175 | 176 | // Updates and writes header. 177 | if (header.totalEntries <= 0) { header.id = id } 178 | md.update(entry.protos.serializedSize.toByteArray()) 179 | md.update(entry.protos.toByteArray()) 180 | header.update(md) 181 | 182 | val raf = RandomAccessFile(file, "rw") 183 | raf.seek(0) 184 | raf.write(header.toByteArray()) 185 | 186 | // Writes new entry. 187 | file.appendBytes(entry.protos.serializedSize.toByteArray()) 188 | file.appendBytes(entry.protos.toByteArray()) 189 | } 190 | } 191 | 192 | // ---------------- 193 | // Iterable 194 | // ---------------- 195 | 196 | /** 197 | * Returns the iterator for each one of the entries in the segment 198 | * file. If the file doesn't exists, an empty iterator is returned 199 | * instead. 200 | * @returns iterator of entries. 201 | */ 202 | override fun iterator(): Iterator { 203 | if (file.exists()) { 204 | return SegmentIterator(file) 205 | } else { 206 | return Collections.emptyIterator() 207 | } 208 | } 209 | 210 | /** 211 | * Iterator for raw entry data. Useful for calculating checksum of the 212 | * segment. 213 | */ 214 | fun rawIterator(): Iterator { 215 | if (file.exists()) { 216 | return RawSegmentIterator(file) 217 | } else { 218 | return Collections.emptyIterator() 219 | } 220 | } 221 | } 222 | 223 | sealed class SegmentException(msg: String) : Throwable(msg) { 224 | class SizeException : SegmentException("Maximum log-file size reached; consider using nextSegment().") 225 | class ChecksumException(before: String, after: String) 226 | : SegmentException("Checksum mismatch. Header is '$before' while data is '$after'.") 227 | } 228 | 229 | /** 230 | * Iterator for raw data of a segment. Useful for building higher-level iterators 231 | * and calculating the checksum. 232 | */ 233 | class RawSegmentIterator(file: File) : Iterator { 234 | val input: DataInputStream = DataInputStream(file.inputStream().buffered()) 235 | var next: ByteArray? = null 236 | init { input.skip(HEADER_SIZE.toLong()) } 237 | 238 | override fun hasNext(): Boolean { 239 | try { 240 | val dataSize = input.readInt() 241 | val data = ByteArray(dataSize) 242 | input.read(data) 243 | next = data 244 | return true 245 | } catch (e: EOFException) { 246 | return false 247 | } 248 | } 249 | 250 | override fun next(): ByteArray { 251 | return next!! 252 | } 253 | } 254 | 255 | /** 256 | * Iterator for entries in a segment file. 257 | * @param file the segment file to use. 258 | */ 259 | class SegmentIterator(file: File) : Iterator { 260 | val rawIterator = RawSegmentIterator(file) 261 | override fun hasNext(): Boolean = rawIterator.hasNext() 262 | override fun next(): Entry = Entry.parseFrom(rawIterator.next()) 263 | } 264 | -------------------------------------------------------------------------------- /src/test/kotlin/org/botellier/command/commands.kt: -------------------------------------------------------------------------------- 1 | package org.botellier.command 2 | 3 | import org.botellier.store.* 4 | import org.botellier.value.* 5 | import org.junit.Assert 6 | import org.junit.Test 7 | 8 | class CommandsTest { 9 | 10 | // Keys. 11 | 12 | @Test 13 | fun delCommand() { 14 | val store = Store() 15 | val del = DelCommand() 16 | 17 | store.transaction().begin { 18 | set("one", IntValue(1)) 19 | set("two", IntValue(2)) 20 | set("three", IntValue(3)) 21 | set("four", IntValue(4)) 22 | } 23 | 24 | del.key = CValue.Primitive.String("one") 25 | del.rest = CValue.Array.String(listOf( 26 | CValue.Primitive.String("two"), 27 | CValue.Primitive.String("three") 28 | )) 29 | Assert.assertEquals(StringValue("OK"), del.execute(store)) 30 | Assert.assertEquals(1, store.size) 31 | } 32 | 33 | @Test 34 | fun existsCommand() { 35 | val store = Store() 36 | val exists = ExistsCommand() 37 | 38 | store.transaction().begin { 39 | set("one", IntValue(1)) 40 | set("two", IntValue(2)) 41 | set("three", IntValue(3)) 42 | } 43 | 44 | exists.key = CValue.Primitive.String("one") 45 | exists.rest = CValue.Array.String(listOf( 46 | CValue.Primitive.String("two"), 47 | CValue.Primitive.String("three"), 48 | CValue.Primitive.String("not key"), 49 | CValue.Primitive.String("not there neither") 50 | )) 51 | Assert.assertEquals(IntValue(3), exists.execute(store)) 52 | } 53 | 54 | @Test 55 | fun keysCommand() { 56 | val store = Store() 57 | val keys = KeysCommand() 58 | 59 | store.transaction().begin { 60 | set("one", IntValue(1)) 61 | set("prone", StringValue("yes")) 62 | set("two", IntValue(2)) 63 | } 64 | 65 | keys.pattern = CValue.Primitive.String(".*ne") 66 | Assert.assertEquals(IntValue(2), (keys.execute(store) as ListValue).size) 67 | } 68 | 69 | @Test 70 | fun renameCommand() { 71 | val store = Store() 72 | val rename = RenameCommand() 73 | 74 | store.transaction().begin { 75 | set("one", IntValue(1)) 76 | } 77 | 78 | rename.key = CValue.Primitive.String("one") 79 | rename.newkey = CValue.Primitive.String("uno") 80 | Assert.assertEquals(StringValue("OK"), rename.execute(store)) 81 | Assert.assertEquals(NilValue(), store.get("one")) 82 | Assert.assertEquals(IntValue(1), store.get("uno")) 83 | 84 | rename.key = rename.newkey 85 | Assert.assertEquals(StringValue("OK"), rename.execute(store)) 86 | Assert.assertEquals(IntValue(1), store.get("uno")) 87 | } 88 | 89 | @Test 90 | fun typeCommand() { 91 | val store = Store() 92 | val type = TypeCommand() 93 | 94 | store.transaction().begin { 95 | set("one", IntValue(1)) 96 | } 97 | 98 | type.key = CValue.Primitive.String("one") 99 | Assert.assertEquals(StringValue("IntValue"), type.execute(store)) 100 | type.key = CValue.Primitive.String("not key") 101 | Assert.assertEquals(StringValue("NilValue"), type.execute(store)) 102 | } 103 | 104 | // Lists. 105 | 106 | @Test 107 | fun lindexCommand() { 108 | val store = Store() 109 | val lindex = LIndexCommand() 110 | 111 | store.transaction().begin { 112 | set("list", listOf(1, 2, 3).map { it.toValue() }.toValue()) 113 | } 114 | 115 | lindex.key = CValue.Primitive.String("list") 116 | lindex.index = CValue.Primitive.Int(2) 117 | Assert.assertEquals(IntValue(3), lindex.execute(store)) 118 | } 119 | 120 | @Test 121 | fun linsertCommand() { 122 | val store = Store() 123 | val linsert = LInsertCommand() 124 | 125 | store.transaction().begin { 126 | set("list", listOf(1, 2, 3).map { it.toValue() }.toValue()) 127 | } 128 | 129 | linsert.key = CValue.Primitive.String("list") 130 | linsert.position = CValue.Primitive.String("BEFORE") 131 | linsert.pivot = CValue.Primitive.Int(1) 132 | linsert.value = CValue.Primitive.Int(0) 133 | Assert.assertEquals(IntValue(4), linsert.execute(store)) 134 | Assert.assertEquals(IntValue(0), (store.get("list") as ListValue).unwrap().get(0)) 135 | 136 | linsert.position = CValue.Primitive.String("AFTER") 137 | linsert.value = CValue.Primitive.Float(1.5) 138 | Assert.assertEquals(IntValue(5), linsert.execute(store)) 139 | Assert.assertEquals(FloatValue(1.5), (store.get("list") as ListValue).unwrap().get(2)) 140 | } 141 | 142 | @Test 143 | fun llenCommand() { 144 | val store = Store() 145 | val llen = LLenCommand() 146 | 147 | store.transaction().begin { 148 | set("list", listOf(1, 2, 3).map { it.toValue() }.toValue()) 149 | } 150 | 151 | llen.key = CValue.Primitive.String("list") 152 | Assert.assertEquals(IntValue(3), llen.execute(store)) 153 | llen.key = CValue.Primitive.String("not key") 154 | Assert.assertEquals(IntValue(0), llen.execute(store)) 155 | } 156 | 157 | @Test 158 | fun lpopCommand() { 159 | val store = Store() 160 | val lpop = LPopCommand() 161 | 162 | store.transaction().begin { 163 | set("list", listOf(1, 2, 3).map { it.toValue() }.toValue()) 164 | } 165 | 166 | lpop.key = CValue.Primitive.String("list") 167 | Assert.assertEquals(IntValue(1), lpop.execute(store)) 168 | Assert.assertEquals(IntValue(2), lpop.execute(store)) 169 | Assert.assertEquals(IntValue(3), lpop.execute(store)) 170 | Assert.assertEquals(NilValue(), lpop.execute(store)) 171 | } 172 | 173 | @Test 174 | fun lpushCommand() { 175 | val store = Store() 176 | val lpush = LPushCommand() 177 | lpush.key = CValue.Primitive.String("new-list") 178 | lpush.value = CValue.Primitive.Int(1) 179 | lpush.rest = CValue.Array.Any(listOf(CValue.Primitive.Float(2.0), CValue.Primitive.String("three"))) 180 | Assert.assertEquals(IntValue(3), lpush.execute(store)) 181 | Assert.assertEquals(IntValue(1), (store.get("new-list") as ListValue).unwrap().last()) 182 | } 183 | 184 | @Test 185 | fun lrangeCommand() { 186 | val store = Store() 187 | val lrange = LRangeCommand() 188 | 189 | store.transaction().begin { 190 | set("list", listOf(1, 2, 3).map { it.toValue() }.toValue()) 191 | } 192 | 193 | lrange.key = CValue.Primitive.String("list") 194 | lrange.start = CValue.Primitive.Int(-2) 195 | lrange.stop = CValue.Primitive.Int(2) 196 | Assert.assertEquals(listOf(2, 3), (lrange.execute(store) as ListValue).toList().map { (it as IntValue).unwrap() }) 197 | } 198 | 199 | @Test 200 | fun lremCommand() { 201 | val store = Store() 202 | val lrem = LRemCommand() 203 | lrem.key = CValue.Primitive.String("list") 204 | 205 | store.transaction().begin { 206 | set("list", listOf(1, 1, 2, 2).map { it.toValue() }.toValue()) 207 | } 208 | 209 | lrem.count = CValue.Primitive.Int(-2) 210 | lrem.value = CValue.Primitive.Int(1) 211 | Assert.assertEquals(IntValue(2), lrem.execute(store)) 212 | Assert.assertEquals(listOf(2, 2), (store.get("list") as ListValue).toList().map { (it as IntValue).unwrap() }) 213 | 214 | store.transaction().begin { 215 | set("list", listOf(1, 1, 2, 2).map { it.toValue() }.toValue()) 216 | } 217 | 218 | lrem.count = CValue.Primitive.Int(2) 219 | lrem.value = CValue.Primitive.Int(2) 220 | Assert.assertEquals(IntValue(2), lrem.execute(store)) 221 | Assert.assertEquals(listOf(1, 1), (store.get("list") as ListValue).toList().map { (it as IntValue).unwrap() }) 222 | } 223 | 224 | @Test 225 | fun lsetCommand() { 226 | val store = Store() 227 | val lset = LSetCommand() 228 | 229 | store.transaction().begin { 230 | set("list", listOf(1, 2, 3).map { it.toValue() }.toValue()) 231 | } 232 | 233 | lset.key = CValue.Primitive.String("list") 234 | lset.index = CValue.Primitive.Int(2) 235 | lset.value = CValue.Primitive.Int(3) 236 | Assert.assertEquals(StringValue("OK"), lset.execute(store)) 237 | Assert.assertEquals(IntValue(3), (store.get("list") as ListValue).unwrap().get(2)) 238 | } 239 | 240 | @Test 241 | fun ltrimCommand() { 242 | val store = Store() 243 | val ltrim = LTrimCommand() 244 | store.transaction().begin { 245 | set("list", listOf(1, 2, 3).map { it.toValue() }.toValue()) 246 | } 247 | ltrim.key = CValue.Primitive.String("list") 248 | ltrim.start = CValue.Primitive.Int(0) 249 | ltrim.stop = CValue.Primitive.Int(1) 250 | Assert.assertEquals(StringValue("OK"), ltrim.execute(store)) 251 | Assert.assertEquals(1, store.size) 252 | ltrim.start = CValue.Primitive.Int(1) 253 | ltrim.stop = CValue.Primitive.Int(0) 254 | Assert.assertEquals(StringValue("OK"), ltrim.execute(store)) 255 | Assert.assertEquals(0, store.size) 256 | } 257 | 258 | @Test 259 | fun rpopCommand() { 260 | val store = Store() 261 | val rpop = RPopCommand() 262 | store.transaction().begin { 263 | set("list", listOf(1, 2, 3).map { it.toValue() }.toValue()) 264 | } 265 | rpop.key = CValue.Primitive.String("list") 266 | Assert.assertEquals(IntValue(3), rpop.execute(store)) 267 | Assert.assertEquals(IntValue(2), rpop.execute(store)) 268 | Assert.assertEquals(IntValue(1), rpop.execute(store)) 269 | Assert.assertEquals(NilValue(), rpop.execute(store)) 270 | } 271 | 272 | @Test 273 | fun rpushCommand() { 274 | val store = Store() 275 | val rpush = RPushCommand() 276 | rpush.key = CValue.Primitive.String("list") 277 | rpush.value = CValue.Primitive.Int(1) 278 | rpush.rest = CValue.Array.Any(listOf(CValue.Primitive.Float(2.0), CValue.Primitive.String("three"))) 279 | Assert.assertEquals(IntValue(3), rpush.execute(store)) 280 | Assert.assertEquals(IntValue(1), (store.get("list") as ListValue).unwrap().first()) 281 | } 282 | 283 | // Strings. 284 | 285 | @Test 286 | fun appendCommand() { 287 | val store = Store() 288 | val append = AppendCommand() 289 | append.key = CValue.Primitive.String("key") 290 | append.value = CValue.Primitive.Int(1) 291 | append.execute(store) 292 | append.value = CValue.Primitive.Float(2.0) 293 | append.execute(store) 294 | append.value = CValue.Primitive.String("three") 295 | Assert.assertEquals(IntValue(9), append.execute(store)) 296 | Assert.assertEquals(StringValue("12.0three"), store.get("key")) 297 | } 298 | 299 | @Test 300 | fun decrCommand() { 301 | val store = Store() 302 | val decr = DecrCommand() 303 | decr.key = CValue.Primitive.String("key") 304 | Assert.assertEquals(IntValue(-1), decr.execute(store)) 305 | Assert.assertEquals(IntValue(-1), store.get("key")) 306 | } 307 | 308 | @Test 309 | fun decrbyCommand() { 310 | val store = Store() 311 | val decrby = DecrbyCommand() 312 | decrby.key = CValue.Primitive.String("key") 313 | decrby.decrement = CValue.Primitive.Int(10) 314 | Assert.assertEquals(IntValue(-10), decrby.execute(store)) 315 | Assert.assertEquals(IntValue(-10), store.get("key")) 316 | } 317 | 318 | @Test 319 | fun getCommand() { 320 | val store = Store() 321 | val get = GetCommand() 322 | store.transaction().begin { 323 | set("key", StringValue("Hello, world!")) 324 | } 325 | get.key = CValue.Primitive.String("key") 326 | Assert.assertEquals(StringValue("Hello, world!"), get.execute(store)) 327 | } 328 | 329 | @Test 330 | fun incrCommand() { 331 | val store = Store() 332 | val incr = IncrCommand() 333 | incr.key = CValue.Primitive.String("key") 334 | Assert.assertEquals(IntValue(1), incr.execute(store)) 335 | Assert.assertEquals(IntValue(1), store.get("key")) 336 | } 337 | 338 | @Test 339 | fun incrbyCommand() { 340 | val store = Store() 341 | val incrby = IncrbyCommand() 342 | incrby.key = CValue.Primitive.String("key") 343 | incrby.increment = CValue.Primitive.Int(10) 344 | Assert.assertEquals(IntValue(10), incrby.execute(store)) 345 | Assert.assertEquals(IntValue(10), store.get("key")) 346 | } 347 | 348 | @Test 349 | fun incrbyfloatCommand() { 350 | val store = Store() 351 | val incrby = IncrbyfloatCommand() 352 | incrby.key = CValue.Primitive.String("key") 353 | incrby.increment = CValue.Primitive.Float(10.0) 354 | Assert.assertEquals(FloatValue(10.0), incrby.execute(store)) 355 | Assert.assertEquals(FloatValue(10.0), store.get("key")) 356 | } 357 | 358 | @Test 359 | fun mgetCommand() { 360 | val store = Store() 361 | val mget = MGetCommand() 362 | store.transaction().begin { 363 | set("one", IntValue(1)) 364 | } 365 | mget.key = CValue.Primitive.String("zero") 366 | mget.rest = CValue.Array.String(listOf("one", "two").map(CValue.Primitive::String)) 367 | 368 | val list = (mget.execute(store) as ListValue).unwrap() 369 | Assert.assertEquals(NilValue(), list.get(0)) 370 | Assert.assertEquals(IntValue(1), list.get(1)) 371 | Assert.assertEquals(NilValue(), list.get(2)) 372 | } 373 | 374 | @Test 375 | fun msetCommand() { 376 | val store = Store() 377 | val mset = MSetCommand() 378 | mset.key = CValue.Primitive.String("key0") 379 | mset.value = CValue.Primitive.Int(0) 380 | mset.rest = CValue.Array.Pair(listOf( 381 | CValue.Pair("key1", CValue.Primitive.Float(1.0)), 382 | CValue.Pair("key2", CValue.Primitive.String("two")) 383 | )) 384 | Assert.assertEquals(StringValue("OK"), mset.execute(store)) 385 | Assert.assertEquals(IntValue(0), store.get("key0")) 386 | Assert.assertEquals(FloatValue(1.0), store.get("key1")) 387 | Assert.assertEquals(StringValue("two"), store.get("key2")) 388 | } 389 | 390 | @Test 391 | fun setCommand() { 392 | val store = Store() 393 | val set = SetCommand() 394 | set.key = CValue.Primitive.String("key") 395 | set.value = CValue.Primitive.String("Hello, world!") 396 | Assert.assertEquals(StringValue("OK"), set.execute(store)) 397 | Assert.assertEquals(StringValue("Hello, world!"), store.get("key")) 398 | } 399 | 400 | @Test 401 | fun strlenCommand() { 402 | val store = Store() 403 | val strlen = StrlenCommand() 404 | store.transaction().begin { 405 | set("message", StringValue("Hello, world!")) 406 | } 407 | strlen.key = CValue.Primitive.String("message") 408 | Assert.assertEquals(IntValue(13), strlen.execute(store)) 409 | } 410 | } 411 | -------------------------------------------------------------------------------- /src/main/kotlin/org/botellier/node/Node.kt: -------------------------------------------------------------------------------- 1 | package org.botellier.node 2 | 3 | import mu.KLogging 4 | import org.apache.zookeeper.* 5 | import org.apache.zookeeper.ZooDefs.Ids.OPEN_ACL_UNSAFE 6 | import java.util.* 7 | 8 | class Node (zooServers: String) : Watcher { 9 | companion object : KLogging() // sets up logging. 10 | 11 | val id = Integer.toHexString(Random().nextInt()) 12 | val zk = ZooKeeper(zooServers, 4000, this) 13 | var type = NodeType.REPLICA 14 | private set 15 | 16 | private var version = 0 17 | private var totalReplicas = 0 // replicas that are not leader or synced follower. 18 | 19 | // ---------------- 20 | // Bootstrap. 21 | // ---------------- 22 | 23 | /** 24 | * Creates all the required persistent directories if they 25 | * don't exist. 26 | */ 27 | fun bootstrap() { 28 | // Leader and synced replica. 29 | createDirectory(Path.leader) 30 | createDirectory(Path.synced) 31 | 32 | // Multi directories. 33 | createDirectory(Path.replicas) 34 | createDirectory(Path.changes) 35 | } 36 | 37 | private fun createDirectory(path: String) { 38 | zk.create(path, null, OPEN_ACL_UNSAFE, CreateMode.PERSISTENT, createDirectoryCb, path) 39 | } 40 | 41 | private val createDirectoryCb = AsyncCallback.StringCallback { rc, path, ctx, name -> 42 | val ecode = KeeperException.Code.get(rc) 43 | val path = ctx as String 44 | when (ecode) { 45 | KeeperException.Code.CONNECTIONLOSS -> { 46 | createDirectory(path) 47 | } 48 | KeeperException.Code.OK -> { 49 | logger.info { "Directory created: $path" } 50 | } 51 | KeeperException.Code.NODEEXISTS -> { 52 | logger.warn { "Directory already exists: $path" } 53 | } 54 | else -> { 55 | logger.error { 56 | "Error when creating $path: ${KeeperException.create(ecode)}" 57 | } 58 | } 59 | } 60 | } 61 | 62 | // ---------------- 63 | // Registration. 64 | // ---------------- 65 | 66 | fun register() { 67 | zk.create(Path.replicaPath(name()), version.toString().toByteArray(), OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL, registerCb, null) 68 | } 69 | 70 | private val registerCb = AsyncCallback.StringCallback { rc, path, ctx, name -> 71 | val ecode = KeeperException.Code.get(rc) 72 | when (ecode) { 73 | KeeperException.Code.CONNECTIONLOSS -> { 74 | register() 75 | } 76 | KeeperException.Code.OK -> { 77 | logger.info { "Node successfully registered: $path" } 78 | } 79 | KeeperException.Code.NODEEXISTS -> { 80 | logger.warn { "Node already registered: $path" } 81 | } 82 | else -> { 83 | logger.error { 84 | "Error when registering $path: ${KeeperException.create(ecode)}" 85 | } 86 | } 87 | } 88 | } 89 | 90 | private fun updateVersion() { 91 | zk.setData(Path.replicaPath(name()), version.toString().toByteArray(), version, updateVersionCb, null) 92 | } 93 | 94 | private val updateVersionCb = AsyncCallback.StatCallback { rc, path, ctx, stat -> 95 | val ecode = KeeperException.Code.get(rc) 96 | when (ecode) { 97 | KeeperException.Code.CONNECTIONLOSS -> { 98 | updateVersion() 99 | } 100 | KeeperException.Code.OK -> { 101 | logger.info { "Node version updated: $path" } 102 | } 103 | else -> { 104 | logger.error { 105 | "Error updating versino of $path: ${KeeperException.create(ecode)}" 106 | } 107 | } 108 | } 109 | } 110 | 111 | // ---------------- 112 | // Synced replica election. 113 | // ---------------- 114 | // Nodes usually try to run for synced replica first. After that, they try to take the leader 115 | // role if the spot is available. 116 | 117 | fun runForSynced() { 118 | ensureReplica { 119 | zk.getData(Path.synced_name, false, runForSyncedCb, null) 120 | } 121 | } 122 | 123 | private val runForSyncedCb = AsyncCallback.DataCallback { rc, path, ctx, data, stat -> 124 | val ecode = KeeperException.Code.get(rc) 125 | when (ecode) { 126 | KeeperException.Code.CONNECTIONLOSS -> { 127 | runForSynced() 128 | } 129 | KeeperException.Code.OK -> { 130 | if (String(data) == name()) { 131 | type = NodeType.SYNCHRONIZED_REPLICA 132 | runForLeader() 133 | } else { 134 | type = NodeType.REPLICA 135 | watchSynced() 136 | } 137 | } 138 | KeeperException.Code.NONODE -> { 139 | checkCandidacy() 140 | } 141 | else -> { 142 | logger.error { 143 | "Error running for synced replica: ${KeeperException.create(ecode)}" 144 | } 145 | } 146 | } 147 | } 148 | 149 | private fun watchSynced() { 150 | ensureReplica { 151 | zk.exists(Path.synced_name, watchSyncedWatcher, watchSyncedCb, null) 152 | } 153 | } 154 | 155 | private val watchSyncedWatcher = Watcher { 156 | when (it.type) { 157 | Watcher.Event.EventType.NodeDeleted -> { 158 | assert(Path.synced_name.equals(it.path)) 159 | runForSynced() 160 | } 161 | } 162 | } 163 | 164 | private val watchSyncedCb = AsyncCallback.StatCallback { rc, path, ctx, stat -> 165 | val ecode = KeeperException.Code.get(rc) 166 | when (ecode) { 167 | KeeperException.Code.CONNECTIONLOSS -> { 168 | watchSynced() 169 | } 170 | KeeperException.Code.OK -> { 171 | if (stat == null) { 172 | runForSynced() 173 | } 174 | } 175 | else -> { 176 | logger.error { "Error checking existence of $path: ${KeeperException.create(ecode)}" } 177 | } 178 | } 179 | } 180 | 181 | private fun checkCandidacy() { 182 | ensureReplica { 183 | zk.getChildren(Path.replicas, false, checkCandidacyCb, null) 184 | } 185 | } 186 | 187 | private val checkCandidacyCb = AsyncCallback.ChildrenCallback { rc, path, ctx, children -> 188 | val ecode = KeeperException.Code.get(rc) 189 | when (ecode) { 190 | KeeperException.Code.CONNECTIONLOSS -> { 191 | checkCandidacy() 192 | } 193 | KeeperException.Code.OK -> { 194 | if (bestCandidate(children) == name()) { 195 | takeReplica() 196 | } else { 197 | runForSynced() 198 | } 199 | } 200 | else -> { 201 | logger.error { 202 | "Error getting candidates: ${KeeperException.create(ecode)}" 203 | } 204 | } 205 | } 206 | } 207 | 208 | private fun bestCandidate(candidates: List): String? { 209 | return candidates.map { 210 | val version = String(getData(Path.replicaPath(it)) ?: byteArrayOf()).toIntOrNull() ?: -1 211 | Pair(it, version) 212 | }.maxBy { it.second }?.first 213 | } 214 | 215 | private fun takeReplica() { 216 | ensureReplica { 217 | zk.multi(listOf( 218 | Op.delete(Path.replicaPath(name()), -1), 219 | Op.create(Path.synced_name, name().toByteArray(), OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL), 220 | Op.create(Path.synced_version, version.toString().toByteArray(), OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL) 221 | ), takeReplicaCb, null) 222 | } 223 | } 224 | 225 | private val takeReplicaCb = AsyncCallback.MultiCallback { rc, path, ctx, opResults -> 226 | val ecode = KeeperException.Code.get(rc) 227 | when (ecode) { 228 | KeeperException.Code.CONNECTIONLOSS -> { 229 | takeReplica() 230 | } 231 | KeeperException.Code.OK -> { 232 | type = NodeType.SYNCHRONIZED_REPLICA 233 | logger.info { "${name()} is now the synced replica." } 234 | runForLeader() 235 | } 236 | KeeperException.Code.NODEEXISTS -> { 237 | runForSynced() 238 | } 239 | else -> { 240 | logger.error { "Error taking role of synced replica on ${name()}: ${KeeperException.create(ecode)}" } 241 | } 242 | } 243 | } 244 | 245 | // ---------------- 246 | // Leader election. 247 | // ---------------- 248 | 249 | fun runForLeader() { 250 | ensureSynced { 251 | zk.getData(Path.leader_name, false, runForLeaderCb, null) 252 | } 253 | } 254 | 255 | private val runForLeaderCb = AsyncCallback.DataCallback { rc, path, ctx, data, stat -> 256 | val ecode = KeeperException.Code.get(rc) 257 | when (ecode) { 258 | KeeperException.Code.CONNECTIONLOSS -> { 259 | runForLeader() 260 | } 261 | KeeperException.Code.OK -> { 262 | if (String(data) == name()) { 263 | type = NodeType.LEADER 264 | } else { 265 | watchLeader() 266 | } 267 | } 268 | KeeperException.Code.NONODE -> { 269 | takeLeader() 270 | } 271 | else -> { 272 | "Error getting data of $path: ${KeeperException.create(ecode)}" 273 | } 274 | } 275 | } 276 | 277 | private fun watchLeader() { 278 | ensureSynced { 279 | zk.exists(Path.leader_name, watchLeaderWatch, watchLeaderCb, null) 280 | } 281 | } 282 | 283 | private val watchLeaderWatch = Watcher { 284 | when (it.type) { 285 | Watcher.Event.EventType.NodeDeleted -> { 286 | assert(Path.leader_name.equals(it.path)) 287 | runForLeader() 288 | } 289 | } 290 | } 291 | 292 | private val watchLeaderCb = AsyncCallback.StatCallback { rc, path, ctx, stat -> 293 | val ecode = KeeperException.Code.get(rc) 294 | when (ecode) { 295 | KeeperException.Code.CONNECTIONLOSS -> { 296 | watchLeader() 297 | } 298 | KeeperException.Code.OK -> { 299 | if (stat == null) { 300 | runForLeader() 301 | } 302 | } 303 | else -> { 304 | logger.error { 305 | "Error checking for leader existence: ${KeeperException.create(ecode)}" 306 | } 307 | } 308 | } 309 | } 310 | 311 | private fun takeLeader() { 312 | ensureSynced { 313 | zk.multi(listOf( 314 | Op.delete(Path.synced_name, -1), 315 | Op.delete(Path.synced_version, -1), 316 | Op.create(Path.leader_name, name().toByteArray(), OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL), 317 | Op.create(Path.leader_version, version.toString().toByteArray(), OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL) 318 | ), takeLeaderCb, null) 319 | } 320 | } 321 | 322 | private val takeLeaderCb = AsyncCallback.MultiCallback { rc, path, ctx, opResults -> 323 | val ecode = KeeperException.Code.get(rc) 324 | when (ecode) { 325 | KeeperException.Code.CONNECTIONLOSS -> { 326 | takeLeader() 327 | } 328 | KeeperException.Code.OK -> { 329 | type = NodeType.LEADER 330 | logger.info { "${name()} is now the leader." } 331 | } 332 | KeeperException.Code.NODEEXISTS -> { 333 | runForLeader() 334 | } 335 | else -> { 336 | logger.error { "Error taking role of leader on ${name()}: ${KeeperException.create(ecode)}" } 337 | } 338 | } 339 | } 340 | 341 | // ---------------- 342 | // Replicas. 343 | // ---------------- 344 | 345 | fun countReplicas() { 346 | zk.getChildren(Path.replicas, countReplicasWatcher, countReplicasCb, null) 347 | } 348 | 349 | private val countReplicasWatcher = Watcher { 350 | when (it.type) { 351 | Watcher.Event.EventType.NodeChildrenChanged -> { 352 | assert(Path.replicas.equals(it.path)) 353 | countReplicas() 354 | } 355 | } 356 | } 357 | 358 | private val countReplicasCb = AsyncCallback.ChildrenCallback { rc, path, ctx, children -> 359 | val ecode = KeeperException.Code.get(rc) 360 | when (ecode) { 361 | KeeperException.Code.CONNECTIONLOSS -> { 362 | countReplicas() 363 | } 364 | KeeperException.Code.OK -> { 365 | if (children == null) { 366 | totalReplicas = 0 367 | } else { 368 | totalReplicas = children.size 369 | } 370 | logger.info { "Total replicas is now: $totalReplicas"} 371 | } 372 | else -> { 373 | logger.error { 374 | "Error counting replicas $path: ${KeeperException.create(ecode)}" 375 | } 376 | } 377 | } 378 | } 379 | 380 | // ---------------- 381 | // Other methods. 382 | // ---------------- 383 | 384 | /** 385 | * Gets the name of the node in the form of 'node-[ID]' 386 | * @return the name of the node. 387 | */ 388 | fun name(): String = "node-$id" 389 | 390 | /** 391 | * Makes sure the current node is a replica 392 | * before running the supplied callback. 393 | */ 394 | private fun ensureReplica(f: () -> Unit) { 395 | if (type == NodeType.REPLICA) { 396 | f() 397 | } 398 | } 399 | 400 | /** 401 | * Makes sure the current node is a synced replica 402 | * before running the supplied callback. 403 | */ 404 | private fun ensureSynced(f: () -> Unit) { 405 | if (type == NodeType.SYNCHRONIZED_REPLICA) { 406 | f() 407 | } else if (type == NodeType.REPLICA) { 408 | runForSynced() 409 | } 410 | } 411 | 412 | // ---------------- 413 | // Zookeeper helpers. 414 | // ---------------- 415 | 416 | /** 417 | * Synchronous function to get node data if it exists. 418 | * 419 | */ 420 | private fun getData(path: String): ByteArray? { 421 | while (true) { 422 | try { 423 | val data = zk.getData(path, false, null) 424 | return data 425 | } catch (e: KeeperException.NoNodeException) { 426 | break 427 | } catch (e: KeeperException.ConnectionLossException) { 428 | continue 429 | } 430 | } 431 | return null 432 | } 433 | 434 | // ---------------- 435 | // Watcher. 436 | // ---------------- 437 | 438 | override fun process(event: WatchedEvent?) { 439 | logger.info { "Event received on ${name()}: $event" } 440 | } 441 | 442 | // ---------------- 443 | // Nested classes. 444 | // ---------------- 445 | 446 | enum class NodeType { 447 | LEADER, SYNCHRONIZED_REPLICA, REPLICA 448 | } 449 | } 450 | -------------------------------------------------------------------------------- /src/main/kotlin/org/botellier/command/commands.kt: -------------------------------------------------------------------------------- 1 | package org.botellier.command 2 | 3 | import com.sun.xml.internal.messaging.saaj.util.ByteOutputStream 4 | import org.botellier.log.* 5 | import org.botellier.server.Client 6 | import org.botellier.server.Server 7 | import org.botellier.store.* 8 | import org.botellier.value.* 9 | import kotlin.reflect.full.createInstance 10 | 11 | // from: https://redis.io/commands 12 | 13 | val COMMANDS = arrayOf( 14 | // Node. 15 | GetLogCommand::class, 16 | // Connection. 17 | AuthCommand::class, 18 | EchoCommand::class, 19 | PingCommand::class, 20 | QuitCommand::class, 21 | SelectCommand::class, 22 | // Keys. 23 | DelCommand::class, 24 | ExistsCommand::class, 25 | KeysCommand::class, 26 | RenameCommand::class, 27 | TypeCommand::class, 28 | // Lists. 29 | LIndexCommand::class, 30 | LInsertCommand::class, 31 | LLenCommand::class, 32 | LPopCommand::class, 33 | LPushCommand::class, 34 | LRangeCommand::class, 35 | LRemCommand::class, 36 | LSetCommand::class, 37 | LTrimCommand::class, 38 | RPopCommand::class, 39 | RPushCommand::class, 40 | // Strings. 41 | AppendCommand::class, 42 | DecrCommand::class, 43 | DecrbyCommand::class, 44 | GetCommand::class, 45 | IncrCommand::class, 46 | IncrbyCommand::class, 47 | IncrbyfloatCommand::class, 48 | MGetCommand::class, 49 | MSetCommand::class, 50 | SetCommand::class, 51 | StrlenCommand::class 52 | ).map { it.createInstance().name to it }.toMap() 53 | 54 | private val OK = StringValue("OK") 55 | 56 | // ---------------- 57 | // Node. 58 | // ---------------- 59 | 60 | /** 61 | * Returns the log history of the given 62 | * base index starting from given version. 63 | */ 64 | // TODO: Should optimize performance while creating response data. Maybe a new StoreValue should be added for raw bytes. 65 | @WithCommand("GETLOG") 66 | class GetLogCommand : ConnCommand() { 67 | @field:Parameter(0) 68 | var index = intValue 69 | 70 | @field:Parameter(1) 71 | var version = intValue 72 | 73 | override fun run(server: Server, client: Client): StoreValue { 74 | try { 75 | val db = server.dbs[index.value] 76 | val query = db.log.query(version.value) 77 | val bos = ByteOutputStream() 78 | 79 | // begin and end mark the start and finish of the sequence. 80 | val begin = buildBeginTransactionEntry(-1) 81 | val end = buildEndTransactionEntry(-1) 82 | 83 | // Write the data. 84 | bos.write(begin.protos.serializedSize.toByteArray()) 85 | bos.write(begin.protos.toByteArray()) 86 | for (entry in query) { 87 | bos.write(entry.protos.serializedSize.toByteArray()) 88 | bos.write(entry.protos.toByteArray()) 89 | } 90 | bos.write(end.protos.serializedSize.toByteArray()) 91 | bos.write(end.protos.toByteArray()) 92 | 93 | return bos.bytes.toValue() 94 | } catch (e: IndexOutOfBoundsException) { 95 | throw CommandException.RuntimeException("Requested DB index does not exists.") 96 | } catch (e: Throwable) { 97 | throw CommandException.RuntimeException("Unexpected error requesting log: $e") 98 | } 99 | } 100 | } 101 | 102 | // ---------------- 103 | // Connection. 104 | // ---------------- 105 | 106 | @WithCommand("AUTH") 107 | class AuthCommand : ConnCommand() { 108 | @field:Parameter(0) 109 | var password = stringValue 110 | 111 | override fun run(server: Server, client: Client): StoreValue { 112 | if (server.password == null || password.value == server.password) { 113 | client.isAuthenticated = true 114 | return OK 115 | } 116 | else { 117 | throw CommandException.RuntimeException("Invalid password") 118 | } 119 | } 120 | } 121 | 122 | @WithCommand("ECHO") 123 | class EchoCommand : ConnCommand() { 124 | @field:Parameter(0) 125 | var message = stringValue 126 | 127 | override fun run(server: Server, client: Client): StoreValue { 128 | return StringValue(message.value) 129 | } 130 | } 131 | 132 | @WithCommand("PING") 133 | class PingCommand : ConnCommand() { 134 | override fun run(server: Server, client: Client): StoreValue { 135 | return StringValue("PONG") 136 | } 137 | } 138 | 139 | @WithCommand("QUIT") 140 | class QuitCommand : ConnCommand() { 141 | override fun run(server: Server, client: Client): StoreValue { 142 | return OK 143 | } 144 | } 145 | 146 | @WithCommand("SELECT") 147 | class SelectCommand : ConnCommand() { 148 | @field:Parameter(0) 149 | var index = intValue 150 | 151 | override fun run(server: Server, client: Client): StoreValue { 152 | if (index.value >= 0 && index.value < server.dbs.size) { 153 | client.dbIndex = index.value 154 | return OK 155 | } 156 | else { 157 | throw CommandException.RuntimeException("Invalid database index.") 158 | } 159 | } 160 | } 161 | 162 | // ---------------- 163 | // Keys. 164 | // ---------------- 165 | 166 | @WithCommand("DEL") 167 | class DelCommand : StoreCommand() { 168 | @field:Parameter(0) 169 | var key = stringValue 170 | 171 | @field:Parameter(1) 172 | var rest = stringArrayValue 173 | 174 | override fun run(transaction: StoreTransaction): StoreValue { 175 | return transaction { 176 | delete(key.value) 177 | rest.value.map { delete(it.value) } 178 | OK 179 | } 180 | } 181 | } 182 | 183 | @WithCommand("EXISTS") 184 | class ExistsCommand : ReadStoreCommand() { 185 | @field:Parameter(0) 186 | var key = stringValue 187 | 188 | @field:Parameter(1) 189 | var rest = stringArrayValue 190 | 191 | override fun run(store: ReadStore): StoreValue { 192 | var total = 0 193 | if (store.get(key.value) !is NilValue) { 194 | total++ 195 | } 196 | rest.value.map { 197 | if (store.get(it.value) !is NilValue) { 198 | total++ 199 | } 200 | } 201 | return IntValue(total) 202 | } 203 | } 204 | 205 | @WithCommand("KEYS") 206 | class KeysCommand : ReadStoreCommand() { 207 | @field:Parameter(0) 208 | var pattern = stringValue 209 | 210 | override fun run(store: ReadStore): StoreValue { 211 | val regex = Regex(pattern.value) 212 | val matching = mutableListOf() 213 | store.keys.map { regex.matches(it) && matching.add(it) } 214 | return ListValue(matching.map { it.toValue() }) 215 | } 216 | } 217 | 218 | @WithCommand("RENAME") 219 | class RenameCommand : StoreCommand() { 220 | @field:Parameter(0) 221 | var key = stringValue 222 | 223 | @field:Parameter(1) 224 | var newkey = stringValue 225 | 226 | override fun run(transaction: StoreTransaction): StoreValue { 227 | val oldValue = transaction.get(key.value) 228 | 229 | return transaction { 230 | delete(key.value) 231 | set(newkey.value, oldValue) 232 | OK 233 | } 234 | } 235 | } 236 | 237 | @WithCommand("TYPE") 238 | class TypeCommand : ReadStoreCommand() { 239 | @field:Parameter(0) 240 | var key = stringValue 241 | 242 | override fun run(store: ReadStore): StoreValue { 243 | return StringValue(store.get(key.value)::class.simpleName ?: "Unknown") 244 | } 245 | } 246 | 247 | /** 248 | * Lists. 249 | */ 250 | 251 | @WithCommand("LINDEX") 252 | class LIndexCommand : ReadStoreCommand() { 253 | @field:Parameter(0) 254 | var key = stringValue 255 | 256 | @field:Parameter(1) 257 | var index = intValue 258 | 259 | override fun run(store: ReadStore): StoreValue { 260 | return requireValue(store, key.value) { 261 | if (index.value >= 0 && index.value <= it.size - 1) { 262 | it.unwrap().get(index.value) 263 | } 264 | else { 265 | NilValue() 266 | } 267 | } 268 | } 269 | } 270 | 271 | @WithCommand("LINSERT") 272 | class LInsertCommand : StoreCommand() { 273 | @field:Parameter(0) 274 | var key = stringValue 275 | 276 | @field:Parameter(1) 277 | var position = stringValue 278 | 279 | @field:Parameter(2) 280 | var pivot = anyValue 281 | 282 | @field:Parameter(2) 283 | var value = anyValue 284 | 285 | override fun run(transaction: StoreTransaction): StoreValue { 286 | if (position.value !in listOf("BEFORE", "AFTER")) { 287 | throw CommandException.RuntimeException("LINSERT [BEFORE|AFTER] expected.") 288 | } 289 | 290 | val updated = transaction.update(key.value) { 291 | val isBefore = position.value == "BEFORE" 292 | val value = value.toValue() 293 | val index = it.indexOf(pivot.toValue()) 294 | 295 | when (index) { 296 | -1 -> return IntValue(-1) 297 | else -> { 298 | it.copy { when { 299 | isBefore -> it.add(index, value) 300 | index < it.size - 1 -> it.add(index + 1, value) 301 | else -> it.rpush(value) 302 | }} 303 | } 304 | } 305 | } 306 | 307 | return IntValue((updated as ListValue).size) 308 | } 309 | } 310 | 311 | @WithCommand("LLEN") 312 | class LLenCommand : ReadStoreCommand() { 313 | @field:Parameter(0) 314 | var key = stringValue 315 | 316 | override fun run(store: ReadStore): StoreValue { 317 | return withValue(store, key.value) { 318 | if (it != null) { 319 | IntValue(it.size) 320 | } 321 | else { 322 | IntValue(0) 323 | } 324 | } 325 | } 326 | } 327 | 328 | @WithCommand("LPOP") 329 | class LPopCommand : StoreCommand() { 330 | @field:Parameter(0) 331 | var key = stringValue 332 | 333 | override fun run(transaction: StoreTransaction): StoreValue { 334 | var ret: StoreValue = NilValue() 335 | 336 | transaction.mupdate(key.value) { 337 | if (it != null && it.size > 0) { 338 | it.copy { 339 | ret = it.lpop() 340 | } 341 | } else { 342 | NilValue() 343 | } 344 | } 345 | 346 | return ret 347 | } 348 | } 349 | 350 | @WithCommand("LPUSH") 351 | class LPushCommand : StoreCommand() { 352 | @field:Parameter(0) 353 | var key = stringValue 354 | 355 | @field:Parameter(1) 356 | var value = anyValue 357 | 358 | @field:Parameter(2) 359 | var rest = anyArrayValue 360 | 361 | override fun run(transaction: StoreTransaction): StoreValue { 362 | val updated = transaction.mupdate(key.value) { 363 | val current = it ?: ListValue() 364 | current.copy { list -> 365 | list.lpush(value.toValue()) 366 | rest.value.map { list.lpush(it.toValue()) } 367 | } 368 | } 369 | 370 | return (updated as ListValue).size.toValue() 371 | } 372 | } 373 | 374 | @WithCommand("LRANGE") 375 | class LRangeCommand : ReadStoreCommand() { 376 | @field:Parameter(0) 377 | var key = stringValue 378 | 379 | @field:Parameter(1) 380 | var start = intValue 381 | 382 | @field:Parameter(2) 383 | var stop = intValue 384 | 385 | override fun run(store: ReadStore): StoreValue { 386 | return requireValue(store, key.value) { 387 | it.unwrap().slice(start.value, stop.value).toValue() 388 | } 389 | } 390 | } 391 | 392 | @WithCommand("LREM") 393 | class LRemCommand : StoreCommand() { 394 | @field:Parameter(0) 395 | var key = stringValue 396 | 397 | @field:Parameter(1) 398 | var count = intValue 399 | 400 | @field:Parameter(2) 401 | var value = anyValue 402 | 403 | override fun run(transaction: StoreTransaction): StoreValue { 404 | val updated = transaction.update(key.value) { 405 | val count = count.value 406 | val value = value.toValue() 407 | 408 | val indices = it.mapIndexed { i, elem -> Pair(i, elem) }.filter { it.second == value }.map { it.first } 409 | when { 410 | count < 0 -> { 411 | it.copy { it.remove(indices.takeLast(Math.abs(count))) } 412 | } 413 | count > 0 -> { 414 | it.copy { it.remove(indices.take(Math.abs(count))) } 415 | } 416 | else -> { 417 | it.copy { it.remove(indices) } 418 | } 419 | } 420 | } 421 | 422 | return IntValue((updated as ListValue).size) 423 | } 424 | } 425 | 426 | @WithCommand("LSET") 427 | class LSetCommand : StoreCommand() { 428 | @field:Parameter(0) 429 | var key = stringValue 430 | 431 | @field:Parameter(1) 432 | var index = intValue 433 | 434 | @field:Parameter(2) 435 | var value = anyValue 436 | 437 | override fun run(transaction: StoreTransaction): StoreValue { 438 | transaction.update(key.value) { 439 | if (index.value < 0 || index.value > it.size - 1) { 440 | throw CommandException.RuntimeException("LSET index out of bounds") 441 | } 442 | 443 | it.copy { 444 | it.set(index.value, value.toValue()) 445 | } 446 | } 447 | 448 | return OK 449 | } 450 | } 451 | 452 | @WithCommand("LTRIM") 453 | class LTrimCommand : StoreCommand() { 454 | @field:Parameter(0) 455 | var key = stringValue 456 | 457 | @field:Parameter(1) 458 | var start = intValue 459 | 460 | @field:Parameter(2) 461 | var stop = intValue 462 | 463 | override fun run(transaction: StoreTransaction): StoreValue { 464 | transaction.update(key.value) { 465 | it.copy { list -> list.trim(start.value, stop.value) } 466 | } 467 | return OK 468 | } 469 | } 470 | 471 | @WithCommand("RPOP") 472 | class RPopCommand : StoreCommand() { 473 | @field:Parameter(0) 474 | var key = stringValue 475 | 476 | override fun run(transaction: StoreTransaction): StoreValue { 477 | var ret: StoreValue = NilValue() 478 | 479 | transaction.mupdate(key.value) { 480 | if (it != null && it.size > 0) { 481 | it.copy { 482 | ret = it.rpop() 483 | } 484 | } else { 485 | NilValue() 486 | } 487 | } 488 | 489 | return ret 490 | } 491 | } 492 | 493 | @WithCommand("RPUSH") 494 | class RPushCommand : StoreCommand() { 495 | @field:Parameter(0) 496 | var key = stringValue 497 | 498 | @field:Parameter(1) 499 | var value = anyValue 500 | 501 | @field:Parameter(2) 502 | var rest = anyArrayValue 503 | 504 | override fun run(transaction: StoreTransaction): StoreValue { 505 | val updated = transaction.mupdate(key.value) { 506 | val current = it ?: ListValue() 507 | current.copy { list -> 508 | list.rpush(value.toValue()) 509 | rest.value.map { list.rpush(it.toValue()) } 510 | } 511 | } 512 | 513 | return IntValue((updated as ListValue).size) 514 | } 515 | } 516 | 517 | /** 518 | * Strings. 519 | */ 520 | 521 | @WithCommand("APPEND") 522 | class AppendCommand : StoreCommand() { 523 | @field:Parameter(0) 524 | var key = stringValue 525 | 526 | @field:Parameter(1) 527 | var value = anyValue 528 | 529 | override fun run(transaction: StoreTransaction): StoreValue { 530 | val updated = transaction.mupdate(key.value) { 531 | val builder = StringBuilder() 532 | if (it != null) { 533 | builder.append(it.unwrap()) 534 | } 535 | builder.append(value.toString()) 536 | builder.toString().toValue() 537 | } 538 | 539 | return (updated as StringValue).unwrap().length.toValue() 540 | } 541 | } 542 | 543 | @WithCommand("DECR") 544 | class DecrCommand : StoreCommand() { 545 | @field:Parameter(0) 546 | var key = stringValue 547 | 548 | override fun run(transaction: StoreTransaction): StoreValue { 549 | return transactionIncr(transaction, key.value, -1) 550 | } 551 | } 552 | 553 | @WithCommand("DECRBY") 554 | class DecrbyCommand : StoreCommand() { 555 | @field:Parameter(0) 556 | var key = stringValue 557 | 558 | @field:Parameter(1) 559 | var decrement = intValue 560 | 561 | override fun run(transaction: StoreTransaction): StoreValue { 562 | return transactionIncr(transaction, key.value, -decrement.value) 563 | } 564 | } 565 | 566 | @WithCommand("GET") 567 | class GetCommand : ReadStoreCommand() { 568 | @field:Parameter(0) 569 | var key = stringValue 570 | override fun run(store: ReadStore): StoreValue { 571 | return withPrimitive(store, key.value) { 572 | it ?: NilValue() 573 | } 574 | } 575 | } 576 | 577 | @WithCommand("INCR") 578 | class IncrCommand : StoreCommand() { 579 | @field:Parameter(0) 580 | var key = stringValue 581 | 582 | override fun run(transaction: StoreTransaction): StoreValue { 583 | return transactionIncr(transaction, key.value, 1) 584 | } 585 | } 586 | 587 | @WithCommand("INCRBY") 588 | class IncrbyCommand : StoreCommand() { 589 | @field:Parameter(0) 590 | var key = stringValue 591 | 592 | @field:Parameter(1) 593 | var increment = intValue 594 | 595 | override fun run(transaction: StoreTransaction): StoreValue { 596 | return transactionIncr(transaction, key.value, increment.value) 597 | } 598 | } 599 | 600 | @WithCommand("INCRBYFLOAT") 601 | class IncrbyfloatCommand : StoreCommand() { 602 | @field:Parameter(0) 603 | var key = stringValue 604 | 605 | @field:Parameter(1) 606 | var increment = floatValue 607 | 608 | override fun run(transaction: StoreTransaction): StoreValue { 609 | return transactionIncrFloat(transaction, key.value, increment.value) 610 | } 611 | } 612 | 613 | @WithCommand("MGET") 614 | class MGetCommand : ReadStoreCommand() { 615 | @field:Parameter(0) 616 | var key = stringValue 617 | 618 | @field:Parameter(1) 619 | var rest = stringArrayValue 620 | 621 | override fun run(store: ReadStore): StoreValue { 622 | val list = mutableListOf() 623 | 624 | list.rpush(withPrimitive(store, key.value) { it ?: NilValue() }) 625 | rest.value.map { 626 | list.rpush(withPrimitive(store, it.value) { it ?: NilValue() }) 627 | } 628 | 629 | return list.map { it as StorePrimitive }.toValue() 630 | } 631 | } 632 | 633 | @WithCommand("MSET") 634 | class MSetCommand : StoreCommand() { 635 | @field:Parameter(0) 636 | var key = stringValue 637 | 638 | @field:Parameter(1) 639 | var value = anyValue 640 | 641 | @field:Parameter(2) 642 | var rest = pairArrayValue 643 | 644 | override fun run(transaction: StoreTransaction): StoreValue { 645 | return transaction { 646 | set(key.value, value.toValue()) 647 | for ((key, value) in rest.value) { 648 | set(key, value.toValue()) 649 | } 650 | OK 651 | } 652 | } 653 | } 654 | 655 | @WithCommand("SET") 656 | class SetCommand : StoreCommand() { 657 | @field:Parameter(0) 658 | var key = stringValue 659 | 660 | @field:Parameter(1) 661 | var value = anyValue 662 | 663 | override fun run(transaction: StoreTransaction): StoreValue { 664 | transaction.set(key.value, value.toValue()) 665 | return OK 666 | } 667 | } 668 | 669 | @WithCommand("STRLEN") 670 | class StrlenCommand : ReadStoreCommand() { 671 | @field:Parameter(0) 672 | var key = stringValue 673 | 674 | override fun run(store: ReadStore): StoreValue { 675 | return withValue(store, key.value) { 676 | if (it != null) { 677 | IntValue(it.unwrap().length) 678 | } 679 | else { 680 | IntValue(0) 681 | } 682 | } 683 | } 684 | } 685 | 686 | // ---------------- 687 | // Utility functions. 688 | // ---------------- 689 | 690 | private fun transactionIncr(transaction: StoreTransaction, key: String, incr: Int): StoreValue { 691 | return transaction.mupdate(key) { 692 | when (it) { 693 | is IntValue -> IntValue(it.unwrap() + incr) 694 | is FloatValue -> FloatValue(it.unwrap() + incr.toFloat()) 695 | else -> IntValue(incr) 696 | } 697 | } 698 | } 699 | 700 | private fun transactionIncrFloat(transaction: StoreTransaction, key: String, incr: Double): StoreValue { 701 | return transaction.mupdate(key) { 702 | when (it) { 703 | is IntValue -> FloatValue(it.unwrap().toFloat() + incr) 704 | is FloatValue -> FloatValue(it.unwrap() + incr) 705 | else -> FloatValue(incr) 706 | } 707 | } 708 | } 709 | 710 | /** 711 | * Utility functions. 712 | */ 713 | 714 | /** 715 | * Gets 'key' from store and throws exception if 'key' doesn't exists. 716 | */ 717 | private inline fun requireType(store: ReadStore, key: String, body: (T) -> R): R { 718 | val value = store.get(key) 719 | if (value !is NilValue) { 720 | if (value is T) { 721 | return body(value) 722 | } 723 | else { 724 | throw CommandException.WrongTypeException(key, value.javaClass.name) 725 | } 726 | } 727 | else { 728 | throw CommandException.WrongTypeException(key, NilValue::class.qualifiedName ?: "NilValue") 729 | } 730 | } 731 | 732 | /** 733 | * Just like [requireType], except that body parameter is nullable (doesn't throw exception if 'key' lookup 734 | * fails). 735 | */ 736 | private inline fun withType(store: ReadStore, key: String, body: (T?) -> R): R { 737 | val value = store.get(key) 738 | if (value !is NilValue) { 739 | if (value is T) { 740 | return body(value) 741 | } 742 | else { 743 | throw CommandException.WrongTypeException(key, value.javaClass.name) 744 | } 745 | } 746 | else { 747 | return body(null) 748 | } 749 | } 750 | 751 | private inline fun withValue(store: ReadStore, key: String, body: (T?) -> StoreValue) 752 | = withType(store, key, body) 753 | 754 | private inline fun withPrimitive(store: ReadStore, key: String, body: (T?) -> StorePrimitive) 755 | = withType(store, key, body) 756 | 757 | private inline fun requireValue(store: ReadStore, key: String, body: (T) -> StoreValue) 758 | = requireType(store, key, body) 759 | --------------------------------------------------------------------------------